diff --git a/cdist/config.py b/cdist/config.py index 6c30db2e..97cc1da6 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -124,6 +124,7 @@ class Config(object): """Remove files and directories for the run""" if self.remove_remote_files_dirs: self._remove_remote_files_dirs() + self.manifest.cleanup() @staticmethod def hosts(source): @@ -787,6 +788,9 @@ class Config(object): self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) try: self.manifest.run_type_manifest(cdist_object) + self.log.trace("[ORDER_DEP] Removing order dep files for %s", + cdist_object) + cdist_object.cleanup() cdist_object.state = core.CdistObject.STATE_PREPARED except cdist.Error as e: raise cdist.CdistObjectError(cdist_object, e) diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 237f0ddd..114a47e0 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -243,6 +243,16 @@ class CdistObject(object): lambda obj: os.path.join(obj.base_path, obj.code_local_path)) code_remote = fsproperty.FileStringProperty( lambda obj: os.path.join(obj.base_path, obj.code_remote_path)) + typeorder = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, 'typeorder')) + typeorder_dep = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, 'typeorder_dep')) + + def cleanup(self): + try: + os.remove(os.path.join(self.absolute_path, 'typeorder_dep')) + except FileNotFoundError: + pass @property def exists(self): diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 07af0ef8..8aeaf860 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -96,6 +96,10 @@ class Manifest(object): """Executes cdist manifests. """ + + ORDER_DEP_STATE_NAME = 'order_dep_state' + TYPEORDER_DEP_NAME = 'typeorder_dep' + def __init__(self, target_host, local, dry_run=False): self.target_host = target_host self.local = local @@ -212,3 +216,13 @@ class Manifest(object): type_manifest, env=self.env_type_manifest(cdist_object), message_prefix=message_prefix) + + def cleanup(self): + def _rm_file(fname): + try: + self.log.trace("[ORDER_DEP] Removing %s", fname) + os.remove(os.path.join(self.local.base_path, fname)) + except FileNotFoundError: + pass + _rm_file(Manifest.ORDER_DEP_STATE_NAME) + _rm_file(Manifest.TYPEORDER_DEP_NAME) diff --git a/cdist/emulator.py b/cdist/emulator.py index 417f2cdd..4800e2a3 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -29,6 +29,7 @@ import sys import cdist from cdist import core from cdist import flock +from cdist.core.manifest import Manifest class MissingRequiredEnvironmentVariableError(cdist.Error): @@ -82,6 +83,11 @@ class Emulator(object): self.object_base_path = os.path.join(self.global_path, "object") self.typeorder_path = os.path.join(self.global_path, "typeorder") + self.typeorder_dep_path = os.path.join(self.global_path, + Manifest.TYPEORDER_DEP_NAME) + self.order_dep_state_path = os.path.join(self.global_path, + Manifest.ORDER_DEP_STATE_NAME) + self.type_name = os.path.basename(argv[0]) self.cdist_type = core.CdistType(self.type_base_path, self.type_name) @@ -206,6 +212,14 @@ class Emulator(object): return params def setup_object(self): + # CDIST_ORDER_DEPENDENCY state + order_dep_on = self._order_dep_on() + order_dep_defined = "CDIST_ORDER_DEPENDENCY" in self.env + if not order_dep_defined and order_dep_on: + self._set_order_dep_state_off() + if order_dep_defined and not order_dep_on: + self._set_order_dep_state_on() + # Create object with given parameters self.parameters = {} for key, value in vars(self.args).items(): @@ -237,6 +251,20 @@ class Emulator(object): # record the created object in typeorder file with open(self.typeorder_path, 'a') as typeorderfile: print(self.cdist_object.name, file=typeorderfile) + # record the created object in parent object typeorder file + __object_name = self.env.get('__object_name', None) + depname = self.cdist_object.name + if __object_name: + parent = self.cdist_object.object_from_name(__object_name) + parent.typeorder.append(self.cdist_object.name) + if self._order_dep_on(): + self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep' + ' for %s'), depname, parent.name) + parent.typeorder_dep.append(depname) + elif self._order_dep_on(): + self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep', + depname) + self._add_typeorder_dep(depname) # Record / Append source self.cdist_object.source.append(self.object_source) @@ -293,45 +321,73 @@ class Emulator(object): return cdist_object.name + def _order_dep_on(self): + return os.path.exists(self.order_dep_state_path) + + def _set_order_dep_state_on(self): + self.log.trace('[ORDER_DEP] Setting order dep state on') + with open(self.order_dep_state_path, 'w'): + pass + + def _set_order_dep_state_off(self): + self.log.trace('[ORDER_DEP] Setting order dep state off') + # remove order dep state file + try: + os.remove(self.order_dep_state_path) + except FileNotFoundError: + pass + # remove typeorder dep file + try: + os.remove(self.typeorder_dep_path) + except FileNotFoundError: + pass + + def _add_typeorder_dep(self, name): + with open(self.typeorder_dep_path, 'a') as f: + print(name, file=f) + + def _read_typeorder_dep(self): + try: + with open(self.typeorder_dep_path, 'r') as f: + return f.readlines() + except FileNotFoundError: + return [] + def record_requirements(self): """Record requirements.""" + order_dep_on = self._order_dep_on() + # Inject the predecessor, but not if its an override # (this would leed to an circular dependency) - if ("CDIST_ORDER_DEPENDENCY" in self.env and - 'CDIST_OVERRIDE' not in self.env): - # load object name created befor this one from typeorder file ... - with open(self.typeorder_path, 'r') as typecreationfile: - typecreationorder = typecreationfile.readlines() - # get the type created before this one ... - try: - lastcreatedtype = typecreationorder[-2].strip() - # __object_name is the name of the object whose type - # manifest is currently executed - __object_name = self.env.get('__object_name', None) - if lastcreatedtype == __object_name: - self.log.debug(("Not injecting require for " - "CDIST_ORDER_DEPENDENCY: %s for %s," - " %s's type manifest is currently" - " being executed"), - lastcreatedtype, - self.cdist_object.name, - lastcreatedtype) - else: - if 'require' in self.env: - appendix = " " + lastcreatedtype - if appendix not in self.env['require']: - self.env['require'] += appendix - else: - self.env['require'] = lastcreatedtype - self.log.debug(("Injecting require for " - "CDIST_ORDER_DEPENDENCY: %s for %s"), - lastcreatedtype, - self.cdist_object.name) - except IndexError: - # if no second last line, we are on the first type, - # so do not set a requirement - pass + if (order_dep_on and 'CDIST_OVERRIDE' not in self.env): + try: + # __object_name is the name of the object whose type + # manifest is currently executed + __object_name = self.env.get('__object_name', None) + # load object name created befor this one from typeorder + # dep file + if __object_name: + parent = self.cdist_object.object_from_name( + __object_name) + typeorder = parent.typeorder_dep + else: + typeorder = self._read_typeorder_dep() + # get the type created before this one + lastcreatedtype = typeorder[-2].strip() + if 'require' in self.env: + if lastcreatedtype not in self.env['require']: + self.env['require'] += " " + lastcreatedtype + else: + self.env['require'] = lastcreatedtype + self.log.debug(("Injecting require for " + "CDIST_ORDER_DEPENDENCY: %s for %s"), + lastcreatedtype, + self.cdist_object.name) + except IndexError: + # if no second last line, we are on the first type, + # so do not set a requirement + pass reqs = set() if "require" in self.env: diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index 5691093c..e375676c 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -24,8 +24,6 @@ import io import os import shutil -import string -import filecmp import random import logging @@ -34,7 +32,6 @@ from cdist import test from cdist.exec import local from cdist import emulator from cdist import core -from cdist import config import os.path as op my_dir = op.abspath(op.dirname(__file__)) @@ -115,7 +112,7 @@ class EmulatorTestCase(test.CdistTestCase): def test_requirement_pattern(self): argv = ['__file', '/tmp/foobar'] self.env['require'] = '__file/etc/*' - emu = emulator.Emulator(argv, env=self.env) + emulator.Emulator(argv, env=self.env) # if we get here all is fine def test_loglevel(self): @@ -172,6 +169,44 @@ class EmulatorTestCase(test.CdistTestCase): self.assertEqual(list(file_object.requirements), ['__planet/mars']) # if we get here all is fine + def test_order_dependency_context(self): + test_seq = ('A', True, 'B', 'C', 'D', False, 'E', 'F', True, 'G', + 'H', False, 'I', ) + expected_requirements = { + 'C': set(('__planet/B', )), + 'D': set(('__planet/C', )), + 'H': set(('__planet/G', )), + } + # Ensure env var is not in env + if 'CDIST_ORDER_DEPENDENCY' in self.env: + del self.env['CDIST_ORDER_DEPENDENCY'] + + for x in test_seq: + if isinstance(x, str): + # Clear because of order dep injection + # In real world, this is not shared over instances + if 'require' in self.env: + del self.env['require'] + argv = ['__planet', x] + emu = emulator.Emulator(argv, env=self.env) + emu.run() + elif isinstance(x, bool): + if x: + self.env['CDIST_ORDER_DEPENDENCY'] = 'on' + elif 'CDIST_ORDER_DEPENDENCY' in self.env: + del self.env['CDIST_ORDER_DEPENDENCY'] + cdist_type = core.CdistType(self.local.type_path, '__planet') + for x in test_seq: + if isinstance(x, str): + obj = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, x) + reqs = set(obj.requirements) + if x in expected_requirements: + self.assertEqual(reqs, expected_requirements[x]) + else: + self.assertTrue(len(reqs) == 0) + # if we get here all is fine + class EmulatorConflictingRequirementsTestCase(test.CdistTestCase):