diff --git a/cdist/config.py b/cdist/config.py index adc460de..56967ef6 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -788,19 +788,31 @@ class Config: def object_prepare(self, cdist_object, transfer_type_explorers=True): """Prepare object: Run type explorer + manifest""" - self._handle_deprecation(cdist_object) - self.log.verbose("Preparing object %s", cdist_object.name) - self.log.verbose("Running manifest and explorers for %s", - cdist_object.name) - self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) - try: - self.manifest.run_type_manifest(cdist_object) - self.log.trace("[ORDER_DEP] Removing order dep files for %s", - cdist_object) - cdist_object.cleanup() - cdist_object.state = core.CdistObject.STATE_PREPARED - except cdist.Error as e: - raise cdist.CdistObjectError(cdist_object, e) + + if cdist_object.processing_preconditions_satisfied(): + self._handle_deprecation(cdist_object) + self.log.verbose("Preparing object %s", cdist_object.name) + self.log.verbose("Running manifest and explorers for %s", + cdist_object.name) + self.explorer.run_type_explorers(cdist_object, + transfer_type_explorers) + try: + self.manifest.run_type_manifest(cdist_object) + self.log.trace("[ORDER_DEP] Removing order dep files for %s", + cdist_object) + cdist_object.cleanup() + cdist_object.state = core.CdistObject.STATE_PREPARED + except cdist.Error as e: + raise cdist.CdistObjectError(cdist_object, e) + else: + cdist_object.state = core.CdistObject.STATE_DONE + cdist_object.skipped = True + cdist_object.skipped_reason = ( + "Processing preconditions not satisfied for object {0}: {1}, " + "skipping {0} object processing".format( + cdist_object.name, cdist_object.processing_preconditions) + ) + self.log.verbose(cdist_object.skipped_reason) def object_run(self, cdist_object): """Run gencode and code for an object""" @@ -817,9 +829,12 @@ class Config: cdist_object) if cdist_object.code_local or cdist_object.code_remote: cdist_object.changed = True + cdist_object.code_generated = True + else: + cdist_object.changed = False # Execute - if cdist_object.code_local or cdist_object.code_remote: + if cdist_object.changed: self.log.info("Processing %s", cdist_object.name) if not self.dry_run: if cdist_object.code_local: diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index ccf7392d..653608cc 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -229,6 +229,8 @@ class CdistObject: requirements = fsproperty.FileListProperty( lambda obj: os.path.join(obj.absolute_path, 'require')) + processing_preconditions = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, 'onchange')) autorequire = fsproperty.FileListProperty( lambda obj: os.path.join(obj.absolute_path, 'autorequire')) parameters = fsproperty.DirectoryDictProperty( @@ -254,6 +256,12 @@ class CdistObject: # types children = fsproperty.FileListProperty( lambda obj: os.path.join(obj.absolute_path, 'children')) + code_generated = fsproperty.FileBooleanProperty( + lambda obj: os.path.join(obj.absolute_path, "code_generated")) + skipped = fsproperty.FileBooleanProperty( + lambda obj: os.path.join(obj.absolute_path, "skipped")) + skipped_reason = fsproperty.FileStringProperty( + lambda obj: os.path.join(obj.absolute_path, "skipped_reason")) def cleanup(self): try: @@ -291,3 +299,24 @@ class CdistObject: object_list.append(cdist_object) return object_list + + def _processing_preconditions_satisfied_dfs(self, current): + # DFS (depth first search) for type that generated code. + obj = self.object_from_name(current) + if obj.code_generated: + return True + # As soon as one child that generated code is found, result with True. + for child in obj.children: + rv = self._processing_preconditions_satisfied_dfs(child) + if rv: + return True + return False + + def processing_preconditions_satisfied(self): + """Return state whether processing preconditions are satisfied""" + + for onchange in self.processing_preconditions: + if self._processing_preconditions_satisfied_dfs(onchange): + return True + + return len(self.processing_preconditions) == 0 diff --git a/cdist/emulator.py b/cdist/emulator.py index c55b47d2..dd6f9dc6 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -104,7 +104,7 @@ class Emulator: with flock.Flock(self.flock_path): self.setup_object() self.save_stdin() - self.record_requirements() + self.record_requirements_and_processing_preconditions() self.record_auto_requirements() self.record_parent_child_relationships() self.log.trace("Finished %s %s", self.cdist_object.path, @@ -294,30 +294,43 @@ class Emulator: except EnvironmentError as e: raise cdist.Error('Failed to read from stdin: {}'.format(e)) - def record_requirement(self, requirement): - """record requirement and return recorded requirement""" + def _record_dep(self, object_name, attr_name, msg): + """ + Record dependency for attr_name and return recorded value. + """ # Raises an error, if object cannot be created try: - cdist_object = self.cdist_object.object_from_name(requirement) + cdist_object = self.cdist_object.object_from_name(object_name) except core.cdist_type.InvalidTypeError as e: - self.log.error("%s requires object %s, but type %s does not" - " exist. Defined at %s", self.cdist_object.name, - requirement, e.name, self.object_source) + self.log.error("%s %s object %s, but type %s does not exist." + " Defined at %s", msg, self.cdist_object.name, + object_name, e.name, self.object_source) raise except core.cdist_object.MissingObjectIdError: - self.log.error("%s requires object %s without object id." - " Defined at %s", self.cdist_object.name, - requirement, self.object_source) + self.log.error("%s %s object %s without object id. Defined at %s", + self.cdist_object.name, msg, object_name, + self.object_source) raise - self.log.debug("Recording requirement %s for %s", - requirement, self.cdist_object.name) + self.log.debug("Recording %s in %s for %s", + object_name, attr_name, self.cdist_object.name) # Save the sanitised version, not the user supplied one # (__file//bar => __file/bar) # This ensures pattern matching is done against sanitised list - self.cdist_object.requirements.append(cdist_object.name) + attr_list = getattr(self.cdist_object, attr_name) + if cdist_object.name not in attr_list: + attr_list.append(cdist_object.name) + + def _record_requirement(self, requirement): + """record requirement and return recorded requirement""" + self._record_dep(requirement, 'requirements', 'requires') + + def _record_processing_precondition(self, onchange): + """record processing precondition and return t""" + self._record_dep(onchange, 'processing_preconditions', + 'has processing precondition') def _order_dep_on(self): return os.path.exists(self.order_dep_state_path) @@ -351,8 +364,25 @@ class Emulator: except FileNotFoundError: return [] - def record_requirements(self): - """Record requirements.""" + def _inject_dependency(self, lastcreatedtype, dep_var): + if dep_var in self.env: + if lastcreatedtype not in self.env[dep_var]: + self.env[dep_var] += " " + lastcreatedtype + else: + self.env[dep_var] = lastcreatedtype + + def _process_dep_var(self, func, dep_var): + if dep_var in self.env: + objects = self.env[dep_var] + self.log.debug("reqs for %s = %s", dep_var, objects) + for requirement in self._parse_require(objects): + # Ignore empty fields - probably the only field anyway + if len(requirement) == 0: + continue + func(requirement) + + def record_requirements_and_processing_preconditions(self): + """Record requirements and processing precondition.""" order_dep_on = self._order_dep_on() @@ -373,27 +403,23 @@ class Emulator: typeorder = self._read_typeorder_dep() # get the type created before this one lastcreatedtype = typeorder[-2].strip() - if 'require' in self.env: - if lastcreatedtype not in self.env['require']: - self.env['require'] += " " + lastcreatedtype - else: - self.env['require'] = lastcreatedtype - self.log.debug("Injecting require for" - " CDIST_ORDER_DEPENDENCY: %s for %s", - lastcreatedtype, self.cdist_object.name) + self.log.debug(("Injecting require for " + "CDIST_ORDER_DEPENDENCY: %s for %s"), + lastcreatedtype, + self.cdist_object.name) + self._inject_dependency(lastcreatedtype, 'require') except IndexError: # if no second last line, we are on the first type, # so do not set a requirement pass - if "require" in self.env: - requirements = self.env['require'] - self.log.debug("reqs = %s", requirements) - for requirement in self._parse_require(requirements): - # Ignore empty fields - probably the only field anyway - if len(requirement) == 0: - continue - self.record_requirement(requirement) + self._process_dep_var(func=self._record_requirement, + dep_var='require') + # onchange implies require + self._process_dep_var(func=self._record_requirement, + dep_var='onchange') + self._process_dep_var(func=self._record_processing_precondition, + dep_var='onchange') def _parse_require(self, require): return re.split(r'[ \t\n]+', require) diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index 4b2bc3ba..29b68d90 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -207,6 +207,40 @@ class EmulatorTestCase(test.CdistTestCase): self.assertTrue(len(reqs) == 0) # if we get here all is fine + def test_onchange(self): + argv = ['__planet', 'erde'] + emu = emulator.Emulator(argv, env=self.env) + emu.run() + argv = ['__planet', 'mars'] + emu = emulator.Emulator(argv, env=self.env) + emu.run() + self.env['onchange'] = '__planet/erde __planet/mars' + argv = ['__file', '/tmp/cdisttest'] + emu = emulator.Emulator(argv, env=self.env) + emu.run() + # now load the objects and verify the require parameter of the objects + cdist_type = core.CdistType(self.local.type_path, '__planet') + erde_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, 'erde') + mars_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, 'mars') + cdist_type = core.CdistType(self.local.type_path, '__file') + file_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + '/tmp/cdisttest') + # now test the recorded exec prerequisites and requirements + self.assertTrue(len(erde_object.requirements) == 0) + self.assertTrue(len(mars_object.requirements) == 0) + self.assertTrue(len(file_object.requirements) > 0) + self.assertTrue(len(erde_object.processing_preconditions) == 0) + self.assertTrue(len(mars_object.processing_preconditions) == 0) + self.assertTrue(len(file_object.processing_preconditions) > 0) + self.assertEqual(list(file_object.requirements), + ['__planet/erde', '__planet/mars']) + self.assertEqual(list(file_object.processing_preconditions), + ['__planet/erde', '__planet/mars']) + # if we get here all is fine + class EmulatorConflictingRequirementsTestCase(test.CdistTestCase): diff --git a/docs/src/cdist-manifest.rst b/docs/src/cdist-manifest.rst index 2e49a721..7c3120bb 100644 --- a/docs/src/cdist-manifest.rst +++ b/docs/src/cdist-manifest.rst @@ -154,6 +154,16 @@ You can find a more in depth description of the flow execution of manifests in `cdist execution stages `_ and of how types work in `cdist type `_. +Processing preconditions +------------------------ +If you want to describe that something should be processed only if specified +objects changed target's state, i.e. specified objects generated either +"code-local" or "code-remote", or both, just setup the variable "onchange" to +contain processing preconditions. "onchange" is specified in the same way as +"require", and it implies "require". If processing preconditions are not +satisfied then it means that object's explorers are not run, manifest is not +run, and gencode scripts are not run. Object processing is skipped. + Create dependencies from execution order ----------------------------------------- You can tell cdist to execute all types in the order in which they are created @@ -375,3 +385,13 @@ Dependencies defined by execution order work as following: require="__some_type_somewhere/id __sample_type/1" __sample_type 2 require="__sample_type/2" __example_type 23 __not_in_order_type 42 + +This example makes use of processing preconditions: + +.. code-block:: sh + + __foo bar + __spam eggs + # execute __baz type only if any of __foo/bar, __spam/eggs has changed + # target's state, i.e. code-local or code-remote is generated (or both) + onchange='__foo/bar __spam/eggs' __baz diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index c0ac2c3e..01f1b963 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -314,6 +314,9 @@ The following environment variables influence the behaviour of cdist: require Setup dependencies between objects (see \`cdist manifest \`_). +onchange + Setup processing preconditions (implies dependencies) between objects (see `cdist manifest `_). + __cdist_log_level cdist log level value. One of: