forked from ungleich-public/cdist
Implement "onchange" support
"onchange" specifies execution prerequisites. It is specified in the same way as "require", it implies "require", but it specifies that object is executed only if any of specified prerequisites has changed target's state. This means that any of specified prerequisites has generated code to be executed, either local or remote or both. Object has generated code if it itself has generated code or if any of its children object has generated code. Resolve #843.
This commit is contained in:
parent
87698395b8
commit
7f32e5855a
6 changed files with 172 additions and 45 deletions
|
@ -788,11 +788,14 @@ class Config:
|
||||||
|
|
||||||
def object_prepare(self, cdist_object, transfer_type_explorers=True):
|
def object_prepare(self, cdist_object, transfer_type_explorers=True):
|
||||||
"""Prepare object: Run type explorer + manifest"""
|
"""Prepare object: Run type explorer + manifest"""
|
||||||
|
|
||||||
|
if cdist_object.processing_preconditions_satisfied():
|
||||||
self._handle_deprecation(cdist_object)
|
self._handle_deprecation(cdist_object)
|
||||||
self.log.verbose("Preparing object %s", cdist_object.name)
|
self.log.verbose("Preparing object %s", cdist_object.name)
|
||||||
self.log.verbose("Running manifest and explorers for %s",
|
self.log.verbose("Running manifest and explorers for %s",
|
||||||
cdist_object.name)
|
cdist_object.name)
|
||||||
self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
|
self.explorer.run_type_explorers(cdist_object,
|
||||||
|
transfer_type_explorers)
|
||||||
try:
|
try:
|
||||||
self.manifest.run_type_manifest(cdist_object)
|
self.manifest.run_type_manifest(cdist_object)
|
||||||
self.log.trace("[ORDER_DEP] Removing order dep files for %s",
|
self.log.trace("[ORDER_DEP] Removing order dep files for %s",
|
||||||
|
@ -801,6 +804,15 @@ class Config:
|
||||||
cdist_object.state = core.CdistObject.STATE_PREPARED
|
cdist_object.state = core.CdistObject.STATE_PREPARED
|
||||||
except cdist.Error as e:
|
except cdist.Error as e:
|
||||||
raise cdist.CdistObjectError(cdist_object, e)
|
raise cdist.CdistObjectError(cdist_object, e)
|
||||||
|
else:
|
||||||
|
cdist_object.state = core.CdistObject.STATE_DONE
|
||||||
|
cdist_object.skipped = True
|
||||||
|
cdist_object.skipped_reason = (
|
||||||
|
"Processing preconditions not satisfied for object {0}: {1}, "
|
||||||
|
"skipping {0} object processing".format(
|
||||||
|
cdist_object.name, cdist_object.processing_preconditions)
|
||||||
|
)
|
||||||
|
self.log.verbose(cdist_object.skipped_reason)
|
||||||
|
|
||||||
def object_run(self, cdist_object):
|
def object_run(self, cdist_object):
|
||||||
"""Run gencode and code for an object"""
|
"""Run gencode and code for an object"""
|
||||||
|
@ -817,9 +829,12 @@ class Config:
|
||||||
cdist_object)
|
cdist_object)
|
||||||
if cdist_object.code_local or cdist_object.code_remote:
|
if cdist_object.code_local or cdist_object.code_remote:
|
||||||
cdist_object.changed = True
|
cdist_object.changed = True
|
||||||
|
cdist_object.code_generated = True
|
||||||
|
else:
|
||||||
|
cdist_object.changed = False
|
||||||
|
|
||||||
# Execute
|
# Execute
|
||||||
if cdist_object.code_local or cdist_object.code_remote:
|
if cdist_object.changed:
|
||||||
self.log.info("Processing %s", cdist_object.name)
|
self.log.info("Processing %s", cdist_object.name)
|
||||||
if not self.dry_run:
|
if not self.dry_run:
|
||||||
if cdist_object.code_local:
|
if cdist_object.code_local:
|
||||||
|
|
|
@ -229,6 +229,8 @@ class CdistObject:
|
||||||
|
|
||||||
requirements = fsproperty.FileListProperty(
|
requirements = fsproperty.FileListProperty(
|
||||||
lambda obj: os.path.join(obj.absolute_path, 'require'))
|
lambda obj: os.path.join(obj.absolute_path, 'require'))
|
||||||
|
processing_preconditions = fsproperty.FileListProperty(
|
||||||
|
lambda obj: os.path.join(obj.absolute_path, 'onchange'))
|
||||||
autorequire = fsproperty.FileListProperty(
|
autorequire = fsproperty.FileListProperty(
|
||||||
lambda obj: os.path.join(obj.absolute_path, 'autorequire'))
|
lambda obj: os.path.join(obj.absolute_path, 'autorequire'))
|
||||||
parameters = fsproperty.DirectoryDictProperty(
|
parameters = fsproperty.DirectoryDictProperty(
|
||||||
|
@ -254,6 +256,12 @@ class CdistObject:
|
||||||
# types
|
# types
|
||||||
children = fsproperty.FileListProperty(
|
children = fsproperty.FileListProperty(
|
||||||
lambda obj: os.path.join(obj.absolute_path, 'children'))
|
lambda obj: os.path.join(obj.absolute_path, 'children'))
|
||||||
|
code_generated = fsproperty.FileBooleanProperty(
|
||||||
|
lambda obj: os.path.join(obj.absolute_path, "code_generated"))
|
||||||
|
skipped = fsproperty.FileBooleanProperty(
|
||||||
|
lambda obj: os.path.join(obj.absolute_path, "skipped"))
|
||||||
|
skipped_reason = fsproperty.FileStringProperty(
|
||||||
|
lambda obj: os.path.join(obj.absolute_path, "skipped_reason"))
|
||||||
|
|
||||||
def cleanup(self):
|
def cleanup(self):
|
||||||
try:
|
try:
|
||||||
|
@ -291,3 +299,24 @@ class CdistObject:
|
||||||
object_list.append(cdist_object)
|
object_list.append(cdist_object)
|
||||||
|
|
||||||
return object_list
|
return object_list
|
||||||
|
|
||||||
|
def _processing_preconditions_satisfied_dfs(self, current):
|
||||||
|
# DFS (depth first search) for type that generated code.
|
||||||
|
obj = self.object_from_name(current)
|
||||||
|
if obj.code_generated:
|
||||||
|
return True
|
||||||
|
# As soon as one child that generated code is found, result with True.
|
||||||
|
for child in obj.children:
|
||||||
|
rv = self._processing_preconditions_satisfied_dfs(child)
|
||||||
|
if rv:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def processing_preconditions_satisfied(self):
|
||||||
|
"""Return state whether processing preconditions are satisfied"""
|
||||||
|
|
||||||
|
for onchange in self.processing_preconditions:
|
||||||
|
if self._processing_preconditions_satisfied_dfs(onchange):
|
||||||
|
return True
|
||||||
|
|
||||||
|
return len(self.processing_preconditions) == 0
|
||||||
|
|
|
@ -104,7 +104,7 @@ class Emulator:
|
||||||
with flock.Flock(self.flock_path):
|
with flock.Flock(self.flock_path):
|
||||||
self.setup_object()
|
self.setup_object()
|
||||||
self.save_stdin()
|
self.save_stdin()
|
||||||
self.record_requirements()
|
self.record_requirements_and_processing_preconditions()
|
||||||
self.record_auto_requirements()
|
self.record_auto_requirements()
|
||||||
self.record_parent_child_relationships()
|
self.record_parent_child_relationships()
|
||||||
self.log.trace("Finished %s %s", self.cdist_object.path,
|
self.log.trace("Finished %s %s", self.cdist_object.path,
|
||||||
|
@ -294,30 +294,43 @@ class Emulator:
|
||||||
except EnvironmentError as e:
|
except EnvironmentError as e:
|
||||||
raise cdist.Error('Failed to read from stdin: {}'.format(e))
|
raise cdist.Error('Failed to read from stdin: {}'.format(e))
|
||||||
|
|
||||||
def record_requirement(self, requirement):
|
def _record_dep(self, object_name, attr_name, msg):
|
||||||
"""record requirement and return recorded requirement"""
|
"""
|
||||||
|
Record dependency for attr_name and return recorded value.
|
||||||
|
"""
|
||||||
|
|
||||||
# Raises an error, if object cannot be created
|
# Raises an error, if object cannot be created
|
||||||
try:
|
try:
|
||||||
cdist_object = self.cdist_object.object_from_name(requirement)
|
cdist_object = self.cdist_object.object_from_name(object_name)
|
||||||
except core.cdist_type.InvalidTypeError as e:
|
except core.cdist_type.InvalidTypeError as e:
|
||||||
self.log.error("%s requires object %s, but type %s does not"
|
self.log.error("%s %s object %s, but type %s does not exist."
|
||||||
" exist. Defined at %s", self.cdist_object.name,
|
" Defined at %s", msg, self.cdist_object.name,
|
||||||
requirement, e.name, self.object_source)
|
object_name, e.name, self.object_source)
|
||||||
raise
|
raise
|
||||||
except core.cdist_object.MissingObjectIdError:
|
except core.cdist_object.MissingObjectIdError:
|
||||||
self.log.error("%s requires object %s without object id."
|
self.log.error("%s %s object %s without object id. Defined at %s",
|
||||||
" Defined at %s", self.cdist_object.name,
|
self.cdist_object.name, msg, object_name,
|
||||||
requirement, self.object_source)
|
self.object_source)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
self.log.debug("Recording requirement %s for %s",
|
self.log.debug("Recording %s in %s for %s",
|
||||||
requirement, self.cdist_object.name)
|
object_name, attr_name, self.cdist_object.name)
|
||||||
|
|
||||||
# Save the sanitised version, not the user supplied one
|
# Save the sanitised version, not the user supplied one
|
||||||
# (__file//bar => __file/bar)
|
# (__file//bar => __file/bar)
|
||||||
# This ensures pattern matching is done against sanitised list
|
# This ensures pattern matching is done against sanitised list
|
||||||
self.cdist_object.requirements.append(cdist_object.name)
|
attr_list = getattr(self.cdist_object, attr_name)
|
||||||
|
if cdist_object.name not in attr_list:
|
||||||
|
attr_list.append(cdist_object.name)
|
||||||
|
|
||||||
|
def _record_requirement(self, requirement):
|
||||||
|
"""record requirement and return recorded requirement"""
|
||||||
|
self._record_dep(requirement, 'requirements', 'requires')
|
||||||
|
|
||||||
|
def _record_processing_precondition(self, onchange):
|
||||||
|
"""record processing precondition and return t"""
|
||||||
|
self._record_dep(onchange, 'processing_preconditions',
|
||||||
|
'has processing precondition')
|
||||||
|
|
||||||
def _order_dep_on(self):
|
def _order_dep_on(self):
|
||||||
return os.path.exists(self.order_dep_state_path)
|
return os.path.exists(self.order_dep_state_path)
|
||||||
|
@ -351,8 +364,25 @@ class Emulator:
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def record_requirements(self):
|
def _inject_dependency(self, lastcreatedtype, dep_var):
|
||||||
"""Record requirements."""
|
if dep_var in self.env:
|
||||||
|
if lastcreatedtype not in self.env[dep_var]:
|
||||||
|
self.env[dep_var] += " " + lastcreatedtype
|
||||||
|
else:
|
||||||
|
self.env[dep_var] = lastcreatedtype
|
||||||
|
|
||||||
|
def _process_dep_var(self, func, dep_var):
|
||||||
|
if dep_var in self.env:
|
||||||
|
objects = self.env[dep_var]
|
||||||
|
self.log.debug("reqs for %s = %s", dep_var, objects)
|
||||||
|
for requirement in self._parse_require(objects):
|
||||||
|
# Ignore empty fields - probably the only field anyway
|
||||||
|
if len(requirement) == 0:
|
||||||
|
continue
|
||||||
|
func(requirement)
|
||||||
|
|
||||||
|
def record_requirements_and_processing_preconditions(self):
|
||||||
|
"""Record requirements and processing precondition."""
|
||||||
|
|
||||||
order_dep_on = self._order_dep_on()
|
order_dep_on = self._order_dep_on()
|
||||||
|
|
||||||
|
@ -373,27 +403,23 @@ class Emulator:
|
||||||
typeorder = self._read_typeorder_dep()
|
typeorder = self._read_typeorder_dep()
|
||||||
# get the type created before this one
|
# get the type created before this one
|
||||||
lastcreatedtype = typeorder[-2].strip()
|
lastcreatedtype = typeorder[-2].strip()
|
||||||
if 'require' in self.env:
|
self.log.debug(("Injecting require for "
|
||||||
if lastcreatedtype not in self.env['require']:
|
"CDIST_ORDER_DEPENDENCY: %s for %s"),
|
||||||
self.env['require'] += " " + lastcreatedtype
|
lastcreatedtype,
|
||||||
else:
|
self.cdist_object.name)
|
||||||
self.env['require'] = lastcreatedtype
|
self._inject_dependency(lastcreatedtype, 'require')
|
||||||
self.log.debug("Injecting require for"
|
|
||||||
" CDIST_ORDER_DEPENDENCY: %s for %s",
|
|
||||||
lastcreatedtype, self.cdist_object.name)
|
|
||||||
except IndexError:
|
except IndexError:
|
||||||
# if no second last line, we are on the first type,
|
# if no second last line, we are on the first type,
|
||||||
# so do not set a requirement
|
# so do not set a requirement
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if "require" in self.env:
|
self._process_dep_var(func=self._record_requirement,
|
||||||
requirements = self.env['require']
|
dep_var='require')
|
||||||
self.log.debug("reqs = %s", requirements)
|
# onchange implies require
|
||||||
for requirement in self._parse_require(requirements):
|
self._process_dep_var(func=self._record_requirement,
|
||||||
# Ignore empty fields - probably the only field anyway
|
dep_var='onchange')
|
||||||
if len(requirement) == 0:
|
self._process_dep_var(func=self._record_processing_precondition,
|
||||||
continue
|
dep_var='onchange')
|
||||||
self.record_requirement(requirement)
|
|
||||||
|
|
||||||
def _parse_require(self, require):
|
def _parse_require(self, require):
|
||||||
return re.split(r'[ \t\n]+', require)
|
return re.split(r'[ \t\n]+', require)
|
||||||
|
|
|
@ -207,6 +207,40 @@ class EmulatorTestCase(test.CdistTestCase):
|
||||||
self.assertTrue(len(reqs) == 0)
|
self.assertTrue(len(reqs) == 0)
|
||||||
# if we get here all is fine
|
# if we get here all is fine
|
||||||
|
|
||||||
|
def test_onchange(self):
|
||||||
|
argv = ['__planet', 'erde']
|
||||||
|
emu = emulator.Emulator(argv, env=self.env)
|
||||||
|
emu.run()
|
||||||
|
argv = ['__planet', 'mars']
|
||||||
|
emu = emulator.Emulator(argv, env=self.env)
|
||||||
|
emu.run()
|
||||||
|
self.env['onchange'] = '__planet/erde __planet/mars'
|
||||||
|
argv = ['__file', '/tmp/cdisttest']
|
||||||
|
emu = emulator.Emulator(argv, env=self.env)
|
||||||
|
emu.run()
|
||||||
|
# now load the objects and verify the require parameter of the objects
|
||||||
|
cdist_type = core.CdistType(self.local.type_path, '__planet')
|
||||||
|
erde_object = core.CdistObject(cdist_type, self.local.object_path,
|
||||||
|
self.local.object_marker_name, 'erde')
|
||||||
|
mars_object = core.CdistObject(cdist_type, self.local.object_path,
|
||||||
|
self.local.object_marker_name, 'mars')
|
||||||
|
cdist_type = core.CdistType(self.local.type_path, '__file')
|
||||||
|
file_object = core.CdistObject(cdist_type, self.local.object_path,
|
||||||
|
self.local.object_marker_name,
|
||||||
|
'/tmp/cdisttest')
|
||||||
|
# now test the recorded exec prerequisites and requirements
|
||||||
|
self.assertTrue(len(erde_object.requirements) == 0)
|
||||||
|
self.assertTrue(len(mars_object.requirements) == 0)
|
||||||
|
self.assertTrue(len(file_object.requirements) > 0)
|
||||||
|
self.assertTrue(len(erde_object.processing_preconditions) == 0)
|
||||||
|
self.assertTrue(len(mars_object.processing_preconditions) == 0)
|
||||||
|
self.assertTrue(len(file_object.processing_preconditions) > 0)
|
||||||
|
self.assertEqual(list(file_object.requirements),
|
||||||
|
['__planet/erde', '__planet/mars'])
|
||||||
|
self.assertEqual(list(file_object.processing_preconditions),
|
||||||
|
['__planet/erde', '__planet/mars'])
|
||||||
|
# if we get here all is fine
|
||||||
|
|
||||||
|
|
||||||
class EmulatorConflictingRequirementsTestCase(test.CdistTestCase):
|
class EmulatorConflictingRequirementsTestCase(test.CdistTestCase):
|
||||||
|
|
||||||
|
|
|
@ -154,6 +154,16 @@ You can find a more in depth description of the flow execution of manifests
|
||||||
in `cdist execution stages <cdist-stages.html>`_ and of how types work in `cdist type <cdist-type.html>`_.
|
in `cdist execution stages <cdist-stages.html>`_ and of how types work in `cdist type <cdist-type.html>`_.
|
||||||
|
|
||||||
|
|
||||||
|
Processing preconditions
|
||||||
|
------------------------
|
||||||
|
If you want to describe that something should be processed only if specified
|
||||||
|
objects changed target's state, i.e. specified objects generated either
|
||||||
|
"code-local" or "code-remote", or both, just setup the variable "onchange" to
|
||||||
|
contain processing preconditions. "onchange" is specified in the same way as
|
||||||
|
"require", and it implies "require". If processing preconditions are not
|
||||||
|
satisfied then it means that object's explorers are not run, manifest is not
|
||||||
|
run, and gencode scripts are not run. Object processing is skipped.
|
||||||
|
|
||||||
Create dependencies from execution order
|
Create dependencies from execution order
|
||||||
-----------------------------------------
|
-----------------------------------------
|
||||||
You can tell cdist to execute all types in the order in which they are created
|
You can tell cdist to execute all types in the order in which they are created
|
||||||
|
@ -375,3 +385,13 @@ Dependencies defined by execution order work as following:
|
||||||
require="__some_type_somewhere/id __sample_type/1" __sample_type 2
|
require="__some_type_somewhere/id __sample_type/1" __sample_type 2
|
||||||
require="__sample_type/2" __example_type 23
|
require="__sample_type/2" __example_type 23
|
||||||
__not_in_order_type 42
|
__not_in_order_type 42
|
||||||
|
|
||||||
|
This example makes use of processing preconditions:
|
||||||
|
|
||||||
|
.. code-block:: sh
|
||||||
|
|
||||||
|
__foo bar
|
||||||
|
__spam eggs
|
||||||
|
# execute __baz type only if any of __foo/bar, __spam/eggs has changed
|
||||||
|
# target's state, i.e. code-local or code-remote is generated (or both)
|
||||||
|
onchange='__foo/bar __spam/eggs' __baz
|
||||||
|
|
|
@ -314,6 +314,9 @@ The following environment variables influence the behaviour of cdist:
|
||||||
require
|
require
|
||||||
Setup dependencies between objects (see \`cdist manifest <cdist-manifest.html>\`_).
|
Setup dependencies between objects (see \`cdist manifest <cdist-manifest.html>\`_).
|
||||||
|
|
||||||
|
onchange
|
||||||
|
Setup processing preconditions (implies dependencies) between objects (see `cdist manifest <cdist-manifest.html>`_).
|
||||||
|
|
||||||
__cdist_log_level
|
__cdist_log_level
|
||||||
cdist log level value. One of:
|
cdist log level value. One of:
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue