forked from ungleich-public/cdist
Compare commits
1 commit
master
...
feature/on
Author | SHA1 | Date | |
---|---|---|---|
|
7f32e5855a |
6 changed files with 172 additions and 45 deletions
|
@ -788,11 +788,14 @@ class Config:
|
||||||
|
|
||||||
def object_prepare(self, cdist_object, transfer_type_explorers=True):
|
def object_prepare(self, cdist_object, transfer_type_explorers=True):
|
||||||
"""Prepare object: Run type explorer + manifest"""
|
"""Prepare object: Run type explorer + manifest"""
|
||||||
|
|
||||||
|
if cdist_object.processing_preconditions_satisfied():
|
||||||
self._handle_deprecation(cdist_object)
|
self._handle_deprecation(cdist_object)
|
||||||
self.log.verbose("Preparing object %s", cdist_object.name)
|
self.log.verbose("Preparing object %s", cdist_object.name)
|
||||||
self.log.verbose("Running manifest and explorers for %s",
|
self.log.verbose("Running manifest and explorers for %s",
|
||||||
cdist_object.name)
|
cdist_object.name)
|
||||||
self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
|
self.explorer.run_type_explorers(cdist_object,
|
||||||
|
transfer_type_explorers)
|
||||||
try:
|
try:
|
||||||
self.manifest.run_type_manifest(cdist_object)
|
self.manifest.run_type_manifest(cdist_object)
|
||||||
self.log.trace("[ORDER_DEP] Removing order dep files for %s",
|
self.log.trace("[ORDER_DEP] Removing order dep files for %s",
|
||||||
|
@ -801,6 +804,15 @@ class Config:
|
||||||
cdist_object.state = core.CdistObject.STATE_PREPARED
|
cdist_object.state = core.CdistObject.STATE_PREPARED
|
||||||
except cdist.Error as e:
|
except cdist.Error as e:
|
||||||
raise cdist.CdistObjectError(cdist_object, e)
|
raise cdist.CdistObjectError(cdist_object, e)
|
||||||
|
else:
|
||||||
|
cdist_object.state = core.CdistObject.STATE_DONE
|
||||||
|
cdist_object.skipped = True
|
||||||
|
cdist_object.skipped_reason = (
|
||||||
|
"Processing preconditions not satisfied for object {0}: {1}, "
|
||||||
|
"skipping {0} object processing".format(
|
||||||
|
cdist_object.name, cdist_object.processing_preconditions)
|
||||||
|
)
|
||||||
|
self.log.verbose(cdist_object.skipped_reason)
|
||||||
|
|
||||||
def object_run(self, cdist_object):
|
def object_run(self, cdist_object):
|
||||||
"""Run gencode and code for an object"""
|
"""Run gencode and code for an object"""
|
||||||
|
@ -817,9 +829,12 @@ class Config:
|
||||||
cdist_object)
|
cdist_object)
|
||||||
if cdist_object.code_local or cdist_object.code_remote:
|
if cdist_object.code_local or cdist_object.code_remote:
|
||||||
cdist_object.changed = True
|
cdist_object.changed = True
|
||||||
|
cdist_object.code_generated = True
|
||||||
|
else:
|
||||||
|
cdist_object.changed = False
|
||||||
|
|
||||||
# Execute
|
# Execute
|
||||||
if cdist_object.code_local or cdist_object.code_remote:
|
if cdist_object.changed:
|
||||||
self.log.info("Processing %s", cdist_object.name)
|
self.log.info("Processing %s", cdist_object.name)
|
||||||
if not self.dry_run:
|
if not self.dry_run:
|
||||||
if cdist_object.code_local:
|
if cdist_object.code_local:
|
||||||
|
|
|
@ -229,6 +229,8 @@ class CdistObject:
|
||||||
|
|
||||||
requirements = fsproperty.FileListProperty(
|
requirements = fsproperty.FileListProperty(
|
||||||
lambda obj: os.path.join(obj.absolute_path, 'require'))
|
lambda obj: os.path.join(obj.absolute_path, 'require'))
|
||||||
|
processing_preconditions = fsproperty.FileListProperty(
|
||||||
|
lambda obj: os.path.join(obj.absolute_path, 'onchange'))
|
||||||
autorequire = fsproperty.FileListProperty(
|
autorequire = fsproperty.FileListProperty(
|
||||||
lambda obj: os.path.join(obj.absolute_path, 'autorequire'))
|
lambda obj: os.path.join(obj.absolute_path, 'autorequire'))
|
||||||
parameters = fsproperty.DirectoryDictProperty(
|
parameters = fsproperty.DirectoryDictProperty(
|
||||||
|
@ -254,6 +256,12 @@ class CdistObject:
|
||||||
# types
|
# types
|
||||||
children = fsproperty.FileListProperty(
|
children = fsproperty.FileListProperty(
|
||||||
lambda obj: os.path.join(obj.absolute_path, 'children'))
|
lambda obj: os.path.join(obj.absolute_path, 'children'))
|
||||||
|
code_generated = fsproperty.FileBooleanProperty(
|
||||||
|
lambda obj: os.path.join(obj.absolute_path, "code_generated"))
|
||||||
|
skipped = fsproperty.FileBooleanProperty(
|
||||||
|
lambda obj: os.path.join(obj.absolute_path, "skipped"))
|
||||||
|
skipped_reason = fsproperty.FileStringProperty(
|
||||||
|
lambda obj: os.path.join(obj.absolute_path, "skipped_reason"))
|
||||||
|
|
||||||
def cleanup(self):
|
def cleanup(self):
|
||||||
try:
|
try:
|
||||||
|
@ -291,3 +299,24 @@ class CdistObject:
|
||||||
object_list.append(cdist_object)
|
object_list.append(cdist_object)
|
||||||
|
|
||||||
return object_list
|
return object_list
|
||||||
|
|
||||||
|
def _processing_preconditions_satisfied_dfs(self, current):
|
||||||
|
# DFS (depth first search) for type that generated code.
|
||||||
|
obj = self.object_from_name(current)
|
||||||
|
if obj.code_generated:
|
||||||
|
return True
|
||||||
|
# As soon as one child that generated code is found, result with True.
|
||||||
|
for child in obj.children:
|
||||||
|
rv = self._processing_preconditions_satisfied_dfs(child)
|
||||||
|
if rv:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def processing_preconditions_satisfied(self):
|
||||||
|
"""Return state whether processing preconditions are satisfied"""
|
||||||
|
|
||||||
|
for onchange in self.processing_preconditions:
|
||||||
|
if self._processing_preconditions_satisfied_dfs(onchange):
|
||||||
|
return True
|
||||||
|
|
||||||
|
return len(self.processing_preconditions) == 0
|
||||||
|
|
|
@ -104,7 +104,7 @@ class Emulator:
|
||||||
with flock.Flock(self.flock_path):
|
with flock.Flock(self.flock_path):
|
||||||
self.setup_object()
|
self.setup_object()
|
||||||
self.save_stdin()
|
self.save_stdin()
|
||||||
self.record_requirements()
|
self.record_requirements_and_processing_preconditions()
|
||||||
self.record_auto_requirements()
|
self.record_auto_requirements()
|
||||||
self.record_parent_child_relationships()
|
self.record_parent_child_relationships()
|
||||||
self.log.trace("Finished %s %s", self.cdist_object.path,
|
self.log.trace("Finished %s %s", self.cdist_object.path,
|
||||||
|
@ -294,30 +294,43 @@ class Emulator:
|
||||||
except EnvironmentError as e:
|
except EnvironmentError as e:
|
||||||
raise cdist.Error('Failed to read from stdin: {}'.format(e))
|
raise cdist.Error('Failed to read from stdin: {}'.format(e))
|
||||||
|
|
||||||
def record_requirement(self, requirement):
|
def _record_dep(self, object_name, attr_name, msg):
|
||||||
"""record requirement and return recorded requirement"""
|
"""
|
||||||
|
Record dependency for attr_name and return recorded value.
|
||||||
|
"""
|
||||||
|
|
||||||
# Raises an error, if object cannot be created
|
# Raises an error, if object cannot be created
|
||||||
try:
|
try:
|
||||||
cdist_object = self.cdist_object.object_from_name(requirement)
|
cdist_object = self.cdist_object.object_from_name(object_name)
|
||||||
except core.cdist_type.InvalidTypeError as e:
|
except core.cdist_type.InvalidTypeError as e:
|
||||||
self.log.error("%s requires object %s, but type %s does not"
|
self.log.error("%s %s object %s, but type %s does not exist."
|
||||||
" exist. Defined at %s", self.cdist_object.name,
|
" Defined at %s", msg, self.cdist_object.name,
|
||||||
requirement, e.name, self.object_source)
|
object_name, e.name, self.object_source)
|
||||||
raise
|
raise
|
||||||
except core.cdist_object.MissingObjectIdError:
|
except core.cdist_object.MissingObjectIdError:
|
||||||
self.log.error("%s requires object %s without object id."
|
self.log.error("%s %s object %s without object id. Defined at %s",
|
||||||
" Defined at %s", self.cdist_object.name,
|
self.cdist_object.name, msg, object_name,
|
||||||
requirement, self.object_source)
|
self.object_source)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
self.log.debug("Recording requirement %s for %s",
|
self.log.debug("Recording %s in %s for %s",
|
||||||
requirement, self.cdist_object.name)
|
object_name, attr_name, self.cdist_object.name)
|
||||||
|
|
||||||
# Save the sanitised version, not the user supplied one
|
# Save the sanitised version, not the user supplied one
|
||||||
# (__file//bar => __file/bar)
|
# (__file//bar => __file/bar)
|
||||||
# This ensures pattern matching is done against sanitised list
|
# This ensures pattern matching is done against sanitised list
|
||||||
self.cdist_object.requirements.append(cdist_object.name)
|
attr_list = getattr(self.cdist_object, attr_name)
|
||||||
|
if cdist_object.name not in attr_list:
|
||||||
|
attr_list.append(cdist_object.name)
|
||||||
|
|
||||||
|
def _record_requirement(self, requirement):
|
||||||
|
"""record requirement and return recorded requirement"""
|
||||||
|
self._record_dep(requirement, 'requirements', 'requires')
|
||||||
|
|
||||||
|
def _record_processing_precondition(self, onchange):
|
||||||
|
"""record processing precondition and return t"""
|
||||||
|
self._record_dep(onchange, 'processing_preconditions',
|
||||||
|
'has processing precondition')
|
||||||
|
|
||||||
def _order_dep_on(self):
|
def _order_dep_on(self):
|
||||||
return os.path.exists(self.order_dep_state_path)
|
return os.path.exists(self.order_dep_state_path)
|
||||||
|
@ -351,8 +364,25 @@ class Emulator:
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def record_requirements(self):
|
def _inject_dependency(self, lastcreatedtype, dep_var):
|
||||||
"""Record requirements."""
|
if dep_var in self.env:
|
||||||
|
if lastcreatedtype not in self.env[dep_var]:
|
||||||
|
self.env[dep_var] += " " + lastcreatedtype
|
||||||
|
else:
|
||||||
|
self.env[dep_var] = lastcreatedtype
|
||||||
|
|
||||||
|
def _process_dep_var(self, func, dep_var):
|
||||||
|
if dep_var in self.env:
|
||||||
|
objects = self.env[dep_var]
|
||||||
|
self.log.debug("reqs for %s = %s", dep_var, objects)
|
||||||
|
for requirement in self._parse_require(objects):
|
||||||
|
# Ignore empty fields - probably the only field anyway
|
||||||
|
if len(requirement) == 0:
|
||||||
|
continue
|
||||||
|
func(requirement)
|
||||||
|
|
||||||
|
def record_requirements_and_processing_preconditions(self):
|
||||||
|
"""Record requirements and processing precondition."""
|
||||||
|
|
||||||
order_dep_on = self._order_dep_on()
|
order_dep_on = self._order_dep_on()
|
||||||
|
|
||||||
|
@ -373,27 +403,23 @@ class Emulator:
|
||||||
typeorder = self._read_typeorder_dep()
|
typeorder = self._read_typeorder_dep()
|
||||||
# get the type created before this one
|
# get the type created before this one
|
||||||
lastcreatedtype = typeorder[-2].strip()
|
lastcreatedtype = typeorder[-2].strip()
|
||||||
if 'require' in self.env:
|
self.log.debug(("Injecting require for "
|
||||||
if lastcreatedtype not in self.env['require']:
|
"CDIST_ORDER_DEPENDENCY: %s for %s"),
|
||||||
self.env['require'] += " " + lastcreatedtype
|
lastcreatedtype,
|
||||||
else:
|
self.cdist_object.name)
|
||||||
self.env['require'] = lastcreatedtype
|
self._inject_dependency(lastcreatedtype, 'require')
|
||||||
self.log.debug("Injecting require for"
|
|
||||||
" CDIST_ORDER_DEPENDENCY: %s for %s",
|
|
||||||
lastcreatedtype, self.cdist_object.name)
|
|
||||||
except IndexError:
|
except IndexError:
|
||||||
# if no second last line, we are on the first type,
|
# if no second last line, we are on the first type,
|
||||||
# so do not set a requirement
|
# so do not set a requirement
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if "require" in self.env:
|
self._process_dep_var(func=self._record_requirement,
|
||||||
requirements = self.env['require']
|
dep_var='require')
|
||||||
self.log.debug("reqs = %s", requirements)
|
# onchange implies require
|
||||||
for requirement in self._parse_require(requirements):
|
self._process_dep_var(func=self._record_requirement,
|
||||||
# Ignore empty fields - probably the only field anyway
|
dep_var='onchange')
|
||||||
if len(requirement) == 0:
|
self._process_dep_var(func=self._record_processing_precondition,
|
||||||
continue
|
dep_var='onchange')
|
||||||
self.record_requirement(requirement)
|
|
||||||
|
|
||||||
def _parse_require(self, require):
|
def _parse_require(self, require):
|
||||||
return re.split(r'[ \t\n]+', require)
|
return re.split(r'[ \t\n]+', require)
|
||||||
|
|
|
@ -207,6 +207,40 @@ class EmulatorTestCase(test.CdistTestCase):
|
||||||
self.assertTrue(len(reqs) == 0)
|
self.assertTrue(len(reqs) == 0)
|
||||||
# if we get here all is fine
|
# if we get here all is fine
|
||||||
|
|
||||||
|
def test_onchange(self):
|
||||||
|
argv = ['__planet', 'erde']
|
||||||
|
emu = emulator.Emulator(argv, env=self.env)
|
||||||
|
emu.run()
|
||||||
|
argv = ['__planet', 'mars']
|
||||||
|
emu = emulator.Emulator(argv, env=self.env)
|
||||||
|
emu.run()
|
||||||
|
self.env['onchange'] = '__planet/erde __planet/mars'
|
||||||
|
argv = ['__file', '/tmp/cdisttest']
|
||||||
|
emu = emulator.Emulator(argv, env=self.env)
|
||||||
|
emu.run()
|
||||||
|
# now load the objects and verify the require parameter of the objects
|
||||||
|
cdist_type = core.CdistType(self.local.type_path, '__planet')
|
||||||
|
erde_object = core.CdistObject(cdist_type, self.local.object_path,
|
||||||
|
self.local.object_marker_name, 'erde')
|
||||||
|
mars_object = core.CdistObject(cdist_type, self.local.object_path,
|
||||||
|
self.local.object_marker_name, 'mars')
|
||||||
|
cdist_type = core.CdistType(self.local.type_path, '__file')
|
||||||
|
file_object = core.CdistObject(cdist_type, self.local.object_path,
|
||||||
|
self.local.object_marker_name,
|
||||||
|
'/tmp/cdisttest')
|
||||||
|
# now test the recorded exec prerequisites and requirements
|
||||||
|
self.assertTrue(len(erde_object.requirements) == 0)
|
||||||
|
self.assertTrue(len(mars_object.requirements) == 0)
|
||||||
|
self.assertTrue(len(file_object.requirements) > 0)
|
||||||
|
self.assertTrue(len(erde_object.processing_preconditions) == 0)
|
||||||
|
self.assertTrue(len(mars_object.processing_preconditions) == 0)
|
||||||
|
self.assertTrue(len(file_object.processing_preconditions) > 0)
|
||||||
|
self.assertEqual(list(file_object.requirements),
|
||||||
|
['__planet/erde', '__planet/mars'])
|
||||||
|
self.assertEqual(list(file_object.processing_preconditions),
|
||||||
|
['__planet/erde', '__planet/mars'])
|
||||||
|
# if we get here all is fine
|
||||||
|
|
||||||
|
|
||||||
class EmulatorConflictingRequirementsTestCase(test.CdistTestCase):
|
class EmulatorConflictingRequirementsTestCase(test.CdistTestCase):
|
||||||
|
|
||||||
|
|
|
@ -154,6 +154,16 @@ You can find a more in depth description of the flow execution of manifests
|
||||||
in `cdist execution stages <cdist-stages.html>`_ and of how types work in `cdist type <cdist-type.html>`_.
|
in `cdist execution stages <cdist-stages.html>`_ and of how types work in `cdist type <cdist-type.html>`_.
|
||||||
|
|
||||||
|
|
||||||
|
Processing preconditions
|
||||||
|
------------------------
|
||||||
|
If you want to describe that something should be processed only if specified
|
||||||
|
objects changed target's state, i.e. specified objects generated either
|
||||||
|
"code-local" or "code-remote", or both, just setup the variable "onchange" to
|
||||||
|
contain processing preconditions. "onchange" is specified in the same way as
|
||||||
|
"require", and it implies "require". If processing preconditions are not
|
||||||
|
satisfied then it means that object's explorers are not run, manifest is not
|
||||||
|
run, and gencode scripts are not run. Object processing is skipped.
|
||||||
|
|
||||||
Create dependencies from execution order
|
Create dependencies from execution order
|
||||||
-----------------------------------------
|
-----------------------------------------
|
||||||
You can tell cdist to execute all types in the order in which they are created
|
You can tell cdist to execute all types in the order in which they are created
|
||||||
|
@ -375,3 +385,13 @@ Dependencies defined by execution order work as following:
|
||||||
require="__some_type_somewhere/id __sample_type/1" __sample_type 2
|
require="__some_type_somewhere/id __sample_type/1" __sample_type 2
|
||||||
require="__sample_type/2" __example_type 23
|
require="__sample_type/2" __example_type 23
|
||||||
__not_in_order_type 42
|
__not_in_order_type 42
|
||||||
|
|
||||||
|
This example makes use of processing preconditions:
|
||||||
|
|
||||||
|
.. code-block:: sh
|
||||||
|
|
||||||
|
__foo bar
|
||||||
|
__spam eggs
|
||||||
|
# execute __baz type only if any of __foo/bar, __spam/eggs has changed
|
||||||
|
# target's state, i.e. code-local or code-remote is generated (or both)
|
||||||
|
onchange='__foo/bar __spam/eggs' __baz
|
||||||
|
|
|
@ -314,6 +314,9 @@ The following environment variables influence the behaviour of cdist:
|
||||||
require
|
require
|
||||||
Setup dependencies between objects (see \`cdist manifest <cdist-manifest.html>\`_).
|
Setup dependencies between objects (see \`cdist manifest <cdist-manifest.html>\`_).
|
||||||
|
|
||||||
|
onchange
|
||||||
|
Setup processing preconditions (implies dependencies) between objects (see `cdist manifest <cdist-manifest.html>`_).
|
||||||
|
|
||||||
__cdist_log_level
|
__cdist_log_level
|
||||||
cdist log level value. One of:
|
cdist log level value. One of:
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue