From a064cc19b3c65c7011e732930c5df25612996ec2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 30 Apr 2013 10:58:23 +0200 Subject: [PATCH 01/10] try new object orientated (hrrr) code instead of stage based Signed-off-by: Nico Schottelius --- cdist/config_install.py | 68 ++++++++++++++++++++++-- cdist/core/cdist_object.py | 18 +++++++ docs/dev/logs/2013-04-12.execution-order | 39 ++++++++++++++ 3 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 docs/dev/logs/2013-04-12.execution-order diff --git a/cdist/config_install.py b/cdist/config_install.py index 7c7a876d..841a19df 100644 --- a/cdist/config_install.py +++ b/cdist/config_install.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# 2010-2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -65,8 +65,13 @@ class ConfigInstall(object): def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" - self.stage_prepare() - self.stage_run() + + # Old Code + # self.stage_prepare() + # self.stage_run() + + # New Code + self.run() def deploy_and_cleanup(self): """Do what is most often done: deploy & cleanup""" @@ -76,6 +81,62 @@ class ConfigInstall(object): self.log.info("Finished successful run in %s seconds", time.time() - start_time) + ###################################################################### + # New code for running on object priority (not stage priority) + # + + def run(self): + """The main runner""" + self.explorer.run_global_explorers(self.context.local.global_explorer_out_path) + self.manifest.run_initial_manifest(self.context.initial_manifest) + self.iterate_until_finished() + + def object_list(self): + """Short name for object list retrieval""" + for cdist_object in core.CdistObject.list_objects(self.context.local.object_path, + self.context.local.type_path): + yield cdist_object + + def iterate_until_finished(self): + # Continue process until no new objects are created anymore + + objects_changed = True + + while objects_changed: + objects_changed = False + + for cdist_object in self.object_list(): + if not cdist_object.requirements_satisfied(cdist_object.requirements): + """We cannot do anything for this poor object""" + continue + + if cdist_object.state == core.CdistObject.STATE_UNDEF: + """Prepare the virgin object""" + + self.object_prepare(cdist_object) + objects_changed = True + + if not cdist_object.requirements_satisfied(cdist_object.autorequire): + """The previous step created objects we depend on - wait for them""" + continue + + if cdist_object.state == core.CdistObject.STATE_PREPARED: + self.object_run(cdist_object) + + # Check whether all objects have been finished + unfinished_object_names = [] + for cdist_object in self.object_list(): + if not cdist_object.state == cdist_object.STATE_DONE: + unfinished_object_names.append(cdist_object.name) + + if unfinished_object_names: + raise cdist.Error("The following objects could not be resolved: %s" % + (" ".join(unfinished_object_names)) + + ###################################################################### + # Stages based code + # + def stage_prepare(self): """Do everything for a deploy, minus the actual code stage""" self.explorer.run_global_explorers(self.context.local.global_explorer_out_path) @@ -89,6 +150,7 @@ class ConfigInstall(object): new_objects_created = False for cdist_object in core.CdistObject.list_objects(self.context.local.object_path, self.context.local.type_path): + if cdist_object.state == core.CdistObject.STATE_PREPARED: self.log.debug("Skipping re-prepare of object %s", cdist_object) continue diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 73537f80..30d002f6 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -61,6 +61,7 @@ class CdistObject(object): """ # Constants for use with Object.state + STATE_UNDEF = "" STATE_PREPARED = "prepared" STATE_RUNNING = "running" STATE_DONE = "done" @@ -223,6 +224,23 @@ class CdistObject(object): except EnvironmentError as error: raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error)) + @property + def requirements_satisfied(self): + """Return state whether normal depedencies are satisfied""" + + satisfied = True + + for requirement in self.requirements: + cdist_object = self.object_from_name(requirement) + + if not cdist_object.state == self.STATE_DONE: + satisfied = False + break + + log.debug("%s is satisfied: %s" % (self.name, satisfied)) + + return satisfied + @property def satisfied_requirements(self): """Return state whether all of our dependencies have been resolved already""" diff --git a/docs/dev/logs/2013-04-12.execution-order b/docs/dev/logs/2013-04-12.execution-order new file mode 100644 index 00000000..1739c287 --- /dev/null +++ b/docs/dev/logs/2013-04-12.execution-order @@ -0,0 +1,39 @@ +Old: + +- global explores (all) +- initial manifest +- for each object + execute type explorers + execute manifest + + continue until all objects (including newly created) + have their type explorers/manifests run +- build dependency tree +- for each object + execute gencode-* + execute code-* + +New: +- run all global explorers +- run initial manifest + creates zero or more cdist_objects +- for each cdist_object + if not cdist_object.has_unfullfilled_requirements: + execute type explorers + execute manifest + may create new objects, resulting in autorequirements + + # Gained requirements during manifest run + if object.has_auto_requirements(): + continue + + cdist_object.execute gencode-* + cdist_object.execute code-* + + +Requirements / Test cases for requirments / resolver: + + - omnipotence + - + +Test From 8a7c64f86afba67dace9c56f0bdd2526496ba29f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 30 Apr 2013 11:04:01 +0200 Subject: [PATCH 02/10] cleanups + indent errors Signed-off-by: Nico Schottelius --- cdist/config_install.py | 4 ++-- cdist/core/cdist_object.py | 48 ++------------------------------------ 2 files changed, 4 insertions(+), 48 deletions(-) diff --git a/cdist/config_install.py b/cdist/config_install.py index 841a19df..b458ad56 100644 --- a/cdist/config_install.py +++ b/cdist/config_install.py @@ -127,11 +127,11 @@ class ConfigInstall(object): unfinished_object_names = [] for cdist_object in self.object_list(): if not cdist_object.state == cdist_object.STATE_DONE: - unfinished_object_names.append(cdist_object.name) + unfinished_object_names.append(cdist_object.name) if unfinished_object_names: raise cdist.Error("The following objects could not be resolved: %s" % - (" ".join(unfinished_object_names)) + (" ".join(unfinished_object_names))) ###################################################################### # Stages based code diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 30d002f6..566cde21 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -224,13 +224,12 @@ class CdistObject(object): except EnvironmentError as error: raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error)) - @property - def requirements_satisfied(self): + def requirements_satisfied(self, requirements): """Return state whether normal depedencies are satisfied""" satisfied = True - for requirement in self.requirements: + for requirement in requirements: cdist_object = self.object_from_name(requirement) if not cdist_object.state == self.STATE_DONE: @@ -241,49 +240,6 @@ class CdistObject(object): return satisfied - @property - def satisfied_requirements(self): - """Return state whether all of our dependencies have been resolved already""" - - satisfied = True - - for requirement in self.all_requirements: - log.debug("%s: Checking requirement %s (%s) .." % (self.name, requirement.name, requirement.state)) - if not requirement.state == self.STATE_DONE: - satisfied = False - break - log.debug("%s is satisfied: %s" % (self.name, satisfied)) - - return satisfied - - - def find_requirements_by_name(self, requirements): - """Takes a list of requirement patterns and returns a list of matching object instances. - - Patterns are expected to be Unix shell-style wildcards for use with fnmatch.filter. - - find_requirements_by_name(['__type/object_id', '__other_type/*']) -> - [, , ] - """ - - - # FIXME: think about where/when to store this - probably not here - self.objects = dict((o.name, o) for o in self.list_objects(self.base_path, self.cdist_type.base_path)) - object_names = self.objects.keys() - - for pattern in requirements: - found = False - for requirement in fnmatch.filter(object_names, pattern): - found = True - yield self.objects[requirement] - if not found: - # FIXME: get rid of the singleton object_id, it should be invisible to the code -> hide it in Object - singleton = os.path.join(pattern, 'singleton') - if singleton in self.objects: - yield self.objects[singleton] - else: - raise RequirementNotFoundError(pattern) - @property def all_requirements(self): """ From 85d24ce2595b972fe5c236ffab1528e19b89b362 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 30 Apr 2013 15:07:27 +0200 Subject: [PATCH 03/10] fix execution order - seems to be fine now Signed-off-by: Nico Schottelius --- cdist/config_install.py | 30 +++++++++++++++++++----- cdist/core/cdist_object.py | 25 ++++---------------- docs/dev/logs/2013-04-12.execution-order | 7 +++++- 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/cdist/config_install.py b/cdist/config_install.py index b458ad56..f33efdf9 100644 --- a/cdist/config_install.py +++ b/cdist/config_install.py @@ -106,7 +106,7 @@ class ConfigInstall(object): objects_changed = False for cdist_object in self.object_list(): - if not cdist_object.requirements_satisfied(cdist_object.requirements): + if cdist_object.requirements_unfinished(cdist_object.requirements): """We cannot do anything for this poor object""" continue @@ -116,22 +116,40 @@ class ConfigInstall(object): self.object_prepare(cdist_object) objects_changed = True - if not cdist_object.requirements_satisfied(cdist_object.autorequire): + if cdist_object.requirements_unfinished(cdist_object.autorequire): """The previous step created objects we depend on - wait for them""" continue if cdist_object.state == core.CdistObject.STATE_PREPARED: self.object_run(cdist_object) + objects_changed = True # Check whether all objects have been finished - unfinished_object_names = [] + unfinished_objects = [] for cdist_object in self.object_list(): if not cdist_object.state == cdist_object.STATE_DONE: - unfinished_object_names.append(cdist_object.name) + unfinished_objects.append(cdist_object) + + if unfinished_objects: + info_string = [] + + for cdist_object in unfinished_objects: + + requirement_names = [] + autorequire_names = [] + + for requirement in cdist_object.requirements_unfinished(cdist_object.requirements): + requirement_names.append(requirement.name) + + for requirement in cdist_object.requirements_unfinished(cdist_object.autorequire): + autorequire_names.append(requirement.name) + + requirements = ", ".join(requirement_names) + autorequire = ", ".join(autorequire_names) + info_string.append("%s requires: %s autorequires: %s" % (cdist_object.name, requirements, autorequire)) - if unfinished_object_names: raise cdist.Error("The following objects could not be resolved: %s" % - (" ".join(unfinished_object_names))) + ("; ".join(info_string))) ###################################################################### # Stages based code diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 566cde21..a9306aaa 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -224,35 +224,18 @@ class CdistObject(object): except EnvironmentError as error: raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error)) - def requirements_satisfied(self, requirements): + def requirements_unfinished(self, requirements): """Return state whether normal depedencies are satisfied""" - satisfied = True + object_list = [] for requirement in requirements: cdist_object = self.object_from_name(requirement) if not cdist_object.state == self.STATE_DONE: - satisfied = False - break - - log.debug("%s is satisfied: %s" % (self.name, satisfied)) - - return satisfied - - @property - def all_requirements(self): - """ - Return resolved autorequirements and requirements so that - a complete list of requirements is returned - """ - - all_reqs= [] - all_reqs.extend(self.find_requirements_by_name(self.requirements)) - all_reqs.extend(self.find_requirements_by_name(self.autorequire)) - - return set(all_reqs) + object_list.append(cdist_object) + return object_list class RequirementNotFoundError(cdist.Error): def __init__(self, requirement): diff --git a/docs/dev/logs/2013-04-12.execution-order b/docs/dev/logs/2013-04-12.execution-order index 1739c287..5100eeda 100644 --- a/docs/dev/logs/2013-04-12.execution-order +++ b/docs/dev/logs/2013-04-12.execution-order @@ -36,4 +36,9 @@ Requirements / Test cases for requirments / resolver: - omnipotence - -Test + +-------------------------------------------------------------------------------- +ERROR: localhost: The following objects could not be resolved: __cdistmarker/singleton requires autorequires ; __directory/etc/sudoers.d requires autorequires ; __file/etc/sudoers.d/nico requires __directory/etc/sudoers.d autorequires ; __file/etc/motd requires autorequires ; __package_pacman/atop requires autorequires ; __package_pacman/screen requires autorequires ; __package_pacman/strace requires autorequires ; __package_pacman/vim requires autorequires ; __package_pacman/zsh requires autorequires ; __package_pacman/lftp requires autorequires ; __package_pacman/nmap requires autorequires ; __package_pacman/ntp requires autorequires ; __package_pacman/rsync requires autorequires ; __package_pacman/rtorrent requires autorequires ; __package_pacman/wget requires autorequires ; __package_pacman/nload requires autorequires ; __package_pacman/iftop requires autorequires ; __package_pacman/mosh requires autorequires ; __package_pacman/git requires autorequires ; __package_pacman/mercurial requires autorequires ; __package_pacman/netcat requires autorequires ; __package_pacman/python-virtualenv requires autorequires ; __package_pacman/wireshark-cli requires autorequires ; __package_pacman/sudo requires autorequires +INFO: Total processing time for 1 host(s): 32.30426597595215 +ERROR: Failed to deploy to the following hosts: localhost + From 2dac681f2561d26e32864752d1400273f53cc00d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 30 Apr 2013 15:07:59 +0200 Subject: [PATCH 04/10] better error message Signed-off-by: Nico Schottelius --- cdist/config_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/config_install.py b/cdist/config_install.py index f33efdf9..7e8eb0f1 100644 --- a/cdist/config_install.py +++ b/cdist/config_install.py @@ -148,7 +148,7 @@ class ConfigInstall(object): autorequire = ", ".join(autorequire_names) info_string.append("%s requires: %s autorequires: %s" % (cdist_object.name, requirements, autorequire)) - raise cdist.Error("The following objects could not be resolved: %s" % + raise cdist.Error("The requirements of the following objects could not be resolved: %s" % ("; ".join(info_string))) ###################################################################### From 956f400da6fca8d904c96f6958c3ea3a5beab9d9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 30 Apr 2013 15:15:22 +0200 Subject: [PATCH 05/10] re-arrange for future cleanup Signed-off-by: Nico Schottelius --- cdist/config_install.py | 77 ++++++++++-------- cdist/resolver.py | 175 ---------------------------------------- 2 files changed, 42 insertions(+), 210 deletions(-) delete mode 100644 cdist/resolver.py diff --git a/cdist/config_install.py b/cdist/config_install.py index 7e8eb0f1..b62af3e0 100644 --- a/cdist/config_install.py +++ b/cdist/config_install.py @@ -63,20 +63,16 @@ class ConfigInstall(object): shutil.rmtree(destination) shutil.move(self.context.local.out_path, destination) - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" + def deploy_and_cleanup(self): + """Do what is most often done: deploy & cleanup""" + start_time = time.time() # Old Code - # self.stage_prepare() - # self.stage_run() + #self.deploy_to() # New Code self.run() - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - start_time = time.time() - self.deploy_to() self.cleanup() self.log.info("Finished successful run in %s seconds", time.time() - start_time) @@ -151,10 +147,48 @@ class ConfigInstall(object): raise cdist.Error("The requirements of the following objects could not be resolved: %s" % ("; ".join(info_string))) + ###################################################################### + # Code required by both methods (which will stay) + # + + def object_run(self, cdist_object, dry_run=False): + """Run gencode and code for an object""" + self.log.debug("Trying to run object " + cdist_object.name) + if cdist_object.state == core.CdistObject.STATE_DONE: + raise cdist.Error("Attempting to run an already finished object: %s", cdist_object) + + cdist_type = cdist_object.cdist_type + + # Generate + self.log.info("Generating and executing code for " + cdist_object.name) + cdist_object.code_local = self.code.run_gencode_local(cdist_object) + cdist_object.code_remote = self.code.run_gencode_remote(cdist_object) + if cdist_object.code_local or cdist_object.code_remote: + cdist_object.changed = True + + # Execute + if not dry_run: + if cdist_object.code_local: + self.code.run_code_local(cdist_object) + if cdist_object.code_remote: + self.code.transfer_code_remote(cdist_object) + self.code.run_code_remote(cdist_object) + + # Mark this object as done + self.log.debug("Finishing run of " + cdist_object.name) + cdist_object.state = core.CdistObject.STATE_DONE + + ###################################################################### # Stages based code # + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + self.stage_prepare() + self.stage_run() + + def stage_prepare(self): """Do everything for a deploy, minus the actual code stage""" self.explorer.run_global_explorers(self.context.local.global_explorer_out_path) @@ -183,33 +217,6 @@ class ConfigInstall(object): self.manifest.run_type_manifest(cdist_object) cdist_object.state = core.CdistObject.STATE_PREPARED - def object_run(self, cdist_object, dry_run=False): - """Run gencode and code for an object""" - self.log.debug("Trying to run object " + cdist_object.name) - if cdist_object.state == core.CdistObject.STATE_DONE: - raise cdist.Error("Attempting to run an already finished object: %s", cdist_object) - - cdist_type = cdist_object.cdist_type - - # Generate - self.log.info("Generating and executing code for " + cdist_object.name) - cdist_object.code_local = self.code.run_gencode_local(cdist_object) - cdist_object.code_remote = self.code.run_gencode_remote(cdist_object) - if cdist_object.code_local or cdist_object.code_remote: - cdist_object.changed = True - - # Execute - if not dry_run: - if cdist_object.code_local: - self.code.run_code_local(cdist_object) - if cdist_object.code_remote: - self.code.transfer_code_remote(cdist_object) - self.code.run_code_remote(cdist_object) - - # Mark this object as done - self.log.debug("Finishing run of " + cdist_object.name) - cdist_object.state = core.CdistObject.STATE_DONE - def stage_run(self): """The final (and real) step of deployment""" self.log.info("Generating and executing code") diff --git a/cdist/resolver.py b/cdist/resolver.py deleted file mode 100644 index ddcf382e..00000000 --- a/cdist/resolver.py +++ /dev/null @@ -1,175 +0,0 @@ -# -*- coding: utf-8 -*- -# -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - -import logging -import os -import itertools -import fnmatch -import pprint - -import cdist - -log = logging.getLogger(__name__) - - -class CircularReferenceError(cdist.Error): - def __init__(self, cdist_object, required_object): - self.cdist_object = cdist_object - self.required_object = required_object - - def __str__(self): - return 'Circular reference detected: %s -> %s' % (self.cdist_object.name, self.required_object.name) - - -class RequirementNotFoundError(cdist.Error): - def __init__(self, requirement): - self.requirement = requirement - - def __str__(self): - return 'Requirement could not be found: %s' % self.requirement - - -class DependencyResolver(object): - """Cdist's dependency resolver. - - Usage: - >> resolver = DependencyResolver(list_of_objects) - # Easy access to the objects we are working with - >> resolver.objects['__some_type/object_id'] - - # Easy access to a specific objects dependencies - >> resolver.dependencies['__some_type/object_id'] - [, ] - # Pretty print the dependency graph - >> from pprint import pprint - >> pprint(resolver.dependencies) - # Iterate over all existing objects in the correct order - >> for cdist_object in resolver: - >> do_something_with(cdist_object) - """ - def __init__(self, objects, logger=None): - self.objects = dict((o.name, o) for o in objects) - self._dependencies = None - self.log = logger or log - - @property - def dependencies(self): - """Build the dependency graph. - - Returns a dict where the keys are the object names and the values are - lists of all dependencies including the key object itself. - """ - if self._dependencies is None: - self.log.info("Resolving dependencies...") - self._dependencies = {} - self._preprocess_requirements() - for name,cdist_object in self.objects.items(): - resolved = [] - unresolved = [] - self._resolve_object_dependencies(cdist_object, resolved, unresolved) - self._dependencies[name] = resolved - self.log.debug(self._dependencies) - return self._dependencies - - def find_requirements_by_name(self, requirements): - """Takes a list of requirement patterns and returns a list of matching object instances. - - Patterns are expected to be Unix shell-style wildcards for use with fnmatch.filter. - - find_requirements_by_name(['__type/object_id', '__other_type/*']) -> - [, , ] - """ - object_names = self.objects.keys() - for pattern in requirements: - found = False - for requirement in fnmatch.filter(object_names, pattern): - found = True - yield self.objects[requirement] - if not found: - # FIXME: get rid of the singleton object_id, it should be invisible to the code -> hide it in Object - singleton = os.path.join(pattern, 'singleton') - if singleton in self.objects: - yield self.objects[singleton] - else: - raise RequirementNotFoundError(pattern) - - def _preprocess_requirements(self): - """Find all autorequire dependencies and merge them to be just requirements - for further processing. - """ - for cdist_object in self.objects.values(): - if cdist_object.autorequire: - # The objects (children) that this cdist_object (parent) defined - # in it's type manifest shall inherit all explicit requirements - # that the parent has so that user defined requirements are - # fullfilled and processed in the expected order. - for auto_requirement in self.find_requirements_by_name(cdist_object.autorequire): - for requirement in self.find_requirements_by_name(cdist_object.requirements): - requirement_object_all_requirements = list(requirement.requirements) + list(requirement.autorequire) - if (requirement.name not in auto_requirement.requirements - and auto_requirement.name not in requirement_object_all_requirements): - self.log.debug('Adding %s to %s.requirements', requirement.name, auto_requirement) - auto_requirement.requirements.append(requirement.name) - # On the other hand the parent shall depend on all the children - # it created so that the user can setup dependencies on it as a - # whole without having to know anything about the parents - # internals. - cdist_object.requirements.extend(cdist_object.autorequire) - # As we changed the object on disc, we have to ensure it is not - # preprocessed again if someone would call us multiple times. - cdist_object.autorequire = [] - - def _resolve_object_dependencies(self, cdist_object, resolved, unresolved): - """Resolve all dependencies for the given cdist_object and store them - in the list which is passed as the 'resolved' arguments. - - e.g. - resolved = [] - unresolved = [] - resolve_object_dependencies(some_object, resolved, unresolved) - print("Dependencies for %s: %s" % (some_object, resolved)) - """ - self.log.debug('Resolving dependencies for: %s' % cdist_object.name) - try: - unresolved.append(cdist_object) - for required_object in self.find_requirements_by_name(cdist_object.requirements): - self.log.debug("Object %s requires %s", cdist_object, required_object) - if required_object not in resolved: - if required_object in unresolved: - error = CircularReferenceError(cdist_object, required_object) - self.log.error('%s: %s', error, pprint.pformat(self._dependencies)) - raise error - self._resolve_object_dependencies(required_object, resolved, unresolved) - resolved.append(cdist_object) - unresolved.remove(cdist_object) - except RequirementNotFoundError as e: - raise cdist.CdistObjectError(cdist_object, "requires non-existing " + e.requirement) - - def __iter__(self): - """Iterate over all unique objects and yield them in the correct order. - """ - iterable = itertools.chain(*self.dependencies.values()) - # Keep record of objects that have already been seen - seen = set() - seen_add = seen.add - for cdist_object in itertools.filterfalse(seen.__contains__, iterable): - seen_add(cdist_object) - yield cdist_object From 4882c2cf190258c98f46578701b603ad306c2b4f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 30 Apr 2013 15:16:38 +0200 Subject: [PATCH 06/10] --resolver in config_install Signed-off-by: Nico Schottelius --- cdist/config_install.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cdist/config_install.py b/cdist/config_install.py index b62af3e0..ceba1887 100644 --- a/cdist/config_install.py +++ b/cdist/config_install.py @@ -32,8 +32,6 @@ import pprint import cdist from cdist import core -from cdist import resolver - class ConfigInstall(object): """Cdist main class to hold arbitrary data""" From f95052e56f5dc453077af716c5fd9aa992c55b5f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 30 Apr 2013 15:18:03 +0200 Subject: [PATCH 07/10] remove unused modules Signed-off-by: Nico Schottelius --- cdist/config_install.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cdist/config_install.py b/cdist/config_install.py index ceba1887..4bf9ad32 100644 --- a/cdist/config_install.py +++ b/cdist/config_install.py @@ -22,12 +22,8 @@ import logging import os -import stat import shutil -import sys -import tempfile import time -import itertools import pprint import cdist From a265d870373ed7fbb91a7ec87eca02978464623d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 2 May 2013 16:41:16 +0200 Subject: [PATCH 08/10] begin dry run in command line Signed-off-by: Nico Schottelius --- cdist/core/cdist_object.py | 1 - scripts/cdist | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index a9306aaa..5d3efbdd 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -50,7 +50,6 @@ class MissingObjectIdError(cdist.Error): def __str__(self): return '%s' % (self.message) - class CdistObject(object): """Represents a cdist object. diff --git a/scripts/cdist b/scripts/cdist index fd18933a..00c55ae8 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -62,6 +62,8 @@ def commandline(): parser['configinstall'].add_argument('-i', '--initial-manifest', help='Path to a cdist manifest or \'-\' to read from stdin.', dest='manifest', required=False) + parser['configinstall'].add_argument('-n', '--dry-run', + help='Do not execute code', action='store_true') parser['configinstall'].add_argument('-p', '--parallel', help='Operate on multiple hosts in parallel', action='store_true', dest='parallel') From 0b4914a7f36ee5dc49e559f01bb055e8b19ee2e5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 15 May 2013 09:16:52 +0200 Subject: [PATCH 09/10] +comment wording Signed-off-by: Nico Schottelius --- cdist/core/cdist_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index a9306aaa..54009f6c 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -225,7 +225,7 @@ class CdistObject(object): raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error)) def requirements_unfinished(self, requirements): - """Return state whether normal depedencies are satisfied""" + """Return state whether requirements are satisfied""" object_list = [] From a9ffa86b74c285e5ad8a0ef73fd6b132efbe34f3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 15 May 2013 09:19:36 +0200 Subject: [PATCH 10/10] remove some old code, merge run into deploy_and_cleanup Signed-off-by: Nico Schottelius --- cdist/config_install.py | 80 +++-------------------------------------- 1 file changed, 5 insertions(+), 75 deletions(-) diff --git a/cdist/config_install.py b/cdist/config_install.py index 4bf9ad32..5adf8b9d 100644 --- a/cdist/config_install.py +++ b/cdist/config_install.py @@ -57,30 +57,18 @@ class ConfigInstall(object): shutil.rmtree(destination) shutil.move(self.context.local.out_path, destination) - def deploy_and_cleanup(self): + def run(self): """Do what is most often done: deploy & cleanup""" start_time = time.time() - # Old Code - #self.deploy_to() - - # New Code - self.run() - - self.cleanup() - self.log.info("Finished successful run in %s seconds", - time.time() - start_time) - - ###################################################################### - # New code for running on object priority (not stage priority) - # - - def run(self): - """The main runner""" self.explorer.run_global_explorers(self.context.local.global_explorer_out_path) self.manifest.run_initial_manifest(self.context.initial_manifest) self.iterate_until_finished() + self.cleanup() + self.log.info("Finished successful run in %s seconds", time.time() - start_time) + + def object_list(self): """Short name for object list retrieval""" for cdist_object in core.CdistObject.list_objects(self.context.local.object_path, @@ -141,10 +129,6 @@ class ConfigInstall(object): raise cdist.Error("The requirements of the following objects could not be resolved: %s" % ("; ".join(info_string))) - ###################################################################### - # Code required by both methods (which will stay) - # - def object_run(self, cdist_object, dry_run=False): """Run gencode and code for an object""" self.log.debug("Trying to run object " + cdist_object.name) @@ -171,57 +155,3 @@ class ConfigInstall(object): # Mark this object as done self.log.debug("Finishing run of " + cdist_object.name) cdist_object.state = core.CdistObject.STATE_DONE - - - ###################################################################### - # Stages based code - # - - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - self.stage_prepare() - self.stage_run() - - - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.explorer.run_global_explorers(self.context.local.global_explorer_out_path) - self.manifest.run_initial_manifest(self.context.initial_manifest) - - self.log.info("Running object manifests and type explorers") - - # Continue process until no new objects are created anymore - new_objects_created = True - while new_objects_created: - new_objects_created = False - for cdist_object in core.CdistObject.list_objects(self.context.local.object_path, - self.context.local.type_path): - - if cdist_object.state == core.CdistObject.STATE_PREPARED: - self.log.debug("Skipping re-prepare of object %s", cdist_object) - continue - else: - self.object_prepare(cdist_object) - new_objects_created = True - - def object_prepare(self, cdist_object): - """Prepare object: Run type explorer + manifest""" - self.log.info("Running manifest and explorers for " + cdist_object.name) - self.explorer.run_type_explorers(cdist_object) - self.manifest.run_type_manifest(cdist_object) - cdist_object.state = core.CdistObject.STATE_PREPARED - - def stage_run(self): - """The final (and real) step of deployment""" - self.log.info("Generating and executing code") - - objects = core.CdistObject.list_objects( - self.context.local.object_path, - self.context.local.type_path) - - dependency_resolver = resolver.DependencyResolver(objects) - self.log.debug(pprint.pformat(dependency_resolver.dependencies)) - - for cdist_object in dependency_resolver: - self.log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object)