From 2732a4ba5c3ea75fccb7aa833b6799d021255140 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 19 Dec 2012 21:10:18 +0100 Subject: [PATCH] finally finish the dynamic resolver Signed-off-by: Nico Schottelius --- cdist/config_install.py | 64 ++++++++------------- cdist/core/cdist_object.py | 6 +- cdist/test/config_install/__init__.py | 82 ++++++++++++++++++++------- cdist/test/object/__init__.py | 25 ++++---- 4 files changed, 105 insertions(+), 72 deletions(-) diff --git a/cdist/config_install.py b/cdist/config_install.py index 9de5ecb8..f1529cc1 100644 --- a/cdist/config_install.py +++ b/cdist/config_install.py @@ -43,29 +43,25 @@ class ConfigInstall(object): self.context = context self.log = logging.getLogger(self.context.target_host) - # For easy access - self.local = context.local - self.remote = context.remote - # Initialise local directory structure - self.local.create_files_dirs() + self.context.local.create_files_dirs() # Initialise remote directory structure - self.remote.create_files_dirs() + self.context.remote.create_files_dirs() - self.explorer = core.Explorer(self.context.target_host, self.local, self.remote) - self.manifest = core.Manifest(self.context.target_host, self.local) - self.code = core.Code(self.context.target_host, self.local, self.remote) + self.explorer = core.Explorer(self.context.target_host, self.context.local, self.context.remote) + self.manifest = core.Manifest(self.context.target_host, self.context.local) + self.code = core.Code(self.context.target_host, self.context.local, self.context.remote) # Add switch to disable code execution self.dry_run = False def cleanup(self): # FIXME: move to local? - destination = os.path.join(self.local.cache_path, self.context.target_host) - self.log.debug("Saving " + self.local.out_path + " to " + destination) + destination = os.path.join(self.context.local.cache_path, self.context.target_host) + self.log.debug("Saving " + self.context.local.out_path + " to " + destination) if os.path.exists(destination): shutil.rmtree(destination) - shutil.move(self.local.out_path, destination) + shutil.move(self.context.local.out_path, destination) def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" @@ -82,7 +78,7 @@ class ConfigInstall(object): def stage_prepare(self): """Do everything for a deploy, minus the actual code stage""" - self.explorer.run_global_explorers(self.local.global_explorer_out_path) + self.explorer.run_global_explorers(self.context.local.global_explorer_out_path) self.manifest.run_initial_manifest(self.context.initial_manifest) self.log.info("Running object manifests and type explorers") @@ -91,8 +87,8 @@ class ConfigInstall(object): new_objects_created = True while new_objects_created: new_objects_created = False - for cdist_object in core.CdistObject.list_objects(self.local.object_path, - self.local.type_path): + for cdist_object in core.CdistObject.list_objects(self.context.local.object_path, + self.context.local.type_path): if cdist_object.state == core.CdistObject.STATE_PREPARED: self.log.debug("Skipping re-prepare of object %s", cdist_object) continue @@ -134,56 +130,46 @@ class ConfigInstall(object): self.log.debug("Finishing run of " + cdist_object.name) cdist_object.state = core.CdistObject.STATE_DONE - def stage_run_prepare(self): - """Prepare the run stage""" - - self.objects = core.CdistObject.list_objects( - self.local.object_path, - self.local.type_path) - - self.all_resolved = False - self.objects_changed = False - - print("srp: %s - %s objects: %s" % (self.local.object_path, self.local.type_path, list(self.objects))) - def stage_run(self): """The final (and real) step of deployment""" self.log.info("Generating and executing code") - self.stage_run_prepare() - # FIXME: - # - think about parallel execution (same for stage_prepare) - # - catch unresolvable trees + # FIXME: think about parallel execution (same for stage_prepare) + self.all_resolved = False while not self.all_resolved: self.stage_run_iterate() def stage_run_iterate(self): - logging.root.setLevel(logging.DEBUG) """ Run one iteration of the run To be repeated until all objects are done """ + objects = list(core.CdistObject.list_objects(self.context.local.object_path, self.context.local.type_path)) + object_state_list=' '.join('%s:%s:%s:%s' % (o, o.state, o.all_requirements, o.satisfied_requirements) for o in objects) - object_state_list=' '.join('%s:%s:%s' % (o, o.state, o.all_requirements()) for o in self.objects) - self.log.debug("Object state (name:state:requirements): %s" % object_state_list) - print("Object state (name:state:requirements): %s" % object_state_list) + self.log.debug("Object state (name:state:requirements:satisfied): %s" % object_state_list) + objects_changed = False self.all_resolved = True - for cdist_object in self.objects: + for cdist_object in objects: if not cdist_object.state == cdist_object.STATE_DONE: self.all_resolved = False + self.log.debug("Object %s not done" % cdist_object.name) if cdist_object.satisfied_requirements: + self.log.debug("Running object %s with satisfied requirements" % cdist_object.name) self.object_run(cdist_object, self.dry_run) - self.objects_changed = True + objects_changed = True + + self.log.debug("All resolved: %s Objects changed: %s" % (self.all_resolved, objects_changed)) # Not all are resolved, but nothing has been changed => bad dependencies! - if not self.objects_changed and not self.all_resolved: + if not objects_changed and not self.all_resolved: # Create list of unfinished objects + their requirements for print evil_objects = [] good_objects = [] - for cdist_object in self.objects: + for cdist_object in objects: if not cdist_object.state == cdist_object.STATE_DONE: evil_objects.append("%s: required: %s, autorequired: %s" % (cdist_object.name, cdist_object.requirements, cdist_object.autorequire)) diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index cc9aeaa5..7beea130 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -211,15 +211,18 @@ class CdistObject(object): except EnvironmentError as error: raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error)) + @property def satisfied_requirements(self): """Return state whether all of our dependencies have been resolved already""" satisfied = True - for requirement in self.all_requirements(): + for requirement in self.all_requirements: + log.debug("%s: Checking requirement %s (%s) .." % (self.name, requirement.name, requirement.state)) if not requirement.state == self.STATE_DONE: satisfied = False break + log.debug("%s is satisfied: %s" % (self.name, satisfied)) return satisfied @@ -251,6 +254,7 @@ class CdistObject(object): else: raise RequirementNotFoundError(pattern) + @property def all_requirements(self): """ Return resolved autorequirements and requirements so that diff --git a/cdist/test/config_install/__init__.py b/cdist/test/config_install/__init__.py index 326632b7..2a05d4c5 100644 --- a/cdist/test/config_install/__init__.py +++ b/cdist/test/config_install/__init__.py @@ -28,6 +28,7 @@ from cdist import core import cdist import cdist.context +import cdist.config import os.path as op my_dir = op.abspath(op.dirname(__file__)) @@ -40,30 +41,40 @@ class ConfigInstallRunTestCase(test.CdistTestCase): def setUp(self): + # Change env for context + self.orig_environ = os.environ + os.environ = os.environ.copy() + self.temp_dir = self.mkdtemp() + + self.out_dir = os.path.join(self.temp_dir, "out") + self.remote_out_dir = os.path.join(self.temp_dir, "remote") + + os.environ['__cdist_out_dir'] = self.out_dir + os.environ['__cdist_remote_out_dir'] = self.remote_out_dir + self.context = cdist.context.Context( target_host=self.target_host, remote_copy=self.remote_copy, remote_exec=self.remote_exec, - initial_manifest=args.manifest, - add_conf_dirs=add_conf_dir, exec_path=test.cdist_exec_path, - debug=False) + debug=True) - self.config = config.Config(self.context) + self.context.local.object_path = object_base_path + self.context.local.type_path = type_base_path + + self.config = cdist.config.Config(self.context) - def setUp(self): self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path)) self.object_index = dict((o.name, o) for o in self.objects) self.object_names = [o.name for o in self.objects] - print(self.objects) - - self.cdist_type = core.CdistType(type_base_path, '__third') - self.cdist_object = core.CdistObject(self.cdist_type, object_base_path, 'moon') - def tearDown(self): for o in self.objects: o.requirements = [] + o.state = "" + + os.environ = self.orig_environ + shutil.rmtree(self.temp_dir) def test_dependency_resolution(self): first = self.object_index['__first/man'] @@ -73,21 +84,50 @@ class ConfigInstallRunTestCase(test.CdistTestCase): first.requirements = [second.name] second.requirements = [third.name] - self.config.stage_run_prepare() - # First run: # solves first and maybe second (depending on the order in the set) self.config.stage_run_iterate() + self.assertTrue(third.state == third.STATE_DONE) - # FIXME :-) - self.assertTrue(False) -# self.assertEqual( -# self.dependency_resolver.dependencies['__first/man'], -# [third_moon, second_on_the, first_man] -# ) + self.config.stage_run_iterate() + self.assertTrue(second.state == second.STATE_DONE) + + + try: + self.config.stage_run_iterate() + except cdist.Error: + # Allow failing, because the third run may or may not be unecessary already, + # depending on the order of the objects + pass + self.assertTrue(first.state == first.STATE_DONE) + + + def test_non_empty_object_list(self): + """Ensure the object list returned is not empty""" + pass def test_requirement_not_found(self): - first_man = self.object_index['__first/man'] - first_man.requirements = ['__does/not/exist'] + """Ensure an exception is thrown for missing depedencies""" + cdist_object = self.object_index['__first/man'] + cdist_object.requirements = ['__does/not/exist'] + with self.assertRaises(core.cdist_object.RequirementNotFoundError): - first_man.find_requirements_by_name(first_man.requirements) + # Use list, as generator does not (yet) raise the error + list(cdist_object.find_requirements_by_name(cdist_object.requirements)) + + def test_unresolvable_requirements(self): + """Ensure an exception is thrown for unresolvable depedencies""" + + # Create to objects depending on each other - no solution possible + first = self.object_index['__first/man'] + second = self.object_index['__second/on-the'] + + first.requirements = [second.name] + second.requirements = [first.name] + + # First round solves __third/moon + self.config.stage_run_iterate() + + # Second round detects it cannot solve the rest + with self.assertRaises(cdist.Error): + self.config.stage_run_iterate() diff --git a/cdist/test/object/__init__.py b/cdist/test/object/__init__.py index ec6e25c1..f3a526aa 100644 --- a/cdist/test/object/__init__.py +++ b/cdist/test/object/__init__.py @@ -36,9 +36,8 @@ type_base_path = op.join(fixtures, 'type') class ObjectClassTestCase(test.CdistTestCase): - def test_list_object_names(self): - found_object_names = sorted(list(core.CdistObject.list_object_names(object_base_path))) - expected_object_names = sorted([ + def setUp(self): + self.expected_object_names = sorted([ '__first/child', '__first/dog', '__first/man', @@ -46,20 +45,24 @@ class ObjectClassTestCase(test.CdistTestCase): '__second/on-the', '__second/under-the', '__third/moon']) - self.assertEqual(found_object_names, expected_object_names) + + self.expected_objects = [] + for cdist_object_name in self.expected_object_names: + cdist_type, cdist_object_id = cdist_object_name.split("/", maxsplit=1) + cdist_object = core.CdistObject(core.CdistType(type_base_path, cdist_type), object_base_path, cdist_object_id) + self.expected_objects.append(cdist_object) + + def test_list_object_names(self): + found_object_names = sorted(list(core.CdistObject.list_object_names(object_base_path))) + self.assertEqual(found_object_names, self.expected_object_names) def test_list_type_names(self): type_names = list(cdist.core.CdistObject.list_type_names(object_base_path)) self.assertEqual(type_names, ['__first', '__second', '__third']) def test_list_objects(self): - objects = list(core.CdistObject.list_objects(object_base_path, type_base_path)) - objects_expected = [ - core.CdistObject(core.CdistType(type_base_path, '__first'), object_base_path, 'man'), - core.CdistObject(core.CdistType(type_base_path, '__second'), object_base_path, 'on-the'), - core.CdistObject(core.CdistType(type_base_path, '__third'), object_base_path, 'moon'), - ] - self.assertEqual(objects, objects_expected) + found_objects = list(core.CdistObject.list_objects(object_base_path, type_base_path)) + self.assertEqual(found_objects, self.expected_objects) class ObjectIdTestCase(test.CdistTestCase):