tom/cdist
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

finally finish the dynamic resolver

Browse source 
Signed-off-by: Nico Schottelius <nico@brief.schottelius.org>
This commit is contained in:
Nico Schottelius 2012-12-19 21:10:18 +01:00
parent 527ec0889b
commit 2732a4ba5c
4 changed files with 105 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

64
cdist/config_install.py
View file

@ -43,29 +43,25 @@ class ConfigInstall(object):
self.context = context self.context = context
self.log = logging.getLogger(self.context.target_host) self.log = logging.getLogger(self.context.target_host)
# For easy access
self.local = context.local
self.remote = context.remote
# Initialise local directory structure # Initialise local directory structure
self.local.create_files_dirs() self.context.local.create_files_dirs()
# Initialise remote directory structure # Initialise remote directory structure
self.remote.create_files_dirs() self.context.remote.create_files_dirs()
self.explorer = core.Explorer(self.context.target_host, self.local, self.remote) self.explorer = core.Explorer(self.context.target_host, self.context.local, self.context.remote)
self.manifest = core.Manifest(self.context.target_host, self.local) self.manifest = core.Manifest(self.context.target_host, self.context.local)
self.code = core.Code(self.context.target_host, self.local, self.remote) self.code = core.Code(self.context.target_host, self.context.local, self.context.remote)
# Add switch to disable code execution # Add switch to disable code execution
self.dry_run = False self.dry_run = False
def cleanup(self): def cleanup(self):
# FIXME: move to local? # FIXME: move to local?
destination = os.path.join(self.local.cache_path, self.context.target_host) destination = os.path.join(self.context.local.cache_path, self.context.target_host)
self.log.debug("Saving " + self.local.out_path + " to " + destination) self.log.debug("Saving " + self.context.local.out_path + " to " + destination)
if os.path.exists(destination): if os.path.exists(destination):
shutil.rmtree(destination) shutil.rmtree(destination)
shutil.move(self.local.out_path, destination) shutil.move(self.context.local.out_path, destination)
def deploy_to(self): def deploy_to(self):
"""Mimic the old deploy to: Deploy to one host""" """Mimic the old deploy to: Deploy to one host"""
@ -82,7 +78,7 @@ class ConfigInstall(object):
def stage_prepare(self): def stage_prepare(self):
"""Do everything for a deploy, minus the actual code stage""" """Do everything for a deploy, minus the actual code stage"""
self.explorer.run_global_explorers(self.local.global_explorer_out_path) self.explorer.run_global_explorers(self.context.local.global_explorer_out_path)
self.manifest.run_initial_manifest(self.context.initial_manifest) self.manifest.run_initial_manifest(self.context.initial_manifest)
self.log.info("Running object manifests and type explorers") self.log.info("Running object manifests and type explorers")
@ -91,8 +87,8 @@ class ConfigInstall(object):
new_objects_created = True new_objects_created = True
while new_objects_created: while new_objects_created:
new_objects_created = False new_objects_created = False
for cdist_object in core.CdistObject.list_objects(self.local.object_path, for cdist_object in core.CdistObject.list_objects(self.context.local.object_path,
self.local.type_path): self.context.local.type_path):
if cdist_object.state == core.CdistObject.STATE_PREPARED: if cdist_object.state == core.CdistObject.STATE_PREPARED:
self.log.debug("Skipping re-prepare of object %s", cdist_object) self.log.debug("Skipping re-prepare of object %s", cdist_object)
continue continue
@ -134,56 +130,46 @@ class ConfigInstall(object):
self.log.debug("Finishing run of " + cdist_object.name) self.log.debug("Finishing run of " + cdist_object.name)
cdist_object.state = core.CdistObject.STATE_DONE cdist_object.state = core.CdistObject.STATE_DONE
def stage_run_prepare(self):
"""Prepare the run stage"""
self.objects = core.CdistObject.list_objects(
self.local.object_path,
self.local.type_path)
self.all_resolved = False
self.objects_changed = False
print("srp: %s - %s objects: %s" % (self.local.object_path, self.local.type_path, list(self.objects)))
def stage_run(self): def stage_run(self):
"""The final (and real) step of deployment""" """The final (and real) step of deployment"""
self.log.info("Generating and executing code") self.log.info("Generating and executing code")
self.stage_run_prepare()
# FIXME: # FIXME: think about parallel execution (same for stage_prepare)
# - think about parallel execution (same for stage_prepare) self.all_resolved = False
# - catch unresolvable trees
while not self.all_resolved: while not self.all_resolved:
self.stage_run_iterate() self.stage_run_iterate()
def stage_run_iterate(self): def stage_run_iterate(self):
logging.root.setLevel(logging.DEBUG)
""" """
Run one iteration of the run Run one iteration of the run
To be repeated until all objects are done To be repeated until all objects are done
""" """
objects = list(core.CdistObject.list_objects(self.context.local.object_path, self.context.local.type_path))
object_state_list=' '.join('%s:%s:%s:%s' % (o, o.state, o.all_requirements, o.satisfied_requirements) for o in objects)
object_state_list=' '.join('%s:%s:%s' % (o, o.state, o.all_requirements()) for o in self.objects) self.log.debug("Object state (name:state:requirements:satisfied): %s" % object_state_list)
self.log.debug("Object state (name:state:requirements): %s" % object_state_list)
print("Object state (name:state:requirements): %s" % object_state_list)
objects_changed = False
self.all_resolved = True self.all_resolved = True
for cdist_object in self.objects: for cdist_object in objects:
if not cdist_object.state == cdist_object.STATE_DONE: if not cdist_object.state == cdist_object.STATE_DONE:
self.all_resolved = False self.all_resolved = False
self.log.debug("Object %s not done" % cdist_object.name)
if cdist_object.satisfied_requirements: if cdist_object.satisfied_requirements:
self.log.debug("Running object %s with satisfied requirements" % cdist_object.name)
self.object_run(cdist_object, self.dry_run) self.object_run(cdist_object, self.dry_run)
self.objects_changed = True objects_changed = True
self.log.debug("All resolved: %s Objects changed: %s" % (self.all_resolved, objects_changed))
# Not all are resolved, but nothing has been changed => bad dependencies! # Not all are resolved, but nothing has been changed => bad dependencies!
if not self.objects_changed and not self.all_resolved: if not objects_changed and not self.all_resolved:
# Create list of unfinished objects + their requirements for print # Create list of unfinished objects + their requirements for print
evil_objects = [] evil_objects = []
good_objects = [] good_objects = []
for cdist_object in self.objects: for cdist_object in objects:
if not cdist_object.state == cdist_object.STATE_DONE: if not cdist_object.state == cdist_object.STATE_DONE:
evil_objects.append("%s: required: %s, autorequired: %s" % evil_objects.append("%s: required: %s, autorequired: %s" %
(cdist_object.name, cdist_object.requirements, cdist_object.autorequire)) (cdist_object.name, cdist_object.requirements, cdist_object.autorequire))

6
cdist/core/cdist_object.py
View file

@ -211,15 +211,18 @@ class CdistObject(object):
except EnvironmentError as error: except EnvironmentError as error:
raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error)) raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error))
@property
def satisfied_requirements(self): def satisfied_requirements(self):
"""Return state whether all of our dependencies have been resolved already""" """Return state whether all of our dependencies have been resolved already"""
satisfied = True satisfied = True
for requirement in self.all_requirements(): for requirement in self.all_requirements:
log.debug("%s: Checking requirement %s (%s) .." % (self.name, requirement.name, requirement.state))
if not requirement.state == self.STATE_DONE: if not requirement.state == self.STATE_DONE:
satisfied = False satisfied = False
break break
log.debug("%s is satisfied: %s" % (self.name, satisfied))
return satisfied return satisfied
@ -251,6 +254,7 @@ class CdistObject(object):
else: else:
raise RequirementNotFoundError(pattern) raise RequirementNotFoundError(pattern)
@property
def all_requirements(self): def all_requirements(self):
""" """
Return resolved autorequirements and requirements so that Return resolved autorequirements and requirements so that

82
cdist/test/config_install/__init__.py
View file

@ -28,6 +28,7 @@ from cdist import core
import cdist import cdist
import cdist.context import cdist.context
import cdist.config
import os.path as op import os.path as op
my_dir = op.abspath(op.dirname(__file__)) my_dir = op.abspath(op.dirname(__file__))
@ -40,30 +41,40 @@ class ConfigInstallRunTestCase(test.CdistTestCase):
def setUp(self): def setUp(self):
# Change env for context
self.orig_environ = os.environ
os.environ = os.environ.copy()
self.temp_dir = self.mkdtemp()
self.out_dir = os.path.join(self.temp_dir, "out")
self.remote_out_dir = os.path.join(self.temp_dir, "remote")
os.environ['__cdist_out_dir'] = self.out_dir
os.environ['__cdist_remote_out_dir'] = self.remote_out_dir
self.context = cdist.context.Context( self.context = cdist.context.Context(
target_host=self.target_host, target_host=self.target_host,
remote_copy=self.remote_copy, remote_copy=self.remote_copy,
remote_exec=self.remote_exec, remote_exec=self.remote_exec,
initial_manifest=args.manifest,
add_conf_dirs=add_conf_dir,
exec_path=test.cdist_exec_path, exec_path=test.cdist_exec_path,
debug=False) debug=True)
self.config = config.Config(self.context) self.context.local.object_path = object_base_path
self.context.local.type_path = type_base_path
self.config = cdist.config.Config(self.context)
def setUp(self):
self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path)) self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
self.object_index = dict((o.name, o) for o in self.objects) self.object_index = dict((o.name, o) for o in self.objects)
self.object_names = [o.name for o in self.objects] self.object_names = [o.name for o in self.objects]
print(self.objects)
self.cdist_type = core.CdistType(type_base_path, '__third')
self.cdist_object = core.CdistObject(self.cdist_type, object_base_path, 'moon')
def tearDown(self): def tearDown(self):
for o in self.objects: for o in self.objects:
o.requirements = [] o.requirements = []
o.state = ""
os.environ = self.orig_environ
shutil.rmtree(self.temp_dir)
def test_dependency_resolution(self): def test_dependency_resolution(self):
first = self.object_index['__first/man'] first = self.object_index['__first/man']
@ -73,21 +84,50 @@ class ConfigInstallRunTestCase(test.CdistTestCase):
first.requirements = [second.name] first.requirements = [second.name]
second.requirements = [third.name] second.requirements = [third.name]
self.config.stage_run_prepare()
# First run: # First run:
# solves first and maybe second (depending on the order in the set) # solves first and maybe second (depending on the order in the set)
self.config.stage_run_iterate() self.config.stage_run_iterate()
self.assertTrue(third.state == third.STATE_DONE)
# FIXME :-) self.config.stage_run_iterate()
self.assertTrue(False) self.assertTrue(second.state == second.STATE_DONE)
# self.assertEqual(
# self.dependency_resolver.dependencies['__first/man'],
# [third_moon, second_on_the, first_man] try:
# ) self.config.stage_run_iterate()
except cdist.Error:
# Allow failing, because the third run may or may not be unecessary already,
# depending on the order of the objects
pass
self.assertTrue(first.state == first.STATE_DONE)
def test_non_empty_object_list(self):
"""Ensure the object list returned is not empty"""
pass
def test_requirement_not_found(self): def test_requirement_not_found(self):
first_man = self.object_index['__first/man'] """Ensure an exception is thrown for missing depedencies"""
first_man.requirements = ['__does/not/exist'] cdist_object = self.object_index['__first/man']
cdist_object.requirements = ['__does/not/exist']
with self.assertRaises(core.cdist_object.RequirementNotFoundError): with self.assertRaises(core.cdist_object.RequirementNotFoundError):
first_man.find_requirements_by_name(first_man.requirements) # Use list, as generator does not (yet) raise the error
list(cdist_object.find_requirements_by_name(cdist_object.requirements))
def test_unresolvable_requirements(self):
"""Ensure an exception is thrown for unresolvable depedencies"""
# Create to objects depending on each other - no solution possible
first = self.object_index['__first/man']
second = self.object_index['__second/on-the']
first.requirements = [second.name]
second.requirements = [first.name]
# First round solves __third/moon
self.config.stage_run_iterate()
# Second round detects it cannot solve the rest
with self.assertRaises(cdist.Error):
self.config.stage_run_iterate()

25
cdist/test/object/__init__.py
View file

@ -36,9 +36,8 @@ type_base_path = op.join(fixtures, 'type')
class ObjectClassTestCase(test.CdistTestCase): class ObjectClassTestCase(test.CdistTestCase):
def test_list_object_names(self): def setUp(self):
found_object_names = sorted(list(core.CdistObject.list_object_names(object_base_path))) self.expected_object_names = sorted([
expected_object_names = sorted([
'__first/child', '__first/child',
'__first/dog', '__first/dog',
'__first/man', '__first/man',
@ -46,20 +45,24 @@ class ObjectClassTestCase(test.CdistTestCase):
'__second/on-the', '__second/on-the',
'__second/under-the', '__second/under-the',
'__third/moon']) '__third/moon'])
self.assertEqual(found_object_names, expected_object_names)
self.expected_objects = []
for cdist_object_name in self.expected_object_names:
cdist_type, cdist_object_id = cdist_object_name.split("/", maxsplit=1)
cdist_object = core.CdistObject(core.CdistType(type_base_path, cdist_type), object_base_path, cdist_object_id)
self.expected_objects.append(cdist_object)
def test_list_object_names(self):
found_object_names = sorted(list(core.CdistObject.list_object_names(object_base_path)))
self.assertEqual(found_object_names, self.expected_object_names)
def test_list_type_names(self): def test_list_type_names(self):
type_names = list(cdist.core.CdistObject.list_type_names(object_base_path)) type_names = list(cdist.core.CdistObject.list_type_names(object_base_path))
self.assertEqual(type_names, ['__first', '__second', '__third']) self.assertEqual(type_names, ['__first', '__second', '__third'])
def test_list_objects(self): def test_list_objects(self):
objects = list(core.CdistObject.list_objects(object_base_path, type_base_path)) found_objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
objects_expected = [ self.assertEqual(found_objects, self.expected_objects)
core.CdistObject(core.CdistType(type_base_path, '__first'), object_base_path, 'man'),
core.CdistObject(core.CdistType(type_base_path, '__second'), object_base_path, 'on-the'),
core.CdistObject(core.CdistType(type_base_path, '__third'), object_base_path, 'moon'),
]
self.assertEqual(objects, objects_expected)
class ObjectIdTestCase(test.CdistTestCase): class ObjectIdTestCase(test.CdistTestCase):