new feature: dependency resolver

This commit is contained in:
Steven Armstrong 2012-01-19 07:51:02 +01:00
parent 9551b2422f
commit 252ae5ea56
22 changed files with 271 additions and 45 deletions

View file

@ -3,7 +3,6 @@ UNASSIGNED TODOS
The following list of todos has not been assigned to any developer.
Feel free to pick one!
TESTS
-----
- multiple defines of object:

View file

@ -27,9 +27,12 @@ import shutil
import sys
import tempfile
import time
import itertools
import pprint
import cdist
from cdist import core
from cdist import resolver
class ConfigInstall(object):
@ -105,29 +108,12 @@ class ConfigInstall(object):
def object_run(self, cdist_object):
"""Run gencode and code for an object"""
self.log.debug("Trying to run object " + cdist_object.name)
if cdist_object.state == core.Object.STATE_RUNNING:
# FIXME: resolve dependency circle / show problem source
raise cdist.Error("Detected circular dependency in " + cdist_object.name)
elif cdist_object.state == core.Object.STATE_DONE:
self.log.debug("Ignoring run of already finished object %s", cdist_object)
return
else:
cdist_object.state = core.Object.STATE_RUNNING
if cdist_object.state == core.Object.STATE_DONE:
# TODO: remove once we are sure that this really never happens.
raise cdist.Error("Attempting to run an already finished object: %s", cdist_object)
cdist_type = cdist_object.type
for requirement in cdist_object.requirements:
self.log.debug("Object %s requires %s", cdist_object, requirement)
required_object = cdist_object.object_from_name(requirement)
# The user may have created dependencies without satisfying them
if not required_object.exists:
raise cdist.Error(cdist_object.name + " requires non-existing " + required_object.name)
else:
self.log.debug("Required object %s exists", required_object.name)
self.object_run(required_object)
# Generate
self.log.info("Generating and executing code for " + cdist_object.name)
cdist_object.code_local = self.code.run_gencode_local(cdist_object)
@ -149,7 +135,14 @@ class ConfigInstall(object):
def stage_run(self):
"""The final (and real) step of deployment"""
self.log.info("Generating and executing code")
for cdist_object in core.Object.list_objects(self.local.object_path,
self.local.type_path):
objects = core.Object.list_objects(
self.local.object_path,
self.local.type_path)
dependency_resolver = resolver.DependencyResolver(objects)
self.log.debug(pprint.pformat(dependency_resolver.graph))
for cdist_object in dependency_resolver:
self.log.debug("Run object: %s", cdist_object)
self.object_run(cdist_object)

View file

@ -96,12 +96,18 @@ class Object(object):
"""
return os.path.join(type_name, object_id)
def __init__(self, cdist_type, base_path, object_id=None):
@staticmethod
def validate_object_id(object_id):
"""Validate the given object_id and raise IllegalObjectIdError if it's not valid.
"""
if object_id:
if object_id.startswith('/'):
raise IllegalObjectIdError(object_id, 'object_id may not start with /')
if OBJECT_MARKER in object_id.split(os.sep):
raise IllegalObjectIdError(object_id, 'object_id may not contain \'%s\'' % OBJECT_MARKER)
def __init__(self, cdist_type, base_path, object_id=None):
self.validate_object_id(object_id)
self.type = cdist_type # instance of Type
self.base_path = base_path
self.object_id = object_id
@ -116,8 +122,12 @@ class Object(object):
return '<Object %s>' % self.name
def __eq__(self, other):
"""define equality as 'attributes are the same'"""
return self.__dict__ == other.__dict__
"""define equality as 'name is the same'"""
return self.name == other.name
def __hash__(self):
return hash(self.name)
def __lt__(self, other):
return isinstance(other, self.__class__) and self.name < other.name

View file

@ -154,30 +154,18 @@ class Emulator(object):
if len(requirement) == 0:
continue
self.log.debug("Recording requirement: " + requirement)
requirement_parts = requirement.split(os.sep, 1)
requirement_type_name = requirement_parts[0]
try:
requirement_object_id = requirement_parts[1]
except IndexError:
# no object id, assume singleton
requirement_object_id = 'singleton'
# Remove leading / from object id
requirement_object_id = requirement_object_id.lstrip('/')
requirement_type_name, requirement_object_id = core.Object.split_name(requirement)
# Instantiate type which fails if type does not exist
requirement_type = core.Type(self.type_base_path, requirement_type_name)
if requirement_object_id == 'singleton' \
and not requirement_type.is_singleton:
if requirement_object_id:
# Validate object_id if any
core.Object.validate_object_id(requirement_object_id)
elif not requirement_type.is_singleton:
# Only singeltons have no object_id
raise IllegalRequirementError(requirement, "Missing object_id and type is not a singleton.")
# Instantiate object which fails if the object_id is illegal
requirement_object = core.Object(requirement_type, self.object_base_path, requirement_object_id)
# Construct cleaned up requirement with only one / :-)
requirement = requirement_type_name + '/' + requirement_object_id
self.log.debug("Recording requirement: " + requirement)
self.cdist_object.requirements.append(requirement)
# Record / Append source

139
lib/cdist/resolver.py Normal file
View file

@ -0,0 +1,139 @@
# -*- coding: utf-8 -*-
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import logging
import os
import itertools
import fnmatch
import cdist
log = logging.getLogger(__name__)
class CircularReferenceError(cdist.Error):
def __init__(self, cdist_object, required_object):
self.cdist_object = cdist_object
self.required_object = required_object
def __str__(self):
return 'Circular reference detected: %s -> %s' % (self.cdist_object.name, self.required_object.name)
class RequirementNotFoundError(cdist.Error):
def __init__(self, requirement):
self.requirement = requirement
def __str__(self):
return 'Requirement could not be found: %s' % self.requirement
class DependencyResolver(object):
"""Cdist's dependency resolver.
Usage:
resolver = DependencyResolver(list_of_objects)
from pprint import pprint
pprint(resolver.graph)
for cdist_object in resolver:
do_something_with(cdist_object)
"""
def __init__(self, objects, logger=None):
self.objects = list(objects) # make sure we store as list, not generator
self._object_index = dict((o.name, o) for o in self.objects)
self._graph = None
self.log = logger or log
@property
def graph(self):
"""Build the dependency graph.
Returns a dict where the keys are the object names and the values are
lists of all dependencies including the key object itself.
"""
if self._graph is None:
graph = {}
for o in self.objects:
resolved = []
unresolved = []
self.resolve_object_dependencies(o, resolved, unresolved)
graph[o.name] = resolved
self._graph = graph
return self._graph
def find_requirements_by_name(self, requirements):
"""Takes a list of requirement patterns and returns a list of matching object instances.
Patterns are expected to be Unix shell-style wildcards for use with fnmatch.filter.
find_requirements_by_name(['__type/object_id', '__other_type/*']) ->
[<Object __type/object_id>, <Object __other_type/any>, <Object __other_type/match>]
"""
object_names = self._object_index.keys()
for pattern in requirements:
found = False
for requirement in fnmatch.filter(object_names, pattern):
found = True
yield self._object_index[requirement]
if not found:
# FIXME: get rid of the singleton object_id, it should be invisible to the code -> hide it in Object
singleton = os.path.join(pattern, 'singleton')
if singleton in self._object_index:
yield self._object_index[singleton]
else:
raise RequirementNotFoundError(pattern)
def resolve_object_dependencies(self, cdist_object, resolved, unresolved):
"""Resolve all dependencies for the given cdist_object and store them
in the list which is passed as the 'resolved' arguments.
e.g.
resolved = []
unresolved = []
resolve_object_dependencies(some_object, resolved, unresolved)
print("Dependencies for %s: %s" % (some_object, resolved))
"""
self.log.debug('Resolving dependencies for: %s' % cdist_object.name)
try:
unresolved.append(cdist_object)
for required_object in self.find_requirements_by_name(cdist_object.requirements):
self.log.debug("Object %s requires %s", cdist_object, required_object)
if required_object not in resolved:
if required_object in unresolved:
raise CircularReferenceError(cdist_object, required_object)
self.resolve_object_dependencies(required_object, resolved, unresolved)
resolved.append(cdist_object)
unresolved.remove(cdist_object)
except RequirementNotFoundError as e:
raise cdist.Error(cdist_object.name + " requires non-existing " + e.requirement)
def __iter__(self):
"""Iterate over all unique objects while resolving dependencies.
"""
iterable = itertools.chain(*self.graph.values())
# Keep record of objects that have already been seen
seen = set()
seen_add = seen.add
for cdist_object in itertools.filterfalse(seen.__contains__, iterable):
seen_add(cdist_object)
yield cdist_object

View file

@ -89,6 +89,13 @@ class EmulatorTestCase(test.CdistTestCase):
emu.run()
# if we get here all is fine
def test_requirement_pattern(self):
argv = ['__file', '/tmp/foobar']
os.environ.update(self.env)
os.environ['require'] = '__file/etc/*'
emu = emulator.Emulator(argv)
# if we get here all is fine
import os.path as op
my_dir = op.abspath(op.dirname(__file__))

View file

@ -0,0 +1,88 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import shutil
import cdist
from cdist import test
from cdist import core
from cdist import resolver
import os.path as op
my_dir = op.abspath(op.dirname(__file__))
fixtures = op.join(my_dir, 'fixtures')
object_base_path = op.join(fixtures, 'object')
type_base_path = op.join(fixtures, 'type')
class ResolverTestCase(test.CdistTestCase):
def setUp(self):
self.objects = list(core.Object.list_objects(object_base_path, type_base_path))
self.object_index = dict((o.name, o) for o in self.objects)
self.dependency_resolver = resolver.DependencyResolver(self.objects)
def tearDown(self):
for o in self.objects:
o.requirements = []
def test_find_requirements_by_name_string(self):
requirements = ['__first/man', '__second/on-the', '__third/moon']
required_objects = [self.object_index[name] for name in requirements]
self.assertEqual(sorted(list(self.dependency_resolver.find_requirements_by_name(requirements))),
sorted(required_objects))
def test_find_requirements_by_name_pattern(self):
requirements = ['__first/*', '__second/*-the', '__third/moon']
requirements_expanded = [
'__first/child', '__first/dog', '__first/man', '__first/woman',
'__second/on-the', '__second/under-the',
'__third/moon'
]
required_objects = [self.object_index[name] for name in requirements_expanded]
self.assertEqual(sorted(list(self.dependency_resolver.find_requirements_by_name(requirements))),
sorted(required_objects))
def test_dependency_resolution(self):
first_man = self.object_index['__first/man']
second_on_the = self.object_index['__second/on-the']
third_moon = self.object_index['__third/moon']
first_man.requirements = [second_on_the.name]
second_on_the.requirements = [third_moon.name]
self.assertEqual(
self.dependency_resolver.graph['__first/man'],
[third_moon, second_on_the, first_man]
)
def test_circular_reference(self):
first_man = self.object_index['__first/man']
first_woman = self.object_index['__first/woman']
first_man.requirements = [first_woman.name]
first_woman.requirements = [first_man.name]
with self.assertRaises(resolver.CircularReferenceError):
self.dependency_resolver.graph
def test_requirement_not_found(self):
first_man = self.object_index['__first/man']
first_man.requirements = ['__does/not/exist']
with self.assertRaises(cdist.Error):
self.dependency_resolver.graph

View file

@ -0,0 +1 @@
Prometheus