384 lines
15 KiB
Python
384 lines
15 KiB
Python
# -*- coding: utf-8 -*-
|
|
#
|
|
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
|
|
# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
|
|
#
|
|
# This file is part of cdist.
|
|
#
|
|
# cdist is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# cdist is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
#
|
|
|
|
import logging
|
|
import os
|
|
import collections
|
|
|
|
import cdist
|
|
import cdist.core
|
|
from cdist.util import fsproperty
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
OBJECT_MARKER = '.cdist'
|
|
|
|
|
|
class IllegalObjectIdError(cdist.Error):
|
|
def __init__(self, object_id, message=None):
|
|
self.object_id = object_id
|
|
self.message = message or 'Illegal object id'
|
|
|
|
def __str__(self):
|
|
return '%s: %s' % (self.message, self.object_id)
|
|
|
|
|
|
class CdistObject(object):
|
|
"""Represents a cdist object.
|
|
|
|
All interaction with objects in cdist should be done through this class.
|
|
Directly accessing an object through the file system from python code is
|
|
a bug.
|
|
|
|
"""
|
|
|
|
# Constants for use with Object.state
|
|
STATE_PREPARED = "prepared"
|
|
STATE_RUNNING = "running"
|
|
STATE_DONE = "done"
|
|
|
|
@classmethod
|
|
def list_objects(cls, object_base_path, type_base_path):
|
|
"""Return a list of object instances"""
|
|
for object_name in cls.list_object_names(object_base_path):
|
|
type_name, object_id = cls.split_name(object_name)
|
|
yield cls(cdist.core.CdistType(type_base_path, type_name), object_base_path, object_id=object_id)
|
|
|
|
@classmethod
|
|
def list_type_names(cls, object_base_path):
|
|
"""Return a list of type names"""
|
|
return os.listdir(object_base_path)
|
|
|
|
@classmethod
|
|
def list_object_names(cls, object_base_path):
|
|
"""Return a list of object names"""
|
|
for path, dirs, files in os.walk(object_base_path):
|
|
if OBJECT_MARKER in dirs:
|
|
yield os.path.relpath(path, object_base_path)
|
|
|
|
@staticmethod
|
|
def split_name(object_name):
|
|
"""split_name('__type_name/the/object_id') -> ('__type_name', 'the/object_id')
|
|
|
|
Split the given object name into it's type and object_id parts.
|
|
|
|
"""
|
|
type_name = object_name.split(os.sep)[0]
|
|
# FIXME: allow object without object_id? e.g. for singleton
|
|
object_id = os.sep.join(object_name.split(os.sep)[1:])
|
|
return type_name, object_id
|
|
|
|
@staticmethod
|
|
def join_name(type_name, object_id):
|
|
"""join_name('__type_name', 'the/object_id') -> __type_name/the/object_id'
|
|
|
|
Join the given type_name and object_id into an object name.
|
|
|
|
"""
|
|
return os.path.join(type_name, object_id)
|
|
|
|
def validate_object_id(self):
|
|
# FIXME: also check that there is no object ID when type is singleton?
|
|
|
|
"""Validate the given object_id and raise IllegalObjectIdError if it's not valid.
|
|
"""
|
|
if self.object_id:
|
|
if OBJECT_MARKER in self.object_id.split(os.sep):
|
|
raise IllegalObjectIdError(self.object_id, 'object_id may not contain \'%s\'' % OBJECT_MARKER)
|
|
if '//' in self.object_id:
|
|
raise IllegalObjectIdError(self.object_id, 'object_id may not contain //')
|
|
|
|
# If no object_id and type is not singleton => error out
|
|
if not self.object_id and not self.cdist_type.is_singleton:
|
|
raise IllegalObjectIdError(self.object_id,
|
|
"Missing object_id and type is not a singleton.")
|
|
|
|
def __init__(self, cdist_type, base_path, object_id=None):
|
|
self.cdist_type = cdist_type # instance of Type
|
|
self.base_path = base_path
|
|
self.object_id = object_id
|
|
|
|
self.validate_object_id()
|
|
self.sanitise_object_id()
|
|
|
|
self.name = self.join_name(self.cdist_type.name, self.object_id)
|
|
self.path = os.path.join(self.cdist_type.path, self.object_id, OBJECT_MARKER)
|
|
self.absolute_path = os.path.join(self.base_path, self.path)
|
|
self.code_local_path = os.path.join(self.path, "code-local")
|
|
self.code_remote_path = os.path.join(self.path, "code-remote")
|
|
self.parameter_path = os.path.join(self.path, "parameter")
|
|
|
|
def object_from_name(self, object_name):
|
|
"""Convenience method for creating an object instance from an object name.
|
|
|
|
Mainly intended to create objects when resolving requirements.
|
|
|
|
e.g:
|
|
<CdistObject __foo/bar>.object_from_name('__other/object') -> <CdistObject __other/object>
|
|
|
|
"""
|
|
|
|
base_path = self.base_path
|
|
type_path = self.cdist_type.base_path
|
|
|
|
type_name, object_id = self.split_name(object_name)
|
|
|
|
cdist_type = self.cdist_type.__class__(type_path, type_name)
|
|
|
|
return self.__class__(cdist_type, base_path, object_id=object_id)
|
|
|
|
def __repr__(self):
|
|
return '<CdistObject %s>' % self.name
|
|
|
|
def __eq__(self, other):
|
|
"""define equality as 'name is the same'"""
|
|
return self.name == other.name
|
|
|
|
def __hash__(self):
|
|
return hash(self.name)
|
|
|
|
def __lt__(self, other):
|
|
return isinstance(other, self.__class__) and self.name < other.name
|
|
|
|
def sanitise_object_id(self):
|
|
"""
|
|
Remove leading and trailing slash (one only)
|
|
"""
|
|
|
|
# Allow empty object id for singletons
|
|
if self.object_id:
|
|
# Remove leading slash
|
|
if self.object_id[0] == '/':
|
|
self.object_id = self.object_id[1:]
|
|
|
|
# Remove trailing slash
|
|
if self.object_id[-1] == '/':
|
|
self.object_id = self.object_id[:-1]
|
|
|
|
# FIXME: still needed?
|
|
@property
|
|
def explorer_path(self):
|
|
"""Create and return the relative path to this objects explorers"""
|
|
# create absolute path
|
|
path = os.path.join(self.absolute_path, "explorer")
|
|
if not os.path.isdir(path):
|
|
os.mkdir(path)
|
|
# return relative path
|
|
return os.path.join(self.path, "explorer")
|
|
|
|
requirements = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, 'require'))
|
|
autorequire = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, 'autorequire'))
|
|
parameters = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.base_path, obj.parameter_path))
|
|
explorers = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.base_path, obj.explorer_path))
|
|
changed = fsproperty.FileBooleanProperty(lambda obj: os.path.join(obj.absolute_path, "changed"))
|
|
state = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.absolute_path, "state"))
|
|
source = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, "source"))
|
|
code_local = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_path, obj.code_local_path))
|
|
code_remote = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_path, obj.code_remote_path))
|
|
|
|
@property
|
|
def exists(self):
|
|
"""Checks wether this cdist object exists on the file systems."""
|
|
return os.path.exists(self.absolute_path)
|
|
|
|
def create(self):
|
|
"""Create this cdist object on the filesystem.
|
|
"""
|
|
try:
|
|
os.makedirs(self.absolute_path, exist_ok=False)
|
|
absolute_parameter_path = os.path.join(self.base_path, self.parameter_path)
|
|
os.makedirs(absolute_parameter_path, exist_ok=False)
|
|
except EnvironmentError as error:
|
|
raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error))
|
|
|
|
def satisfied_requirements(self):
|
|
|
|
|
|
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
|
|
#
|
|
# This file is part of cdist.
|
|
#
|
|
# cdist is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# cdist is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
#
|
|
|
|
import logging
|
|
import os
|
|
import itertools
|
|
import fnmatch
|
|
|
|
import cdist
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class CircularReferenceError(cdist.Error):
|
|
def __init__(self, cdist_object, required_object):
|
|
self.cdist_object = cdist_object
|
|
self.required_object = required_object
|
|
|
|
def __str__(self):
|
|
return 'Circular reference detected: %s -> %s' % (self.cdist_object.name, self.required_object.name)
|
|
|
|
|
|
class RequirementNotFoundError(cdist.Error):
|
|
def __init__(self, requirement):
|
|
self.requirement = requirement
|
|
|
|
def __str__(self):
|
|
return 'Requirement could not be found: %s' % self.requirement
|
|
|
|
|
|
class DependencyResolver(object):
|
|
"""Cdist's dependency resolver.
|
|
|
|
Usage:
|
|
>> resolver = DependencyResolver(list_of_objects)
|
|
# Easy access to the objects we are working with
|
|
>> resolver.objects['__some_type/object_id']
|
|
<CdistObject __some_type/object_id>
|
|
# Easy access to a specific objects dependencies
|
|
>> resolver.dependencies['__some_type/object_id']
|
|
[<CdistObject __other_type/dependency>, <CdistObject __some_type/object_id>]
|
|
# Pretty print the dependency graph
|
|
>> from pprint import pprint
|
|
>> pprint(resolver.dependencies)
|
|
# Iterate over all existing objects in the correct order
|
|
>> for cdist_object in resolver:
|
|
>> do_something_with(cdist_object)
|
|
"""
|
|
def __init__(self, objects, logger=None):
|
|
self.objects = dict((o.name, o) for o in objects)
|
|
self._dependencies = None
|
|
self.log = logger or log
|
|
|
|
@property
|
|
def dependencies(self):
|
|
"""Build the dependency graph.
|
|
|
|
Returns a dict where the keys are the object names and the values are
|
|
lists of all dependencies including the key object itself.
|
|
"""
|
|
if self._dependencies is None:
|
|
log.info("Resolving dependencies...")
|
|
self._dependencies = d = {}
|
|
self._preprocess_requirements()
|
|
for name,cdist_object in self.objects.items():
|
|
resolved = []
|
|
unresolved = []
|
|
self._resolve_object_dependencies(cdist_object, resolved, unresolved)
|
|
d[name] = resolved
|
|
return self._dependencies
|
|
|
|
def find_requirements_by_name(self, requirements):
|
|
"""Takes a list of requirement patterns and returns a list of matching object instances.
|
|
|
|
Patterns are expected to be Unix shell-style wildcards for use with fnmatch.filter.
|
|
|
|
find_requirements_by_name(['__type/object_id', '__other_type/*']) ->
|
|
[<Object __type/object_id>, <Object __other_type/any>, <Object __other_type/match>]
|
|
"""
|
|
object_names = self.objects.keys()
|
|
for pattern in requirements:
|
|
found = False
|
|
for requirement in fnmatch.filter(object_names, pattern):
|
|
found = True
|
|
yield self.objects[requirement]
|
|
if not found:
|
|
# FIXME: get rid of the singleton object_id, it should be invisible to the code -> hide it in Object
|
|
singleton = os.path.join(pattern, 'singleton')
|
|
if singleton in self.objects:
|
|
yield self.objects[singleton]
|
|
else:
|
|
raise RequirementNotFoundError(pattern)
|
|
|
|
def _preprocess_requirements(self):
|
|
"""Find all autorequire dependencies and merge them to be just requirements
|
|
for further processing.
|
|
"""
|
|
for cdist_object in self.objects.values():
|
|
if cdist_object.autorequire:
|
|
# The objects (children) that this cdist_object (parent) defined
|
|
# in it's type manifest shall inherit all explicit requirements
|
|
# that the parent has so that user defined requirements are
|
|
# fullfilled and processed in the expected order.
|
|
for auto_requirement in self.find_requirements_by_name(cdist_object.autorequire):
|
|
for requirement in cdist_object.requirements:
|
|
if requirement not in auto_requirement.requirements:
|
|
auto_requirement.requirements.append(requirement)
|
|
# On the other hand the parent shall depend on all the children
|
|
# it created so that the user can setup dependencies on it as a
|
|
# whole without having to know anything about the parents
|
|
# internals.
|
|
cdist_object.requirements.extend(cdist_object.autorequire)
|
|
# As we changed the object on disc, we have to ensure it is not
|
|
# preprocessed again if someone would call us multiple times.
|
|
cdist_object.autorequire = []
|
|
|
|
def _resolve_object_dependencies(self, cdist_object, resolved, unresolved):
|
|
"""Resolve all dependencies for the given cdist_object and store them
|
|
in the list which is passed as the 'resolved' arguments.
|
|
|
|
e.g.
|
|
resolved = []
|
|
unresolved = []
|
|
resolve_object_dependencies(some_object, resolved, unresolved)
|
|
print("Dependencies for %s: %s" % (some_object, resolved))
|
|
"""
|
|
self.log.debug('Resolving dependencies for: %s' % cdist_object.name)
|
|
try:
|
|
unresolved.append(cdist_object)
|
|
for required_object in self.find_requirements_by_name(cdist_object.requirements):
|
|
self.log.debug("Object %s requires %s", cdist_object, required_object)
|
|
if required_object not in resolved:
|
|
if required_object in unresolved:
|
|
raise CircularReferenceError(cdist_object, required_object)
|
|
self._resolve_object_dependencies(required_object, resolved, unresolved)
|
|
resolved.append(cdist_object)
|
|
unresolved.remove(cdist_object)
|
|
except RequirementNotFoundError as e:
|
|
raise cdist.CdistObjectError(cdist_object, "requires non-existing " + e.requirement)
|
|
|
|
def __iter__(self):
|
|
"""Iterate over all unique objects and yield them in the correct order.
|
|
"""
|
|
iterable = itertools.chain(*self.dependencies.values())
|
|
# Keep record of objects that have already been seen
|
|
seen = set()
|
|
seen_add = seen.add
|
|
for cdist_object in itertools.filterfalse(seen.__contains__, iterable):
|
|
seen_add(cdist_object)
|
|
yield cdist_object
|