Merge branch 'master' into type__rbenv

build

@@ -341,7 +341,7 @@ eof
 
     web)
         set -e
-        $0 web-doc
+        "$0" web-doc
         # Fix ikiwiki, which does not like symlinks for pseudo security
         ssh tee.schottelius.org \
           "cd /home/services/www/nico/www.nico.schottelius.org/www/software/cdist/man &&
@@ -349,7 +349,7 @@ eof
     ;;
 
      p|pu|pub)
-          for remote in "" github sf ethz; do
+          for remote in "" github sf; do
                 echo "Pushing to $remote"
                 git push --mirror $remote
           done

cdist/conf/type/__jail/gencode-remote

@@ -35,7 +35,8 @@ fi
 state="$(cat "$__object/parameter/state")"
 
 started="true"
-[ -f "$__object/parameter/stopped" ] && started="false"
+# If the user wants the jail gone, it implies it shouldn't be started.
+[ -f "$__object/parameter/stopped" -o "$state" = "absent" ] && started="false"
 
 if [ -f "$__object/parameter/ip" ]; then
    ip="$(cat "$__object/parameter/ip")"
@@ -92,14 +93,6 @@ fi
 present="$(cat "$__object/explorer/present")"
 status="$(cat "$__object/explorer/status")"
 
-# Defining a jail as absent and started at the same time
-#    makes no sense. Treat this as an error.
-if [ "$started" = "true" -a "$state" = "absent" ]; then
-   exec >&2
-   echo "Can't have --state absent and --started true together\!"
-   exit 1
-fi
-
 stopJail() {
 # Check $status before issuing command
    if [ "$status" = "STARTED" ]; then

cdist/conf/type/__package_pip/gencode-remote

@@ -46,10 +46,10 @@ fi
 
 case "$state_should" in
     present)
-        echo $pip install -q pyro
+        echo $pip install -q "$name"
     ;;
     absent)
-        echo $pip uninstall -q -y pyro
+        echo $pip uninstall -q -y "$name"
     ;;
     *)
         echo "Unknown state: $state_should" >&2

cdist/config_install.py

@@ -43,26 +43,25 @@ class ConfigInstall(object):
         self.context = context
         self.log = logging.getLogger(self.context.target_host)
 
-        # For easy access
-        self.local = context.local
-        self.remote = context.remote
-
         # Initialise local directory structure
-        self.local.create_files_dirs()
+        self.context.local.create_files_dirs()
         # Initialise remote directory structure
-        self.remote.create_files_dirs()
+        self.context.remote.create_files_dirs()
 
-        self.explorer = core.Explorer(self.context.target_host, self.local, self.remote)
+        self.explorer = core.Explorer(self.context.target_host, self.context.local, self.context.remote)
-        self.manifest = core.Manifest(self.context.target_host, self.local)
+        self.manifest = core.Manifest(self.context.target_host, self.context.local)
-        self.code = core.Code(self.context.target_host, self.local, self.remote)
+        self.code = core.Code(self.context.target_host, self.context.local, self.context.remote)
+
+        # Add switch to disable code execution
+        self.dry_run = False
 
     def cleanup(self):
         # FIXME: move to local?
-        destination = os.path.join(self.local.cache_path, self.context.target_host)
+        destination = os.path.join(self.context.local.cache_path, self.context.target_host)
-        self.log.debug("Saving " + self.local.out_path + " to " + destination)
+        self.log.debug("Saving " + self.context.local.out_path + " to " + destination)
         if os.path.exists(destination):
             shutil.rmtree(destination)
-        shutil.move(self.local.out_path, destination)
+        shutil.move(self.context.local.out_path, destination)
 
     def deploy_to(self):
         """Mimic the old deploy to: Deploy to one host"""
@@ -79,7 +78,7 @@ class ConfigInstall(object):
 
     def stage_prepare(self):
         """Do everything for a deploy, minus the actual code stage"""
-        self.explorer.run_global_explorers(self.local.global_explorer_out_path)
+        self.explorer.run_global_explorers(self.context.local.global_explorer_out_path)
         self.manifest.run_initial_manifest(self.context.initial_manifest)
 
         self.log.info("Running object manifests and type explorers")
@@ -88,8 +87,8 @@ class ConfigInstall(object):
         new_objects_created = True
         while new_objects_created:
             new_objects_created = False
-            for cdist_object in core.CdistObject.list_objects(self.local.object_path,
+            for cdist_object in core.CdistObject.list_objects(self.context.local.object_path,
-                                                         self.local.type_path):
+                                                         self.context.local.type_path):
                 if cdist_object.state == core.CdistObject.STATE_PREPARED:
                     self.log.debug("Skipping re-prepare of object %s", cdist_object)
                     continue
@@ -104,11 +103,10 @@ class ConfigInstall(object):
         self.manifest.run_type_manifest(cdist_object)
         cdist_object.state = core.CdistObject.STATE_PREPARED
 
-    def object_run(self, cdist_object):
+    def object_run(self, cdist_object, dry_run=False):
         """Run gencode and code for an object"""
         self.log.debug("Trying to run object " + cdist_object.name)
         if cdist_object.state == core.CdistObject.STATE_DONE:
-            # TODO: remove once we are sure that this really never happens.
             raise cdist.Error("Attempting to run an already finished object: %s", cdist_object)
 
         cdist_type = cdist_object.cdist_type
@@ -121,11 +119,12 @@ class ConfigInstall(object):
             cdist_object.changed = True
 
         # Execute
-        if cdist_object.code_local:
+        if not dry_run:
-            self.code.run_code_local(cdist_object)
+            if cdist_object.code_local:
-        if cdist_object.code_remote:
+                self.code.run_code_local(cdist_object)
-            self.code.transfer_code_remote(cdist_object)
+            if cdist_object.code_remote:
-            self.code.run_code_remote(cdist_object)
+                self.code.transfer_code_remote(cdist_object)
+                self.code.run_code_remote(cdist_object)
 
         # Mark this object as done
         self.log.debug("Finishing run of " + cdist_object.name)
@@ -135,13 +134,49 @@ class ConfigInstall(object):
         """The final (and real) step of deployment"""
         self.log.info("Generating and executing code")
 
-        objects = core.CdistObject.list_objects(
+        # FIXME: think about parallel execution (same for stage_prepare)
-            self.local.object_path,
+        self.all_resolved = False
-            self.local.type_path)
+        while not self.all_resolved:
+            self.stage_run_iterate()
 
-        dependency_resolver = resolver.DependencyResolver(objects)
+    def stage_run_iterate(self):
-        self.log.debug(pprint.pformat(dependency_resolver.dependencies))
+        """
+        Run one iteration of the run
 
-        for cdist_object in dependency_resolver:
+        To be repeated until all objects are done
-            self.log.debug("Run object: %s", cdist_object)
+        """
-            self.object_run(cdist_object)
+        objects = list(core.CdistObject.list_objects(self.context.local.object_path, self.context.local.type_path))
+        object_state_list=' '.join('%s:%s:%s:%s' % (o, o.state, o.all_requirements, o.satisfied_requirements) for o in objects)
+
+        self.log.debug("Object state (name:state:requirements:satisfied): %s" % object_state_list)
+
+        objects_changed = False
+        self.all_resolved = True
+        for cdist_object in objects:
+            if not cdist_object.state == cdist_object.STATE_DONE:
+                self.all_resolved = False
+                self.log.debug("Object %s not done" % cdist_object.name)
+                if cdist_object.satisfied_requirements:
+                    self.log.debug("Running object %s with satisfied requirements" % cdist_object.name)
+                    self.object_run(cdist_object, self.dry_run)
+                    objects_changed = True
+
+        self.log.debug("All resolved: %s Objects changed: %s" % (self.all_resolved, objects_changed))
+
+        # Not all are resolved, but nothing has been changed => bad dependencies!
+        if not objects_changed and not self.all_resolved:
+            # Create list of unfinished objects + their requirements for print
+
+            evil_objects = []
+            good_objects = []
+            for cdist_object in objects:
+                if not cdist_object.state == cdist_object.STATE_DONE:
+                    evil_objects.append("%s: required: %s, autorequired: %s" %
+                        (cdist_object.name, cdist_object.requirements, cdist_object.autorequire))
+                else:
+                    evil_objects.append("%s (%s): required: %s, autorequired: %s" %
+                        (cdist_object.state, cdist_object.name, 
+                        cdist_object.requirements, cdist_object.autorequire))
+
+            errormessage = "Cannot solve requirements for the following objects: %s - solved: %s" % (",".join(evil_objects), ",".join(good_objects))
+            raise cdist.Error(errormessage)

cdist/core/cdist_object.py

@@ -20,6 +20,7 @@
 #
 #
 
+import fnmatch
 import logging
 import os
 import collections
@@ -56,6 +57,21 @@ class CdistObject(object):
     STATE_RUNNING = "running"
     STATE_DONE = "done"
 
+    def __init__(self, cdist_type, base_path, object_id=None):
+        self.cdist_type = cdist_type # instance of Type
+        self.base_path = base_path
+        self.object_id = object_id
+
+        self.validate_object_id()
+        self.sanitise_object_id()
+
+        self.name = self.join_name(self.cdist_type.name, self.object_id)
+        self.path = os.path.join(self.cdist_type.path, self.object_id, OBJECT_MARKER)
+        self.absolute_path = os.path.join(self.base_path, self.path)
+        self.code_local_path = os.path.join(self.path, "code-local")
+        self.code_remote_path = os.path.join(self.path, "code-remote")
+        self.parameter_path = os.path.join(self.path, "parameter")
+
     @classmethod
     def list_objects(cls, object_base_path, type_base_path):
         """Return a list of object instances"""
@@ -112,21 +128,6 @@ class CdistObject(object):
             raise IllegalObjectIdError(self.object_id,
                 "Missing object_id and type is not a singleton.")
 
-    def __init__(self, cdist_type, base_path, object_id=None):
-        self.cdist_type = cdist_type # instance of Type
-        self.base_path = base_path
-        self.object_id = object_id
-
-        self.validate_object_id()
-        self.sanitise_object_id()
-
-        self.name = self.join_name(self.cdist_type.name, self.object_id)
-        self.path = os.path.join(self.cdist_type.path, self.object_id, OBJECT_MARKER)
-        self.absolute_path = os.path.join(self.base_path, self.path)
-        self.code_local_path = os.path.join(self.path, "code-local")
-        self.code_remote_path = os.path.join(self.path, "code-remote")
-        self.parameter_path = os.path.join(self.path, "parameter")
-
     def object_from_name(self, object_name):
         """Convenience method for creating an object instance from an object name.
 
@@ -209,3 +210,67 @@ class CdistObject(object):
             os.makedirs(absolute_parameter_path, exist_ok=False)
         except EnvironmentError as error:
             raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error))
+
+    @property
+    def satisfied_requirements(self):
+        """Return state whether all of our dependencies have been resolved already"""
+
+        satisfied = True
+
+        for requirement in self.all_requirements:
+            log.debug("%s: Checking requirement %s (%s) .." % (self.name, requirement.name, requirement.state))
+            if not requirement.state == self.STATE_DONE:
+                satisfied = False
+                break
+        log.debug("%s is satisfied: %s" % (self.name, satisfied))
+
+        return satisfied
+
+
+    def find_requirements_by_name(self, requirements):
+        """Takes a list of requirement patterns and returns a list of matching object instances.
+
+        Patterns are expected to be Unix shell-style wildcards for use with fnmatch.filter.
+
+        find_requirements_by_name(['__type/object_id', '__other_type/*']) -> 
+            [<Object __type/object_id>, <Object __other_type/any>, <Object __other_type/match>]
+        """
+
+
+        # FIXME: think about where/when to store this - probably not here
+        self.objects = dict((o.name, o) for o in self.list_objects(self.base_path, self.cdist_type.base_path))
+        object_names = self.objects.keys()
+
+        for pattern in requirements:
+            found = False
+            for requirement in fnmatch.filter(object_names, pattern):
+                found = True
+                yield self.objects[requirement]
+            if not found:
+                # FIXME: get rid of the singleton object_id, it should be invisible to the code -> hide it in Object
+                singleton = os.path.join(pattern, 'singleton')
+                if singleton in self.objects:
+                    yield self.objects[singleton]
+                else:
+                    raise RequirementNotFoundError(pattern)
+
+    @property
+    def all_requirements(self):
+        """
+        Return resolved autorequirements and requirements so that
+        a complete list of requirements is returned
+        """
+
+        all_reqs= []
+        all_reqs.extend(self.find_requirements_by_name(self.requirements))
+        all_reqs.extend(self.find_requirements_by_name(self.autorequire))
+
+        return set(all_reqs)
+
+
+class RequirementNotFoundError(cdist.Error):
+    def __init__(self, requirement):
+        self.requirement = requirement
+
+    def __str__(self):
+        return 'Requirement could not be found: %s' % self.requirement

cdist/core/cdist_type.py

@@ -42,6 +42,26 @@ class CdistType(object):
 
     """
 
+    def __init__(self, base_path, name):
+        self.base_path = base_path
+        self.name = name
+        self.path = self.name
+        self.absolute_path = os.path.join(self.base_path, self.path)
+        if not os.path.isdir(self.absolute_path):
+            raise NoSuchTypeError(self.path, self.absolute_path)
+        self.manifest_path = os.path.join(self.name, "manifest")
+        self.explorer_path = os.path.join(self.name, "explorer")
+        self.gencode_local_path = os.path.join(self.name, "gencode-local")
+        self.gencode_remote_path = os.path.join(self.name, "gencode-remote")
+        self.manifest_path = os.path.join(self.name, "manifest")
+
+        self.__explorers = None
+        self.__required_parameters = None
+        self.__required_multiple_parameters = None
+        self.__optional_parameters = None
+        self.__optional_multiple_parameters = None
+        self.__boolean_parameters = None
+
     @classmethod
     def list_types(cls, base_path):
         """Return a list of type instances"""
@@ -65,26 +85,6 @@ class CdistType(object):
             # return instance so __init__ is called
         return cls._instances[name]
 
-    def __init__(self, base_path, name):
-        self.base_path = base_path
-        self.name = name
-        self.path = self.name
-        self.absolute_path = os.path.join(self.base_path, self.path)
-        if not os.path.isdir(self.absolute_path):
-            raise NoSuchTypeError(self.path, self.absolute_path)
-        self.manifest_path = os.path.join(self.name, "manifest")
-        self.explorer_path = os.path.join(self.name, "explorer")
-        self.gencode_local_path = os.path.join(self.name, "gencode-local")
-        self.gencode_remote_path = os.path.join(self.name, "gencode-remote")
-        self.manifest_path = os.path.join(self.name, "manifest")
-
-        self.__explorers = None
-        self.__required_parameters = None
-        self.__required_multiple_parameters = None
-        self.__optional_parameters = None
-        self.__optional_multiple_parameters = None
-        self.__boolean_parameters = None
-
     def __repr__(self):
         return '<CdistType %s>' % self.name
 

cdist/test/autorequire/__init__.py

@@ -65,10 +65,10 @@ class AutorequireTestCase(test.CdistTestCase):
         shutil.rmtree(self.temp_dir)
 
     def test_implicit_dependencies(self):
-        self.context.initial_manifest = os.path.join(self.config.local.manifest_path, 'implicit_dependencies')
+        self.context.initial_manifest = os.path.join(self.context.local.manifest_path, 'implicit_dependencies')
         self.config.stage_prepare()
 
-        objects = core.CdistObject.list_objects(self.config.local.object_path, self.config.local.type_path)
+        objects = core.CdistObject.list_objects(self.context.local.object_path, self.context.local.type_path)
         dependency_resolver = resolver.DependencyResolver(objects)
         expected_dependencies = [
             dependency_resolver.objects['__package_special/b'],
@@ -79,7 +79,7 @@ class AutorequireTestCase(test.CdistTestCase):
         self.assertEqual(resolved_dependencies, expected_dependencies)
 
     def test_circular_dependency(self):
-        self.context.initial_manifest = os.path.join(self.config.local.manifest_path, 'circular_dependency')
+        self.context.initial_manifest = os.path.join(self.context.local.manifest_path, 'circular_dependency')
         self.config.stage_prepare()
         # raises CircularDependecyError
         self.config.stage_run()

cdist/test/config_install/__init__.py

@@ -0,0 +1,119 @@
+# -*- coding: utf-8 -*-
+#
+# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2012 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import os
+import shutil
+
+from cdist import test
+from cdist import core
+
+import cdist
+import cdist.context
+import cdist.config
+
+import os.path as op
+my_dir = op.abspath(op.dirname(__file__))
+fixtures = op.join(my_dir, 'fixtures')
+object_base_path = op.join(fixtures, 'object')
+type_base_path = op.join(fixtures, 'type')
+add_conf_dir = op.join(fixtures, 'conf')
+
+class ConfigInstallRunTestCase(test.CdistTestCase):
+
+    def setUp(self):
+
+        # Change env for context
+        self.orig_environ = os.environ
+        os.environ = os.environ.copy()
+        self.temp_dir = self.mkdtemp()
+
+        self.out_dir = os.path.join(self.temp_dir, "out")
+        self.remote_out_dir = os.path.join(self.temp_dir, "remote")
+
+        os.environ['__cdist_out_dir'] = self.out_dir
+        os.environ['__cdist_remote_out_dir'] = self.remote_out_dir
+
+        self.context = cdist.context.Context(
+            target_host=self.target_host,
+            remote_copy=self.remote_copy,
+            remote_exec=self.remote_exec,
+            exec_path=test.cdist_exec_path,
+            debug=True)
+
+        self.context.local.object_path = object_base_path
+        self.context.local.type_path = type_base_path
+
+        self.config = cdist.config.Config(self.context)
+
+        self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
+        self.object_index = dict((o.name, o) for o in self.objects)
+        self.object_names = [o.name for o in self.objects]
+
+    def tearDown(self):
+        for o in self.objects:
+            o.requirements = []
+            o.state = ""
+
+        os.environ = self.orig_environ
+        shutil.rmtree(self.temp_dir)
+
+    def test_dependency_resolution(self):
+        first   = self.object_index['__first/man']
+        second  = self.object_index['__second/on-the']
+        third   = self.object_index['__third/moon']
+
+        first.requirements = [second.name]
+        second.requirements = [third.name]
+
+        # First run: 
+        # solves first and maybe second (depending on the order in the set)
+        self.config.stage_run_iterate()
+        self.assertTrue(third.state == third.STATE_DONE)
+
+        self.config.stage_run_iterate()
+        self.assertTrue(second.state == second.STATE_DONE)
+
+
+        try:
+            self.config.stage_run_iterate()
+        except cdist.Error:
+            # Allow failing, because the third run may or may not be unecessary already,
+            # depending on the order of the objects
+            pass
+        self.assertTrue(first.state == first.STATE_DONE)
+
+    def test_unresolvable_requirements(self):
+        """Ensure an exception is thrown for unresolvable depedencies"""
+
+        # Create to objects depending on each other - no solution possible
+        first   = self.object_index['__first/man']
+        second  = self.object_index['__second/on-the']
+
+        first.requirements = [second.name]
+        second.requirements = [first.name]
+
+        # First round solves __third/moon
+        self.config.stage_run_iterate()
+
+        # Second round detects it cannot solve the rest
+        with self.assertRaises(cdist.Error):
+            self.config.stage_run_iterate()

cdist/test/config_install/fixtures/object/__third/moon/.cdist/parameter/name

@@ -0,0 +1 @@
+Prometheus

cdist/test/config_install/fixtures/object/__third/moon/.cdist/parameter/planet

@@ -0,0 +1 @@
+Saturn

cdist/test/object/__init__.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2012 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -35,22 +36,33 @@ type_base_path = op.join(fixtures, 'type')
 
 class ObjectClassTestCase(test.CdistTestCase):
 
+    def setUp(self):
+        self.expected_object_names = sorted([
+            '__first/child',
+            '__first/dog',
+            '__first/man',
+            '__first/woman',
+            '__second/on-the',
+            '__second/under-the',
+            '__third/moon'])
+
+        self.expected_objects = []
+        for cdist_object_name in self.expected_object_names:
+            cdist_type, cdist_object_id = cdist_object_name.split("/", maxsplit=1)
+            cdist_object = core.CdistObject(core.CdistType(type_base_path, cdist_type), object_base_path, cdist_object_id)
+            self.expected_objects.append(cdist_object)
+ 
     def test_list_object_names(self):
-        object_names = list(core.CdistObject.list_object_names(object_base_path))
+       found_object_names = sorted(list(core.CdistObject.list_object_names(object_base_path)))
-        self.assertEqual(object_names, ['__first/man', '__second/on-the', '__third/moon'])
+       self.assertEqual(found_object_names, self.expected_object_names)
 
     def test_list_type_names(self):
         type_names = list(cdist.core.CdistObject.list_type_names(object_base_path))
         self.assertEqual(type_names, ['__first', '__second', '__third'])
 
     def test_list_objects(self):
-        objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
+        found_objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
-        objects_expected = [
+        self.assertEqual(found_objects, self.expected_objects)
-            core.CdistObject(core.CdistType(type_base_path, '__first'), object_base_path, 'man'),
-            core.CdistObject(core.CdistType(type_base_path, '__second'), object_base_path, 'on-the'),
-            core.CdistObject(core.CdistType(type_base_path, '__third'), object_base_path, 'moon'),
-        ]
-        self.assertEqual(objects, objects_expected)
 
 
 class ObjectIdTestCase(test.CdistTestCase):
@@ -200,3 +212,54 @@ class ObjectTestCase(test.CdistTestCase):
         self.assertTrue(isinstance(other_object, core.CdistObject))
         self.assertEqual(other_object.cdist_type.name, '__first')
         self.assertEqual(other_object.object_id, 'man')
+
+
+
+class ObjectResolveRequirementsTestCase(test.CdistTestCase):
+
+    def setUp(self):
+        self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
+        self.object_index = dict((o.name, o) for o in self.objects)
+        self.object_names = [o.name for o in self.objects]
+
+        print(self.objects)
+
+        self.cdist_type = core.CdistType(type_base_path, '__third')
+        self.cdist_object = core.CdistObject(self.cdist_type, object_base_path, 'moon') 
+
+    def tearDown(self):
+        for o in self.objects:
+            o.requirements = []
+
+    def test_find_requirements_by_name_string(self):
+        """Check that resolving requirements by name works (require all objects)"""
+        requirements = self.object_names
+
+        self.cdist_object.requirements = requirements
+
+        found_requirements = sorted(self.cdist_object.find_requirements_by_name(self.cdist_object.requirements))
+        expected_requirements = sorted(self.objects)
+
+        self.assertEqual(found_requirements, expected_requirements)
+
+    def test_find_requirements_by_name_pattern(self):
+        """Test whether pattern matching on requirements works"""
+
+        # Matches all objects in the end
+        requirements = ['__first/*', '__second/*-the', '__third/moon']
+
+        self.cdist_object.requirements = requirements
+
+        expected_requirements = sorted(self.objects)
+        found_requirements = sorted(self.cdist_object.find_requirements_by_name(self.cdist_object.requirements))
+
+        self.assertEqual(expected_requirements, found_requirements)
+
+    def test_requirement_not_found(self):
+        """Ensure an exception is thrown for missing depedencies"""
+        cdist_object = self.object_index['__first/man']
+        cdist_object.requirements = ['__does/not/exist']
+
+        with self.assertRaises(core.cdist_object.RequirementNotFoundError):
+            # Use list, as generator does not (yet) raise the error
+            list(cdist_object.find_requirements_by_name(cdist_object.requirements))

cdist/test/resolver/__init__.py

@@ -1,88 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-import os
-import shutil
-
-import cdist
-from cdist import test
-from cdist import core
-from cdist import resolver
-
-import os.path as op
-my_dir = op.abspath(op.dirname(__file__))
-fixtures = op.join(my_dir, 'fixtures')
-object_base_path = op.join(fixtures, 'object')
-type_base_path = op.join(fixtures, 'type')
-
-
-class ResolverTestCase(test.CdistTestCase):
-
-    def setUp(self):
-        self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
-        self.object_index = dict((o.name, o) for o in self.objects)
-        self.dependency_resolver = resolver.DependencyResolver(self.objects)
-
-    def tearDown(self):
-        for o in self.objects:
-            o.requirements = []
-
-    def test_find_requirements_by_name_string(self):
-        requirements = ['__first/man', '__second/on-the', '__third/moon']
-        required_objects = [self.object_index[name] for name in requirements]
-        self.assertEqual(sorted(list(self.dependency_resolver.find_requirements_by_name(requirements))),
-            sorted(required_objects))
-
-    def test_find_requirements_by_name_pattern(self):
-        requirements = ['__first/*', '__second/*-the', '__third/moon']
-        requirements_expanded = [
-            '__first/child', '__first/dog', '__first/man', '__first/woman',
-            '__second/on-the', '__second/under-the',
-            '__third/moon'
-        ]
-        required_objects = [self.object_index[name] for name in requirements_expanded]
-        self.assertEqual(sorted(list(self.dependency_resolver.find_requirements_by_name(requirements))),
-            sorted(required_objects))
-
-    def test_dependency_resolution(self):
-        first_man = self.object_index['__first/man']
-        second_on_the = self.object_index['__second/on-the']
-        third_moon = self.object_index['__third/moon']
-        first_man.requirements = [second_on_the.name]
-        second_on_the.requirements = [third_moon.name]
-        self.assertEqual(
-            self.dependency_resolver.dependencies['__first/man'],
-            [third_moon, second_on_the, first_man]
-        )
-
-    def test_circular_reference(self):
-        first_man = self.object_index['__first/man']
-        first_woman = self.object_index['__first/woman']
-        first_man.requirements = [first_woman.name]
-        first_woman.requirements = [first_man.name]
-        with self.assertRaises(resolver.CircularReferenceError):
-            self.dependency_resolver.dependencies
-
-    def test_requirement_not_found(self):
-        first_man = self.object_index['__first/man']
-        first_man.requirements = ['__does/not/exist']
-        with self.assertRaises(cdist.Error):
-            self.dependency_resolver.dependencies

docs/changelog

@@ -4,7 +4,11 @@ Changelog
 	* Changes are always commented with their author in (braces)
 	* Exception: No braces means author == Nico Schottelius
 
-2.1.0:
+next:
+	* Type __jail: State absent should implies stopped (Jake Guffey)
+	* Core: Use dynamic dependency resolver to allow indirect self dependencies
+
+2.1.0: 2012-12-09
 	* Core: Ensure global explorers are executable
 	* Core: Ensure type explorers are executable (Steven Armstrong)
 	* New Type: __git
@@ -16,6 +20,7 @@ Changelog
 	* Type __jail: Change optional parameter "started" to boolean "stopped" parameter,
 		change optional parameter "devfs-enable" to boolean "devfs-disable" parameter and
 		change optional parameter "onboot" to boolean.
+	* Type __package_pip: Bugfix: Installeded the package, not pyro
 	* Remove Type __ssh_authorized_key: Superseeded by __ssh_authorized_keys
 	* Support for CDIST_PATH (Steven Armstrong)
 

docs/dev/logs/2012-12-11.dependencies

@@ -0,0 +1,72 @@
+2.1.0 behaviour:
+
+
+__git foo
+    __package git --state present
+
+__git bar
+    __package git --state present
+
+
+require="__git/foo" git bar:
+
+    __git bar
+        __git foo
+            __package git --state present
+        __package git --state present
+            __git foo
+                __package git --state present
+
+    -> detects circular dependency
+
+
+--------------------------------------------------------------------------------
+
+__package abc
+    __package_apt abc
+
+__sometype def
+    __package abc
+        __package_apt abc
+
+
+--------------------------------------------------------------------------------
+
+Change proposal:
+
+Each object only depends on the objects it directly requires, tree build to
+ensure correct running behaviour:
+
+
+__git bar
+    __git foo
+    __package git --state present
+
+__git foo
+    __package git --state present
+
+Order:
+
+1)
+    __package/git (leaf node!)
+
+2)
+    __git/foo (new leaf node!)
+
+3)
+    __git/bar (new leaf node!)
+
+
+For __package:
+
+__sometype def
+    __package abc
+
+__package abc
+    __package_apt abc
+
+1) __package_apt/abc (leaf node)
+
+2) __package/abc (new leaf node)
+
+3) __sometype/def (new leaf node)

docs/man/cdist-reference.text.sh

@@ -59,62 +59,75 @@ cat << eof
 
 PATHS
 -----
-If not specified otherwise, all paths are relative to the checkout directory.
+\$HOME/.cdist::
+    The standard cdist configuration directory relative to your home directory
+    This is usually the place you want to store your site specific configuration
 
-conf/::
+cdist/conf/::
-    Contains the (static) configuration like manifests, types and explorers.  
+    The distribution configuration directory
+    This contains types and explorers to be used
 
-conf/manifest/init::
+confdir::
+    Cdist will use all available configuration directories and create
+    a temporary confdir containing links to the real configuration directories.
+    This way it is possible to merge configuration directories.
+    By default it consists of everything in \$HOME/.cdist and cdist/conf/.
+    For more details see cdist(1)
+
+confdir/manifest/init::
     This is the central entry point.
     It is an executable (+x bit set) shell script that can use
     values from the explorers to decide which configuration to create
     for the specified target host.
     Its intent is to used to define mapping from configurations to hosts.
 
-conf/manifest/*::
+confdir/manifest/*::
     All other files in this directory are not directly used by cdist, but you
     can seperate configuration mappings, if you have a lot of code in the
     conf/manifest/init file. This may also be helpful to have different admins
     maintain different groups of hosts.
 
-conf/explorer/<name>::
+confdir/explorer/<name>::
     Contains explorers to be run on the target hosts, see cdist-explorer(7).
 
-conf/type/::
+confdir/type/::
     Contains all available types, which are used to provide
     some kind of functionality. See cdist-type(7).
 
-conf/type/<name>/::
+confdir/type/<name>/::
     Home of the type <name>.
-
     This directory is referenced by the variable __type (see below).
 
-conf/type/<name>/man.text::
+confdir/type/<name>/man.text::
     Manpage in Asciidoc format (required for inclusion into upstream)
 
-conf/type/<name>/manifest::
+confdir/type/<name>/manifest::
     Used to generate additional objects from a type.
 
-conf/type/<name>/gencode-local::
+confdir/type/<name>/gencode-local::
     Used to generate code to be executed on the source host
 
-conf/type/<name>/gencode-remote::
+confdir/type/<name>/gencode-remote::
     Used to generate code to be executed on the target host
 
-conf/type/<name>/parameter/required::
+confdir/type/<name>/parameter/required::
     Parameters required by type, \n seperated list.
 
-conf/type/<name>/parameter/optional::
+confdir/type/<name>/parameter/optional::
     Parameters optionally accepted by type, \n seperated list.
 
-conf/type/<name>/parameter/boolean::
+confdir/type/<name>/parameter/boolean::
    Boolean parameters accepted by type, \n seperated list.
 
-conf/type/<name>/explorer::
+confdir/type/<name>/explorer::
     Location of the type specific explorers.
     This directory is referenced by the variable __type_explorer (see below).
     See cdist-explorer(7).
 
+confdir/type/<name>/files::
+    This directory is reserved for user data and will not be used
+    by cdist at any time
+
 out/::
     This directory contains output of cdist and is usually located
     in a temporary directory and thus will be removed after the run.
@@ -179,10 +192,8 @@ __object::
 __object_id::
     The type unique object id.
     Available for: type manifest, type explorer, type gencode
-
     Note: The leading and the trailing "/" will always be stripped (caused by
     the filesystem database and ensured by the core).
-
     Note: Double slashes ("//") will not be fixed and result in an error.
 __object_name::
     The full qualified name of the current object.

docs/man/man1/cdist.text

@@ -78,7 +78,7 @@ EXAMPLES
 # Configure ikq05.ethz.ch with debug enabled
 cdist config -d ikq05.ethz.ch
 
-# Configure hosts in parallel and use a different home directory
+# Configure hosts in parallel and use a different configuration directory
 cdist config -c ~/p/cdist-nutzung \
     -p ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch
 

docs/man/man7/cdist-hacker.text

@@ -44,7 +44,7 @@ work nor kill the authors brain:
 
 - All files should contain the usual header (Author, Copying, etc.)
 - Code submission must be done via git
-- Do not add conf/manifest/init - This file should only be touched in your
+- Do not add cdist/conf/manifest/init - This file should only be touched in your
   private branch!
 - Code to be included should be branched of the upstream "master" branch
    - Exception: Bugfixes to a version branch

docs/man/man7/cdist-manifest.text

@@ -16,8 +16,8 @@ An object is represented by the combination of
 **type + slash + object name**: **__file/etc/cdist-configured** is an
 object of the type ***__file*** with the name ***etc/cdist-configured***.
 
-All available types can be found in the **conf/type/** directory,
+All available types can be found in the **cdist/conf/type/** directory,
-use **ls conf/type** to get the list of available types. If you have
+use **ls cdist/conf/type** to get the list of available types. If you have
 setup the MANPATH correctly, you can use **man cdist-reference** to access
 the reference with pointers to the manpages.
 
@@ -57,7 +57,7 @@ DEFINE STATE IN THE INITIAL MANIFEST
 ------------------------------------
 The **initial manifest** is the entry point for cdist to find out, which
 **objects** to configure on the selected host.
-Cdist searches for the initial manifest at **conf/manifest/init**.
+Cdist searches for the initial manifest at **cdist/conf/manifest/init**.
 
 Within this initial manifest, you define, which objects should be
 created on which host. To distinguish between hosts, you can use the
@@ -88,7 +88,7 @@ command.
 SPLITTING UP THE INITIAL MANIFEST
 ---------------------------------
 If you want to split up your initial manifest, you can create other shell
-scripts in **conf/manifest/** and include them in **conf/manifest/init**.
+scripts in **cdist/conf/manifest/** and include them in **cdist/conf/manifest/init**.
 Cdist provides the environment variable ***__manifest*** to reference to
 the directory containing the initial manifest (see cdist-reference(7)).
 

docs/man/man7/cdist-type.text

@@ -64,10 +64,10 @@ A type consists of
 - explorer  (optional)
 - gencode   (optional)
 
-Types are stored below conf/type/. Their name should always be prefixed with
+Types are stored below cdist/conf/type/. Their name should always be prefixed with
 two underscores (__) to prevent collisions with other executables in $PATH.
 
-To begin a new type, just create the directory **conf/type/__NAME**.
+To begin a new type, just create the directory **cdist/conf/type/__NAME**.
 
 
 DEFINING PARAMETERS
@@ -84,10 +84,10 @@ or no parameters at all.
 
 Example:
 --------------------------------------------------------------------------------
-echo servername >> conf/type/__nginx_vhost/parameter/required
+echo servername >> cdist/conf/type/__nginx_vhost/parameter/required
-echo logdirectory >> conf/type/__nginx_vhost/parameter/optional
+echo logdirectory >> cdist/conf/type/__nginx_vhost/parameter/optional
-echo server_alias >> conf/type/__nginx_vhost/parameter/optional_multiple
+echo server_alias >> cdist/conf/type/__nginx_vhost/parameter/optional_multiple
-echo use_ssl >> conf/type/__nginx_vhost/parameter/boolean
+echo use_ssl >> cdist/conf/type/__nginx_vhost/parameter/boolean
 --------------------------------------------------------------------------------
 
 
@@ -98,7 +98,7 @@ The parameters given to a type can be accessed and used in all type scripts
 represented by file existence. File exists -> True,
 file does not exist -> False
 
-Example: (e.g. in conf/type/__nginx_vhost/manifest)
+Example: (e.g. in cdist/conf/type/__nginx_vhost/manifest)
 --------------------------------------------------------------------------------
 # required parameter
 servername="$(cat "$__object/parameter/servername")"
@@ -129,7 +129,7 @@ INPUT FROM STDIN
 Every type can access what has been written on stdin when it has been called.
 The result is saved into the ***stdin*** file in the object directory.
 
-Example use of a type: (e.g. in conf/type/__archlinux_hostname)
+Example use of a type: (e.g. in cdist/conf/type/__archlinux_hostname)
 --------------------------------------------------------------------------------
 __file /etc/rc.conf --source - << eof
 ...
@@ -186,7 +186,7 @@ mark it as a singleton: Just create the (empty) file "singleton" in your type
 directory:
 
 --------------------------------------------------------------------------------
-touch conf/type/__NAME/singleton
+touch cdist/conf/type/__NAME/singleton
 --------------------------------------------------------------------------------
 
 This will also change the way your type must be called:

docs/web/cdist/update.mdwn

@@ -24,6 +24,8 @@ To upgrade to the lastet version do
 
 ### Updating from 2.0 to 2.1
 
+Have a look at the update guide for [[2.0 to 2.1|2.0-to-2.1]].
+
  * Type **\_\_package* and \_\_process** use --state **present** or **absent**.
    The states **removed/installed** and **stopped/running** have been removed.
    Support for the new states is already present in 2.0.

docs/web/cdist/update/2.0-to-2.1.mdwn

@@ -0,0 +1,118 @@
+[[!meta title="Update Guide for 2.0 to 2.1"]]
+
+## Introduction
+
+When changing your installation from 2.0 to 2.1, there are
+a lot of changes coming up. 2.1 is mainly a cleanup release,
+which removes long time deprecated behaviour, but also makes
+a lot of things more consistent and allows you to split off your types,
+explorers and manifest to custom directories.
+
+This document will guide you to a successful update.
+
+## Preperation
+
+As for every software and system you use in production, you should first of 
+all make a backup of your data. To prevent any breakage, it is
+recommended to create a new git branch to do the update on:
+
+    % git checkout -b update_to_2.1
+
+This also ensure that whenever you need to do a change in your
+2.0 based tree, you can simply go back to that branch, apply the change
+and configure your systems - independently of your update progress!
+
+Next fetch the latest upstream changes, I assume that
+origin refers to one of the upstream mirrors (change origin if you use
+another remote name for upstream cdist):
+
+    % git fetch -v origin
+
+## Merge the changes
+
+Now try to merge upstream into the new branch.
+
+    % git merge origin/2.1
+
+Fix any conflicts that may have been occurred due to local changes
+and then **git add** and *git commit** those changes. This should seldomly
+occur and if, it's mostly for people hacking on the cdist core.
+
+## Move "conf" directory
+
+One of the biggest changes in cdist 2.1 is that you can have multiple
+**conf** directories: Indeed, the new default behaviour of cdist is to
+search for conf directories
+
+ * below the python module (cdist/conf in the source tree or in the installed location)
+ * at ~/.cdist/ (on conf suffix there)
+
+So you can now choose, where to store your types.
+
+### Integrate your conf/ back into the tree
+
+If you choose to store your types together with the upstream types,
+you can just move all your stuff below **cdist/conf**:
+
+    % git mv conf/type/* cdist/conf/type
+    % git mv conf/manifest/* cdist/conf/manifest
+    % git mv conf/explorer/* cdist/conf/explorer
+    % git commit -m "Re-Integrate my conf directory into cdist 2.1 tree"
+
+### Move your conf/ directory to ~/.cdist
+
+If you want to store your site specific
+configuration outside of the cdist tree, you
+can move your conf/ directory to your homedirectory ($HOME) under ~/.cdist:
+
+    % mv conf ~/.cdist
+    % git rm -r conf
+    % git commit -m "Move my conf directory to ~/.cdist"
+
+It it still recommended to use a version control system like git in it:
+
+    % cd ~/.cdist
+    % git init
+    % git add .
+    % git commit -m "Create new git repository containing my cdist configuration"
+
+## Test the migration
+
+Some of the types shipped with upstream were changed, so you may want to test 
+the result by running cdist on one of your staging target hosts:
+
+    % ./bin/cdist config -v staging-host
+
+All incompatibilities are listed on the [[cdist update page|software/cdist/update]],
+so you can browse through the list and update your configuration.
+
+## Final Cleanups
+
+When everything is tested, there are some cleanups to be done to finalise the update.
+
+### When continuing to keep conf/ in the tree
+
+You can then merge back your changes into the master tree and continue to work
+as normal.
+
+### When using ~/.cdist
+
+If you decided to move your site specific code to ~/.cdist, you can now switch your
+**master** branch or version branch to upstream directly. Assumnig you are in the
+cdist directory, having your previous branch checked out, you can create a clean
+state using the following commands:
+
+    % upstream_branch=2.1
+    % current_branch=$(git rev-parse --abbrev-ref HEAD)
+    % git checkout -b archive_my_own_tree
+    % git branch -D "$current_branch"
+    % git checkout -b "$current_branch" "origin/$upstream_branch"
+
+Afther these commands, your previous main branch is accessible at
+**archive_my_own_tree** and your branch is now tracking upstream.
+
+## Questions? Critics? Hints?
+
+If you think this manual helped or misses some information, do not 
+hesitate to contact us on any of the usual ways (irc, mailinglist,
+github issue tracker, ...).

docs/web/cdist/why.mdwn

@@ -41,7 +41,7 @@ Cdist requires very litte on a target system. Even better,
 in almost all cases all dependencies are usually fulfilled.
 Cdist does not require an agent or a high level programming
 languages on the target host: it will run on any host that
-has an **ssh server running** and a posix compatible shell
+has a **ssh server running** and a posix compatible shell
 (**/bin/sh**).
 
 ## Push based distribution