Merge branch 'master' into type__rbenv

This commit is contained in:
Nico Schottelius 2013-01-02 22:32:22 +01:00
commit 6c9606330c
35 changed files with 611 additions and 214 deletions

4
build
View file

@ -341,7 +341,7 @@ eof
web) web)
set -e set -e
$0 web-doc "$0" web-doc
# Fix ikiwiki, which does not like symlinks for pseudo security # Fix ikiwiki, which does not like symlinks for pseudo security
ssh tee.schottelius.org \ ssh tee.schottelius.org \
"cd /home/services/www/nico/www.nico.schottelius.org/www/software/cdist/man && "cd /home/services/www/nico/www.nico.schottelius.org/www/software/cdist/man &&
@ -349,7 +349,7 @@ eof
;; ;;
p|pu|pub) p|pu|pub)
for remote in "" github sf ethz; do for remote in "" github sf; do
echo "Pushing to $remote" echo "Pushing to $remote"
git push --mirror $remote git push --mirror $remote
done done

View file

@ -35,7 +35,8 @@ fi
state="$(cat "$__object/parameter/state")" state="$(cat "$__object/parameter/state")"
started="true" started="true"
[ -f "$__object/parameter/stopped" ] && started="false" # If the user wants the jail gone, it implies it shouldn't be started.
[ -f "$__object/parameter/stopped" -o "$state" = "absent" ] && started="false"
if [ -f "$__object/parameter/ip" ]; then if [ -f "$__object/parameter/ip" ]; then
ip="$(cat "$__object/parameter/ip")" ip="$(cat "$__object/parameter/ip")"
@ -92,14 +93,6 @@ fi
present="$(cat "$__object/explorer/present")" present="$(cat "$__object/explorer/present")"
status="$(cat "$__object/explorer/status")" status="$(cat "$__object/explorer/status")"
# Defining a jail as absent and started at the same time
# makes no sense. Treat this as an error.
if [ "$started" = "true" -a "$state" = "absent" ]; then
exec >&2
echo "Can't have --state absent and --started true together\!"
exit 1
fi
stopJail() { stopJail() {
# Check $status before issuing command # Check $status before issuing command
if [ "$status" = "STARTED" ]; then if [ "$status" = "STARTED" ]; then

View file

@ -46,10 +46,10 @@ fi
case "$state_should" in case "$state_should" in
present) present)
echo $pip install -q pyro echo $pip install -q "$name"
;; ;;
absent) absent)
echo $pip uninstall -q -y pyro echo $pip uninstall -q -y "$name"
;; ;;
*) *)
echo "Unknown state: $state_should" >&2 echo "Unknown state: $state_should" >&2

View file

@ -43,26 +43,25 @@ class ConfigInstall(object):
self.context = context self.context = context
self.log = logging.getLogger(self.context.target_host) self.log = logging.getLogger(self.context.target_host)
# For easy access
self.local = context.local
self.remote = context.remote
# Initialise local directory structure # Initialise local directory structure
self.local.create_files_dirs() self.context.local.create_files_dirs()
# Initialise remote directory structure # Initialise remote directory structure
self.remote.create_files_dirs() self.context.remote.create_files_dirs()
self.explorer = core.Explorer(self.context.target_host, self.local, self.remote) self.explorer = core.Explorer(self.context.target_host, self.context.local, self.context.remote)
self.manifest = core.Manifest(self.context.target_host, self.local) self.manifest = core.Manifest(self.context.target_host, self.context.local)
self.code = core.Code(self.context.target_host, self.local, self.remote) self.code = core.Code(self.context.target_host, self.context.local, self.context.remote)
# Add switch to disable code execution
self.dry_run = False
def cleanup(self): def cleanup(self):
# FIXME: move to local? # FIXME: move to local?
destination = os.path.join(self.local.cache_path, self.context.target_host) destination = os.path.join(self.context.local.cache_path, self.context.target_host)
self.log.debug("Saving " + self.local.out_path + " to " + destination) self.log.debug("Saving " + self.context.local.out_path + " to " + destination)
if os.path.exists(destination): if os.path.exists(destination):
shutil.rmtree(destination) shutil.rmtree(destination)
shutil.move(self.local.out_path, destination) shutil.move(self.context.local.out_path, destination)
def deploy_to(self): def deploy_to(self):
"""Mimic the old deploy to: Deploy to one host""" """Mimic the old deploy to: Deploy to one host"""
@ -79,7 +78,7 @@ class ConfigInstall(object):
def stage_prepare(self): def stage_prepare(self):
"""Do everything for a deploy, minus the actual code stage""" """Do everything for a deploy, minus the actual code stage"""
self.explorer.run_global_explorers(self.local.global_explorer_out_path) self.explorer.run_global_explorers(self.context.local.global_explorer_out_path)
self.manifest.run_initial_manifest(self.context.initial_manifest) self.manifest.run_initial_manifest(self.context.initial_manifest)
self.log.info("Running object manifests and type explorers") self.log.info("Running object manifests and type explorers")
@ -88,8 +87,8 @@ class ConfigInstall(object):
new_objects_created = True new_objects_created = True
while new_objects_created: while new_objects_created:
new_objects_created = False new_objects_created = False
for cdist_object in core.CdistObject.list_objects(self.local.object_path, for cdist_object in core.CdistObject.list_objects(self.context.local.object_path,
self.local.type_path): self.context.local.type_path):
if cdist_object.state == core.CdistObject.STATE_PREPARED: if cdist_object.state == core.CdistObject.STATE_PREPARED:
self.log.debug("Skipping re-prepare of object %s", cdist_object) self.log.debug("Skipping re-prepare of object %s", cdist_object)
continue continue
@ -104,11 +103,10 @@ class ConfigInstall(object):
self.manifest.run_type_manifest(cdist_object) self.manifest.run_type_manifest(cdist_object)
cdist_object.state = core.CdistObject.STATE_PREPARED cdist_object.state = core.CdistObject.STATE_PREPARED
def object_run(self, cdist_object): def object_run(self, cdist_object, dry_run=False):
"""Run gencode and code for an object""" """Run gencode and code for an object"""
self.log.debug("Trying to run object " + cdist_object.name) self.log.debug("Trying to run object " + cdist_object.name)
if cdist_object.state == core.CdistObject.STATE_DONE: if cdist_object.state == core.CdistObject.STATE_DONE:
# TODO: remove once we are sure that this really never happens.
raise cdist.Error("Attempting to run an already finished object: %s", cdist_object) raise cdist.Error("Attempting to run an already finished object: %s", cdist_object)
cdist_type = cdist_object.cdist_type cdist_type = cdist_object.cdist_type
@ -121,11 +119,12 @@ class ConfigInstall(object):
cdist_object.changed = True cdist_object.changed = True
# Execute # Execute
if cdist_object.code_local: if not dry_run:
self.code.run_code_local(cdist_object) if cdist_object.code_local:
if cdist_object.code_remote: self.code.run_code_local(cdist_object)
self.code.transfer_code_remote(cdist_object) if cdist_object.code_remote:
self.code.run_code_remote(cdist_object) self.code.transfer_code_remote(cdist_object)
self.code.run_code_remote(cdist_object)
# Mark this object as done # Mark this object as done
self.log.debug("Finishing run of " + cdist_object.name) self.log.debug("Finishing run of " + cdist_object.name)
@ -135,13 +134,49 @@ class ConfigInstall(object):
"""The final (and real) step of deployment""" """The final (and real) step of deployment"""
self.log.info("Generating and executing code") self.log.info("Generating and executing code")
objects = core.CdistObject.list_objects( # FIXME: think about parallel execution (same for stage_prepare)
self.local.object_path, self.all_resolved = False
self.local.type_path) while not self.all_resolved:
self.stage_run_iterate()
dependency_resolver = resolver.DependencyResolver(objects) def stage_run_iterate(self):
self.log.debug(pprint.pformat(dependency_resolver.dependencies)) """
Run one iteration of the run
for cdist_object in dependency_resolver: To be repeated until all objects are done
self.log.debug("Run object: %s", cdist_object) """
self.object_run(cdist_object) objects = list(core.CdistObject.list_objects(self.context.local.object_path, self.context.local.type_path))
object_state_list=' '.join('%s:%s:%s:%s' % (o, o.state, o.all_requirements, o.satisfied_requirements) for o in objects)
self.log.debug("Object state (name:state:requirements:satisfied): %s" % object_state_list)
objects_changed = False
self.all_resolved = True
for cdist_object in objects:
if not cdist_object.state == cdist_object.STATE_DONE:
self.all_resolved = False
self.log.debug("Object %s not done" % cdist_object.name)
if cdist_object.satisfied_requirements:
self.log.debug("Running object %s with satisfied requirements" % cdist_object.name)
self.object_run(cdist_object, self.dry_run)
objects_changed = True
self.log.debug("All resolved: %s Objects changed: %s" % (self.all_resolved, objects_changed))
# Not all are resolved, but nothing has been changed => bad dependencies!
if not objects_changed and not self.all_resolved:
# Create list of unfinished objects + their requirements for print
evil_objects = []
good_objects = []
for cdist_object in objects:
if not cdist_object.state == cdist_object.STATE_DONE:
evil_objects.append("%s: required: %s, autorequired: %s" %
(cdist_object.name, cdist_object.requirements, cdist_object.autorequire))
else:
evil_objects.append("%s (%s): required: %s, autorequired: %s" %
(cdist_object.state, cdist_object.name,
cdist_object.requirements, cdist_object.autorequire))
errormessage = "Cannot solve requirements for the following objects: %s - solved: %s" % (",".join(evil_objects), ",".join(good_objects))
raise cdist.Error(errormessage)

View file

@ -20,6 +20,7 @@
# #
# #
import fnmatch
import logging import logging
import os import os
import collections import collections
@ -56,6 +57,21 @@ class CdistObject(object):
STATE_RUNNING = "running" STATE_RUNNING = "running"
STATE_DONE = "done" STATE_DONE = "done"
def __init__(self, cdist_type, base_path, object_id=None):
self.cdist_type = cdist_type # instance of Type
self.base_path = base_path
self.object_id = object_id
self.validate_object_id()
self.sanitise_object_id()
self.name = self.join_name(self.cdist_type.name, self.object_id)
self.path = os.path.join(self.cdist_type.path, self.object_id, OBJECT_MARKER)
self.absolute_path = os.path.join(self.base_path, self.path)
self.code_local_path = os.path.join(self.path, "code-local")
self.code_remote_path = os.path.join(self.path, "code-remote")
self.parameter_path = os.path.join(self.path, "parameter")
@classmethod @classmethod
def list_objects(cls, object_base_path, type_base_path): def list_objects(cls, object_base_path, type_base_path):
"""Return a list of object instances""" """Return a list of object instances"""
@ -112,21 +128,6 @@ class CdistObject(object):
raise IllegalObjectIdError(self.object_id, raise IllegalObjectIdError(self.object_id,
"Missing object_id and type is not a singleton.") "Missing object_id and type is not a singleton.")
def __init__(self, cdist_type, base_path, object_id=None):
self.cdist_type = cdist_type # instance of Type
self.base_path = base_path
self.object_id = object_id
self.validate_object_id()
self.sanitise_object_id()
self.name = self.join_name(self.cdist_type.name, self.object_id)
self.path = os.path.join(self.cdist_type.path, self.object_id, OBJECT_MARKER)
self.absolute_path = os.path.join(self.base_path, self.path)
self.code_local_path = os.path.join(self.path, "code-local")
self.code_remote_path = os.path.join(self.path, "code-remote")
self.parameter_path = os.path.join(self.path, "parameter")
def object_from_name(self, object_name): def object_from_name(self, object_name):
"""Convenience method for creating an object instance from an object name. """Convenience method for creating an object instance from an object name.
@ -209,3 +210,67 @@ class CdistObject(object):
os.makedirs(absolute_parameter_path, exist_ok=False) os.makedirs(absolute_parameter_path, exist_ok=False)
except EnvironmentError as error: except EnvironmentError as error:
raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error)) raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error))
@property
def satisfied_requirements(self):
"""Return state whether all of our dependencies have been resolved already"""
satisfied = True
for requirement in self.all_requirements:
log.debug("%s: Checking requirement %s (%s) .." % (self.name, requirement.name, requirement.state))
if not requirement.state == self.STATE_DONE:
satisfied = False
break
log.debug("%s is satisfied: %s" % (self.name, satisfied))
return satisfied
def find_requirements_by_name(self, requirements):
"""Takes a list of requirement patterns and returns a list of matching object instances.
Patterns are expected to be Unix shell-style wildcards for use with fnmatch.filter.
find_requirements_by_name(['__type/object_id', '__other_type/*']) ->
[<Object __type/object_id>, <Object __other_type/any>, <Object __other_type/match>]
"""
# FIXME: think about where/when to store this - probably not here
self.objects = dict((o.name, o) for o in self.list_objects(self.base_path, self.cdist_type.base_path))
object_names = self.objects.keys()
for pattern in requirements:
found = False
for requirement in fnmatch.filter(object_names, pattern):
found = True
yield self.objects[requirement]
if not found:
# FIXME: get rid of the singleton object_id, it should be invisible to the code -> hide it in Object
singleton = os.path.join(pattern, 'singleton')
if singleton in self.objects:
yield self.objects[singleton]
else:
raise RequirementNotFoundError(pattern)
@property
def all_requirements(self):
"""
Return resolved autorequirements and requirements so that
a complete list of requirements is returned
"""
all_reqs= []
all_reqs.extend(self.find_requirements_by_name(self.requirements))
all_reqs.extend(self.find_requirements_by_name(self.autorequire))
return set(all_reqs)
class RequirementNotFoundError(cdist.Error):
def __init__(self, requirement):
self.requirement = requirement
def __str__(self):
return 'Requirement could not be found: %s' % self.requirement

View file

@ -42,6 +42,26 @@ class CdistType(object):
""" """
def __init__(self, base_path, name):
self.base_path = base_path
self.name = name
self.path = self.name
self.absolute_path = os.path.join(self.base_path, self.path)
if not os.path.isdir(self.absolute_path):
raise NoSuchTypeError(self.path, self.absolute_path)
self.manifest_path = os.path.join(self.name, "manifest")
self.explorer_path = os.path.join(self.name, "explorer")
self.gencode_local_path = os.path.join(self.name, "gencode-local")
self.gencode_remote_path = os.path.join(self.name, "gencode-remote")
self.manifest_path = os.path.join(self.name, "manifest")
self.__explorers = None
self.__required_parameters = None
self.__required_multiple_parameters = None
self.__optional_parameters = None
self.__optional_multiple_parameters = None
self.__boolean_parameters = None
@classmethod @classmethod
def list_types(cls, base_path): def list_types(cls, base_path):
"""Return a list of type instances""" """Return a list of type instances"""
@ -65,26 +85,6 @@ class CdistType(object):
# return instance so __init__ is called # return instance so __init__ is called
return cls._instances[name] return cls._instances[name]
def __init__(self, base_path, name):
self.base_path = base_path
self.name = name
self.path = self.name
self.absolute_path = os.path.join(self.base_path, self.path)
if not os.path.isdir(self.absolute_path):
raise NoSuchTypeError(self.path, self.absolute_path)
self.manifest_path = os.path.join(self.name, "manifest")
self.explorer_path = os.path.join(self.name, "explorer")
self.gencode_local_path = os.path.join(self.name, "gencode-local")
self.gencode_remote_path = os.path.join(self.name, "gencode-remote")
self.manifest_path = os.path.join(self.name, "manifest")
self.__explorers = None
self.__required_parameters = None
self.__required_multiple_parameters = None
self.__optional_parameters = None
self.__optional_multiple_parameters = None
self.__boolean_parameters = None
def __repr__(self): def __repr__(self):
return '<CdistType %s>' % self.name return '<CdistType %s>' % self.name

View file

@ -65,10 +65,10 @@ class AutorequireTestCase(test.CdistTestCase):
shutil.rmtree(self.temp_dir) shutil.rmtree(self.temp_dir)
def test_implicit_dependencies(self): def test_implicit_dependencies(self):
self.context.initial_manifest = os.path.join(self.config.local.manifest_path, 'implicit_dependencies') self.context.initial_manifest = os.path.join(self.context.local.manifest_path, 'implicit_dependencies')
self.config.stage_prepare() self.config.stage_prepare()
objects = core.CdistObject.list_objects(self.config.local.object_path, self.config.local.type_path) objects = core.CdistObject.list_objects(self.context.local.object_path, self.context.local.type_path)
dependency_resolver = resolver.DependencyResolver(objects) dependency_resolver = resolver.DependencyResolver(objects)
expected_dependencies = [ expected_dependencies = [
dependency_resolver.objects['__package_special/b'], dependency_resolver.objects['__package_special/b'],
@ -79,7 +79,7 @@ class AutorequireTestCase(test.CdistTestCase):
self.assertEqual(resolved_dependencies, expected_dependencies) self.assertEqual(resolved_dependencies, expected_dependencies)
def test_circular_dependency(self): def test_circular_dependency(self):
self.context.initial_manifest = os.path.join(self.config.local.manifest_path, 'circular_dependency') self.context.initial_manifest = os.path.join(self.context.local.manifest_path, 'circular_dependency')
self.config.stage_prepare() self.config.stage_prepare()
# raises CircularDependecyError # raises CircularDependecyError
self.config.stage_run() self.config.stage_run()

View file

@ -0,0 +1,119 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import shutil
from cdist import test
from cdist import core
import cdist
import cdist.context
import cdist.config
import os.path as op
my_dir = op.abspath(op.dirname(__file__))
fixtures = op.join(my_dir, 'fixtures')
object_base_path = op.join(fixtures, 'object')
type_base_path = op.join(fixtures, 'type')
add_conf_dir = op.join(fixtures, 'conf')
class ConfigInstallRunTestCase(test.CdistTestCase):
def setUp(self):
# Change env for context
self.orig_environ = os.environ
os.environ = os.environ.copy()
self.temp_dir = self.mkdtemp()
self.out_dir = os.path.join(self.temp_dir, "out")
self.remote_out_dir = os.path.join(self.temp_dir, "remote")
os.environ['__cdist_out_dir'] = self.out_dir
os.environ['__cdist_remote_out_dir'] = self.remote_out_dir
self.context = cdist.context.Context(
target_host=self.target_host,
remote_copy=self.remote_copy,
remote_exec=self.remote_exec,
exec_path=test.cdist_exec_path,
debug=True)
self.context.local.object_path = object_base_path
self.context.local.type_path = type_base_path
self.config = cdist.config.Config(self.context)
self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
self.object_index = dict((o.name, o) for o in self.objects)
self.object_names = [o.name for o in self.objects]
def tearDown(self):
for o in self.objects:
o.requirements = []
o.state = ""
os.environ = self.orig_environ
shutil.rmtree(self.temp_dir)
def test_dependency_resolution(self):
first = self.object_index['__first/man']
second = self.object_index['__second/on-the']
third = self.object_index['__third/moon']
first.requirements = [second.name]
second.requirements = [third.name]
# First run:
# solves first and maybe second (depending on the order in the set)
self.config.stage_run_iterate()
self.assertTrue(third.state == third.STATE_DONE)
self.config.stage_run_iterate()
self.assertTrue(second.state == second.STATE_DONE)
try:
self.config.stage_run_iterate()
except cdist.Error:
# Allow failing, because the third run may or may not be unecessary already,
# depending on the order of the objects
pass
self.assertTrue(first.state == first.STATE_DONE)
def test_unresolvable_requirements(self):
"""Ensure an exception is thrown for unresolvable depedencies"""
# Create to objects depending on each other - no solution possible
first = self.object_index['__first/man']
second = self.object_index['__second/on-the']
first.requirements = [second.name]
second.requirements = [first.name]
# First round solves __third/moon
self.config.stage_run_iterate()
# Second round detects it cannot solve the rest
with self.assertRaises(cdist.Error):
self.config.stage_run_iterate()

View file

@ -0,0 +1 @@
Prometheus

View file

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2012 Nico Schottelius (nico-cdist at schottelius.org)
# #
# This file is part of cdist. # This file is part of cdist.
# #
@ -35,22 +36,33 @@ type_base_path = op.join(fixtures, 'type')
class ObjectClassTestCase(test.CdistTestCase): class ObjectClassTestCase(test.CdistTestCase):
def setUp(self):
self.expected_object_names = sorted([
'__first/child',
'__first/dog',
'__first/man',
'__first/woman',
'__second/on-the',
'__second/under-the',
'__third/moon'])
self.expected_objects = []
for cdist_object_name in self.expected_object_names:
cdist_type, cdist_object_id = cdist_object_name.split("/", maxsplit=1)
cdist_object = core.CdistObject(core.CdistType(type_base_path, cdist_type), object_base_path, cdist_object_id)
self.expected_objects.append(cdist_object)
def test_list_object_names(self): def test_list_object_names(self):
object_names = list(core.CdistObject.list_object_names(object_base_path)) found_object_names = sorted(list(core.CdistObject.list_object_names(object_base_path)))
self.assertEqual(object_names, ['__first/man', '__second/on-the', '__third/moon']) self.assertEqual(found_object_names, self.expected_object_names)
def test_list_type_names(self): def test_list_type_names(self):
type_names = list(cdist.core.CdistObject.list_type_names(object_base_path)) type_names = list(cdist.core.CdistObject.list_type_names(object_base_path))
self.assertEqual(type_names, ['__first', '__second', '__third']) self.assertEqual(type_names, ['__first', '__second', '__third'])
def test_list_objects(self): def test_list_objects(self):
objects = list(core.CdistObject.list_objects(object_base_path, type_base_path)) found_objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
objects_expected = [ self.assertEqual(found_objects, self.expected_objects)
core.CdistObject(core.CdistType(type_base_path, '__first'), object_base_path, 'man'),
core.CdistObject(core.CdistType(type_base_path, '__second'), object_base_path, 'on-the'),
core.CdistObject(core.CdistType(type_base_path, '__third'), object_base_path, 'moon'),
]
self.assertEqual(objects, objects_expected)
class ObjectIdTestCase(test.CdistTestCase): class ObjectIdTestCase(test.CdistTestCase):
@ -200,3 +212,54 @@ class ObjectTestCase(test.CdistTestCase):
self.assertTrue(isinstance(other_object, core.CdistObject)) self.assertTrue(isinstance(other_object, core.CdistObject))
self.assertEqual(other_object.cdist_type.name, '__first') self.assertEqual(other_object.cdist_type.name, '__first')
self.assertEqual(other_object.object_id, 'man') self.assertEqual(other_object.object_id, 'man')
class ObjectResolveRequirementsTestCase(test.CdistTestCase):
def setUp(self):
self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
self.object_index = dict((o.name, o) for o in self.objects)
self.object_names = [o.name for o in self.objects]
print(self.objects)
self.cdist_type = core.CdistType(type_base_path, '__third')
self.cdist_object = core.CdistObject(self.cdist_type, object_base_path, 'moon')
def tearDown(self):
for o in self.objects:
o.requirements = []
def test_find_requirements_by_name_string(self):
"""Check that resolving requirements by name works (require all objects)"""
requirements = self.object_names
self.cdist_object.requirements = requirements
found_requirements = sorted(self.cdist_object.find_requirements_by_name(self.cdist_object.requirements))
expected_requirements = sorted(self.objects)
self.assertEqual(found_requirements, expected_requirements)
def test_find_requirements_by_name_pattern(self):
"""Test whether pattern matching on requirements works"""
# Matches all objects in the end
requirements = ['__first/*', '__second/*-the', '__third/moon']
self.cdist_object.requirements = requirements
expected_requirements = sorted(self.objects)
found_requirements = sorted(self.cdist_object.find_requirements_by_name(self.cdist_object.requirements))
self.assertEqual(expected_requirements, found_requirements)
def test_requirement_not_found(self):
"""Ensure an exception is thrown for missing depedencies"""
cdist_object = self.object_index['__first/man']
cdist_object.requirements = ['__does/not/exist']
with self.assertRaises(core.cdist_object.RequirementNotFoundError):
# Use list, as generator does not (yet) raise the error
list(cdist_object.find_requirements_by_name(cdist_object.requirements))

View file

@ -1,88 +0,0 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import shutil
import cdist
from cdist import test
from cdist import core
from cdist import resolver
import os.path as op
my_dir = op.abspath(op.dirname(__file__))
fixtures = op.join(my_dir, 'fixtures')
object_base_path = op.join(fixtures, 'object')
type_base_path = op.join(fixtures, 'type')
class ResolverTestCase(test.CdistTestCase):
def setUp(self):
self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
self.object_index = dict((o.name, o) for o in self.objects)
self.dependency_resolver = resolver.DependencyResolver(self.objects)
def tearDown(self):
for o in self.objects:
o.requirements = []
def test_find_requirements_by_name_string(self):
requirements = ['__first/man', '__second/on-the', '__third/moon']
required_objects = [self.object_index[name] for name in requirements]
self.assertEqual(sorted(list(self.dependency_resolver.find_requirements_by_name(requirements))),
sorted(required_objects))
def test_find_requirements_by_name_pattern(self):
requirements = ['__first/*', '__second/*-the', '__third/moon']
requirements_expanded = [
'__first/child', '__first/dog', '__first/man', '__first/woman',
'__second/on-the', '__second/under-the',
'__third/moon'
]
required_objects = [self.object_index[name] for name in requirements_expanded]
self.assertEqual(sorted(list(self.dependency_resolver.find_requirements_by_name(requirements))),
sorted(required_objects))
def test_dependency_resolution(self):
first_man = self.object_index['__first/man']
second_on_the = self.object_index['__second/on-the']
third_moon = self.object_index['__third/moon']
first_man.requirements = [second_on_the.name]
second_on_the.requirements = [third_moon.name]
self.assertEqual(
self.dependency_resolver.dependencies['__first/man'],
[third_moon, second_on_the, first_man]
)
def test_circular_reference(self):
first_man = self.object_index['__first/man']
first_woman = self.object_index['__first/woman']
first_man.requirements = [first_woman.name]
first_woman.requirements = [first_man.name]
with self.assertRaises(resolver.CircularReferenceError):
self.dependency_resolver.dependencies
def test_requirement_not_found(self):
first_man = self.object_index['__first/man']
first_man.requirements = ['__does/not/exist']
with self.assertRaises(cdist.Error):
self.dependency_resolver.dependencies

View file

@ -4,7 +4,11 @@ Changelog
* Changes are always commented with their author in (braces) * Changes are always commented with their author in (braces)
* Exception: No braces means author == Nico Schottelius * Exception: No braces means author == Nico Schottelius
2.1.0: next:
* Type __jail: State absent should implies stopped (Jake Guffey)
* Core: Use dynamic dependency resolver to allow indirect self dependencies
2.1.0: 2012-12-09
* Core: Ensure global explorers are executable * Core: Ensure global explorers are executable
* Core: Ensure type explorers are executable (Steven Armstrong) * Core: Ensure type explorers are executable (Steven Armstrong)
* New Type: __git * New Type: __git
@ -16,6 +20,7 @@ Changelog
* Type __jail: Change optional parameter "started" to boolean "stopped" parameter, * Type __jail: Change optional parameter "started" to boolean "stopped" parameter,
change optional parameter "devfs-enable" to boolean "devfs-disable" parameter and change optional parameter "devfs-enable" to boolean "devfs-disable" parameter and
change optional parameter "onboot" to boolean. change optional parameter "onboot" to boolean.
* Type __package_pip: Bugfix: Installeded the package, not pyro
* Remove Type __ssh_authorized_key: Superseeded by __ssh_authorized_keys * Remove Type __ssh_authorized_key: Superseeded by __ssh_authorized_keys
* Support for CDIST_PATH (Steven Armstrong) * Support for CDIST_PATH (Steven Armstrong)

View file

@ -0,0 +1,72 @@
2.1.0 behaviour:
__git foo
__package git --state present
__git bar
__package git --state present
require="__git/foo" git bar:
__git bar
__git foo
__package git --state present
__package git --state present
__git foo
__package git --state present
-> detects circular dependency
--------------------------------------------------------------------------------
__package abc
__package_apt abc
__sometype def
__package abc
__package_apt abc
--------------------------------------------------------------------------------
Change proposal:
Each object only depends on the objects it directly requires, tree build to
ensure correct running behaviour:
__git bar
__git foo
__package git --state present
__git foo
__package git --state present
Order:
1)
__package/git (leaf node!)
2)
__git/foo (new leaf node!)
3)
__git/bar (new leaf node!)
For __package:
__sometype def
__package abc
__package abc
__package_apt abc
1) __package_apt/abc (leaf node)
2) __package/abc (new leaf node)
3) __sometype/def (new leaf node)

View file

@ -59,62 +59,75 @@ cat << eof
PATHS PATHS
----- -----
If not specified otherwise, all paths are relative to the checkout directory. \$HOME/.cdist::
The standard cdist configuration directory relative to your home directory
This is usually the place you want to store your site specific configuration
conf/:: cdist/conf/::
Contains the (static) configuration like manifests, types and explorers. The distribution configuration directory
This contains types and explorers to be used
conf/manifest/init:: confdir::
Cdist will use all available configuration directories and create
a temporary confdir containing links to the real configuration directories.
This way it is possible to merge configuration directories.
By default it consists of everything in \$HOME/.cdist and cdist/conf/.
For more details see cdist(1)
confdir/manifest/init::
This is the central entry point. This is the central entry point.
It is an executable (+x bit set) shell script that can use It is an executable (+x bit set) shell script that can use
values from the explorers to decide which configuration to create values from the explorers to decide which configuration to create
for the specified target host. for the specified target host.
Its intent is to used to define mapping from configurations to hosts. Its intent is to used to define mapping from configurations to hosts.
conf/manifest/*:: confdir/manifest/*::
All other files in this directory are not directly used by cdist, but you All other files in this directory are not directly used by cdist, but you
can seperate configuration mappings, if you have a lot of code in the can seperate configuration mappings, if you have a lot of code in the
conf/manifest/init file. This may also be helpful to have different admins conf/manifest/init file. This may also be helpful to have different admins
maintain different groups of hosts. maintain different groups of hosts.
conf/explorer/<name>:: confdir/explorer/<name>::
Contains explorers to be run on the target hosts, see cdist-explorer(7). Contains explorers to be run on the target hosts, see cdist-explorer(7).
conf/type/:: confdir/type/::
Contains all available types, which are used to provide Contains all available types, which are used to provide
some kind of functionality. See cdist-type(7). some kind of functionality. See cdist-type(7).
conf/type/<name>/:: confdir/type/<name>/::
Home of the type <name>. Home of the type <name>.
This directory is referenced by the variable __type (see below). This directory is referenced by the variable __type (see below).
conf/type/<name>/man.text:: confdir/type/<name>/man.text::
Manpage in Asciidoc format (required for inclusion into upstream) Manpage in Asciidoc format (required for inclusion into upstream)
conf/type/<name>/manifest:: confdir/type/<name>/manifest::
Used to generate additional objects from a type. Used to generate additional objects from a type.
conf/type/<name>/gencode-local:: confdir/type/<name>/gencode-local::
Used to generate code to be executed on the source host Used to generate code to be executed on the source host
conf/type/<name>/gencode-remote:: confdir/type/<name>/gencode-remote::
Used to generate code to be executed on the target host Used to generate code to be executed on the target host
conf/type/<name>/parameter/required:: confdir/type/<name>/parameter/required::
Parameters required by type, \n seperated list. Parameters required by type, \n seperated list.
conf/type/<name>/parameter/optional:: confdir/type/<name>/parameter/optional::
Parameters optionally accepted by type, \n seperated list. Parameters optionally accepted by type, \n seperated list.
conf/type/<name>/parameter/boolean:: confdir/type/<name>/parameter/boolean::
Boolean parameters accepted by type, \n seperated list. Boolean parameters accepted by type, \n seperated list.
conf/type/<name>/explorer:: confdir/type/<name>/explorer::
Location of the type specific explorers. Location of the type specific explorers.
This directory is referenced by the variable __type_explorer (see below). This directory is referenced by the variable __type_explorer (see below).
See cdist-explorer(7). See cdist-explorer(7).
confdir/type/<name>/files::
This directory is reserved for user data and will not be used
by cdist at any time
out/:: out/::
This directory contains output of cdist and is usually located This directory contains output of cdist and is usually located
in a temporary directory and thus will be removed after the run. in a temporary directory and thus will be removed after the run.
@ -179,10 +192,8 @@ __object::
__object_id:: __object_id::
The type unique object id. The type unique object id.
Available for: type manifest, type explorer, type gencode Available for: type manifest, type explorer, type gencode
Note: The leading and the trailing "/" will always be stripped (caused by Note: The leading and the trailing "/" will always be stripped (caused by
the filesystem database and ensured by the core). the filesystem database and ensured by the core).
Note: Double slashes ("//") will not be fixed and result in an error. Note: Double slashes ("//") will not be fixed and result in an error.
__object_name:: __object_name::
The full qualified name of the current object. The full qualified name of the current object.

View file

@ -78,7 +78,7 @@ EXAMPLES
# Configure ikq05.ethz.ch with debug enabled # Configure ikq05.ethz.ch with debug enabled
cdist config -d ikq05.ethz.ch cdist config -d ikq05.ethz.ch
# Configure hosts in parallel and use a different home directory # Configure hosts in parallel and use a different configuration directory
cdist config -c ~/p/cdist-nutzung \ cdist config -c ~/p/cdist-nutzung \
-p ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch -p ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch

View file

@ -44,7 +44,7 @@ work nor kill the authors brain:
- All files should contain the usual header (Author, Copying, etc.) - All files should contain the usual header (Author, Copying, etc.)
- Code submission must be done via git - Code submission must be done via git
- Do not add conf/manifest/init - This file should only be touched in your - Do not add cdist/conf/manifest/init - This file should only be touched in your
private branch! private branch!
- Code to be included should be branched of the upstream "master" branch - Code to be included should be branched of the upstream "master" branch
- Exception: Bugfixes to a version branch - Exception: Bugfixes to a version branch

View file

@ -16,8 +16,8 @@ An object is represented by the combination of
**type + slash + object name**: **__file/etc/cdist-configured** is an **type + slash + object name**: **__file/etc/cdist-configured** is an
object of the type ***__file*** with the name ***etc/cdist-configured***. object of the type ***__file*** with the name ***etc/cdist-configured***.
All available types can be found in the **conf/type/** directory, All available types can be found in the **cdist/conf/type/** directory,
use **ls conf/type** to get the list of available types. If you have use **ls cdist/conf/type** to get the list of available types. If you have
setup the MANPATH correctly, you can use **man cdist-reference** to access setup the MANPATH correctly, you can use **man cdist-reference** to access
the reference with pointers to the manpages. the reference with pointers to the manpages.
@ -57,7 +57,7 @@ DEFINE STATE IN THE INITIAL MANIFEST
------------------------------------ ------------------------------------
The **initial manifest** is the entry point for cdist to find out, which The **initial manifest** is the entry point for cdist to find out, which
**objects** to configure on the selected host. **objects** to configure on the selected host.
Cdist searches for the initial manifest at **conf/manifest/init**. Cdist searches for the initial manifest at **cdist/conf/manifest/init**.
Within this initial manifest, you define, which objects should be Within this initial manifest, you define, which objects should be
created on which host. To distinguish between hosts, you can use the created on which host. To distinguish between hosts, you can use the
@ -88,7 +88,7 @@ command.
SPLITTING UP THE INITIAL MANIFEST SPLITTING UP THE INITIAL MANIFEST
--------------------------------- ---------------------------------
If you want to split up your initial manifest, you can create other shell If you want to split up your initial manifest, you can create other shell
scripts in **conf/manifest/** and include them in **conf/manifest/init**. scripts in **cdist/conf/manifest/** and include them in **cdist/conf/manifest/init**.
Cdist provides the environment variable ***__manifest*** to reference to Cdist provides the environment variable ***__manifest*** to reference to
the directory containing the initial manifest (see cdist-reference(7)). the directory containing the initial manifest (see cdist-reference(7)).

View file

@ -64,10 +64,10 @@ A type consists of
- explorer (optional) - explorer (optional)
- gencode (optional) - gencode (optional)
Types are stored below conf/type/. Their name should always be prefixed with Types are stored below cdist/conf/type/. Their name should always be prefixed with
two underscores (__) to prevent collisions with other executables in $PATH. two underscores (__) to prevent collisions with other executables in $PATH.
To begin a new type, just create the directory **conf/type/__NAME**. To begin a new type, just create the directory **cdist/conf/type/__NAME**.
DEFINING PARAMETERS DEFINING PARAMETERS
@ -84,10 +84,10 @@ or no parameters at all.
Example: Example:
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
echo servername >> conf/type/__nginx_vhost/parameter/required echo servername >> cdist/conf/type/__nginx_vhost/parameter/required
echo logdirectory >> conf/type/__nginx_vhost/parameter/optional echo logdirectory >> cdist/conf/type/__nginx_vhost/parameter/optional
echo server_alias >> conf/type/__nginx_vhost/parameter/optional_multiple echo server_alias >> cdist/conf/type/__nginx_vhost/parameter/optional_multiple
echo use_ssl >> conf/type/__nginx_vhost/parameter/boolean echo use_ssl >> cdist/conf/type/__nginx_vhost/parameter/boolean
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
@ -98,7 +98,7 @@ The parameters given to a type can be accessed and used in all type scripts
represented by file existence. File exists -> True, represented by file existence. File exists -> True,
file does not exist -> False file does not exist -> False
Example: (e.g. in conf/type/__nginx_vhost/manifest) Example: (e.g. in cdist/conf/type/__nginx_vhost/manifest)
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
# required parameter # required parameter
servername="$(cat "$__object/parameter/servername")" servername="$(cat "$__object/parameter/servername")"
@ -129,7 +129,7 @@ INPUT FROM STDIN
Every type can access what has been written on stdin when it has been called. Every type can access what has been written on stdin when it has been called.
The result is saved into the ***stdin*** file in the object directory. The result is saved into the ***stdin*** file in the object directory.
Example use of a type: (e.g. in conf/type/__archlinux_hostname) Example use of a type: (e.g. in cdist/conf/type/__archlinux_hostname)
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
__file /etc/rc.conf --source - << eof __file /etc/rc.conf --source - << eof
... ...
@ -186,7 +186,7 @@ mark it as a singleton: Just create the (empty) file "singleton" in your type
directory: directory:
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
touch conf/type/__NAME/singleton touch cdist/conf/type/__NAME/singleton
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
This will also change the way your type must be called: This will also change the way your type must be called:

View file

@ -24,6 +24,8 @@ To upgrade to the lastet version do
### Updating from 2.0 to 2.1 ### Updating from 2.0 to 2.1
Have a look at the update guide for [[2.0 to 2.1|2.0-to-2.1]].
* Type **\_\_package* and \_\_process** use --state **present** or **absent**. * Type **\_\_package* and \_\_process** use --state **present** or **absent**.
The states **removed/installed** and **stopped/running** have been removed. The states **removed/installed** and **stopped/running** have been removed.
Support for the new states is already present in 2.0. Support for the new states is already present in 2.0.

View file

@ -0,0 +1,118 @@
[[!meta title="Update Guide for 2.0 to 2.1"]]
## Introduction
When changing your installation from 2.0 to 2.1, there are
a lot of changes coming up. 2.1 is mainly a cleanup release,
which removes long time deprecated behaviour, but also makes
a lot of things more consistent and allows you to split off your types,
explorers and manifest to custom directories.
This document will guide you to a successful update.
## Preperation
As for every software and system you use in production, you should first of
all make a backup of your data. To prevent any breakage, it is
recommended to create a new git branch to do the update on:
% git checkout -b update_to_2.1
This also ensure that whenever you need to do a change in your
2.0 based tree, you can simply go back to that branch, apply the change
and configure your systems - independently of your update progress!
Next fetch the latest upstream changes, I assume that
origin refers to one of the upstream mirrors (change origin if you use
another remote name for upstream cdist):
% git fetch -v origin
## Merge the changes
Now try to merge upstream into the new branch.
% git merge origin/2.1
Fix any conflicts that may have been occurred due to local changes
and then **git add** and *git commit** those changes. This should seldomly
occur and if, it's mostly for people hacking on the cdist core.
## Move "conf" directory
One of the biggest changes in cdist 2.1 is that you can have multiple
**conf** directories: Indeed, the new default behaviour of cdist is to
search for conf directories
* below the python module (cdist/conf in the source tree or in the installed location)
* at ~/.cdist/ (on conf suffix there)
So you can now choose, where to store your types.
### Integrate your conf/ back into the tree
If you choose to store your types together with the upstream types,
you can just move all your stuff below **cdist/conf**:
% git mv conf/type/* cdist/conf/type
% git mv conf/manifest/* cdist/conf/manifest
% git mv conf/explorer/* cdist/conf/explorer
% git commit -m "Re-Integrate my conf directory into cdist 2.1 tree"
### Move your conf/ directory to ~/.cdist
If you want to store your site specific
configuration outside of the cdist tree, you
can move your conf/ directory to your homedirectory ($HOME) under ~/.cdist:
% mv conf ~/.cdist
% git rm -r conf
% git commit -m "Move my conf directory to ~/.cdist"
It it still recommended to use a version control system like git in it:
% cd ~/.cdist
% git init
% git add .
% git commit -m "Create new git repository containing my cdist configuration"
## Test the migration
Some of the types shipped with upstream were changed, so you may want to test
the result by running cdist on one of your staging target hosts:
% ./bin/cdist config -v staging-host
All incompatibilities are listed on the [[cdist update page|software/cdist/update]],
so you can browse through the list and update your configuration.
## Final Cleanups
When everything is tested, there are some cleanups to be done to finalise the update.
### When continuing to keep conf/ in the tree
You can then merge back your changes into the master tree and continue to work
as normal.
### When using ~/.cdist
If you decided to move your site specific code to ~/.cdist, you can now switch your
**master** branch or version branch to upstream directly. Assumnig you are in the
cdist directory, having your previous branch checked out, you can create a clean
state using the following commands:
% upstream_branch=2.1
% current_branch=$(git rev-parse --abbrev-ref HEAD)
% git checkout -b archive_my_own_tree
% git branch -D "$current_branch"
% git checkout -b "$current_branch" "origin/$upstream_branch"
Afther these commands, your previous main branch is accessible at
**archive_my_own_tree** and your branch is now tracking upstream.
## Questions? Critics? Hints?
If you think this manual helped or misses some information, do not
hesitate to contact us on any of the usual ways (irc, mailinglist,
github issue tracker, ...).

View file

@ -41,7 +41,7 @@ Cdist requires very litte on a target system. Even better,
in almost all cases all dependencies are usually fulfilled. in almost all cases all dependencies are usually fulfilled.
Cdist does not require an agent or a high level programming Cdist does not require an agent or a high level programming
languages on the target host: it will run on any host that languages on the target host: it will run on any host that
has an **ssh server running** and a posix compatible shell has a **ssh server running** and a posix compatible shell
(**/bin/sh**). (**/bin/sh**).
## Push based distribution ## Push based distribution