Merge branch 'master' into autorequire

Conflicts:
	lib/cdist/test/emulator/__init__.py

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
This commit is contained in:
Steven Armstrong 2011-11-15 10:47:16 +01:00
commit f36d09289d
17 changed files with 445 additions and 21 deletions

View file

@ -24,7 +24,7 @@
if [ -f "$__object/parameter/name" ]; then if [ -f "$__object/parameter/name" ]; then
name="$(cat "$__object/parameter/name")" name="$(cat "$__object/parameter/name")"
else else
name="/$__object_id" name="$__object_id"
fi fi
pgrep -x -f "$name" || true pgrep -x -f "$name" || true

View file

@ -50,6 +50,10 @@ __process /usr/sbin/sshd --state stopped --stop "/etc/rc.d/sshd stop"
# Ensure cups is running, which runs with -C ...: # Ensure cups is running, which runs with -C ...:
__process cups --start "/etc/rc.d/cups start" --state running \ __process cups --start "/etc/rc.d/cups start" --state running \
--name "/usr/sbin/cupsd -C /etc/cups/cupsd.conf" --name "/usr/sbin/cupsd -C /etc/cups/cupsd.conf"
# Ensure rpc.statd is running (which usually runs with -L) using a regexp
__process rpcstatd --state running --start "/etc/init.d/statd start" \
--name "rpc.statd.*"
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------

View file

@ -3,6 +3,7 @@
* Cleanup: __object_fq variable removed (never used) * Cleanup: __object_fq variable removed (never used)
* Cleanup: Environment variable __self DEPRECATED, use __object_name instead * Cleanup: Environment variable __self DEPRECATED, use __object_name instead
* Cleanup: Environment variable __self scheduled for removal in cdist 2.1 * Cleanup: Environment variable __self scheduled for removal in cdist 2.1
* New Type: __cron (Steven Armstrong)
2.0.3: 2011-10-18 2.0.3: 2011-10-18
* Improved logging, added --verbose, by more quiet by default * Improved logging, added --verbose, by more quiet by default

View file

@ -71,7 +71,7 @@ class ConfigInstall(object):
start_time = time.time() start_time = time.time()
self.deploy_to() self.deploy_to()
self.cleanup() self.cleanup()
self.log.info("Finished run in %s seconds", self.log.info("Finished successful run in %s seconds",
time.time() - start_time) time.time() - start_time)
def stage_prepare(self): def stage_prepare(self):
@ -106,7 +106,7 @@ class ConfigInstall(object):
"""Run gencode and code for an object""" """Run gencode and code for an object"""
self.log.debug("Trying to run object " + cdist_object.name) self.log.debug("Trying to run object " + cdist_object.name)
if cdist_object.state == core.Object.STATE_RUNNING: if cdist_object.state == core.Object.STATE_RUNNING:
# FIXME: resolve dependency circle # FIXME: resolve dependency circle / show problem source
raise cdist.Error("Detected circular dependency in " + cdist_object.name) raise cdist.Error("Detected circular dependency in " + cdist_object.name)
elif cdist_object.state == core.Object.STATE_DONE: elif cdist_object.state == core.Object.STATE_DONE:
self.log.debug("Ignoring run of already finished object %s", cdist_object) self.log.debug("Ignoring run of already finished object %s", cdist_object)
@ -119,6 +119,13 @@ class ConfigInstall(object):
for requirement in cdist_object.requirements: for requirement in cdist_object.requirements:
self.log.debug("Object %s requires %s", cdist_object, requirement) self.log.debug("Object %s requires %s", cdist_object, requirement)
required_object = cdist_object.object_from_name(requirement) required_object = cdist_object.object_from_name(requirement)
# The user may have created dependencies without satisfying them
if not required_object.exists:
raise cdist.Error(cdist_object.name + " requires non-existing " + required_object.name)
else:
self.log.debug("Required object %s exists", required_object.name)
self.object_run(required_object) self.object_run(required_object)
# Generate # Generate

View file

@ -23,6 +23,7 @@ from cdist.core.type import Type
from cdist.core.type import NoSuchTypeError from cdist.core.type import NoSuchTypeError
from cdist.core.object import Object from cdist.core.object import Object
from cdist.core.object import IllegalObjectIdError from cdist.core.object import IllegalObjectIdError
from cdist.core.object import OBJECT_MARKER
from cdist.core.explorer import Explorer from cdist.core.explorer import Explorer
from cdist.core.manifest import Manifest from cdist.core.manifest import Manifest
from cdist.core.code import Code from cdist.core.code import Code

View file

@ -30,7 +30,7 @@ from cdist.util import fsproperty
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
DOT_CDIST = '.cdist' OBJECT_MARKER = '.cdist'
class IllegalObjectIdError(cdist.Error): class IllegalObjectIdError(cdist.Error):
@ -72,7 +72,7 @@ class Object(object):
def list_object_names(cls, object_base_path): def list_object_names(cls, object_base_path):
"""Return a list of object names""" """Return a list of object names"""
for path, dirs, files in os.walk(object_base_path): for path, dirs, files in os.walk(object_base_path):
if DOT_CDIST in dirs: if OBJECT_MARKER in dirs:
yield os.path.relpath(path, object_base_path) yield os.path.relpath(path, object_base_path)
@staticmethod @staticmethod
@ -100,13 +100,13 @@ class Object(object):
if object_id: if object_id:
if object_id.startswith('/'): if object_id.startswith('/'):
raise IllegalObjectIdError(object_id, 'object_id may not start with /') raise IllegalObjectIdError(object_id, 'object_id may not start with /')
if '.cdist' in object_id: if OBJECT_MARKER in object_id.split(os.sep):
raise IllegalObjectIdError(object_id, 'object_id may not contain \'.cdist\'') raise IllegalObjectIdError(object_id, 'object_id may not contain \'%s\'' % OBJECT_MARKER)
self.type = cdist_type # instance of Type self.type = cdist_type # instance of Type
self.base_path = base_path self.base_path = base_path
self.object_id = object_id self.object_id = object_id
self.name = self.join_name(self.type.name, self.object_id) self.name = self.join_name(self.type.name, self.object_id)
self.path = os.path.join(self.type.path, self.object_id, DOT_CDIST) self.path = os.path.join(self.type.path, self.object_id, OBJECT_MARKER)
self.absolute_path = os.path.join(self.base_path, self.path) self.absolute_path = os.path.join(self.base_path, self.path)
self.code_local_path = os.path.join(self.path, "code-local") self.code_local_path = os.path.join(self.path, "code-local")
self.code_remote_path = os.path.join(self.path, "code-remote") self.code_remote_path = os.path.join(self.path, "code-remote")
@ -132,9 +132,9 @@ class Object(object):
""" """
type_path = self.type.base_path type_path = self.type.base_path
object_path = self.base_path base_path = self.base_path
type_name, object_id = self.split_name(object_name) type_name, object_id = self.split_name(object_name)
return self.__class__(self.type.__class__(type_path, type_name), object_path, object_id=object_id) return self.__class__(self.type.__class__(type_path, type_name), base_path, object_id=object_id)
# FIXME: still needed? # FIXME: still needed?
@property @property

View file

@ -157,19 +157,22 @@ class Emulator(object):
self.log.debug("Recording requirement: " + requirement) self.log.debug("Recording requirement: " + requirement)
requirement_parts = requirement.split(os.sep, 1) requirement_parts = requirement.split(os.sep, 1)
requirement_type_name = requirement_parts[0] requirement_type_name = requirement_parts[0]
requirement_object_id = requirement_parts[1] try:
requirement_object_id = requirement_parts[1]
# FIXME: Add support for omitted object id == singleton except IndexError:
#if len(requirement_parts) == 1: # no object id, assume singleton
#except IndexError: requirement_object_id = 'singleton'
# # no object id, must be singleton
# requirement_object_id = 'singleton' # Remove leading / from object id
# Remove / if existent in object id
requirement_object_id = requirement_object_id.lstrip('/') requirement_object_id = requirement_object_id.lstrip('/')
# Instantiate type which fails if type does not exist # Instantiate type which fails if type does not exist
requirement_type = core.Type(self.type_base_path, requirement_type_name) requirement_type = core.Type(self.type_base_path, requirement_type_name)
if requirement_object_id == 'singleton' \
and not requirement_type.is_singleton:
raise IllegalRequirementError(requirement, "Missing object_id and type is not a singleton.")
# Instantiate object which fails if the object_id is illegal # Instantiate object which fails if the object_id is illegal
requirement_object = core.Object(requirement_type, self.object_base_path, requirement_object_id) requirement_object = core.Object(requirement_type, self.object_base_path, requirement_object_id)

View file

@ -74,6 +74,21 @@ class EmulatorTestCase(test.CdistTestCase):
emu = emulator.Emulator(argv) emu = emulator.Emulator(argv)
self.assertRaises(core.IllegalObjectIdError, emu.run) self.assertRaises(core.IllegalObjectIdError, emu.run)
def test_missing_object_id_requirement(self):
argv = ['__file', '/tmp/foobar']
os.environ.update(self.env)
os.environ['require'] = '__file'
emu = emulator.Emulator(argv)
self.assertRaises(emulator.IllegalRequirementError, emu.run)
def test_singleton_object_requirement(self):
argv = ['__file', '/tmp/foobar']
os.environ.update(self.env)
os.environ['require'] = '__issue'
emu = emulator.Emulator(argv)
emu.run()
# if we get here all is fine
import os.path as op import os.path as op
my_dir = op.abspath(op.dirname(__file__)) my_dir = op.abspath(op.dirname(__file__))

View file

@ -58,12 +58,18 @@ class ObjectIdTestCase(test.CdistTestCase):
with self.assertRaises(core.IllegalObjectIdError): with self.assertRaises(core.IllegalObjectIdError):
core.Object(cdist_type, object_base_path, illegal_object_id) core.Object(cdist_type, object_base_path, illegal_object_id)
def test_object_id_contains_dotcdist(self): def test_object_id_contains_object_marker(self):
cdist_type = core.Type(type_base_path, '__third') cdist_type = core.Type(type_base_path, '__third')
illegal_object_id = 'object_id/may/not/contain/.cdist/anywhere' illegal_object_id = 'object_id/may/not/contain/%s/anywhere' % core.OBJECT_MARKER
with self.assertRaises(core.IllegalObjectIdError): with self.assertRaises(core.IllegalObjectIdError):
core.Object(cdist_type, object_base_path, illegal_object_id) core.Object(cdist_type, object_base_path, illegal_object_id)
def test_object_id_contains_object_marker_string(self):
cdist_type = core.Type(type_base_path, '__third')
illegal_object_id = 'object_id/may/contain_%s_in_filename' % core.OBJECT_MARKER
core.Object(cdist_type, object_base_path, illegal_object_id)
# if we get here, the test passed
class ObjectTestCase(test.CdistTestCase): class ObjectTestCase(test.CdistTestCase):

View file

@ -0,0 +1,2 @@
Some examples of using alternative __remote_copy and __remote_exec prefixes.
This allows you to change how cdist interacts with the target host (or directory, or whatever :-)

View file

@ -0,0 +1,47 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# __remote_copy script to run cdist against a local chroot instead of via ssh
# to a remote target host.
#
# Usage:
# __remote_copy="/path/to/this/script /path/to/your/chroot" cdist config target-id
#
log() {
#echo "$@" | logger -t "cdist-chroot-copy"
:
}
chroot="$1"; shift
target_host="$__target_host"
# replace target_host with chroot location
code="$(echo "$@" | sed "s|$target_host:|$chroot|g")"
log "target_host: $target_host"
log "chroot: $chroot"
log "$@"
log "$code"
# copy files into chroot
cp $code
log "-----"

View file

@ -0,0 +1,55 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# __remote_exec script to run cdist against a local chroot instead of via ssh
# on a remote target host.
#
# Usage:
# __remote_exec="/path/to/this/script /path/to/your/chroot" cdist config target-id
#
log() {
#echo "$@" | logger -t "cdist-chroot-exec"
:
}
chroot="$1"; shift
target_host="$1"; shift
script=$(mktemp "${chroot}/tmp/chroot-${0##*/}.XXXXXXXXXX")
trap cleanup INT TERM EXIT
cleanup() {
[ $__cdist_debug ] || rm "$script"
}
log "target_host: $target_host"
log "script: $script"
log "@: $@"
echo "#!/bin/sh -l" > "$script"
echo "$@" >> "$script"
chmod +x "$script"
relative_script="${script#$chroot}"
log "relative_script: $relative_script"
# run in chroot
chroot "$chroot" "$relative_script"
log "-----"

132
other/examples/remote/schroot-uri Executable file
View file

@ -0,0 +1,132 @@
#!/bin/sh -e
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# __remote_{exec,copy} script to run cdist against a schroot target uri
#
# Usage:
# __remote_exec="/path/to/this/script exec" cdist config target_uri
# __remote_copy="/path/to/this/script copy" cdist config target_uri
#
# # target_uri examples:
# schroot:///chroot-name
# schroot://foo.ethz.ch/chroot-name
# schroot://user-name@foo.ethz.ch/chroot-name
#
# # and how to match them in .../manifest/init
# case "$target_host" in
# schroot://*)
# # any schroot
# ;;
# schroot://foo.ethz.ch/*)
# # any schroot on specific host
# ;;
# schroot://foo.ethz.ch/chroot-name)
# # specific schroot on specific host
# ;;
# schroot:///chroot-name)
# # specific schroot on localhost
# ;;
# esac
my_name="${0##*/}"
mode="$1"; shift
log() {
#echo "$@" | logger -t "cdist-$my_name-$mode"
:
}
die() {
echo "$@" >&2
exit 1
}
uri="$__target_host"
scheme="${uri%%:*}"; rest="${uri#$scheme:}"; rest="${rest#//}"
authority="${rest%%/*}"; rest="${rest#$authority}"
path="${rest%\?*}"; rest="${rest#$path}"
schroot_name="${path#/}"
[ "$scheme" = "schroot" ] || die "Failed to parse scheme from __target_host ($__target_host). Expected 'schroot', got '$scheme'"
[ -n "$schroot_name" ] || die "Failed to parse schroot name from __target_host: $__target_host"
case "$authority" in
'')
# authority is empty, neither user nor host given
user=""
host=""
;;
*@*)
# authority contains @, take user from authority
user="${authority%@*}"
host="${authority#*@}"
;;
*)
# no user in authority, default to root
user="root"
host="$authority"
;;
esac
log "mode: $mode"
log "@: $@"
log "uri: $uri"
log "scheme: $scheme"
log "authority: $authority"
log "user: $user"
log "host: $host"
log "path: $path"
log "schroot_name: $schroot_name"
exec_prefix=""
copy_prefix=""
if [ -n "$host" ]; then
# we are working on a remote host
exec_prefix="ssh -o User=$user -q $host"
copy_prefix="scp -o User=$user -q"
copy_destination_prefix="$host:"
else
# working on local machine
copy_prefix="cp"
copy_destination_prefix=""
fi
case "$mode" in
exec)
code="$exec_prefix schroot -c $schroot_name -- $@"
;;
copy)
# get directory for given chroot_name
schroot_directory="$($exec_prefix schroot $chroot_name --config | awk -F = '/directory=/ {print $2}')"
[ -n "$schroot_directory" ] || die "Failed to retreive schroot directory for schroot: $schroot_name"
# prefix destination with chroot
code="$copy_prefix $(echo "$@" | sed "s|$uri:|${copy_destination_prefix}${schroot_directory}|g")"
;;
*) die "Unknown mode: $mode";;
esac
log "code: $code"
# Run the code
$code
log "-----"

View file

@ -0,0 +1,50 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# __remote_copy script to run cdist against a chroot on the target host over ssh.
#
# Usage:
# __remote_copy="/path/to/this/script schroot-chroot-name" cdist config target_host
#
log() {
#echo "$@" | logger -t "cdist-schroot-copy"
:
}
chroot_name="$1"; shift
target_host="$__target_host"
# get directory for given chroot_name
chroot="$(ssh -o User=root -q $target_host schroot $chroot_name --config | awk -F = '/directory=/ {print $2}')"
# prefix destination with chroot
code="$(echo "$@" | sed "s|$target_host:|$target_host:$chroot|g")"
log "target_host: $target_host"
log "chroot_name: $chroot_name"
log "chroot: $chroot"
log "@: $@"
log "code: $code"
# copy files into remote chroot
scp -o User=root -q $code
log "-----"

View file

@ -0,0 +1,45 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# __remote_exec script to run cdist against a chroot on the target host over ssh.
#
# Usage:
# __remote_exec="/path/to/this/script schroot-chroot-name" cdist config target_host
#
log() {
#echo "$@" | logger -t "cdist-schroot-exec"
:
}
chroot_name="$1"; shift
target_host="$1"; shift
code="ssh -o User=root -q $target_host schroot -c $chroot_name -- $@"
log "target_host: $target_host"
log "chroot_name: $chroot_name"
log "@: $@"
log "code: $code"
# run in remote chroot
$code
log "-----"

28
other/examples/remote/ssh/copy Executable file
View file

@ -0,0 +1,28 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# same as cdist default
#
# Usage:
# __remote_copy="/path/to/this/script" cdist config target_host
#
#echo "$@" | logger -t "cdist-ssh-copy"
scp -o User=root -q $@

28
other/examples/remote/ssh/exec Executable file
View file

@ -0,0 +1,28 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# same as cdist default
#
# Usage:
# __remote_exec="/path/to/this/script" cdist config target_host
#
#echo "$@" | logger -t "cdist-ssh-exec"
ssh -o User=root -q $@