Merge branch 'master' into autorequire

Conflicts:
	lib/cdist/test/emulator/__init__.py

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
This commit is contained in:
Steven Armstrong 2011-11-15 10:47:16 +01:00
commit f36d09289d
17 changed files with 445 additions and 21 deletions

View File

@ -24,7 +24,7 @@
if [ -f "$__object/parameter/name" ]; then
name="$(cat "$__object/parameter/name")"
else
name="/$__object_id"
name="$__object_id"
fi
pgrep -x -f "$name" || true

View File

@ -50,6 +50,10 @@ __process /usr/sbin/sshd --state stopped --stop "/etc/rc.d/sshd stop"
# Ensure cups is running, which runs with -C ...:
__process cups --start "/etc/rc.d/cups start" --state running \
--name "/usr/sbin/cupsd -C /etc/cups/cupsd.conf"
# Ensure rpc.statd is running (which usually runs with -L) using a regexp
__process rpcstatd --state running --start "/etc/init.d/statd start" \
--name "rpc.statd.*"
--------------------------------------------------------------------------------

View File

@ -3,6 +3,7 @@
* Cleanup: __object_fq variable removed (never used)
* Cleanup: Environment variable __self DEPRECATED, use __object_name instead
* Cleanup: Environment variable __self scheduled for removal in cdist 2.1
* New Type: __cron (Steven Armstrong)
2.0.3: 2011-10-18
* Improved logging, added --verbose, by more quiet by default

View File

@ -71,7 +71,7 @@ class ConfigInstall(object):
start_time = time.time()
self.deploy_to()
self.cleanup()
self.log.info("Finished run in %s seconds",
self.log.info("Finished successful run in %s seconds",
time.time() - start_time)
def stage_prepare(self):
@ -106,7 +106,7 @@ class ConfigInstall(object):
"""Run gencode and code for an object"""
self.log.debug("Trying to run object " + cdist_object.name)
if cdist_object.state == core.Object.STATE_RUNNING:
# FIXME: resolve dependency circle
# FIXME: resolve dependency circle / show problem source
raise cdist.Error("Detected circular dependency in " + cdist_object.name)
elif cdist_object.state == core.Object.STATE_DONE:
self.log.debug("Ignoring run of already finished object %s", cdist_object)
@ -119,6 +119,13 @@ class ConfigInstall(object):
for requirement in cdist_object.requirements:
self.log.debug("Object %s requires %s", cdist_object, requirement)
required_object = cdist_object.object_from_name(requirement)
# The user may have created dependencies without satisfying them
if not required_object.exists:
raise cdist.Error(cdist_object.name + " requires non-existing " + required_object.name)
else:
self.log.debug("Required object %s exists", required_object.name)
self.object_run(required_object)
# Generate

View File

@ -23,6 +23,7 @@ from cdist.core.type import Type
from cdist.core.type import NoSuchTypeError
from cdist.core.object import Object
from cdist.core.object import IllegalObjectIdError
from cdist.core.object import OBJECT_MARKER
from cdist.core.explorer import Explorer
from cdist.core.manifest import Manifest
from cdist.core.code import Code

View File

@ -30,7 +30,7 @@ from cdist.util import fsproperty
log = logging.getLogger(__name__)
DOT_CDIST = '.cdist'
OBJECT_MARKER = '.cdist'
class IllegalObjectIdError(cdist.Error):
@ -72,7 +72,7 @@ class Object(object):
def list_object_names(cls, object_base_path):
"""Return a list of object names"""
for path, dirs, files in os.walk(object_base_path):
if DOT_CDIST in dirs:
if OBJECT_MARKER in dirs:
yield os.path.relpath(path, object_base_path)
@staticmethod
@ -100,13 +100,13 @@ class Object(object):
if object_id:
if object_id.startswith('/'):
raise IllegalObjectIdError(object_id, 'object_id may not start with /')
if '.cdist' in object_id:
raise IllegalObjectIdError(object_id, 'object_id may not contain \'.cdist\'')
if OBJECT_MARKER in object_id.split(os.sep):
raise IllegalObjectIdError(object_id, 'object_id may not contain \'%s\'' % OBJECT_MARKER)
self.type = cdist_type # instance of Type
self.base_path = base_path
self.object_id = object_id
self.name = self.join_name(self.type.name, self.object_id)
self.path = os.path.join(self.type.path, self.object_id, DOT_CDIST)
self.path = os.path.join(self.type.path, self.object_id, OBJECT_MARKER)
self.absolute_path = os.path.join(self.base_path, self.path)
self.code_local_path = os.path.join(self.path, "code-local")
self.code_remote_path = os.path.join(self.path, "code-remote")
@ -132,9 +132,9 @@ class Object(object):
"""
type_path = self.type.base_path
object_path = self.base_path
base_path = self.base_path
type_name, object_id = self.split_name(object_name)
return self.__class__(self.type.__class__(type_path, type_name), object_path, object_id=object_id)
return self.__class__(self.type.__class__(type_path, type_name), base_path, object_id=object_id)
# FIXME: still needed?
@property

View File

@ -157,19 +157,22 @@ class Emulator(object):
self.log.debug("Recording requirement: " + requirement)
requirement_parts = requirement.split(os.sep, 1)
requirement_type_name = requirement_parts[0]
requirement_object_id = requirement_parts[1]
# FIXME: Add support for omitted object id == singleton
#if len(requirement_parts) == 1:
#except IndexError:
# # no object id, must be singleton
# requirement_object_id = 'singleton'
# Remove / if existent in object id
try:
requirement_object_id = requirement_parts[1]
except IndexError:
# no object id, assume singleton
requirement_object_id = 'singleton'
# Remove leading / from object id
requirement_object_id = requirement_object_id.lstrip('/')
# Instantiate type which fails if type does not exist
requirement_type = core.Type(self.type_base_path, requirement_type_name)
if requirement_object_id == 'singleton' \
and not requirement_type.is_singleton:
raise IllegalRequirementError(requirement, "Missing object_id and type is not a singleton.")
# Instantiate object which fails if the object_id is illegal
requirement_object = core.Object(requirement_type, self.object_base_path, requirement_object_id)

View File

@ -74,6 +74,21 @@ class EmulatorTestCase(test.CdistTestCase):
emu = emulator.Emulator(argv)
self.assertRaises(core.IllegalObjectIdError, emu.run)
def test_missing_object_id_requirement(self):
argv = ['__file', '/tmp/foobar']
os.environ.update(self.env)
os.environ['require'] = '__file'
emu = emulator.Emulator(argv)
self.assertRaises(emulator.IllegalRequirementError, emu.run)
def test_singleton_object_requirement(self):
argv = ['__file', '/tmp/foobar']
os.environ.update(self.env)
os.environ['require'] = '__issue'
emu = emulator.Emulator(argv)
emu.run()
# if we get here all is fine
import os.path as op
my_dir = op.abspath(op.dirname(__file__))

View File

@ -58,12 +58,18 @@ class ObjectIdTestCase(test.CdistTestCase):
with self.assertRaises(core.IllegalObjectIdError):
core.Object(cdist_type, object_base_path, illegal_object_id)
def test_object_id_contains_dotcdist(self):
def test_object_id_contains_object_marker(self):
cdist_type = core.Type(type_base_path, '__third')
illegal_object_id = 'object_id/may/not/contain/.cdist/anywhere'
illegal_object_id = 'object_id/may/not/contain/%s/anywhere' % core.OBJECT_MARKER
with self.assertRaises(core.IllegalObjectIdError):
core.Object(cdist_type, object_base_path, illegal_object_id)
def test_object_id_contains_object_marker_string(self):
cdist_type = core.Type(type_base_path, '__third')
illegal_object_id = 'object_id/may/contain_%s_in_filename' % core.OBJECT_MARKER
core.Object(cdist_type, object_base_path, illegal_object_id)
# if we get here, the test passed
class ObjectTestCase(test.CdistTestCase):

View File

@ -0,0 +1,2 @@
Some examples of using alternative __remote_copy and __remote_exec prefixes.
This allows you to change how cdist interacts with the target host (or directory, or whatever :-)

View File

@ -0,0 +1,47 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# __remote_copy script to run cdist against a local chroot instead of via ssh
# to a remote target host.
#
# Usage:
# __remote_copy="/path/to/this/script /path/to/your/chroot" cdist config target-id
#
log() {
#echo "$@" | logger -t "cdist-chroot-copy"
:
}
chroot="$1"; shift
target_host="$__target_host"
# replace target_host with chroot location
code="$(echo "$@" | sed "s|$target_host:|$chroot|g")"
log "target_host: $target_host"
log "chroot: $chroot"
log "$@"
log "$code"
# copy files into chroot
cp $code
log "-----"

View File

@ -0,0 +1,55 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# __remote_exec script to run cdist against a local chroot instead of via ssh
# on a remote target host.
#
# Usage:
# __remote_exec="/path/to/this/script /path/to/your/chroot" cdist config target-id
#
log() {
#echo "$@" | logger -t "cdist-chroot-exec"
:
}
chroot="$1"; shift
target_host="$1"; shift
script=$(mktemp "${chroot}/tmp/chroot-${0##*/}.XXXXXXXXXX")
trap cleanup INT TERM EXIT
cleanup() {
[ $__cdist_debug ] || rm "$script"
}
log "target_host: $target_host"
log "script: $script"
log "@: $@"
echo "#!/bin/sh -l" > "$script"
echo "$@" >> "$script"
chmod +x "$script"
relative_script="${script#$chroot}"
log "relative_script: $relative_script"
# run in chroot
chroot "$chroot" "$relative_script"
log "-----"

132
other/examples/remote/schroot-uri Executable file
View File

@ -0,0 +1,132 @@
#!/bin/sh -e
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# __remote_{exec,copy} script to run cdist against a schroot target uri
#
# Usage:
# __remote_exec="/path/to/this/script exec" cdist config target_uri
# __remote_copy="/path/to/this/script copy" cdist config target_uri
#
# # target_uri examples:
# schroot:///chroot-name
# schroot://foo.ethz.ch/chroot-name
# schroot://user-name@foo.ethz.ch/chroot-name
#
# # and how to match them in .../manifest/init
# case "$target_host" in
# schroot://*)
# # any schroot
# ;;
# schroot://foo.ethz.ch/*)
# # any schroot on specific host
# ;;
# schroot://foo.ethz.ch/chroot-name)
# # specific schroot on specific host
# ;;
# schroot:///chroot-name)
# # specific schroot on localhost
# ;;
# esac
my_name="${0##*/}"
mode="$1"; shift
log() {
#echo "$@" | logger -t "cdist-$my_name-$mode"
:
}
die() {
echo "$@" >&2
exit 1
}
uri="$__target_host"
scheme="${uri%%:*}"; rest="${uri#$scheme:}"; rest="${rest#//}"
authority="${rest%%/*}"; rest="${rest#$authority}"
path="${rest%\?*}"; rest="${rest#$path}"
schroot_name="${path#/}"
[ "$scheme" = "schroot" ] || die "Failed to parse scheme from __target_host ($__target_host). Expected 'schroot', got '$scheme'"
[ -n "$schroot_name" ] || die "Failed to parse schroot name from __target_host: $__target_host"
case "$authority" in
'')
# authority is empty, neither user nor host given
user=""
host=""
;;
*@*)
# authority contains @, take user from authority
user="${authority%@*}"
host="${authority#*@}"
;;
*)
# no user in authority, default to root
user="root"
host="$authority"
;;
esac
log "mode: $mode"
log "@: $@"
log "uri: $uri"
log "scheme: $scheme"
log "authority: $authority"
log "user: $user"
log "host: $host"
log "path: $path"
log "schroot_name: $schroot_name"
exec_prefix=""
copy_prefix=""
if [ -n "$host" ]; then
# we are working on a remote host
exec_prefix="ssh -o User=$user -q $host"
copy_prefix="scp -o User=$user -q"
copy_destination_prefix="$host:"
else
# working on local machine
copy_prefix="cp"
copy_destination_prefix=""
fi
case "$mode" in
exec)
code="$exec_prefix schroot -c $schroot_name -- $@"
;;
copy)
# get directory for given chroot_name
schroot_directory="$($exec_prefix schroot $chroot_name --config | awk -F = '/directory=/ {print $2}')"
[ -n "$schroot_directory" ] || die "Failed to retreive schroot directory for schroot: $schroot_name"
# prefix destination with chroot
code="$copy_prefix $(echo "$@" | sed "s|$uri:|${copy_destination_prefix}${schroot_directory}|g")"
;;
*) die "Unknown mode: $mode";;
esac
log "code: $code"
# Run the code
$code
log "-----"

View File

@ -0,0 +1,50 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# __remote_copy script to run cdist against a chroot on the target host over ssh.
#
# Usage:
# __remote_copy="/path/to/this/script schroot-chroot-name" cdist config target_host
#
log() {
#echo "$@" | logger -t "cdist-schroot-copy"
:
}
chroot_name="$1"; shift
target_host="$__target_host"
# get directory for given chroot_name
chroot="$(ssh -o User=root -q $target_host schroot $chroot_name --config | awk -F = '/directory=/ {print $2}')"
# prefix destination with chroot
code="$(echo "$@" | sed "s|$target_host:|$target_host:$chroot|g")"
log "target_host: $target_host"
log "chroot_name: $chroot_name"
log "chroot: $chroot"
log "@: $@"
log "code: $code"
# copy files into remote chroot
scp -o User=root -q $code
log "-----"

View File

@ -0,0 +1,45 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# __remote_exec script to run cdist against a chroot on the target host over ssh.
#
# Usage:
# __remote_exec="/path/to/this/script schroot-chroot-name" cdist config target_host
#
log() {
#echo "$@" | logger -t "cdist-schroot-exec"
:
}
chroot_name="$1"; shift
target_host="$1"; shift
code="ssh -o User=root -q $target_host schroot -c $chroot_name -- $@"
log "target_host: $target_host"
log "chroot_name: $chroot_name"
log "@: $@"
log "code: $code"
# run in remote chroot
$code
log "-----"

28
other/examples/remote/ssh/copy Executable file
View File

@ -0,0 +1,28 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# same as cdist default
#
# Usage:
# __remote_copy="/path/to/this/script" cdist config target_host
#
#echo "$@" | logger -t "cdist-ssh-copy"
scp -o User=root -q $@

28
other/examples/remote/ssh/exec Executable file
View File

@ -0,0 +1,28 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# same as cdist default
#
# Usage:
# __remote_exec="/path/to/this/script" cdist config target_host
#
#echo "$@" | logger -t "cdist-ssh-exec"
ssh -o User=root -q $@