Merge branch 'master' into autorequire

Conflicts:
	lib/cdist/test/emulator/__init__.py

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>

conf/type/__process/explorer/runs

@@ -24,7 +24,7 @@
 if [ -f "$__object/parameter/name" ]; then
    name="$(cat "$__object/parameter/name")"
 else
-   name="/$__object_id"
+   name="$__object_id"
 fi
 
 pgrep -x -f "$name" || true

conf/type/__process/man.text

@@ -50,6 +50,10 @@ __process /usr/sbin/sshd --state stopped --stop "/etc/rc.d/sshd stop"
 # Ensure cups is running, which runs with -C ...:
 __process cups --start "/etc/rc.d/cups start" --state running \
    --name "/usr/sbin/cupsd -C /etc/cups/cupsd.conf"
+
+# Ensure rpc.statd is running (which usually runs with -L) using a regexp
+__process rpcstatd --state running --start "/etc/init.d/statd start" \
+    --name "rpc.statd.*"
 --------------------------------------------------------------------------------
 
 

doc/changelog

@@ -3,6 +3,7 @@
 	* Cleanup: __object_fq variable removed (never used)
 	* Cleanup: Environment variable __self DEPRECATED, use __object_name instead
 	* Cleanup: Environment variable __self scheduled for removal in cdist 2.1
+	* New Type: __cron (Steven Armstrong)
 
 2.0.3: 2011-10-18
 	* Improved logging, added --verbose, by more quiet by default

lib/cdist/config_install.py

@@ -71,7 +71,7 @@ class ConfigInstall(object):
         start_time = time.time()
         self.deploy_to()
         self.cleanup()
-        self.log.info("Finished run in %s seconds",
+        self.log.info("Finished successful run in %s seconds",
             time.time() - start_time)
 
     def stage_prepare(self):
@@ -106,7 +106,7 @@ class ConfigInstall(object):
         """Run gencode and code for an object"""
         self.log.debug("Trying to run object " + cdist_object.name)
         if cdist_object.state == core.Object.STATE_RUNNING:
-            # FIXME: resolve dependency circle
+            # FIXME: resolve dependency circle / show problem source
             raise cdist.Error("Detected circular dependency in " + cdist_object.name)
         elif cdist_object.state == core.Object.STATE_DONE:
             self.log.debug("Ignoring run of already finished object %s", cdist_object)
@@ -119,6 +119,13 @@ class ConfigInstall(object):
         for requirement in cdist_object.requirements:
             self.log.debug("Object %s requires %s", cdist_object, requirement)
             required_object = cdist_object.object_from_name(requirement)
+
+            # The user may have created dependencies without satisfying them
+            if not required_object.exists:
+                raise cdist.Error(cdist_object.name + " requires non-existing " + required_object.name)
+            else:
+                self.log.debug("Required object %s exists", required_object.name)
+
             self.object_run(required_object)
 
         # Generate

lib/cdist/core/__init__.py

@@ -23,6 +23,7 @@ from cdist.core.type import Type
 from cdist.core.type import NoSuchTypeError
 from cdist.core.object import Object
 from cdist.core.object import IllegalObjectIdError
+from cdist.core.object import OBJECT_MARKER
 from cdist.core.explorer import Explorer
 from cdist.core.manifest import Manifest
 from cdist.core.code import Code

lib/cdist/core/object.py

@@ -30,7 +30,7 @@ from cdist.util import fsproperty
 
 log = logging.getLogger(__name__)
 
-DOT_CDIST = '.cdist'
+OBJECT_MARKER = '.cdist'
 
 
 class IllegalObjectIdError(cdist.Error):
@@ -72,7 +72,7 @@ class Object(object):
     def list_object_names(cls, object_base_path):
         """Return a list of object names"""
         for path, dirs, files in os.walk(object_base_path):
-            if DOT_CDIST in dirs:
+            if OBJECT_MARKER in dirs:
                 yield os.path.relpath(path, object_base_path)
 
     @staticmethod
@@ -100,13 +100,13 @@ class Object(object):
         if object_id:
             if object_id.startswith('/'):
                 raise IllegalObjectIdError(object_id, 'object_id may not start with /')
-            if '.cdist' in object_id:
-                raise IllegalObjectIdError(object_id, 'object_id may not contain \'.cdist\'')
+            if OBJECT_MARKER in object_id.split(os.sep):
+                raise IllegalObjectIdError(object_id, 'object_id may not contain \'%s\'' % OBJECT_MARKER)
         self.type = cdist_type # instance of Type
         self.base_path = base_path
         self.object_id = object_id
         self.name = self.join_name(self.type.name, self.object_id)
-        self.path = os.path.join(self.type.path, self.object_id, DOT_CDIST)
+        self.path = os.path.join(self.type.path, self.object_id, OBJECT_MARKER)
         self.absolute_path = os.path.join(self.base_path, self.path)
         self.code_local_path = os.path.join(self.path, "code-local")
         self.code_remote_path = os.path.join(self.path, "code-remote")
@@ -132,9 +132,9 @@ class Object(object):
 
         """
         type_path = self.type.base_path
-        object_path = self.base_path
+        base_path = self.base_path
         type_name, object_id = self.split_name(object_name)
-        return self.__class__(self.type.__class__(type_path, type_name), object_path, object_id=object_id)
+        return self.__class__(self.type.__class__(type_path, type_name), base_path, object_id=object_id)
 
     # FIXME: still needed?
     @property

lib/cdist/emulator.py

@@ -157,19 +157,22 @@ class Emulator(object):
                 self.log.debug("Recording requirement: " + requirement)
                 requirement_parts = requirement.split(os.sep, 1)
                 requirement_type_name = requirement_parts[0]
+                try:
                     requirement_object_id = requirement_parts[1]
+                except IndexError:
+                    # no object id, assume singleton
+                    requirement_object_id = 'singleton'
                 
-                # FIXME: Add support for omitted object id == singleton
-                #if len(requirement_parts) == 1:
-                #except IndexError:
-                #    # no object id, must be singleton
-                #    requirement_object_id = 'singleton'
-
-                # Remove / if existent in object id
+                # Remove leading / from object id
                 requirement_object_id = requirement_object_id.lstrip('/')
 
                 # Instantiate type which fails if type does not exist
                 requirement_type = core.Type(self.type_base_path, requirement_type_name)
+
+                if requirement_object_id == 'singleton' \
+                    and not requirement_type.is_singleton:
+                    raise IllegalRequirementError(requirement, "Missing object_id and type is not a singleton.")
+
                 # Instantiate object which fails if the object_id is illegal
                 requirement_object = core.Object(requirement_type, self.object_base_path, requirement_object_id)
 

lib/cdist/test/emulator/__init__.py

@@ -74,6 +74,21 @@ class EmulatorTestCase(test.CdistTestCase):
         emu = emulator.Emulator(argv)
         self.assertRaises(core.IllegalObjectIdError, emu.run)
 
+    def test_missing_object_id_requirement(self):
+        argv = ['__file', '/tmp/foobar']
+        os.environ.update(self.env)
+        os.environ['require'] = '__file'
+        emu = emulator.Emulator(argv)
+        self.assertRaises(emulator.IllegalRequirementError, emu.run)
+
+    def test_singleton_object_requirement(self):
+        argv = ['__file', '/tmp/foobar']
+        os.environ.update(self.env)
+        os.environ['require'] = '__issue'
+        emu = emulator.Emulator(argv)
+        emu.run()
+        # if we get here all is fine
+
 
 import os.path as op
 my_dir = op.abspath(op.dirname(__file__))

lib/cdist/test/object/__init__.py

@@ -58,12 +58,18 @@ class ObjectIdTestCase(test.CdistTestCase):
         with self.assertRaises(core.IllegalObjectIdError):
             core.Object(cdist_type, object_base_path, illegal_object_id)
 
-    def test_object_id_contains_dotcdist(self):
+    def test_object_id_contains_object_marker(self):
         cdist_type = core.Type(type_base_path, '__third')
-        illegal_object_id = 'object_id/may/not/contain/.cdist/anywhere'
+        illegal_object_id = 'object_id/may/not/contain/%s/anywhere' % core.OBJECT_MARKER
         with self.assertRaises(core.IllegalObjectIdError):
             core.Object(cdist_type, object_base_path, illegal_object_id)
 
+    def test_object_id_contains_object_marker_string(self):
+        cdist_type = core.Type(type_base_path, '__third')
+        illegal_object_id = 'object_id/may/contain_%s_in_filename' % core.OBJECT_MARKER
+        core.Object(cdist_type, object_base_path, illegal_object_id)
+        # if we get here, the test passed
+
 
 class ObjectTestCase(test.CdistTestCase):
 

other/examples/remote/README

@@ -0,0 +1,2 @@
+Some examples of using alternative __remote_copy and __remote_exec prefixes.
+This allows you to change how cdist interacts with the target host (or directory, or whatever :-)

other/examples/remote/chroot/copy

@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# __remote_copy script to run cdist against a local chroot instead of via ssh 
+# to a remote target host.
+#
+# Usage:
+#  __remote_copy="/path/to/this/script /path/to/your/chroot" cdist config target-id
+#
+
+log() {
+   #echo "$@" | logger -t "cdist-chroot-copy"
+   :
+}
+
+chroot="$1"; shift
+target_host="$__target_host"
+
+# replace target_host with chroot location
+code="$(echo "$@" | sed "s|$target_host:|$chroot|g")"
+
+log "target_host: $target_host"
+log "chroot: $chroot"
+log "$@"
+log "$code"
+
+# copy files into chroot
+cp $code
+
+log "-----"

other/examples/remote/chroot/exec

@@ -0,0 +1,55 @@
+#!/bin/sh
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# __remote_exec script to run cdist against a local chroot instead of via ssh 
+# on a remote target host.
+#
+# Usage:
+#  __remote_exec="/path/to/this/script /path/to/your/chroot" cdist config target-id
+#
+
+log() {
+   #echo "$@" | logger -t "cdist-chroot-exec"
+   :
+}
+
+chroot="$1"; shift
+target_host="$1"; shift
+
+script=$(mktemp "${chroot}/tmp/chroot-${0##*/}.XXXXXXXXXX")
+trap cleanup INT TERM EXIT
+cleanup() {
+   [ $__cdist_debug ] || rm "$script"
+}
+
+log "target_host: $target_host"
+log "script: $script"
+log "@: $@"
+echo "#!/bin/sh -l" > "$script"
+echo "$@" >> "$script"
+chmod +x "$script"
+
+relative_script="${script#$chroot}"
+log "relative_script: $relative_script"
+
+# run in chroot
+chroot "$chroot" "$relative_script"
+
+log "-----"

other/examples/remote/schroot-uri

@@ -0,0 +1,132 @@
+#!/bin/sh -e
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# __remote_{exec,copy} script to run cdist against a schroot target uri
+#
+# Usage:
+#  __remote_exec="/path/to/this/script exec" cdist config target_uri
+#  __remote_copy="/path/to/this/script copy" cdist config target_uri
+#
+# # target_uri examples:
+# schroot:///chroot-name
+# schroot://foo.ethz.ch/chroot-name
+# schroot://user-name@foo.ethz.ch/chroot-name
+#
+# # and how to match them in .../manifest/init
+# case "$target_host" in
+#    schroot://*)
+#       # any schroot
+#    ;;
+#    schroot://foo.ethz.ch/*)
+#       # any schroot on specific host
+#    ;;
+#    schroot://foo.ethz.ch/chroot-name)
+#       # specific schroot on specific host
+#    ;;
+#    schroot:///chroot-name)
+#       # specific schroot on localhost
+#    ;;
+# esac
+
+my_name="${0##*/}"
+mode="$1"; shift
+
+log() {
+   #echo "$@" | logger -t "cdist-$my_name-$mode"
+   :
+}
+
+die() {
+   echo "$@" >&2
+   exit 1
+}
+
+
+uri="$__target_host"
+
+scheme="${uri%%:*}"; rest="${uri#$scheme:}"; rest="${rest#//}"
+authority="${rest%%/*}"; rest="${rest#$authority}"
+path="${rest%\?*}"; rest="${rest#$path}"
+schroot_name="${path#/}"
+
+[ "$scheme" = "schroot" ] || die "Failed to parse scheme from __target_host ($__target_host). Expected 'schroot', got '$scheme'"
+[ -n "$schroot_name" ] || die "Failed to parse schroot name from __target_host: $__target_host"
+
+case "$authority" in
+   '')
+      # authority is empty, neither user nor host given
+      user=""
+      host=""
+   ;; 
+   *@*)
+      # authority contains @, take user from authority
+      user="${authority%@*}"
+      host="${authority#*@}"
+   ;; 
+   *) 
+      # no user in authority, default to root
+      user="root"
+      host="$authority"
+   ;;
+esac
+
+log "mode: $mode"
+log "@: $@"
+log "uri: $uri"
+log "scheme: $scheme"
+log "authority: $authority"
+log "user: $user"
+log "host: $host"
+log "path: $path"
+log "schroot_name: $schroot_name"
+
+exec_prefix=""
+copy_prefix=""
+if [ -n "$host" ]; then
+   # we are working on a remote host
+   exec_prefix="ssh -o User=$user -q $host"
+   copy_prefix="scp -o User=$user -q"
+   copy_destination_prefix="$host:"
+else
+   # working on local machine
+   copy_prefix="cp"
+   copy_destination_prefix=""
+fi
+
+case "$mode" in
+   exec)
+      code="$exec_prefix schroot -c $schroot_name -- $@"
+   ;;
+   copy)
+      # get directory for given chroot_name
+      schroot_directory="$($exec_prefix schroot $chroot_name --config | awk -F = '/directory=/ {print $2}')"
+      [ -n "$schroot_directory" ] || die "Failed to retreive schroot directory for schroot: $schroot_name"
+      # prefix destination with chroot
+      code="$copy_prefix $(echo "$@" | sed "s|$uri:|${copy_destination_prefix}${schroot_directory}|g")"
+   ;;
+   *) die "Unknown mode: $mode";;
+esac
+
+log "code: $code"
+
+# Run the code
+$code
+
+log "-----"

other/examples/remote/schroot/copy

@@ -0,0 +1,50 @@
+#!/bin/sh
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# __remote_copy script to run cdist against a chroot on the target host over ssh.
+#
+# Usage:
+#  __remote_copy="/path/to/this/script schroot-chroot-name" cdist config target_host
+#
+
+log() {
+   #echo "$@" | logger -t "cdist-schroot-copy"
+   :
+}
+
+chroot_name="$1"; shift
+target_host="$__target_host"
+
+# get directory for given chroot_name
+chroot="$(ssh -o User=root -q $target_host schroot $chroot_name --config | awk -F = '/directory=/ {print $2}')"
+
+# prefix destination with chroot
+code="$(echo "$@" | sed "s|$target_host:|$target_host:$chroot|g")"
+
+log "target_host: $target_host"
+log "chroot_name: $chroot_name"
+log "chroot: $chroot"
+log "@: $@"
+log "code: $code"
+
+# copy files into remote chroot
+scp -o User=root -q $code
+
+log "-----"

other/examples/remote/schroot/exec

@@ -0,0 +1,45 @@
+#!/bin/sh
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# __remote_exec script to run cdist against a chroot on the target host over ssh.
+#
+# Usage:
+#  __remote_exec="/path/to/this/script schroot-chroot-name" cdist config target_host
+#
+
+log() {
+   #echo "$@" | logger -t "cdist-schroot-exec"
+   :
+}
+
+chroot_name="$1"; shift
+target_host="$1"; shift
+
+code="ssh -o User=root -q $target_host schroot -c $chroot_name -- $@"
+
+log "target_host: $target_host"
+log "chroot_name: $chroot_name"
+log "@: $@"
+log "code: $code"
+
+# run in remote chroot
+$code
+
+log "-----"

other/examples/remote/ssh/copy

@@ -0,0 +1,28 @@
+#!/bin/sh
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# same as cdist default
+#
+# Usage:
+#  __remote_copy="/path/to/this/script" cdist config target_host
+#
+
+#echo "$@" | logger -t "cdist-ssh-copy"
+scp -o User=root -q $@

other/examples/remote/ssh/exec

@@ -0,0 +1,28 @@
+#!/bin/sh
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# same as cdist default
+#
+# Usage:
+#  __remote_exec="/path/to/this/script" cdist config target_host
+#
+
+#echo "$@" | logger -t "cdist-ssh-exec"
+ssh -o User=root -q $@