diff --git a/README b/README index 3ab11928..965138c4 100644 --- a/README +++ b/README @@ -50,10 +50,11 @@ UNIX, simplicity, familar environment | cdist is configured in POSIX shell ### Documentation -The cdist documentation is included as manpages in the distribution. +The cdist documentation is included as manpages in the distribution. +You can browse the documentation online as well: - * You can [browse the documentation of the latest version online](man) as well. - * Have a look at the [given speeches](speeches) + * [latest version](man/latest) + * [all versions (>= 2.0.4)](man) ### OS support @@ -88,7 +89,7 @@ cdist was tested or is know to run on at least ## Installation -### Preperation +### Preparation Ensure you have Python 3.2 installed on the machine you use to **deploy to the targets** (the ***source host***). @@ -135,7 +136,13 @@ If you want to ensure nothing breaks you must set back the python version to wha #### Max OS X -Ensure you have port installed and configured (http://www.macports.org/install.php). +You can choose between Homebrew and Macports, either way works: + +[Homebrew](http://mxcl.github.com/homebrew/) variant: + + brew install python3 + +[Macports](http://www.macports.org/install.php) variant: port install python32 ln -s /opt/local/bin/python3.2 /opt/local/bin/python3 diff --git a/build b/build index 5cc59b6d..b994ce00 100755 --- a/build +++ b/build @@ -85,7 +85,7 @@ case "$1" in ;; release) - "$0" clean && "$0" man && "$0" web + ./doc/dev/releasechecklist ;; speeches) @@ -99,7 +99,7 @@ case "$1" in webmain) cp README ${WEBPAGE} - cd ${WEBDIR} && git commit -m "cdist update" ${WEBPAGE} + cd ${WEBDIR} && git commit -m "cdist main update" ${WEBPAGE} cd ${WEBDIR} && make pub ;; @@ -119,6 +119,11 @@ case "$1" in cd ${WEBDIR} && git add ${WEBBASE} cd ${WEBDIR} && git commit -m "cdist update" ${WEBBASE} ${WEBPAGE} cd ${WEBDIR} && make pub + + # Fix ikiwiki, which does not like symlinks for pseudo security + ssh tee.schottelius.org \ + "cd /home/services/www/nico/www.nico.schottelius.org/www/software/cdist/man && + rm -f latest && ln -sf "$version" latest" ;; p|pu|pub) diff --git a/conf/explorer/os b/conf/explorer/os index 1aafb468..3f3ce266 100755 --- a/conf/explorer/os +++ b/conf/explorer/os @@ -18,13 +18,14 @@ # along with cdist. If not, see . # # -# All os variables are lower case -# +# All os variables are lower case. Keep this file in alphabetical +# order by os variable except in cases where order otherwise matters, +# in which case keep the primary os and its derivatives together in +# a block (see Debian and Redhat examples below). # -# Ubuntu is also Debian, thus return if Ubuntu was found -if grep -q ^DISTRIB_ID=Ubuntu /etc/lsb-release 2>/dev/null; then - echo ubuntu +if grep -q ^Amazon /etc/system-release 2>/dev/null; then + echo amazon exit 0 fi @@ -33,45 +34,52 @@ if [ -f /etc/arch-release ]; then exit 0 fi +if [ -f /etc/cdist-preos ]; then + echo cdist-preos + exit 0 +fi + +### Debian and derivatives +if grep -q ^DISTRIB_ID=Ubuntu /etc/lsb-release 2>/dev/null; then + echo ubuntu + exit 0 +fi + if [ -f /etc/debian_version ]; then echo debian exit 0 fi +### if [ -f /etc/gentoo-release ]; then echo gentoo exit 0 fi -# Fedora is also Redhat, thus return before redhat! -if grep -q ^Fedora /etc/redhat-release 2>/dev/null; then - echo fedora - exit 0 -fi - -# CentOS is also based on Redhat, thus return before redhat! -if grep -q ^CentOS /etc/redhat-release 2>/dev/null; then - echo centos - exit 0 -fi - -if [ -f /etc/redhat-release ]; then - echo redhat - exit 0 -fi - -if [ -f /etc/SuSE-release ]; then - echo suse - exit 0 -fi - if [ -f /etc/owl-release ]; then echo owl exit 0 fi -if [ -f /etc/cdist-preos ]; then - echo cdist-preos +### Redhat and derivatives +if grep -q ^CentOS /etc/redhat-release 2>/dev/null; then + echo centos + exit 0 +fi + +if grep -q ^Fedora /etc/redhat-release 2>/dev/null; then + echo fedora + exit 0 +fi + +if [ -f /etc/redhat-release ]; then + echo redhat + exit 0 +fi +### + +if [ -f /etc/SuSE-release ]; then + echo suse exit 0 fi diff --git a/conf/explorer/os_version b/conf/explorer/os_version index ef80e8fc..73d3ecd7 100755 --- a/conf/explorer/os_version +++ b/conf/explorer/os_version @@ -23,6 +23,9 @@ # case "$($__explorer/os)" in + amazon) + cat /etc/system-release + ;; archlinux) # empty, but well... cat /etc/arch-release diff --git a/conf/type/__file/gencode-remote b/conf/type/__file/gencode-remote index 9e700934..2b4c7e45 100755 --- a/conf/type/__file/gencode-remote +++ b/conf/type/__file/gencode-remote @@ -34,11 +34,6 @@ case "$state_should" in fi fi - # Mode settings - if [ -f "$__object/parameter/mode" ]; then - echo chmod \"$(cat "$__object/parameter/mode")\" \"$destination\" - fi - # Group if [ -f "$__object/parameter/group" ]; then echo chgrp \"$(cat "$__object/parameter/group")\" \"$destination\" @@ -48,6 +43,12 @@ case "$state_should" in if [ -f "$__object/parameter/owner" ]; then echo chown \"$(cat "$__object/parameter/owner")\" \"$destination\" fi + + # Mode - needs to happen last as a chown/chgrp can alter mode by + # clearing S_ISUID and S_ISGID bits (see chown(2)) + if [ -f "$__object/parameter/mode" ]; then + echo chmod \"$(cat "$__object/parameter/mode")\" \"$destination\" + fi ;; absent) diff --git a/conf/type/__group/gencode-remote b/conf/type/__group/gencode-remote index 20e08738..cf26a437 100755 --- a/conf/type/__group/gencode-remote +++ b/conf/type/__group/gencode-remote @@ -29,7 +29,7 @@ if grep -q "^${name}:" "$__object/explorer/group"; then for property in $(ls .); do new_value="$(cat "$property")" - case "$key" in + case "$property" in password) current_value="$(awk -F: '{ print $2 }' < "$__object/explorer/gshadow")" ;; diff --git a/conf/type/__key_value/explorer/state b/conf/type/__key_value/explorer/state new file mode 100755 index 00000000..94a5ea7f --- /dev/null +++ b/conf/type/__key_value/explorer/state @@ -0,0 +1,53 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +key="$(cat "$__object/parameter/key" 2>/dev/null \ + || echo "$__object_id")" +state="$(cat "$__object/parameter/state" 2>/dev/null \ + || echo "present")" +file="$(cat "$__object/parameter/file")" +delimiter="$(cat "$__object/parameter/delimiter")" +value="$(cat "$__object/parameter/value" 2>/dev/null \ + || echo "__CDIST_NOTSET__")" + +case "$state" in + absent) + if grep -q -E "^$key$delimiter+" "$file"; then + # if the key exists, with whatever value, we will have to remove it + # so report it as present + echo present + else + # key does not exist + echo absent + fi + ;; + present) + if grep -q -E "^$key$delimiter+$value$" "$file"; then + # key exists and value is same + echo present + elif grep -q -E "^$key$delimiter+" "$file"; then + # key exists, but value is empty or different + echo wrongvalue + else + # key does not exist + echo absent + fi + ;; +esac diff --git a/conf/type/__key_value/gencode-remote b/conf/type/__key_value/gencode-remote index eff0925c..0846dca1 100755 --- a/conf/type/__key_value/gencode-remote +++ b/conf/type/__key_value/gencode-remote @@ -18,35 +18,40 @@ # along with cdist. If not, see . # -value_is="$(cat "$__object/explorer/value")" -value_should="$(cat "$__object/parameter/value")" - key="$(cat "$__object/parameter/key")" file="$(cat "$__object/parameter/file")" delimiter="$(cat "$__object/parameter/delimiter")" +value="$(cat "$__object/parameter/value")" -if [ "$value_is" != "$value_should" ]; then - case "$value_is" in - __NOTSET__) - # add key and value - echo "echo \"${key}${delimiter}${value_should}\" >> \"$file\"" - ;; - *) - if [ "$value_should" = '__NOTSET__' ]; then - # remove key and value - cat << DONE -sed -i '/^${key}/d' "$file" -DONE - else - # change value - cat << DONE -awk -F "$delimiter" ' -/${key}${delimiter}*/{gsub("$value_is", "$value_should")};{print}' "$file" > "${file}+" \ -&& mv "${file}+" "$file" +state_is="$(cat "$__object/explorer/state")" +state_should="$(cat "$__object/parameter/state")" -DONE - fi - ;; - esac +if [ "$state_is" = "$state_should" ]; then + # nothing to do + exit 0 fi +case "$state_should" in + absent) + # remove lines starting with key + echo "sed -i '/^$key\($delimiter\+\)/d' \"$file\"" + ;; + present) + case "$state_is" in + absent) + # add new key and value + echo "echo \"${key}${delimiter}${value}\" >> \"$file\"" + ;; + wrongvalue) + # change exisiting value + echo "sed -i \"s|^$key\($delimiter\+\).*|$key\1$value|\" \"$file\"" + ;; + *) + echo "Unknown explorer state: $state_is" >&2 + exit 1 + esac + ;; + *) + echo "Unknown state: $state_should" >&2 + exit 1 +esac diff --git a/conf/type/__key_value/man.text b/conf/type/__key_value/man.text index 3e4e8013..1423fc7d 100644 --- a/conf/type/__key_value/man.text +++ b/conf/type/__key_value/man.text @@ -16,9 +16,6 @@ file. REQUIRED PARAMETERS ------------------- -value:: - The value for the key. Setting the value to `__NOTSET__` will remove the key - from the file. file:: The file to operate on. delimiter:: @@ -27,8 +24,13 @@ delimiter:: OPTIONAL PARAMETERS ------------------- +state:: + present or absent, defaults to present. If present, sets the key to value, + if absent, removes the key from the file. key:: The key to change. Defaults to object_id. +value:: + The value for the key. Optional if state=absent, required otherwise. EXAMPLES @@ -45,6 +47,9 @@ __key_value my-fancy-id --file /etc/login.defs --key SYS_UID_MAX --value 666 \ # Enable packet forwarding __key_value net.ipv4.ip_forward --file /etc/sysctl.conf --value 1 \ --delimiter '=' + +# Remove existing key/value +__key_value LEGACY_KEY --file /etc/somefile --state absent --delimiter '=' -------------------------------------------------------------------------------- diff --git a/conf/type/__key_value/manifest b/conf/type/__key_value/manifest index 706b0b0d..2e75e175 100755 --- a/conf/type/__key_value/manifest +++ b/conf/type/__key_value/manifest @@ -18,9 +18,13 @@ # along with cdist. If not, see . # -if [ -f "$__object/parameter/key" ]; then - key="$(cat "$__object/parameter/key")" -else - echo "$__object_id" > "$__object/parameter/key" -fi +# set defaults +key="$(cat "$__object/parameter/key" 2>/dev/null \ + || echo "$__object_id" | tee "$__object/parameter/key")" +state="$(cat "$__object/parameter/state" 2>/dev/null \ + || echo "present" | tee "$__object/parameter/state")" +if [ "$state" = "present" -a ! -f "$__object/parameter/value" ]; then + echo "Missing required parameter 'value'" >&2 + exit 1 +fi diff --git a/conf/type/__key_value/parameter/optional b/conf/type/__key_value/parameter/optional index 06bfde49..483e3192 100644 --- a/conf/type/__key_value/parameter/optional +++ b/conf/type/__key_value/parameter/optional @@ -1 +1,3 @@ key +value +state diff --git a/conf/type/__key_value/parameter/required b/conf/type/__key_value/parameter/required index 8f4aa53c..3ae10da3 100644 --- a/conf/type/__key_value/parameter/required +++ b/conf/type/__key_value/parameter/required @@ -1,3 +1,2 @@ -value file delimiter diff --git a/conf/type/__link/explorer/state b/conf/type/__link/explorer/state new file mode 100755 index 00000000..a9220a3c --- /dev/null +++ b/conf/type/__link/explorer/state @@ -0,0 +1,62 @@ +#!/bin/sh +# +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +destination="/$__object_id" +type="$(cat "$__object/parameter/type")" +source="$(cat "$__object/parameter/source")" + +# no destination? -> state is absent +if [ ! -e "$destination" ]; then + echo absent + exit 0 +fi + +destination_dir="${destination%/*}" + +case "$type" in + symbolic) + cd "$destination_dir" + source_is=$(ls -l "$destination" | sed 's/.*-> //g') + if [ -h "$destination" -a "$source_is" = "$source" ]; then + echo present + else + echo absent + fi + ;; + hard) + cd "$destination_dir" + # check source relative to destination_dir + if [ ! -e "$source" ]; then + echo sourcemissing + exit 0 + fi + destination_inode=$(ls -i "$destination" | awk '{print $1}') + source_inode=$(ls -i "$source" | awk '{print $1}') + if [ "$destination_inode" -eq "$source_inode" ]; then + echo present + else + echo absent + fi + ;; + *) + echo "Unknown type: $type" >&2 + exit 1 + ;; +esac diff --git a/conf/type/__link/gencode-remote b/conf/type/__link/gencode-remote index 0a367654..8d4cc3d5 100755 --- a/conf/type/__link/gencode-remote +++ b/conf/type/__link/gencode-remote @@ -39,7 +39,14 @@ case "$type" in ;; esac +state_is="$(cat "$__object/explorer/state")" state_should="$(cat "$__object/parameter/state")" + +if [ "$state_should" = "$state_is" ]; then + # nothing to do + exit 0 +fi + case "$state_should" in present) echo ln ${lnopt} -f \"$source\" \"$destination\" diff --git a/conf/type/__package/manifest b/conf/type/__package/manifest index 48818dd8..181da077 100755 --- a/conf/type/__package/manifest +++ b/conf/type/__package/manifest @@ -33,7 +33,7 @@ else archlinux) type="pacman" ;; debian|ubuntu) type="apt" ;; gentoo) type="emerge" ;; - fedora|redhat|centos) type="yum" ;; + amazon|centos|fedora|redhat) type="yum" ;; *) echo "Don't know how to manage packages on: $os" >&2 exit 1 diff --git a/conf/type/__package_pip/explorer/state b/conf/type/__package_pip/explorer/state new file mode 100644 index 00000000..5be07280 --- /dev/null +++ b/conf/type/__package_pip/explorer/state @@ -0,0 +1,49 @@ +#!/bin/sh +# +# 2012 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Development supported by Local AG (www.local.ch) +# + +nameparam="$__object/parameter/name" +if [ -f "$nameparam" ]; then + name=$(cat "$nameparam") +else + name="$__object_id" +fi + +pipparam="$__object/parameter/pip" +if [ -f "$pipparam" ]; then + pip=$(cat "$pipparam") +else + pip="pip" +fi + +# If there is no pip, it may get created from somebody else. +# If it will be created, there is probably no package installed. +if ! command -v "$pip" >/dev/null 2>&1; then + echo absent +else + + if "$pip" freeze | grep -i -q "^$name=="; then + echo present + else + echo absent + fi +fi diff --git a/conf/type/__key_value/explorer/value b/conf/type/__package_pip/gencode-remote old mode 100755 new mode 100644 similarity index 52% rename from conf/type/__key_value/explorer/value rename to conf/type/__package_pip/gencode-remote index 3afc7cc5..e60d74c5 --- a/conf/type/__key_value/explorer/value +++ b/conf/type/__package_pip/gencode-remote @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2012 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -18,20 +18,33 @@ # along with cdist. If not, see . # # -# Get the current value of key or __NOTSET__ if the key doesn't exist. +# Development supported by Local AG (www.local.ch) # -if [ -f "$__object/parameter/key" ]; then - key="$(cat "$__object/parameter/key")" +state_is=$(cat "$__object/explorer/state") +state_should=$(cat "$__object/parameter/state") + +[ "$state_is" = "$state_should" ] && exit 0 + +nameparam="$__object/parameter/name" +if [ -f "$nameparam" ]; then + name=$(cat "$nameparam") else - key="$__object_id" + name="$__object_id" fi -file="$(cat "$__object/parameter/file")" -delimiter="$(cat "$__object/parameter/delimiter")" -awk -F "$delimiter" ' -BEGIN { found=0 } -/^'$key'/ { print $2; found=1 } -END { if (found) exit 0; else exit 1 }' "$file" \ -|| echo "__NOTSET__" +pipparam="$__object/parameter/pip" +if [ -f "$pipparam" ]; then + pip=$(cat "$pipparam") +else + pip="pip" +fi +case "$state_should" in + present) + echo $pip install -q pyro + ;; + absent) + echo $pip uninstall -q -y pyro + ;; +esac diff --git a/conf/type/__package_pip/man.text b/conf/type/__package_pip/man.text new file mode 100644 index 00000000..5ce45c50 --- /dev/null +++ b/conf/type/__package_pip/man.text @@ -0,0 +1,53 @@ +cdist-type__package_pip(7) +========================== +Nico Schottelius + + +NAME +---- +cdist-type__package_pip - Manage packages with pip + + +DESCRIPTION +----------- +Pip is used in Python environments to install packages. +It is also included in the python virtualenv environment. + + +REQUIRED PARAMETERS +------------------- +state:: + Either "present" or "absent". + + +OPTIONAL PARAMETERS +------------------- +name:: + If supplied, use the name and not the object id as the package name. + +pip:: + Instead of using pip from PATH, use the specific pip path. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# Install a package +__package_pip pyro --state present + +# Use pip in a virtualenv located at /root/shinken_virtualenv +__package_pip pyro --state present --pip /root/shinken_virtualenv/bin/pip +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- cdist-type__package(7) + + +COPYING +------- +Copyright \(C) 2012 Nico Schottelius. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/conf/type/__package_pip/parameter/optional b/conf/type/__package_pip/parameter/optional new file mode 100644 index 00000000..a1b589e3 --- /dev/null +++ b/conf/type/__package_pip/parameter/optional @@ -0,0 +1 @@ +pip diff --git a/conf/type/__package_pip/parameter/required b/conf/type/__package_pip/parameter/required new file mode 100644 index 00000000..ff72b5c7 --- /dev/null +++ b/conf/type/__package_pip/parameter/required @@ -0,0 +1 @@ +state diff --git a/conf/type/__package_yum/gencode-remote b/conf/type/__package_yum/gencode-remote index e43712f8..b24ed220 100755 --- a/conf/type/__package_yum/gencode-remote +++ b/conf/type/__package_yum/gencode-remote @@ -29,7 +29,7 @@ fi state="$(cat "$__object/parameter/state")" -if grep -q -E "(centos|redhat)" "$__global/explorer/os"; then +if grep -q -E "(centos|redhat|amazon)" "$__global/explorer/os"; then opts="-y --quiet" else opts="--assumeyes --quiet" diff --git a/conf/type/__user/gencode-remote b/conf/type/__user/gencode-remote index ce50359b..8979c56e 100755 --- a/conf/type/__user/gencode-remote +++ b/conf/type/__user/gencode-remote @@ -28,6 +28,7 @@ cd "$__object/parameter" if grep -q "^${name}:" "$__object/explorer/passwd"; then for property in $(ls .); do new_value="$(cat "$property")" + unset current_value file="$__object/explorer/passwd" @@ -36,13 +37,20 @@ if grep -q "^${name}:" "$__object/explorer/passwd"; then if $(echo "$new_value" | grep -q '^[0-9][0-9]*$'); then field=4 else - # group name - file="$__object/explorer/group" - field=1 + # We were passed a group name. Compare the gid in + # the user's /etc/passwd entry with the gid of the + # group returned by the group explorer. + gid_from_group=$(awk -F: '{ print $3 }' "$__object/explorer/group") + gid_from_passwd=$(awk -F: '{ print $4 }' "$file") + if [ "$gid_from_group" != "$gid_from_passwd" ]; then + current_value="$gid_from_passwd" + else + current_value="$new_value" + fi fi ;; password) - field=3 + field=2 file="$__object/explorer/shadow" ;; comment) field=5 ;; @@ -51,8 +59,12 @@ if grep -q "^${name}:" "$__object/explorer/passwd"; then uid) field=3 ;; esac - export field - current_value="$(awk -F: '{ print $ENVIRON["field"] }' < "$file")" + # If we haven't already set $current_value above, pull it from the + # appropriate file/field. + if [ -z "$current_value" ]; then + export field + current_value="$(awk -F: '{ print $ENVIRON["field"] }' < "$file")" + fi if [ "$new_value" != "$current_value" ]; then set -- "$@" "--$property" \'$new_value\' diff --git a/doc/changelog b/doc/changelog index d5a2fd7a..2d2d7a48 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,15 +1,37 @@ -2.0.6: +Changelog +--------- + + * Changes are always commented with their author in (braces) + * Exception: No braces means author == Nico Schottelius + +2.0.8: + * Cleanup: Better hint to source of error + * Cleanup: Remove support for __debug variable in manifests (Type != Core + debugging) + +2.0.7: 2012-02-13 + * Bugfix __file: Use chmod after chown/chgrp (Matt Coddington) + * Bugfix __user: Correct shadow field in explorer (Matt Coddington) + * Bugfix __link: Properly handle existing links (Steven Armstrong) + * Bugfix __key_value: More robust implementation (Steven Armstrong) + * Bugfix __user: Fix for changing a user's group by name (Matt Coddington) + * New Type: __package_pip + * Bugfix/Cleanup: Correctly allow Object ID to start and end with /, but + not contain //. + +2.0.6: 2012-01-28 * Bugfix __apt_ppa: Also remove the [ppa-name].list file, if empty. (Tim Kersten) + * Bugfix __group: + Referenced wrong variable name (Matt Coddington) * Feature __package_apt: Initial support for virtual packages (Evax Software) + * Feature Core: Added new dependency resolver (Steven Armstrong) + * Feature Explorer, __package_yum: Support Amazon Linux (Matt Coddington) * New Type: __rvm (Evax Software) * New Type: __rvm_gem (Evax Software) * New Type: __rvm_gemset (Evax Software) * New Type: __rvm_ruby (Evax Software) - * New Explorer: runlevel - * Documentation: Update of reference (environment variables) - * Feature core: Added new dependency resolver (Steven Armstrong) 2.0.5: 2012-01-18 * Bugfix __key_value: Use correct delimiters diff --git a/doc/dev/logs/2012-02-08.explorer-depends-on-another-type b/doc/dev/logs/2012-02-08.explorer-depends-on-another-type new file mode 100644 index 00000000..c76626d5 --- /dev/null +++ b/doc/dev/logs/2012-02-08.explorer-depends-on-another-type @@ -0,0 +1,36 @@ +If a type explorer depends on a command that will be generated by another type, +the operation fails, as can be seen below. + +This may be a corner case, but is hapenning with __package_pip and +__python_virtualenv. + +[19:10] brief:cdist% ./bin/cdist config -v loch +INFO: loch: Running global explorers +INFO: loch: Running initial manifest /home/users/nico/privat/firmen/local.ch/vcs/cdist/conf/manifest +INFO: loch: Running object manifests and type explorers +INFO: loch: Running manifest and explorers for __git/root/shinken +INFO: loch: Running manifest and explorers for __package_pip/pyro +/var/lib/cdist/conf/type/__package_pip/explorer/state: line 38: /root/shinken_virtualenv/bin/pip: No such file or directory +INFO: loch: Running manifest and explorers for __python_virtualenv/root/shinken_virtualenv +INFO: loch: Running manifest and explorers for __directory/pyro +INFO: loch: Running manifest and explorers for __directory/root/shinken +INFO: loch: Running manifest and explorers for __directory/root/shinken_virtualenv +INFO: loch: Running manifest and explorers for __package/git +INFO: loch: Running manifest and explorers for __package/python-virtualenv +INFO: loch: Running manifest and explorers for __package_pacman/git +INFO: loch: Running manifest and explorers for __package_pacman/python-virtualenv +INFO: loch: Generating and executing code +INFO: loch: Generating and executing code for __package_pacman/git +INFO: loch: Generating and executing code for __package/git +INFO: loch: Generating and executing code for __directory/root/shinken +INFO: loch: Generating and executing code for __git/root/shinken +fatal: write error: No space left on device +fatal: index-pack failed +ERROR: loch: Code that raised the error: +git clone --quiet "git://github.com/naparuba/shinken.git" "/root/shinken" + +ERROR: Remote script execution failed: ssh -o User=root -q loch /bin/sh -e /var/lib/cdist/object/__git/root/shinken/.cdist/code-remote +WARNING: Failed to deploy to the following hosts: loch +INFO: Total processing time for 1 host(s): 340.62370681762695 +[19:17] brief:cdist% ./bin/cdist config -v loch + diff --git a/doc/dev/logs/2012-02-10.object_id-and-slashes b/doc/dev/logs/2012-02-10.object_id-and-slashes new file mode 100644 index 00000000..de46a1f8 --- /dev/null +++ b/doc/dev/logs/2012-02-10.object_id-and-slashes @@ -0,0 +1,18 @@ +__typename /foo/bar # possible, usual use case +require="__a//b" __typename /foo/bar # possible and happens often for __a/$id in loops + +__typename /foo/bar/ # trailing slash will be stripped, can be documented + +__typename /foo//bar//baz # // will be converted to / implicitly through fs; error prone; disallow + +require="__a//b//c" __typename # // will be converted to / implicitly through fs; error prone; disallow + + +Solution: + + 1) allow require __a//b: type __a, object id /b + => strip first slash of object id, as we do in non-dep-mode + 2) allow _one_ trailing /: __type /foo/bar/ and require="__foo/abc/" + => strip one leading slash of object id + 3) disallow // within object id + 4) disallow starting or ending / after 1) and 2) diff --git a/doc/dev/logs/2012-02-13.dependencies b/doc/dev/logs/2012-02-13.dependencies new file mode 100644 index 00000000..3b0f3f21 --- /dev/null +++ b/doc/dev/logs/2012-02-13.dependencies @@ -0,0 +1,23 @@ +possible dependencies: + + - unix pattern __foo/* + - object: __foo//bar, __foo/bar + - singleton with object_id: __foo/singleton + - singleton without object_id: __foo/ + +solving dependencies: + + solve_dep(object, run_list): + - list = [me] + - if status == IN_DEPENDENCY: + fail: circular dependency + - status = IN_DEPENDENCY + - create_list_of_deps(object) + - try pattern expansion + - for each dependency: + if object does not exist: + fail + else: + list.append(solve_dep(object, run_list)): + - status == IN_LIST + - return [me, dependencies [, dependencies of dependencies]] diff --git a/doc/dev/releasechecklist b/doc/dev/releasechecklist index cb6d610d..eba81dc0 100755 --- a/doc/dev/releasechecklist +++ b/doc/dev/releasechecklist @@ -11,15 +11,25 @@ echo "Testing documentation..." ./build clean && ./build man || exit 1 # get version -changelog_version=$(head -n1 doc/changelog | sed 's/:.*//') +changelog_version=$(grep '^[[:digit:]]' doc/changelog | head -n1 | sed 's/:.*//') #git_version=$(git describe) lib_version=$(grep ^VERSION lib/cdist/__init__.py | sed -e 's/.*= //' -e 's/"//g') +# get date +date_today="$(date +%Y-%m-%d)" +date_changelog=$(grep '^[[:digit:]]' doc/changelog | head -n1 | sed 's/.*: //') + echo "Ensure you fixed/prepared version files: $files" echo "changelog: $changelog_version" #echo "git: $git_version" echo "lib: $lib_version" +if [ "$date_today" != "$date_changelog" ]; then + echo "Messed up date, not releasing:" + echo "Changelog: $date_changelog" + exit 1 +fi + if [ "$lib_version" != "$changelog_version" ]; then echo "Messed up versions, not releasing" exit 1 diff --git a/doc/dev/todo/TAKEME b/doc/dev/todo/TAKEME index abcd5097..b40936f6 100644 --- a/doc/dev/todo/TAKEME +++ b/doc/dev/todo/TAKEME @@ -34,7 +34,6 @@ USER INTERFACE TYPES ------ +- Add testing framework (proposed by Evax Software) - __user add option to include --create-home -- ensure that all types, which support --state support - present and absent (consistent look and feel) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 4543b32a..f81c7570 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,9 +1,17 @@ -- __init_script? - to enable/disable startup of init stuff - http://linuxhelp.blogspot.com/2006/04/enabling-and-disabling-services-during_01.html +2.0.8 features / cleanups: + +- cleanup object_id handling + - have a look at singletons + +- remove useless + ERROR: monitoring02: Code that raised the error: + +- ensure that all types, which support --state support + present and absent (consistent look and feel) + +-------------------------------------------------------------------------------- - update/create docs - - document __remote_copy and __remote_exec - cdist-cache:: How to get use information about the hosts we have been working on [advanced] - cdist-scaling-tuning:: @@ -17,3 +25,4 @@ - exec flag is not true for manifest anymore - SSH HINTS - ssh agent + diff --git a/doc/man/cdist-reference.text.sh b/doc/man/cdist-reference.text.sh index c6016333..abef255f 100755 --- a/doc/man/cdist-reference.text.sh +++ b/doc/man/cdist-reference.text.sh @@ -49,10 +49,10 @@ The following global explorers are available: eof ( - cd ../../conf/explorer - for explorer in *; do - echo "- $explorer" - done + cd ../../conf/explorer + for explorer in *; do + echo "- $explorer" + done ) cat << eof @@ -62,77 +62,77 @@ PATHS If not specified otherwise, all paths are relative to the checkout directory. conf/:: - Contains the (static) configuration like manifests, types and explorers. + Contains the (static) configuration like manifests, types and explorers. conf/manifest/init:: - This is the central entry point used by cdist-manifest-init(1). - It is an executable (+x bit set) shell script that can use - values from the explorers to decide which configuration to create - for the specified target host. - It should be primary used to define mapping from configurations to hosts. + This is the central entry point used by cdist-manifest-init(1). + It is an executable (+x bit set) shell script that can use + values from the explorers to decide which configuration to create + for the specified target host. + It should be primary used to define mapping from configurations to hosts. conf/manifest/*:: - All other files in this directory are not directly used by cdist, but you - can seperate configuration mappings, if you have a lot of code in the - manifest/init file. This may also be helpful to have different admins - maintain different groups of hosts. + All other files in this directory are not directly used by cdist, but you + can seperate configuration mappings, if you have a lot of code in the + manifest/init file. This may also be helpful to have different admins + maintain different groups of hosts. conf/explorer/:: - Contains explorers to be run on the target hosts, see cdist-explorer(7). + Contains explorers to be run on the target hosts, see cdist-explorer(7). conf/type/:: - Contains all available types, which are used to provide - some kind of functionality. See cdist-type(7). + Contains all available types, which are used to provide + some kind of functionality. See cdist-type(7). conf/type//:: - Home of the type . + Home of the type . - This directory is referenced by the variable __type (see below). + This directory is referenced by the variable __type (see below). conf/type//man.text:: - Manpage in Asciidoc format (required for inclusion into upstream) + Manpage in Asciidoc format (required for inclusion into upstream) conf/type//manifest:: - Used to generate additional objects from a type. + Used to generate additional objects from a type. conf/type//gencode-local:: - Used to generate code to be executed on the server. + Used to generate code to be executed on the server. conf/type//gencode-remote:: - Used to generate code to be executed on the client. + Used to generate code to be executed on the client. conf/type//parameters/required:: - Parameters required by type, \n seperated list. + Parameters required by type, \n seperated list. conf/type//parameters/optional:: - Parameters optionally accepted by type, \n seperated list. + Parameters optionally accepted by type, \n seperated list. conf/type//explorer:: - Location of the type specific explorers. - This directory is referenced by the variable __type_explorer (see below). - See cdist-explorer(7). + Location of the type specific explorers. + This directory is referenced by the variable __type_explorer (see below). + See cdist-explorer(7). out/:: - This directory contains output of cdist and is usually located - in a temporary directory and thus will be removed after the run. - This directory is referenced by the variable __global (see below). + This directory contains output of cdist and is usually located + in a temporary directory and thus will be removed after the run. + This directory is referenced by the variable __global (see below). out/explorer:: - Output of general explorers. + Output of general explorers. out/object:: - Objects created for the host. + Objects created for the host. out/object/:: - Contains all object specific information. - This directory is referenced by the variable __object (see below). + Contains all object specific information. + This directory is referenced by the variable __object (see below). out/object//explorers:: - Output of type specific explorers, per object. + Output of type specific explorers, per object. tmp_dir:: - A tempdir and a tempfile is used by cdist internally, - which will be removed when the scripts end automatically. + A tempdir and a tempfile is used by cdist internally, + which will be removed when the scripts end automatically. TYPES ----- @@ -141,13 +141,13 @@ The following types are available: eof for type in man7/cdist-type__*.text; do - no_dir="${type#man7/}"; - no_type="${no_dir#cdist-type}"; - name="${no_type%.text}"; - name_no_underline="$(echo $name | sed 's/^__/\\__/g')" - man="${no_dir%.text}(7)" + no_dir="${type#man7/}"; + no_type="${no_dir#cdist-type}"; + name="${no_type%.text}"; + name_no_underline="$(echo $name | sed 's/^__/\\__/g')" + man="${no_dir%.text}(7)" - echo "- $name_no_underline" "($man)" + echo "- $name_no_underline" "($man)" done cat << eof @@ -159,8 +159,8 @@ For object to object communication and tests, the following paths are usable within a object directory: changed:: - This empty file exists in an object directory, if the object has - code to be excuted (either remote or local) + This empty file exists in an object directory, if the object has + code to be excuted (either remote or local) ENVIRONMENT VARIABLES @@ -169,33 +169,37 @@ __explorer:: Directory that contains all global explorers. Available for: explorer, type explorer __manifest:: - Directory that contains the initial manifest. - Available for: initial manifest + Directory that contains the initial manifest. + Available for: initial manifest __global:: - Directory that contains generic output like explorer. - Available for: initial manifest, type manifest, type gencode + Directory that contains generic output like explorer. + Available for: initial manifest, type manifest, type gencode __object:: - Directory that contains the current object. - Available for: type manifest, type explorer, type gencode + Directory that contains the current object. + Available for: type manifest, type explorer, type gencode __object_id:: - The type unique object id. - Available for: type manifest, type explorer, type gencode - Note: The leading "/" will always be stripped. + The type unique object id. + Available for: type manifest, type explorer, type gencode + + Note: The leading and the trailing "/" will always be stripped (caused by + the filesystem database and ensured by the core). + + Note: Double slashes ("//") will not be fixed and result in an error. __self:: - DEPRECATED: Same as __object_name, do not use anymore, use __object_name instead. - Will be removed in cdist 3.x. + DEPRECATED: Same as __object_name, do not use anymore, use __object_name instead. + Will be removed in cdist 3.x. __object_name:: - The full qualified name of the current object. - Available for: type manifest, type explorer, type gencode + The full qualified name of the current object. + Available for: type manifest, type explorer, type gencode __target_host:: The host we are deploying to. Available for: explorer, initial manifest, type explorer, type manifest, type gencode __type:: - Path to the current type. - Available for: type manifest, type gencode + Path to the current type. + Available for: type manifest, type gencode __type_explorer:: - Directory that contains the type explorers. - Available for: type explorer + Directory that contains the type explorers. + Available for: type explorer SEE ALSO diff --git a/doc/man/man7/cdist-hacker.text b/doc/man/man7/cdist-hacker.text index 9bdf63d4..646439a3 100644 --- a/doc/man/man7/cdist-hacker.text +++ b/doc/man/man7/cdist-hacker.text @@ -61,12 +61,19 @@ including it. HOW TO SUBMIT A NEW TYPE ------------------------ +For detailled information about types, see cdist-type(7). + Submitting a type works as described above, with the additional requirement that a corresponding manpage named man.text in asciidoc format with the manpage-name "cdist-type__NAME" is included in the type directory AND asciidoc is able to compile it (i.e. do NOT have to many "=" in the second line). +Warning: Submitting "exec" or "run" types that simply echo their parameter in +gencode* will not be accepted, because they are of no use. Every type can output +code and thus such a type introduces redundant functionality that is given by +core cdist already. + SEE ALSO -------- diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py index 09e4aacc..4742a937 100644 --- a/lib/cdist/__init__.py +++ b/lib/cdist/__init__.py @@ -19,6 +19,8 @@ # # +VERSION = "2.0.7" + BANNER = """ .. . .x+=:. s dF @88> z` ^% :8 @@ -34,7 +36,6 @@ BANNER = """ "P' "" "" """ DOT_CDIST = ".cdist" -VERSION = "2.0.5" import os @@ -43,15 +44,17 @@ class Error(Exception): """Base exception class for this project""" pass +class CdistObjectError(Error): + """Something went wrong with an object""" + + def __init__(self, cdist_object, message): + self.name = cdist_object.name + self.source = " ".join(cdist_object.source) + self.message = message -class MissingEnvironmentVariableError(Error): - """Raised when a required environment variable is not set.""" - - def __init__(self, name): - self.name = name def __str__(self): - return 'Missing required environment variable: ' + str(self.name) + return '%s: %s (defined at %s)' % (self.name, self.message, self.source) def file_to_list(filename): """Return list from \n seperated file""" diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 542f2024..7cce240e 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -89,9 +89,9 @@ class ConfigInstall(object): new_objects_created = True while new_objects_created: new_objects_created = False - for cdist_object in core.Object.list_objects(self.local.object_path, + for cdist_object in core.CdistObject.list_objects(self.local.object_path, self.local.type_path): - if cdist_object.state == core.Object.STATE_PREPARED: + if cdist_object.state == core.CdistObject.STATE_PREPARED: self.log.debug("Skipping re-prepare of object %s", cdist_object) continue else: @@ -103,16 +103,16 @@ class ConfigInstall(object): self.log.info("Running manifest and explorers for " + cdist_object.name) self.explorer.run_type_explorers(cdist_object) self.manifest.run_type_manifest(cdist_object) - cdist_object.state = core.Object.STATE_PREPARED + cdist_object.state = core.CdistObject.STATE_PREPARED def object_run(self, cdist_object): """Run gencode and code for an object""" self.log.debug("Trying to run object " + cdist_object.name) - if cdist_object.state == core.Object.STATE_DONE: + if cdist_object.state == core.CdistObject.STATE_DONE: # TODO: remove once we are sure that this really never happens. raise cdist.Error("Attempting to run an already finished object: %s", cdist_object) - cdist_type = cdist_object.type + cdist_type = cdist_object.cdist_type # Generate self.log.info("Generating and executing code for " + cdist_object.name) @@ -130,13 +130,13 @@ class ConfigInstall(object): # Mark this object as done self.log.debug("Finishing run of " + cdist_object.name) - cdist_object.state = core.Object.STATE_DONE + cdist_object.state = core.CdistObject.STATE_DONE def stage_run(self): """The final (and real) step of deployment""" self.log.info("Generating and executing code") - objects = core.Object.list_objects( + objects = core.CdistObject.list_objects( self.local.object_path, self.local.type_path) diff --git a/lib/cdist/core/__init__.py b/lib/cdist/core/__init__.py index c61c659b..66ee00a5 100644 --- a/lib/cdist/core/__init__.py +++ b/lib/cdist/core/__init__.py @@ -19,11 +19,11 @@ # # -from cdist.core.type import Type -from cdist.core.type import NoSuchTypeError -from cdist.core.object import Object -from cdist.core.object import IllegalObjectIdError -from cdist.core.object import OBJECT_MARKER -from cdist.core.explorer import Explorer -from cdist.core.manifest import Manifest -from cdist.core.code import Code +from cdist.core.cdist_type import CdistType +from cdist.core.cdist_type import NoSuchTypeError +from cdist.core.cdist_object import CdistObject +from cdist.core.cdist_object import IllegalObjectIdError +from cdist.core.cdist_object import OBJECT_MARKER +from cdist.core.explorer import Explorer +from cdist.core.manifest import Manifest +from cdist.core.code import Code diff --git a/lib/cdist/core/object.py b/lib/cdist/core/cdist_object.py similarity index 74% rename from lib/cdist/core/object.py rename to lib/cdist/core/cdist_object.py index da2f21a6..e12bcfbd 100644 --- a/lib/cdist/core/object.py +++ b/lib/cdist/core/cdist_object.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -42,7 +42,7 @@ class IllegalObjectIdError(cdist.Error): return '%s: %s' % (self.message, self.object_id) -class Object(object): +class CdistObject(object): """Represents a cdist object. All interaction with objects in cdist should be done through this class. @@ -61,7 +61,7 @@ class Object(object): """Return a list of object instances""" for object_name in cls.list_object_names(object_base_path): type_name, object_id = cls.split_name(object_name) - yield cls(cdist.core.Type(type_base_path, type_name), object_base_path, object_id=object_id) + yield cls(cdist.core.CdistType(type_base_path, type_name), object_base_path, object_id=object_id) @classmethod def list_type_names(cls, object_base_path): @@ -96,30 +96,58 @@ class Object(object): """ return os.path.join(type_name, object_id) - @staticmethod - def validate_object_id(object_id): + def validate_object_id(self): + # FIXME: also check that there is no object ID when type is singleton? + """Validate the given object_id and raise IllegalObjectIdError if it's not valid. """ - if object_id: - if object_id.startswith('/'): - raise IllegalObjectIdError(object_id, 'object_id may not start with /') - if OBJECT_MARKER in object_id.split(os.sep): - raise IllegalObjectIdError(object_id, 'object_id may not contain \'%s\'' % OBJECT_MARKER) + if self.object_id: + if OBJECT_MARKER in self.object_id.split(os.sep): + raise IllegalObjectIdError(self.object_id, 'object_id may not contain \'%s\'' % OBJECT_MARKER) + if '//' in self.object_id: + raise IllegalObjectIdError(self.object_id, 'object_id may not contain //') + + # If no object_id and type is not singleton => error out + if not self.object_id and not self.cdist_type.is_singleton: + raise IllegalObjectIdError(self.object_id, + "Missing object_id and type is not a singleton.") def __init__(self, cdist_type, base_path, object_id=None): - self.validate_object_id(object_id) - self.type = cdist_type # instance of Type + self.cdist_type = cdist_type # instance of Type self.base_path = base_path self.object_id = object_id - self.name = self.join_name(self.type.name, self.object_id) - self.path = os.path.join(self.type.path, self.object_id, OBJECT_MARKER) + + self.validate_object_id() + self.sanitise_object_id() + + self.name = self.join_name(self.cdist_type.name, self.object_id) + self.path = os.path.join(self.cdist_type.path, self.object_id, OBJECT_MARKER) self.absolute_path = os.path.join(self.base_path, self.path) self.code_local_path = os.path.join(self.path, "code-local") self.code_remote_path = os.path.join(self.path, "code-remote") self.parameter_path = os.path.join(self.path, "parameter") + def object_from_name(self, object_name): + """Convenience method for creating an object instance from an object name. + + Mainly intended to create objects when resolving requirements. + + e.g: + .object_from_name('__other/object') -> + + """ + + base_path = self.base_path + type_path = self.cdist_type.base_path + + type_name, object_id = self.split_name(object_name) + + cdist_type = self.cdist_type.__class__(type_path, type_name) + + return self.__class__(cdist_type, base_path, object_id=object_id) + def __repr__(self): - return '' % self.name + return '' % self.name def __eq__(self, other): """define equality as 'name is the same'""" @@ -128,23 +156,23 @@ class Object(object): def __hash__(self): return hash(self.name) - def __lt__(self, other): return isinstance(other, self.__class__) and self.name < other.name - def object_from_name(self, object_name): - """Convenience method for creating an object instance from an object name. - - Mainly intended to create objects when resolving requirements. - - e.g: - .object_from_name('__other/object') -> - + def sanitise_object_id(self): """ - type_path = self.type.base_path - base_path = self.base_path - type_name, object_id = self.split_name(object_name) - return self.__class__(self.type.__class__(type_path, type_name), base_path, object_id=object_id) + Remove leading and trailing slash (one only) + """ + + # Allow empty object id for singletons + if self.object_id: + # Remove leading slash + if self.object_id[0] == '/': + self.object_id = self.object_id[1:] + + # Remove trailing slash + if self.object_id[-1] == '/': + self.object_id = self.object_id[:-1] # FIXME: still needed? @property diff --git a/lib/cdist/core/type.py b/lib/cdist/core/cdist_type.py similarity index 97% rename from lib/cdist/core/type.py rename to lib/cdist/core/cdist_type.py index 20365b8d..55609e7e 100644 --- a/lib/cdist/core/type.py +++ b/lib/cdist/core/cdist_type.py @@ -34,7 +34,7 @@ class NoSuchTypeError(cdist.Error): return "Type '%s' does not exist at %s" % (self.type_path, self.type_absolute_path) -class Type(object): +class CdistType(object): """Represents a cdist type. All interaction with types in cdist should be done through this class. @@ -61,7 +61,7 @@ class Type(object): # name is second argument name = args[1] if not name in cls._instances: - instance = super(Type, cls).__new__(cls) + instance = super(CdistType, cls).__new__(cls) cls._instances[name] = instance # return instance so __init__ is called return cls._instances[name] @@ -84,7 +84,7 @@ class Type(object): self.__optional_parameters = None def __repr__(self): - return '' % self.name + return '' % self.name def __eq__(self, other): return isinstance(other, self.__class__) and self.name == other.name diff --git a/lib/cdist/core/code.py b/lib/cdist/core/code.py index 51912559..2ffef9cf 100644 --- a/lib/cdist/core/code.py +++ b/lib/cdist/core/code.py @@ -92,17 +92,14 @@ class Code(object): '__global': self.local.out_path, } - if log.getEffectiveLevel() == logging.DEBUG: - self.env.update({'__debug': "yes" }) - def _run_gencode(self, cdist_object, which): - cdist_type = cdist_object.type + cdist_type = cdist_object.cdist_type script = os.path.join(self.local.type_path, getattr(cdist_type, 'gencode_%s_path' % which)) if os.path.isfile(script): env = os.environ.copy() env.update(self.env) env.update({ - '__type': cdist_object.type.absolute_path, + '__type': cdist_object.cdist_type.absolute_path, '__object': cdist_object.absolute_path, '__object_id': cdist_object.object_id, '__object_name': cdist_object.name, diff --git a/lib/cdist/core/explorer.py b/lib/cdist/core/explorer.py index 19833d92..d49b7ac4 100644 --- a/lib/cdist/core/explorer.py +++ b/lib/cdist/core/explorer.py @@ -73,9 +73,6 @@ class Explorer(object): '__target_host': self.target_host, '__explorer': self.remote.global_explorer_path, } - # FIXME: remove soon with new logging infrastructure - if self.log.getEffectiveLevel() == logging.DEBUG: - self.env.update({'__debug': "yes" }) self._type_explorers_transferred = [] ### global @@ -122,15 +119,28 @@ class Explorer(object): in the object. """ - self.log.debug("Transfering type explorers for type: %s", cdist_object.type) - self.transfer_type_explorers(cdist_object.type) + self.log.debug("Transfering type explorers for type: %s", cdist_object.cdist_type) + self.transfer_type_explorers(cdist_object.cdist_type) self.log.debug("Transfering object parameters for object: %s", cdist_object.name) self.transfer_object_parameters(cdist_object) - for explorer in self.list_type_explorer_names(cdist_object.type): + for explorer in self.list_type_explorer_names(cdist_object.cdist_type): output = self.run_type_explorer(explorer, cdist_object) self.log.debug("Running type explorer '%s' for object '%s'", explorer, cdist_object.name) cdist_object.explorers[explorer] = output + def run_type_explorer(self, explorer, cdist_object): + """Run the given type explorer for the given object and return it's output.""" + cdist_type = cdist_object.cdist_type + env = self.env.copy() + env.update({ + '__object': os.path.join(self.remote.object_path, cdist_object.path), + '__object_id': cdist_object.object_id, + '__object_fq': cdist_object.path, + '__type_explorer': os.path.join(self.remote.type_path, cdist_type.explorer_path) + }) + script = os.path.join(self.remote.type_path, cdist_type.explorer_path, explorer) + return self.remote.run_script(script, env=env, return_output=True) + def transfer_type_explorers(self, cdist_type): """Transfer the type explorers for the given type to the remote side.""" if cdist_type.explorers: @@ -150,16 +160,3 @@ class Explorer(object): destination = os.path.join(self.remote.object_path, cdist_object.parameter_path) self.remote.mkdir(destination) self.remote.transfer(source, destination) - - def run_type_explorer(self, explorer, cdist_object): - """Run the given type explorer for the given object and return it's output.""" - cdist_type = cdist_object.type - env = self.env.copy() - env.update({ - '__object': os.path.join(self.remote.object_path, cdist_object.path), - '__object_id': cdist_object.object_id, - '__object_fq': cdist_object.path, - '__type_explorer': os.path.join(self.remote.type_path, cdist_type.explorer_path) - }) - script = os.path.join(self.remote.type_path, cdist_type.explorer_path, explorer) - return self.remote.run_script(script, env=env, return_output=True) diff --git a/lib/cdist/core/manifest.py b/lib/cdist/core/manifest.py index 704a3978..8b784229 100644 --- a/lib/cdist/core/manifest.py +++ b/lib/cdist/core/manifest.py @@ -87,7 +87,7 @@ class Manifest(object): self.local.run_script(script, env=env) def run_type_manifest(self, cdist_object): - script = os.path.join(self.local.type_path, cdist_object.type.manifest_path) + script = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path) if os.path.isfile(script): env = os.environ.copy() env.update(self.env) @@ -96,7 +96,7 @@ class Manifest(object): '__object_id': cdist_object.object_id, '__object_name': cdist_object.name, '__self': cdist_object.name, - '__type': cdist_object.type.absolute_path, + '__type': cdist_object.cdist_type.absolute_path, '__cdist_manifest': script, }) self.local.run_script(script, env=env) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 05202a39..0979182a 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -26,29 +26,22 @@ import os import cdist from cdist import core - -class IllegalRequirementError(cdist.Error): - def __init__(self, requirement, message=None): - self.requirement = requirement - self.message = message or 'Illegal requirement' - - def __str__(self): - return '%s: %s' % (self.message, self.requirement) - class Emulator(object): def __init__(self, argv): self.argv = argv self.object_id = False self.global_path = os.environ['__global'] - self.object_source = os.environ['__cdist_manifest'] self.target_host = os.environ['__target_host'] + + # Internally only + self.object_source = os.environ['__cdist_manifest'] self.type_base_path = os.environ['__cdist_type_base_path'] self.object_base_path = os.path.join(self.global_path, "object") self.type_name = os.path.basename(argv[0]) - self.cdist_type = core.Type(self.type_base_path, self.type_name) + self.cdist_type = core.CdistType(self.type_base_path, self.type_name) self.__init_log() @@ -113,20 +106,15 @@ class Emulator(object): def setup_object(self): - # FIXME: verify object id - - # Setup object_id + # Setup object_id - FIXME: unset / do not setup anymore! if self.cdist_type.is_singleton: self.object_id = "singleton" else: self.object_id = self.args.object_id[0] del self.args.object_id - # strip leading slash from object_id - self.object_id = self.object_id.lstrip('/') - # Instantiate the cdist object we are defining - self.cdist_object = core.Object(self.cdist_type, self.object_base_path, self.object_id) + self.cdist_object = core.CdistObject(self.cdist_type, self.object_base_path, self.object_id) # Create object with given parameters self.parameters = {} @@ -143,6 +131,9 @@ class Emulator(object): self.cdist_object.create() self.cdist_object.parameters = self.parameters + # Record / Append source + self.cdist_object.source.append(self.object_source) + def record_requirements(self): """record requirements""" @@ -151,25 +142,17 @@ class Emulator(object): self.log.debug("reqs = " + requirements) for requirement in requirements.split(" "): # Ignore empty fields - probably the only field anyway - if len(requirement) == 0: - continue + if len(requirement) == 0: continue - requirement_type_name, requirement_object_id = core.Object.split_name(requirement) - # Instantiate type which fails if type does not exist - requirement_type = core.Type(self.type_base_path, requirement_type_name) - - if requirement_object_id: - # Validate object_id if any - core.Object.validate_object_id(requirement_object_id) - elif not requirement_type.is_singleton: - # Only singeltons have no object_id - raise IllegalRequirementError(requirement, "Missing object_id and type is not a singleton.") + # Raises an error, if object cannot be created + cdist_object = self.cdist_object.object_from_name(requirement) self.log.debug("Recording requirement: " + requirement) - self.cdist_object.requirements.append(requirement) - # Record / Append source - self.cdist_object.source.append(self.object_source) + # Save the sanitised version, not the user supplied one + # (__file//bar => __file/bar) + # This ensures pattern matching is done against sanitised list + self.cdist_object.requirements.append(cdist_object.name) def record_auto_requirements(self): """An object shall automatically depend on all objects that it defined in it's type manifest. diff --git a/lib/cdist/exec/local.py b/lib/cdist/exec/local.py index 613f5cf2..cdf06205 100644 --- a/lib/cdist/exec/local.py +++ b/lib/cdist/exec/local.py @@ -143,7 +143,7 @@ class Local(object): def link_emulator(self, exec_path): """Link emulator to types""" src = os.path.abspath(exec_path) - for cdist_type in core.Type.list_types(self.type_path): + for cdist_type in core.CdistType.list_types(self.type_path): dst = os.path.join(self.bin_path, cdist_type.name) self.log.debug("Linking emulator: %s to %s", src, dst) diff --git a/lib/cdist/exec/remote.py b/lib/cdist/exec/remote.py index 87db7273..69dc5dda 100644 --- a/lib/cdist/exec/remote.py +++ b/lib/cdist/exec/remote.py @@ -20,8 +20,6 @@ # # -# FIXME: common base class with Local? - import io import os import sys @@ -148,18 +146,17 @@ class Remote(object): os_environ = os.environ.copy() os_environ['__target_host'] = self.target_host + self.log.debug("Remote run script: %s", command) + # can't pass environment to remote side, so prepend command with # variable declarations if env: + self.log.debug("Remote run script env: %s", env) command.extend(["%s=%s" % item for item in env.items()]) command.extend(["/bin/sh", "-e"]) command.append(script) - self.log.debug("Remote run script: %s", command) - if env: - self.log.debug("Remote run script env: %s", env) - try: if return_output: return subprocess.check_output(command, env=os_environ).decode() diff --git a/lib/cdist/resolver.py b/lib/cdist/resolver.py index 24a5e496..368c9eb8 100644 --- a/lib/cdist/resolver.py +++ b/lib/cdist/resolver.py @@ -125,7 +125,7 @@ class DependencyResolver(object): resolved.append(cdist_object) unresolved.remove(cdist_object) except RequirementNotFoundError as e: - raise cdist.Error(cdist_object.name + " requires non-existing " + e.requirement) + raise cdist.CdistObjectError(cdist_object, "requires non-existing " + e.requirement) def __iter__(self): """Iterate over all unique objects while resolving dependencies. diff --git a/lib/cdist/test/code/__init__.py b/lib/cdist/test/code/__init__.py index 2f061086..dc701cce 100644 --- a/lib/cdist/test/code/__init__.py +++ b/lib/cdist/test/code/__init__.py @@ -54,8 +54,8 @@ class CodeTestCase(test.CdistTestCase): self.code = code.Code(self.target_host, self.local, self.remote) - self.cdist_type = core.Type(self.local.type_path, '__dump_environment') - self.cdist_object = core.Object(self.cdist_type, self.local.object_path, 'whatever') + self.cdist_type = core.CdistType(self.local.type_path, '__dump_environment') + self.cdist_object = core.CdistObject(self.cdist_type, self.local.object_path, 'whatever') self.cdist_object.create() self.log = logging.getLogger("cdist") diff --git a/lib/cdist/test/emulator/__init__.py b/lib/cdist/test/emulator/__init__.py index e67bed4a..f1ea5832 100644 --- a/lib/cdist/test/emulator/__init__.py +++ b/lib/cdist/test/emulator/__init__.py @@ -79,7 +79,7 @@ class EmulatorTestCase(test.CdistTestCase): os.environ.update(self.env) os.environ['require'] = '__file' emu = emulator.Emulator(argv) - self.assertRaises(emulator.IllegalRequirementError, emu.run) + self.assertRaises(core.IllegalObjectIdError, emu.run) def test_singleton_object_requirement(self): argv = ['__file', '/tmp/foobar'] @@ -119,8 +119,8 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase): def test_autorequire(self): initial_manifest = os.path.join(self.local.manifest_path, "init") self.manifest.run_initial_manifest(initial_manifest) - cdist_type = core.Type(self.local.type_path, '__saturn') - cdist_object = core.Object(cdist_type, self.local.object_path, 'singleton') + cdist_type = core.CdistType(self.local.type_path, '__saturn') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'singleton') self.manifest.run_type_manifest(cdist_object) expected = ['__planet/Saturn', '__moon/Prometheus'] self.assertEqual(sorted(cdist_object.requirements), sorted(expected)) @@ -156,6 +156,6 @@ class ArgumentsWithDashesTestCase(test.CdistTestCase): emu = emulator.Emulator(argv) emu.run() - cdist_type = core.Type(self.local.type_path, '__arguments_with_dashes') - cdist_object = core.Object(cdist_type, self.local.object_path, 'some-id') + cdist_type = core.CdistType(self.local.type_path, '__arguments_with_dashes') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'some-id') self.assertTrue('with-dash' in cdist_object.parameters) diff --git a/lib/cdist/test/explorer/__init__.py b/lib/cdist/test/explorer/__init__.py index cafe34fc..257ad8a9 100644 --- a/lib/cdist/test/explorer/__init__.py +++ b/lib/cdist/test/explorer/__init__.py @@ -83,19 +83,19 @@ class ExplorerClassTestCase(test.CdistTestCase): shutil.rmtree(out_path) def test_list_type_explorer_names(self): - cdist_type = core.Type(self.local.type_path, '__test_type') + cdist_type = core.CdistType(self.local.type_path, '__test_type') expected = cdist_type.explorers self.assertEqual(self.explorer.list_type_explorer_names(cdist_type), expected) def test_transfer_type_explorers(self): - cdist_type = core.Type(self.local.type_path, '__test_type') + cdist_type = core.CdistType(self.local.type_path, '__test_type') self.explorer.transfer_type_explorers(cdist_type) source = os.path.join(self.local.type_path, cdist_type.explorer_path) destination = os.path.join(self.remote.type_path, cdist_type.explorer_path) self.assertEqual(os.listdir(source), os.listdir(destination)) def test_transfer_type_explorers_only_once(self): - cdist_type = core.Type(self.local.type_path, '__test_type') + cdist_type = core.CdistType(self.local.type_path, '__test_type') # first transfer self.explorer.transfer_type_explorers(cdist_type) source = os.path.join(self.local.type_path, cdist_type.explorer_path) @@ -109,8 +109,8 @@ class ExplorerClassTestCase(test.CdistTestCase): self.assertFalse(os.listdir(destination)) def test_transfer_object_parameters(self): - cdist_type = core.Type(self.local.type_path, '__test_type') - cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever') + cdist_type = core.CdistType(self.local.type_path, '__test_type') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever') cdist_object.create() cdist_object.parameters = {'first': 'first value', 'second': 'second value'} self.explorer.transfer_object_parameters(cdist_object) @@ -119,15 +119,15 @@ class ExplorerClassTestCase(test.CdistTestCase): self.assertEqual(sorted(os.listdir(source)), sorted(os.listdir(destination))) def test_run_type_explorer(self): - cdist_type = core.Type(self.local.type_path, '__test_type') - cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever') + cdist_type = core.CdistType(self.local.type_path, '__test_type') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever') self.explorer.transfer_type_explorers(cdist_type) output = self.explorer.run_type_explorer('world', cdist_object) self.assertEqual(output, 'hello\n') def test_run_type_explorers(self): - cdist_type = core.Type(self.local.type_path, '__test_type') - cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever') + cdist_type = core.CdistType(self.local.type_path, '__test_type') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever') cdist_object.create() self.explorer.run_type_explorers(cdist_object) self.assertEqual(cdist_object.explorers, {'world': 'hello'}) diff --git a/lib/cdist/test/manifest/__init__.py b/lib/cdist/test/manifest/__init__.py index 2383adf7..a188c788 100644 --- a/lib/cdist/test/manifest/__init__.py +++ b/lib/cdist/test/manifest/__init__.py @@ -79,8 +79,8 @@ class ManifestTestCase(test.CdistTestCase): self.assertEqual(output_dict['__manifest'], self.local.manifest_path) def test_type_manifest_environment(self): - cdist_type = core.Type(self.local.type_path, '__dump_environment') - cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever') + cdist_type = core.CdistType(self.local.type_path, '__dump_environment') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever') handle, output_file = self.mkstemp(dir=self.temp_dir) os.close(handle) os.environ['__cdist_test_out'] = output_file diff --git a/lib/cdist/test/object/__init__.py b/lib/cdist/test/object/__init__.py index f199ffb5..3a91f709 100644 --- a/lib/cdist/test/object/__init__.py +++ b/lib/cdist/test/object/__init__.py @@ -25,6 +25,8 @@ import shutil from cdist import test from cdist import core +import cdist + import os.path as op my_dir = op.abspath(op.dirname(__file__)) fixtures = op.join(my_dir, 'fixtures') @@ -34,48 +36,48 @@ type_base_path = op.join(fixtures, 'type') class ObjectClassTestCase(test.CdistTestCase): def test_list_object_names(self): - object_names = list(core.Object.list_object_names(object_base_path)) + object_names = list(core.CdistObject.list_object_names(object_base_path)) self.assertEqual(object_names, ['__first/man', '__second/on-the', '__third/moon']) def test_list_type_names(self): - type_names = list(core.Object.list_type_names(object_base_path)) + type_names = list(cdist.core.CdistObject.list_type_names(object_base_path)) self.assertEqual(type_names, ['__first', '__second', '__third']) def test_list_objects(self): - objects = list(core.Object.list_objects(object_base_path, type_base_path)) + objects = list(core.CdistObject.list_objects(object_base_path, type_base_path)) objects_expected = [ - core.Object(core.Type(type_base_path, '__first'), object_base_path, 'man'), - core.Object(core.Type(type_base_path, '__second'), object_base_path, 'on-the'), - core.Object(core.Type(type_base_path, '__third'), object_base_path, 'moon'), + core.CdistObject(core.CdistType(type_base_path, '__first'), object_base_path, 'man'), + core.CdistObject(core.CdistType(type_base_path, '__second'), object_base_path, 'on-the'), + core.CdistObject(core.CdistType(type_base_path, '__third'), object_base_path, 'moon'), ] self.assertEqual(objects, objects_expected) class ObjectIdTestCase(test.CdistTestCase): - def test_object_id_starts_with_slash(self): - cdist_type = core.Type(type_base_path, '__third') - illegal_object_id = '/object_id/may/not/start/with/slash' + def test_object_id_contains_double_slash(self): + cdist_type = core.CdistType(type_base_path, '__third') + illegal_object_id = '/object_id//may/not/contain/double/slash' with self.assertRaises(core.IllegalObjectIdError): - core.Object(cdist_type, object_base_path, illegal_object_id) + core.CdistObject(cdist_type, object_base_path, illegal_object_id) def test_object_id_contains_object_marker(self): - cdist_type = core.Type(type_base_path, '__third') + cdist_type = core.CdistType(type_base_path, '__third') illegal_object_id = 'object_id/may/not/contain/%s/anywhere' % core.OBJECT_MARKER with self.assertRaises(core.IllegalObjectIdError): - core.Object(cdist_type, object_base_path, illegal_object_id) + core.CdistObject(cdist_type, object_base_path, illegal_object_id) def test_object_id_contains_object_marker_string(self): - cdist_type = core.Type(type_base_path, '__third') + cdist_type = core.CdistType(type_base_path, '__third') illegal_object_id = 'object_id/may/contain_%s_in_filename' % core.OBJECT_MARKER - core.Object(cdist_type, object_base_path, illegal_object_id) + core.CdistObject(cdist_type, object_base_path, illegal_object_id) # if we get here, the test passed class ObjectTestCase(test.CdistTestCase): def setUp(self): - self.cdist_type = core.Type(type_base_path, '__third') - self.cdist_object = core.Object(self.cdist_type, object_base_path, 'moon') + self.cdist_type = core.CdistType(type_base_path, '__third') + self.cdist_object = core.CdistObject(self.cdist_type, object_base_path, 'moon') def tearDown(self): self.cdist_object.changed = False @@ -159,16 +161,16 @@ class ObjectTestCase(test.CdistTestCase): self.assertEqual(self.cdist_object.state, '') def test_state_prepared(self): - self.cdist_object.state = core.Object.STATE_PREPARED - self.assertEqual(self.cdist_object.state, core.Object.STATE_PREPARED) + self.cdist_object.state = core.CdistObject.STATE_PREPARED + self.assertEqual(self.cdist_object.state, core.CdistObject.STATE_PREPARED) def test_state_running(self): - self.cdist_object.state = core.Object.STATE_RUNNING - self.assertEqual(self.cdist_object.state, core.Object.STATE_RUNNING) + self.cdist_object.state = core.CdistObject.STATE_RUNNING + self.assertEqual(self.cdist_object.state, core.CdistObject.STATE_RUNNING) def test_state_done(self): - self.cdist_object.state = core.Object.STATE_DONE - self.assertEqual(self.cdist_object.state, core.Object.STATE_DONE) + self.cdist_object.state = core.CdistObject.STATE_DONE + self.assertEqual(self.cdist_object.state, core.CdistObject.STATE_DONE) def test_source(self): self.assertEqual(list(self.cdist_object.source), []) @@ -195,6 +197,6 @@ class ObjectTestCase(test.CdistTestCase): self.cdist_object.code_remote = 'Hello World' other_name = '__first/man' other_object = self.cdist_object.object_from_name(other_name) - self.assertTrue(isinstance(other_object, core.Object)) - self.assertEqual(other_object.type.name, '__first') + self.assertTrue(isinstance(other_object, core.CdistObject)) + self.assertEqual(other_object.cdist_type.name, '__first') self.assertEqual(other_object.object_id, 'man') diff --git a/lib/cdist/test/resolver/__init__.py b/lib/cdist/test/resolver/__init__.py index cca058a4..ae8f6915 100644 --- a/lib/cdist/test/resolver/__init__.py +++ b/lib/cdist/test/resolver/__init__.py @@ -37,7 +37,7 @@ type_base_path = op.join(fixtures, 'type') class ResolverTestCase(test.CdistTestCase): def setUp(self): - self.objects = list(core.Object.list_objects(object_base_path, type_base_path)) + self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path)) self.object_index = dict((o.name, o) for o in self.objects) self.dependency_resolver = resolver.DependencyResolver(self.objects) diff --git a/lib/cdist/test/type/__init__.py b/lib/cdist/test/type/__init__.py index 7bb8654c..2ef14a4c 100644 --- a/lib/cdist/test/type/__init__.py +++ b/lib/cdist/test/type/__init__.py @@ -33,115 +33,115 @@ class TypeTestCase(test.CdistTestCase): def test_list_type_names(self): base_path = op.join(fixtures, 'list_types') - type_names = core.Type.list_type_names(base_path) + type_names = core.CdistType.list_type_names(base_path) self.assertEqual(type_names, ['__first', '__second', '__third']) def test_list_types(self): base_path = op.join(fixtures, 'list_types') - types = list(core.Type.list_types(base_path)) + types = list(core.CdistType.list_types(base_path)) types_expected = [ - core.Type(base_path, '__first'), - core.Type(base_path, '__second'), - core.Type(base_path, '__third'), + core.CdistType(base_path, '__first'), + core.CdistType(base_path, '__second'), + core.CdistType(base_path, '__third'), ] self.assertEqual(types, types_expected) def test_only_one_instance(self): base_path = fixtures - cdist_type1 = core.Type(base_path, '__name_path') - cdist_type2 = core.Type(base_path, '__name_path') + cdist_type1 = core.CdistType(base_path, '__name_path') + cdist_type2 = core.CdistType(base_path, '__name_path') self.assertEqual(id(cdist_type1), id(cdist_type2)) def test_nonexistent_type(self): base_path = fixtures - self.assertRaises(core.NoSuchTypeError, core.Type, base_path, '__i-dont-exist') + self.assertRaises(core.NoSuchTypeError, core.CdistType, base_path, '__i-dont-exist') def test_name(self): base_path = fixtures - cdist_type = core.Type(base_path, '__name_path') + cdist_type = core.CdistType(base_path, '__name_path') self.assertEqual(cdist_type.name, '__name_path') def test_path(self): base_path = fixtures - cdist_type = core.Type(base_path, '__name_path') + cdist_type = core.CdistType(base_path, '__name_path') self.assertEqual(cdist_type.path, '__name_path') def test_base_path(self): base_path = fixtures - cdist_type = core.Type(base_path, '__name_path') + cdist_type = core.CdistType(base_path, '__name_path') self.assertEqual(cdist_type.base_path, base_path) def test_absolute_path(self): base_path = fixtures - cdist_type = core.Type(base_path, '__name_path') + cdist_type = core.CdistType(base_path, '__name_path') self.assertEqual(cdist_type.absolute_path, os.path.join(base_path, '__name_path')) def test_manifest_path(self): base_path = fixtures - cdist_type = core.Type(base_path, '__name_path') + cdist_type = core.CdistType(base_path, '__name_path') self.assertEqual(cdist_type.manifest_path, os.path.join('__name_path', 'manifest')) def test_explorer_path(self): base_path = fixtures - cdist_type = core.Type(base_path, '__name_path') + cdist_type = core.CdistType(base_path, '__name_path') self.assertEqual(cdist_type.explorer_path, os.path.join('__name_path', 'explorer')) def test_gencode_local_path(self): base_path = fixtures - cdist_type = core.Type(base_path, '__name_path') + cdist_type = core.CdistType(base_path, '__name_path') self.assertEqual(cdist_type.gencode_local_path, os.path.join('__name_path', 'gencode-local')) def test_gencode_remote_path(self): base_path = fixtures - cdist_type = core.Type(base_path, '__name_path') + cdist_type = core.CdistType(base_path, '__name_path') self.assertEqual(cdist_type.gencode_remote_path, os.path.join('__name_path', 'gencode-remote')) def test_singleton_is_singleton(self): base_path = fixtures - cdist_type = core.Type(base_path, '__singleton') + cdist_type = core.CdistType(base_path, '__singleton') self.assertTrue(cdist_type.is_singleton) def test_not_singleton_is_singleton(self): base_path = fixtures - cdist_type = core.Type(base_path, '__not_singleton') + cdist_type = core.CdistType(base_path, '__not_singleton') self.assertFalse(cdist_type.is_singleton) def test_install_is_install(self): base_path = fixtures - cdist_type = core.Type(base_path, '__install') + cdist_type = core.CdistType(base_path, '__install') self.assertTrue(cdist_type.is_install) def test_not_install_is_install(self): base_path = fixtures - cdist_type = core.Type(base_path, '__not_install') + cdist_type = core.CdistType(base_path, '__not_install') self.assertFalse(cdist_type.is_install) def test_with_explorers(self): base_path = fixtures - cdist_type = core.Type(base_path, '__with_explorers') + cdist_type = core.CdistType(base_path, '__with_explorers') self.assertEqual(cdist_type.explorers, ['whatever']) def test_without_explorers(self): base_path = fixtures - cdist_type = core.Type(base_path, '__without_explorers') + cdist_type = core.CdistType(base_path, '__without_explorers') self.assertEqual(cdist_type.explorers, []) def test_with_required_parameters(self): base_path = fixtures - cdist_type = core.Type(base_path, '__with_required_parameters') + cdist_type = core.CdistType(base_path, '__with_required_parameters') self.assertEqual(cdist_type.required_parameters, ['required1', 'required2']) def test_without_required_parameters(self): base_path = fixtures - cdist_type = core.Type(base_path, '__without_required_parameters') + cdist_type = core.CdistType(base_path, '__without_required_parameters') self.assertEqual(cdist_type.required_parameters, []) def test_with_optional_parameters(self): base_path = fixtures - cdist_type = core.Type(base_path, '__with_optional_parameters') + cdist_type = core.CdistType(base_path, '__with_optional_parameters') self.assertEqual(cdist_type.optional_parameters, ['optional1', 'optional2']) def test_without_optional_parameters(self): base_path = fixtures - cdist_type = core.Type(base_path, '__without_optional_parameters') + cdist_type = core.CdistType(base_path, '__without_optional_parameters') self.assertEqual(cdist_type.optional_parameters, []) diff --git a/other/types_submitted_for_inclusion/__rsyncer/README.inclusion b/other/types_submitted_for_inclusion/__rsyncer/README.inclusion new file mode 100644 index 00000000..eff48995 --- /dev/null +++ b/other/types_submitted_for_inclusion/__rsyncer/README.inclusion @@ -0,0 +1,22 @@ +Description: + + Type that supports transfer of huge data, which is a general problem in + configuration management systems. + + Good solution using standardised rsync approach. + +Problem: + + Uses root@$__target_host:$destination notation for transfer. + This breaks the concept of being able to replace __remote_exec and + __remote_copy and then doing chroot or different stuff. + + This breaks for instance, if __remote_copy = cp and the destination is + a local chroot. + +Solutions: + + - Have cdist provide support for rsync syntax? + - Integrate __rsyncer more in line with philosohpy of other components + - Think about the general way of __rsyncer and what cdist would need + to provide for general solution. diff --git a/other/types_submitted_for_inclusion/__rsyncer/gencode-local b/other/types_submitted_for_inclusion/__rsyncer/gencode-local new file mode 100755 index 00000000..9635707b --- /dev/null +++ b/other/types_submitted_for_inclusion/__rsyncer/gencode-local @@ -0,0 +1,48 @@ +#!/bin/sh +# +# Copyright (C) 2011 Daniel Maher (phrawzty+cdist at gmail.com) +# +# This file is part of cdist (https://github.com/telmich/cdist/). +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +source="$(cat "$__object/parameter/source")" + +if [ -f "$__object/parameter/destination" ]; then + destination="$(cat "$__object/parameter/destination")" +else + destination="/$__object_id" +fi + +# The system binary is probably ok, but if not... +if [ -f "$__object/parameter/rsyncbin" ]; then + rsyncbin="$(cat "$__object/parameter/rsyncbin")" +else + rsyncbin=`which rsync` +fi + +args='-a' + +# If the --delete argument should be passed to rsync. +if [ -f "$__object/parameter/delete" ]; then + args="$args --delete" +fi + +# FIXME: +# - using root@ may break - find a good way to avoid this +# - align with __remote_{exec,copy} variables? + +# Run rsync (locally). +echo "$rsyncbin $args $source root@$__target_host:$destination" diff --git a/other/types_submitted_for_inclusion/__rsyncer/man.text b/other/types_submitted_for_inclusion/__rsyncer/man.text new file mode 100644 index 00000000..6fab9fd3 --- /dev/null +++ b/other/types_submitted_for_inclusion/__rsyncer/man.text @@ -0,0 +1,62 @@ +cdist-type__rsyncer(7) +====================== +Daniel Maher + + +NAME +---- +cdist-type__rsyncer - Use rsync to copy files. + + +DESCRIPTION +----------- +This type is used to trigger rsync to copy files from the machine running cdist +(source) to the target machine in question (destination). The likely usage is +the rapid deployment of full directory trees, the cohorency of which can be +guarunteed with the optional --delete argument, which will remove any files +from the destination which are not present on the source. + + +REQUIRED PARAMETERS +------------------- +source:: + The full path of the source from which to copy. This is passed directly + to rsync. + + +OPTIONAL PARAMETERS +------------------- +destination:: + The full path of the destination. This is passed directly to rsync. + Default: object_id + +delete:: + If true, remove files from destination which are not in source. This is + effectively the --delete argument of rsync. + Default: false + +rsyncbin:: + Specify the full path to the rsync binary. + Default: `which rsync` + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# Basic example +__rsyncer '/home/foo' --source '/opt/dist/foo' + +# Fancier example +__rsyncer FOO --source '/opt/dist/foo' --destination '/home/foo/' --delete true +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2011 Daniel Maher. Free use of this software is granted under +the terms of the GNU General Public License version 3 (GPLv3). diff --git a/other/types_submitted_for_inclusion/__rsyncer/parameter/optional b/other/types_submitted_for_inclusion/__rsyncer/parameter/optional new file mode 100644 index 00000000..3bcb4dc7 --- /dev/null +++ b/other/types_submitted_for_inclusion/__rsyncer/parameter/optional @@ -0,0 +1,3 @@ +destination +delete +rsyncbin diff --git a/other/types_submitted_for_inclusion/__rsyncer/parameter/required b/other/types_submitted_for_inclusion/__rsyncer/parameter/required new file mode 100644 index 00000000..5a18cd2f --- /dev/null +++ b/other/types_submitted_for_inclusion/__rsyncer/parameter/required @@ -0,0 +1 @@ +source