stephan/cdist
1
0
Fork 0
forked from ungleich-public/cdist
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'master' into feature_init_process

Browse source 
Conflicts:
	doc/changelog
	doc/dev/todo/niconext
	doc/man/cdist-reference.text.sh
	lib/cdist/core/explorer.py

doc/man/cdist-reference.text.sh documents better reachability of
variables - also suitable for master?

Signed-off-by: Nico Schottelius <nico@brief.schottelius.org>
This commit is contained in:
Nico Schottelius 2012-02-14 13:53:16 +01:00
commit 2b380b7dc1
55 changed files with 921 additions and 360 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
README
View file

@ -51,9 +51,10 @@ UNIX, simplicity, familar environment | cdist is configured in POSIX shell
### Documentation
The cdist documentation is included as manpages in the distribution.
You can browse the documentation online as well:
* You can [browse the documentation of the latest version online](man) as well.
* Have a look at the [given speeches](speeches)
* [latest version](man/latest)
* [all versions (>= 2.0.4)](man)
### OS support
@ -88,7 +89,7 @@ cdist was tested or is know to run on at least
## Installation
### Preperation
### Preparation
Ensure you have Python 3.2 installed on the machine you use to **deploy to the targets**
(the ***source host***).
@ -135,7 +136,13 @@ If you want to ensure nothing breaks you must set back the python version to wha
#### Max OS X
Ensure you have port installed and configured (http://www.macports.org/install.php).
You can choose between Homebrew and Macports, either way works:
[Homebrew](http://mxcl.github.com/homebrew/) variant:
brew install python3
[Macports](http://www.macports.org/install.php) variant:
port install python32
ln -s /opt/local/bin/python3.2 /opt/local/bin/python3

9
build
View file

@ -85,7 +85,7 @@ case "$1" in
;;
release)
"$0" clean && "$0" man && "$0" web
./doc/dev/releasechecklist
;;
speeches)
@ -99,7 +99,7 @@ case "$1" in
webmain)
cp README ${WEBPAGE}
cd ${WEBDIR} && git commit -m "cdist update" ${WEBPAGE}
cd ${WEBDIR} && git commit -m "cdist main update" ${WEBPAGE}
cd ${WEBDIR} && make pub
;;
@ -119,6 +119,11 @@ case "$1" in
cd ${WEBDIR} && git add ${WEBBASE}
cd ${WEBDIR} && git commit -m "cdist update" ${WEBBASE} ${WEBPAGE}
cd ${WEBDIR} && make pub
# Fix ikiwiki, which does not like symlinks for pseudo security
ssh tee.schottelius.org \
"cd /home/services/www/nico/www.nico.schottelius.org/www/software/cdist/man &&
rm -f latest && ln -sf "$version" latest"
;;
p|pu|pub)

66
conf/explorer/os
View file

@ -18,13 +18,14 @@
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# All os variables are lower case
#
# All os variables are lower case. Keep this file in alphabetical
# order by os variable except in cases where order otherwise matters,
# in which case keep the primary os and its derivatives together in
# a block (see Debian and Redhat examples below).
#
# Ubuntu is also Debian, thus return if Ubuntu was found
if grep -q ^DISTRIB_ID=Ubuntu /etc/lsb-release 2>/dev/null; then
echo ubuntu
if grep -q ^Amazon /etc/system-release 2>/dev/null; then
echo amazon
exit 0
fi
@ -33,45 +34,52 @@ if [ -f /etc/arch-release ]; then
exit 0
fi
if [ -f /etc/cdist-preos ]; then
echo cdist-preos
exit 0
fi
### Debian and derivatives
if grep -q ^DISTRIB_ID=Ubuntu /etc/lsb-release 2>/dev/null; then
echo ubuntu
exit 0
fi
if [ -f /etc/debian_version ]; then
echo debian
exit 0
fi
###
if [ -f /etc/gentoo-release ]; then
echo gentoo
exit 0
fi
# Fedora is also Redhat, thus return before redhat!
if grep -q ^Fedora /etc/redhat-release 2>/dev/null; then
echo fedora
exit 0
fi
# CentOS is also based on Redhat, thus return before redhat!
if grep -q ^CentOS /etc/redhat-release 2>/dev/null; then
echo centos
exit 0
fi
if [ -f /etc/redhat-release ]; then
echo redhat
exit 0
fi
if [ -f /etc/SuSE-release ]; then
echo suse
exit 0
fi
if [ -f /etc/owl-release ]; then
echo owl
exit 0
fi
if [ -f /etc/cdist-preos ]; then
echo cdist-preos
### Redhat and derivatives
if grep -q ^CentOS /etc/redhat-release 2>/dev/null; then
echo centos
exit 0
fi
if grep -q ^Fedora /etc/redhat-release 2>/dev/null; then
echo fedora
exit 0
fi
if [ -f /etc/redhat-release ]; then
echo redhat
exit 0
fi
###
if [ -f /etc/SuSE-release ]; then
echo suse
exit 0
fi

3
conf/explorer/os_version
View file

@ -23,6 +23,9 @@
#
case "$($__explorer/os)" in
amazon)
cat /etc/system-release
;;
archlinux)
# empty, but well...
cat /etc/arch-release

11
conf/type/__file/gencode-remote
View file

@ -34,11 +34,6 @@ case "$state_should" in
fi
fi
# Mode settings
if [ -f "$__object/parameter/mode" ]; then
echo chmod \"$(cat "$__object/parameter/mode")\" \"$destination\"
fi
# Group
if [ -f "$__object/parameter/group" ]; then
echo chgrp \"$(cat "$__object/parameter/group")\" \"$destination\"
@ -48,6 +43,12 @@ case "$state_should" in
if [ -f "$__object/parameter/owner" ]; then
echo chown \"$(cat "$__object/parameter/owner")\" \"$destination\"
fi
# Mode - needs to happen last as a chown/chgrp can alter mode by
# clearing S_ISUID and S_ISGID bits (see chown(2))
if [ -f "$__object/parameter/mode" ]; then
echo chmod \"$(cat "$__object/parameter/mode")\" \"$destination\"
fi
;;
absent)

2
conf/type/__group/gencode-remote
View file

@ -29,7 +29,7 @@ if grep -q "^${name}:" "$__object/explorer/group"; then
for property in $(ls .); do
new_value="$(cat "$property")"
case "$key" in
case "$property" in
password)
current_value="$(awk -F: '{ print $2 }' < "$__object/explorer/gshadow")"
;;

53
conf/type/__key_value/explorer/state Executable file
View file

@ -0,0 +1,53 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
key="$(cat "$__object/parameter/key" 2>/dev/null \
|| echo "$__object_id")"
state="$(cat "$__object/parameter/state" 2>/dev/null \
|| echo "present")"
file="$(cat "$__object/parameter/file")"
delimiter="$(cat "$__object/parameter/delimiter")"
value="$(cat "$__object/parameter/value" 2>/dev/null \
|| echo "__CDIST_NOTSET__")"
case "$state" in
absent)
if grep -q -E "^$key$delimiter+" "$file"; then
# if the key exists, with whatever value, we will have to remove it
# so report it as present
echo present
else
# key does not exist
echo absent
fi
;;
present)
if grep -q -E "^$key$delimiter+$value$" "$file"; then
# key exists and value is same
echo present
elif grep -q -E "^$key$delimiter+" "$file"; then
# key exists, but value is empty or different
echo wrongvalue
else
# key does not exist
echo absent
fi
;;
esac

55
conf/type/__key_value/gencode-remote
View file

@ -18,35 +18,40 @@
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
value_is="$(cat "$__object/explorer/value")"
value_should="$(cat "$__object/parameter/value")"
key="$(cat "$__object/parameter/key")"
file="$(cat "$__object/parameter/file")"
delimiter="$(cat "$__object/parameter/delimiter")"
value="$(cat "$__object/parameter/value")"
if [ "$value_is" != "$value_should" ]; then
case "$value_is" in
__NOTSET__)
# add key and value
echo "echo \"${key}${delimiter}${value_should}\" >> \"$file\""
state_is="$(cat "$__object/explorer/state")"
state_should="$(cat "$__object/parameter/state")"
if [ "$state_is" = "$state_should" ]; then
# nothing to do
exit 0
fi
case "$state_should" in
absent)
# remove lines starting with key
echo "sed -i '/^$key\($delimiter\+\)/d' \"$file\""
;;
present)
case "$state_is" in
absent)
# add new key and value
echo "echo \"${key}${delimiter}${value}\" >> \"$file\""
;;
wrongvalue)
# change exisiting value
echo "sed -i \"s|^$key\($delimiter\+\).*|$key\1$value|\" \"$file\""
;;
*)
if [ "$value_should" = '__NOTSET__' ]; then
# remove key and value
cat << DONE
sed -i '/^${key}/d' "$file"
DONE
else
# change value
cat << DONE
awk -F "$delimiter" '
/${key}${delimiter}*/{gsub("$value_is", "$value_should")};{print}' "$file" > "${file}+" \
&& mv "${file}+" "$file"
DONE
fi
;;
echo "Unknown explorer state: $state_is" >&2
exit 1
esac
;;
*)
echo "Unknown state: $state_should" >&2
exit 1
esac
fi

11
conf/type/__key_value/man.text
View file

@ -16,9 +16,6 @@ file.
REQUIRED PARAMETERS
-------------------
value::
The value for the key. Setting the value to `__NOTSET__` will remove the key
from the file.
file::
The file to operate on.
delimiter::
@ -27,8 +24,13 @@ delimiter::
OPTIONAL PARAMETERS
-------------------
state::
present or absent, defaults to present. If present, sets the key to value,
if absent, removes the key from the file.
key::
The key to change. Defaults to object_id.
value::
The value for the key. Optional if state=absent, required otherwise.
EXAMPLES
@ -45,6 +47,9 @@ __key_value my-fancy-id --file /etc/login.defs --key SYS_UID_MAX --value 666 \
# Enable packet forwarding
__key_value net.ipv4.ip_forward --file /etc/sysctl.conf --value 1 \
--delimiter '='
# Remove existing key/value
__key_value LEGACY_KEY --file /etc/somefile --state absent --delimiter '='
--------------------------------------------------------------------------------

14
conf/type/__key_value/manifest
View file

@ -18,9 +18,13 @@
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
if [ -f "$__object/parameter/key" ]; then
key="$(cat "$__object/parameter/key")"
else
echo "$__object_id" > "$__object/parameter/key"
fi
# set defaults
key="$(cat "$__object/parameter/key" 2>/dev/null \
|| echo "$__object_id" | tee "$__object/parameter/key")"
state="$(cat "$__object/parameter/state" 2>/dev/null \
|| echo "present" | tee "$__object/parameter/state")"
if [ "$state" = "present" -a ! -f "$__object/parameter/value" ]; then
echo "Missing required parameter 'value'" >&2
exit 1
fi

2
conf/type/__key_value/parameter/optional
View file

@ -1 +1,3 @@
key
value
state

1
conf/type/__key_value/parameter/required
View file

@ -1,3 +1,2 @@
value
file
delimiter

62
conf/type/__link/explorer/state Executable file
View file

@ -0,0 +1,62 @@
#!/bin/sh
#
# 2012 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
destination="/$__object_id"
type="$(cat "$__object/parameter/type")"
source="$(cat "$__object/parameter/source")"
# no destination? -> state is absent
if [ ! -e "$destination" ]; then
echo absent
exit 0
fi
destination_dir="${destination%/*}"
case "$type" in
symbolic)
cd "$destination_dir"
source_is=$(ls -l "$destination" | sed 's/.*-> //g')
if [ -h "$destination" -a "$source_is" = "$source" ]; then
echo present
else
echo absent
fi
;;
hard)
cd "$destination_dir"
# check source relative to destination_dir
if [ ! -e "$source" ]; then
echo sourcemissing
exit 0
fi
destination_inode=$(ls -i "$destination" | awk '{print $1}')
source_inode=$(ls -i "$source" | awk '{print $1}')
if [ "$destination_inode" -eq "$source_inode" ]; then
echo present
else
echo absent
fi
;;
*)
echo "Unknown type: $type" >&2
exit 1
;;
esac

7
conf/type/__link/gencode-remote
View file

@ -39,7 +39,14 @@ case "$type" in
;;
esac
state_is="$(cat "$__object/explorer/state")"
state_should="$(cat "$__object/parameter/state")"
if [ "$state_should" = "$state_is" ]; then
# nothing to do
exit 0
fi
case "$state_should" in
present)
echo ln ${lnopt} -f \"$source\" \"$destination\"

2
conf/type/__package/manifest
View file

@ -33,7 +33,7 @@ else
archlinux) type="pacman" ;;
debian|ubuntu) type="apt" ;;
gentoo) type="emerge" ;;
fedora|redhat|centos) type="yum" ;;
amazon|centos|fedora|redhat) type="yum" ;;
*)
echo "Don't know how to manage packages on: $os" >&2
exit 1

49
conf/type/__package_pip/explorer/state Normal file
View file

@ -0,0 +1,49 @@
#!/bin/sh
#
# 2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# Development supported by Local AG (www.local.ch)
#
nameparam="$__object/parameter/name"
if [ -f "$nameparam" ]; then
name=$(cat "$nameparam")
else
name="$__object_id"
fi
pipparam="$__object/parameter/pip"
if [ -f "$pipparam" ]; then
pip=$(cat "$pipparam")
else
pip="pip"
fi
# If there is no pip, it may get created from somebody else.
# If it will be created, there is probably no package installed.
if ! command -v "$pip" >/dev/null 2>&1; then
echo absent
else
if "$pip" freeze | grep -i -q "^$name=="; then
echo present
else
echo absent
fi
fi

37
conf/type/__key_value/explorer/value → conf/type/__package_pip/gencode-remote Executable file → Normal file
View file

@ -1,6 +1,6 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
@ -18,20 +18,33 @@
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# Get the current value of key or __NOTSET__ if the key doesn't exist.
# Development supported by Local AG (www.local.ch)
#
if [ -f "$__object/parameter/key" ]; then
key="$(cat "$__object/parameter/key")"
state_is=$(cat "$__object/explorer/state")
state_should=$(cat "$__object/parameter/state")
[ "$state_is" = "$state_should" ] && exit 0
nameparam="$__object/parameter/name"
if [ -f "$nameparam" ]; then
name=$(cat "$nameparam")
else
key="$__object_id"
name="$__object_id"
fi
file="$(cat "$__object/parameter/file")"
delimiter="$(cat "$__object/parameter/delimiter")"
awk -F "$delimiter" '
BEGIN { found=0 }
/^'$key'/ { print $2; found=1 }
END { if (found) exit 0; else exit 1 }' "$file" \
|| echo "__NOTSET__"
pipparam="$__object/parameter/pip"
if [ -f "$pipparam" ]; then
pip=$(cat "$pipparam")
else
pip="pip"
fi
case "$state_should" in
present)
echo $pip install -q pyro
;;
absent)
echo $pip uninstall -q -y pyro
;;
esac

53
conf/type/__package_pip/man.text Normal file
View file

@ -0,0 +1,53 @@
cdist-type__package_pip(7)
==========================
Nico Schottelius <nico-cdist--@--schottelius.org>
NAME
----
cdist-type__package_pip - Manage packages with pip
DESCRIPTION
-----------
Pip is used in Python environments to install packages.
It is also included in the python virtualenv environment.
REQUIRED PARAMETERS
-------------------
state::
Either "present" or "absent".
OPTIONAL PARAMETERS
-------------------
name::
If supplied, use the name and not the object id as the package name.
pip::
Instead of using pip from PATH, use the specific pip path.
EXAMPLES
--------
--------------------------------------------------------------------------------
# Install a package
__package_pip pyro --state present
# Use pip in a virtualenv located at /root/shinken_virtualenv
__package_pip pyro --state present --pip /root/shinken_virtualenv/bin/pip
--------------------------------------------------------------------------------
SEE ALSO
--------
- cdist-type(7)
- cdist-type__package(7)
COPYING
-------
Copyright \(C) 2012 Nico Schottelius. Free use of this software is
granted under the terms of the GNU General Public License version 3 (GPLv3).

1
conf/type/__package_pip/parameter/optional Normal file
View file

@ -0,0 +1 @@
pip

1
conf/type/__package_pip/parameter/required Normal file
View file

@ -0,0 +1 @@
state

2
conf/type/__package_yum/gencode-remote
View file

@ -29,7 +29,7 @@ fi
state="$(cat "$__object/parameter/state")"
if grep -q -E "(centos|redhat)" "$__global/explorer/os"; then
if grep -q -E "(centos|redhat|amazon)" "$__global/explorer/os"; then
opts="-y --quiet"
else
opts="--assumeyes --quiet"

20
conf/type/__user/gencode-remote
View file

@ -28,6 +28,7 @@ cd "$__object/parameter"
if grep -q "^${name}:" "$__object/explorer/passwd"; then
for property in $(ls .); do
new_value="$(cat "$property")"
unset current_value
file="$__object/explorer/passwd"
@ -36,13 +37,20 @@ if grep -q "^${name}:" "$__object/explorer/passwd"; then
if $(echo "$new_value" | grep -q '^[0-9][0-9]*$'); then
field=4
else
# group name
file="$__object/explorer/group"
field=1
# We were passed a group name. Compare the gid in
# the user's /etc/passwd entry with the gid of the
# group returned by the group explorer.
gid_from_group=$(awk -F: '{ print $3 }' "$__object/explorer/group")
gid_from_passwd=$(awk -F: '{ print $4 }' "$file")
if [ "$gid_from_group" != "$gid_from_passwd" ]; then
current_value="$gid_from_passwd"
else
current_value="$new_value"
fi
fi
;;
password)
field=3
field=2
file="$__object/explorer/shadow"
;;
comment) field=5 ;;
@ -51,8 +59,12 @@ if grep -q "^${name}:" "$__object/explorer/passwd"; then
uid) field=3 ;;
esac
# If we haven't already set $current_value above, pull it from the
# appropriate file/field.
if [ -z "$current_value" ]; then
export field
current_value="$(awk -F: '{ print $ENVIRON["field"] }' < "$file")"
fi
if [ "$new_value" != "$current_value" ]; then
set -- "$@" "--$property" \'$new_value\'

30
doc/changelog
View file

@ -1,15 +1,37 @@
2.0.6:
Changelog
---------
* Changes are always commented with their author in (braces)
* Exception: No braces means author == Nico Schottelius
2.0.8:
* Cleanup: Better hint to source of error
* Cleanup: Remove support for __debug variable in manifests (Type != Core
debugging)
2.0.7: 2012-02-13
* Bugfix __file: Use chmod after chown/chgrp (Matt Coddington)
* Bugfix __user: Correct shadow field in explorer (Matt Coddington)
* Bugfix __link: Properly handle existing links (Steven Armstrong)
* Bugfix __key_value: More robust implementation (Steven Armstrong)
* Bugfix __user: Fix for changing a user's group by name (Matt Coddington)
* New Type: __package_pip
* Bugfix/Cleanup: Correctly allow Object ID to start and end with /, but
not contain //.
2.0.6: 2012-01-28
* Bugfix __apt_ppa:
Also remove the [ppa-name].list file, if empty. (Tim Kersten)
* Bugfix __group:
Referenced wrong variable name (Matt Coddington)
* Feature __package_apt:
Initial support for virtual packages (Evax Software)
* Feature Core: Added new dependency resolver (Steven Armstrong)
* Feature Explorer, __package_yum: Support Amazon Linux (Matt Coddington)
* New Type: __rvm (Evax Software)
* New Type: __rvm_gem (Evax Software)
* New Type: __rvm_gemset (Evax Software)
* New Type: __rvm_ruby (Evax Software)
* New Explorer: runlevel
* Documentation: Update of reference (environment variables)
* Feature core: Added new dependency resolver (Steven Armstrong)
2.0.5: 2012-01-18
* Bugfix __key_value: Use correct delimiters

36
doc/dev/logs/2012-02-08.explorer-depends-on-another-type Normal file
View file

@ -0,0 +1,36 @@
If a type explorer depends on a command that will be generated by another type,
the operation fails, as can be seen below.
This may be a corner case, but is hapenning with __package_pip and
__python_virtualenv.
[19:10] brief:cdist% ./bin/cdist config -v loch
INFO: loch: Running global explorers
INFO: loch: Running initial manifest /home/users/nico/privat/firmen/local.ch/vcs/cdist/conf/manifest
INFO: loch: Running object manifests and type explorers
INFO: loch: Running manifest and explorers for __git/root/shinken
INFO: loch: Running manifest and explorers for __package_pip/pyro
/var/lib/cdist/conf/type/__package_pip/explorer/state: line 38: /root/shinken_virtualenv/bin/pip: No such file or directory
INFO: loch: Running manifest and explorers for __python_virtualenv/root/shinken_virtualenv
INFO: loch: Running manifest and explorers for __directory/pyro
INFO: loch: Running manifest and explorers for __directory/root/shinken
INFO: loch: Running manifest and explorers for __directory/root/shinken_virtualenv
INFO: loch: Running manifest and explorers for __package/git
INFO: loch: Running manifest and explorers for __package/python-virtualenv
INFO: loch: Running manifest and explorers for __package_pacman/git
INFO: loch: Running manifest and explorers for __package_pacman/python-virtualenv
INFO: loch: Generating and executing code
INFO: loch: Generating and executing code for __package_pacman/git
INFO: loch: Generating and executing code for __package/git
INFO: loch: Generating and executing code for __directory/root/shinken
INFO: loch: Generating and executing code for __git/root/shinken
fatal: write error: No space left on device
fatal: index-pack failed
ERROR: loch: Code that raised the error:
git clone --quiet "git://github.com/naparuba/shinken.git" "/root/shinken"
ERROR: Remote script execution failed: ssh -o User=root -q loch /bin/sh -e /var/lib/cdist/object/__git/root/shinken/.cdist/code-remote
WARNING: Failed to deploy to the following hosts: loch
INFO: Total processing time for 1 host(s): 340.62370681762695
[19:17] brief:cdist% ./bin/cdist config -v loch

18
doc/dev/logs/2012-02-10.object_id-and-slashes Normal file
View file

@ -0,0 +1,18 @@
__typename /foo/bar # possible, usual use case
require="__a//b" __typename /foo/bar # possible and happens often for __a/$id in loops
__typename /foo/bar/ # trailing slash will be stripped, can be documented
__typename /foo//bar//baz # // will be converted to / implicitly through fs; error prone; disallow
require="__a//b//c" __typename # // will be converted to / implicitly through fs; error prone; disallow
Solution:
1) allow require __a//b: type __a, object id /b
=> strip first slash of object id, as we do in non-dep-mode
2) allow _one_ trailing /: __type /foo/bar/ and require="__foo/abc/"
=> strip one leading slash of object id
3) disallow // within object id
4) disallow starting or ending / after 1) and 2)

23
doc/dev/logs/2012-02-13.dependencies Normal file
View file

@ -0,0 +1,23 @@
possible dependencies:
- unix pattern __foo/*
- object: __foo//bar, __foo/bar
- singleton with object_id: __foo/singleton
- singleton without object_id: __foo/
solving dependencies:
solve_dep(object, run_list):
- list = [me]
- if status == IN_DEPENDENCY:
fail: circular dependency
- status = IN_DEPENDENCY
- create_list_of_deps(object)
- try pattern expansion
- for each dependency:
if object does not exist:
fail
else:
list.append(solve_dep(object, run_list)):
- status == IN_LIST
- return [me, dependencies [, dependencies of dependencies]]

12
doc/dev/releasechecklist
View file

@ -11,15 +11,25 @@ echo "Testing documentation..."
./build clean && ./build man || exit 1
# get version
changelog_version=$(head -n1 doc/changelog | sed 's/:.*//')
changelog_version=$(grep '^[[:digit:]]' doc/changelog | head -n1 | sed 's/:.*//')
#git_version=$(git describe)
lib_version=$(grep ^VERSION lib/cdist/__init__.py | sed -e 's/.*= //' -e 's/"//g')
# get date
date_today="$(date +%Y-%m-%d)"
date_changelog=$(grep '^[[:digit:]]' doc/changelog | head -n1 | sed 's/.*: //')
echo "Ensure you fixed/prepared version files: $files"
echo "changelog: $changelog_version"
#echo "git: $git_version"
echo "lib: $lib_version"
if [ "$date_today" != "$date_changelog" ]; then
echo "Messed up date, not releasing:"
echo "Changelog: $date_changelog"
exit 1
fi
if [ "$lib_version" != "$changelog_version" ]; then
echo "Messed up versions, not releasing"
exit 1

3
doc/dev/todo/TAKEME
View file

@ -34,7 +34,6 @@ USER INTERFACE
TYPES
------
- Add testing framework (proposed by Evax Software)
- __user
add option to include --create-home
- ensure that all types, which support --state support
present and absent (consistent look and feel)

17
doc/dev/todo/niconext
View file

@ -1,9 +1,17 @@
- __init_script?
to enable/disable startup of init stuff
http://linuxhelp.blogspot.com/2006/04/enabling-and-disabling-services-during_01.html
2.0.8 features / cleanups:
- cleanup object_id handling
- have a look at singletons
- remove useless
ERROR: monitoring02: Code that raised the error:
- ensure that all types, which support --state support
present and absent (consistent look and feel)
--------------------------------------------------------------------------------
- update/create docs
- document __remote_copy and __remote_exec
- cdist-cache::
How to get use information about the hosts we have been working on [advanced]
- cdist-scaling-tuning::
@ -17,3 +25,4 @@
- exec flag is not true for manifest anymore
- SSH HINTS - ssh agent

6
doc/man/cdist-reference.text.sh
View file

@ -180,7 +180,11 @@ __object::
__object_id::
The type unique object id.
Available for: type manifest, type explorer, type gencode
Note: The leading "/" will always be stripped.
Note: The leading and the trailing "/" will always be stripped (caused by
the filesystem database and ensured by the core).
Note: Double slashes ("//") will not be fixed and result in an error.
__self::
DEPRECATED: Same as __object_name, do not use anymore, use __object_name instead.
Will be removed in cdist 3.x.

7
doc/man/man7/cdist-hacker.text
View file

@ -61,12 +61,19 @@ including it.
HOW TO SUBMIT A NEW TYPE
------------------------
For detailled information about types, see cdist-type(7).
Submitting a type works as described above, with the additional requirement
that a corresponding manpage named man.text in asciidoc format with
the manpage-name "cdist-type__NAME" is included in the type directory
AND asciidoc is able to compile it (i.e. do NOT have to many "=" in the second
line).
Warning: Submitting "exec" or "run" types that simply echo their parameter in
gencode* will not be accepted, because they are of no use. Every type can output
code and thus such a type introduces redundant functionality that is given by
core cdist already.
SEE ALSO
--------

15
lib/cdist/__init__.py
View file

@ -19,6 +19,8 @@
#
#
VERSION = "2.0.7"
BANNER = """
.. . .x+=:. s
dF @88> z` ^% :8
@ -34,7 +36,6 @@ BANNER = """
"P' "" ""
"""
DOT_CDIST = ".cdist"
VERSION = "2.0.5"
import os
@ -43,15 +44,17 @@ class Error(Exception):
"""Base exception class for this project"""
pass
class CdistObjectError(Error):
"""Something went wrong with an object"""
class MissingEnvironmentVariableError(Error):
"""Raised when a required environment variable is not set."""
def __init__(self, cdist_object, message):
self.name = cdist_object.name
self.source = " ".join(cdist_object.source)
self.message = message
def __init__(self, name):
self.name = name
def __str__(self):
return 'Missing required environment variable: ' + str(self.name)
return '%s: %s (defined at %s)' % (self.name, self.message, self.source)
def file_to_list(filename):
"""Return list from \n seperated file"""

14
lib/cdist/config_install.py
View file

@ -89,9 +89,9 @@ class ConfigInstall(object):
new_objects_created = True
while new_objects_created:
new_objects_created = False
for cdist_object in core.Object.list_objects(self.local.object_path,
for cdist_object in core.CdistObject.list_objects(self.local.object_path,
self.local.type_path):
if cdist_object.state == core.Object.STATE_PREPARED:
if cdist_object.state == core.CdistObject.STATE_PREPARED:
self.log.debug("Skipping re-prepare of object %s", cdist_object)
continue
else:
@ -103,16 +103,16 @@ class ConfigInstall(object):
self.log.info("Running manifest and explorers for " + cdist_object.name)
self.explorer.run_type_explorers(cdist_object)
self.manifest.run_type_manifest(cdist_object)
cdist_object.state = core.Object.STATE_PREPARED
cdist_object.state = core.CdistObject.STATE_PREPARED
def object_run(self, cdist_object):
"""Run gencode and code for an object"""
self.log.debug("Trying to run object " + cdist_object.name)
if cdist_object.state == core.Object.STATE_DONE:
if cdist_object.state == core.CdistObject.STATE_DONE:
# TODO: remove once we are sure that this really never happens.
raise cdist.Error("Attempting to run an already finished object: %s", cdist_object)
cdist_type = cdist_object.type
cdist_type = cdist_object.cdist_type
# Generate
self.log.info("Generating and executing code for " + cdist_object.name)
@ -130,13 +130,13 @@ class ConfigInstall(object):
# Mark this object as done
self.log.debug("Finishing run of " + cdist_object.name)
cdist_object.state = core.Object.STATE_DONE
cdist_object.state = core.CdistObject.STATE_DONE
def stage_run(self):
"""The final (and real) step of deployment"""
self.log.info("Generating and executing code")
objects = core.Object.list_objects(
objects = core.CdistObject.list_objects(
self.local.object_path,
self.local.type_path)

10
lib/cdist/core/__init__.py
View file

@ -19,11 +19,11 @@
#
#
from cdist.core.type import Type
from cdist.core.type import NoSuchTypeError
from cdist.core.object import Object
from cdist.core.object import IllegalObjectIdError
from cdist.core.object import OBJECT_MARKER
from cdist.core.cdist_type import CdistType
from cdist.core.cdist_type import NoSuchTypeError
from cdist.core.cdist_object import CdistObject
from cdist.core.cdist_object import IllegalObjectIdError
from cdist.core.cdist_object import OBJECT_MARKER
from cdist.core.explorer import Explorer
from cdist.core.manifest import Manifest
from cdist.core.code import Code

84
lib/cdist/core/object.py → lib/cdist/core/cdist_object.py
View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
@ -42,7 +42,7 @@ class IllegalObjectIdError(cdist.Error):
return '%s: %s' % (self.message, self.object_id)
class Object(object):
class CdistObject(object):
"""Represents a cdist object.
All interaction with objects in cdist should be done through this class.
@ -61,7 +61,7 @@ class Object(object):
"""Return a list of object instances"""
for object_name in cls.list_object_names(object_base_path):
type_name, object_id = cls.split_name(object_name)
yield cls(cdist.core.Type(type_base_path, type_name), object_base_path, object_id=object_id)
yield cls(cdist.core.CdistType(type_base_path, type_name), object_base_path, object_id=object_id)
@classmethod
def list_type_names(cls, object_base_path):
@ -96,30 +96,58 @@ class Object(object):
"""
return os.path.join(type_name, object_id)
@staticmethod
def validate_object_id(object_id):
def validate_object_id(self):
# FIXME: also check that there is no object ID when type is singleton?
"""Validate the given object_id and raise IllegalObjectIdError if it's not valid.
"""
if object_id:
if object_id.startswith('/'):
raise IllegalObjectIdError(object_id, 'object_id may not start with /')
if OBJECT_MARKER in object_id.split(os.sep):
raise IllegalObjectIdError(object_id, 'object_id may not contain \'%s\'' % OBJECT_MARKER)
if self.object_id:
if OBJECT_MARKER in self.object_id.split(os.sep):
raise IllegalObjectIdError(self.object_id, 'object_id may not contain \'%s\'' % OBJECT_MARKER)
if '//' in self.object_id:
raise IllegalObjectIdError(self.object_id, 'object_id may not contain //')
# If no object_id and type is not singleton => error out
if not self.object_id and not self.cdist_type.is_singleton:
raise IllegalObjectIdError(self.object_id,
"Missing object_id and type is not a singleton.")
def __init__(self, cdist_type, base_path, object_id=None):
self.validate_object_id(object_id)
self.type = cdist_type # instance of Type
self.cdist_type = cdist_type # instance of Type
self.base_path = base_path
self.object_id = object_id
self.name = self.join_name(self.type.name, self.object_id)
self.path = os.path.join(self.type.path, self.object_id, OBJECT_MARKER)
self.validate_object_id()
self.sanitise_object_id()
self.name = self.join_name(self.cdist_type.name, self.object_id)
self.path = os.path.join(self.cdist_type.path, self.object_id, OBJECT_MARKER)
self.absolute_path = os.path.join(self.base_path, self.path)
self.code_local_path = os.path.join(self.path, "code-local")
self.code_remote_path = os.path.join(self.path, "code-remote")
self.parameter_path = os.path.join(self.path, "parameter")
def object_from_name(self, object_name):
"""Convenience method for creating an object instance from an object name.
Mainly intended to create objects when resolving requirements.
e.g:
<CdistObject __foo/bar>.object_from_name('__other/object') -> <CdistObject __other/object>
"""
base_path = self.base_path
type_path = self.cdist_type.base_path
type_name, object_id = self.split_name(object_name)
cdist_type = self.cdist_type.__class__(type_path, type_name)
return self.__class__(cdist_type, base_path, object_id=object_id)
def __repr__(self):
return '<Object %s>' % self.name
return '<CdistObject %s>' % self.name
def __eq__(self, other):
"""define equality as 'name is the same'"""
@ -128,23 +156,23 @@ class Object(object):
def __hash__(self):
return hash(self.name)
def __lt__(self, other):
return isinstance(other, self.__class__) and self.name < other.name
def object_from_name(self, object_name):
"""Convenience method for creating an object instance from an object name.
Mainly intended to create objects when resolving requirements.
e.g:
<Object __foo/bar>.object_from_name('__other/object') -> <Object __other/object>
def sanitise_object_id(self):
"""
type_path = self.type.base_path
base_path = self.base_path
type_name, object_id = self.split_name(object_name)
return self.__class__(self.type.__class__(type_path, type_name), base_path, object_id=object_id)
Remove leading and trailing slash (one only)
"""
# Allow empty object id for singletons
if self.object_id:
# Remove leading slash
if self.object_id[0] == '/':
self.object_id = self.object_id[1:]
# Remove trailing slash
if self.object_id[-1] == '/':
self.object_id = self.object_id[:-1]
# FIXME: still needed?
@property

6
lib/cdist/core/type.py → lib/cdist/core/cdist_type.py
View file

@ -34,7 +34,7 @@ class NoSuchTypeError(cdist.Error):
return "Type '%s' does not exist at %s" % (self.type_path, self.type_absolute_path)
class Type(object):
class CdistType(object):
"""Represents a cdist type.
All interaction with types in cdist should be done through this class.
@ -61,7 +61,7 @@ class Type(object):
# name is second argument
name = args[1]
if not name in cls._instances:
instance = super(Type, cls).__new__(cls)
instance = super(CdistType, cls).__new__(cls)
cls._instances[name] = instance
# return instance so __init__ is called
return cls._instances[name]
@ -84,7 +84,7 @@ class Type(object):
self.__optional_parameters = None
def __repr__(self):
return '<Type %s>' % self.name
return '<CdistType %s>' % self.name
def __eq__(self, other):
return isinstance(other, self.__class__) and self.name == other.name

7
lib/cdist/core/code.py
View file

@ -92,17 +92,14 @@ class Code(object):
'__global': self.local.out_path,
}
if log.getEffectiveLevel() == logging.DEBUG:
self.env.update({'__debug': "yes" })
def _run_gencode(self, cdist_object, which):
cdist_type = cdist_object.type
cdist_type = cdist_object.cdist_type
script = os.path.join(self.local.type_path, getattr(cdist_type, 'gencode_%s_path' % which))
if os.path.isfile(script):
env = os.environ.copy()
env.update(self.env)
env.update({
'__type': cdist_object.type.absolute_path,
'__type': cdist_object.cdist_type.absolute_path,
'__object': cdist_object.absolute_path,
'__object_id': cdist_object.object_id,
'__object_name': cdist_object.name,

35
lib/cdist/core/explorer.py
View file

@ -73,9 +73,6 @@ class Explorer(object):
'__target_host': self.target_host,
'__explorer': self.remote.global_explorer_path,
}
# FIXME: remove soon with new logging infrastructure
if self.log.getEffectiveLevel() == logging.DEBUG:
self.env.update({'__debug': "yes" })
self._type_explorers_transferred = []
### global
@ -122,15 +119,28 @@ class Explorer(object):
in the object.
"""
self.log.debug("Transfering type explorers for type: %s", cdist_object.type)
self.transfer_type_explorers(cdist_object.type)
self.log.debug("Transfering type explorers for type: %s", cdist_object.cdist_type)
self.transfer_type_explorers(cdist_object.cdist_type)
self.log.debug("Transfering object parameters for object: %s", cdist_object.name)
self.transfer_object_parameters(cdist_object)
for explorer in self.list_type_explorer_names(cdist_object.type):
for explorer in self.list_type_explorer_names(cdist_object.cdist_type):
output = self.run_type_explorer(explorer, cdist_object)
self.log.debug("Running type explorer '%s' for object '%s'", explorer, cdist_object.name)
cdist_object.explorers[explorer] = output
def run_type_explorer(self, explorer, cdist_object):
"""Run the given type explorer for the given object and return it's output."""
cdist_type = cdist_object.cdist_type
env = self.env.copy()
env.update({
'__object': os.path.join(self.remote.object_path, cdist_object.path),
'__object_id': cdist_object.object_id,
'__object_fq': cdist_object.path,
'__type_explorer': os.path.join(self.remote.type_path, cdist_type.explorer_path)
})
script = os.path.join(self.remote.type_path, cdist_type.explorer_path, explorer)
return self.remote.run_script(script, env=env, return_output=True)
def transfer_type_explorers(self, cdist_type):
"""Transfer the type explorers for the given type to the remote side."""
if cdist_type.explorers:
@ -150,16 +160,3 @@ class Explorer(object):
destination = os.path.join(self.remote.object_path, cdist_object.parameter_path)
self.remote.mkdir(destination)
self.remote.transfer(source, destination)
def run_type_explorer(self, explorer, cdist_object):
"""Run the given type explorer for the given object and return it's output."""
cdist_type = cdist_object.type
env = self.env.copy()
env.update({
'__object': os.path.join(self.remote.object_path, cdist_object.path),
'__object_id': cdist_object.object_id,
'__object_fq': cdist_object.path,
'__type_explorer': os.path.join(self.remote.type_path, cdist_type.explorer_path)
})
script = os.path.join(self.remote.type_path, cdist_type.explorer_path, explorer)
return self.remote.run_script(script, env=env, return_output=True)

4
lib/cdist/core/manifest.py
View file

@ -87,7 +87,7 @@ class Manifest(object):
self.local.run_script(script, env=env)
def run_type_manifest(self, cdist_object):
script = os.path.join(self.local.type_path, cdist_object.type.manifest_path)
script = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path)
if os.path.isfile(script):
env = os.environ.copy()
env.update(self.env)
@ -96,7 +96,7 @@ class Manifest(object):
'__object_id': cdist_object.object_id,
'__object_name': cdist_object.name,
'__self': cdist_object.name,
'__type': cdist_object.type.absolute_path,
'__type': cdist_object.cdist_type.absolute_path,
'__cdist_manifest': script,
})
self.local.run_script(script, env=env)

51
lib/cdist/emulator.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
@ -26,29 +26,22 @@ import os
import cdist
from cdist import core
class IllegalRequirementError(cdist.Error):
def __init__(self, requirement, message=None):
self.requirement = requirement
self.message = message or 'Illegal requirement'
def __str__(self):
return '%s: %s' % (self.message, self.requirement)
class Emulator(object):
def __init__(self, argv):
self.argv = argv
self.object_id = False
self.global_path = os.environ['__global']
self.object_source = os.environ['__cdist_manifest']
self.target_host = os.environ['__target_host']
# Internally only
self.object_source = os.environ['__cdist_manifest']
self.type_base_path = os.environ['__cdist_type_base_path']
self.object_base_path = os.path.join(self.global_path, "object")
self.type_name = os.path.basename(argv[0])
self.cdist_type = core.Type(self.type_base_path, self.type_name)
self.cdist_type = core.CdistType(self.type_base_path, self.type_name)
self.__init_log()
@ -113,20 +106,15 @@ class Emulator(object):
def setup_object(self):
# FIXME: verify object id
# Setup object_id
# Setup object_id - FIXME: unset / do not setup anymore!
if self.cdist_type.is_singleton:
self.object_id = "singleton"
else:
self.object_id = self.args.object_id[0]
del self.args.object_id
# strip leading slash from object_id
self.object_id = self.object_id.lstrip('/')
# Instantiate the cdist object we are defining
self.cdist_object = core.Object(self.cdist_type, self.object_base_path, self.object_id)
self.cdist_object = core.CdistObject(self.cdist_type, self.object_base_path, self.object_id)
# Create object with given parameters
self.parameters = {}
@ -143,6 +131,9 @@ class Emulator(object):
self.cdist_object.create()
self.cdist_object.parameters = self.parameters
# Record / Append source
self.cdist_object.source.append(self.object_source)
def record_requirements(self):
"""record requirements"""
@ -151,25 +142,17 @@ class Emulator(object):
self.log.debug("reqs = " + requirements)
for requirement in requirements.split(" "):
# Ignore empty fields - probably the only field anyway
if len(requirement) == 0:
continue
if len(requirement) == 0: continue
requirement_type_name, requirement_object_id = core.Object.split_name(requirement)
# Instantiate type which fails if type does not exist
requirement_type = core.Type(self.type_base_path, requirement_type_name)
if requirement_object_id:
# Validate object_id if any
core.Object.validate_object_id(requirement_object_id)
elif not requirement_type.is_singleton:
# Only singeltons have no object_id
raise IllegalRequirementError(requirement, "Missing object_id and type is not a singleton.")
# Raises an error, if object cannot be created
cdist_object = self.cdist_object.object_from_name(requirement)
self.log.debug("Recording requirement: " + requirement)
self.cdist_object.requirements.append(requirement)
# Record / Append source
self.cdist_object.source.append(self.object_source)
# Save the sanitised version, not the user supplied one
# (__file//bar => __file/bar)
# This ensures pattern matching is done against sanitised list
self.cdist_object.requirements.append(cdist_object.name)
def record_auto_requirements(self):
"""An object shall automatically depend on all objects that it defined in it's type manifest.

2
lib/cdist/exec/local.py
View file

@ -143,7 +143,7 @@ class Local(object):
def link_emulator(self, exec_path):
"""Link emulator to types"""
src = os.path.abspath(exec_path)
for cdist_type in core.Type.list_types(self.type_path):
for cdist_type in core.CdistType.list_types(self.type_path):
dst = os.path.join(self.bin_path, cdist_type.name)
self.log.debug("Linking emulator: %s to %s", src, dst)

9
lib/cdist/exec/remote.py
View file

@ -20,8 +20,6 @@
#
#
# FIXME: common base class with Local?
import io
import os
import sys
@ -148,18 +146,17 @@ class Remote(object):
os_environ = os.environ.copy()
os_environ['__target_host'] = self.target_host
self.log.debug("Remote run script: %s", command)
# can't pass environment to remote side, so prepend command with
# variable declarations
if env:
self.log.debug("Remote run script env: %s", env)
command.extend(["%s=%s" % item for item in env.items()])
command.extend(["/bin/sh", "-e"])
command.append(script)
self.log.debug("Remote run script: %s", command)
if env:
self.log.debug("Remote run script env: %s", env)
try:
if return_output:
return subprocess.check_output(command, env=os_environ).decode()

2
lib/cdist/resolver.py
View file

@ -125,7 +125,7 @@ class DependencyResolver(object):
resolved.append(cdist_object)
unresolved.remove(cdist_object)
except RequirementNotFoundError as e:
raise cdist.Error(cdist_object.name + " requires non-existing " + e.requirement)
raise cdist.CdistObjectError(cdist_object, "requires non-existing " + e.requirement)
def __iter__(self):
"""Iterate over all unique objects while resolving dependencies.

4
lib/cdist/test/code/__init__.py
View file

@ -54,8 +54,8 @@ class CodeTestCase(test.CdistTestCase):
self.code = code.Code(self.target_host, self.local, self.remote)
self.cdist_type = core.Type(self.local.type_path, '__dump_environment')
self.cdist_object = core.Object(self.cdist_type, self.local.object_path, 'whatever')
self.cdist_type = core.CdistType(self.local.type_path, '__dump_environment')
self.cdist_object = core.CdistObject(self.cdist_type, self.local.object_path, 'whatever')
self.cdist_object.create()
self.log = logging.getLogger("cdist")

10
lib/cdist/test/emulator/__init__.py
View file

@ -79,7 +79,7 @@ class EmulatorTestCase(test.CdistTestCase):
os.environ.update(self.env)
os.environ['require'] = '__file'
emu = emulator.Emulator(argv)
self.assertRaises(emulator.IllegalRequirementError, emu.run)
self.assertRaises(core.IllegalObjectIdError, emu.run)
def test_singleton_object_requirement(self):
argv = ['__file', '/tmp/foobar']
@ -119,8 +119,8 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase):
def test_autorequire(self):
initial_manifest = os.path.join(self.local.manifest_path, "init")
self.manifest.run_initial_manifest(initial_manifest)
cdist_type = core.Type(self.local.type_path, '__saturn')
cdist_object = core.Object(cdist_type, self.local.object_path, 'singleton')
cdist_type = core.CdistType(self.local.type_path, '__saturn')
cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'singleton')
self.manifest.run_type_manifest(cdist_object)
expected = ['__planet/Saturn', '__moon/Prometheus']
self.assertEqual(sorted(cdist_object.requirements), sorted(expected))
@ -156,6 +156,6 @@ class ArgumentsWithDashesTestCase(test.CdistTestCase):
emu = emulator.Emulator(argv)
emu.run()
cdist_type = core.Type(self.local.type_path, '__arguments_with_dashes')
cdist_object = core.Object(cdist_type, self.local.object_path, 'some-id')
cdist_type = core.CdistType(self.local.type_path, '__arguments_with_dashes')
cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'some-id')
self.assertTrue('with-dash' in cdist_object.parameters)

18
lib/cdist/test/explorer/__init__.py
View file

@ -83,19 +83,19 @@ class ExplorerClassTestCase(test.CdistTestCase):
shutil.rmtree(out_path)
def test_list_type_explorer_names(self):
cdist_type = core.Type(self.local.type_path, '__test_type')
cdist_type = core.CdistType(self.local.type_path, '__test_type')
expected = cdist_type.explorers
self.assertEqual(self.explorer.list_type_explorer_names(cdist_type), expected)
def test_transfer_type_explorers(self):
cdist_type = core.Type(self.local.type_path, '__test_type')
cdist_type = core.CdistType(self.local.type_path, '__test_type')
self.explorer.transfer_type_explorers(cdist_type)
source = os.path.join(self.local.type_path, cdist_type.explorer_path)
destination = os.path.join(self.remote.type_path, cdist_type.explorer_path)
self.assertEqual(os.listdir(source), os.listdir(destination))
def test_transfer_type_explorers_only_once(self):
cdist_type = core.Type(self.local.type_path, '__test_type')
cdist_type = core.CdistType(self.local.type_path, '__test_type')
# first transfer
self.explorer.transfer_type_explorers(cdist_type)
source = os.path.join(self.local.type_path, cdist_type.explorer_path)
@ -109,8 +109,8 @@ class ExplorerClassTestCase(test.CdistTestCase):
self.assertFalse(os.listdir(destination))
def test_transfer_object_parameters(self):
cdist_type = core.Type(self.local.type_path, '__test_type')
cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever')
cdist_type = core.CdistType(self.local.type_path, '__test_type')
cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever')
cdist_object.create()
cdist_object.parameters = {'first': 'first value', 'second': 'second value'}
self.explorer.transfer_object_parameters(cdist_object)
@ -119,15 +119,15 @@ class ExplorerClassTestCase(test.CdistTestCase):
self.assertEqual(sorted(os.listdir(source)), sorted(os.listdir(destination)))
def test_run_type_explorer(self):
cdist_type = core.Type(self.local.type_path, '__test_type')
cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever')
cdist_type = core.CdistType(self.local.type_path, '__test_type')
cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever')
self.explorer.transfer_type_explorers(cdist_type)
output = self.explorer.run_type_explorer('world', cdist_object)
self.assertEqual(output, 'hello\n')
def test_run_type_explorers(self):
cdist_type = core.Type(self.local.type_path, '__test_type')
cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever')
cdist_type = core.CdistType(self.local.type_path, '__test_type')
cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever')
cdist_object.create()
self.explorer.run_type_explorers(cdist_object)
self.assertEqual(cdist_object.explorers, {'world': 'hello'})

4
lib/cdist/test/manifest/__init__.py
View file

@ -79,8 +79,8 @@ class ManifestTestCase(test.CdistTestCase):
self.assertEqual(output_dict['__manifest'], self.local.manifest_path)
def test_type_manifest_environment(self):
cdist_type = core.Type(self.local.type_path, '__dump_environment')
cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever')
cdist_type = core.CdistType(self.local.type_path, '__dump_environment')
cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever')
handle, output_file = self.mkstemp(dir=self.temp_dir)
os.close(handle)
os.environ['__cdist_test_out'] = output_file

50
lib/cdist/test/object/__init__.py
View file

@ -25,6 +25,8 @@ import shutil
from cdist import test
from cdist import core
import cdist
import os.path as op
my_dir = op.abspath(op.dirname(__file__))
fixtures = op.join(my_dir, 'fixtures')
@ -34,48 +36,48 @@ type_base_path = op.join(fixtures, 'type')
class ObjectClassTestCase(test.CdistTestCase):
def test_list_object_names(self):
object_names = list(core.Object.list_object_names(object_base_path))
object_names = list(core.CdistObject.list_object_names(object_base_path))
self.assertEqual(object_names, ['__first/man', '__second/on-the', '__third/moon'])
def test_list_type_names(self):
type_names = list(core.Object.list_type_names(object_base_path))
type_names = list(cdist.core.CdistObject.list_type_names(object_base_path))
self.assertEqual(type_names, ['__first', '__second', '__third'])
def test_list_objects(self):
objects = list(core.Object.list_objects(object_base_path, type_base_path))
objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
objects_expected = [
core.Object(core.Type(type_base_path, '__first'), object_base_path, 'man'),
core.Object(core.Type(type_base_path, '__second'), object_base_path, 'on-the'),
core.Object(core.Type(type_base_path, '__third'), object_base_path, 'moon'),
core.CdistObject(core.CdistType(type_base_path, '__first'), object_base_path, 'man'),
core.CdistObject(core.CdistType(type_base_path, '__second'), object_base_path, 'on-the'),
core.CdistObject(core.CdistType(type_base_path, '__third'), object_base_path, 'moon'),
]
self.assertEqual(objects, objects_expected)
class ObjectIdTestCase(test.CdistTestCase):
def test_object_id_starts_with_slash(self):
cdist_type = core.Type(type_base_path, '__third')
illegal_object_id = '/object_id/may/not/start/with/slash'
def test_object_id_contains_double_slash(self):
cdist_type = core.CdistType(type_base_path, '__third')
illegal_object_id = '/object_id//may/not/contain/double/slash'
with self.assertRaises(core.IllegalObjectIdError):
core.Object(cdist_type, object_base_path, illegal_object_id)
core.CdistObject(cdist_type, object_base_path, illegal_object_id)
def test_object_id_contains_object_marker(self):
cdist_type = core.Type(type_base_path, '__third')
cdist_type = core.CdistType(type_base_path, '__third')
illegal_object_id = 'object_id/may/not/contain/%s/anywhere' % core.OBJECT_MARKER
with self.assertRaises(core.IllegalObjectIdError):
core.Object(cdist_type, object_base_path, illegal_object_id)
core.CdistObject(cdist_type, object_base_path, illegal_object_id)
def test_object_id_contains_object_marker_string(self):
cdist_type = core.Type(type_base_path, '__third')
cdist_type = core.CdistType(type_base_path, '__third')
illegal_object_id = 'object_id/may/contain_%s_in_filename' % core.OBJECT_MARKER
core.Object(cdist_type, object_base_path, illegal_object_id)
core.CdistObject(cdist_type, object_base_path, illegal_object_id)
# if we get here, the test passed
class ObjectTestCase(test.CdistTestCase):
def setUp(self):
self.cdist_type = core.Type(type_base_path, '__third')
self.cdist_object = core.Object(self.cdist_type, object_base_path, 'moon')
self.cdist_type = core.CdistType(type_base_path, '__third')
self.cdist_object = core.CdistObject(self.cdist_type, object_base_path, 'moon')
def tearDown(self):
self.cdist_object.changed = False
@ -159,16 +161,16 @@ class ObjectTestCase(test.CdistTestCase):
self.assertEqual(self.cdist_object.state, '')
def test_state_prepared(self):
self.cdist_object.state = core.Object.STATE_PREPARED
self.assertEqual(self.cdist_object.state, core.Object.STATE_PREPARED)
self.cdist_object.state = core.CdistObject.STATE_PREPARED
self.assertEqual(self.cdist_object.state, core.CdistObject.STATE_PREPARED)
def test_state_running(self):
self.cdist_object.state = core.Object.STATE_RUNNING
self.assertEqual(self.cdist_object.state, core.Object.STATE_RUNNING)
self.cdist_object.state = core.CdistObject.STATE_RUNNING
self.assertEqual(self.cdist_object.state, core.CdistObject.STATE_RUNNING)
def test_state_done(self):
self.cdist_object.state = core.Object.STATE_DONE
self.assertEqual(self.cdist_object.state, core.Object.STATE_DONE)
self.cdist_object.state = core.CdistObject.STATE_DONE
self.assertEqual(self.cdist_object.state, core.CdistObject.STATE_DONE)
def test_source(self):
self.assertEqual(list(self.cdist_object.source), [])
@ -195,6 +197,6 @@ class ObjectTestCase(test.CdistTestCase):
self.cdist_object.code_remote = 'Hello World'
other_name = '__first/man'
other_object = self.cdist_object.object_from_name(other_name)
self.assertTrue(isinstance(other_object, core.Object))
self.assertEqual(other_object.type.name, '__first')
self.assertTrue(isinstance(other_object, core.CdistObject))
self.assertEqual(other_object.cdist_type.name, '__first')
self.assertEqual(other_object.object_id, 'man')

2
lib/cdist/test/resolver/__init__.py
View file

@ -37,7 +37,7 @@ type_base_path = op.join(fixtures, 'type')
class ResolverTestCase(test.CdistTestCase):
def setUp(self):
self.objects = list(core.Object.list_objects(object_base_path, type_base_path))
self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
self.object_index = dict((o.name, o) for o in self.objects)
self.dependency_resolver = resolver.DependencyResolver(self.objects)

52
lib/cdist/test/type/__init__.py
View file

@ -33,115 +33,115 @@ class TypeTestCase(test.CdistTestCase):
def test_list_type_names(self):
base_path = op.join(fixtures, 'list_types')
type_names = core.Type.list_type_names(base_path)
type_names = core.CdistType.list_type_names(base_path)
self.assertEqual(type_names, ['__first', '__second', '__third'])
def test_list_types(self):
base_path = op.join(fixtures, 'list_types')
types = list(core.Type.list_types(base_path))
types = list(core.CdistType.list_types(base_path))
types_expected = [
core.Type(base_path, '__first'),
core.Type(base_path, '__second'),
core.Type(base_path, '__third'),
core.CdistType(base_path, '__first'),
core.CdistType(base_path, '__second'),
core.CdistType(base_path, '__third'),
]
self.assertEqual(types, types_expected)
def test_only_one_instance(self):
base_path = fixtures
cdist_type1 = core.Type(base_path, '__name_path')
cdist_type2 = core.Type(base_path, '__name_path')
cdist_type1 = core.CdistType(base_path, '__name_path')
cdist_type2 = core.CdistType(base_path, '__name_path')
self.assertEqual(id(cdist_type1), id(cdist_type2))
def test_nonexistent_type(self):
base_path = fixtures
self.assertRaises(core.NoSuchTypeError, core.Type, base_path, '__i-dont-exist')
self.assertRaises(core.NoSuchTypeError, core.CdistType, base_path, '__i-dont-exist')
def test_name(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.name, '__name_path')
def test_path(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.path, '__name_path')
def test_base_path(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.base_path, base_path)
def test_absolute_path(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.absolute_path, os.path.join(base_path, '__name_path'))
def test_manifest_path(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.manifest_path, os.path.join('__name_path', 'manifest'))
def test_explorer_path(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.explorer_path, os.path.join('__name_path', 'explorer'))
def test_gencode_local_path(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.gencode_local_path, os.path.join('__name_path', 'gencode-local'))
def test_gencode_remote_path(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.gencode_remote_path, os.path.join('__name_path', 'gencode-remote'))
def test_singleton_is_singleton(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__singleton')
cdist_type = core.CdistType(base_path, '__singleton')
self.assertTrue(cdist_type.is_singleton)
def test_not_singleton_is_singleton(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__not_singleton')
cdist_type = core.CdistType(base_path, '__not_singleton')
self.assertFalse(cdist_type.is_singleton)
def test_install_is_install(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__install')
cdist_type = core.CdistType(base_path, '__install')
self.assertTrue(cdist_type.is_install)
def test_not_install_is_install(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__not_install')
cdist_type = core.CdistType(base_path, '__not_install')
self.assertFalse(cdist_type.is_install)
def test_with_explorers(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__with_explorers')
cdist_type = core.CdistType(base_path, '__with_explorers')
self.assertEqual(cdist_type.explorers, ['whatever'])
def test_without_explorers(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__without_explorers')
cdist_type = core.CdistType(base_path, '__without_explorers')
self.assertEqual(cdist_type.explorers, [])
def test_with_required_parameters(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__with_required_parameters')
cdist_type = core.CdistType(base_path, '__with_required_parameters')
self.assertEqual(cdist_type.required_parameters, ['required1', 'required2'])
def test_without_required_parameters(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__without_required_parameters')
cdist_type = core.CdistType(base_path, '__without_required_parameters')
self.assertEqual(cdist_type.required_parameters, [])
def test_with_optional_parameters(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__with_optional_parameters')
cdist_type = core.CdistType(base_path, '__with_optional_parameters')
self.assertEqual(cdist_type.optional_parameters, ['optional1', 'optional2'])
def test_without_optional_parameters(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__without_optional_parameters')
cdist_type = core.CdistType(base_path, '__without_optional_parameters')
self.assertEqual(cdist_type.optional_parameters, [])

22
other/types_submitted_for_inclusion/__rsyncer/README.inclusion Normal file
View file

@ -0,0 +1,22 @@
Description:
Type that supports transfer of huge data, which is a general problem in
configuration management systems.
Good solution using standardised rsync approach.
Problem:
Uses root@$__target_host:$destination notation for transfer.
This breaks the concept of being able to replace __remote_exec and
__remote_copy and then doing chroot or different stuff.
This breaks for instance, if __remote_copy = cp and the destination is
a local chroot.
Solutions:
- Have cdist provide support for rsync syntax?
- Integrate __rsyncer more in line with philosohpy of other components
- Think about the general way of __rsyncer and what cdist would need
to provide for general solution.

48
other/types_submitted_for_inclusion/__rsyncer/gencode-local Executable file
View file

@ -0,0 +1,48 @@
#!/bin/sh
#
# Copyright (C) 2011 Daniel Maher (phrawzty+cdist at gmail.com)
#
# This file is part of cdist (https://github.com/telmich/cdist/).
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
source="$(cat "$__object/parameter/source")"
if [ -f "$__object/parameter/destination" ]; then
destination="$(cat "$__object/parameter/destination")"
else
destination="/$__object_id"
fi
# The system binary is probably ok, but if not...
if [ -f "$__object/parameter/rsyncbin" ]; then
rsyncbin="$(cat "$__object/parameter/rsyncbin")"
else
rsyncbin=`which rsync`
fi
args='-a'
# If the --delete argument should be passed to rsync.
if [ -f "$__object/parameter/delete" ]; then
args="$args --delete"
fi
# FIXME:
# - using root@ may break - find a good way to avoid this
# - align with __remote_{exec,copy} variables?
# Run rsync (locally).
echo "$rsyncbin $args $source root@$__target_host:$destination"

62
other/types_submitted_for_inclusion/__rsyncer/man.text Normal file
View file

@ -0,0 +1,62 @@
cdist-type__rsyncer(7)
======================
Daniel Maher <phrawzty+cdist at gmail.com>
NAME
----
cdist-type__rsyncer - Use rsync to copy files.
DESCRIPTION
-----------
This type is used to trigger rsync to copy files from the machine running cdist
(source) to the target machine in question (destination). The likely usage is
the rapid deployment of full directory trees, the cohorency of which can be
guarunteed with the optional --delete argument, which will remove any files
from the destination which are not present on the source.
REQUIRED PARAMETERS
-------------------
source::
The full path of the source from which to copy. This is passed directly
to rsync.
OPTIONAL PARAMETERS
-------------------
destination::
The full path of the destination. This is passed directly to rsync.
Default: object_id
delete::
If true, remove files from destination which are not in source. This is
effectively the --delete argument of rsync.
Default: false
rsyncbin::
Specify the full path to the rsync binary.
Default: `which rsync`
EXAMPLES
--------
--------------------------------------------------------------------------------
# Basic example
__rsyncer '/home/foo' --source '/opt/dist/foo'
# Fancier example
__rsyncer FOO --source '/opt/dist/foo' --destination '/home/foo/' --delete true
--------------------------------------------------------------------------------
SEE ALSO
--------
- cdist-type(7)
COPYING
-------
Copyright \(C) 2011 Daniel Maher. Free use of this software is granted under
the terms of the GNU General Public License version 3 (GPLv3).

3
other/types_submitted_for_inclusion/__rsyncer/parameter/optional Normal file
View file

@ -0,0 +1,3 @@
destination
delete
rsyncbin

1
other/types_submitted_for_inclusion/__rsyncer/parameter/required Normal file
View file

@ -0,0 +1 @@
source