matthijs/cdist
1
0
Fork 0
forked from ungleich-public/cdist
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'master' into __package_absent_present

Browse source
This commit is contained in:
Nico Schottelius 2012-02-16 16:35:47 +01:00
commit 605eaeb039
51 changed files with 1002 additions and 427 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README
View file

@ -89,7 +89,7 @@ cdist was tested or is know to run on at least
## Installation
### Preperation
### Preparation
Ensure you have Python 3.2 installed on the machine you use to **deploy to the targets**
(the ***source host***).

8
build
View file

@ -85,7 +85,7 @@ case "$1" in
;;
release)
"$0" clean && "$0" man && "$0" web
./doc/dev/releasechecklist
;;
speeches)
@ -113,8 +113,8 @@ case "$1" in
# cp ${SPEECHESDIR}/*.pdf ${WEBDIR}/${WEBBASE}/speeches
# git describe > ${WEBDIR}/${WEBBASE}/man/VERSION
cp ${MAN1DSTDIR}/*.html ${WEBMAN}/man1
cp ${MAN7DSTDIR}/*.html ${WEBMAN}/man7
cp ${MAN1DSTDIR}/*.html ${MAN1DSTDIR}/*.css ${WEBMAN}/man1
cp ${MAN7DSTDIR}/*.html ${MAN7DSTDIR}/*.css ${WEBMAN}/man7
cd ${WEBDIR} && git add ${WEBBASE}
cd ${WEBDIR} && git commit -m "cdist update" ${WEBBASE} ${WEBPAGE}
@ -123,7 +123,7 @@ case "$1" in
# Fix ikiwiki, which does not like symlinks for pseudo security
ssh tee.schottelius.org \
"cd /home/services/www/nico/www.nico.schottelius.org/www/software/cdist/man &&
ln -sf "$version" latest"
rm -f latest && ln -sf "$version" latest"
;;
p|pu|pub)

26
conf/explorer/runlevel Executable file
View file

@ -0,0 +1,26 @@
#!/bin/sh
#
# 2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
set +e
executable=$(which runlevel 2>/dev/null)
if [ -x "$executable" ]; then
"$executable" | awk '{ print $2 }'
fi

53
conf/type/__key_value/explorer/state Executable file
View file

@ -0,0 +1,53 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
key="$(cat "$__object/parameter/key" 2>/dev/null \
|| echo "$__object_id")"
state="$(cat "$__object/parameter/state" 2>/dev/null \
|| echo "present")"
file="$(cat "$__object/parameter/file")"
delimiter="$(cat "$__object/parameter/delimiter")"
value="$(cat "$__object/parameter/value" 2>/dev/null \
|| echo "__CDIST_NOTSET__")"
case "$state" in
absent)
if grep -q -E "^$key$delimiter+" "$file"; then
# if the key exists, with whatever value, we will have to remove it
# so report it as present
echo present
else
# key does not exist
echo absent
fi
;;
present)
if grep -q -E "^$key$delimiter+$value$" "$file"; then
# key exists and value is same
echo present
elif grep -q -E "^$key$delimiter+" "$file"; then
# key exists, but value is empty or different
echo wrongvalue
else
# key does not exist
echo absent
fi
;;
esac

55
conf/type/__key_value/gencode-remote
View file

@ -18,35 +18,40 @@
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
value_is="$(cat "$__object/explorer/value")"
value_should="$(cat "$__object/parameter/value")"
key="$(cat "$__object/parameter/key")"
file="$(cat "$__object/parameter/file")"
delimiter="$(cat "$__object/parameter/delimiter")"
value="$(cat "$__object/parameter/value")"
if [ "$value_is" != "$value_should" ]; then
case "$value_is" in
__NOTSET__)
# add key and value
echo "echo \"${key}${delimiter}${value_should}\" >> \"$file\""
;;
*)
if [ "$value_should" = '__NOTSET__' ]; then
# remove key and value
cat << DONE
sed -i '/^${key}/d' "$file"
DONE
else
# change value
cat << DONE
awk -F "$delimiter" '
/${key}${delimiter}*/{gsub("$value_is", "$value_should")};{print}' "$file" > "${file}+" \
&& mv "${file}+" "$file"
state_is="$(cat "$__object/explorer/state")"
state_should="$(cat "$__object/parameter/state")"
DONE
fi
;;
esac
if [ "$state_is" = "$state_should" ]; then
# nothing to do
exit 0
fi
case "$state_should" in
absent)
# remove lines starting with key
echo "sed -i '/^$key\($delimiter\+\)/d' \"$file\""
;;
present)
case "$state_is" in
absent)
# add new key and value
echo "echo \"${key}${delimiter}${value}\" >> \"$file\""
;;
wrongvalue)
# change exisiting value
echo "sed -i \"s|^$key\($delimiter\+\).*|$key\1$value|\" \"$file\""
;;
*)
echo "Unknown explorer state: $state_is" >&2
exit 1
esac
;;
*)
echo "Unknown state: $state_should" >&2
exit 1
esac

11
conf/type/__key_value/man.text
View file

@ -16,9 +16,6 @@ file.
REQUIRED PARAMETERS
-------------------
value::
The value for the key. Setting the value to `__NOTSET__` will remove the key
from the file.
file::
The file to operate on.
delimiter::
@ -27,8 +24,13 @@ delimiter::
OPTIONAL PARAMETERS
-------------------
state::
present or absent, defaults to present. If present, sets the key to value,
if absent, removes the key from the file.
key::
The key to change. Defaults to object_id.
value::
The value for the key. Optional if state=absent, required otherwise.
EXAMPLES
@ -45,6 +47,9 @@ __key_value my-fancy-id --file /etc/login.defs --key SYS_UID_MAX --value 666 \
# Enable packet forwarding
__key_value net.ipv4.ip_forward --file /etc/sysctl.conf --value 1 \
--delimiter '='
# Remove existing key/value
__key_value LEGACY_KEY --file /etc/somefile --state absent --delimiter '='
--------------------------------------------------------------------------------

14
conf/type/__key_value/manifest
View file

@ -18,9 +18,13 @@
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
if [ -f "$__object/parameter/key" ]; then
key="$(cat "$__object/parameter/key")"
else
echo "$__object_id" > "$__object/parameter/key"
fi
# set defaults
key="$(cat "$__object/parameter/key" 2>/dev/null \
|| echo "$__object_id" | tee "$__object/parameter/key")"
state="$(cat "$__object/parameter/state" 2>/dev/null \
|| echo "present" | tee "$__object/parameter/state")"
if [ "$state" = "present" -a ! -f "$__object/parameter/value" ]; then
echo "Missing required parameter 'value'" >&2
exit 1
fi

2
conf/type/__key_value/parameter/optional
View file

@ -1 +1,3 @@
key
value
state

1
conf/type/__key_value/parameter/required
View file

@ -1,3 +1,2 @@
value
file
delimiter

17
conf/type/__package_pip/explorer/state
View file

@ -35,14 +35,15 @@ else
pip="pip"
fi
# which is not posix, but command is :-)
# If there is no pip, it may get created from somebody else.
# If it will be created, there is probably no package installed.
if ! command -v "$pip" >/dev/null 2>&1; then
echo "No usable pip found at path \"$pip\"" >&2
exit 1
fi
if "$pip" freeze | grep -i -q "^$name=="; then
echo present
else
echo absent
else
if "$pip" freeze | grep -i -q "^$name=="; then
echo present
else
echo absent
fi
fi

2
conf/type/__package_pip/man.text
View file

@ -1,5 +1,5 @@
cdist-type__package_pip(7)
=============================
==========================
Nico Schottelius <nico-cdist--@--schottelius.org>

64
conf/type/__start_on_boot/explorer/state Executable file
View file

@ -0,0 +1,64 @@
#!/bin/sh
#
# 2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# Check whether the given name will be started on boot or not
#
os=$("$__explorer/os")
runlevel=$("$__explorer/runlevel")
name="$__object_id"
case "$os" in
archlinux)
# convert bash array to shell
daemons=$(grep ^DAEMONS /etc/rc.conf | sed -e 's/^.*=(//' -e 's/)$//')
# absent, as long as not found
state="absent"
# iterate, last one wins.
for daemon in $daemons; do
if [ "$daemon" = "$name" -o "$daemon" = "@${name}" ]; then
state="present"
elif [ "$daemon" = "!${name}" ]; then
state="absent"
fi
done
;;
debian|ubuntu)
state="present"
[ -f "/etc/rc$runlevel.d/S"??"$name" ] || state="absent"
;;
centos|fedora|owl|redhat)
state="present"
state=$(chkconfig --level "$runlevel" \"$name\" || echo absent)
[ "$state" ] || state="present"
;;
*)
echo "Unsupported os: $os" >&2
exit 1
;;
esac
echo $state

89
conf/type/__start_on_boot/gencode-remote Executable file
View file

@ -0,0 +1,89 @@
#!/bin/sh
#
# 2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
state_should="$(cat "$__object/parameter/state")"
state_is=$(cat "$__object/explorer/state")
# Nothing todo, go away
[ "$state_should" = "$state_is" ] && exit 0
os=$(cat "$__global/explorer/os")
name="$__object_id"
case "$state_should" in
present)
case "$os" in
archlinux)
echo "sed -i /etc/rc.conf \'s/^\\(DAEMONS=.*\\))/\\1 $name)/\'"
;;
debian|ubuntu)
echo "update-rc.d \"$name\" defaults >/dev/null"
;;
# FIXME: Disabled until the explorer is checked
# gentoo)
# echo rc-update add \"$name\" default
# ;;
centos|fedora|owl|redhat)
echo chkconfig \"$name\" on
;;
*)
echo "Unsupported os: $os" >&2
exit 1
;;
esac
;;
absent)
case "$os" in
archlinux)
# Replace a) at the beginning b) in the middle c) end d) only
# Support @name as well...makes it more ugly, but well...
echo "sed -i /etc/rc.conf -e 's/^\\(DAEMONS=(\\)@\\{0,1\\}$name /\\1/' -e 's/^\\(DAEMONS=(.* \\)@\\{0,1\\}$name \\(.*\\)/\\1\\2/' -e 's/^\\(DAEMONS=(.*\\) @\\{0,1\\}$name)/\\1)/' -e 's/^\\(DAEMONS=(\\)@\\{0,1\\}$name)/\\1)/'"
;;
debian|ubuntu)
echo update-rc.d -f \"$name\" remove
;;
# FIXME: Disabled until the explorer is checked
# gentoo)
# echo rc-update del \"$name\"
# ;;
centos|fedora|owl|redhat)
echo chkconfig \"$name\" off
;;
*)
echo "Unsupported os: $os" >&2
exit 1
;;
esac
;;
*)
echo "Unknown state: $state_should" >&2
exit 1
;;
esac

53
conf/type/__start_on_boot/man.text Normal file
View file

@ -0,0 +1,53 @@
cdist-type__start_on_boot(7)
============================
Nico Schottelius <nico-cdist--@--schottelius.org>
NAME
----
cdist-type__start_on_boot - Manage stuff to be started at boot
DESCRIPTION
-----------
This cdist type allows you to enable or disable stuff to be started
at boot of your operating system.
Warning: This type has not been tested intensively and is not fully
supported (i.e. gentoo and *bsd are not implemented).
REQUIRED PARAMETERS
-------------------
None.
OPTIONAL PARAMETERS
-------------------
state::
'present' or 'absent', defaults to 'present'
EXAMPLES
--------
--------------------------------------------------------------------------------
# Ensure snmpd is started at boot
__start_on_boot snmpd
# Same, but more explicit
__start_on_boot snmpd --state present
# Ensure legacy configuration management will not be started
__start_on_boot puppet --state absent
--------------------------------------------------------------------------------
SEE ALSO
--------
- cdist-type(7)
COPYING
-------
Copyright \(C) 2012 Nico Schottelius. Free use of this software is
granted under the terms of the GNU General Public License version 3 (GPLv3).

21
conf/type/__key_value/explorer/value → conf/type/__start_on_boot/manifest
View file

@ -1,6 +1,7 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
@ -17,21 +18,7 @@
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# Get the current value of key or __NOTSET__ if the key doesn't exist.
#
if [ -f "$__object/parameter/key" ]; then
key="$(cat "$__object/parameter/key")"
else
key="$__object_id"
fi
file="$(cat "$__object/parameter/file")"
delimiter="$(cat "$__object/parameter/delimiter")"
awk -F "$delimiter" '
BEGIN { found=0 }
/^'$key'/ { print $2; found=1 }
END { if (found) exit 0; else exit 1 }' "$file" \
|| echo "__NOTSET__"
# set default: present, if not setup
statefile="$__object/parameter/state"
[ -f "$statefile" ] || echo present > "$statefile"

1
conf/type/__start_on_boot/parameter/optional Normal file
View file

@ -0,0 +1 @@
state

18
conf/type/__user/gencode-remote
View file

@ -28,6 +28,7 @@ cd "$__object/parameter"
if grep -q "^${name}:" "$__object/explorer/passwd"; then
for property in $(ls .); do
new_value="$(cat "$property")"
unset current_value
file="$__object/explorer/passwd"
@ -36,9 +37,16 @@ if grep -q "^${name}:" "$__object/explorer/passwd"; then
if $(echo "$new_value" | grep -q '^[0-9][0-9]*$'); then
field=4
else
# group name
file="$__object/explorer/group"
field=1
# We were passed a group name. Compare the gid in
# the user's /etc/passwd entry with the gid of the
# group returned by the group explorer.
gid_from_group=$(awk -F: '{ print $3 }' "$__object/explorer/group")
gid_from_passwd=$(awk -F: '{ print $4 }' "$file")
if [ "$gid_from_group" != "$gid_from_passwd" ]; then
current_value="$gid_from_passwd"
else
current_value="$new_value"
fi
fi
;;
password)
@ -51,8 +59,12 @@ if grep -q "^${name}:" "$__object/explorer/passwd"; then
uid) field=3 ;;
esac
# If we haven't already set $current_value above, pull it from the
# appropriate file/field.
if [ -z "$current_value" ]; then
export field
current_value="$(awk -F: '{ print $ENVIRON["field"] }' < "$file")"
fi
if [ "$new_value" != "$current_value" ]; then
set -- "$@" "--$property" \'$new_value\'

13
doc/changelog
View file

@ -4,11 +4,22 @@ Changelog
* Changes are always commented with their author in (braces)
* Exception: No braces means author == Nico Schottelius
2.0.7:
2.0.8:
* Cleanup: Better hint to source of error
* Cleanup: Do not output failing script, but path to script only
* Cleanup: Remove support for __debug variable in manifests (Type != Core
debugging)
* Feature Core: Support boolean parameters (Steven Armstrong)
2.0.7: 2012-02-13
* Bugfix __file: Use chmod after chown/chgrp (Matt Coddington)
* Bugfix __user: Correct shadow field in explorer (Matt Coddington)
* Bugfix __link: Properly handle existing links (Steven Armstrong)
* Bugfix __key_value: More robust implementation (Steven Armstrong)
* Bugfix __user: Fix for changing a user's group by name (Matt Coddington)
* New Type: __package_pip
* Bugfix/Cleanup: Correctly allow Object ID to start and end with /, but
not contain //.
2.0.6: 2012-01-28
* Bugfix __apt_ppa:

18
doc/dev/logs/2012-02-10.object_id-and-slashes Normal file
View file

@ -0,0 +1,18 @@
__typename /foo/bar # possible, usual use case
require="__a//b" __typename /foo/bar # possible and happens often for __a/$id in loops
__typename /foo/bar/ # trailing slash will be stripped, can be documented
__typename /foo//bar//baz # // will be converted to / implicitly through fs; error prone; disallow
require="__a//b//c" __typename # // will be converted to / implicitly through fs; error prone; disallow
Solution:
1) allow require __a//b: type __a, object id /b
=> strip first slash of object id, as we do in non-dep-mode
2) allow _one_ trailing /: __type /foo/bar/ and require="__foo/abc/"
=> strip one leading slash of object id
3) disallow // within object id
4) disallow starting or ending / after 1) and 2)

23
doc/dev/logs/2012-02-13.dependencies Normal file
View file

@ -0,0 +1,23 @@
possible dependencies:
- unix pattern __foo/*
- object: __foo//bar, __foo/bar
- singleton with object_id: __foo/singleton
- singleton without object_id: __foo/
solving dependencies:
solve_dep(object, run_list):
- list = [me]
- if status == IN_DEPENDENCY:
fail: circular dependency
- status = IN_DEPENDENCY
- create_list_of_deps(object)
- try pattern expansion
- for each dependency:
if object does not exist:
fail
else:
list.append(solve_dep(object, run_list)):
- status == IN_LIST
- return [me, dependencies [, dependencies of dependencies]]

132
doc/dev/logs/2012-02-15.steven Normal file
View file

@ -0,0 +1,132 @@
- parameter/setting default from manifest
==> BRANCH[feature_default_parameters],
==> PERSON[Steven or Nico]
==> PROPOSAL(1)
- current bug
- proposal 1: parameter/default/$name (for optional ones)
- new way
- catches --state absent|present
- needs changes of types
- also possible for explorer
- support for it in core?
- handling of ${o} $o "$o" ?
- handling which variables?
- introduction of "templating language"
- aka macros
- possible problems:
- inconsistency
- redoing shell functionality
- raising expectations for more templating from users
- possible benefit
- no need for eval
- once in core, not everytime in type
- OTOH: one extra word.
- a=$(cat $__object/parameter/name) vs. $(eval $(cat $__object/parameter/name))
- only possible for static defaults
- --name overrides name not possible vs. object_id
- Is this the only case????
- if yes: don't care.
- possible solution:
- echo '/$__object_id' > typename/parameter/default/name
- eval $(cat $__object/parameter/name)
- probably allows code injection
- is possible anyway???
- $(cat /etc/shadow)
- other eval side effects???
- none: go for it
- some: have headache
- many: don't do
- proposal 2: 2 dbs (user input vs. stuff changable by type)
- explicit 2nd db [parameter_user and parameter/]
- not very clean (both agreed)
- proposal 3: parameter are read-only
- breaks current types (in core probably elsewhere)
- can't enforce, but user is on his own => breaks, her problem
+ clean seperation between core and type (nico)
- parameter belongs to type not core (steven)
- proposal 4: core ignores changes in parameter/* of object
- implicit 2nd db [see automagic below]
- steven+++
- does not work with divergent emulator not being in core
- because emulators primary db __is__ fs.
1 manifest:
__foo bar == emulator
echo present > $__global/object/__foo/bar/parameter/state
# fails
__foo bar == emulator
! automagic / filesystem
! fsproperty:
- kill, write explicitly to disk
==> BRANCH[cleanup_fsproperty]
==> PERSON[Steven]
==> PROPOSAL(just cleanup)
- implicit/automatic writes/read to fs
- explicit interfaces are better then implicit
- same problems as in cdist 1.x to 2.x move! (environment!)
- format on disk should not change/dictate code flow
- degrade python to shell (nico++! steven--)
- user should not care about python, ruby, .net or ASM implementation (steven++ nico++)
? proposal 1: diverge emulator / core
- emulator verifies input
- emulator writes to fs
- core reads/syncs from/to fs before passing control to user
? proposal 2: emulator is dumb and passes data to core
- core creates objects
- no fs involved
- core reads/syncs from/to fs before passing control to user
- passing:
- full objects via pickle
- parameters only
- how???
- unix socket?
- not everywhere possible?
- tcp / ip
- not everywhere possible
- chroot / local only
- rfc 1149
- not everywhere possible
- missing avian carriers
- 0mq
- not everywhere possible
- not installed
- shm (ipcs and friends)
- not everywhere possible
- no /dev/shm, different libraries? cleanups needed...
- what speaks against FS?
- emulator_input/.../
- nico: to fancy probably
! boolean implementation
==> BRANCH[feature_boolean_parameter]
==> PERSON[Steven]
- nico:
- parameters/boolean: document
- argparse changes (consider parameters/boolean)
- create
- can be implemented with changes in emulator
- use store_true, del if false => never seen by core
- INDEPENDENT INDEPENDT OF FS.PROPERTIES!!111111!
- emulator:
- how much integrated into core
- also: using CdistObject????
- dependency on filesystem: good (nico) | bad (steven)
- singleton / support without object_id
- not discussed
- __apt_ppa:
==> BRANCH[bugfix_do_not_change_state_in_manifest]
==> PERSON[Nico]
- logging divergent between emulator / core
- no problem (nico)
- may be helpful (steven)

10
doc/dev/releasechecklist
View file

@ -15,11 +15,21 @@ changelog_version=$(grep '^[[:digit:]]' doc/changelog | head -n1 | sed 's/:.*//'
#git_version=$(git describe)
lib_version=$(grep ^VERSION lib/cdist/__init__.py | sed -e 's/.*= //' -e 's/"//g')
# get date
date_today="$(date +%Y-%m-%d)"
date_changelog=$(grep '^[[:digit:]]' doc/changelog | head -n1 | sed 's/.*: //')
echo "Ensure you fixed/prepared version files: $files"
echo "changelog: $changelog_version"
#echo "git: $git_version"
echo "lib: $lib_version"
if [ "$date_today" != "$date_changelog" ]; then
echo "Messed up date, not releasing:"
echo "Changelog: $date_changelog"
exit 1
fi
if [ "$lib_version" != "$changelog_version" ]; then
echo "Messed up versions, not releasing"
exit 1

2
doc/dev/todo/TAKEME
View file

@ -35,3 +35,5 @@ USER INTERFACE
TYPES
------
- Add testing framework (proposed by Evax Software)
- __user
add option to include --create-home

10
doc/dev/todo/niconext
View file

@ -1,11 +1,13 @@
- __file/foo/bar//bar
- fails later
- introduce default parameters
- cleanup object_id handling
- have a look at singletons
- __user
add option to include --create-home
- ensure that all types, which support --state support
present and absent (consistent look and feel)
--------------------------------------------------------------------------------
- update/create docs
- cdist-cache::
How to get use information about the hosts we have been working on [advanced]

17
doc/man/cdist-reference.text.sh
View file

@ -101,12 +101,15 @@ conf/type/<name>/gencode-local::
conf/type/<name>/gencode-remote::
Used to generate code to be executed on the client.
conf/type/<name>/parameters/required::
conf/type/<name>/parameter/required::
Parameters required by type, \n seperated list.
conf/type/<name>/parameters/optional::
conf/type/<name>/parameter/optional::
Parameters optionally accepted by type, \n seperated list.
conf/type/<name>/parameter/boolean::
Boolean parameters accepted by type, \n seperated list.
conf/type/<name>/explorer::
Location of the type specific explorers.
This directory is referenced by the variable __type_explorer (see below).
@ -167,7 +170,7 @@ ENVIRONMENT VARIABLES
---------------------
__explorer::
Directory that contains all global explorers.
Available for: explorer
Available for: explorer, type explorer
__manifest::
Directory that contains the initial manifest.
Available for: initial manifest
@ -180,7 +183,11 @@ __object::
__object_id::
The type unique object id.
Available for: type manifest, type explorer, type gencode
Note: The leading "/" will always be stripped.
Note: The leading and the trailing "/" will always be stripped (caused by
the filesystem database and ensured by the core).
Note: Double slashes ("//") will not be fixed and result in an error.
__self::
DEPRECATED: Same as __object_name, do not use anymore, use __object_name instead.
Will be removed in cdist 3.x.
@ -189,7 +196,7 @@ __object_name::
Available for: type manifest, type explorer, type gencode
__target_host::
The host we are deploying to.
Available for: initial manifest, type manifest, type gencode
Available for: explorer, initial manifest, type explorer, type manifest, type gencode
__type::
Path to the current type.
Available for: type manifest, type gencode

7
doc/man/man7/cdist-hacker.text
View file

@ -61,12 +61,19 @@ including it.
HOW TO SUBMIT A NEW TYPE
------------------------
For detailled information about types, see cdist-type(7).
Submitting a type works as described above, with the additional requirement
that a corresponding manpage named man.text in asciidoc format with
the manpage-name "cdist-type__NAME" is included in the type directory
AND asciidoc is able to compile it (i.e. do NOT have to many "=" in the second
line).
Warning: Submitting "exec" or "run" types that simply echo their parameter in
gencode* will not be accepted, because they are of no use. Every type can output
code and thus such a type introduces redundant functionality that is given by
core cdist already.
SEE ALSO
--------

35
doc/man/man7/cdist-type.text
View file

@ -72,15 +72,42 @@ To begin a new type, just create the directory **conf/type/__NAME**.
DEFINING PARAMETERS
-------------------
Every type consists of optional and required parameters, which must
be created in a newline seperated file in ***parameters/required*** and
***parameters/optional***. If either or both missing, the type will have
no required, no optional or no parameters at all.
Every type consists of required, optional and boolean parameters, which must
be created in a newline seperated file in ***parameter/required***,
***parameter/optional*** and ***parameter/boolean***. If either is missing,
the type will have no required, no optional, no boolean or no parameters at
all.
Example:
--------------------------------------------------------------------------------
echo servername >> conf/type/__nginx_vhost/parameter/required
echo logdirectory >> conf/type/__nginx_vhost/parameter/optional
echo use_ssl >> conf/type/__nginx_vhost/parameter/boolean
--------------------------------------------------------------------------------
USING PARAMETERS
----------------
The parameters given to a type can be accessed and used in all type scripts
(e.g manifest, gencode-*, explorer/*). Note that boolean parameters are
represented by file existence. File exists -> True,
file does not exist -> False
Example: (e.g. in conf/type/__nginx_vhost/manifest)
--------------------------------------------------------------------------------
# required parameter
servername="$(cat "$__object/parameter/servername")"
# optional parameter
if [ -f "$__object/parameter/logdirectory" ]; then
logdirectory="$(cat "$__object/parameter/logdirectory")"
fi
# boolean parameter
if [ -f "$__object/parameter/use_ssl" ]; then
# file exists -> True
# do some fancy ssl stuff
fi
--------------------------------------------------------------------------------

14
lib/cdist/__init__.py
View file

@ -19,7 +19,7 @@
#
#
VERSION = "2.0.6"
VERSION = "2.0.7"
BANNER = """
.. . .x+=:. s
@ -44,15 +44,17 @@ class Error(Exception):
"""Base exception class for this project"""
pass
class CdistObjectError(Error):
"""Something went wrong with an object"""
class MissingEnvironmentVariableError(Error):
"""Raised when a required environment variable is not set."""
def __init__(self, cdist_object, message):
self.name = cdist_object.name
self.source = " ".join(cdist_object.source)
self.message = message
def __init__(self, name):
self.name = name
def __str__(self):
return 'Missing required environment variable: ' + str(self.name)
return '%s: %s (defined at %s)' % (self.name, self.message, self.source)
def file_to_list(filename):
"""Return list from \n seperated file"""

14
lib/cdist/config_install.py
View file

@ -89,9 +89,9 @@ class ConfigInstall(object):
new_objects_created = True
while new_objects_created:
new_objects_created = False
for cdist_object in core.Object.list_objects(self.local.object_path,
for cdist_object in core.CdistObject.list_objects(self.local.object_path,
self.local.type_path):
if cdist_object.state == core.Object.STATE_PREPARED:
if cdist_object.state == core.CdistObject.STATE_PREPARED:
self.log.debug("Skipping re-prepare of object %s", cdist_object)
continue
else:
@ -103,16 +103,16 @@ class ConfigInstall(object):
self.log.info("Running manifest and explorers for " + cdist_object.name)
self.explorer.run_type_explorers(cdist_object)
self.manifest.run_type_manifest(cdist_object)
cdist_object.state = core.Object.STATE_PREPARED
cdist_object.state = core.CdistObject.STATE_PREPARED
def object_run(self, cdist_object):
"""Run gencode and code for an object"""
self.log.debug("Trying to run object " + cdist_object.name)
if cdist_object.state == core.Object.STATE_DONE:
if cdist_object.state == core.CdistObject.STATE_DONE:
# TODO: remove once we are sure that this really never happens.
raise cdist.Error("Attempting to run an already finished object: %s", cdist_object)
cdist_type = cdist_object.type
cdist_type = cdist_object.cdist_type
# Generate
self.log.info("Generating and executing code for " + cdist_object.name)
@ -130,13 +130,13 @@ class ConfigInstall(object):
# Mark this object as done
self.log.debug("Finishing run of " + cdist_object.name)
cdist_object.state = core.Object.STATE_DONE
cdist_object.state = core.CdistObject.STATE_DONE
def stage_run(self):
"""The final (and real) step of deployment"""
self.log.info("Generating and executing code")
objects = core.Object.list_objects(
objects = core.CdistObject.list_objects(
self.local.object_path,
self.local.type_path)

10
lib/cdist/core/__init__.py
View file

@ -19,11 +19,11 @@
#
#
from cdist.core.type import Type
from cdist.core.type import NoSuchTypeError
from cdist.core.object import Object
from cdist.core.object import IllegalObjectIdError
from cdist.core.object import OBJECT_MARKER
from cdist.core.cdist_type import CdistType
from cdist.core.cdist_type import NoSuchTypeError
from cdist.core.cdist_object import CdistObject
from cdist.core.cdist_object import IllegalObjectIdError
from cdist.core.cdist_object import OBJECT_MARKER
from cdist.core.explorer import Explorer
from cdist.core.manifest import Manifest
from cdist.core.code import Code

84
lib/cdist/core/object.py → lib/cdist/core/cdist_object.py
View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
@ -42,7 +42,7 @@ class IllegalObjectIdError(cdist.Error):
return '%s: %s' % (self.message, self.object_id)
class Object(object):
class CdistObject(object):
"""Represents a cdist object.
All interaction with objects in cdist should be done through this class.
@ -61,7 +61,7 @@ class Object(object):
"""Return a list of object instances"""
for object_name in cls.list_object_names(object_base_path):
type_name, object_id = cls.split_name(object_name)
yield cls(cdist.core.Type(type_base_path, type_name), object_base_path, object_id=object_id)
yield cls(cdist.core.CdistType(type_base_path, type_name), object_base_path, object_id=object_id)
@classmethod
def list_type_names(cls, object_base_path):
@ -96,30 +96,58 @@ class Object(object):
"""
return os.path.join(type_name, object_id)
@staticmethod
def validate_object_id(object_id):
def validate_object_id(self):
# FIXME: also check that there is no object ID when type is singleton?
"""Validate the given object_id and raise IllegalObjectIdError if it's not valid.
"""
if object_id:
if object_id.startswith('/'):
raise IllegalObjectIdError(object_id, 'object_id may not start with /')
if OBJECT_MARKER in object_id.split(os.sep):
raise IllegalObjectIdError(object_id, 'object_id may not contain \'%s\'' % OBJECT_MARKER)
if self.object_id:
if OBJECT_MARKER in self.object_id.split(os.sep):
raise IllegalObjectIdError(self.object_id, 'object_id may not contain \'%s\'' % OBJECT_MARKER)
if '//' in self.object_id:
raise IllegalObjectIdError(self.object_id, 'object_id may not contain //')
# If no object_id and type is not singleton => error out
if not self.object_id and not self.cdist_type.is_singleton:
raise IllegalObjectIdError(self.object_id,
"Missing object_id and type is not a singleton.")
def __init__(self, cdist_type, base_path, object_id=None):
self.validate_object_id(object_id)
self.type = cdist_type # instance of Type
self.cdist_type = cdist_type # instance of Type
self.base_path = base_path
self.object_id = object_id
self.name = self.join_name(self.type.name, self.object_id)
self.path = os.path.join(self.type.path, self.object_id, OBJECT_MARKER)
self.validate_object_id()
self.sanitise_object_id()
self.name = self.join_name(self.cdist_type.name, self.object_id)
self.path = os.path.join(self.cdist_type.path, self.object_id, OBJECT_MARKER)
self.absolute_path = os.path.join(self.base_path, self.path)
self.code_local_path = os.path.join(self.path, "code-local")
self.code_remote_path = os.path.join(self.path, "code-remote")
self.parameter_path = os.path.join(self.path, "parameter")
def object_from_name(self, object_name):
"""Convenience method for creating an object instance from an object name.
Mainly intended to create objects when resolving requirements.
e.g:
<CdistObject __foo/bar>.object_from_name('__other/object') -> <CdistObject __other/object>
"""
base_path = self.base_path
type_path = self.cdist_type.base_path
type_name, object_id = self.split_name(object_name)
cdist_type = self.cdist_type.__class__(type_path, type_name)
return self.__class__(cdist_type, base_path, object_id=object_id)
def __repr__(self):
return '<Object %s>' % self.name
return '<CdistObject %s>' % self.name
def __eq__(self, other):
"""define equality as 'name is the same'"""
@ -128,23 +156,23 @@ class Object(object):
def __hash__(self):
return hash(self.name)
def __lt__(self, other):
return isinstance(other, self.__class__) and self.name < other.name
def object_from_name(self, object_name):
"""Convenience method for creating an object instance from an object name.
Mainly intended to create objects when resolving requirements.
e.g:
<Object __foo/bar>.object_from_name('__other/object') -> <Object __other/object>
def sanitise_object_id(self):
"""
type_path = self.type.base_path
base_path = self.base_path
type_name, object_id = self.split_name(object_name)
return self.__class__(self.type.__class__(type_path, type_name), base_path, object_id=object_id)
Remove leading and trailing slash (one only)
"""
# Allow empty object id for singletons
if self.object_id:
# Remove leading slash
if self.object_id[0] == '/':
self.object_id = self.object_id[1:]
# Remove trailing slash
if self.object_id[-1] == '/':
self.object_id = self.object_id[:-1]
# FIXME: still needed?
@property

23
lib/cdist/core/type.py → lib/cdist/core/cdist_type.py
View file

@ -34,7 +34,7 @@ class NoSuchTypeError(cdist.Error):
return "Type '%s' does not exist at %s" % (self.type_path, self.type_absolute_path)
class Type(object):
class CdistType(object):
"""Represents a cdist type.
All interaction with types in cdist should be done through this class.
@ -61,7 +61,7 @@ class Type(object):
# name is second argument
name = args[1]
if not name in cls._instances:
instance = super(Type, cls).__new__(cls)
instance = super(CdistType, cls).__new__(cls)
cls._instances[name] = instance
# return instance so __init__ is called
return cls._instances[name]
@ -82,9 +82,10 @@ class Type(object):
self.__explorers = None
self.__required_parameters = None
self.__optional_parameters = None
self.__boolean_parameters = None
def __repr__(self):
return '<Type %s>' % self.name
return '<CdistType %s>' % self.name
def __eq__(self, other):
return isinstance(other, self.__class__) and self.name == other.name
@ -144,3 +145,19 @@ class Type(object):
finally:
self.__optional_parameters = parameters
return self.__optional_parameters
@property
def boolean_parameters(self):
"""Return a list of boolean parameters"""
if not self.__boolean_parameters:
parameters = []
try:
with open(os.path.join(self.absolute_path, "parameter", "boolean")) as fd:
for line in fd:
parameters.append(line.strip())
except EnvironmentError:
# error ignored
pass
finally:
self.__boolean_parameters = parameters
return self.__boolean_parameters

7
lib/cdist/core/code.py
View file

@ -92,17 +92,14 @@ class Code(object):
'__global': self.local.out_path,
}
if log.getEffectiveLevel() == logging.DEBUG:
self.env.update({'__debug': "yes" })
def _run_gencode(self, cdist_object, which):
cdist_type = cdist_object.type
cdist_type = cdist_object.cdist_type
script = os.path.join(self.local.type_path, getattr(cdist_type, 'gencode_%s_path' % which))
if os.path.isfile(script):
env = os.environ.copy()
env.update(self.env)
env.update({
'__type': cdist_object.type.absolute_path,
'__type': cdist_object.cdist_type.absolute_path,
'__object': cdist_object.absolute_path,
'__object_id': cdist_object.object_id,
'__object_name': cdist_object.name,

34
lib/cdist/core/explorer.py
View file

@ -73,8 +73,6 @@ class Explorer(object):
'__target_host': self.target_host,
'__explorer': self.remote.global_explorer_path,
}
if self.log.getEffectiveLevel() == logging.DEBUG:
self.env.update({'__debug': "yes" })
self._type_explorers_transferred = []
### global
@ -121,15 +119,28 @@ class Explorer(object):
in the object.
"""
self.log.debug("Transfering type explorers for type: %s", cdist_object.type)
self.transfer_type_explorers(cdist_object.type)
self.log.debug("Transfering type explorers for type: %s", cdist_object.cdist_type)
self.transfer_type_explorers(cdist_object.cdist_type)
self.log.debug("Transfering object parameters for object: %s", cdist_object.name)
self.transfer_object_parameters(cdist_object)
for explorer in self.list_type_explorer_names(cdist_object.type):
for explorer in self.list_type_explorer_names(cdist_object.cdist_type):
output = self.run_type_explorer(explorer, cdist_object)
self.log.debug("Running type explorer '%s' for object '%s'", explorer, cdist_object.name)
cdist_object.explorers[explorer] = output
def run_type_explorer(self, explorer, cdist_object):
"""Run the given type explorer for the given object and return it's output."""
cdist_type = cdist_object.cdist_type
env = self.env.copy()
env.update({
'__object': os.path.join(self.remote.object_path, cdist_object.path),
'__object_id': cdist_object.object_id,
'__object_fq': cdist_object.path,
'__type_explorer': os.path.join(self.remote.type_path, cdist_type.explorer_path)
})
script = os.path.join(self.remote.type_path, cdist_type.explorer_path, explorer)
return self.remote.run_script(script, env=env, return_output=True)
def transfer_type_explorers(self, cdist_type):
"""Transfer the type explorers for the given type to the remote side."""
if cdist_type.explorers:
@ -149,16 +160,3 @@ class Explorer(object):
destination = os.path.join(self.remote.object_path, cdist_object.parameter_path)
self.remote.mkdir(destination)
self.remote.transfer(source, destination)
def run_type_explorer(self, explorer, cdist_object):
"""Run the given type explorer for the given object and return it's output."""
cdist_type = cdist_object.type
env = self.env.copy()
env.update({
'__object': os.path.join(self.remote.object_path, cdist_object.path),
'__object_id': cdist_object.object_id,
'__object_fq': cdist_object.path,
'__type_explorer': os.path.join(self.remote.type_path, cdist_type.explorer_path)
})
script = os.path.join(self.remote.type_path, cdist_type.explorer_path, explorer)
return self.remote.run_script(script, env=env, return_output=True)

4
lib/cdist/core/manifest.py
View file

@ -87,7 +87,7 @@ class Manifest(object):
self.local.run_script(script, env=env)
def run_type_manifest(self, cdist_object):
script = os.path.join(self.local.type_path, cdist_object.type.manifest_path)
script = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path)
if os.path.isfile(script):
env = os.environ.copy()
env.update(self.env)
@ -96,7 +96,7 @@ class Manifest(object):
'__object_id': cdist_object.object_id,
'__object_name': cdist_object.name,
'__self': cdist_object.name,
'__type': cdist_object.type.absolute_path,
'__type': cdist_object.cdist_type.absolute_path,
'__cdist_manifest': script,
})
self.local.run_script(script, env=env)

58
lib/cdist/emulator.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
@ -26,29 +26,22 @@ import os
import cdist
from cdist import core
class IllegalRequirementError(cdist.Error):
def __init__(self, requirement, message=None):
self.requirement = requirement
self.message = message or 'Illegal requirement'
def __str__(self):
return '%s: %s' % (self.message, self.requirement)
class Emulator(object):
def __init__(self, argv):
self.argv = argv
self.object_id = False
self.global_path = os.environ['__global']
self.object_source = os.environ['__cdist_manifest']
self.target_host = os.environ['__target_host']
# Internally only
self.object_source = os.environ['__cdist_manifest']
self.type_base_path = os.environ['__cdist_type_base_path']
self.object_base_path = os.path.join(self.global_path, "object")
self.type_name = os.path.basename(argv[0])
self.cdist_type = core.Type(self.type_base_path, self.type_name)
self.cdist_type = core.CdistType(self.type_base_path, self.type_name)
self.__init_log()
@ -94,7 +87,7 @@ class Emulator(object):
def commandline(self):
"""Parse command line"""
parser = argparse.ArgumentParser(add_help=False)
parser = argparse.ArgumentParser(add_help=False, argument_default=argparse.SUPPRESS)
for parameter in self.cdist_type.optional_parameters:
argument = "--" + parameter
@ -102,6 +95,9 @@ class Emulator(object):
for parameter in self.cdist_type.required_parameters:
argument = "--" + parameter
parser.add_argument(argument, dest=parameter, action='store', required=True)
for parameter in self.cdist_type.boolean_parameters:
argument = "--" + parameter
parser.add_argument(argument, dest=parameter, action='store_const', const='')
# If not singleton support one positional parameter
if not self.cdist_type.is_singleton:
@ -113,20 +109,15 @@ class Emulator(object):
def setup_object(self):
# FIXME: verify object id
# Setup object_id
# Setup object_id - FIXME: unset / do not setup anymore!
if self.cdist_type.is_singleton:
self.object_id = "singleton"
else:
self.object_id = self.args.object_id[0]
del self.args.object_id
# strip leading slash from object_id
self.object_id = self.object_id.lstrip('/')
# Instantiate the cdist object we are defining
self.cdist_object = core.Object(self.cdist_type, self.object_base_path, self.object_id)
self.cdist_object = core.CdistObject(self.cdist_type, self.object_base_path, self.object_id)
# Create object with given parameters
self.parameters = {}
@ -137,12 +128,15 @@ class Emulator(object):
if self.cdist_object.exists:
if self.cdist_object.parameters != self.parameters:
raise cdist.Error("Object %s already exists with conflicting parameters:\n%s: %s\n%s: %s"
% (self.cdist_object, " ".join(self.cdist_object.source), self.cdist_object.parameters, self.object_source, self.parameters)
% (self.cdist_object.name, " ".join(self.cdist_object.source), self.cdist_object.parameters, self.object_source, self.parameters)
)
else:
self.cdist_object.create()
self.cdist_object.parameters = self.parameters
# Record / Append source
self.cdist_object.source.append(self.object_source)
def record_requirements(self):
"""record requirements"""
@ -151,25 +145,17 @@ class Emulator(object):
self.log.debug("reqs = " + requirements)
for requirement in requirements.split(" "):
# Ignore empty fields - probably the only field anyway
if len(requirement) == 0:
continue
if len(requirement) == 0: continue
requirement_type_name, requirement_object_id = core.Object.split_name(requirement)
# Instantiate type which fails if type does not exist
requirement_type = core.Type(self.type_base_path, requirement_type_name)
if requirement_object_id:
# Validate object_id if any
core.Object.validate_object_id(requirement_object_id)
elif not requirement_type.is_singleton:
# Only singeltons have no object_id
raise IllegalRequirementError(requirement, "Missing object_id and type is not a singleton.")
# Raises an error, if object cannot be created
cdist_object = self.cdist_object.object_from_name(requirement)
self.log.debug("Recording requirement: " + requirement)
self.cdist_object.requirements.append(requirement)
# Record / Append source
self.cdist_object.source.append(self.object_source)
# Save the sanitised version, not the user supplied one
# (__file//bar => __file/bar)
# This ensures pattern matching is done against sanitised list
self.cdist_object.requirements.append(cdist_object.name)
def record_auto_requirements(self):
"""An object shall automatically depend on all objects that it defined in it's type manifest.

37
lib/cdist/exec/local.py
View file

@ -32,18 +32,6 @@ import logging
import cdist
from cdist import core
class LocalScriptError(cdist.Error):
def __init__(self, script, command, script_content):
self.script = script
self.command = command
self.script_content = script_content
def __str__(self):
plain_command = " ".join(self.command)
return "Local script execution failed: %s" % plain_command
class Local(object):
"""Execute commands locally.
@ -119,35 +107,16 @@ class Local(object):
command = ["/bin/sh", "-e"]
command.append(script)
self.log.debug("Local run script: %s", command)
if env is None:
env = os.environ.copy()
# Export __target_host for use in __remote_{copy,exec} scripts
env['__target_host'] = self.target_host
self.log.debug("Local run script env: %s", env)
try:
if return_output:
return subprocess.check_output(command, env=env).decode()
else:
subprocess.check_call(command, env=env)
except subprocess.CalledProcessError as error:
script_content = self.run(["cat", script], return_output=True)
self.log.error("Code that raised the error:\n%s", script_content)
raise LocalScriptError(script, command, script_content)
except EnvironmentError as error:
raise cdist.Error(" ".join(command) + ": " + error.args[1])
return self.run(command, env, return_output)
def link_emulator(self, exec_path):
"""Link emulator to types"""
src = os.path.abspath(exec_path)
for cdist_type in core.Type.list_types(self.type_path):
for cdist_type in core.CdistType.list_types(self.type_path):
dst = os.path.join(self.bin_path, cdist_type.name)
self.log.debug("Linking emulator: %s to %s", src, dst)
try:
os.symlink(src, dst)
except OSError as e:
raise cdist.Error("Linking emulator from " + src + " to " + dst + " failed: " + e.__str__())
raise cdist.Error("Linking emulator from %s to %s failed: %s" % (src, dst, e.__str__()))

76
lib/cdist/exec/remote.py
View file

@ -20,8 +20,6 @@
#
#
# FIXME: common base class with Local?
import io
import os
import sys
@ -30,17 +28,6 @@ import logging
import cdist
class RemoteScriptError(cdist.Error):
def __init__(self, script, command, script_content):
self.script = script
self.command = command
self.script_content = script_content
def __str__(self):
plain_command = " ".join(self.command)
return "Remote script execution failed: %s" % plain_command
class DecodeError(cdist.Error):
def __init__(self, command):
self.command = command
@ -93,6 +80,17 @@ class Remote(object):
command.extend(["-r", source, self.target_host + ":" + destination])
self._run_command(command)
def run_script(self, script, env=None, return_output=False):
"""Run the given script with the given environment on the remote side.
Return the output as a string.
"""
command = ["/bin/sh", "-e"]
command.append(script)
return self.run(command, env, return_output)
def run(self, command, env=None, return_output=False):
"""Run the given command with the given environment on the remote side.
Return the output as a string.
@ -101,7 +99,15 @@ class Remote(object):
# prefix given command with remote_exec
cmd = self._exec.split()
cmd.append(self.target_host)
# can't pass environment to remote side, so prepend command with
# variable declarations
if env:
remote_env = ["%s=%s" % item for item in env.items()]
cmd.extend(remote_env)
cmd.extend(command)
return self._run_command(cmd, env=env, return_output=return_output)
def _run_command(self, command, env=None, return_output=False):
@ -115,14 +121,6 @@ class Remote(object):
os_environ = os.environ.copy()
os_environ['__target_host'] = self.target_host
# can't pass environment to remote side, so prepend command with
# variable declarations
if env:
cmd = ["%s=%s" % item for item in env.items()]
cmd.extend(command)
else:
cmd = command
self.log.debug("Remote run: %s", command)
try:
if return_output:
@ -135,39 +133,3 @@ class Remote(object):
raise cdist.Error(" ".join(*args) + ": " + error.args[1])
except UnicodeDecodeError:
raise DecodeError(command)
def run_script(self, script, env=None, return_output=False):
"""Run the given script with the given environment on the remote side.
Return the output as a string.
"""
command = self._exec.split()
command.append(self.target_host)
# export target_host for use in __remote_{exec,copy} scripts
os_environ = os.environ.copy()
os_environ['__target_host'] = self.target_host
# can't pass environment to remote side, so prepend command with
# variable declarations
if env:
command.extend(["%s=%s" % item for item in env.items()])
command.extend(["/bin/sh", "-e"])
command.append(script)
self.log.debug("Remote run script: %s", command)
if env:
self.log.debug("Remote run script env: %s", env)
try:
if return_output:
return subprocess.check_output(command, env=os_environ).decode()
else:
subprocess.check_call(command, env=os_environ)
except subprocess.CalledProcessError as error:
script_content = self.run(["cat", script], return_output=True)
self.log.error("Code that raised the error:\n%s", script_content)
raise RemoteScriptError(script, command, script_content)
except EnvironmentError as error:
raise cdist.Error(" ".join(command) + ": " + error.args[1])

2
lib/cdist/resolver.py
View file

@ -125,7 +125,7 @@ class DependencyResolver(object):
resolved.append(cdist_object)
unresolved.remove(cdist_object)
except RequirementNotFoundError as e:
raise cdist.Error(cdist_object.name + " requires non-existing " + e.requirement)
raise cdist.CdistObjectError(cdist_object, "requires non-existing " + e.requirement)
def __iter__(self):
"""Iterate over all unique objects while resolving dependencies.

4
lib/cdist/test/code/__init__.py
View file

@ -54,8 +54,8 @@ class CodeTestCase(test.CdistTestCase):
self.code = code.Code(self.target_host, self.local, self.remote)
self.cdist_type = core.Type(self.local.type_path, '__dump_environment')
self.cdist_object = core.Object(self.cdist_type, self.local.object_path, 'whatever')
self.cdist_type = core.CdistType(self.local.type_path, '__dump_environment')
self.cdist_object = core.CdistObject(self.cdist_type, self.local.object_path, 'whatever')
self.cdist_object.create()
self.log = logging.getLogger("cdist")

68
lib/cdist/test/emulator/__init__.py
View file

@ -79,7 +79,7 @@ class EmulatorTestCase(test.CdistTestCase):
os.environ.update(self.env)
os.environ['require'] = '__file'
emu = emulator.Emulator(argv)
self.assertRaises(emulator.IllegalRequirementError, emu.run)
self.assertRaises(core.IllegalObjectIdError, emu.run)
def test_singleton_object_requirement(self):
argv = ['__file', '/tmp/foobar']
@ -119,14 +119,14 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase):
def test_autorequire(self):
initial_manifest = os.path.join(self.local.manifest_path, "init")
self.manifest.run_initial_manifest(initial_manifest)
cdist_type = core.Type(self.local.type_path, '__saturn')
cdist_object = core.Object(cdist_type, self.local.object_path, 'singleton')
cdist_type = core.CdistType(self.local.type_path, '__saturn')
cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'singleton')
self.manifest.run_type_manifest(cdist_object)
expected = ['__planet/Saturn', '__moon/Prometheus']
self.assertEqual(sorted(cdist_object.requirements), sorted(expected))
class ArgumentsWithDashesTestCase(test.CdistTestCase):
class ArgumentsTestCase(test.CdistTestCase):
def setUp(self):
self.temp_dir = self.mkdtemp()
@ -156,6 +156,62 @@ class ArgumentsWithDashesTestCase(test.CdistTestCase):
emu = emulator.Emulator(argv)
emu.run()
cdist_type = core.Type(self.local.type_path, '__arguments_with_dashes')
cdist_object = core.Object(cdist_type, self.local.object_path, 'some-id')
cdist_type = core.CdistType(self.local.type_path, '__arguments_with_dashes')
cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'some-id')
self.assertTrue('with-dash' in cdist_object.parameters)
def test_boolean(self):
type_name = '__arguments_boolean'
object_id = 'some-id'
argv = [type_name, object_id, '--boolean1']
os.environ.update(self.env)
emu = emulator.Emulator(argv)
emu.run()
cdist_type = core.CdistType(self.local.type_path, type_name)
cdist_object = core.CdistObject(cdist_type, self.local.object_path, object_id)
self.assertTrue('boolean1' in cdist_object.parameters)
self.assertFalse('boolean2' in cdist_object.parameters)
# empty file -> True
self.assertTrue(cdist_object.parameters['boolean1'] == '')
def test_required(self):
type_name = '__arguments_required'
object_id = 'some-id'
value = 'some value'
argv = [type_name, object_id, '--required1', value, '--required2', value]
os.environ.update(self.env)
emu = emulator.Emulator(argv)
emu.run()
cdist_type = core.CdistType(self.local.type_path, type_name)
cdist_object = core.CdistObject(cdist_type, self.local.object_path, object_id)
self.assertTrue('required1' in cdist_object.parameters)
self.assertTrue('required2' in cdist_object.parameters)
self.assertEqual(cdist_object.parameters['required1'], value)
self.assertEqual(cdist_object.parameters['required2'], value)
# def test_required_missing(self):
# type_name = '__arguments_required'
# object_id = 'some-id'
# value = 'some value'
# argv = [type_name, object_id, '--required1', value]
# os.environ.update(self.env)
# emu = emulator.Emulator(argv)
#
# self.assertRaises(SystemExit, emu.run)
def test_optional(self):
type_name = '__arguments_optional'
object_id = 'some-id'
value = 'some value'
argv = [type_name, object_id, '--optional1', value]
os.environ.update(self.env)
emu = emulator.Emulator(argv)
emu.run()
cdist_type = core.CdistType(self.local.type_path, type_name)
cdist_object = core.CdistObject(cdist_type, self.local.object_path, object_id)
self.assertTrue('optional1' in cdist_object.parameters)
self.assertFalse('optional2' in cdist_object.parameters)
self.assertEqual(cdist_object.parameters['optional1'], value)

2
lib/cdist/test/emulator/fixtures/conf/type/__arguments_boolean/parameter/boolean Normal file
View file

@ -0,0 +1,2 @@
boolean1
boolean2

1
lib/cdist/test/emulator/fixtures/conf/type/__arguments_optional/parameter/optional Normal file
View file

@ -0,0 +1 @@
optional1

2
lib/cdist/test/emulator/fixtures/conf/type/__arguments_required/parameter/required Normal file
View file

@ -0,0 +1,2 @@
required1
required2

18
lib/cdist/test/explorer/__init__.py
View file

@ -83,19 +83,19 @@ class ExplorerClassTestCase(test.CdistTestCase):
shutil.rmtree(out_path)
def test_list_type_explorer_names(self):
cdist_type = core.Type(self.local.type_path, '__test_type')
cdist_type = core.CdistType(self.local.type_path, '__test_type')
expected = cdist_type.explorers
self.assertEqual(self.explorer.list_type_explorer_names(cdist_type), expected)
def test_transfer_type_explorers(self):
cdist_type = core.Type(self.local.type_path, '__test_type')
cdist_type = core.CdistType(self.local.type_path, '__test_type')
self.explorer.transfer_type_explorers(cdist_type)
source = os.path.join(self.local.type_path, cdist_type.explorer_path)
destination = os.path.join(self.remote.type_path, cdist_type.explorer_path)
self.assertEqual(os.listdir(source), os.listdir(destination))
def test_transfer_type_explorers_only_once(self):
cdist_type = core.Type(self.local.type_path, '__test_type')
cdist_type = core.CdistType(self.local.type_path, '__test_type')
# first transfer
self.explorer.transfer_type_explorers(cdist_type)
source = os.path.join(self.local.type_path, cdist_type.explorer_path)
@ -109,8 +109,8 @@ class ExplorerClassTestCase(test.CdistTestCase):
self.assertFalse(os.listdir(destination))
def test_transfer_object_parameters(self):
cdist_type = core.Type(self.local.type_path, '__test_type')
cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever')
cdist_type = core.CdistType(self.local.type_path, '__test_type')
cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever')
cdist_object.create()
cdist_object.parameters = {'first': 'first value', 'second': 'second value'}
self.explorer.transfer_object_parameters(cdist_object)
@ -119,15 +119,15 @@ class ExplorerClassTestCase(test.CdistTestCase):
self.assertEqual(sorted(os.listdir(source)), sorted(os.listdir(destination)))
def test_run_type_explorer(self):
cdist_type = core.Type(self.local.type_path, '__test_type')
cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever')
cdist_type = core.CdistType(self.local.type_path, '__test_type')
cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever')
self.explorer.transfer_type_explorers(cdist_type)
output = self.explorer.run_type_explorer('world', cdist_object)
self.assertEqual(output, 'hello\n')
def test_run_type_explorers(self):
cdist_type = core.Type(self.local.type_path, '__test_type')
cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever')
cdist_type = core.CdistType(self.local.type_path, '__test_type')
cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever')
cdist_object.create()
self.explorer.run_type_explorers(cdist_object)
self.assertEqual(cdist_object.explorers, {'world': 'hello'})

4
lib/cdist/test/manifest/__init__.py
View file

@ -79,8 +79,8 @@ class ManifestTestCase(test.CdistTestCase):
self.assertEqual(output_dict['__manifest'], self.local.manifest_path)
def test_type_manifest_environment(self):
cdist_type = core.Type(self.local.type_path, '__dump_environment')
cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever')
cdist_type = core.CdistType(self.local.type_path, '__dump_environment')
cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever')
handle, output_file = self.mkstemp(dir=self.temp_dir)
os.close(handle)
os.environ['__cdist_test_out'] = output_file

50
lib/cdist/test/object/__init__.py
View file

@ -25,6 +25,8 @@ import shutil
from cdist import test
from cdist import core
import cdist
import os.path as op
my_dir = op.abspath(op.dirname(__file__))
fixtures = op.join(my_dir, 'fixtures')
@ -34,48 +36,48 @@ type_base_path = op.join(fixtures, 'type')
class ObjectClassTestCase(test.CdistTestCase):
def test_list_object_names(self):
object_names = list(core.Object.list_object_names(object_base_path))
object_names = list(core.CdistObject.list_object_names(object_base_path))
self.assertEqual(object_names, ['__first/man', '__second/on-the', '__third/moon'])
def test_list_type_names(self):
type_names = list(core.Object.list_type_names(object_base_path))
type_names = list(cdist.core.CdistObject.list_type_names(object_base_path))
self.assertEqual(type_names, ['__first', '__second', '__third'])
def test_list_objects(self):
objects = list(core.Object.list_objects(object_base_path, type_base_path))
objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
objects_expected = [
core.Object(core.Type(type_base_path, '__first'), object_base_path, 'man'),
core.Object(core.Type(type_base_path, '__second'), object_base_path, 'on-the'),
core.Object(core.Type(type_base_path, '__third'), object_base_path, 'moon'),
core.CdistObject(core.CdistType(type_base_path, '__first'), object_base_path, 'man'),
core.CdistObject(core.CdistType(type_base_path, '__second'), object_base_path, 'on-the'),
core.CdistObject(core.CdistType(type_base_path, '__third'), object_base_path, 'moon'),
]
self.assertEqual(objects, objects_expected)
class ObjectIdTestCase(test.CdistTestCase):
def test_object_id_starts_with_slash(self):
cdist_type = core.Type(type_base_path, '__third')
illegal_object_id = '/object_id/may/not/start/with/slash'
def test_object_id_contains_double_slash(self):
cdist_type = core.CdistType(type_base_path, '__third')
illegal_object_id = '/object_id//may/not/contain/double/slash'
with self.assertRaises(core.IllegalObjectIdError):
core.Object(cdist_type, object_base_path, illegal_object_id)
core.CdistObject(cdist_type, object_base_path, illegal_object_id)
def test_object_id_contains_object_marker(self):
cdist_type = core.Type(type_base_path, '__third')
cdist_type = core.CdistType(type_base_path, '__third')
illegal_object_id = 'object_id/may/not/contain/%s/anywhere' % core.OBJECT_MARKER
with self.assertRaises(core.IllegalObjectIdError):
core.Object(cdist_type, object_base_path, illegal_object_id)
core.CdistObject(cdist_type, object_base_path, illegal_object_id)
def test_object_id_contains_object_marker_string(self):
cdist_type = core.Type(type_base_path, '__third')
cdist_type = core.CdistType(type_base_path, '__third')
illegal_object_id = 'object_id/may/contain_%s_in_filename' % core.OBJECT_MARKER
core.Object(cdist_type, object_base_path, illegal_object_id)
core.CdistObject(cdist_type, object_base_path, illegal_object_id)
# if we get here, the test passed
class ObjectTestCase(test.CdistTestCase):
def setUp(self):
self.cdist_type = core.Type(type_base_path, '__third')
self.cdist_object = core.Object(self.cdist_type, object_base_path, 'moon')
self.cdist_type = core.CdistType(type_base_path, '__third')
self.cdist_object = core.CdistObject(self.cdist_type, object_base_path, 'moon')
def tearDown(self):
self.cdist_object.changed = False
@ -159,16 +161,16 @@ class ObjectTestCase(test.CdistTestCase):
self.assertEqual(self.cdist_object.state, '')
def test_state_prepared(self):
self.cdist_object.state = core.Object.STATE_PREPARED
self.assertEqual(self.cdist_object.state, core.Object.STATE_PREPARED)
self.cdist_object.state = core.CdistObject.STATE_PREPARED
self.assertEqual(self.cdist_object.state, core.CdistObject.STATE_PREPARED)
def test_state_running(self):
self.cdist_object.state = core.Object.STATE_RUNNING
self.assertEqual(self.cdist_object.state, core.Object.STATE_RUNNING)
self.cdist_object.state = core.CdistObject.STATE_RUNNING
self.assertEqual(self.cdist_object.state, core.CdistObject.STATE_RUNNING)
def test_state_done(self):
self.cdist_object.state = core.Object.STATE_DONE
self.assertEqual(self.cdist_object.state, core.Object.STATE_DONE)
self.cdist_object.state = core.CdistObject.STATE_DONE
self.assertEqual(self.cdist_object.state, core.CdistObject.STATE_DONE)
def test_source(self):
self.assertEqual(list(self.cdist_object.source), [])
@ -195,6 +197,6 @@ class ObjectTestCase(test.CdistTestCase):
self.cdist_object.code_remote = 'Hello World'
other_name = '__first/man'
other_object = self.cdist_object.object_from_name(other_name)
self.assertTrue(isinstance(other_object, core.Object))
self.assertEqual(other_object.type.name, '__first')
self.assertTrue(isinstance(other_object, core.CdistObject))
self.assertEqual(other_object.cdist_type.name, '__first')
self.assertEqual(other_object.object_id, 'man')

2
lib/cdist/test/resolver/__init__.py
View file

@ -37,7 +37,7 @@ type_base_path = op.join(fixtures, 'type')
class ResolverTestCase(test.CdistTestCase):
def setUp(self):
self.objects = list(core.Object.list_objects(object_base_path, type_base_path))
self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path))
self.object_index = dict((o.name, o) for o in self.objects)
self.dependency_resolver = resolver.DependencyResolver(self.objects)

63
lib/cdist/test/type/__init__.py
View file

@ -33,115 +33,126 @@ class TypeTestCase(test.CdistTestCase):
def test_list_type_names(self):
base_path = op.join(fixtures, 'list_types')
type_names = core.Type.list_type_names(base_path)
type_names = core.CdistType.list_type_names(base_path)
self.assertEqual(type_names, ['__first', '__second', '__third'])
def test_list_types(self):
base_path = op.join(fixtures, 'list_types')
types = list(core.Type.list_types(base_path))
types = list(core.CdistType.list_types(base_path))
types_expected = [
core.Type(base_path, '__first'),
core.Type(base_path, '__second'),
core.Type(base_path, '__third'),
core.CdistType(base_path, '__first'),
core.CdistType(base_path, '__second'),
core.CdistType(base_path, '__third'),
]
self.assertEqual(types, types_expected)
def test_only_one_instance(self):
base_path = fixtures
cdist_type1 = core.Type(base_path, '__name_path')
cdist_type2 = core.Type(base_path, '__name_path')
cdist_type1 = core.CdistType(base_path, '__name_path')
cdist_type2 = core.CdistType(base_path, '__name_path')
self.assertEqual(id(cdist_type1), id(cdist_type2))
def test_nonexistent_type(self):
base_path = fixtures
self.assertRaises(core.NoSuchTypeError, core.Type, base_path, '__i-dont-exist')
self.assertRaises(core.NoSuchTypeError, core.CdistType, base_path, '__i-dont-exist')
def test_name(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.name, '__name_path')
def test_path(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.path, '__name_path')
def test_base_path(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.base_path, base_path)
def test_absolute_path(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.absolute_path, os.path.join(base_path, '__name_path'))
def test_manifest_path(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.manifest_path, os.path.join('__name_path', 'manifest'))
def test_explorer_path(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.explorer_path, os.path.join('__name_path', 'explorer'))
def test_gencode_local_path(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.gencode_local_path, os.path.join('__name_path', 'gencode-local'))
def test_gencode_remote_path(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__name_path')
cdist_type = core.CdistType(base_path, '__name_path')
self.assertEqual(cdist_type.gencode_remote_path, os.path.join('__name_path', 'gencode-remote'))
def test_singleton_is_singleton(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__singleton')
cdist_type = core.CdistType(base_path, '__singleton')
self.assertTrue(cdist_type.is_singleton)
def test_not_singleton_is_singleton(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__not_singleton')
cdist_type = core.CdistType(base_path, '__not_singleton')
self.assertFalse(cdist_type.is_singleton)
def test_install_is_install(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__install')
cdist_type = core.CdistType(base_path, '__install')
self.assertTrue(cdist_type.is_install)
def test_not_install_is_install(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__not_install')
cdist_type = core.CdistType(base_path, '__not_install')
self.assertFalse(cdist_type.is_install)
def test_with_explorers(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__with_explorers')
cdist_type = core.CdistType(base_path, '__with_explorers')
self.assertEqual(cdist_type.explorers, ['whatever'])
def test_without_explorers(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__without_explorers')
cdist_type = core.CdistType(base_path, '__without_explorers')
self.assertEqual(cdist_type.explorers, [])
def test_with_required_parameters(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__with_required_parameters')
cdist_type = core.CdistType(base_path, '__with_required_parameters')
self.assertEqual(cdist_type.required_parameters, ['required1', 'required2'])
def test_without_required_parameters(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__without_required_parameters')
cdist_type = core.CdistType(base_path, '__without_required_parameters')
self.assertEqual(cdist_type.required_parameters, [])
def test_with_optional_parameters(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__with_optional_parameters')
cdist_type = core.CdistType(base_path, '__with_optional_parameters')
self.assertEqual(cdist_type.optional_parameters, ['optional1', 'optional2'])
def test_without_optional_parameters(self):
base_path = fixtures
cdist_type = core.Type(base_path, '__without_optional_parameters')
cdist_type = core.CdistType(base_path, '__without_optional_parameters')
self.assertEqual(cdist_type.optional_parameters, [])
def test_with_boolean_parameters(self):
base_path = fixtures
cdist_type = core.CdistType(base_path, '__with_boolean_parameters')
self.assertEqual(cdist_type.boolean_parameters, ['boolean1', 'boolean2'])
def test_without_boolean_parameters(self):
base_path = fixtures
cdist_type = core.CdistType(base_path, '__without_boolean_parameters')
self.assertEqual(cdist_type.boolean_parameters, [])

2
lib/cdist/test/type/fixtures/__with_boolean_parameters/parameter/boolean Normal file
View file

@ -0,0 +1,2 @@
boolean1
boolean2

0
lib/cdist/test/type/fixtures/__without_boolean_parameters/.keep Normal file
View file