Merge branch '2.1'

Signed-off-by: Nico Schottelius <nico@brief.schottelius.org>

Conflicts:
	docs/changelog
This commit is contained in:
Nico Schottelius 2012-11-02 15:59:32 +01:00
commit 9eef95a0b6
518 changed files with 1087 additions and 1092 deletions

29
.gitignore vendored
View File

@ -2,20 +2,27 @@
.*.swp
# Ignore generated manpages
doc/man/.marker
doc/man/man1/*.1
doc/man/man7/*.7
doc/man/man*/*.html
doc/man/man*/*.xml
doc/man/man7/cdist-type__*.text
doc/man/man7/cdist-reference.text
doc/man/man*/docbook-xsl.css
docs/man/.marker
docs/man/man1/*.1
docs/man/man7/*.7
docs/man/man*/*.html
docs/man/man*/*.xml
docs/man/man*/docbook-xsl.css
docs/man/man7/cdist-type__*.text
docs/man/man7/cdist-reference.text
# Ignore cdist cache for version control
/cache/
# Python / cache
# Python: cache, distutils, distribution in general
__pycache__/
MANIFEST
dist/
cdist/version.py
# Is static and will never be included
lib/cdist/version_static.py
# Packaging: Archlinux
/PKGBUILD
/cdist-*.pkg.tar.xz
/cdist-*.tar.gz
/pkg
/src

View File

@ -1 +1 @@
2.0.14
2.1.0-pre1

3
MANIFEST.in Normal file
View File

@ -0,0 +1,3 @@
include docs/changelog
recursive-include docs/gfx *.png *.text
recursive-include docs *.text *.html *.1 *.7

26
PKGBUILD.in Executable file
View File

@ -0,0 +1,26 @@
#!/bin/sh
version=$(git describe)
outfile=${0%.in}
cat << eof > "${outfile}"
pkgname=cdist
pkgver=$version
pkgrel=1
pkgdesc='A Usable Configuration Management System"'
arch=('any')
url='http://www.nico.schottelius.org/software/cdist/'
license=('GPL3')
depends=('python>=3.2.0')
source=("http://pypi.python.org/packages/source/c/cdist/cdist-\${pkgver}.tar.gz")
package() {
cd cdist-\${pkgver}
python3 setup.py build install --root="\${pkgdir}"
mv "\${pkgdir}"/usr/bin/cdist.py "\${pkgdir}"/usr/bin/cdist
#install -Dm644 offlineimap.1 "\${pkgdir}"/usr/share/man/man1/offlineimap.1
}
eof
makepkg -g >> "${outfile}"

235
bin/cdist
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3
#!/bin/sh
# -*- coding: utf-8 -*-
#
# 2010-2012 Nico Schottelius (nico-cdist at schottelius.org)
# 2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
@ -20,229 +20,10 @@
#
#
def commandline():
"""Parse command line"""
import argparse
# Wrapper for real script to allow execution from checkout
dir=${0%/*}
import cdist.banner
import cdist.config
import cdist.install
# Construct parser others can reuse
parser = {}
# Options _all_ parsers have in common
parser['loglevel'] = argparse.ArgumentParser(add_help=False)
parser['loglevel'].add_argument('-d', '--debug',
help='Set log level to debug', action='store_true',
default=False)
parser['loglevel'].add_argument('-v', '--verbose',
help='Set log level to info, be more verbose',
action='store_true', default=False)
# Main subcommand parser
parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION,
parents=[parser['loglevel']])
parser['main'].add_argument('-V', '--version',
help='Show version', action='version',
version='%(prog)s ' + cdist.VERSION)
parser['sub'] = parser['main'].add_subparsers(title="Commands")
# Banner
parser['banner'] = parser['sub'].add_parser('banner',
parents=[parser['loglevel']])
parser['banner'].set_defaults(func=cdist.banner.banner)
# Config and install (common stuff)
parser['configinstall'] = argparse.ArgumentParser(add_help=False)
parser['configinstall'].add_argument('host', nargs='+',
help='one or more hosts to operate on')
parser['configinstall'].add_argument('-c', '--cdist-home',
help='Change cdist home (default: .. from bin directory)',
action='store')
parser['configinstall'].add_argument('-i', '--initial-manifest',
help='Path to a cdist manifest or \'-\' to read from stdin.',
dest='manifest', required=False)
parser['configinstall'].add_argument('-p', '--parallel',
help='Operate on multiple hosts in parallel',
action='store_true', dest='parallel')
parser['configinstall'].add_argument('-s', '--sequential',
help='Operate on multiple hosts sequentially (default)',
action='store_false', dest='parallel')
parser['configinstall'].add_argument('--remote-copy',
help='Command to use for remote copy (should behave like scp)',
action='store', dest='remote_copy',
default="scp -o User=root -q")
parser['configinstall'].add_argument('--remote-exec',
help='Command to use for remote execution (should behave like ssh)',
action='store', dest='remote_exec',
default="ssh -o User=root -q")
# Config
parser['config'] = parser['sub'].add_parser('config',
parents=[parser['loglevel'], parser['configinstall']])
parser['config'].set_defaults(func=config)
# Install
# 20120525/sar: commented until it actually does something
#parser['install'] = parser['sub'].add_parser('install',
# parents=[parser['loglevel'], parser['configinstall']])
#parser['install'].set_defaults(func=install)
for p in parser:
parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
args = parser['main'].parse_args(sys.argv[1:])
# Loglevels are handled globally in here and debug wins over verbose
if args.verbose:
logging.root.setLevel(logging.INFO)
if args.debug:
logging.root.setLevel(logging.DEBUG)
log.debug(args)
args.func(args)
def config(args):
configinstall(args, mode=cdist.config.Config)
def install(args):
configinstall(args, mode=cdist.install.Install)
def configinstall(args, mode):
"""Configure or install remote system"""
import multiprocessing
import time
initial_manifest_tempfile = None
if args.manifest == '-':
# read initial manifest from stdin
import tempfile
try:
handle, initial_manifest_temp_path = tempfile.mkstemp(prefix='cdist.stdin.')
with os.fdopen(handle, 'w') as fd:
fd.write(sys.stdin.read())
except (IOError, OSError) as e:
raise cdist.Error("Creating tempfile for stdin data failed: %s" % e)
args.manifest = initial_manifest_temp_path
import atexit
atexit.register(lambda: os.remove(initial_manifest_temp_path))
process = {}
failed_hosts = []
time_start = time.time()
for host in args.host:
if args.parallel:
log.debug("Creating child process for %s", host)
process[host] = multiprocessing.Process(target=configinstall_onehost, args=(host, args, mode, True))
process[host].start()
else:
try:
configinstall_onehost(host, args, mode, parallel=False)
except cdist.Error as e:
failed_hosts.append(host)
# Catch errors in parallel mode when joining
if args.parallel:
for host in process.keys():
log.debug("Joining process %s", host)
process[host].join()
if not process[host].exitcode == 0:
failed_hosts.append(host)
time_end = time.time()
log.info("Total processing time for %s host(s): %s", len(args.host),
(time_end - time_start))
if len(failed_hosts) > 0:
raise cdist.Error("Failed to deploy to the following hosts: " +
" ".join(failed_hosts))
def configinstall_onehost(host, args, mode, parallel):
"""Configure or install ONE remote system"""
try:
import cdist.context
context = cdist.context.Context(
target_host=host,
remote_copy=args.remote_copy,
remote_exec=args.remote_exec,
initial_manifest=args.manifest,
base_path=args.cdist_home,
exec_path=sys.argv[0],
debug=args.debug)
c = mode(context)
c.deploy_and_cleanup()
context.cleanup()
except cdist.Error as e:
# We are running in our own process here, need to sys.exit!
if parallel:
log.error(e)
sys.exit(1)
else:
raise
except KeyboardInterrupt:
# Ignore in parallel mode, we are existing anyway
if parallel:
sys.exit(0)
# Pass back to controlling code in sequential mode
else:
raise
def emulator():
"""Prepare and run emulator"""
import cdist.emulator
emulator = cdist.emulator.Emulator(sys.argv)
return emulator.run()
if __name__ == "__main__":
# Sys is needed for sys.exit()
import sys
cdistpythonversion = '3.2'
if sys.version < cdistpythonversion:
print('Cdist requires Python >= ' + cdistpythonversion +
' on the source host.', file=sys.stderr)
sys.exit(1)
exit_code = 0
try:
import logging
import os
import re
# Ensure our /lib/ is included into PYTHON_PATH
sys.path.insert(0, os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')))
# And now import our stuff
import cdist
log = logging.getLogger("cdist")
logging.basicConfig(format='%(levelname)s: %(message)s')
if re.match("__", os.path.basename(sys.argv[0])):
emulator()
else:
commandline()
except KeyboardInterrupt:
pass
except cdist.Error as e:
log.error(e)
exit_code = 1
# Determine exit code by return value of function
sys.exit(exit_code)
# Ensure version is present - the bundled/shipped version contains a static version,
# the git version contains a dynamic version
"$dir/../build" version
PYTHONPATH="${dir}/../" "$dir/../scripts/cdist" "$@"

416
build
View File

@ -26,6 +26,7 @@
# exit on any error
#set -e
basedir=${0%/*}
version=$(git describe)
# Manpage and HTML
@ -39,145 +40,296 @@ WEBMAN=$WEBBASE/man/$version
WEBPAGE=${WEBBASE}.mdwn
# Documentation
MANDIR=doc/man
MANDIR=docs/man
MAN1DSTDIR=${MANDIR}/man1
MAN7DSTDIR=${MANDIR}/man7
SPEECHESDIR=doc/speeches
SPEECHESDIR=docs/speeches
# Change to checkout directory
cd "$basedir"
case "$1" in
man)
set -e
"$0" mangen
"$0" mantype
"$0" manbuild
;;
manbuild)
trap abort INT
abort() {
kill 0
}
for section in 1 7; do
for src in ${MANDIR}/man${section}/*.text; do
manpage="${src%.text}.$section"
if [ ! -f "$manpage" -o "$manpage" -ot "$src" ]; then
echo "Compiling man page for $src"
$A2XM "$src"
fi
htmlpage="${src%.text}.html"
if [ ! -f "$htmlpage" -o "$htmlpage" -ot "$src" ]; then
echo "Compiling html page for $src"
$A2XH "$src"
fi
done
done
;;
mantype)
for mansrc in conf/type/*/man.text; do
dst="$(echo $mansrc | sed -e 's;conf/;cdist-;' -e 's;/;;' -e 's;/man;;' -e 's;^;doc/man/man7/;')"
ln -sf "../../../$mansrc" "$dst"
done
;;
mangen)
${MANDIR}/cdist-reference.text.sh
;;
release)
./doc/dev/releasechecklist
;;
speeches)
cd "$SPEECHESDIR"
for speech in *tex; do
pdflatex "$speech"
pdflatex "$speech"
pdflatex "$speech"
done
;;
webmain)
cp README ${WEBPAGE}
cd ${WEBDIR} && git commit -m "cdist main update" ${WEBPAGE}
cd ${WEBDIR} && make pub
;;
web)
cp README ${WEBPAGE}
rm -rf ${WEBMAN}
mkdir -p ${WEBMAN}/man1 ${WEBMAN}/man7
# old stuff
# rm -rf ${WEBDIR}/${WEBBASE}/speeches && mkdir ${WEBDIR}/${WEBBASE}/speeches
# cp ${SPEECHESDIR}/*.pdf ${WEBDIR}/${WEBBASE}/speeches
# git describe > ${WEBDIR}/${WEBBASE}/man/VERSION
cp ${MAN1DSTDIR}/*.html ${MAN1DSTDIR}/*.css ${WEBMAN}/man1
cp ${MAN7DSTDIR}/*.html ${MAN7DSTDIR}/*.css ${WEBMAN}/man7
cd ${WEBDIR} && git add ${WEBBASE}
cd ${WEBDIR} && git commit -m "cdist update" ${WEBBASE} ${WEBPAGE}
cd ${WEBDIR} && make pub
# Fix ikiwiki, which does not like symlinks for pseudo security
ssh tee.schottelius.org \
"cd /home/services/www/nico/www.nico.schottelius.org/www/software/cdist/man &&
rm -f latest && ln -sf "$version" latest"
;;
p|pu|pub)
for remote in "" github sf ethz; do
echo "Pushing to $remote"
git push --mirror $remote
done
;;
clean)
rm -f ${MAN7DSTDIR}/cdist-reference.text lib/cdist/version.py
find "${MANDIR}" -mindepth 2 -type l \
-o -name "*.1" \
-o -name "*.7" \
-o -name "*.html" \
-o -name "*.xml" \
| xargs rm -f
;;
test)
shift # skip t
export PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib
if [ $# -lt 1 ]; then
python3 -m cdist.test
else
python3 -m unittest "$@"
fi
;;
version-dynamic)
cd lib/cdist/
ln -sf version_dynamic.py version.py
man)
set -e
"$0" mangen
"$0" mantype
"$0" manbuild
;;
version-dist)
version=$(cat .version)
cd lib/cdist/
echo "VERSION=\"$version\"" > version_static.py
ln -sf version_static.py version.py
manbuild)
trap abort INT
abort() {
kill 0
}
for section in 1 7; do
for src in ${MANDIR}/man${section}/*.text; do
manpage="${src%.text}.$section"
if [ ! -f "$manpage" -o "$manpage" -ot "$src" ]; then
echo "Compiling man page for $src"
$A2XM "$src"
fi
htmlpage="${src%.text}.html"
if [ ! -f "$htmlpage" -o "$htmlpage" -ot "$src" ]; then
echo "Compiling html page for $src"
$A2XH "$src"
fi
done
done
;;
mantype)
for mansrc in cdist/conf/type/*/man.text; do
dst="$(echo $mansrc | sed -e 's;cdist/conf/;cdist-;' -e 's;/;;' -e 's;/man;;' -e 's;^;docs/man/man7/;')"
ln -sf "../../../$mansrc" "$dst"
done
;;
mangen)
${MANDIR}/cdist-reference.text.sh
;;
dist)
set -e
# Do the checks
$0 dist-check
# Git changes - everything depends on this
$0 dist-tag
$0 dist-branch-merge
# Pypi first - is the base for others
$0 dist-pypi
# Archlinux depends on successful pypi ;-)
$0 dist-archlinux
# Update website (includes documentation)
$0 web
$0 pub
$0 dist-freecode
$0 dist-post
;;
changelog-version)
# get version from changelog and ensure it's not already present
grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/:.*//'
;;
dist-check)
set -e
echo "Verifying documentation building works ..."
$0 clean
$0 man
changelog_version=$($0 changelog-version)
echo "Target version from changelog: $changelog_version"
if git show --quiet $changelog_version >/dev/null 2>&1; then
echo "Version $changelog_version already exists, aborting."
exit 1
fi
# verify date in changelog
date_today="$(date +%Y-%m-%d)"
date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //')
if [ "$date_today" != "$date_changelog" ]; then
echo "Date in changelog is not today"
echo "Changelog: $date_changelog"
exit 1
fi
;;
dist-post)
cat << notes
To be done manually...
- freecode release
- blog entry
- linkedin entry
- mailinglist update
notes
;;
dist-tag)
version=$($0 changelog-version)
# add tag
printf "Enter tag description for %s> " "$version"
read tagmessage
git tag "$version" -m "$tagmessage"
;;
dist-branch-merge)
version=$($0 changelog-version)
target_branch=${version%\.*}
current_branch=$(git rev-parse --abbrev-ref HEAD)
if [ "$target_branch" = "$current_branch" ]; then
echo "Skipping merge, already on destination branch"
else
printf "Press enter to git merge $current_branch into \"$target_branch\" > "
read prompt
git checkout "$target_branch"
git merge "$current_branch"
git checkout "$current_branch"
fi
;;
dist-archlinux)
$0 dist-archlinux-makepkg
$0 dist-archlinux-aur-upload
;;
dist-archlinux-makepkg)
./PKGBUILD.in
makepkg -c --source
;;
dist-archlinux-aur-upload)
version=$($0 changelog-version)
tar=cdist-${version}-1.src.tar.gz
burp -c system "$tar"
;;
dist-freecode)
version=$($0 changelog-version)
api_token=$(awk '/machine freecode login/ { print $8 }' ~/.netrc)
printf "Enter tag list for freecode release %s> " "$version"
read taglist
printf "Enter changelog for freecode release %s> " "$version"
read changelog
echo "Submit preview"
cat << eof
tag_list = $taglist
changelog = $changelog
version = $version
eof
printf "Press enter to submit to freecode> "
read dummy
cat << eof | cfreecode-api release-add cdist
{
"auth_code": "$api_token",
"release": {
"tag_list": "$taglist",
"version": "$version",
"changelog": "$changelog",
"hidden_from_frontpage": false
}
}
eof
;;
dist-pypi)
$0 man
$0 version
python3 setup.py sdist upload
;;
speeches)
cd "$SPEECHESDIR"
for speech in *tex; do
pdflatex "$speech"
pdflatex "$speech"
pdflatex "$speech"
done
;;
webmain)
cp README ${WEBPAGE}
cd ${WEBDIR} && git commit -m "cdist main update" ${WEBPAGE}
cd ${WEBDIR} && make pub
;;
web)
cp README ${WEBPAGE}
rm -rf ${WEBMAN}
mkdir -p ${WEBMAN}/man1 ${WEBMAN}/man7
cp ${MAN1DSTDIR}/*.html ${MAN1DSTDIR}/*.css ${WEBMAN}/man1
cp ${MAN7DSTDIR}/*.html ${MAN7DSTDIR}/*.css ${WEBMAN}/man7
cd ${WEBDIR} && git add ${WEBBASE}
cd ${WEBDIR} && git commit -m "cdist update" ${WEBBASE} ${WEBPAGE}
cd ${WEBDIR} && make pub
# Fix ikiwiki, which does not like symlinks for pseudo security
ssh tee.schottelius.org \
"cd /home/services/www/nico/www.nico.schottelius.org/www/software/cdist/man &&
rm -f latest && ln -sf "$version" latest"
;;
p|pu|pub)
for remote in "" github sf ethz; do
echo "Pushing to $remote"
git push --mirror $remote
done
;;
clean)
rm -f ${MAN7DSTDIR}/cdist-reference.text
find "${MANDIR}" -mindepth 2 -type l \
-o -name "*.1" \
-o -name "*.7" \
-o -name "*.html" \
-o -name "*.xml" \
| xargs rm -f
find * -name __pycache__ | xargs rm -rf
;;
clean-dist)
rm -f cdist/version.py MANIFEST PKGBUILD
rm -rf cache/ dist/
# Archlinux
rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz
rm -rf pkg/ src/
;;
very-clean)
$0 clean
$0 clean-dist
;;
test)
shift # skip t
export PYTHONPATH="$(pwd -P)"
if [ $# -lt 1 ]; then
python3 -m cdist.test
else
python3 -m unittest "$@"
fi
;;
version)
echo "VERSION=\"$version\"" > cdist/version.py
;;
*)
echo ''
echo 'Welcome to cdist!'
echo ''
echo 'Here are the possible targets:'
echo ''
echo ' clean: Remove build stuff'
echo ' man: Build manpages (requires Asciidoc)'
echo ' test: Run tests'
echo ''
echo ''
echo "Unknown target, \"$1\"" >&2
exit 1
;;
echo ''
echo 'Welcome to cdist!'
echo ''
echo 'Here are the possible targets:'
echo ''
echo ' clean: Remove build stuff'
echo ' man: Build manpages (requires Asciidoc)'
echo ' test: Run tests'
echo ''
echo ''
echo "Unknown target, \"$1\"" >&2
exit 1
;;
esac

View File

@ -0,0 +1,62 @@
## #
## # Sample manifest from cdist distribution
## #
##
## # Every machine becomes a marker, so sysadmins know that automatic
## # configurations are happening
## __file /etc/cdist-configured
## __cdistmarker
##
## case "$__target_host" in
## # Everybody has this
## localhost)
## require="__file/etc/cdist-configured" __link /tmp/cdist-testfile \
## --source /etc/cdist-configured --type symbolic
## require="__directory/tmp/cdist-test-dir" __file /tmp/cdist-test-dir/test-file \
## --mode 0750 --owner nobody --group root
## __directory /tmp/cdist-test-dir --mode 4777
##
## require="__file/etc/cdist-configured __link/tmp/cdist-testfile" \
## __file /tmp/cdist-another-testfile
##
## ;;
##
## #
## # Use an alias in /etc/hosts for localhost to use these hosts:
## #
## # 127.0.0.1 localhost.localdomain localhost cdist-archlinux
## #
## cdist-archlinux)
## # This is the specific package type for pacman
## __package_pacman zsh --state installed
##
## # The __package type autoselect the right type based on the os
## __package vim --state installed
##
## # If the type is a singleton, it does not take an object id
## __issue
## ;;
## # This is how it would look like on gentoo
## cdist-gentoo)
## # Same stuff for gentoo
## __package tree --state installed
## ;;
##
## cdist-debian)
## __package_apt atop --state installed
## __package apache2 --state removed
## ;;
##
## cdist-redhat)
## __issue
## __motd
## ;;
##
## # Real machines may be used with their hostname or fqdn,
## # depending on how you call cdist
## # ...
## # ;;
## # machine.example.org)
## # ...
## # ;;
## esac

View File

@ -18,7 +18,6 @@
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
# Check state and exit if nothing needs to be done
state_should="present"
[ -f "$__object/parameter/state" ] && state_should="$(cat "$__object/parameter/state")"
state_is="$(cat "$__object/explorer/state")"
@ -27,9 +26,9 @@ state_is="$(cat "$__object/explorer/state")"
destination="/$__object_id"
mkdiropt=""
grep yes "$__object/parameter/parents" >/dev/null 2>&1 && mkdiropt="-p"
[ -f "$__object/parameter/parents" ] && mkdiropt="-p"
recursive=""
grep yes "$__object/parameter/recursive" >/dev/null 2>&1 && recursive="-R"
[ -f "$__object/parameter/recursive" ] && recursive="-R"
case "$state_should" in
present)

View File

@ -32,13 +32,15 @@ mode::
owner::
User to chown to.
BOOLEAN PARAMETERS
------------------
parents::
Whether to create parents as well (mkdir -p behaviour). Must be yes or no.
Whether to create parents as well (mkdir -p behaviour)
recursive::
If supplied the chgrp and chown call will run recursively.
This does *not* influence the behaviour of chmod.
Must be yes or no.
EXAMPLES
@ -55,13 +57,18 @@ __directory /tmp/foobar --state absent
__directory /etc --owner root --group root --mode 0755
# Create nfs service directory, including parents
__directory /home/services/nfs --parents yes
__directory /home/services/nfs --parents
# Change permissions recursively
__directory /home/services --recursive yes --owner root --group root
__directory /home/services --recursive --owner root --group root
# Setup a temp directory
__directory /local --mode 1777
# Take it all
__directory /home/services/kvm --recursive --parents \
--owner root --group root --mode 0755 --state present
--------------------------------------------------------------------------------

View File

@ -0,0 +1,2 @@
parents
recursive

View File

@ -2,5 +2,3 @@ state
group
mode
owner
parents
recursive

Some files were not shown because too many files have changed in this diff Show More