diff --git a/.gitignore b/.gitignore index 6e2d4437..76ed1fcb 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ cdist/version.py /cdist-*.tar.gz /pkg /src +build +.lock-* +.git-current-branch diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..5910ab2e --- /dev/null +++ b/Makefile @@ -0,0 +1,252 @@ +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +A2XM=a2x -f manpage --no-xmllint -a encoding=UTF-8 +A2XH=a2x -f xhtml --no-xmllint -a encoding=UTF-8 +helper=./bin/build-helper + +MANDIR=docs/man +SPEECHDIR=docs/speeches +TYPEDIR=cdist/conf/type + +WEBSRCDIR=docs/web + +WEBDIR=$$HOME/www.nico.schottelius.org +WEBBLOG=$(WEBDIR)/blog +WEBBASE=$(WEBDIR)/software/cdist +WEBPAGE=$(WEBBASE).mdwn + +CHANGELOG_VERSION=$(shell $(helper) changelog-version) +CHANGELOG_FILE=docs/changelog + +PYTHON_VERSION=cdist/version.py + +################################################################################ +# Manpages +# +MAN1DSTDIR=$(MANDIR)/man1 +MAN7DSTDIR=$(MANDIR)/man7 + +# Manpages #1: Types +# Use shell / ls to get complete list - $(TYPEDIR)/*/man.text does not work +MANTYPESRC=$(shell ls $(TYPEDIR)/*/man.text) + +# replace first path component +MANTYPEPREFIX=$(subst $(TYPEDIR)/,$(MAN7DSTDIR)/cdist-type,$(MANTYPESRC)) + +# replace man.text with .7 or .html +MANTYPEMAN=$(subst /man.text,.7,$(MANTYPEPREFIX)) +MANTYPEHTML=$(subst /man.text,.html,$(MANTYPEPREFIX)) +MANTYPEALL=$(MANTYPEMAN) $(MANTYPEHTML) + +# Link manpage so A2XH does not create man.html but correct named file +$(MAN7DSTDIR)/cdist-type%.text: $(TYPEDIR)/%/man.text + ln -sf "../../../$^" $@ + +# Manpages #2: reference +MANREF=$(MAN7DSTDIR)/cdist-reference.text +MANREFSH=$(MANDIR)/cdist-reference.text.sh +MANREFMAN=$(MANREF:.text=.7) +MANREFHTML=$(MANREF:.text=.html) +MANREFALL=$(MANREFMAN) $(MANREFHTML) + +$(MANREF): $(MANREFSH) + $(MANREFSH) + +# Manpages #3: static pages +MAN1STATIC=$(shell ls $(MAN1DSTDIR)/*.text) +MAN7STATIC=$(shell ls $(MAN7DSTDIR)/*.text) +MANSTATICMAN=$(MAN1STATIC:.text=.1) $(MAN7STATIC:.text=.7) +MANSTATICHTML=$(MAN1STATIC:.text=.html) $(MAN7STATIC:.text=.html) +MANSTATICALL=$(MANSTATICMAN) $(MANSTATICHTML) + +# Manpages #4: generic part + +# Creating the type manpage +%.1 %.7: %.text + $(A2XM) $^ + +# Creating the type html page +%.html: %.text + $(A2XH) $^ + +man: $(MANTYPEALL) $(MANREFALL) $(MANSTATICALL) + +# Manpages #5: release part +MANWEBDIR=$(WEBBASE)/man/$(CHANGELOG_VERSION) + +man-dist: man check-date + rm -rf "${MANWEBDIR}" + mkdir -p "${MANWEBDIR}/man1" "${MANWEBDIR}/man7" + cp ${MAN1DSTDIR}/*.html ${MAN1DSTDIR}/*.css ${MANWEBDIR}/man1 + cp ${MAN7DSTDIR}/*.html ${MAN7DSTDIR}/*.css ${MANWEBDIR}/man7 + cd ${MANWEBDIR} && git add . && git commit -m "cdist manpages update: $(CHANGELOG_VERSION)" || true + +man-fix-link: web-pub + # Fix ikiwiki, which does not like symlinks for pseudo security + ssh tee.schottelius.org \ + "cd /home/services/www/nico/www.nico.schottelius.org/www/software/cdist/man && rm -f latest && ln -sf "$(CHANGELOG_VERSION)" latest" + +################################################################################ +# Speeches +# +SPEECHESOURCES=$(SPEECHDIR)/*.tex +SPEECHES=$(SPEECHESOURCES:.tex=.pdf) +SPEECHESWEBDIR=$(WEBBASE)/speeches + +# Create speeches and ensure Toc is up-to-date +$(SPEECHDIR)/%.pdf: $(SPEECHDIR)/%.tex + pdflatex -output-directory $(SPEECHDIR) $^ + pdflatex -output-directory $(SPEECHDIR) $^ + pdflatex -output-directory $(SPEECHDIR) $^ + +speeches: $(SPEECHES) + +speeches-dist: speeches + rm -rf "${SPEECHESWEBDIR}" + mkdir -p "${SPEECHESWEBDIR}" + cp ${SPEECHES} "${SPEECHESWEBDIR}" + cd ${SPEECHESWEBDIR} && git add . && git commit -m "cdist speeches updated" || true + +################################################################################ +# Website +# + +BLOGFILE=$(WEBBLOG)/cdist-$(CHANGELOG_VERSION)-released.mdwn + +$(BLOGFILE): $(CHANGELOG_FILE) + $(helper) blog $(CHANGELOG_VERSION) $(BLOGFILE) + +web-blog: $(BLOGFILE) + +web-doc: + # Go to top level, because of cdist.mdwn + rsync -av "$(WEBSRCDIR)/" "${WEBBASE}/.." + cd "${WEBBASE}/.." && git add cdist* && git commit -m "cdist doc update" cdist* || true + +web-dist: web-blog web-doc + +web-pub: web-dist man-dist speeches-dist + cd "${WEBDIR}" && make pub + +web-release-all: man-fix-link + +################################################################################ +# Release: Mailinglist +# +ML_FILE=.lock-ml + +# Only send mail once - lock until new changelog things happened +$(ML_FILE): $(CHANGELOG_FILE) + $(helper) ml-release $(CHANGELOG_VERSION) + touch $@ + +ml-release: $(ML_FILE) + + +################################################################################ +# Release: Freecode +# +FREECODE_FILE=.lock-freecode + +$(FREECODE_FILE): $(CHANGELOG_FILE) + $(helper) freecode-release $(CHANGELOG_VERSION) + touch $@ + +freecode-release: $(FREECODE_FILE) + +################################################################################ +# pypi +# +pypi-release: man $(PYTHON_VERSION) + python3 setup.py sdist upload + touch $@ + +################################################################################ +# archlinux +# +ARCHLINUX_FILE=.lock-archlinux +ARCHLINUXTAR=cdist-$(CHANGELOG_VERSION)-1.src.tar.gz + +$(ARCHLINUXTAR): PKGBUILD + makepkg -c --source + +PKGBUILD: PKGBUILD.in $(PYTHON_VERSION) + ./PKGBUILD.in $(CHANGELOG_VERSION) + +$(ARCHLINUX_FILE): $(ARCHLINUXTAR) $(PYTHON_VERSION) + burp -c system $(ARCHLINUXTAR) + touch $@ + +archlinux-release: $(ARCHLINUX_FILE) + +################################################################################ +# Release +# + +$(PYTHON_VERSION): .git/refs/heads/master + $(helper) version + +# Code that is better handled in a shell script +check-%: + $(helper) $@ + +release: + $(helper) $@ + +################################################################################ +# Cleanup +# + +clean: + rm -f $(MAN7DSTDIR)/cdist-reference.text + + find "$(MANDIR)" -mindepth 2 -type l \ + -o -name "*.1" \ + -o -name "*.7" \ + -o -name "*.html" \ + -o -name "*.xml" \ + | xargs rm -f + + find * -name __pycache__ | xargs rm -rf + + # Archlinux + rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz + rm -rf pkg/ src/ + + rm -f MANIFEST PKGBUILD + rm -rf dist/ + +distclean: clean + rm -f cdist/version.py + +################################################################################ +# Misc +# + +# The pub is Nico's "push to all git remotes" way ("make pub") +pub: + for remote in "" github sf; do \ + echo "Pushing to $$remote"; \ + git push --mirror $$remote; \ + done + +test: + $(helper) $@ diff --git a/PKGBUILD.in b/PKGBUILD.in index 68bd6add..e3ae4619 100755 --- a/PKGBUILD.in +++ b/PKGBUILD.in @@ -1,6 +1,6 @@ #!/bin/sh -version=$(git describe) +version="$1" outfile=${0%.in} cat << eof > "${outfile}" diff --git a/bin/build-helper b/bin/build-helper new file mode 100755 index 00000000..b97528f1 --- /dev/null +++ b/bin/build-helper @@ -0,0 +1,298 @@ +#!/bin/sh +# +# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# This file contains the heavy lifting found usually in the Makefile +# + +basedir=${0%/*}/../ +# Change to checkout directory +cd "$basedir" + +version=$(git describe) + +option=$1; shift + +case "$option" in + changelog-changes) + if [ "$#" -eq 1 ]; then + start=$1 + else + start="[[:digit:]]" + fi + + end="[[:digit:]]" + + awk -F: "BEGIN { start=0 } + { + if(start == 0) { + if (\$0 ~ /^$start/) { + start = 1 + } + } else { + if (\$0 ~ /^$end/) { + exit + } else { + print \$0 + } + } + }" "$basedir/docs/changelog" + ;; + + changelog-version) + # get version from changelog + grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/:.*//' + ;; + + check-date) + # verify date in changelog is today + date_today="$(date +%Y-%m-%d)" + date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //') + + if [ "$date_today" != "$date_changelog" ]; then + echo "Date in changelog is not today" + echo "Changelog: $date_changelog" + exit 1 + fi + ;; + + check-unittest) + "$0" test + ;; + + blog) + version=$1; shift + blogfile=$1; shift + dir=${blogfile%/*} + file=${blogfile##*/} + + + cat << eof > "$blogfile" +[[!meta title="Cdist $version released"]] + +Here's a short overview about the changes found in version ${version}: + +eof + + $0 changelog-changes "$version" >> "$blogfile" + + cat << eof >> "$blogfile" +For more information visit the [[cdist homepage|software/cdist]]. + +[[!tag cdist config unix]] +eof + cd "$dir" + git add "$file" + # Allow git commit to fail if there are no changes + git commit -m "cdist blog update: $version" "$blogfile" || true + ;; + + ml-release) + version=$1; shift + + to_a=cdist + to_d=l.schottelius.org + to=${to_a}@${to_d} + + from_a=nico-cdist + from_d=schottelius.org + from=${from_a}@${from_d} + + ( + cat << eof +From: Nico -telmich- Schottelius <$from> +To: cdist mailing list <$to> +Subject: cdist $version released + +Hello .*, + +cdist $version has been released with the following changes: + +eof + + "$0" changelog-changes "$version" + cat << eof + +Cheers, + +Nico + +-- +Automatisation at its best level. With cdist. +eof + ) | /usr/sbin/sendmail -f "$from" "$to" + ;; + + + freecode-release) + version=$1; shift + api_token=$(awk '/machine freecode login/ { print $8 }' ~/.netrc) + + printf "Enter tag list for freecode release %s> " "$version" + read taglist + + printf "Enter changelog for freecode release %s> " "$version" + read changelog + + echo "Submit preview" + cat << eof +tag_list = $taglist +changelog = $changelog +version = $version +eof + printf "Press enter to submit to freecode> " + read dummy + + cat << eof | cfreecode-api release-add cdist + { + "auth_code": "$api_token", + "release": { + "tag_list": "$taglist", + "version": "$version", + "changelog": "$changelog", + "hidden_from_frontpage": false + } + } +eof + + ;; + + release-git-tag) + target_version=$($0 changelog-version) + if git rev-parse --verify refs/tags/$target_version; then + echo "Tag for $target_version exists, aborting" + exit 1 + fi + printf "Enter tag description for ${target_version}: " + read tagmessage + git tag "$target_version" -m "$$tagmessage" + ;; + + release) + set -e + target_version=$($0 changelog-version) + target_branch=$($0 version-branch) + + echo "Beginning release process for $target_version" + + # First check everything is sane + "$0" check-date + "$0" check-unittest + + # Generate version file to be included in packaging + "$0" version + + # Ensure the git status is clean, else abort + if ! git diff-index --name-only --exit-code HEAD ; then + echo "Unclean tree, see files above, aborting" + exit 1 + fi + + # Ensure we are on the master branch + if [ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then + echo "Releases are happening from the master branch, aborting" + exit 1 + fi + + # Ensure version branch exists + if ! git rev-parse --verify refs/heads/$target_branch 2>/dev/null; then + git branch "$target_branch" + fi + + # Merge master branch into version branch + git checkout "$target_branch" + git merge master + + # Verify that after the merge everything works + "$0" check-date + "$0" check-unittest + + # Generate man pages (indirect check if they build) + make man + + # Generate speeches (indirect check if they build) + make speeches + + ############################################################# + # Everything green, let's do the release + + # Tag the current commit + "$0" release-git-tag + + # Also merge back the version branch + git checkout master + git merge "$target_branch" + + # Publish git changes + make pub + + # publish man, speeches, website + make web-release-all + + # Ensure that pypi release has the right version + "$0" version + + # Create and publish package for pypi + make pypi-release + + # Archlinux release is based on pypi + make archlinux-release + + # Announce change on Freecode + make freecode-release + + # Announce change on ML + make ml-release + + cat << eof +Manual steps post release: + + - linkedin + - hackernews + - reddit + - twitter + +eof + + ;; + + test) + export PYTHONPATH="$(pwd -P)" + + if [ $# -lt 1 ]; then + python3 -m cdist.test + else + python3 -m unittest "$@" + fi + ;; + + version-branch) + "$0" changelog-version | cut -d. -f '1,2' + ;; + + version) + echo "VERSION = \"$(git describe)\"" > cdist/version.py + ;; + + *) + echo "Unknown helper target $@ - aborting" + exit 1 + ;; + +esac diff --git a/bin/cdist b/bin/cdist index dfe4fa00..645020a1 100755 --- a/bin/cdist +++ b/bin/cdist @@ -25,7 +25,7 @@ dir=${0%/*} # Ensure version is present - the bundled/shipped version contains a static version, # the git version contains a dynamic version -"$dir/../build" version +"$dir/build-helper" version libdir=$(cd "${dir}/../" && pwd -P) export PYTHONPATH="${libdir}" diff --git a/build b/build deleted file mode 100755 index 1f408b94..00000000 --- a/build +++ /dev/null @@ -1,414 +0,0 @@ -#!/bin/sh -# -# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Push a directory to a target, both sides have the same name (i.e. explorers) -# or -# Pull a directory from a target, both sides have the same name (i.e. explorers) -# - -# exit on any error -#set -e - -basedir=${0%/*} -version=$(cd "$basedir" && git describe) - -# Manpage and HTML -A2XM="a2x -f manpage --no-xmllint -a encoding=UTF-8" -A2XH="a2x -f xhtml --no-xmllint -a encoding=UTF-8" - -# Developer webbase -WEBDIR=$HOME/www.nico.schottelius.org -WEBBLOG=$WEBDIR/blog -WEBTOPDIR=$WEBDIR/software -WEBBASE=$WEBTOPDIR/cdist -WEBMAN=$WEBBASE/man/$version -WEBPAGE=${WEBBASE}.mdwn - -# Documentation -MANDIR=docs/man -MAN1DSTDIR=${MANDIR}/man1 -MAN7DSTDIR=${MANDIR}/man7 -SPEECHESDIR=docs/speeches - -# Change to checkout directory -cd "$basedir" - -case "$1" in - man) - set -e - "$0" mangen - "$0" mantype - "$0" manbuild - ;; - - manbuild) - trap abort INT - abort() { - kill 0 - } - for section in 1 7; do - for src in ${MANDIR}/man${section}/*.text; do - manpage="${src%.text}.$section" - if [ ! -f "$manpage" -o "$manpage" -ot "$src" ]; then - echo "Compiling man page for $src" - $A2XM "$src" - fi - htmlpage="${src%.text}.html" - if [ ! -f "$htmlpage" -o "$htmlpage" -ot "$src" ]; then - echo "Compiling html page for $src" - $A2XH "$src" - fi - done - done - ;; - - mantype) - for mansrc in cdist/conf/type/*/man.text; do - dst="$(echo $mansrc | sed -e 's;cdist/conf/;cdist-;' -e 's;/;;' -e 's;/man;;' -e 's;^;docs/man/man7/;')" - ln -sf "../../../$mansrc" "$dst" - done - ;; - - mangen) - ${MANDIR}/cdist-reference.text.sh - ;; - - man-pub) - $0 man - - version=$($0 changelog-version) - - rm -rf "${WEBMAN}" - mkdir -p "${WEBMAN}/man1" "${WEBMAN}/man7" - cp ${MAN1DSTDIR}/*.html ${MAN1DSTDIR}/*.css ${WEBMAN}/man1 - cp ${MAN7DSTDIR}/*.html ${MAN7DSTDIR}/*.css ${WEBMAN}/man7 - cd ${WEBMAN} && git add . && git commit -m "Cdist Manpage update: $version" - ;; - - dist) - set -e - # Do the checks - $0 dist-check - - # Git changes - everything depends on this - $0 dist-tag - $0 dist-branch-merge - - # Pypi first - is the base for others - $0 dist-pypi - - # Archlinux depends on successful pypi ;-) - $0 dist-archlinux - - # Update website (includes documentation) - $0 web - - # Update manpages on website - $0 man-pub - - # update git repos - $0 pub - - $0 dist-blog - $0 dist-freecode - $0 dist-ml - $0 dist-manual - ;; - - changelog-changes) - awk -F: 'BEGIN { start=0 } { if ($0 ~ /^[[:digit:]]/) { if(start == 0) {start = 1 } else { exit } } else { if(start==1) {print $0 }} }' "$basedir/docs/changelog" - ;; - - changelog-version) - # get version from changelog and ensure it's not already present - grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/:.*//' - ;; - - dist-check) - set -e - echo "Verifying documentation building works ..." - $0 clean - $0 man - - changelog_version=$($0 changelog-version) - echo "Target version from changelog: $changelog_version" - - if git show --quiet $changelog_version >/dev/null 2>&1; then - echo "Version $changelog_version already exists, aborting." - exit 1 - fi - - # verify date in changelog - date_today="$(date +%Y-%m-%d)" - date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //') - - if [ "$date_today" != "$date_changelog" ]; then - echo "Date in changelog is not today" - echo "Changelog: $date_changelog" - exit 1 - fi - - ;; - - blog) - version=$($0 changelog-version) - blogfile=$WEBBLOG/cdist-${version}-released.mdwn - cat << eof > "$blogfile" -[[!meta title="Cdist $version released"]] - -Here's a short overview about the changes found in this release: - -eof - - $0 changelog-changes >> "$blogfile" - - cat << eof >> "$blogfile" -For more information visit the [[cdist homepage|software/cdist]]. - -[[!tag cdist config unix]] -eof - ;; - - dist-blog) - $0 blog - version=$($0 changelog-version) - file=cdist-${version}-released.mdwn - cd "$WEBBLOG" - git add "$file" - git commit -m "New cdist version (blogentry): $version" "$file" - git push - ;; - - dist-ml) - $0 blog - version=$($0 changelog-version) - to_a=cdist - to_d=l.schottelius.org - to=${to_a}@${to_d} - - from_a=nico-cdist - from_d=schottelius.org - from=${from_a}@${from_d} - - ( - cat << eof -From: Nico -telmich- Schottelius <$from> -To: cdist mailing list <$to> -Subject: cdist $version released - -Hello .*, - -cdist $version has been released with the following changes: - -eof - - "$0" changelog-changes - cat << eof - -Cheers, - -Nico - --- -Automatisation at its best level. With cdist. -eof - ) | /usr/sbin/sendmail -f "$from" "$to" - ;; - - - dist-manual) - cat << notes - - To be done manually... - - - linkedin entry -notes - - ;; - - dist-tag) - version=$($0 changelog-version) - # add tag - printf "Enter tag description for %s> " "$version" - read tagmessage - git tag "$version" -m "$tagmessage" - ;; - - dist-branch-merge) - version=$($0 changelog-version) - target_branch=${version%\.*} - current_branch=$(git rev-parse --abbrev-ref HEAD) - - if [ "$target_branch" = "$current_branch" ]; then - echo "Skipping merge, already on destination branch" - else - printf "Press enter to git merge $current_branch into \"$target_branch\" > " - read prompt - git checkout "$target_branch" - git merge "$current_branch" - git checkout "$current_branch" - fi - ;; - - dist-archlinux) - $0 dist-archlinux-makepkg - $0 dist-archlinux-aur-upload - ;; - - dist-archlinux-makepkg) - ./PKGBUILD.in - makepkg -c --source - ;; - - dist-archlinux-aur-upload) - version=$($0 changelog-version) - tar=cdist-${version}-1.src.tar.gz - burp -c system "$tar" - ;; - - dist-freecode) - version=$($0 changelog-version) - api_token=$(awk '/machine freecode login/ { print $8 }' ~/.netrc) - - printf "Enter tag list for freecode release %s> " "$version" - read taglist - - printf "Enter changelog for freecode release %s> " "$version" - read changelog - - echo "Submit preview" - cat << eof -tag_list = $taglist -changelog = $changelog -version = $version -eof - printf "Press enter to submit to freecode> " - read dummy - - cat << eof | cfreecode-api release-add cdist - { - "auth_code": "$api_token", - "release": { - "tag_list": "$taglist", - "version": "$version", - "changelog": "$changelog", - "hidden_from_frontpage": false - } - } -eof - - ;; - - dist-pypi) - $0 man - $0 version - python3 setup.py sdist upload - ;; - - speeches) - cd "$SPEECHESDIR" - for speech in *tex; do - pdflatex "$speech" - pdflatex "$speech" - pdflatex "$speech" - done - ;; - - web-doc) - rsync -av "${basedir}/docs/web/" "${WEBTOPDIR}" - - cd "${WEBDIR}" && git add "${WEBBASE}" - cd "${WEBDIR}" && git commit -m "cdist update" "${WEBBASE}" "${WEBPAGE}" - cd "${WEBDIR}" && make pub - ;; - - web) - set -e - "$0" web-doc - # Fix ikiwiki, which does not like symlinks for pseudo security - ssh tee.schottelius.org \ - "cd /home/services/www/nico/www.nico.schottelius.org/www/software/cdist/man && - rm -f latest && ln -sf "$version" latest" - ;; - - p|pu|pub) - for remote in "" github sf; do - echo "Pushing to $remote" - git push --mirror $remote - done - ;; - - clean) - rm -f ${MAN7DSTDIR}/cdist-reference.text - - find "${MANDIR}" -mindepth 2 -type l \ - -o -name "*.1" \ - -o -name "*.7" \ - -o -name "*.html" \ - -o -name "*.xml" \ - | xargs rm -f - - find * -name __pycache__ | xargs rm -rf - ;; - clean-dist) - rm -f cdist/version.py MANIFEST PKGBUILD - rm -rf cache/ dist/ - - # Archlinux - rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz - rm -rf pkg/ src/ - ;; - - very-clean) - $0 clean - $0 clean-dist - ;; - - test) - shift # skip t - export PYTHONPATH="$(pwd -P)" - - if [ $# -lt 1 ]; then - python3 -m cdist.test - else - python3 -m unittest "$@" - fi - ;; - - version) - echo "VERSION=\"$version\"" > cdist/version.py - ;; - - *) - echo '' - echo 'Welcome to cdist!' - echo '' - echo 'Here are the possible targets:' - echo '' - echo ' clean: Remove build stuff' - echo ' man: Build manpages (requires Asciidoc)' - echo ' test: Run tests' - echo '' - echo '' - echo "Unknown target, \"$1\"" >&2 - exit 1 - ;; - -esac diff --git a/cdist/__init__.py b/cdist/__init__.py index 02d708b1..20c76b31 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -40,13 +40,20 @@ BANNER = """ "8888P' `"888*"" R888" ` ^"F 'Y" "P' "" "" """ + DOT_CDIST = ".cdist" +REMOTE_COPY = "scp -o User=root -q" +REMOTE_EXEC = "ssh -o User=root -q" class Error(Exception): """Base exception class for this project""" pass +class UnresolvableRequirementsError(cdist.Error): + """Resolving requirements failed""" + pass + class CdistObjectError(Error): """Something went wrong with an object""" diff --git a/cdist/conf/explorer/hostname b/cdist/conf/explorer/hostname index 2ae23759..881c910a 100755 --- a/cdist/conf/explorer/hostname +++ b/cdist/conf/explorer/hostname @@ -20,6 +20,6 @@ # # -if command -v hostname; then - hostname +if command -v hostname >/dev/null; then + hostname -f fi diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os index e67d87ab..053177eb 100755 --- a/cdist/conf/explorer/os +++ b/cdist/conf/explorer/os @@ -88,6 +88,11 @@ if [ -f /etc/SuSE-release ]; then exit 0 fi +if [ -f /etc/slackware-version ]; then + echo slackware + exit 0 +fi + uname_s="$(uname -s)" # Assume there is no tr on the client -> do lower case ourselves diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 8e6d37d3..50889429 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -54,6 +54,9 @@ case "$($__explorer/os)" in redhat|centos) cat /etc/redhat-release ;; + slackware) + cat /etc/slackware-version + ;; suse) cat /etc/SuSE-release ;; diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index 0ea8011c..300a0e1e 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -22,7 +22,7 @@ name="$__object_id" state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" -if [ "$state_should" == "$state_is" ]; then +if [ "$state_should" = "$state_is" ]; then # Nothing to do, move along exit 0 fi diff --git a/cdist/conf/type/__cdist/man.text b/cdist/conf/type/__cdist/man.text new file mode 100644 index 00000000..0805598e --- /dev/null +++ b/cdist/conf/type/__cdist/man.text @@ -0,0 +1,63 @@ +cdist-type__cdist(7) +==================== +Nico Schottelius + + +NAME +---- +cdist-type__cdist - Manage cdist installations + + +DESCRIPTION +----------- +This cdist type allows you to easily setup cdist +on another box, to allow the other box to configure +systems. + +This type is *NOT* required by target hosts. +It is only helpful to build FROM which you configure +other hosts. + +This type will use git to clone + + +REQUIRED PARAMETERS +------------------- + +OPTIONAL PARAMETERS +------------------- +username:: + Select the user to create for the cdist installation. + Defaults to "cdist". + +source:: + Select the source from which to clone cdist from. + Defaults to "git://github.com/telmich/cdist.git". + + +branch:: + Select the branch to checkout from. + Defaults to "master". + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# Install cdist for user cdist in her home as subfolder cdist +__cdist /home/cdist/cdist + +# Use alternative source +__cdist --source "git://git.schottelius.org/cdist" /home/cdist/cdist +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2013 Nico Schottelius. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__cdist/manifest b/cdist/conf/type/__cdist/manifest new file mode 100755 index 00000000..44d62f6c --- /dev/null +++ b/cdist/conf/type/__cdist/manifest @@ -0,0 +1,59 @@ +#!/bin/sh +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +directory="$__object_id" + +if [ -f "$__object/parameter/shell" ]; then + shell="--shell $(cat "$__object/parameter/shell")" +else + shell="" +fi + +if [ -f "$__object/parameter/username" ]; then + username="$(cat "$__object/parameter/username")" +else + username="cdist" +fi + +if [ -f "$__object/parameter/branch" ]; then + branch="$(cat "$__object/parameter/branch")" +else + branch="master" +fi + +if [ -f "$__object/parameter/source" ]; then + source="$(cat "$__object/parameter/source")" +else + source="git://github.com/telmich/cdist.git" +fi + +# Currently hardcoded - if anyone cares, make a parameter +# out of it +home=/home/$username + +__user "$username" --home "$home" $shell + +require="__user/$username" __directory "$home" \ + --owner "$username" + +require="__user/$username __directory/$home" __git "$directory" \ + --source "$source" \ + --owner "$username" --branch "$branch" diff --git a/cdist/conf/type/__cdist/parameter/optional b/cdist/conf/type/__cdist/parameter/optional new file mode 100644 index 00000000..a5f14343 --- /dev/null +++ b/cdist/conf/type/__cdist/parameter/optional @@ -0,0 +1,4 @@ +branch +source +username +shell diff --git a/cdist/conf/type/__cron/explorer/entry b/cdist/conf/type/__cron/explorer/entry index 1b4bec42..c3bf02d2 100755 --- a/cdist/conf/type/__cron/explorer/entry +++ b/cdist/conf/type/__cron/explorer/entry @@ -1,6 +1,7 @@ #!/bin/sh # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) +# 2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -18,22 +19,7 @@ # along with cdist. If not, see . # -name="$__object_id" +name="$__object_name" user="$(cat "$__object/parameter/user")" -prefix="#cdist:__cron/$name" -suffix="#/cdist:__cron/$name" - -crontab -u $user -l 2>/dev/null | awk -v prefix="$prefix" -v suffix="$suffix" ' -{ - if (index($0,prefix)) { - triggered=1 - } - if (triggered) { - if (index($0,suffix)) { - triggered=0 - } - print - } -} -' +crontab -u $user -l 2>/dev/null | grep "# $name\$" || true diff --git a/cdist/conf/type/__cron/gencode-remote b/cdist/conf/type/__cron/gencode-remote index 37e0dc15..c04a7245 100755 --- a/cdist/conf/type/__cron/gencode-remote +++ b/cdist/conf/type/__cron/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -18,39 +19,43 @@ # along with cdist. If not, see . # -os="$(cat "$__global/explorer/os")" +name="$__object_name" user="$(cat "$__object/parameter/user")" -state_should="$(cat "$__object/parameter/state")" -state_is=$(diff -q "$__object/parameter/entry" "$__object/explorer/entry" \ - && echo present \ - || echo absent -) +command="$(cat "$__object/parameter/command")" -# FreeBSD mktemp doesn't allow execution without at least one param -if [ "$os" = "freebsd" ]; then - mktemp="mktemp -t tmp" +if [ -f "$__object/parameter/raw" ]; then + raw="$(cat "$__object/parameter/raw")" + entry="$raw $command" else - mktemp="mktemp" + minute="$(cat "$__object/parameter/minute" 2>/dev/null || echo "*")" + hour="$(cat "$__object/parameter/hour" 2>/dev/null || echo "*")" + day_of_month="$(cat "$__object/parameter/day_of_month" 2>/dev/null || echo "*")" + month="$(cat "$__object/parameter/month" 2>/dev/null || echo "*")" + day_of_week="$(cat "$__object/parameter/day_of_week" 2>/dev/null || echo "*")" + entry="$minute $hour $day_of_month $month $day_of_week $command" fi -if [ "$state_is" != "$state_should" ]; then - case "$state_should" in - present) - cat << DONE -tmp=\$($mktemp) -crontab -u $user -l > \$tmp -cat >> \$tmp << EOC -$(cat "$__object/parameter/entry") -EOC -crontab -u $user \$tmp -rm \$tmp -DONE - ;; - absent) - # defined in type manifest - prefix="$(cat "$__object/parameter/prefix")" - suffix="$(cat "$__object/parameter/suffix")" - cat << DONE +entry="$entry # $name" +mkdir "$__object/files" +echo "$entry" > "$__object/files/entry" + +if diff -q "$__object/files/entry" "$__object/explorer/entry" >/dev/null; then + state_is=present +else + state_is=absent +fi + +state_should="$(cat "$__object/parameter/state" 2>/dev/null || echo "present")" + +[ "$state_is" = "$state_should" ] && exit 0 + +# If anything is going to change, ensure the old entries are +# not present anymore + +# These are the old markers +prefix="#cdist:__cron/$__object_id" +suffix="#/cdist:__cron/$__object_id" +cat << DONE crontab -u $user -l | awk -v prefix="$prefix" -v suffix="$suffix" ' { if (index(\$0,prefix)) { @@ -66,6 +71,16 @@ crontab -u $user -l | awk -v prefix="$prefix" -v suffix="$suffix" ' } ' | crontab -u $user - DONE - ;; - esac -fi + +case "$state_should" in + present) + echo "(" + echo "crontab -u $user -l 2>/dev/null || true" + echo "echo '$entry'" + echo ") | crontab -u $user -" + ;; + absent) + echo "( crontab -u $user -l 2>/dev/null || true ) | \\" + echo "grep -v \"# $name\\$\" | crontab -u $user -" + ;; +esac diff --git a/cdist/conf/type/__cron/man.text b/cdist/conf/type/__cron/man.text index 47f47456..22627234 100644 --- a/cdist/conf/type/__cron/man.text +++ b/cdist/conf/type/__cron/man.text @@ -68,5 +68,5 @@ SEE ALSO COPYING ------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is +Copyright \(C) 2011-2013 Steven Armstrong. Free use of this software is granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__cron/manifest b/cdist/conf/type/__cron/manifest deleted file mode 100755 index 7aca41ff..00000000 --- a/cdist/conf/type/__cron/manifest +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh -# -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -name="$__object_id" -user="$(cat "$__object/parameter/user")" -command="$(cat "$__object/parameter/command")" - -# set defaults -test -f "$__object/parameter/state" || echo "present" > "$__object/parameter/state" - -if [ -f "$__object/parameter/raw" ]; then - raw="$(cat "$__object/parameter/raw")" - entry="$raw $command" -else - minute="$(cat "$__object/parameter/minute" 2>/dev/null || echo "*")" - hour="$(cat "$__object/parameter/hour" 2>/dev/null || echo "*")" - day_of_month="$(cat "$__object/parameter/day_of_month" 2>/dev/null || echo "*")" - month="$(cat "$__object/parameter/month" 2>/dev/null || echo "*")" - day_of_week="$(cat "$__object/parameter/day_of_week" 2>/dev/null || echo "*")" - entry="$minute $hour $day_of_month $month $day_of_week $command" -fi - -# NOTE: if changed, also change in explorers -prefix="#cdist:__cron/$name" -suffix="#/cdist:__cron/$name" -echo "$prefix" | tee "$__object/parameter/prefix" > "$__object/parameter/entry" -echo "$entry" >> "$__object/parameter/entry" -echo "$suffix" | tee "$__object/parameter/suffix" >> "$__object/parameter/entry" diff --git a/cdist/conf/type/__debconf_set_selections/man.text b/cdist/conf/type/__debconf_set_selections/man.text index b6b2ad18..f1e13a8e 100644 --- a/cdist/conf/type/__debconf_set_selections/man.text +++ b/cdist/conf/type/__debconf_set_selections/man.text @@ -17,7 +17,7 @@ to setup configuration parameters. REQUIRED PARAMETERS ------------------- file:: - If supplied, use the given filename as input for debconf-set-selections(1) + Use the given filename as input for debconf-set-selections(1) EXAMPLES @@ -35,9 +35,11 @@ __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd" SEE ALSO -------- - cdist-type(7) +- cdist-type__update_alternatives(7) +- debconf-set-selections(1) COPYING ------- -Copyright \(C) 2011 Nico Schottelius. Free use of this software is +Copyright \(C) 2011-2013 Nico Schottelius. Free use of this software is granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__directory/explorer/group b/cdist/conf/type/__directory/explorer/group new file mode 100644 index 00000000..e5be37da --- /dev/null +++ b/cdist/conf/type/__directory/explorer/group @@ -0,0 +1,39 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Check whether file exists or not +# + +destination="/$__object_id" +os=$("$__explorer/os") + +case "$os" in + "freebsd") + cmd="stat -f %Sg" + ;; + *) + cmd="stat -c %G" + ;; +esac + +if [ -e "$destination" ]; then + $cmd "$destination" +fi + diff --git a/cdist/conf/type/__directory/explorer/mode b/cdist/conf/type/__directory/explorer/mode new file mode 100644 index 00000000..f75b282b --- /dev/null +++ b/cdist/conf/type/__directory/explorer/mode @@ -0,0 +1,39 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Check whether file exists or not +# + +destination="/$__object_id" +os=$("$__explorer/os") + +case "$os" in + "freebsd") + cmd="stat -f %Op" + ;; + *) + cmd="stat -c %a" + ;; +esac + +if [ -e "$destination" ]; then + $cmd "$destination" +fi + diff --git a/cdist/conf/type/__directory/explorer/owner b/cdist/conf/type/__directory/explorer/owner new file mode 100644 index 00000000..cebd199b --- /dev/null +++ b/cdist/conf/type/__directory/explorer/owner @@ -0,0 +1,39 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Check whether file exists or not +# + +destination="/$__object_id" +os=$("$__explorer/os") + +case "$os" in + "freebsd") + cmd="stat -f %Su" + ;; + *) + cmd="stat -c %U" + ;; +esac + +if [ -e "$destination" ]; then + $cmd "$destination" +fi + diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote index 21f4c5b6..f46a5967 100755 --- a/cdist/conf/type/__directory/gencode-remote +++ b/cdist/conf/type/__directory/gencode-remote @@ -18,39 +18,51 @@ # along with cdist. If not, see . # -state_should="present" -[ -f "$__object/parameter/state" ] && state_should="$(cat "$__object/parameter/state")" -state_is="$(cat "$__object/explorer/state")" -[ "$state_should" = "$state_is" ] && exit 0 - destination="/$__object_id" +state_is="$(cat "$__object/explorer/state")" +owner_is="$(cat "$__object/explorer/owner")" +group_is="$(cat "$__object/explorer/group")" +mode_is="$(cat "$__object/explorer/mode")" + +state_should="present" +[ -f "$__object/parameter/state" ] && state_should="$(cat "$__object/parameter/state")" +mode="" +[ -f "$__object/parameter/mode" ] && mode="$(cat "$__object/parameter/mode")" +owner="" +[ -f "$__object/parameter/owner" ] && owner="$(cat "$__object/parameter/owner")" +group="" +[ -f "$__object/parameter/group" ] && group="$(cat "$__object/parameter/group")" mkdiropt="" -[ -f "$__object/parameter/parents" ] && mkdiropt="-p" +[ -f "$__object/parameter/parents" ] && mkdiropt="-p" recursive="" [ -f "$__object/parameter/recursive" ] && recursive="-R" case "$state_should" in present) - echo mkdir $mkdiropt \"$destination\" + if [ "$state_is" != "present" ]; then + echo mkdir $mkdiropt \"$destination\" + fi # Mode settings - if [ -f "$__object/parameter/mode" ]; then - echo chmod \"$(cat "$__object/parameter/mode")\" \"$destination\" + if [ "$mode" ] && [ "$mode_is" != "$mode" -o -n "$recursive" ]; then + echo chmod $recursive \"$mode\" \"$destination\" fi # Group - if [ -f "$__object/parameter/group" ]; then - echo chgrp $recursive \"$(cat "$__object/parameter/group")\" \"$destination\" + if [ "$group" ] && [ "$group_is" != "$group" -o -n "$recursive" ]; then + echo chgrp $recursive \"$group\" \"$destination\" fi # Owner - if [ -f "$__object/parameter/owner" ]; then - echo chown $recursive \"$(cat "$__object/parameter/owner")\" \"$destination\" + if [ "$owner" ] && [ "$owner_is" != "$owner" -o -n "$recursive" ]; then + echo chown $recursive \"$owner\" \"$destination\" fi ;; absent) - echo rm -rf \"$destination\" + if [ "$state_is" != "absent" ]; then + echo rm -rf \"$destination\" + fi ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__directory/man.text b/cdist/conf/type/__directory/man.text index 1f4def7d..cc327af2 100644 --- a/cdist/conf/type/__directory/man.text +++ b/cdist/conf/type/__directory/man.text @@ -36,7 +36,11 @@ owner:: BOOLEAN PARAMETERS ------------------ parents:: - Whether to create parents as well (mkdir -p behaviour) + Whether to create parents as well (mkdir -p behaviour). + Warning: all intermediate directory permissions default + to whatever mkdir -p does. + + Usually this means root:root, 0700. recursive:: If supplied the chgrp and chown call will run recursively. diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group new file mode 100644 index 00000000..1308c710 --- /dev/null +++ b/cdist/conf/type/__git/explorer/group @@ -0,0 +1,5 @@ +#!/bin/sh + +destination="/$__object_id/.git" + +stat --print "%G" ${destination} 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__git/explorer/owner b/cdist/conf/type/__git/explorer/owner new file mode 100644 index 00000000..8c36b035 --- /dev/null +++ b/cdist/conf/type/__git/explorer/owner @@ -0,0 +1,5 @@ +#!/bin/sh + +destination="/$__object_id/.git" + +stat --print "%U" ${destination} 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__git/gencode-remote b/cdist/conf/type/__git/gencode-remote index 0f665d59..d719a492 100644 --- a/cdist/conf/type/__git/gencode-remote +++ b/cdist/conf/type/__git/gencode-remote @@ -20,6 +20,9 @@ # state_is="$(cat "$__object/explorer/state")" +owner_is="$(cat "$__object/explorer/owner")" +group_is="$(cat "$__object/explorer/group")" + state_should=present [ -f "$__object/parameter/state" ] && state_should="$(cat "$__object/parameter/state")" @@ -30,11 +33,31 @@ source="$(cat "$__object/parameter/source")" destination="/$__object_id" -[ "$state_should" = "$state_is" ] && exit 0 +owner="" +[ -f "$__object/parameter/owner" ] && owner="$(cat "$__object/parameter/owner")" +group="" +[ -f "$__object/parameter/group" ] && group="$(cat "$__object/parameter/group")" +mode="" +[ -f "$__object/parameter/mode" ] && mode="$(cat "$__object/parameter/mode")" + +[ "$state_should" = "$state_is" -a \ + "$owner" = "$owner_is" -a \ + "$group" = "$group_is" -a \ + -n "$mode" ] && exit 0 case $state_should in present) - echo git clone --quiet --branch "$branch" "$source" "$destination" + + if [ "$state_should" != "$state_is" ]; then + echo git clone --quiet --branch "$branch" "$source" "$destination" + fi + if [ \( -n "$owner" -a "$owner_is" != "$owner" \) -o \ + \( -n "$group" -a "$group_is" != "$group" \) ]; then + echo chown -R "${owner}:${group}" "$destination" + fi + if [ -n "$mode" ]; then + echo chmod -R "$mode" "$destination" + fi ;; # Handled in manifest absent) diff --git a/cdist/conf/type/__git/man.text b/cdist/conf/type/__git/man.text index 5597a52d..7c6b83cd 100644 --- a/cdist/conf/type/__git/man.text +++ b/cdist/conf/type/__git/man.text @@ -27,6 +27,15 @@ state:: branch:: Create this branch by checking out the remote branch of this name +group:: + Group to chgrp to. + +mode:: + Unix permissions, suitable for chmod. + +owner:: + User to chown to. + EXAMPLES -------- diff --git a/cdist/conf/type/__git/manifest b/cdist/conf/type/__git/manifest index e8c9b233..8d6a29e4 100644 --- a/cdist/conf/type/__git/manifest +++ b/cdist/conf/type/__git/manifest @@ -26,14 +26,11 @@ __package git --state present state_should=present [ -f "$__object/parameter/state" ] && state_should="$(cat "$__object/parameter/state")" -[ -f "$__object/parameter/owner" ] && dirparams="$dirparams --owner $(cat "$__object/parameter/owner")" -[ -f "$__object/parameter/group" ] && dirparams="$dirparams --group $(cat "$__object/parameter/group")" - # Let __directory handle removal of git repos case "$state_should" in present) - __directory "$__object_id" --state present $dirparams --recursive + : ;; absent) diff --git a/cdist/conf/type/__git/parameter/optional b/cdist/conf/type/__git/parameter/optional index d9684aaa..3c409162 100644 --- a/cdist/conf/type/__git/parameter/optional +++ b/cdist/conf/type/__git/parameter/optional @@ -2,3 +2,4 @@ state branch group owner +mode diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script new file mode 100644 index 00000000..2dc952e9 --- /dev/null +++ b/cdist/conf/type/__iptables_apply/files/init-script @@ -0,0 +1,48 @@ +#!/bin/sh +# Nico Schottelius +# Zürisee, Mon Sep 2 18:38:27 CEST 2013 +# +### BEGIN INIT INFO +# Provides: iptables +# Required-Start: $local_fs $remote_fs +# Required-Stop: $local_fs $remote_fs +# X-Start-Before: fail2ban +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Applies iptables ruleset +# Description: Applies all rules found in /etc/iptables.d +# and saves/restores previous status +### END INIT INFO + + +basedir=/etc/iptables.d +status="${basedir}/.pre-start" + +case $1 in + start) + # Save status + iptables-save > "$status" + + # Apply our ruleset + cd "$basedir" + count="$(ls -1 | wc -l)" + + # Only do something if there are rules + if [ "$count" -ge 1 ]; then + for rule in *; do + echo "Applying iptables rule $rule ..." + iptables $(cat "$rule") + done + fi + ;; + + stop) + # Restore from status before, if there is something to restore + if [ -f "$status" ]; then + iptables-restore < "$status" + fi + ;; + restart) + "$0" stop && "$0" start + ;; +esac diff --git a/cdist/conf/type/__iptables_apply/gencode-remote b/cdist/conf/type/__iptables_apply/gencode-remote new file mode 100644 index 00000000..0773b452 --- /dev/null +++ b/cdist/conf/type/__iptables_apply/gencode-remote @@ -0,0 +1,2 @@ +# Rebuild rules - FIXME: do conditionally as soon as cdist supports it +echo /etc/init.d/iptables restart diff --git a/cdist/conf/type/__iptables_apply/man.text b/cdist/conf/type/__iptables_apply/man.text new file mode 100644 index 00000000..87f4b4ee --- /dev/null +++ b/cdist/conf/type/__iptables_apply/man.text @@ -0,0 +1,42 @@ +cdist-type__iptables_apply(7) +============================= +Nico Schottelius + + +NAME +---- +cdist-type__iptables_apply - Apply the rules + + +DESCRIPTION +----------- +This cdist type deploys an init script that triggers +the configured rules and also re-applies them on +configuration. + + +REQUIRED PARAMETERS +------------------- +None + +OPTIONAL PARAMETERS +------------------- +None + +EXAMPLES +-------- + +None (__iptables_apply is used by __iptables_rule) + + +SEE ALSO +-------- +- cdist-type(7) +- cdist-type__iptables_rule(7) +- iptables(8) + + +COPYING +------- +Copyright \(C) 2013 Nico Schottelius. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__iptables_apply/manifest b/cdist/conf/type/__iptables_apply/manifest new file mode 100644 index 00000000..a22901ba --- /dev/null +++ b/cdist/conf/type/__iptables_apply/manifest @@ -0,0 +1,26 @@ +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +__file /etc/init.d/iptables \ + --source "$__type/files/init-script" \ + --state present \ + --mode 0755 + +require="__file/etc/init.d/iptables" __start_on_boot iptables diff --git a/cdist/test/type/fixtures/__singleton/singleton b/cdist/conf/type/__iptables_apply/singleton similarity index 100% rename from cdist/test/type/fixtures/__singleton/singleton rename to cdist/conf/type/__iptables_apply/singleton diff --git a/cdist/conf/type/__iptables_rule/man.text b/cdist/conf/type/__iptables_rule/man.text new file mode 100644 index 00000000..eb230093 --- /dev/null +++ b/cdist/conf/type/__iptables_rule/man.text @@ -0,0 +1,64 @@ +cdist-type__iptables_rule(7) +============================ +Nico Schottelius + + +NAME +---- +cdist-type__iptables_rule - Deploy iptable rulesets + + +DESCRIPTION +----------- +This cdist type allows you to manage iptable rules +in a distribution independent manner. + + +REQUIRED PARAMETERS +------------------- +rule:: + The rule to apply. Essentially an iptables command + line without iptables in front of it. + + +OPTIONAL PARAMETERS +------------------- +state:: + 'present' or 'absent', defaults to 'present' + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# Deploy some policies +__iptables_rule policy-in --rule "-P INPUT DROP" +__iptables_rule policy-out --rule "-P OUTPUT ACCEPT" +__iptables_rule policy-fwd --rule "-P FORWARD DROP" + +# The usual established rule +__iptables_rule established --rule "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT" + +# Some service rules +__iptables_rule http --rule "-A INPUT -p tcp --dport 80 -j ACCEPT" +__iptables_rule ssh --rule "-A INPUT -p tcp --dport 80 -j ACCEPT" +__iptables_rule https --rule "-A INPUT -p tcp --dport 443 -j ACCEPT" + +# Ensure some rules are not present anymore +__iptables_rule munin --rule "-A INPUT -p tcp --dport 4949 -j ACCEPT" \ + --state absent + +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- cdist-type__iptables_apply(7) +- iptables(8) + + +COPYING +------- +Copyright \(C) 2013 Nico Schottelius. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest new file mode 100644 index 00000000..a6abbd5e --- /dev/null +++ b/cdist/conf/type/__iptables_rule/manifest @@ -0,0 +1,46 @@ +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +base_dir=/etc/iptables.d + +name="$__object_id" + +if [ -f "$__object/parameter/state" ]; then + state="$(cat "$__object/parameter/state")" +else + state="present" +fi + +################################################################################ +# Basic setup +# + +__directory "$base_dir" --state present + +# Have apply do the real job +require="$__object_name" __iptables_apply + +################################################################################ +# The rule +# + +require="__directory/$base_dir" __file "$base_dir/${name}" \ + --source "$__object/parameter/rule" \ + --state "$state" diff --git a/cdist/conf/type/__postgres_database/parameter/required b/cdist/conf/type/__iptables_rule/parameter/optional similarity index 100% rename from cdist/conf/type/__postgres_database/parameter/required rename to cdist/conf/type/__iptables_rule/parameter/optional diff --git a/cdist/conf/type/__iptables_rule/parameter/required b/cdist/conf/type/__iptables_rule/parameter/required new file mode 100644 index 00000000..2b254dff --- /dev/null +++ b/cdist/conf/type/__iptables_rule/parameter/required @@ -0,0 +1 @@ +rule diff --git a/cdist/conf/type/__jail/gencode-remote b/cdist/conf/type/__jail/gencode-remote index 7491754c..b044e4b0 100755 --- a/cdist/conf/type/__jail/gencode-remote +++ b/cdist/conf/type/__jail/gencode-remote @@ -92,6 +92,20 @@ fi present="$(cat "$__object/explorer/present")" status="$(cat "$__object/explorer/status")" +# Handle ip="iface|addr, iface|addr" format +if [ $(expr "${ip}" : ".*|.*") -gt "0" ]; then + # If we have multiple IPs defined, $interface doesn't make sense because ip="iface|addr, iface|addr" implies it + interface="" + SAVE_IFS="$IFS" + IFS=", " + for cur_ip in ${ip}; do + # Just get the last IP address for SSH to listen on + mgmt_ip=$(echo "${ip}" | sed -E -e 's/^.*\|(.*)\/[0-9]+$/\1/') + done + IFS="$SAVE_IFS" +else + mgmt_ip=$(echo "${ip}" | cut '-d ' -f1) +fi stopJail() { # Check $status before issuing command @@ -160,10 +174,10 @@ EOF createJail() { # Create the jail directory cat <>/etc/rc.conf <>"${jaildir}/rw/${name}/etc/rc.conf" EOF # Configure SSHd's listening address cat < \"$file.cdist-tmp\"" + printf 'sed "s|^%s\(%s\+\).*|%s\\1%s|" "%s" > "%s.cdist-tmp"\n' \ + "$key" "$delimiter" "$key" "$value" "$file" "$file" echo "mv \"$file.cdist-tmp\" \"$file\"" ;; *) diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state index d240bf4d..d04d5d09 100755 --- a/cdist/conf/type/__line/explorer/state +++ b/cdist/conf/type/__line/explorer/state @@ -1,6 +1,6 @@ #!/bin/sh # -# 2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -24,16 +24,18 @@ file="/$__object_id" if [ -f "$__object/parameter/regex" ]; then regex=$(cat "$__object/parameter/regex") + greparg="" else if [ ! -f "$__object/parameter/line" ]; then echo "Parameter line and regex missing - cannot explore" >&2 exit 1 fi - regex="^$(cat "$__object/parameter/line")\$" + regex="$(cat "$__object/parameter/line")" + greparg="-F -x" fi # Allow missing file - thus 2>/dev/null -if grep -q "$regex" "$file" 2>/dev/null; then +if grep -q $greparg "$regex" "$file" 2>/dev/null; then echo present else echo absent diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index 8ac273e2..1c46c16c 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -38,7 +38,19 @@ case "$state_should" in exit 1 fi - echo "echo \"$line\" >> $file" + #echo "echo \"$line\" >> $file" + #line_sanitised=$(cat "$__object/parameter/line" | sed 's/"/\"/g') + # Idea: replace ' in the string: + # '"'"' + # |------> ': end the string + # |-|---> "'": create ' in the output string + # |--> ': continue the string + # + # Replace all \ so \t and other combinations are not interpreted + # + + line_sanitised=$(cat "$__object/parameter/line" | sed -e "s/'/'\"'\"'/g" -e 's/\\/\\\\/g') + echo "printf '%s\n' '$line_sanitised' >> $file" ;; absent) @@ -47,13 +59,16 @@ case "$state_should" in exit 1 fi - [ "$line" ] && regex="^$line\$" + greparg="" + if [ "$line" ]; then + regex="$line" + greparg="-F -x" + fi cat << eof tmp=\$(mktemp) -sed '/$regex/d' "$file" > \$tmp && cat "\$tmp" > "$file" && rm -f "\$tmp" +grep -v $greparg '$regex' '$file' > \$tmp && cat "\$tmp" > '$file' && rm -f "\$tmp" eof - #echo "echo q | ex -c \"/${line}/d|w|q\" \"${file}\"" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__line/man.text b/cdist/conf/type/__line/man.text index e1a5941c..f39ee929 100644 --- a/cdist/conf/type/__line/man.text +++ b/cdist/conf/type/__line/man.text @@ -32,11 +32,11 @@ regex:: given line, if the given regular expression does not match. In case of absent, ensure all lines matching the - regular expression are absent (cannot be combined with - the line parameter, if state is absent). + regular expression are absent. - If the regular expression contains / (slashes), they need - to be escaped with \ (backslash): / becomes \/. + The regular expression is interpreted by grep. + + Must not be combined with line, if state is absent. file:: If supplied, use this as the destination file. @@ -64,9 +64,10 @@ __line legacy_timezone --file /etc/rc.conf --regex 'TIMEZONE=.*' --state absent SEE ALSO -------- - cdist-type(7) +- grep(1) COPYING ------- -Copyright \(C) 2012 Nico Schottelius. Free use of this software is +Copyright \(C) 2012-2013 Nico Schottelius. Free use of this software is granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__locale/files/locale.gen b/cdist/conf/type/__locale/files/locale.gen new file mode 100644 index 00000000..cf8e8651 --- /dev/null +++ b/cdist/conf/type/__locale/files/locale.gen @@ -0,0 +1,3 @@ +de_CH.UTF-8 UTF-8 +de_DE.UTF-8 UTF-8 +en_US.UTF-8 UTF-8 diff --git a/cdist/conf/type/__locale/gencode-remote b/cdist/conf/type/__locale/gencode-remote new file mode 100644 index 00000000..538ce2cd --- /dev/null +++ b/cdist/conf/type/__locale/gencode-remote @@ -0,0 +1,51 @@ +#!/bin/sh +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Let localedef do the magic +# + +locale="$__object_id" + +# Hardcoded, create a pull request with +# branching on $os in case it is at another location +alias=/usr/share/locale/locale.alias + +input=$(echo "$locale" | cut -d . -f 1) +charmap=$(echo "$locale" | cut -d . -f 2) + +# Adding locale? The name is de_CH.UTF-8 +# Removing locale? The name is de_CH.utf8. +# W-T-F! +locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/') + +state=$(cat "$__object/parameter/state") + +case "$state" in + present) + echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale" + ;; + absent) + echo localedef --delete-from-archive "$locale_remove" + ;; + *) + echo "Unsupported state: $state" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__locale/man.text b/cdist/conf/type/__locale/man.text new file mode 100644 index 00000000..f76c2059 --- /dev/null +++ b/cdist/conf/type/__locale/man.text @@ -0,0 +1,47 @@ +cdist-type__locale(7) +===================== +Nico Schottelius + + +NAME +---- +cdist-type__locale - Configure locales + + +DESCRIPTION +----------- +This cdist type allows you to setup locales. + + +OPTIONAL PARAMETERS +------------------- +state:: + 'present' or 'absent' + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# Add locale de_CH.UTF-8 +__locale de_CH.UTF-8 + +# Same as above, but more explicit +__locale de_CH.UTF-8 --state present + +# Remove colourful British English +__locale en_GB.UTF-8 --state absent +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- locale(1) +- localedef(1) +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2013 Nico Schottelius. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__locale/manifest b/cdist/conf/type/__locale/manifest new file mode 100644 index 00000000..5dd5fd8f --- /dev/null +++ b/cdist/conf/type/__locale/manifest @@ -0,0 +1,32 @@ +#!/bin/sh +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Install required packages +# + +os=$(cat "$__global/explorer/os") + + +case "$os" in + debian) + # Debian needs a seperate package + __package locales --state present + ;; +esac diff --git a/cdist/conf/type/__locale/parameter/default/state b/cdist/conf/type/__locale/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__locale/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__postgres_role/parameter/required b/cdist/conf/type/__locale/parameter/optional similarity index 100% rename from cdist/conf/type/__postgres_role/parameter/required rename to cdist/conf/type/__locale/parameter/optional diff --git a/cdist/conf/type/__motd/gencode-remote b/cdist/conf/type/__motd/gencode-remote new file mode 100755 index 00000000..2aa84902 --- /dev/null +++ b/cdist/conf/type/__motd/gencode-remote @@ -0,0 +1,33 @@ +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +os=$(cat "$__global/explorer/os") + +case "$os" in + debian|ubuntu) + + # Debian and Ubuntu need to be updated, + # as seen in /etc/init.d/bootlogs + echo "uname -snrvm > /var/run/motd" + echo "cat /etc/motd.tail >> /var/run/motd" + ;; + *) + exit 0 + ;; +esac diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index a80d707e..7aba76d5 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -42,7 +42,7 @@ case "$state_is" in ;; esac -aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes" +aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes --no-install-recommends -o DPkg::Options::=\"--force-confold\"" [ "$state_is" = "$state_should" ] && exit 0 diff --git a/cdist/conf/type/__package_opkg/gencode-remote b/cdist/conf/type/__package_opkg/gencode-remote index 43f1ad8a..1fb78fbe 100755 --- a/cdist/conf/type/__package_opkg/gencode-remote +++ b/cdist/conf/type/__package_opkg/gencode-remote @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2011,2013 Nico Schottelius (nico-cdist at schottelius.org) # 2012 Giel van Schijndel (giel plus cdist at mortis dot eu) # # This file is part of cdist. @@ -42,20 +42,20 @@ case "$state_is" in ;; esac -if [ "$state_is" != "$state_should" ]; then - case "$state_should" in - present) - if [ "$present" = "notpresent" ]; then +[ "$state_is" = "$state_should" ] && exit 0 + +case "$state_should" in + present) + if [ "$present" = "notpresent" ]; then echo opkg --verbosity=0 update - fi - echo opkg --verbosity=0 install \"$name\" - ;; - absent) - echo opkg --verbosity=0 remove \"$name\" - ;; - *) - echo "Unknown state: $state" >&2 - exit 1 - ;; - esac -fi + fi + echo opkg --verbosity=0 install \"$name\" + ;; + absent) + echo opkg --verbosity=0 remove \"$name\" + ;; + *) + echo "Unknown state: $state" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__postfix/man.text b/cdist/conf/type/__postfix/man.text new file mode 100644 index 00000000..1a91723a --- /dev/null +++ b/cdist/conf/type/__postfix/man.text @@ -0,0 +1,42 @@ +cdist-type__postfix(7) +====================== +Steven Armstrong + + +NAME +---- +cdist-type__postfix - install postfix + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__postfix +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2012 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__postfix/manifest b/cdist/conf/type/__postfix/manifest new file mode 100755 index 00000000..2dc70ce2 --- /dev/null +++ b/cdist/conf/type/__postfix/manifest @@ -0,0 +1,33 @@ +#!/bin/sh +# +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + ubuntu|debian|archlinux) + __package postfix --state present + ;; + *) + echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 + ;; +esac diff --git a/cdist/test/autorequire/fixtures/conf/explorer/.keep b/cdist/conf/type/__postfix/singleton similarity index 100% rename from cdist/test/autorequire/fixtures/conf/explorer/.keep rename to cdist/conf/type/__postfix/singleton diff --git a/cdist/conf/type/__postfix_master/explorer/entry b/cdist/conf/type/__postfix_master/explorer/entry new file mode 100755 index 00000000..9d6b1514 --- /dev/null +++ b/cdist/conf/type/__postfix_master/explorer/entry @@ -0,0 +1,39 @@ +#!/bin/sh +# +# 2011 - 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +config="/etc/postfix/master.cf" + +# no master.cf, nothing we could do +[ -f "$config" ] || exit 0 + +# NOTE: keep variables in sync in manifest,explorer,gencode-* +prefix="#cdist:$__object_name" +suffix="#/cdist:$__object_name" +awk -v prefix="$prefix" -v suffix="$suffix" '{ + if (index($0,prefix)) { + triggered=1 + } + if (triggered) { + if (index($0,suffix)) { + triggered=0 + } + print + } +}' "$config" diff --git a/cdist/conf/type/__postfix_master/gencode-remote b/cdist/conf/type/__postfix_master/gencode-remote new file mode 100755 index 00000000..51edc668 --- /dev/null +++ b/cdist/conf/type/__postfix_master/gencode-remote @@ -0,0 +1,78 @@ +#!/bin/sh +# +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +config="/etc/postfix/master.cf" +entry="$__object/files/entry" +state_should="$(cat "$__object/parameter/state")" +if [ ! -s "$__object/explorer/entry" ]; then + state_is='absent' +else + state_is=$(diff -q "$entry" "$__object/explorer/entry" >/dev/null \ + && echo present \ + || echo changed + ) +fi + +if [ "$state_should" = "$state_is" ]; then + # Nothing to do, move along + exit 0 +fi + + +remove_entry() { + # NOTE: keep variables in sync in manifest/explorer/gencode-* + prefix="#cdist:$__object_name" + suffix="#/cdist:$__object_name" + cat << DONE +tmpfile=\$(mktemp ${config}.cdist.XXXXXXXXXX) +# preserve ownership and permissions of existing file +cp -p "$config" "\$tmpfile" +awk -v prefix="$prefix" -v suffix="$suffix" ' +{ + if (index(\$0,prefix)) { + triggered=1 + } + if (triggered) { + if (index(\$0,suffix)) { + triggered=0 + } + } else { + print + } +}' "$config" > "\$tmpfile" +mv -f "\$tmpfile" "$config" +DONE +} + +case "$state_should" in + present) + if [ "$state_is" = "changed" ]; then + remove_entry + fi + cat << DONE +cat >> "$config" << ${__type##*/}_DONE +$(cat "$entry") +${__type##*/}_DONE +DONE + ;; + absent) + remove_entry + ;; +esac diff --git a/cdist/conf/type/__postfix_master/man.text b/cdist/conf/type/__postfix_master/man.text new file mode 100644 index 00000000..0ec78752 --- /dev/null +++ b/cdist/conf/type/__postfix_master/man.text @@ -0,0 +1,73 @@ +cdist-type__postfix_master(7) +============================= +Steven Armstrong + + +NAME +---- +cdist-type__postfix_master - configure postfix master.cf + + +DESCRIPTION +----------- +See master(5) for more information. + + +REQUIRED PARAMETERS +------------------- +type:: + See master(5) +command:: + See master(5) + + +BOOLEAN PARAMETERS +------------------ +noreload:: + don't reload postfix after changes + + +OPTIONAL PARAMETERS +------------------- +state:: + present or absent, defaults to present +service:: +private:: +unpriv:: +chroot:: +wakeup:: +maxproc:: +option:: + Pass an option to a service. Same as using -o in master.cf. + Can be specified multiple times. +comment:: + a textual comment to add with the master.cf entry + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__postfix_master smtp --type inet --command smtpd + +__postfix_master smtp --type inet --chroot y --command smtpd \ + --option smtpd_enforce_tls=yes \ + --option smtpd_sasl_auth_enable=yes \ + --option smtpd_client_restrictions=permit_sasl_authenticated,reject + +__postfix_master submission --type inet --command smtpd \ + --comment "Run alternative smtp on submission port" +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- master(5) + + +COPYING +------- +Copyright \(C) 2012 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). + diff --git a/cdist/conf/type/__postfix_master/manifest b/cdist/conf/type/__postfix_master/manifest new file mode 100755 index 00000000..1642e91b --- /dev/null +++ b/cdist/conf/type/__postfix_master/manifest @@ -0,0 +1,81 @@ +#!/bin/sh +# +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + ubuntu|debian|archlinux) + : + ;; + *) + echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 + ;; +esac + + +# Default to object_id +service="$(cat "$__object/parameter/service" 2>/dev/null || echo "$__object_id")" +state="$(cat "$__object/parameter/state")" + +# NOTE: keep variables in sync in manifest,explorer,gencode-* +prefix="#cdist:$__object_name" +suffix="#/cdist:$__object_name" + +# Generate entry for inclusion in master.cf +mkdir "$__object/files" +entry="$__object/files/entry" +( + echo "$prefix" + if [ -f "$__object/parameter/comment" ]; then + echo "# $(cat "$__object/parameter/comment")" + fi + printf "%s " "$service" + printf "%s " "$type" + for parameter in type private unpriv chroot wakeup maxproc; do + printf "%s " "$(cat "$__object/parameter/$parameter")" + done + command="$(cat "$__object/parameter/command")" + # ensure we have a trailing newline + echo "$command" + options="$(cat "$__object/parameter/option" 2>/dev/null || true)" + for option in $options; do + echo " -o $option" + done + echo "$suffix" +) > "$entry" + +# Reload postfix after changes +if [ ! -f "$__object/parameter/noreload" ]; then + state_should="$(cat "$__object/parameter/state")" + if [ ! -s "$__object/explorer/entry" ]; then + state_is='absent' + else + state_is=$(diff -q "$entry" "$__object/explorer/entry" >/dev/null \ + && echo present \ + || echo changed + ) + fi + if [ "$state_is" != "$state_should" ]; then + require="$__object_name" __postfix_reload + fi +fi diff --git a/cdist/conf/type/__postfix_master/parameter/boolean b/cdist/conf/type/__postfix_master/parameter/boolean new file mode 100644 index 00000000..862edc87 --- /dev/null +++ b/cdist/conf/type/__postfix_master/parameter/boolean @@ -0,0 +1 @@ +noreload diff --git a/cdist/conf/type/__postfix_master/parameter/default/chroot b/cdist/conf/type/__postfix_master/parameter/default/chroot new file mode 100644 index 00000000..39cdd0de --- /dev/null +++ b/cdist/conf/type/__postfix_master/parameter/default/chroot @@ -0,0 +1 @@ +- diff --git a/cdist/conf/type/__postfix_master/parameter/default/maxproc b/cdist/conf/type/__postfix_master/parameter/default/maxproc new file mode 100644 index 00000000..39cdd0de --- /dev/null +++ b/cdist/conf/type/__postfix_master/parameter/default/maxproc @@ -0,0 +1 @@ +- diff --git a/cdist/conf/type/__postfix_master/parameter/default/private b/cdist/conf/type/__postfix_master/parameter/default/private new file mode 100644 index 00000000..39cdd0de --- /dev/null +++ b/cdist/conf/type/__postfix_master/parameter/default/private @@ -0,0 +1 @@ +- diff --git a/cdist/conf/type/__postfix_master/parameter/default/state b/cdist/conf/type/__postfix_master/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__postfix_master/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__postfix_master/parameter/default/unpriv b/cdist/conf/type/__postfix_master/parameter/default/unpriv new file mode 100644 index 00000000..39cdd0de --- /dev/null +++ b/cdist/conf/type/__postfix_master/parameter/default/unpriv @@ -0,0 +1 @@ +- diff --git a/cdist/conf/type/__postfix_master/parameter/default/wakeup b/cdist/conf/type/__postfix_master/parameter/default/wakeup new file mode 100644 index 00000000..39cdd0de --- /dev/null +++ b/cdist/conf/type/__postfix_master/parameter/default/wakeup @@ -0,0 +1 @@ +- diff --git a/cdist/conf/type/__postfix_master/parameter/optional b/cdist/conf/type/__postfix_master/parameter/optional new file mode 100644 index 00000000..792b42c5 --- /dev/null +++ b/cdist/conf/type/__postfix_master/parameter/optional @@ -0,0 +1,9 @@ +service +private +unpriv +chroot +wakeup +maxproc +option +comment +state diff --git a/cdist/conf/type/__postfix_master/parameter/required b/cdist/conf/type/__postfix_master/parameter/required new file mode 100644 index 00000000..24c14146 --- /dev/null +++ b/cdist/conf/type/__postfix_master/parameter/required @@ -0,0 +1,2 @@ +type +command diff --git a/cdist/conf/type/__postfix_postconf/explorer/value b/cdist/conf/type/__postfix_postconf/explorer/value new file mode 100755 index 00000000..edf48b48 --- /dev/null +++ b/cdist/conf/type/__postfix_postconf/explorer/value @@ -0,0 +1,37 @@ +#!/bin/sh +# +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$("$__explorer/os") + +case "$os" in + ubuntu|debian|archlinux) + : + ;; + *) + echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 + ;; +esac + +key="$(cat "$__object/parameter/key" 2>/dev/null || echo "$__object_id")" + +postconf -h "$key" diff --git a/cdist/conf/type/__postfix_postconf/gencode-remote b/cdist/conf/type/__postfix_postconf/gencode-remote new file mode 100755 index 00000000..60143590 --- /dev/null +++ b/cdist/conf/type/__postfix_postconf/gencode-remote @@ -0,0 +1,60 @@ +#!/bin/sh +# +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$(cat "$__global/explorer/os") + +case "$os" in + ubuntu|debian|archlinux) + : + ;; + *) + echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 + ;; +esac + +state_should="$(cat "$__object/parameter/state")" +if [ ! -s "$__object/explorer/value" ]; then + state_is='absent' +else + state_is=$(diff -q "$__object/parameter/value" "$__object/explorer/value" >/dev/null \ + && echo present \ + || echo changed + ) +fi + +if [ "$state_should" = "$state_is" ]; then + # Nothing to do, move along + exit 0 +fi + +key="$(cat "$__object/parameter/key" 2>/dev/null || echo "$__object_id")" +value="$(cat "$__object/parameter/value")" + +case "$state_should" in + absent) + # revert parameter to its default value + echo "postconf -# $key" + ;; + present) + echo "postconf -e '$key=$value'" + ;; +esac diff --git a/cdist/conf/type/__postfix_postconf/man.text b/cdist/conf/type/__postfix_postconf/man.text new file mode 100644 index 00000000..727637b1 --- /dev/null +++ b/cdist/conf/type/__postfix_postconf/man.text @@ -0,0 +1,51 @@ +cdist-type__postfix_postconf(7) +=============================== +Steven Armstrong + + +NAME +---- +cdist-type__postfix_postconf - configure postfix main.cf + + +DESCRIPTION +----------- +See postconf(5) for possible keys and values. + +Note that this type directly runs the postconf executable. +It does not make changes to /etc/postfix/main.cf itself. + + +REQUIRED PARAMETERS +------------------- +value:: + the value for the postfix parameter + + +OPTIONAL PARAMETERS +------------------- +key:: + the name of the parameter. Defaults to __object_id + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__postfix_postconf mydomain --value somedomain.com + +__postfix_postconf bind-to-special-ip --key smtp_bind_address --value 127.0.0.5 + +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- postconf(5) + + +COPYING +------- +Copyright \(C) 2012 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__postfix_postconf/parameter/default/state b/cdist/conf/type/__postfix_postconf/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__postfix_postconf/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__process/parameter/required b/cdist/conf/type/__postfix_postconf/parameter/optional similarity index 60% rename from cdist/conf/type/__process/parameter/required rename to cdist/conf/type/__postfix_postconf/parameter/optional index ff72b5c7..6ada755a 100644 --- a/cdist/conf/type/__process/parameter/required +++ b/cdist/conf/type/__postfix_postconf/parameter/optional @@ -1 +1,2 @@ +key state diff --git a/cdist/conf/type/__postfix_postconf/parameter/required b/cdist/conf/type/__postfix_postconf/parameter/required new file mode 100644 index 00000000..6d4e1507 --- /dev/null +++ b/cdist/conf/type/__postfix_postconf/parameter/required @@ -0,0 +1 @@ +value diff --git a/cdist/conf/type/__postfix_postmap/gencode-remote b/cdist/conf/type/__postfix_postmap/gencode-remote new file mode 100755 index 00000000..1b370001 --- /dev/null +++ b/cdist/conf/type/__postfix_postmap/gencode-remote @@ -0,0 +1,21 @@ +#!/bin/sh +# +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +echo "postmap /$__object_id" diff --git a/cdist/conf/type/__postfix_postmap/man.text b/cdist/conf/type/__postfix_postmap/man.text new file mode 100644 index 00000000..37060d04 --- /dev/null +++ b/cdist/conf/type/__postfix_postmap/man.text @@ -0,0 +1,42 @@ +cdist-type__postfix_postmap(7) +============================== +Steven Armstrong + + +NAME +---- +cdist-type__postfix_postmap - run postmap on the given file + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__postfix_postmap /etc/postfix/generic +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2012 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__postfix_reload/gencode-remote b/cdist/conf/type/__postfix_reload/gencode-remote new file mode 100755 index 00000000..5822f1e3 --- /dev/null +++ b/cdist/conf/type/__postfix_reload/gencode-remote @@ -0,0 +1,33 @@ +#!/bin/sh +# +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + ubuntu|debian|archlinux) + echo "postfix reload" + ;; + *) + echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__postfix_reload/man.text b/cdist/conf/type/__postfix_reload/man.text new file mode 100644 index 00000000..c63356b5 --- /dev/null +++ b/cdist/conf/type/__postfix_reload/man.text @@ -0,0 +1,42 @@ +cdist-type__postfix_reload(7) +============================= +Steven Armstrong + + +NAME +---- +cdist-type__postfix_reload - tell postfix to reload its configuration + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__postfix_reload +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2012 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/test/autorequire/fixtures/conf/type/__addifnosuchline/.keep b/cdist/conf/type/__postfix_reload/singleton similarity index 100% rename from cdist/test/autorequire/fixtures/conf/type/__addifnosuchline/.keep rename to cdist/conf/type/__postfix_reload/singleton diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote index c097efce..0ffc842a 100755 --- a/cdist/conf/type/__postgres_database/gencode-remote +++ b/cdist/conf/type/__postgres_database/gencode-remote @@ -19,7 +19,8 @@ # name="$__object_id" -state_should="$(cat "$__object/parameter/state")" +state_should="present" +[ -f "$__object/parameter/state" ] && state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" if [ "$state_should" != "$state_is" ]; then diff --git a/cdist/conf/type/__postgres_database/man.text b/cdist/conf/type/__postgres_database/man.text index d01ca8f6..88259b6f 100644 --- a/cdist/conf/type/__postgres_database/man.text +++ b/cdist/conf/type/__postgres_database/man.text @@ -13,14 +13,11 @@ DESCRIPTION This cdist type allows you to create or drop postgres databases. -REQUIRED PARAMETERS +OPTIONAL PARAMETERS ------------------- state:: either 'present' or 'absent' - -OPTIONAL PARAMETERS -------------------- owner:: the role owning this database @@ -29,7 +26,7 @@ EXAMPLES -------- -------------------------------------------------------------------------------- -__postgres_database mydbname --state present --owner mydbusername +__postgres_database mydbname --owner mydbusername -------------------------------------------------------------------------------- diff --git a/cdist/conf/type/__postgres_database/parameter/optional b/cdist/conf/type/__postgres_database/parameter/optional index 7ee3bde8..d86b6469 100644 --- a/cdist/conf/type/__postgres_database/parameter/optional +++ b/cdist/conf/type/__postgres_database/parameter/optional @@ -1 +1,2 @@ +state owner diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index c9de4707..65a9d588 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -20,14 +20,15 @@ name="$__object_id" state_is="$(cat "$__object/explorer/state")" -state_should="$(cat "$__object/parameter/state")" +state_should="present" +[ -f "$__object/parameter/state" ] && state_should="$(cat "$__object/parameter/state")" [ "$state_is" = "$state_should" ] && exit 0 case "$state_should" in present) if [ -f "$__object/parameter/password" ]; then - password="$(cat "$__object/parameter/$parameter")" + password="$(cat "$__object/parameter/password")" fi booleans="" for boolean in login createdb createrole superuser; do diff --git a/cdist/conf/type/__postgres_role/man.text b/cdist/conf/type/__postgres_role/man.text index 904f0831..ac87754b 100644 --- a/cdist/conf/type/__postgres_role/man.text +++ b/cdist/conf/type/__postgres_role/man.text @@ -13,15 +13,12 @@ DESCRIPTION This cdist type allows you to create or drop postgres roles. -REQUIRED PARAMETERS +OPTIONAL PARAMETERS ------------------- state:: Either "present" or "absent", defaults to "present" - -OPTIONAL PARAMETERS -------------------- -All parameter map directly to the corresponding postgres createrole +All other parameters map directly to the corresponding postgres createrole parameters. password:: @@ -41,13 +38,13 @@ EXAMPLES -------- -------------------------------------------------------------------------------- -__postgres_role myrole --state present +__postgres_role myrole -__postgres_role myrole --state present --password 'secret' +__postgres_role myrole --password 'secret' -__postgres_role admin --state present --password 'very-secret' --superuser +__postgres_role admin --password 'very-secret' --superuser -__postgres_role dbcustomer --state present --password 'bla' --createdb +__postgres_role dbcustomer --password 'bla' --createdb -------------------------------------------------------------------------------- diff --git a/cdist/conf/type/__postgres_role/parameter/optional b/cdist/conf/type/__postgres_role/parameter/optional index f3097ab1..cb9b2c48 100644 --- a/cdist/conf/type/__postgres_role/parameter/optional +++ b/cdist/conf/type/__postgres_role/parameter/optional @@ -1 +1,2 @@ +state password diff --git a/cdist/conf/type/__process/gencode-remote b/cdist/conf/type/__process/gencode-remote index fdb6033a..41bc5381 100755 --- a/cdist/conf/type/__process/gencode-remote +++ b/cdist/conf/type/__process/gencode-remote @@ -25,7 +25,12 @@ else name="$__object_id" fi -state_should="$(cat "$__object/parameter/state")" +parameter_state="$__object/parameter/state" +if [ -f "$_parameter_state" ]; then + state_should=$(cat "$__object/parameter/state") +else + state_should="present" +fi runs="$(cat "$__object/explorer/runs")" if [ "$runs" ]; then diff --git a/cdist/conf/type/__process/man.text b/cdist/conf/type/__process/man.text index 0d457ead..2fdd27aa 100644 --- a/cdist/conf/type/__process/man.text +++ b/cdist/conf/type/__process/man.text @@ -13,14 +13,11 @@ DESCRIPTION This cdist type allows you to define the state of a process. -REQUIRED PARAMETERS +OPTIONAL PARAMETERS ------------------- state:: Either "present" or "absent", defaults to "present" - -OPTIONAL PARAMETERS -------------------- name:: Process name to match on when using pgrep -f -x. diff --git a/cdist/conf/type/__process/parameter/optional b/cdist/conf/type/__process/parameter/optional index 3411afb4..85fe8805 100644 --- a/cdist/conf/type/__process/parameter/optional +++ b/cdist/conf/type/__process/parameter/optional @@ -1,3 +1,4 @@ name stop start +state diff --git a/cdist/conf/type/__ssh_authorized_keys/gencode-remote b/cdist/conf/type/__ssh_authorized_keys/gencode-remote index cc86cc19..7fcb59c6 100755 --- a/cdist/conf/type/__ssh_authorized_keys/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_keys/gencode-remote @@ -1,6 +1,6 @@ #!/bin/sh # -# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# 2012-2013 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -24,7 +24,7 @@ if [ -f "$__object/parameter/file" ]; then else home="$(cut -d':' -f 6 "$__object/explorer/passwd")" file="$home/.ssh/authorized_keys" -fi +fi entry="$__object/files/entry" if [ ! -s "$__object/explorer/entry" ]; then @@ -47,7 +47,9 @@ remove_entry() { prefix="#cdist:$__object_name" suffix="#/cdist:$__object_name" cat << DONE -tmpfile=\$(mktemp) +tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX) +# preserve ownership and permissions by copying existing file over tmpfile +cp -p "$file" "\$tmpfile" awk -v prefix="$prefix" -v suffix="$suffix" ' { if (index(\$0,prefix)) { diff --git a/cdist/conf/type/__ssh_authorized_keys/manifest b/cdist/conf/type/__ssh_authorized_keys/manifest index 268b1fbe..47cdf746 100755 --- a/cdist/conf/type/__ssh_authorized_keys/manifest +++ b/cdist/conf/type/__ssh_authorized_keys/manifest @@ -19,7 +19,7 @@ # owner="$(cat "$__object/parameter/owner" 2>/dev/null || echo "$__object_id")" -state="$(cat "$__object/parameter/present" 2>/dev/null || echo "present")" +state="$(cat "$__object/parameter/state" 2>/dev/null || echo "present")" if [ -f "$__object/parameter/file" ]; then file="$(cat "$__object/parameter/file")" else diff --git a/cdist/conf/type/__start_on_boot/explorer/state b/cdist/conf/type/__start_on_boot/explorer/state index 6fd0ea92..4e0c82c2 100755 --- a/cdist/conf/type/__start_on_boot/explorer/state +++ b/cdist/conf/type/__start_on_boot/explorer/state @@ -1,6 +1,6 @@ #!/bin/sh # -# 2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -28,20 +28,9 @@ name="$__object_id" case "$os" in archlinux) - # convert bash array to shell - daemons=$(grep ^DAEMONS /etc/rc.conf | sed -e 's/^.*=(//' -e 's/)$//') - - # absent, as long as not found - state="absent" - - # iterate, last one wins. - for daemon in $daemons; do - if [ "$daemon" = "$name" -o "$daemon" = "@${name}" ]; then - state="present" - elif [ "$daemon" = "!${name}" ]; then - state="absent" - fi - done + state=$(systemctl is-enabled "$name" >/dev/null 2>&1 \ + && echo present \ + || echo absent) ;; debian|ubuntu|openwrt) @@ -54,7 +43,7 @@ case "$os" in [ "$state" ] || state="present" ;; - *) + *) echo "Unsupported os: $os" >&2 exit 1 ;; diff --git a/cdist/conf/type/__start_on_boot/gencode-remote b/cdist/conf/type/__start_on_boot/gencode-remote index 7724e8c7..58ff6a4a 100755 --- a/cdist/conf/type/__start_on_boot/gencode-remote +++ b/cdist/conf/type/__start_on_boot/gencode-remote @@ -1,6 +1,6 @@ #!/bin/sh # -# 2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -32,8 +32,7 @@ case "$state_should" in present) case "$os" in archlinux) - echo "sed 's/^\\(DAEMONS=.*\\))/\\1 $name)/' /etc/rc.conf > /etc/rc.conf.cdist-tmp" - echo "mv /etc/rc.conf.cdist-tmp /etc/rc.conf" + echo "systemctl enable \"$name\"" ;; debian|ubuntu) echo "update-rc.d \"$name\" defaults >/dev/null" @@ -65,10 +64,7 @@ case "$state_should" in absent) case "$os" in archlinux) - # Replace a) at the beginning b) in the middle c) end d) only - # Support @name as well...makes it more ugly, but well... - echo "sed /etc/rc.conf -e 's/^\\(DAEMONS=(\\)@\\{0,1\\}$name /\\1/' -e 's/^\\(DAEMONS=(.* \\)@\\{0,1\\}$name \\(.*\\)/\\1\\2/' -e 's/^\\(DAEMONS=(.*\\) @\\{0,1\\}$name)/\\1)/' -e 's/^\\(DAEMONS=(\\)@\\{0,1\\}$name)/\\1)/' > /etc/rc.conf.cdist-tmp" - echo "mv /etc/rc.conf.cdist-tmp /etc/rc.conf" + echo "systemctl disable \"$name\"" ;; debian|ubuntu) echo update-rc.d -f \"$name\" remove diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote new file mode 100755 index 00000000..19ea9968 --- /dev/null +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -0,0 +1,26 @@ +#!/bin/sh +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Setup alternative - no standard way to create, always set +# + +path="$(cat "$__object/parameter/path")" +name="$__object_id" +echo "update-alternatives --quiet --set '$name' '$path'" diff --git a/cdist/conf/type/__update_alternatives/man.text b/cdist/conf/type/__update_alternatives/man.text new file mode 100644 index 00000000..2bcc1874 --- /dev/null +++ b/cdist/conf/type/__update_alternatives/man.text @@ -0,0 +1,43 @@ +cdist-type__update_alternatives(7) +================================== +Nico Schottelius + + +NAME +---- +cdist-type__update_alternatives - Configure alternatives + + +DESCRIPTION +----------- +On Debian and alike systems update-alternatives(1) can be used +to setup alternatives for various programs. +One of the most common used targets is the "editor". + + +REQUIRED PARAMETERS +------------------- +path:: + Use this path for the given alternative + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# Setup vim as the default editor +__update_alternatives editor --path /usr/bin/vim.basic +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- cdist-type__debconf_set_selections(7) +- update-alternatives(8) + + +COPYING +------- +Copyright \(C) 2013 Nico Schottelius. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__update_alternatives/parameter/required b/cdist/conf/type/__update_alternatives/parameter/required new file mode 100644 index 00000000..e7a8fd4d --- /dev/null +++ b/cdist/conf/type/__update_alternatives/parameter/required @@ -0,0 +1 @@ +path diff --git a/cdist/conf/type/__user/gencode-remote b/cdist/conf/type/__user/gencode-remote index baa6f354..a2cdfd22 100755 --- a/cdist/conf/type/__user/gencode-remote +++ b/cdist/conf/type/__user/gencode-remote @@ -39,6 +39,7 @@ shorten_property() { password) ret="-p";; shell) ret="-s";; uid) ret="-u";; + create-home) ret="-m";; esac echo "$ret" } @@ -76,6 +77,7 @@ if grep -q "^${name}:" "$__object/explorer/passwd"; then home) field=6 ;; shell) field=7 ;; uid) field=3 ;; + create-home) continue;; # Does not apply to user modification esac # If we haven't already set $current_value above, pull it from the @@ -102,7 +104,11 @@ if grep -q "^${name}:" "$__object/explorer/passwd"; then else for property in $(ls .); do new_value="$(cat "$property")" - set -- "$@" "$(shorten_property $property)" \'$new_value\' + if [ -z "$new_value" ];then # Boolean values have no value + set -- "$@" "$(shorten_property $property)" + else + set -- "$@" "$(shorten_property $property)" \'$new_value\' + fi done if [ "$os" = "freebsd" ]; then diff --git a/cdist/conf/type/__user/parameter/boolean b/cdist/conf/type/__user/parameter/boolean new file mode 100644 index 00000000..e0517c6a --- /dev/null +++ b/cdist/conf/type/__user/parameter/boolean @@ -0,0 +1 @@ +create-home diff --git a/cdist/config.py b/cdist/config.py index 9af25b75..7e003835 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -20,7 +20,252 @@ # # -import cdist.config_install +import logging +import os +import shutil +import sys +import time +import pprint -class Config(cdist.config_install.ConfigInstall): - pass +import cdist + +import cdist.exec.local +import cdist.exec.remote + +from cdist import core + +class Config(object): + """Cdist main class to hold arbitrary data""" + + def __init__(self, local, remote, dry_run=False): + + self.local = local + self.remote = remote + self.log = logging.getLogger(self.local.target_host) + self.dry_run = dry_run + + self.explorer = core.Explorer(self.local.target_host, self.local, self.remote) + self.manifest = core.Manifest(self.local.target_host, self.local) + self.code = core.Code(self.local.target_host, self.local, self.remote) + + def _init_files_dirs(self): + """Prepare files and directories for the run""" + self.local.create_files_dirs() + self.remote.create_files_dirs() + + @classmethod + def commandline(cls, args): + """Configure remote system""" + import multiprocessing + + # FIXME: Refactor relict - remove later + log = logging.getLogger("cdist") + + initial_manifest_tempfile = None + if args.manifest == '-': + # read initial manifest from stdin + import tempfile + try: + handle, initial_manifest_temp_path = tempfile.mkstemp(prefix='cdist.stdin.') + with os.fdopen(handle, 'w') as fd: + fd.write(sys.stdin.read()) + except (IOError, OSError) as e: + raise cdist.Error("Creating tempfile for stdin data failed: %s" % e) + + args.manifest = initial_manifest_temp_path + import atexit + atexit.register(lambda: os.remove(initial_manifest_temp_path)) + + process = {} + failed_hosts = [] + time_start = time.time() + + for host in args.host: + if args.parallel: + log.debug("Creating child process for %s", host) + process[host] = multiprocessing.Process(target=cls.onehost, args=(host, args, True)) + process[host].start() + else: + try: + cls.onehost(host, args, parallel=False) + except cdist.Error as e: + failed_hosts.append(host) + + # Catch errors in parallel mode when joining + if args.parallel: + for host in process.keys(): + log.debug("Joining process %s", host) + process[host].join() + + if not process[host].exitcode == 0: + failed_hosts.append(host) + + time_end = time.time() + log.info("Total processing time for %s host(s): %s", len(args.host), + (time_end - time_start)) + + if len(failed_hosts) > 0: + raise cdist.Error("Failed to configure the following hosts: " + + " ".join(failed_hosts)) + + @classmethod + def onehost(cls, host, args, parallel): + """Configure ONE system""" + + log = logging.getLogger(host) + + try: + local = cdist.exec.local.Local( + target_host=host, + initial_manifest=args.manifest, + base_path=args.out_path, + add_conf_dirs=args.conf_dir) + + remote = cdist.exec.remote.Remote( + target_host=host, + remote_exec=args.remote_exec, + remote_copy=args.remote_copy) + + c = cls(local, remote, dry_run=args.dry_run) + c.run() + + except cdist.Error as e: + log.error(e) + if parallel: + # We are running in our own process here, need to sys.exit! + sys.exit(1) + else: + raise + + except KeyboardInterrupt: + # Ignore in parallel mode, we are existing anyway + if parallel: + sys.exit(0) + # Pass back to controlling code in sequential mode + else: + raise + + def run(self): + """Do what is most often done: deploy & cleanup""" + start_time = time.time() + + self._init_files_dirs() + + self.explorer.run_global_explorers(self.local.global_explorer_out_path) + self.manifest.run_initial_manifest(self.local.initial_manifest) + self.iterate_until_finished() + + self.local.save_cache() + self.log.info("Finished successful run in %s seconds", time.time() - start_time) + + + def object_list(self): + """Short name for object list retrieval""" + for cdist_object in core.CdistObject.list_objects(self.local.object_path, + self.local.type_path): + yield cdist_object + + def iterate_once(self): + """ + Iterate over the objects once - helper method for + iterate_until_finished + """ + objects_changed = False + + for cdist_object in self.object_list(): + if cdist_object.requirements_unfinished(cdist_object.requirements): + """We cannot do anything for this poor object""" + continue + + if cdist_object.state == core.CdistObject.STATE_UNDEF: + """Prepare the virgin object""" + + self.object_prepare(cdist_object) + objects_changed = True + + if cdist_object.requirements_unfinished(cdist_object.autorequire): + """The previous step created objects we depend on - wait for them""" + continue + + if cdist_object.state == core.CdistObject.STATE_PREPARED: + self.object_run(cdist_object) + objects_changed = True + + return objects_changed + + + def iterate_until_finished(self): + """ + Go through all objects and solve them + one after another + """ + + objects_changed = True + + while objects_changed: + objects_changed = self.iterate_once() + + # Check whether all objects have been finished + unfinished_objects = [] + for cdist_object in self.object_list(): + if not cdist_object.state == cdist_object.STATE_DONE: + unfinished_objects.append(cdist_object) + + if unfinished_objects: + info_string = [] + + for cdist_object in unfinished_objects: + + requirement_names = [] + autorequire_names = [] + + for requirement in cdist_object.requirements_unfinished(cdist_object.requirements): + requirement_names.append(requirement.name) + + for requirement in cdist_object.requirements_unfinished(cdist_object.autorequire): + autorequire_names.append(requirement.name) + + requirements = ", ".join(requirement_names) + autorequire = ", ".join(autorequire_names) + info_string.append("%s requires: %s autorequires: %s" % (cdist_object.name, requirements, autorequire)) + + raise cdist.UnresolvableRequirementsError("The requirements of the following objects could not be resolved: %s" % + ("; ".join(info_string))) + + def object_prepare(self, cdist_object): + """Prepare object: Run type explorer + manifest""" + self.log.info("Running manifest and explorers for " + cdist_object.name) + self.explorer.run_type_explorers(cdist_object) + self.manifest.run_type_manifest(cdist_object) + cdist_object.state = core.CdistObject.STATE_PREPARED + + def object_run(self, cdist_object): + """Run gencode and code for an object""" + + self.log.debug("Trying to run object %s" % (cdist_object.name)) + if cdist_object.state == core.CdistObject.STATE_DONE: + raise cdist.Error("Attempting to run an already finished object: %s", cdist_object) + + cdist_type = cdist_object.cdist_type + + # Generate + self.log.info("Generating and executing code for %s" % (cdist_object.name)) + cdist_object.code_local = self.code.run_gencode_local(cdist_object) + cdist_object.code_remote = self.code.run_gencode_remote(cdist_object) + if cdist_object.code_local or cdist_object.code_remote: + cdist_object.changed = True + + # Execute + if not self.dry_run: + if cdist_object.code_local: + self.code.run_code_local(cdist_object) + if cdist_object.code_remote: + self.code.transfer_code_remote(cdist_object) + self.code.run_code_remote(cdist_object) + else: + self.log.info("Skipping code execution due to DRY RUN") + + + # Mark this object as done + self.log.debug("Finishing run of " + cdist_object.name) + cdist_object.state = core.CdistObject.STATE_DONE diff --git a/cdist/config_install.py b/cdist/config_install.py deleted file mode 100644 index f1529cc1..00000000 --- a/cdist/config_install.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# 2010-2012 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - -import logging -import os -import stat -import shutil -import sys -import tempfile -import time -import itertools -import pprint - -import cdist -from cdist import core -from cdist import resolver - - -class ConfigInstall(object): - """Cdist main class to hold arbitrary data""" - - def __init__(self, context): - - self.context = context - self.log = logging.getLogger(self.context.target_host) - - # Initialise local directory structure - self.context.local.create_files_dirs() - # Initialise remote directory structure - self.context.remote.create_files_dirs() - - self.explorer = core.Explorer(self.context.target_host, self.context.local, self.context.remote) - self.manifest = core.Manifest(self.context.target_host, self.context.local) - self.code = core.Code(self.context.target_host, self.context.local, self.context.remote) - - # Add switch to disable code execution - self.dry_run = False - - def cleanup(self): - # FIXME: move to local? - destination = os.path.join(self.context.local.cache_path, self.context.target_host) - self.log.debug("Saving " + self.context.local.out_path + " to " + destination) - if os.path.exists(destination): - shutil.rmtree(destination) - shutil.move(self.context.local.out_path, destination) - - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - self.stage_prepare() - self.stage_run() - - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - start_time = time.time() - self.deploy_to() - self.cleanup() - self.log.info("Finished successful run in %s seconds", - time.time() - start_time) - - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.explorer.run_global_explorers(self.context.local.global_explorer_out_path) - self.manifest.run_initial_manifest(self.context.initial_manifest) - - self.log.info("Running object manifests and type explorers") - - # Continue process until no new objects are created anymore - new_objects_created = True - while new_objects_created: - new_objects_created = False - for cdist_object in core.CdistObject.list_objects(self.context.local.object_path, - self.context.local.type_path): - if cdist_object.state == core.CdistObject.STATE_PREPARED: - self.log.debug("Skipping re-prepare of object %s", cdist_object) - continue - else: - self.object_prepare(cdist_object) - new_objects_created = True - - def object_prepare(self, cdist_object): - """Prepare object: Run type explorer + manifest""" - self.log.info("Running manifest and explorers for " + cdist_object.name) - self.explorer.run_type_explorers(cdist_object) - self.manifest.run_type_manifest(cdist_object) - cdist_object.state = core.CdistObject.STATE_PREPARED - - def object_run(self, cdist_object, dry_run=False): - """Run gencode and code for an object""" - self.log.debug("Trying to run object " + cdist_object.name) - if cdist_object.state == core.CdistObject.STATE_DONE: - raise cdist.Error("Attempting to run an already finished object: %s", cdist_object) - - cdist_type = cdist_object.cdist_type - - # Generate - self.log.info("Generating and executing code for " + cdist_object.name) - cdist_object.code_local = self.code.run_gencode_local(cdist_object) - cdist_object.code_remote = self.code.run_gencode_remote(cdist_object) - if cdist_object.code_local or cdist_object.code_remote: - cdist_object.changed = True - - # Execute - if not dry_run: - if cdist_object.code_local: - self.code.run_code_local(cdist_object) - if cdist_object.code_remote: - self.code.transfer_code_remote(cdist_object) - self.code.run_code_remote(cdist_object) - - # Mark this object as done - self.log.debug("Finishing run of " + cdist_object.name) - cdist_object.state = core.CdistObject.STATE_DONE - - def stage_run(self): - """The final (and real) step of deployment""" - self.log.info("Generating and executing code") - - # FIXME: think about parallel execution (same for stage_prepare) - self.all_resolved = False - while not self.all_resolved: - self.stage_run_iterate() - - def stage_run_iterate(self): - """ - Run one iteration of the run - - To be repeated until all objects are done - """ - objects = list(core.CdistObject.list_objects(self.context.local.object_path, self.context.local.type_path)) - object_state_list=' '.join('%s:%s:%s:%s' % (o, o.state, o.all_requirements, o.satisfied_requirements) for o in objects) - - self.log.debug("Object state (name:state:requirements:satisfied): %s" % object_state_list) - - objects_changed = False - self.all_resolved = True - for cdist_object in objects: - if not cdist_object.state == cdist_object.STATE_DONE: - self.all_resolved = False - self.log.debug("Object %s not done" % cdist_object.name) - if cdist_object.satisfied_requirements: - self.log.debug("Running object %s with satisfied requirements" % cdist_object.name) - self.object_run(cdist_object, self.dry_run) - objects_changed = True - - self.log.debug("All resolved: %s Objects changed: %s" % (self.all_resolved, objects_changed)) - - # Not all are resolved, but nothing has been changed => bad dependencies! - if not objects_changed and not self.all_resolved: - # Create list of unfinished objects + their requirements for print - - evil_objects = [] - good_objects = [] - for cdist_object in objects: - if not cdist_object.state == cdist_object.STATE_DONE: - evil_objects.append("%s: required: %s, autorequired: %s" % - (cdist_object.name, cdist_object.requirements, cdist_object.autorequire)) - else: - evil_objects.append("%s (%s): required: %s, autorequired: %s" % - (cdist_object.state, cdist_object.name, - cdist_object.requirements, cdist_object.autorequire)) - - errormessage = "Cannot solve requirements for the following objects: %s - solved: %s" % (",".join(evil_objects), ",".join(good_objects)) - raise cdist.Error(errormessage) diff --git a/cdist/context.py b/cdist/context.py deleted file mode 100644 index e0391be8..00000000 --- a/cdist/context.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# 2010-2012 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - -import logging -import os -import sys -import tempfile -import shutil - -from cdist.exec import local -from cdist.exec import remote - - -class Context(object): - """Hold information about current context""" - - def __init__(self, - target_host, - remote_copy, - remote_exec, - initial_manifest=False, - add_conf_dirs=None, - exec_path=sys.argv[0], - debug=False): - - self.debug = debug - self.target_host = target_host - self.exec_path = exec_path - - # Context logging - self.log = logging.getLogger(self.target_host) - self.log.addFilter(self) - - # Local temp directory - # FIXME: if __cdist_out_dir can be given from the outside, the same directory will be used for all hosts - if '__cdist_out_dir' in os.environ: - self.out_path = os.environ['__cdist_out_dir'] - self.temp_dir = None - else: - self.temp_dir = tempfile.mkdtemp() - self.out_path = os.path.join(self.temp_dir, "out") - - self.local = local.Local(self.target_host, self.out_path, self.exec_path, add_conf_dirs=add_conf_dirs) - - self.initial_manifest = (initial_manifest or - os.path.join(self.local.manifest_path, "init")) - - self._init_remote(remote_copy, remote_exec) - - # Remote stuff - def _init_remote(self, remote_copy, remote_exec): - - self.remote_base_path = os.environ.get('__cdist_remote_out_dir', "/var/lib/cdist") - self.remote_copy = remote_copy - self.remote_exec = remote_exec - - os.environ['__remote_copy'] = self.remote_copy - os.environ['__remote_exec'] = self.remote_exec - - self.remote = remote.Remote(self.target_host, self.remote_base_path, - self.remote_exec, self.remote_copy) - - def cleanup(self): - """Remove temp stuff""" - if self.temp_dir: - shutil.rmtree(self.temp_dir) - - def filter(self, record): - """Add hostname to logs via logging Filter""" - - record.msg = self.target_host + ": " + str(record.msg) - - return True diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 7beea130..e3c1c532 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -42,6 +42,13 @@ class IllegalObjectIdError(cdist.Error): def __str__(self): return '%s: %s' % (self.message, self.object_id) +class MissingObjectIdError(cdist.Error): + def __init__(self, type_name): + self.type_name = type_name + self.message = "Type %s requires object id (is not a singleton type)" % self.type_name + + def __str__(self): + return '%s' % (self.message) class CdistObject(object): """Represents a cdist object. @@ -53,11 +60,12 @@ class CdistObject(object): """ # Constants for use with Object.state + STATE_UNDEF = "" STATE_PREPARED = "prepared" STATE_RUNNING = "running" STATE_DONE = "done" - def __init__(self, cdist_type, base_path, object_id=None): + def __init__(self, cdist_type, base_path, object_id=''): self.cdist_type = cdist_type # instance of Type self.base_path = base_path self.object_id = object_id @@ -99,7 +107,6 @@ class CdistObject(object): """ type_name = object_name.split(os.sep)[0] - # FIXME: allow object without object_id? e.g. for singleton object_id = os.sep.join(object_name.split(os.sep)[1:]) return type_name, object_id @@ -125,8 +132,12 @@ class CdistObject(object): # If no object_id and type is not singleton => error out if not self.object_id and not self.cdist_type.is_singleton: - raise IllegalObjectIdError(self.object_id, - "Missing object_id and type is not a singleton.") + raise MissingObjectIdError(self.cdist_type.name) + + # Does not work: AttributeError: 'CdistObject' object has no attribute 'parameter_path' + + #"Type %s is not a singleton type - missing object id (parameters: %s)" % + # (self.cdist_type.name, self.parameters)) def object_from_name(self, object_name): """Convenience method for creating an object instance from an object name. @@ -211,66 +222,15 @@ class CdistObject(object): except EnvironmentError as error: raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error)) - @property - def satisfied_requirements(self): - """Return state whether all of our dependencies have been resolved already""" + def requirements_unfinished(self, requirements): + """Return state whether requirements are satisfied""" - satisfied = True + object_list = [] - for requirement in self.all_requirements: - log.debug("%s: Checking requirement %s (%s) .." % (self.name, requirement.name, requirement.state)) - if not requirement.state == self.STATE_DONE: - satisfied = False - break - log.debug("%s is satisfied: %s" % (self.name, satisfied)) + for requirement in requirements: + cdist_object = self.object_from_name(requirement) - return satisfied + if not cdist_object.state == self.STATE_DONE: + object_list.append(cdist_object) - - def find_requirements_by_name(self, requirements): - """Takes a list of requirement patterns and returns a list of matching object instances. - - Patterns are expected to be Unix shell-style wildcards for use with fnmatch.filter. - - find_requirements_by_name(['__type/object_id', '__other_type/*']) -> - [, , ] - """ - - - # FIXME: think about where/when to store this - probably not here - self.objects = dict((o.name, o) for o in self.list_objects(self.base_path, self.cdist_type.base_path)) - object_names = self.objects.keys() - - for pattern in requirements: - found = False - for requirement in fnmatch.filter(object_names, pattern): - found = True - yield self.objects[requirement] - if not found: - # FIXME: get rid of the singleton object_id, it should be invisible to the code -> hide it in Object - singleton = os.path.join(pattern, 'singleton') - if singleton in self.objects: - yield self.objects[singleton] - else: - raise RequirementNotFoundError(pattern) - - @property - def all_requirements(self): - """ - Return resolved autorequirements and requirements so that - a complete list of requirements is returned - """ - - all_reqs= [] - all_reqs.extend(self.find_requirements_by_name(self.requirements)) - all_reqs.extend(self.find_requirements_by_name(self.autorequire)) - - return set(all_reqs) - - -class RequirementNotFoundError(cdist.Error): - def __init__(self, requirement): - self.requirement = requirement - - def __str__(self): - return 'Requirement could not be found: %s' % self.requirement + return object_list diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index 0efb10f4..46e126f9 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -25,7 +25,8 @@ import os import cdist class NoSuchTypeError(cdist.Error): - def __init__(self, type_path, type_absolute_path): + def __init__(self, name, type_path, type_absolute_path): + self.name = name self.type_path = type_path self.type_absolute_path = type_absolute_path @@ -48,7 +49,7 @@ class CdistType(object): self.path = self.name self.absolute_path = os.path.join(self.base_path, self.path) if not os.path.isdir(self.absolute_path): - raise NoSuchTypeError(self.path, self.absolute_path) + raise NoSuchTypeError(self.name, self.path, self.absolute_path) self.manifest_path = os.path.join(self.name, "manifest") self.explorer_path = os.path.join(self.name, "explorer") self.gencode_local_path = os.path.join(self.name, "gencode-local") @@ -61,6 +62,7 @@ class CdistType(object): self.__optional_parameters = None self.__optional_multiple_parameters = None self.__boolean_parameters = None + self.__parameter_defaults = None @classmethod def list_types(cls, base_path): @@ -99,11 +101,6 @@ class CdistType(object): """Check whether a type is a singleton.""" return os.path.isfile(os.path.join(self.absolute_path, "singleton")) - @property - def is_install(self): - """Check whether a type is used for installation (if not: for configuration)""" - return os.path.isfile(os.path.join(self.absolute_path, "install")) - @property def explorers(self): """Return a list of available explorers""" @@ -194,3 +191,19 @@ class CdistType(object): finally: self.__boolean_parameters = parameters return self.__boolean_parameters + + @property + def parameter_defaults(self): + if not self.__parameter_defaults: + defaults = {} + try: + defaults_dir = os.path.join(self.absolute_path, "parameter", "default") + for name in os.listdir(defaults_dir): + with open(os.path.join(defaults_dir, name)) as fd: + defaults[name] = fd.read().strip() + except EnvironmentError: + # error ignored + pass + finally: + self.__parameter_defaults = defaults + return self.__parameter_defaults diff --git a/cdist/core/code.py b/cdist/core/code.py index fa1ed3c1..d5f59094 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -89,7 +89,7 @@ class Code(object): self.remote = remote self.env = { '__target_host': self.target_host, - '__global': self.local.out_path, + '__global': self.local.base_path, } def _run_gencode(self, cdist_object, which): diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 19639618..97121474 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -94,7 +94,7 @@ class Manifest(object): self.env = { 'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']), '__cdist_type_base_path': self.local.type_path, # for use in type emulator - '__global': self.local.out_path, + '__global': self.local.base_path, '__target_host': self.target_host, } if self.log.getEffectiveLevel() == logging.DEBUG: @@ -106,6 +106,7 @@ class Manifest(object): env.update(self.env) env['__cdist_manifest'] = initial_manifest env['__manifest'] = self.local.manifest_path + env['__explorer'] = self.local.global_explorer_out_path return env diff --git a/cdist/emulator.py b/cdist/emulator.py index a9c760cb..b1cd8f2d 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2012 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. @@ -28,20 +28,33 @@ import sys import cdist from cdist import core +class MissingRequiredEnvironmentVariableError(cdist.Error): + def __init__(self, name): + self.name = name + self.message = "Emulator requires the environment variable %s to be setup" % self.name + + def __str__(self): + return self.message + + class Emulator(object): def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ): self.argv = argv self.stdin = stdin self.env = env - self.object_id = False + self.object_id = '' - self.global_path = self.env['__global'] - self.target_host = self.env['__target_host'] + try: + self.global_path = self.env['__global'] + self.target_host = self.env['__target_host'] - # Internally only - self.object_source = self.env['__cdist_manifest'] - self.type_base_path = self.env['__cdist_type_base_path'] + # Internally only + self.object_source = self.env['__cdist_manifest'] + self.type_base_path = self.env['__cdist_type_base_path'] + + except KeyError as e: + raise MissingRequiredEnvironmentVariableError(e.args[0]) self.object_base_path = os.path.join(self.global_path, "object") @@ -50,26 +63,9 @@ class Emulator(object): self.__init_log() - def filter(self, record): - """Add hostname and object to logs via logging Filter""" - - prefix = self.target_host + ": (emulator)" - - if self.object_id: - prefix = prefix + " " + self.type_name + "/" + self.object_id - - record.msg = prefix + ": " + record.msg - - return True - def run(self): """Emulate type commands (i.e. __file and co)""" - if '__install' in self.env: - if not self.cdist_type.is_install: - self.log.debug("Running in install mode, ignoring non install type") - return True - self.commandline() self.setup_object() self.save_stdin() @@ -79,16 +75,13 @@ class Emulator(object): def __init_log(self): """Setup logging facility""" - logformat = '%(levelname)s: %(message)s' - logging.basicConfig(format=logformat) if '__cdist_debug' in self.env: logging.root.setLevel(logging.DEBUG) else: logging.root.setLevel(logging.INFO) - self.log = logging.getLogger(__name__) - self.log.addFilter(self) + self.log = logging.getLogger(self.target_host) def commandline(self): """Parse command line""" @@ -103,10 +96,12 @@ class Emulator(object): parser.add_argument(argument, dest=parameter, action='append', required=True) for parameter in self.cdist_type.optional_parameters: argument = "--" + parameter - parser.add_argument(argument, dest=parameter, action='store', required=False) + parser.add_argument(argument, dest=parameter, action='store', required=False, + default=self.cdist_type.parameter_defaults.get(parameter, None)) for parameter in self.cdist_type.optional_multiple_parameters: argument = "--" + parameter - parser.add_argument(argument, dest=parameter, action='append', required=False) + parser.add_argument(argument, dest=parameter, action='append', required=False, + default=self.cdist_type.parameter_defaults.get(parameter, None)) for parameter in self.cdist_type.boolean_parameters: argument = "--" + parameter parser.add_argument(argument, dest=parameter, action='store_const', const='') @@ -122,9 +117,7 @@ class Emulator(object): def setup_object(self): # Setup object_id - FIXME: unset / do not setup anymore! - if self.cdist_type.is_singleton: - self.object_id = "singleton" - else: + if not self.cdist_type.is_singleton: self.object_id = self.args.object_id[0] del self.args.object_id @@ -184,7 +177,12 @@ class Emulator(object): if len(requirement) == 0: continue # Raises an error, if object cannot be created - cdist_object = self.cdist_object.object_from_name(requirement) + try: + cdist_object = self.cdist_object.object_from_name(requirement) + except core.cdist_type.NoSuchTypeError as e: + self.log.error("%s requires object %s, but type %s does not exist (definded at %s)" % (self.cdist_object.name, requirement, e.name, self.object_source)) + raise + self.log.debug("Recording requirement: " + requirement) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 7f640411..3a3ac706 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -27,6 +27,7 @@ import re import subprocess import shutil import logging +import tempfile import cdist from cdist import core @@ -38,20 +39,35 @@ class Local(object): Directly accessing the local side from python code is a bug. """ - def __init__(self, target_host, out_path, exec_path, add_conf_dirs=None, cache_dir=None): + def __init__(self, + target_host, + exec_path=sys.argv[0], + initial_manifest=None, + base_path=None, + add_conf_dirs=None): self.target_host = target_host - self.out_path = out_path + + # FIXME: stopped: create base that does not require moving later + if base_path: + self.base_path = base_path + else: + self.base_path = tempfile.mkdtemp() + + # FIXME: as well + self._init_cache_dir(None) + self.exec_path = exec_path + self.custom_initial_manifest = initial_manifest self._add_conf_dirs = add_conf_dirs self._init_log() self._init_permissions() self._init_paths() - self._init_cache_dir(cache_dir) self._init_conf_dirs() + @property def dist_conf_dir(self): return os.path.abspath(os.path.join(os.path.dirname(cdist.__file__), "conf")) @@ -72,22 +88,23 @@ class Local(object): def _init_paths(self): # Depending on out_path - self.bin_path = os.path.join(self.out_path, "bin") - self.conf_path = os.path.join(self.out_path, "conf") - self.global_explorer_out_path = os.path.join(self.out_path, "explorer") - self.object_path = os.path.join(self.out_path, "object") + self.bin_path = os.path.join(self.base_path, "bin") + self.conf_path = os.path.join(self.base_path, "conf") + self.global_explorer_out_path = os.path.join(self.base_path, "explorer") + self.object_path = os.path.join(self.base_path, "object") # Depending on conf_path self.global_explorer_path = os.path.join(self.conf_path, "explorer") self.manifest_path = os.path.join(self.conf_path, "manifest") + self.initial_manifest = (self.custom_initial_manifest or + os.path.join(self.manifest_path, "init")) + self.type_path = os.path.join(self.conf_path, "type") def _init_conf_dirs(self): self.conf_dirs = [] - # Comes with the distribution - system_conf_dir = os.path.abspath(os.path.join(os.path.dirname(cdist.__file__), "conf")) - self.conf_dirs.append(system_conf_dir) + self.conf_dirs.append(self.dist_conf_dir) # Is the default place for user created explorer, type and manifest if self.home_dir: @@ -99,10 +116,21 @@ class Local(object): cdist_path_dirs.reverse() self.conf_dirs.extend(cdist_path_dirs) - # Add user supplied directories + # Add command line supplied directories if self._add_conf_dirs: self.conf_dirs.extend(self._add_conf_dirs) + def _init_directories(self): + self.mkdir(self.conf_path) + self.mkdir(self.global_explorer_out_path) + self.mkdir(self.bin_path) + + def create_files_dirs(self): + self._init_directories() + self._create_conf_path_and_link_conf_dirs() + self._link_types_for_emulator() + + def _init_cache_dir(self, cache_dir): if cache_dir: self.cache_path = cache_dir @@ -127,8 +155,8 @@ class Local(object): Return the output as a string. """ - assert isinstance(command, (list, tuple)), "list or tuple argument expected, got: %s" % command self.log.debug("Local run: %s", command) + assert isinstance(command, (list, tuple)), "list or tuple argument expected, got: %s" % command if env is None: env = os.environ.copy() @@ -155,17 +183,12 @@ class Local(object): return self.run(command, env, return_output) - def create_files_dirs(self): - self._create_context_dirs() - self._create_conf_path_and_link_conf_dirs() - self._link_types_for_emulator() - - def _create_context_dirs(self): - self.mkdir(self.out_path) - - self.mkdir(self.conf_path) - self.mkdir(self.global_explorer_out_path) - self.mkdir(self.bin_path) + def save_cache(self): + destination = os.path.join(self.cache_path, self.target_host) + self.log.debug("Saving " + self.base_path + " to " + destination) + if os.path.exists(destination): + shutil.rmtree(destination) + shutil.move(self.base_path, destination) def _create_conf_path_and_link_conf_dirs(self): # Link destination directories @@ -184,7 +207,7 @@ class Local(object): for entry in os.listdir(current_dir): rel_entry_path = os.path.join(sub_dir, entry) - src = os.path.join(conf_dir, sub_dir, entry) + src = os.path.abspath(os.path.join(conf_dir, sub_dir, entry)) dst = os.path.join(self.conf_path, sub_dir, entry) # Already exists? remove and link diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index d4d2cb2b..2c42201a 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -43,12 +43,20 @@ class Remote(object): Directly accessing the remote side from python code is a bug. """ - def __init__(self, target_host, remote_base_path, remote_exec, remote_copy): + def __init__(self, + target_host, + remote_exec, + remote_copy, + base_path=None): self.target_host = target_host - self.base_path = remote_base_path self._exec = remote_exec self._copy = remote_copy + if base_path: + self.base_path = base_path + else: + self.base_path = "/var/lib/cdist" + self.conf_path = os.path.join(self.base_path, "conf") self.object_path = os.path.join(self.base_path, "object") @@ -57,9 +65,19 @@ class Remote(object): self.log = logging.getLogger(self.target_host) + self._init_env() + + def _init_env(self): + """Setup environment for scripts - HERE????""" + # FIXME: better do so in exec functions that require it! + os.environ['__remote_copy'] = self._copy + os.environ['__remote_exec'] = self._exec + + def create_files_dirs(self): self.rmdir(self.base_path) self.mkdir(self.base_path) + self.run(["chmod", "0700", self.base_path]) self.mkdir(self.conf_path) def rmdir(self, path): @@ -100,9 +118,6 @@ class Remote(object): cmd = self._exec.split() cmd.append(self.target_host) - # Always call umask before actual call to ensure proper file permissions - cmd.append("umask 077;") - # FIXME: replace this by -o SendEnv name -o SendEnv name ... to ssh? # can't pass environment to remote side, so prepend command with # variable declarations @@ -134,6 +149,6 @@ class Remote(object): except subprocess.CalledProcessError: raise cdist.Error("Command failed: " + " ".join(command)) except OSError as error: - raise cdist.Error(" ".join(*args) + ": " + error.args[1]) + raise cdist.Error(" ".join(command) + ": " + error.args[1]) except UnicodeDecodeError: raise DecodeError(command) diff --git a/cdist/install.py b/cdist/log.py similarity index 65% rename from cdist/install.py rename to cdist/log.py index 0f06f5e7..8c3aac79 100644 --- a/cdist/install.py +++ b/cdist/log.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -20,14 +20,19 @@ # # -import os -import cdist.config_install +import logging -class Install(cdist.config_install.ConfigInstall): - def __init__(self, *args, **kargs): - """Enhance config install with install support""" +class Log(logging.Logger): - # Setup environ to be used in emulator - os.environ['__install'] = "yes" + def __init__(self, name): - super().__init__(*args, **kargs) + self.name = name + super().__init__(name) + self.addFilter(self) + + def filter(self, record): + """Prefix messages with logger name""" + + record.msg = self.name + ": " + str(record.msg) + + return True diff --git a/cdist/resolver.py b/cdist/resolver.py deleted file mode 100644 index c1b2c292..00000000 --- a/cdist/resolver.py +++ /dev/null @@ -1,168 +0,0 @@ -# -*- coding: utf-8 -*- -# -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - -import logging -import os -import itertools -import fnmatch - -import cdist - -log = logging.getLogger(__name__) - - -class CircularReferenceError(cdist.Error): - def __init__(self, cdist_object, required_object): - self.cdist_object = cdist_object - self.required_object = required_object - - def __str__(self): - return 'Circular reference detected: %s -> %s' % (self.cdist_object.name, self.required_object.name) - - -class RequirementNotFoundError(cdist.Error): - def __init__(self, requirement): - self.requirement = requirement - - def __str__(self): - return 'Requirement could not be found: %s' % self.requirement - - -class DependencyResolver(object): - """Cdist's dependency resolver. - - Usage: - >> resolver = DependencyResolver(list_of_objects) - # Easy access to the objects we are working with - >> resolver.objects['__some_type/object_id'] - - # Easy access to a specific objects dependencies - >> resolver.dependencies['__some_type/object_id'] - [, ] - # Pretty print the dependency graph - >> from pprint import pprint - >> pprint(resolver.dependencies) - # Iterate over all existing objects in the correct order - >> for cdist_object in resolver: - >> do_something_with(cdist_object) - """ - def __init__(self, objects, logger=None): - self.objects = dict((o.name, o) for o in objects) - self._dependencies = None - self.log = logger or log - - @property - def dependencies(self): - """Build the dependency graph. - - Returns a dict where the keys are the object names and the values are - lists of all dependencies including the key object itself. - """ - if self._dependencies is None: - log.info("Resolving dependencies...") - self._dependencies = d = {} - self._preprocess_requirements() - for name,cdist_object in self.objects.items(): - resolved = [] - unresolved = [] - self._resolve_object_dependencies(cdist_object, resolved, unresolved) - d[name] = resolved - return self._dependencies - - def find_requirements_by_name(self, requirements): - """Takes a list of requirement patterns and returns a list of matching object instances. - - Patterns are expected to be Unix shell-style wildcards for use with fnmatch.filter. - - find_requirements_by_name(['__type/object_id', '__other_type/*']) -> - [, , ] - """ - object_names = self.objects.keys() - for pattern in requirements: - found = False - for requirement in fnmatch.filter(object_names, pattern): - found = True - yield self.objects[requirement] - if not found: - # FIXME: get rid of the singleton object_id, it should be invisible to the code -> hide it in Object - singleton = os.path.join(pattern, 'singleton') - if singleton in self.objects: - yield self.objects[singleton] - else: - raise RequirementNotFoundError(pattern) - - def _preprocess_requirements(self): - """Find all autorequire dependencies and merge them to be just requirements - for further processing. - """ - for cdist_object in self.objects.values(): - if cdist_object.autorequire: - # The objects (children) that this cdist_object (parent) defined - # in it's type manifest shall inherit all explicit requirements - # that the parent has so that user defined requirements are - # fullfilled and processed in the expected order. - for auto_requirement in self.find_requirements_by_name(cdist_object.autorequire): - for requirement in cdist_object.requirements: - if requirement not in auto_requirement.requirements: - auto_requirement.requirements.append(requirement) - # On the other hand the parent shall depend on all the children - # it created so that the user can setup dependencies on it as a - # whole without having to know anything about the parents - # internals. - cdist_object.requirements.extend(cdist_object.autorequire) - # As we changed the object on disc, we have to ensure it is not - # preprocessed again if someone would call us multiple times. - cdist_object.autorequire = [] - - def _resolve_object_dependencies(self, cdist_object, resolved, unresolved): - """Resolve all dependencies for the given cdist_object and store them - in the list which is passed as the 'resolved' arguments. - - e.g. - resolved = [] - unresolved = [] - resolve_object_dependencies(some_object, resolved, unresolved) - print("Dependencies for %s: %s" % (some_object, resolved)) - """ - self.log.debug('Resolving dependencies for: %s' % cdist_object.name) - try: - unresolved.append(cdist_object) - for required_object in self.find_requirements_by_name(cdist_object.requirements): - self.log.debug("Object %s requires %s", cdist_object, required_object) - if required_object not in resolved: - if required_object in unresolved: - raise CircularReferenceError(cdist_object, required_object) - self._resolve_object_dependencies(required_object, resolved, unresolved) - resolved.append(cdist_object) - unresolved.remove(cdist_object) - except RequirementNotFoundError as e: - raise cdist.CdistObjectError(cdist_object, "requires non-existing " + e.requirement) - - def __iter__(self): - """Iterate over all unique objects and yield them in the correct order. - """ - iterable = itertools.chain(*self.dependencies.values()) - # Keep record of objects that have already been seen - seen = set() - seen_add = seen.add - for cdist_object in itertools.filterfalse(seen.__contains__, iterable): - seen_add(cdist_object) - yield cdist_object diff --git a/cdist/shell.py b/cdist/shell.py new file mode 100644 index 00000000..ebf9f434 --- /dev/null +++ b/cdist/shell.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging +import os +import subprocess + +# initialise cdist +import cdist.exec.local + + +import cdist.config + +log = logging.getLogger(__name__) + +class Shell(object): + + def __init__(self, shell=None): + + self.shell = shell + + self.target_host = "cdist-shell-no-target-host" + self.local = cdist.exec.local.Local( + target_host=self.target_host) + + def _init_shell(self): + """Select shell to execute, if not specified by user""" + + if not self.shell: + if 'SHELL' in os.environ: + self.shell = os.environ['SHELL'] + else: + self.shell = "/bin/sh" + + def _init_files_dirs(self): + self.local.create_files_dirs() + + def _init_environment(self): + self.env = os.environ.copy() + additional_env = { + 'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']), + '__cdist_type_base_path': self.local.type_path, # for use in type emulator + '__cdist_manifest': "cdist shell", + '__global': self.local.base_path, + '__target_host': self.target_host, + '__manifest': self.local.manifest_path, + '__explorer': self.local.global_explorer_path, + } + + self.env.update(additional_env) + + def run(self): + self._init_shell() + self._init_files_dirs() + self._init_environment() + + log.info("Starting shell...") + self.local.run([self.shell], self.env) + log.info("Finished shell.") + + @classmethod + def commandline(cls, args): + shell = cls(args.shell) + shell.run() diff --git a/cdist/test/autorequire/__init__.py b/cdist/test/autorequire/__init__.py deleted file mode 100644 index 330680df..00000000 --- a/cdist/test/autorequire/__init__.py +++ /dev/null @@ -1,85 +0,0 @@ -# -*- coding: utf-8 -*- -# -# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2012 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - -import os -import shutil - -import cdist -from cdist import test -from cdist.exec import local -from cdist import core -from cdist.core import manifest -from cdist import resolver -from cdist import config -import cdist.context - -import os.path as op -my_dir = op.abspath(op.dirname(__file__)) -fixtures = op.join(my_dir, 'fixtures') -add_conf_dir = op.join(fixtures, 'conf') - -class AutorequireTestCase(test.CdistTestCase): - - def setUp(self): - self.orig_environ = os.environ - os.environ = os.environ.copy() - self.temp_dir = self.mkdtemp() - - self.out_dir = os.path.join(self.temp_dir, "out") - self.remote_out_dir = os.path.join(self.temp_dir, "remote") - - os.environ['__cdist_out_dir'] = self.out_dir - os.environ['__cdist_remote_out_dir'] = self.remote_out_dir - - self.context = cdist.context.Context( - target_host=self.target_host, - remote_copy=self.remote_copy, - remote_exec=self.remote_exec, - add_conf_dirs=[add_conf_dir], - exec_path=test.cdist_exec_path, - debug=False) - - self.config = config.Config(self.context) - - def tearDown(self): - os.environ = self.orig_environ - shutil.rmtree(self.temp_dir) - - def test_implicit_dependencies(self): - self.context.initial_manifest = os.path.join(self.context.local.manifest_path, 'implicit_dependencies') - self.config.stage_prepare() - - objects = core.CdistObject.list_objects(self.context.local.object_path, self.context.local.type_path) - dependency_resolver = resolver.DependencyResolver(objects) - expected_dependencies = [ - dependency_resolver.objects['__package_special/b'], - dependency_resolver.objects['__package/b'], - dependency_resolver.objects['__package_special/a'] - ] - resolved_dependencies = dependency_resolver.dependencies['__package_special/a'] - self.assertEqual(resolved_dependencies, expected_dependencies) - - def test_circular_dependency(self): - self.context.initial_manifest = os.path.join(self.context.local.manifest_path, 'circular_dependency') - self.config.stage_prepare() - # raises CircularDependecyError - self.config.stage_run() diff --git a/cdist/test/autorequire/fixtures/conf/manifest/circular_dependency b/cdist/test/autorequire/fixtures/conf/manifest/circular_dependency deleted file mode 100755 index 6ea308d1..00000000 --- a/cdist/test/autorequire/fixtures/conf/manifest/circular_dependency +++ /dev/null @@ -1,2 +0,0 @@ -# this has triggered CircularReferenceError -__nfsroot_client test diff --git a/cdist/test/autorequire/fixtures/conf/manifest/implicit_dependencies b/cdist/test/autorequire/fixtures/conf/manifest/implicit_dependencies deleted file mode 100755 index 7d68aed2..00000000 --- a/cdist/test/autorequire/fixtures/conf/manifest/implicit_dependencies +++ /dev/null @@ -1,3 +0,0 @@ -# this creates implicit dependencies through autorequire. -# this failed because autorequired dependencies where not aware of their anchestors dependencies -__top test diff --git a/cdist/test/autorequire/fixtures/conf/type/__nfsroot_client/manifest b/cdist/test/autorequire/fixtures/conf/type/__nfsroot_client/manifest deleted file mode 100755 index f6cb7833..00000000 --- a/cdist/test/autorequire/fixtures/conf/type/__nfsroot_client/manifest +++ /dev/null @@ -1,3 +0,0 @@ -__user root -__root_ssh_authorized_key john -__root_ssh_authorized_key frank diff --git a/cdist/test/autorequire/fixtures/conf/type/__package/manifest b/cdist/test/autorequire/fixtures/conf/type/__package/manifest deleted file mode 100755 index 6f70e627..00000000 --- a/cdist/test/autorequire/fixtures/conf/type/__package/manifest +++ /dev/null @@ -1 +0,0 @@ -__package_special "$__object_id" diff --git a/cdist/test/autorequire/fixtures/conf/type/__root_ssh_authorized_key/manifest b/cdist/test/autorequire/fixtures/conf/type/__root_ssh_authorized_key/manifest deleted file mode 100755 index 6224629f..00000000 --- a/cdist/test/autorequire/fixtures/conf/type/__root_ssh_authorized_key/manifest +++ /dev/null @@ -1,4 +0,0 @@ -user="$__object_id" -__directory /root/.ssh -require="__directory/root/.ssh" \ - __addifnosuchline "ssh-root-$user" diff --git a/cdist/test/autorequire/fixtures/conf/type/__top/manifest b/cdist/test/autorequire/fixtures/conf/type/__top/manifest deleted file mode 100755 index d6968c25..00000000 --- a/cdist/test/autorequire/fixtures/conf/type/__top/manifest +++ /dev/null @@ -1,2 +0,0 @@ -__package b -require="__package/b" __package a diff --git a/cdist/test/object/__init__.py b/cdist/test/cdist_object/__init__.py similarity index 81% rename from cdist/test/object/__init__.py rename to cdist/test/cdist_object/__init__.py index c4f46cd1..ffb2ba79 100644 --- a/cdist/test/object/__init__.py +++ b/cdist/test/cdist_object/__init__.py @@ -64,6 +64,16 @@ class ObjectClassTestCase(test.CdistTestCase): found_objects = list(core.CdistObject.list_objects(object_base_path, type_base_path)) self.assertEqual(found_objects, self.expected_objects) + def test_create_singleton(self): + """Check whether creating an object without id (singleton) works""" + singleton = self.expected_objects[0].object_from_name("__test_singleton") + # came here - everything fine + + def test_create_singleton_not_singleton_type(self): + """try to create an object of a type that is not a singleton + without an object id""" + with self.assertRaises(cdist.core.cdist_object.MissingObjectIdError): + self.expected_objects[0].object_from_name("__first") class ObjectIdTestCase(test.CdistTestCase): def test_object_id_contains_double_slash(self): @@ -212,54 +222,3 @@ class ObjectTestCase(test.CdistTestCase): self.assertTrue(isinstance(other_object, core.CdistObject)) self.assertEqual(other_object.cdist_type.name, '__first') self.assertEqual(other_object.object_id, 'man') - - - -class ObjectResolveRequirementsTestCase(test.CdistTestCase): - - def setUp(self): - self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path)) - self.object_index = dict((o.name, o) for o in self.objects) - self.object_names = [o.name for o in self.objects] - - print(self.objects) - - self.cdist_type = core.CdistType(type_base_path, '__third') - self.cdist_object = core.CdistObject(self.cdist_type, object_base_path, 'moon') - - def tearDown(self): - for o in self.objects: - o.requirements = [] - - def test_find_requirements_by_name_string(self): - """Check that resolving requirements by name works (require all objects)""" - requirements = self.object_names - - self.cdist_object.requirements = requirements - - found_requirements = sorted(self.cdist_object.find_requirements_by_name(self.cdist_object.requirements)) - expected_requirements = sorted(self.objects) - - self.assertEqual(found_requirements, expected_requirements) - - def test_find_requirements_by_name_pattern(self): - """Test whether pattern matching on requirements works""" - - # Matches all objects in the end - requirements = ['__first/*', '__second/*-the', '__third/moon'] - - self.cdist_object.requirements = requirements - - expected_requirements = sorted(self.objects) - found_requirements = sorted(self.cdist_object.find_requirements_by_name(self.cdist_object.requirements)) - - self.assertEqual(expected_requirements, found_requirements) - - def test_requirement_not_found(self): - """Ensure an exception is thrown for missing depedencies""" - cdist_object = self.object_index['__first/man'] - cdist_object.requirements = ['__does/not/exist'] - - with self.assertRaises(core.cdist_object.RequirementNotFoundError): - # Use list, as generator does not (yet) raise the error - list(cdist_object.find_requirements_by_name(cdist_object.requirements)) diff --git a/cdist/test/autorequire/fixtures/conf/type/__directory/.keep b/cdist/test/cdist_object/fixtures/object/__first/.keep similarity index 100% rename from cdist/test/autorequire/fixtures/conf/type/__directory/.keep rename to cdist/test/cdist_object/fixtures/object/__first/.keep diff --git a/cdist/test/autorequire/fixtures/conf/type/__package_special/.keep b/cdist/test/cdist_object/fixtures/object/__first/child/.cdist/.keep similarity index 100% rename from cdist/test/autorequire/fixtures/conf/type/__package_special/.keep rename to cdist/test/cdist_object/fixtures/object/__first/child/.cdist/.keep diff --git a/cdist/test/autorequire/fixtures/conf/type/__user/.keep b/cdist/test/cdist_object/fixtures/object/__first/dog/.cdist/.keep similarity index 100% rename from cdist/test/autorequire/fixtures/conf/type/__user/.keep rename to cdist/test/cdist_object/fixtures/object/__first/dog/.cdist/.keep diff --git a/cdist/test/config_install/fixtures/object/__first/.keep b/cdist/test/cdist_object/fixtures/object/__first/man/.cdist/.keep similarity index 100% rename from cdist/test/config_install/fixtures/object/__first/.keep rename to cdist/test/cdist_object/fixtures/object/__first/man/.cdist/.keep diff --git a/cdist/test/config_install/fixtures/object/__first/man/.cdist/.keep b/cdist/test/cdist_object/fixtures/object/__first/woman/.cdist/.keep similarity index 100% rename from cdist/test/config_install/fixtures/object/__first/man/.cdist/.keep rename to cdist/test/cdist_object/fixtures/object/__first/woman/.cdist/.keep diff --git a/cdist/test/config_install/fixtures/object/__second/.keep b/cdist/test/cdist_object/fixtures/object/__second/.keep similarity index 100% rename from cdist/test/config_install/fixtures/object/__second/.keep rename to cdist/test/cdist_object/fixtures/object/__second/.keep diff --git a/cdist/test/config_install/fixtures/object/__second/on-the/.cdist/.keep b/cdist/test/cdist_object/fixtures/object/__second/on-the/.cdist/.keep similarity index 100% rename from cdist/test/config_install/fixtures/object/__second/on-the/.cdist/.keep rename to cdist/test/cdist_object/fixtures/object/__second/on-the/.cdist/.keep diff --git a/cdist/test/config_install/fixtures/object/__third/.keep b/cdist/test/cdist_object/fixtures/object/__second/under-the/.cdist/.keep similarity index 100% rename from cdist/test/config_install/fixtures/object/__third/.keep rename to cdist/test/cdist_object/fixtures/object/__second/under-the/.cdist/.keep diff --git a/cdist/test/config_install/fixtures/object/__third/moon/.cdist/.keep b/cdist/test/cdist_object/fixtures/object/__third/.keep similarity index 100% rename from cdist/test/config_install/fixtures/object/__third/moon/.cdist/.keep rename to cdist/test/cdist_object/fixtures/object/__third/.keep diff --git a/cdist/test/config_install/fixtures/type/__first/.keep b/cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/.keep similarity index 100% rename from cdist/test/config_install/fixtures/type/__first/.keep rename to cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/.keep diff --git a/cdist/test/config_install/fixtures/object/__third/moon/.cdist/parameter/name b/cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/parameter/name similarity index 100% rename from cdist/test/config_install/fixtures/object/__third/moon/.cdist/parameter/name rename to cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/parameter/name diff --git a/cdist/test/config_install/fixtures/object/__third/moon/.cdist/parameter/planet b/cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/parameter/planet similarity index 100% rename from cdist/test/config_install/fixtures/object/__third/moon/.cdist/parameter/planet rename to cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/parameter/planet diff --git a/cdist/test/config_install/fixtures/type/__second/.keep b/cdist/test/cdist_object/fixtures/type/__first/.keep similarity index 100% rename from cdist/test/config_install/fixtures/type/__second/.keep rename to cdist/test/cdist_object/fixtures/type/__first/.keep diff --git a/cdist/test/config_install/fixtures/type/__third/.keep b/cdist/test/cdist_object/fixtures/type/__second/.keep similarity index 100% rename from cdist/test/config_install/fixtures/type/__third/.keep rename to cdist/test/cdist_object/fixtures/type/__second/.keep diff --git a/cdist/test/object/fixtures/object/__first/.keep b/cdist/test/cdist_object/fixtures/type/__test_singleton/singleton similarity index 100% rename from cdist/test/object/fixtures/object/__first/.keep rename to cdist/test/cdist_object/fixtures/type/__test_singleton/singleton diff --git a/cdist/test/object/fixtures/object/__first/child/.cdist/.keep b/cdist/test/cdist_object/fixtures/type/__third/.keep similarity index 100% rename from cdist/test/object/fixtures/object/__first/child/.cdist/.keep rename to cdist/test/cdist_object/fixtures/type/__third/.keep diff --git a/cdist/test/type/__init__.py b/cdist/test/cdist_type/__init__.py similarity index 94% rename from cdist/test/type/__init__.py rename to cdist/test/cdist_type/__init__.py index 5e774aa9..79f824d3 100644 --- a/cdist/test/type/__init__.py +++ b/cdist/test/cdist_type/__init__.py @@ -106,16 +106,6 @@ class TypeTestCase(test.CdistTestCase): cdist_type = core.CdistType(base_path, '__not_singleton') self.assertFalse(cdist_type.is_singleton) - def test_install_is_install(self): - base_path = fixtures - cdist_type = core.CdistType(base_path, '__install') - self.assertTrue(cdist_type.is_install) - - def test_not_install_is_install(self): - base_path = fixtures - cdist_type = core.CdistType(base_path, '__not_install') - self.assertFalse(cdist_type.is_install) - def test_with_explorers(self): base_path = fixtures cdist_type = core.CdistType(base_path, '__with_explorers') @@ -156,3 +146,10 @@ class TypeTestCase(test.CdistTestCase): cdist_type = core.CdistType(base_path, '__without_boolean_parameters') self.assertEqual(cdist_type.boolean_parameters, []) + def test_with_parameter_defaults(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, '__with_parameter_defaults') + self.assertTrue('optional1' in cdist_type.parameter_defaults) + self.assertFalse('optional2' in cdist_type.parameter_defaults) + self.assertEqual(cdist_type.parameter_defaults['optional1'], 'value1') + diff --git a/cdist/test/type/fixtures/__install/install b/cdist/test/cdist_type/fixtures/__install/install similarity index 100% rename from cdist/test/type/fixtures/__install/install rename to cdist/test/cdist_type/fixtures/__install/install diff --git a/cdist/test/object/fixtures/object/__first/dog/.cdist/.keep b/cdist/test/cdist_type/fixtures/__name_path/.keep similarity index 100% rename from cdist/test/object/fixtures/object/__first/dog/.cdist/.keep rename to cdist/test/cdist_type/fixtures/__name_path/.keep diff --git a/cdist/test/object/fixtures/object/__first/man/.cdist/.keep b/cdist/test/cdist_type/fixtures/__not_install/.keep similarity index 100% rename from cdist/test/object/fixtures/object/__first/man/.cdist/.keep rename to cdist/test/cdist_type/fixtures/__not_install/.keep diff --git a/cdist/test/object/fixtures/object/__first/woman/.cdist/.keep b/cdist/test/cdist_type/fixtures/__not_singleton/.keep similarity index 100% rename from cdist/test/object/fixtures/object/__first/woman/.cdist/.keep rename to cdist/test/cdist_type/fixtures/__not_singleton/.keep diff --git a/cdist/test/object/fixtures/object/__second/.keep b/cdist/test/cdist_type/fixtures/__singleton/singleton similarity index 100% rename from cdist/test/object/fixtures/object/__second/.keep rename to cdist/test/cdist_type/fixtures/__singleton/singleton diff --git a/cdist/test/type/fixtures/__with_boolean_parameters/parameter/boolean b/cdist/test/cdist_type/fixtures/__with_boolean_parameters/parameter/boolean similarity index 100% rename from cdist/test/type/fixtures/__with_boolean_parameters/parameter/boolean rename to cdist/test/cdist_type/fixtures/__with_boolean_parameters/parameter/boolean diff --git a/cdist/test/type/fixtures/__with_explorers/explorer/whatever b/cdist/test/cdist_type/fixtures/__with_explorers/explorer/whatever similarity index 100% rename from cdist/test/type/fixtures/__with_explorers/explorer/whatever rename to cdist/test/cdist_type/fixtures/__with_explorers/explorer/whatever diff --git a/cdist/test/type/fixtures/__with_optional_parameters/parameter/optional b/cdist/test/cdist_type/fixtures/__with_optional_parameters/parameter/optional similarity index 100% rename from cdist/test/type/fixtures/__with_optional_parameters/parameter/optional rename to cdist/test/cdist_type/fixtures/__with_optional_parameters/parameter/optional diff --git a/cdist/test/cdist_type/fixtures/__with_parameter_defaults/parameter/default/optional1 b/cdist/test/cdist_type/fixtures/__with_parameter_defaults/parameter/default/optional1 new file mode 100644 index 00000000..ef208405 --- /dev/null +++ b/cdist/test/cdist_type/fixtures/__with_parameter_defaults/parameter/default/optional1 @@ -0,0 +1 @@ +value1 diff --git a/cdist/test/cdist_type/fixtures/__with_parameter_defaults/parameter/optional b/cdist/test/cdist_type/fixtures/__with_parameter_defaults/parameter/optional new file mode 100644 index 00000000..8174d2a9 --- /dev/null +++ b/cdist/test/cdist_type/fixtures/__with_parameter_defaults/parameter/optional @@ -0,0 +1,2 @@ +optional1 +optional2 diff --git a/cdist/test/type/fixtures/__with_required_parameters/parameter/required b/cdist/test/cdist_type/fixtures/__with_required_parameters/parameter/required similarity index 100% rename from cdist/test/type/fixtures/__with_required_parameters/parameter/required rename to cdist/test/cdist_type/fixtures/__with_required_parameters/parameter/required diff --git a/cdist/test/object/fixtures/object/__second/on-the/.cdist/.keep b/cdist/test/cdist_type/fixtures/__without_boolean_parameters/.keep similarity index 100% rename from cdist/test/object/fixtures/object/__second/on-the/.cdist/.keep rename to cdist/test/cdist_type/fixtures/__without_boolean_parameters/.keep diff --git a/cdist/test/object/fixtures/object/__second/under-the/.cdist/.keep b/cdist/test/cdist_type/fixtures/__without_explorers/.keep similarity index 100% rename from cdist/test/object/fixtures/object/__second/under-the/.cdist/.keep rename to cdist/test/cdist_type/fixtures/__without_explorers/.keep diff --git a/cdist/test/object/fixtures/object/__third/.keep b/cdist/test/cdist_type/fixtures/__without_optional_parameters/.keep similarity index 100% rename from cdist/test/object/fixtures/object/__third/.keep rename to cdist/test/cdist_type/fixtures/__without_optional_parameters/.keep diff --git a/cdist/test/object/fixtures/object/__third/moon/.cdist/.keep b/cdist/test/cdist_type/fixtures/__without_required_parameters/.keep similarity index 100% rename from cdist/test/object/fixtures/object/__third/moon/.cdist/.keep rename to cdist/test/cdist_type/fixtures/__without_required_parameters/.keep diff --git a/cdist/test/object/fixtures/type/__first/.keep b/cdist/test/cdist_type/fixtures/list_types/__first/.keep similarity index 100% rename from cdist/test/object/fixtures/type/__first/.keep rename to cdist/test/cdist_type/fixtures/list_types/__first/.keep diff --git a/cdist/test/object/fixtures/type/__second/.keep b/cdist/test/cdist_type/fixtures/list_types/__second/.keep similarity index 100% rename from cdist/test/object/fixtures/type/__second/.keep rename to cdist/test/cdist_type/fixtures/list_types/__second/.keep diff --git a/cdist/test/object/fixtures/type/__third/.keep b/cdist/test/cdist_type/fixtures/list_types/__third/.keep similarity index 100% rename from cdist/test/object/fixtures/type/__third/.keep rename to cdist/test/cdist_type/fixtures/list_types/__third/.keep diff --git a/cdist/test/code/__init__.py b/cdist/test/code/__init__.py index 284ef9c0..796e8a51 100644 --- a/cdist/test/code/__init__.py +++ b/cdist/test/code/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -39,21 +39,23 @@ conf_dir = op.join(fixtures, 'conf') class CodeTestCase(test.CdistTestCase): def setUp(self): - self.target_host = 'localhost' - - self.out_path = self.mkdtemp() + self.local_dir = self.mkdtemp() self.local = local.Local( target_host=self.target_host, - out_path = self.out_path, + base_path = self.local_dir, exec_path = cdist.test.cdist_exec_path, add_conf_dirs=[conf_dir]) self.local.create_files_dirs() - self.remote_base_path = self.mkdtemp() + self.remote_dir = self.mkdtemp() remote_exec = self.remote_exec remote_copy = self.remote_copy - self.remote = remote.Remote(self.target_host, self.remote_base_path, remote_exec, remote_copy) + self.remote = remote.Remote( + target_host=self.target_host, + remote_exec=remote_exec, + remote_copy=remote_copy, + base_path=self.remote_dir) self.remote.create_files_dirs() self.code = code.Code(self.target_host, self.local, self.remote) @@ -63,8 +65,8 @@ class CodeTestCase(test.CdistTestCase): self.cdist_object.create() def tearDown(self): - shutil.rmtree(self.out_path) - shutil.rmtree(self.remote_base_path) + shutil.rmtree(self.local_dir) + shutil.rmtree(self.remote_dir) def test_run_gencode_local_environment(self): output_string = self.code.run_gencode_local(self.cdist_object) @@ -75,7 +77,7 @@ class CodeTestCase(test.CdistTestCase): key = junk.split(' ')[1] output_dict[key] = value self.assertEqual(output_dict['__target_host'], self.local.target_host) - self.assertEqual(output_dict['__global'], self.local.out_path) + self.assertEqual(output_dict['__global'], self.local.base_path) self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path) self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path) self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id) @@ -90,7 +92,7 @@ class CodeTestCase(test.CdistTestCase): key = junk.split(' ')[1] output_dict[key] = value self.assertEqual(output_dict['__target_host'], self.local.target_host) - self.assertEqual(output_dict['__global'], self.local.out_path) + self.assertEqual(output_dict['__global'], self.local.base_path) self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path) self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path) self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id) diff --git a/cdist/test/config_install/__init__.py b/cdist/test/config/__init__.py similarity index 53% rename from cdist/test/config_install/__init__.py rename to cdist/test/config/__init__.py index 2abf7614..80a45d9b 100644 --- a/cdist/test/config_install/__init__.py +++ b/cdist/test/config/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -27,8 +27,9 @@ from cdist import test from cdist import core import cdist -import cdist.context import cdist.config +import cdist.core.cdist_type +import cdist.core.cdist_object import os.path as op my_dir = op.abspath(op.dirname(__file__)) @@ -37,7 +38,7 @@ object_base_path = op.join(fixtures, 'object') type_base_path = op.join(fixtures, 'type') add_conf_dir = op.join(fixtures, 'conf') -class ConfigInstallRunTestCase(test.CdistTestCase): +class ConfigRunTestCase(test.CdistTestCase): def setUp(self): @@ -46,23 +47,24 @@ class ConfigInstallRunTestCase(test.CdistTestCase): os.environ = os.environ.copy() self.temp_dir = self.mkdtemp() - self.out_dir = os.path.join(self.temp_dir, "out") - self.remote_out_dir = os.path.join(self.temp_dir, "remote") + self.local_dir = os.path.join(self.temp_dir, "local") + os.mkdir(self.local_dir) + self.local = cdist.exec.local.Local( + target_host=self.target_host, + base_path=self.local_dir) - os.environ['__cdist_out_dir'] = self.out_dir - os.environ['__cdist_remote_out_dir'] = self.remote_out_dir - - self.context = cdist.context.Context( + self.remote_dir = os.path.join(self.temp_dir, "remote") + os.mkdir(self.remote_dir) + self.remote = cdist.exec.remote.Remote( target_host=self.target_host, remote_copy=self.remote_copy, remote_exec=self.remote_exec, - exec_path=test.cdist_exec_path, - debug=True) + base_path=self.remote_dir) - self.context.local.object_path = object_base_path - self.context.local.type_path = type_base_path + self.local.object_path = object_base_path + self.local.type_path = type_base_path - self.config = cdist.config.Config(self.context) + self.config = cdist.config.Config(self.local, self.remote) self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path)) self.object_index = dict((o.name, o) for o in self.objects) @@ -86,15 +88,15 @@ class ConfigInstallRunTestCase(test.CdistTestCase): # First run: # solves first and maybe second (depending on the order in the set) - self.config.stage_run_iterate() + self.config.iterate_once() self.assertTrue(third.state == third.STATE_DONE) - self.config.stage_run_iterate() + self.config.iterate_once() self.assertTrue(second.state == second.STATE_DONE) try: - self.config.stage_run_iterate() + self.config.iterate_once() except cdist.Error: # Allow failing, because the third run may or may not be unecessary already, # depending on the order of the objects @@ -111,9 +113,37 @@ class ConfigInstallRunTestCase(test.CdistTestCase): first.requirements = [second.name] second.requirements = [first.name] - # First round solves __third/moon - self.config.stage_run_iterate() + with self.assertRaises(cdist.UnresolvableRequirementsError): + self.config.iterate_until_finished() - # Second round detects it cannot solve the rest - with self.assertRaises(cdist.Error): - self.config.stage_run_iterate() + def test_missing_requirements(self): + """Throw an error if requiring something non-existing""" + first = self.object_index['__first/man'] + first.requirements = ['__first/not/exist'] + with self.assertRaises(cdist.UnresolvableRequirementsError): + self.config.iterate_until_finished() + + def test_requirement_broken_type(self): + """Unknown type should be detected in the resolving process""" + first = self.object_index['__first/man'] + first.requirements = ['__nosuchtype/not/exist'] + with self.assertRaises(cdist.core.cdist_type.NoSuchTypeError): + self.config.iterate_until_finished() + + def test_requirement_singleton_where_no_singleton(self): + """Missing object id should be detected in the resolving process""" + first = self.object_index['__first/man'] + first.requirements = ['__first'] + with self.assertRaises(cdist.core.cdist_object.MissingObjectIdError): + self.config.iterate_until_finished() + +# Currently the resolving code will simply detect that this object does +# not exist. It should probably check if the type is a singleton as well +# - but maybe only in the emulator - to be discussed. +# +# def test_requirement_no_singleton_where_singleton(self): +# """Missing object id should be detected in the resolving process""" +# first = self.object_index['__first/man'] +# first.requirements = ['__singleton_test/foo'] +# with self.assertRaises(cdist.core.?????): +# self.config.iterate_until_finished() diff --git a/cdist/test/resolver/fixtures/object/__first/.keep b/cdist/test/config/fixtures/object/__first/.keep similarity index 100% rename from cdist/test/resolver/fixtures/object/__first/.keep rename to cdist/test/config/fixtures/object/__first/.keep diff --git a/cdist/test/resolver/fixtures/object/__first/child/.cdist/.keep b/cdist/test/config/fixtures/object/__first/man/.cdist/.keep similarity index 100% rename from cdist/test/resolver/fixtures/object/__first/child/.cdist/.keep rename to cdist/test/config/fixtures/object/__first/man/.cdist/.keep diff --git a/cdist/test/resolver/fixtures/object/__first/dog/.cdist/.keep b/cdist/test/config/fixtures/object/__second/.keep similarity index 100% rename from cdist/test/resolver/fixtures/object/__first/dog/.cdist/.keep rename to cdist/test/config/fixtures/object/__second/.keep diff --git a/cdist/test/resolver/fixtures/object/__first/man/.cdist/.keep b/cdist/test/config/fixtures/object/__second/on-the/.cdist/.keep similarity index 100% rename from cdist/test/resolver/fixtures/object/__first/man/.cdist/.keep rename to cdist/test/config/fixtures/object/__second/on-the/.cdist/.keep diff --git a/cdist/test/resolver/fixtures/object/__first/woman/.cdist/.keep b/cdist/test/config/fixtures/object/__third/.keep similarity index 100% rename from cdist/test/resolver/fixtures/object/__first/woman/.cdist/.keep rename to cdist/test/config/fixtures/object/__third/.keep diff --git a/cdist/test/resolver/fixtures/object/__second/.keep b/cdist/test/config/fixtures/object/__third/moon/.cdist/.keep similarity index 100% rename from cdist/test/resolver/fixtures/object/__second/.keep rename to cdist/test/config/fixtures/object/__third/moon/.cdist/.keep diff --git a/cdist/test/object/fixtures/object/__third/moon/.cdist/parameter/name b/cdist/test/config/fixtures/object/__third/moon/.cdist/parameter/name similarity index 100% rename from cdist/test/object/fixtures/object/__third/moon/.cdist/parameter/name rename to cdist/test/config/fixtures/object/__third/moon/.cdist/parameter/name diff --git a/cdist/test/object/fixtures/object/__third/moon/.cdist/parameter/planet b/cdist/test/config/fixtures/object/__third/moon/.cdist/parameter/planet similarity index 100% rename from cdist/test/object/fixtures/object/__third/moon/.cdist/parameter/planet rename to cdist/test/config/fixtures/object/__third/moon/.cdist/parameter/planet diff --git a/cdist/test/resolver/fixtures/object/__second/on-the/.cdist/.keep b/cdist/test/config/fixtures/type/__first/.keep similarity index 100% rename from cdist/test/resolver/fixtures/object/__second/on-the/.cdist/.keep rename to cdist/test/config/fixtures/type/__first/.keep diff --git a/cdist/test/resolver/fixtures/object/__second/under-the/.cdist/.keep b/cdist/test/config/fixtures/type/__second/.keep similarity index 100% rename from cdist/test/resolver/fixtures/object/__second/under-the/.cdist/.keep rename to cdist/test/config/fixtures/type/__second/.keep diff --git a/cdist/test/resolver/fixtures/object/__third/.keep b/cdist/test/config/fixtures/type/__singleton_test/singleton similarity index 100% rename from cdist/test/resolver/fixtures/object/__third/.keep rename to cdist/test/config/fixtures/type/__singleton_test/singleton diff --git a/cdist/test/resolver/fixtures/object/__third/moon/.cdist/.keep b/cdist/test/config/fixtures/type/__third/.keep similarity index 100% rename from cdist/test/resolver/fixtures/object/__third/moon/.cdist/.keep rename to cdist/test/config/fixtures/type/__third/.keep diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index fc0b6695..8c2dcd72 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -33,7 +33,6 @@ from cdist.exec import local from cdist import emulator from cdist import core from cdist import config -import cdist.context import os.path as op my_dir = op.abspath(op.dirname(__file__)) @@ -46,11 +45,11 @@ class EmulatorTestCase(test.CdistTestCase): self.temp_dir = self.mkdtemp() handle, self.script = self.mkstemp(dir=self.temp_dir) os.close(handle) - out_path = self.temp_dir + base_path = self.temp_dir self.local = local.Local( target_host=self.target_host, - out_path=out_path, + base_path=base_path, exec_path=test.cdist_exec_path, add_conf_dirs=[conf_dir]) self.local.create_files_dirs() @@ -63,13 +62,13 @@ class EmulatorTestCase(test.CdistTestCase): def test_nonexistent_type_exec(self): argv = ['__does-not-exist'] - self.assertRaises(core.NoSuchTypeError, emulator.Emulator, argv, env=self.env) + self.assertRaises(core.cdist_type.NoSuchTypeError, emulator.Emulator, argv, env=self.env) def test_nonexistent_type_requirement(self): argv = ['__file', '/tmp/foobar'] self.env['require'] = '__does-not-exist/some-id' emu = emulator.Emulator(argv, env=self.env) - self.assertRaises(core.NoSuchTypeError, emu.run) + self.assertRaises(core.cdist_type.NoSuchTypeError, emu.run) def test_illegal_object_id_requirement(self): argv = ['__file', '/tmp/foobar'] @@ -81,7 +80,14 @@ class EmulatorTestCase(test.CdistTestCase): argv = ['__file', '/tmp/foobar'] self.env['require'] = '__file' emu = emulator.Emulator(argv, env=self.env) - self.assertRaises(core.IllegalObjectIdError, emu.run) + self.assertRaises(core.cdist_object.MissingObjectIdError, emu.run) + + def test_no_singleton_no_requirement(self): + argv = ['__file', '/tmp/foobar'] + self.env['require'] = '__test_singleton' + emu = emulator.Emulator(argv, env=self.env) + emu.run() + # If reached here, everything is fine def test_singleton_object_requirement(self): argv = ['__file', '/tmp/foobar'] @@ -101,11 +107,11 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase): def setUp(self): self.temp_dir = self.mkdtemp() - out_path = os.path.join(self.temp_dir, "out") + base_path = os.path.join(self.temp_dir, "out") self.local = local.Local( target_host=self.target_host, - out_path=out_path, + base_path=base_path, exec_path=test.cdist_exec_path, add_conf_dirs=[conf_dir]) self.local.create_files_dirs() @@ -118,7 +124,7 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase): initial_manifest = os.path.join(self.local.manifest_path, "init") self.manifest.run_initial_manifest(initial_manifest) cdist_type = core.CdistType(self.local.type_path, '__saturn') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'singleton') + cdist_object = core.CdistObject(cdist_type, self.local.object_path) self.manifest.run_type_manifest(cdist_object) expected = ['__planet/Saturn', '__moon/Prometheus'] self.assertEqual(sorted(cdist_object.autorequire), sorted(expected)) @@ -128,13 +134,13 @@ class ArgumentsTestCase(test.CdistTestCase): def setUp(self): self.temp_dir = self.mkdtemp() - out_path = self.temp_dir + base_path = self.temp_dir handle, self.script = self.mkstemp(dir=self.temp_dir) os.close(handle) self.local = local.Local( target_host=self.target_host, - out_path=out_path, + base_path=base_path, exec_path=test.cdist_exec_path, add_conf_dirs=[conf_dir]) self.local.create_files_dirs() @@ -170,11 +176,13 @@ class ArgumentsTestCase(test.CdistTestCase): # empty file -> True self.assertTrue(cdist_object.parameters['boolean1'] == '') - def test_required(self): + def test_required_arguments(self): + """check whether assigning required parameter works""" type_name = '__arguments_required' object_id = 'some-id' value = 'some value' argv = [type_name, object_id, '--required1', value, '--required2', value] + print(self.env) os.environ.update(self.env) emu = emulator.Emulator(argv) emu.run() @@ -211,6 +219,21 @@ class ArgumentsTestCase(test.CdistTestCase): self.assertFalse('optional2' in cdist_object.parameters) self.assertEqual(cdist_object.parameters['optional1'], value) + def test_argument_defaults(self): + type_name = '__argument_defaults' + object_id = 'some-id' + value = 'value1' + argv = [type_name, object_id] + os.environ.update(self.env) + emu = emulator.Emulator(argv) + emu.run() + + cdist_type = core.CdistType(self.local.type_path, type_name) + cdist_object = core.CdistObject(cdist_type, self.local.object_path, object_id) + self.assertTrue('optional1' in cdist_object.parameters) + self.assertFalse('optional2' in cdist_object.parameters) + self.assertEqual(cdist_object.parameters['optional1'], value) + class StdinTestCase(test.CdistTestCase): @@ -219,11 +242,11 @@ class StdinTestCase(test.CdistTestCase): os.environ = os.environ.copy() self.temp_dir = self.mkdtemp() - out_path = os.path.join(self.temp_dir, "out") + base_path = os.path.join(self.temp_dir, "out") self.local = local.Local( target_host=self.target_host, - out_path=out_path, + base_path=base_path, exec_path=test.cdist_exec_path, add_conf_dirs=[conf_dir]) diff --git a/cdist/test/emulator/fixtures/conf/type/__argument_defaults/parameter/default/optional1 b/cdist/test/emulator/fixtures/conf/type/__argument_defaults/parameter/default/optional1 new file mode 100644 index 00000000..ef208405 --- /dev/null +++ b/cdist/test/emulator/fixtures/conf/type/__argument_defaults/parameter/default/optional1 @@ -0,0 +1 @@ +value1 diff --git a/cdist/test/emulator/fixtures/conf/type/__argument_defaults/parameter/optional b/cdist/test/emulator/fixtures/conf/type/__argument_defaults/parameter/optional new file mode 100644 index 00000000..8174d2a9 --- /dev/null +++ b/cdist/test/emulator/fixtures/conf/type/__argument_defaults/parameter/optional @@ -0,0 +1,2 @@ +optional1 +optional2 diff --git a/cdist/test/resolver/fixtures/type/__first/.keep b/cdist/test/emulator/fixtures/conf/type/__test_singleton/singleton similarity index 100% rename from cdist/test/resolver/fixtures/type/__first/.keep rename to cdist/test/emulator/fixtures/conf/type/__test_singleton/singleton diff --git a/cdist/test/explorer/__init__.py b/cdist/test/explorer/__init__.py index a97b538a..92ef75a3 100644 --- a/cdist/test/explorer/__init__.py +++ b/cdist/test/explorer/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -39,26 +39,25 @@ conf_dir = op.join(fixtures, "conf") class ExplorerClassTestCase(test.CdistTestCase): def setUp(self): - self.target_host = 'localhost' - - self.temp_dir = self.mkdtemp() - self.out_path = os.path.join(self.temp_dir, "out") - self.remote_base_path = os.path.join(self.temp_dir, "remote") + self.temp_dir = self.mkdtemp() + self.local_path = os.path.join(self.temp_dir, "local") + self.remote_base_path = os.path.join(self.temp_dir, "remote") os.makedirs(self.remote_base_path) self.local = local.Local( target_host=self.target_host, - out_path=self.out_path, + base_path=self.local_path, exec_path=test.cdist_exec_path, - add_conf_dirs=[conf_dir]) + add_conf_dirs=[conf_dir], + ) self.local.create_files_dirs() self.remote = remote.Remote( - self.target_host, - self.remote_base_path, - self.remote_exec, - self.remote_copy) + target_host=self.target_host, + remote_exec=self.remote_exec, + remote_copy=self.remote_copy, + base_path=self.remote_base_path) self.remote.create_files_dirs() self.explorer = explorer.Explorer( @@ -70,11 +69,13 @@ class ExplorerClassTestCase(test.CdistTestCase): shutil.rmtree(self.temp_dir) def test_list_global_explorer_names(self): + """Ensure that all explorers are listed""" names = self.explorer.list_global_explorer_names() self.assertIn("foobar", names) self.assertIn("global", names) def test_transfer_global_explorers(self): + """Ensure transferring explorers to remote works""" self.explorer.transfer_global_explorers() source = self.local.global_explorer_path destination = self.remote.global_explorer_path @@ -86,7 +87,7 @@ class ExplorerClassTestCase(test.CdistTestCase): output = self.explorer.run_global_explorer('global') self.assertEqual(output, 'global\n') - def test_run_global_explorers(self): + def test_global_explorer_output(self): """Ensure output is created for every global explorer""" out_path = self.mkdtemp() @@ -103,6 +104,7 @@ class ExplorerClassTestCase(test.CdistTestCase): self.assertEqual(self.explorer.list_type_explorer_names(cdist_type), expected) def test_transfer_type_explorers(self): + """Test if transferring type explorers works""" cdist_type = core.CdistType(self.local.type_path, '__test_type') self.explorer.transfer_type_explorers(cdist_type) source = os.path.join(self.local.type_path, cdist_type.explorer_path) diff --git a/cdist/test/fixtures/remote b/cdist/test/fixtures/remote deleted file mode 120000 index c5db6358..00000000 --- a/cdist/test/fixtures/remote +++ /dev/null @@ -1 +0,0 @@ -../../../other/examples/remote/local \ No newline at end of file diff --git a/cdist/test/fixtures/remote/README b/cdist/test/fixtures/remote/README new file mode 100644 index 00000000..cfd350f9 --- /dev/null +++ b/cdist/test/fixtures/remote/README @@ -0,0 +1,3 @@ +This effectively turns remote calling into local calling. + +Probably most useful for the unittesting. diff --git a/cdist/test/fixtures/remote/copy b/cdist/test/fixtures/remote/copy new file mode 100755 index 00000000..a4627716 --- /dev/null +++ b/cdist/test/fixtures/remote/copy @@ -0,0 +1,23 @@ +#!/bin/sh +# +# 2012-2013 Nico Schottelius (nico-cdist schottelius.org) +# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +code="$(echo "$@" | sed "s|\([[:space:]]\)$__target_host:|\1|g")" +cp --dereference $code diff --git a/cdist/test/fixtures/remote/exec b/cdist/test/fixtures/remote/exec new file mode 100755 index 00000000..838513a9 --- /dev/null +++ b/cdist/test/fixtures/remote/exec @@ -0,0 +1,23 @@ +#!/bin/sh +# +# 2012 Nico Schottelius (nico-cdist schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +target_host=$1; shift +echo "$@" | /bin/sh diff --git a/cdist/test/manifest/__init__.py b/cdist/test/manifest/__init__.py index 727017e7..c375a80f 100644 --- a/cdist/test/manifest/__init__.py +++ b/cdist/test/manifest/__init__.py @@ -46,11 +46,11 @@ class ManifestTestCase(test.CdistTestCase): self.orig_environ = os.environ os.environ = os.environ.copy() self.temp_dir = self.mkdtemp() - self.target_host = 'localhost' + out_path = self.temp_dir self.local = local.Local( target_host=self.target_host, - out_path=out_path, + base_path=out_path, exec_path = cdist.test.cdist_exec_path, add_conf_dirs=[conf_dir]) self.local.create_files_dirs() @@ -78,7 +78,7 @@ class ManifestTestCase(test.CdistTestCase): output_dict[key] = value self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path)) self.assertEqual(output_dict['__target_host'], self.local.target_host) - self.assertEqual(output_dict['__global'], self.local.out_path) + self.assertEqual(output_dict['__global'], self.local.base_path) self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_path) self.assertEqual(output_dict['__manifest'], self.local.manifest_path) @@ -99,7 +99,7 @@ class ManifestTestCase(test.CdistTestCase): output_dict[key] = value self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path)) self.assertEqual(output_dict['__target_host'], self.local.target_host) - self.assertEqual(output_dict['__global'], self.local.out_path) + self.assertEqual(output_dict['__global'], self.local.base_path) self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_path) self.assertEqual(output_dict['__type'], cdist_type.absolute_path) self.assertEqual(output_dict['__object'], cdist_object.absolute_path) diff --git a/cdist/test/resolver/fixtures/object/__third/moon/.cdist/parameter/name b/cdist/test/resolver/fixtures/object/__third/moon/.cdist/parameter/name deleted file mode 100644 index 4129a761..00000000 --- a/cdist/test/resolver/fixtures/object/__third/moon/.cdist/parameter/name +++ /dev/null @@ -1 +0,0 @@ -Prometheus diff --git a/cdist/test/resolver/fixtures/object/__third/moon/.cdist/parameter/planet b/cdist/test/resolver/fixtures/object/__third/moon/.cdist/parameter/planet deleted file mode 100644 index 8e6ee422..00000000 --- a/cdist/test/resolver/fixtures/object/__third/moon/.cdist/parameter/planet +++ /dev/null @@ -1 +0,0 @@ -Saturn diff --git a/cdist/test/resolver/fixtures/type/__second/.keep b/cdist/test/resolver/fixtures/type/__second/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/resolver/fixtures/type/__third/.keep b/cdist/test/resolver/fixtures/type/__third/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/type/fixtures/__name_path/.keep b/cdist/test/type/fixtures/__name_path/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/type/fixtures/__not_install/.keep b/cdist/test/type/fixtures/__not_install/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/type/fixtures/__not_singleton/.keep b/cdist/test/type/fixtures/__not_singleton/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/type/fixtures/__without_boolean_parameters/.keep b/cdist/test/type/fixtures/__without_boolean_parameters/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/type/fixtures/__without_explorers/.keep b/cdist/test/type/fixtures/__without_explorers/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/type/fixtures/__without_optional_parameters/.keep b/cdist/test/type/fixtures/__without_optional_parameters/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/type/fixtures/__without_required_parameters/.keep b/cdist/test/type/fixtures/__without_required_parameters/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/type/fixtures/list_types/__first/.keep b/cdist/test/type/fixtures/list_types/__first/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/type/fixtures/list_types/__second/.keep b/cdist/test/type/fixtures/list_types/__second/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/type/fixtures/list_types/__third/.keep b/cdist/test/type/fixtures/list_types/__third/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/changelog b/docs/changelog index 12baa1a2..e7d90609 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,9 +4,84 @@ Changelog * Changes are always commented with their author in (braces) * Exception: No braces means author == Nico Schottelius -next: - * Type __jail: State absent should implies stopped (Jake Guffey) +2.3.6: 2013-11-25 + * New Type: __locale + * Type __line: Ensure special characters are not interpreted + +2.3.5: 2013-10-10 + * Core: Unit test fix for remote_copy (Steven Armstrong) + * Documentation: Updated manpages of __package and __file (Alex Greif) + * Documentation: Add more examples to cdist-manifest (Dan Levin) + * Type __package_apt: Do not install recommends by default + +2.3.4: 2013-10-03 + * Core: Add missing bits to support dry run (Steven Armstrong) + * Core: Make unit test remote copy more compatible with scp (Steven Armstrong) + * New Type: __postfix (Steven Armstrong) + * New Type: __postfix_master (Steven Armstrong) + * New Type: __postfix_postconf (Steven Armstrong) + * New Type: __postfix_postmap (Steven Armstrong) + * New Type: __postfix_reload (Steven Armstrong) + * Type __line: Ensure regex does not contain / + * Type __ssh_authorized_keys: Bugfix: Preserve ownership (Steven Armstrong) + +2.3.3: 2013-09-09 + * Core: Add support for default values of optional parameters (Steven Armstrong) + * Type __start_on_boot: Bugfix for systemd (Steven Armstrong) + +2.3.2: 2013-09-05 + * Build: Ensure tests don't change attributes of non-test files + * Core: Fix typo in argument parser + * Core: Code cleanup: Remove old install code (Steven Armstrong) + * Core: Improve error message when using non-existing type in requirement + * New Type: __iptables_rule + * New Type: __iptables_apply + * Type __cdist: Also create home directory + * Type __cdist: Add support for --shell parameter + * Type __motd: Regenerate motd on Debian and Ubuntu + +2.3.1: 2013-08-28 + * Core: Support relative paths for configuration directories + * Core: Code cleanup (removed context class, added log class) + * Documentation: Add more best practises + * Documentation: Add troubleshooting chapter + * Type __key_value: Fix quoting problem (Steven Armstrong) + +2.3.0: 2013-08-12 + * Core: Added support for cdist shell + * Documentation: Improved some manpages + +2.2.0: 2013-07-12 + * Build: Cleanup the Makefile + * Type __package_opkg: Use shortcut version + * Core: Remove old pseudo object id "singleton" (Steven Armstrong) + +2.1.2: 2013-07-09 + * Build: Change clean-dist target to "distclean" + * Build: Moved a lot of build logic into Makefile for dependency resolution + * Core: Make global explorers available to initial manifest (Arkaitz Jimenez) + * Core: Change execution order to run object as one unit + * Documentation: Improved documentation (Tomáš Pospíšek) + * New Remote Example: Add support for sudo operations (Chase James) + * New Type: __update_alternatives + * New Type: __cdist + * Type __apt_ppa: Fix comparison operator (Tyler Akins) + * Type __start_on_boot: Archlinux changed to use systemd - adapt type + * Type __git: Missing quotes added (Chase James) + * Type __postgres_database: Make state parameter optional (Chase James) + * Type __postgres_role: Make state parameter optional, fix password bug (Chase James) + * Type __process: Make state parameter optional + * Type __cron: Simplyfied and syntax change + +2.1.1: 2013-04-08 * Core: Use dynamic dependency resolver to allow indirect self dependencies + * Core: Remove umask call - protect /var/lib/cdist only (Arkaitz Jimenez) + * Explorer os: Added Slackware support (Eivind Uggedal) + * Type __git: Support mode and fix owner/group settings (contradict) + * Type __jail: State absent should implies stopped (Jake Guffey) + * Type __directory: Make stat call compatible with FreeBSD (Jake Guffey) + * Type __cron: Allow crontab without entries (Arkaitz Jimenez) + * Type __user: Add support for creating user home (Arkaitz Jimenez) 2.1.0: 2012-12-09 * Core: Ensure global explorers are executable diff --git a/docs/changelog.future b/docs/changelog.future new file mode 100644 index 00000000..12adf8c3 --- /dev/null +++ b/docs/changelog.future @@ -0,0 +1,10 @@ +Changelog +--------- + + * Changes are always commented with their author in (braces) + * Exception: No braces means author == Nico Schottelius + +future (maybe 3.x?): + * Type __cron: Dropped support for old internal format + Using this version prior to running cdist 2.1.2 will + break add the cron entries twice. diff --git a/docs/dev/logs/2012-09-03.dep-ideas.xoj b/docs/dev/logs/2012-09-03.dep-ideas.xoj new file mode 100644 index 00000000..b2ab927e Binary files /dev/null and b/docs/dev/logs/2012-09-03.dep-ideas.xoj differ diff --git a/docs/dev/logs/2012-11-02.cdist-vs-centralised-development.xoj b/docs/dev/logs/2012-11-02.cdist-vs-centralised-development.xoj new file mode 100644 index 00000000..56303893 Binary files /dev/null and b/docs/dev/logs/2012-11-02.cdist-vs-centralised-development.xoj differ diff --git a/docs/dev/logs/2012-11-15.cdist-sexy-interaction.svg b/docs/dev/logs/2012-11-15.cdist-sexy-interaction.svg new file mode 100644 index 00000000..788de83c --- /dev/null +++ b/docs/dev/logs/2012-11-15.cdist-sexy-interaction.svg @@ -0,0 +1,282 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + cdist + + + + configures hosts + + + + + sexy + + + + + + manages inventory + + + + + + installs hosts(missing) + + + + interact + + + + + + + + visualises inventory(missing) + + + + + + diff --git a/docs/dev/logs/2013-01-03.dependencies-visualised.xoj b/docs/dev/logs/2013-01-03.dependencies-visualised.xoj new file mode 100644 index 00000000..715d2583 Binary files /dev/null and b/docs/dev/logs/2013-01-03.dependencies-visualised.xoj differ diff --git a/docs/dev/logs/2013-02-05.debugging-wrong-singleton-type-parameter b/docs/dev/logs/2013-02-05.debugging-wrong-singleton-type-parameter new file mode 100644 index 00000000..5169b678 --- /dev/null +++ b/docs/dev/logs/2013-02-05.debugging-wrong-singleton-type-parameter @@ -0,0 +1,34 @@ +Traceback (most recent call last): + File "/home/users/nico/p/cdist/cdist/bin/../scripts/cdist", line 230, in + commandline() + File "/home/users/nico/p/cdist/cdist/bin/../scripts/cdist", line 104, in commandline + args.func(args) + File "/home/users/nico/p/cdist/cdist/bin/../scripts/cdist", line 107, in config + configinstall(args, mode=cdist.config.Config) + File "/home/users/nico/p/cdist/cdist/bin/../scripts/cdist", line 143, in configinstall + configinstall_onehost(host, args, mode, parallel=False) + File "/home/users/nico/p/cdist/cdist/bin/../scripts/cdist", line 180, in configinstall_onehost + c.deploy_and_cleanup() + File "/home/users/nico/oeffentlich/rechner/projekte/cdist/cdist/cdist/config_install.py", line 74, in deploy_and_cleanup + self.deploy_to() + File "/home/users/nico/oeffentlich/rechner/projekte/cdist/cdist/cdist/config_install.py", line 68, in deploy_to + self.stage_prepare() + File "/home/users/nico/oeffentlich/rechner/projekte/cdist/cdist/cdist/config_install.py", line 91, in stage_prepare + self.context.local.type_path): + File "/home/users/nico/oeffentlich/rechner/projekte/cdist/cdist/cdist/core/cdist_object.py", line 80, in list_objects + yield cls(cdist.core.CdistType(type_base_path, type_name), object_base_path, object_id=object_id) + File "/home/users/nico/oeffentlich/rechner/projekte/cdist/cdist/cdist/core/cdist_object.py", line 65, in __init__ + self.validate_object_id() + File "/home/users/nico/oeffentlich/rechner/projekte/cdist/cdist/cdist/core/cdist_object.py", line 130, in validate_object_id + (self.cdist_type.name, self.parameters)) + File "/home/users/nico/oeffentlich/rechner/projekte/cdist/cdist/cdist/util/fsproperty.py", line 210, in __get__ + return self._get_attribute(instance, owner) + File "/home/users/nico/oeffentlich/rechner/projekte/cdist/cdist/cdist/util/fsproperty.py", line 202, in _get_attribute + path = self._get_path(instance) + File "/home/users/nico/oeffentlich/rechner/projekte/cdist/cdist/cdist/util/fsproperty.py", line 190, in _get_path + path = path(instance) + File "/home/users/nico/oeffentlich/rechner/projekte/cdist/cdist/cdist/core/cdist_object.py", line 192, in + parameters = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.base_path, obj.parameter_path)) +AttributeError: 'CdistObject' object has no attribute 'parameter_path' +[22:37] brief:~% + diff --git a/docs/dev/logs/2013-02-05.weird-notsingleton-type-error b/docs/dev/logs/2013-02-05.weird-notsingleton-type-error new file mode 100644 index 00000000..23693777 --- /dev/null +++ b/docs/dev/logs/2013-02-05.weird-notsingleton-type-error @@ -0,0 +1,15 @@ + +Hard to find the source bug/problem: + +DEBUG: solr.petspremium.de: (emulator) __file//etc/solr/solr.xml: Finished __file/etc/solr/solr.xml/.cdist {'mode': '0644', 'source': '/home/users/nico/.tmp/tmpn27s24/out/conf/type/__petspremium_solr/files/solr/solr.xml'} ++ for file in '$(find . -type f | sed '\''s,^./,,'\'')' ++ dfile=/etc/solr/web.xml ++ reqdir=/etc/solr ++ require=__directory/etc/solr ++ __file /etc/solr/web.xml --source /home/users/nico/.tmp/tmpn27s24/out/conf/type/__petspremium_solr/files/solr/web.xml --mode 0644 +DEBUG: solr.petspremium.de: (emulator): /home/users/nico/.tmp/tmpn27s24/out/bin/__file: Namespace(mode='0644', object_id=['/etc/solr/web.xml'], source='/home/users/nico/.tmp/tmpn27s24/out/conf/type/__petspremium_solr/files/solr/web.xml') +DEBUG: solr.petspremium.de: (emulator) __file//etc/solr/web.xml: Recording requirement: __directory/etc/solr +DEBUG: solr.petspremium.de: (emulator) __file//etc/solr/web.xml: Finished __file/etc/solr/web.xml/.cdist {'source': '/home/users/nico/.tmp/tmpn27s24/out/conf/type/__petspremium_solr/files/solr/web.xml', 'mode': '0644'} +ERROR: solr.petspremium.de: Type __directory requires object id (is not a singleton type) +INFO: Total processing time for 1 host(s): 9.756716251373291 +ERROR: Failed to deploy to the following hosts: solr.petspremium.de diff --git a/docs/dev/logs/2013-04-03.dependency-discussion b/docs/dev/logs/2013-04-03.dependency-discussion new file mode 100644 index 00000000..29b5b5b5 --- /dev/null +++ b/docs/dev/logs/2013-04-03.dependency-discussion @@ -0,0 +1,30 @@ +Steven, Nico + +Discussion raised due to proposal from Arkaitz Jimenez + +-------------------------------------------------------------------------------- + +Proposal changes back to cdist behaviour as of 2011 (see commit 61b7b68). + +Change would introduce: + +- no direct stage based running +- stages only in object (not globally) +- cannot build full dependency list before beginning + - Thus wildcard requirements (require="__file/*") don't work anymore + +Accepting this or similar approaches means: + +- Drop wildcard requirements (is undocumented anyway) +- Type execution is closed (again) + +Furthermore/other points: + +- Change cdist to continue run as long as possible + - Don't stop if an object fails + - Record failure, print at the end (and exit non zero) + +- Logging + - Catch output of manifest, gencode, code, do not display directly + - Print at the end + - Prefix with hostname as usual! diff --git a/docs/dev/logs/2013-04-08.execution-graph.xoj b/docs/dev/logs/2013-04-08.execution-graph.xoj new file mode 100644 index 00000000..25d48d3b Binary files /dev/null and b/docs/dev/logs/2013-04-08.execution-graph.xoj differ diff --git a/docs/dev/logs/2013-04-10.discussion b/docs/dev/logs/2013-04-10.discussion new file mode 100644 index 00000000..648ed470 --- /dev/null +++ b/docs/dev/logs/2013-04-10.discussion @@ -0,0 +1,77 @@ +Steven, Nico (ETH office) + +- Try out patch for dependency resolver changing from [nico] + - Add tests + - Cleanup code: + - remove all old resolver parts (including tests!) + - remve wildcard matching pattern code + +- Cache: [nobody] + - Should cache be usable by types? + - Should all run outputs be stored? + - Different caches for install and config + +- Replace fsproperties with cconfig [steven] + +- Maybe support "rerun from previous version (cache)"? [nobody] + - need to include initial manifest(s!) + - copy/link types + - save remote-{exec,copy} parameters (copy or save argument list) + + - cdist replay / oldconfig ? + +- Support diffing two configurations [nobody] + - cdist diff ? + +- Nested Types [both] + - Motivation: + - Put everything related into one directory + - Have a look at it when Arkaitz pushes out pull request + - Implementations: + + 1) Arkaitz + + Folder structure Call Object + __package/ __package abc __package/abc + __package/type/pkg __package.pkg abc __package.pkg/abc + __package/type/pkg/type/green __package.pkg.green abc __package.pkg.green/abc + + ... + + __package.pkg __package.pkg abc __package.pkg/abc + + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + => Need to forbid types with "." in the name! + + 2) Steven (earlier version) + + Folder structure Call Object + __package/.type __package abc __package/abc + __package/pkg/.type __package.pkg abc __package.pkg/abc + __package/pkg/green/.type __package.pkg.green abc __package.pkg.green/abc + + - Clashes: + - if __. and __ and subtype exist both (in both implementations) + +- Install [nobody] + - Merge into master? + - Needs some cleanups + +- PreOS [nobody] + - cdist preos / preos-generate + --output= + --arch=[i386|amd64|arm??] + --type=[usb, cdrom/iso, floppy, pxe] + --other-params (?) + + - Maybe implement using cdist config indirectly and a type __preos + + - Can be: + - Internally only (devs) + - Usable by end users + + - Requirements: + - git + - buildchain + - toolchain for target arch + - ... diff --git a/docs/dev/logs/2013-04-12.execution-order b/docs/dev/logs/2013-04-12.execution-order new file mode 100644 index 00000000..5100eeda --- /dev/null +++ b/docs/dev/logs/2013-04-12.execution-order @@ -0,0 +1,44 @@ +Old: + +- global explores (all) +- initial manifest +- for each object + execute type explorers + execute manifest + + continue until all objects (including newly created) + have their type explorers/manifests run +- build dependency tree +- for each object + execute gencode-* + execute code-* + +New: +- run all global explorers +- run initial manifest + creates zero or more cdist_objects +- for each cdist_object + if not cdist_object.has_unfullfilled_requirements: + execute type explorers + execute manifest + may create new objects, resulting in autorequirements + + # Gained requirements during manifest run + if object.has_auto_requirements(): + continue + + cdist_object.execute gencode-* + cdist_object.execute code-* + + +Requirements / Test cases for requirments / resolver: + + - omnipotence + - + + +-------------------------------------------------------------------------------- +ERROR: localhost: The following objects could not be resolved: __cdistmarker/singleton requires autorequires ; __directory/etc/sudoers.d requires autorequires ; __file/etc/sudoers.d/nico requires __directory/etc/sudoers.d autorequires ; __file/etc/motd requires autorequires ; __package_pacman/atop requires autorequires ; __package_pacman/screen requires autorequires ; __package_pacman/strace requires autorequires ; __package_pacman/vim requires autorequires ; __package_pacman/zsh requires autorequires ; __package_pacman/lftp requires autorequires ; __package_pacman/nmap requires autorequires ; __package_pacman/ntp requires autorequires ; __package_pacman/rsync requires autorequires ; __package_pacman/rtorrent requires autorequires ; __package_pacman/wget requires autorequires ; __package_pacman/nload requires autorequires ; __package_pacman/iftop requires autorequires ; __package_pacman/mosh requires autorequires ; __package_pacman/git requires autorequires ; __package_pacman/mercurial requires autorequires ; __package_pacman/netcat requires autorequires ; __package_pacman/python-virtualenv requires autorequires ; __package_pacman/wireshark-cli requires autorequires ; __package_pacman/sudo requires autorequires +INFO: Total processing time for 1 host(s): 32.30426597595215 +ERROR: Failed to deploy to the following hosts: localhost + diff --git a/docs/dev/logs/2013-05-04.ssh b/docs/dev/logs/2013-05-04.ssh new file mode 100644 index 00000000..176e5b62 --- /dev/null +++ b/docs/dev/logs/2013-05-04.ssh @@ -0,0 +1,340 @@ +- analysis of ssh connections for callback + SSH_CLIENT='::1 38502 22' + SSH_CONNECTION='::1 38502 ::1 22' + + -> callback possible to source host + + + +[ target host ] <--------------| + | | + | | + | | + | trigger | configuration + | | + v | +[ configuration host ] ----| + + +- dynamic port allocation for tunneling + + [1:37] bento:~% ssh -R 0:localhost:22 localhost + Warning: Permanently added 'localhost' (ECDSA) to the list of known hosts. + Allocated port 53161 for remote forward to localhost:22 + + SSH_AUTH_SOCK=/tmp/ssh-zDCWbUVcUK/agent.30749 + SSH_CLIENT='::1 38587 22' + SSH_CONNECTION='::1 38587 ::1 22' + SSH_TTY=/dev/pts/21 + + +- ssh_config: + DynamicForward + LocalForward + RemoteForward + +- testing + +[1:52] bento:cdist% netstat -anp | grep 56844 +(Not all processes could be identified, non-owned process info + will not be shown, you would have to be root to see it all.) +tcp 0 0 127.0.0.1:56844 0.0.0.0:* LISTEN - +tcp6 0 0 ::1:56844 :::* LISTEN - +[1:53] bento:cdist% + + +[1:48] bento:~% ssh -R 0:localhost:22 localhost +Allocated port 56844 for remote forward to localhost:22 +... + +- chatting + +01:42 -!- Irssi: Join to #openssh was synced in 0 secs +01:42 < telmich> good evening +01:43 < telmich> I am trying to make use of remote port forwarding using dynamic port + allocation (port=0) -- I am wondering if there is an easy way to + access the port number on the remote side easily? +01:44 < telmich> background for this question is: I'd like to allow various clients to + login to a configuration server, which then configures the clients by + using the tunnel the client provides for the server to ssh back into +02:07 < BasketCase> telmich: afaik you need to use a tool like ss/netstat/lsof to see what port it has open + +- ssh debug + +[11:37] bento:~% ssh -R 0:localhost:22 localhost +Allocated port 33562 for remote forward to localhost:22 + + .. . .x+=:. s + dF @88> z` ^% :8 + '88bu. %8P . server aes128-ctr hmac-md5-etm@openssh.com zlib@openssh.com [preauth] +debug1: kex: server->client aes128-ctr hmac-md5-etm@openssh.com zlib@openssh.com [preauth] +debug1: expecting SSH2_MSG_KEX_ECDH_INIT [preauth] +debug1: SSH2_MSG_NEWKEYS sent [preauth] +debug1: expecting SSH2_MSG_NEWKEYS [preauth] +debug1: SSH2_MSG_NEWKEYS received [preauth] +debug1: KEX done [preauth] +debug1: userauth-request for user root service ssh-connection method none [preauth] +debug1: attempt 0 failures 0 [preauth] +debug1: PAM: initializing for "root" +debug1: PAM: setting PAM_RHOST to "localhost.localdomain" +debug1: PAM: setting PAM_TTY to "ssh" +debug1: userauth-request for user root service ssh-connection method publickey [preauth] +debug1: attempt 1 failures 0 [preauth] +debug1: test whether pkalg/pkblob are acceptable [preauth] +debug1: temporarily_use_uid: 0/0 (e=0/0) +debug1: trying public key file /root/.ssh/authorized_keys +debug1: fd 4 clearing O_NONBLOCK +debug1: matching key found: file /root/.ssh/authorized_keys, line 2 +Found matching RSA key: 2e:1b:3f:10:01:1d:21:6c:6c:1e:3d:a9:33:ba:3c:f7 +debug1: restore_uid: 0/0 +Postponed publickey for root from ::1 port 57848 ssh2 [preauth] +debug1: userauth-request for user root service ssh-connection method publickey [preauth] +debug1: attempt 2 failures 0 [preauth] +debug1: temporarily_use_uid: 0/0 (e=0/0) +debug1: trying public key file /root/.ssh/authorized_keys +debug1: fd 4 clearing O_NONBLOCK +debug1: matching key found: file /root/.ssh/authorized_keys, line 2 +Found matching RSA key: 2e:1b:3f:10:01:1d:21:6c:6c:1e:3d:a9:33:ba:3c:f7 +debug1: restore_uid: 0/0 +debug1: ssh_rsa_verify: signature correct +debug1: do_pam_account: called +Accepted publickey for root from ::1 port 57848 ssh2 +debug1: monitor_child_preauth: root has been authenticated by privileged process +debug1: Enabling compression at level 6. [preauth] +debug1: monitor_read_log: child log fd closed +debug1: PAM: establishing credentials +debug1: Entering interactive session for SSH2. +debug1: server_init_dispatch_20 +debug1: server_input_global_request: rtype tcpip-forward want_reply 1 +debug1: server_input_global_request: tcpip-forward listen localhost port 0 +debug1: Local forwarding listening on ::1 port 0. +debug1: Allocated listen port 33562 +debug1: channel 0: new [port listener] +debug1: Local forwarding listening on 127.0.0.1 port 33562. +debug1: channel 1: new [port listener] +debug1: server_input_channel_open: ctype session rchan 0 win 1048576 max 16384 +debug1: input_session_request +debug1: channel 2: new [server-session] +debug1: session_new: session 0 +debug1: session_open: channel 2 +debug1: session_open: session 0: link with channel 2 +debug1: server_input_channel_open: confirm session +debug1: server_input_global_request: rtype no-more-sessions@openssh.com want_reply 0 +debug1: server_input_channel_req: channel 2 request auth-agent-req@openssh.com reply 0 +debug1: session_by_channel: session 0 channel 2 +debug1: session_input_channel_req: session 0 req auth-agent-req@openssh.com +debug1: temporarily_use_uid: 0/0 (e=0/0) +debug1: restore_uid: 0/0 +debug1: channel 3: new [auth socket] +debug1: server_input_channel_req: channel 2 request pty-req reply 1 +debug1: session_by_channel: session 0 channel 2 +debug1: session_input_channel_req: session 0 req pty-req +debug1: Allocating pty. +debug1: session_pty_req: session 0 alloc /dev/pts/32 +debug1: server_input_channel_req: channel 2 request shell reply 1 +debug1: session_by_channel: session 0 channel 2 +debug1: session_input_channel_req: session 0 req shell +debug1: Setting controlling tty using TIOCSCTTY. + +-------------------------------------------------------------------------------- +debug1: server_input_global_request: rtype tcpip-forward want_reply 1 +debug1: server_input_global_request: tcpip-forward listen localhost port 0 +debug1: Local forwarding listening on ::1 port 0. +debug1: Allocated listen port 33562 +debug1: channel 0: new [port listener] +debug1: Local forwarding listening on 127.0.0.1 port 33562. + +[11:49] bento:openssh-6.2p1% grep "Allocated listen port" -r . +./channels.c: debug("Allocated listen port %d", +[11:49] bento:openssh-6.2p1% + + +-------------------------------------------------------------------------------- +[11:54] bento:~% ssh -R 0:localhost:22 -R 0:192.168.1.1:33 localhost +Allocated port 48392 for remote forward to localhost:22 +Allocated port 37515 for remote forward to 192.168.1.1:33 + + + + +debug1: server_input_global_request: rtype tcpip-forward want_reply 1 +debug1: server_input_global_request: tcpip-forward listen localhost port 0 +debug1: Local forwarding listening on ::1 port 0. +debug1: Allocated listen port 48392 +debug1: channel 0: new [port listener] +debug1: Local forwarding listening on 127.0.0.1 port 48392. +debug1: channel 1: new [port listener] +debug1: server_input_global_request: rtype tcpip-forward want_reply 1 +debug1: server_input_global_request: tcpip-forward listen localhost port 0 +debug1: Local forwarding listening on ::1 port 0. +debug1: Allocated listen port 37515 +debug1: channel 2: new [port listener] +debug1: Local forwarding listening on 127.0.0.1 port 37515. +debug1: channel 3: new [port listener] +debug1: server_input_channel_open: ctype session rchan 0 win 1048576 max 16384 +debug1: input_session_request +debug1: channel 4: new [server-session] +debug1: session_new: session 0 +debug1: session_open: channel 4 +debug1: session_open: session 0: link with channel 4 + +debug1: Local forwarding listening on ::1 port 5555. +debug1: channel 0: new [port listener] +debug1: Local forwarding listening on 127.0.0.1 port 5555. +debug1: channel 1: new [port listener] +debug1: server_input_global_request: rtype tcpip-forward want_reply 1 +debug1: server_input_global_request: tcpip-forward listen localhost port 4444 +debug1: Local forwarding listening on ::1 port 4444. +debug1: channel 2: new [port listener] +debug1: Local forwarding listening on 127.0.0.1 port 4444. +debug1: channel 3: new [port listener] +debug1: server_input_channel_open: ctype session rchan 0 win 1048576 max 16384 +debug1: input_session_request +debug1: channel 4: new [server-session] +debug1: session_new: session 0 +debug1: session_open: channel 4 + +-------------------------------------------------------------------------------- + +[12:06] bento:openssh-6.2p1% grep SSH_CONNECTION -r * +audit-bsm.c: case SSH_CONNECTION_CLOSE: +audit.c: {SSH_CONNECTION_CLOSE, "CONNECTION_CLOSE"}, +audit.c: {SSH_CONNECTION_ABANDON, "CONNECTION_ABANDON"}, +audit.h: SSH_CONNECTION_CLOSE, /* closed after attempting auth or session */ +audit.h: SSH_CONNECTION_ABANDON, /* closed without completing auth */ +audit-linux.c: case SSH_CONNECTION_CLOSE: +monitor.c: case SSH_CONNECTION_CLOSE: +regress/proxy-connect.sh: SSH_CONNECTION=`${SSH} -$p -F $OBJ/ssh_proxy 999.999.999.999 'echo $SSH_CONNECTION'` +regress/proxy-connect.sh: if [ "$SSH_CONNECTION" != "UNKNOWN 65535 UNKNOWN 65535" ]; then +regress/proxy-connect.sh: fail "bad SSH_CONNECTION" +session.c: child_set_env(&env, &envsize, "SSH_CONNECTION", buf); +sftp-server.c: if ((cp = getenv("SSH_CONNECTION")) != NULL) { +sftp-server.c: error("Malformed SSH_CONNECTION variable: \"%s\"", +sftp-server.c: getenv("SSH_CONNECTION")); +ssh.0: SSH_CONNECTION Identifies the client and server ends of the +ssh.1:.It Ev SSH_CONNECTION +sshd.c: PRIVSEP(audit_event(SSH_CONNECTION_CLOSE)); +sshd.c: audit_event(SSH_CONNECTION_ABANDON); +[12:06] bento:openssh-6.2p1% + +-------------------------------------------------------------------------------- +debug1: Remote connections from LOCALHOST:5555 forwarded to local address localhost:22 + +-------------------------------------------------------------------------------- +[12:42] bento:openssh-6.2p1% grep tcpip-forward * +channels.c: packet_put_cstring("tcpip-forward"); +channels.c: packet_put_cstring("cancel-tcpip-forward"); +Binary file channels.o matches +grep: contrib: Is a directory +Binary file libssh.a matches +grep: openbsd-compat: Is a directory +grep: regress: Is a directory +grep: scard: Is a directory +serverloop.c: if (strcmp(rtype, "tcpip-forward") == 0) { +serverloop.c: debug("server_input_global_request: tcpip-forward listen %s port %d", +serverloop.c: } else if (strcmp(rtype, "cancel-tcpip-forward") == 0) { +serverloop.c: debug("%s: cancel-tcpip-forward addr %s port %d", __func__, +Binary file serverloop.o matches +Binary file ssh matches +Binary file sshd matches +Binary file ssh-keyscan matches +Binary file ssh-keysign matches +[12:42] bento:openssh-6.2p1% + +-------------------------------------------------------------------------------- +Channel information for (remote) forwarding: + + c = channel_new("port listener", type, sock, sock, -1, + CHAN_TCP_WINDOW_DEFAULT, CHAN_TCP_PACKET_DEFAULT, + 0, "port listener", 1); + c->path = xstrdup(host); + c->host_port = port_to_connect; + c->listening_addr = addr == NULL ? NULL : xstrdup(addr); + if (listen_port == 0 && allocated_listen_port != NULL && + !(datafellows & SSH_BUG_DYNAMIC_RPORT)) + c->listening_port = *allocated_listen_port; + else + c->listening_port = listen_port; + +-------------------------------------------------------------------------------- + +Code handling remote forwarding in the client: +- ssh_init_forwarding + - channel_request_remote_forwarding + Sends hostname + port for ssh1 only - not send in ssh2 + +Code handling forwarding / listening in the server: + +- channel_new: creates channels, 2 per listener (ipv4/ipv6) + - channels_alloc contains number of channels +- server_input_global_request + Reads only listen port, not hostname/port to connect to + - channel_setup_remote_fwd_listener + - channel_setup_remote_fwd_listener + +Code handling environment variables: + +- child_set_env +1236 child_set_env(&env, &envsize, "SSH_CONNECTION", buf); + diff --git a/docs/dev/logs/2013-05-17.ssh-callback-socat b/docs/dev/logs/2013-05-17.ssh-callback-socat new file mode 100644 index 00000000..69428309 --- /dev/null +++ b/docs/dev/logs/2013-05-17.ssh-callback-socat @@ -0,0 +1,40 @@ + +start ssh +to controlhost, +bind other side to +localhost:22 + + +targethost ------> ssh ------> controlhost + | + | + socat: connect stdin/stdout to ? + start cdist with port information + added + + +Use + +socat + + +-------------------------------------------------------------------------------- + TCP:: + Connects to [TCP service] on [IP address] using TCP/IP version 4 or 6 depending on address specifi‐ + cation, name resolution, or option pf. + Option groups: FD,SOCKET,IP4,IP6,TCP,RETRY + Useful options: crnl, bind, pf, connect-timeout, tos, mtudiscover, mss, nodelay, nonblock, sourceport, retry, + readbytes + See also: TCP4, TCP6, TCP-LISTEN, UDP, SCTP-CONNECT, UNIX-CONNECT + +forever +-------------------------------------------------------------------------------- +[root@nico-dev-vm-snr01 yum.repos.d]# ps aux | grep socat +nico 25035 0.0 0.0 41640 1524 ? Ss 13:27 0:00 socat - TCP-LISTEN:1234 +root 25037 0.0 0.0 103240 836 pts/1 S+ 13:27 0:00 grep socat +[root@nico-dev-vm-snr01 yum.repos.d]# + + + +-------------------------------------------------------------------------------- + diff --git a/docs/dev/logs/2013-07-12.release b/docs/dev/logs/2013-07-12.release new file mode 100644 index 00000000..da4b296c --- /dev/null +++ b/docs/dev/logs/2013-07-12.release @@ -0,0 +1,38 @@ +- setup release date in docs/changelog to today manually + +- checkout master branch + [ + x check if date is correct in docs/changelog + x ensure all unittests work + - requires (wrong/outdated) versionfile! + x compile manpages + x compile speeches + ] + [ + x add manpages to website repo + x add speeches to website repo + x rsync cdist docs to website repo & add to website repo + x create blog entry & add to website repo + ] + x upload website + x fix latest link for manpages + x send mail to mailinglist -> also requires git tag & git release + x should also require web-release including blog! + + - create PKGBUILD for archlinux release + + x create git tag / read description + t if necessary create version branch + x change to version branch and merge tag! + x update git repos + x update website from repo + x create release on freecode + + x create versionfile + x make pypi release + x make archlinux release + +manual last steps: + +- announce on linkedin +- announce on twitter diff --git a/docs/dev/logs/2013-07-25.source-error-does-not-stop-cdist b/docs/dev/logs/2013-07-25.source-error-does-not-stop-cdist new file mode 100644 index 00000000..b0c43971 --- /dev/null +++ b/docs/dev/logs/2013-07-25.source-error-does-not-stop-cdist @@ -0,0 +1,56 @@ +Symptom: + running something in a manifest and that fails does not exist + the cdist run + +Analysis: + Find out what the shell does: + + [23:56] bento:testshell% cat a.sh + # source something that fails + . b.sh + [23:57] bento:testshell% cat b.sh + nosuchcommand + [23:57] bento:testshell% sh -e a.sh + a.sh: 2: .: b.sh: not found + [23:57] bento:testshell% echo $? + 2 + + -> exit 2 -> looks good + + + Find out what the python does: + + [23:57] bento:testshell% python3 + Python 3.3.2 (default, May 21 2013, 15:40:45) + [GCC 4.8.0 20130502 (prerelease)] on linux + Type "help", "copyright", "credits" or "license" for more information. + >>> import subprocess + >>> subprocess.check_call(["/bin/sh", "-e", "a.sh"]) + a.sh: 2: .: b.sh: not found + Traceback (most recent call last): + File "", line 1, in + File "/usr/lib/python3.3/subprocess.py", line 544, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['/bin/sh', '-e', 'a.sh']' returned non-zero exit status 2 + >>> + + +Conclusion: + Manifests that execute (!) other shell scripts does + not necessarily give the -e flag to the other script + -> called script can have failures, but exit 0 + if something the last thing executed does exit 0! + +Solution: + Instead of doing stuff like + "$__manifest/special" + + use + sh -e "$__manifest/special" + + or source the script: + . "$__manifest/special" + + (runs the script in the same namespace/process as everything in the + calling script) + diff --git a/docs/dev/logs/2013-08-07.shell b/docs/dev/logs/2013-08-07.shell new file mode 100644 index 00000000..3fa9f139 --- /dev/null +++ b/docs/dev/logs/2013-08-07.shell @@ -0,0 +1,2 @@ +What about having a cdist shell to have a shell with all available types? +Let's give it a try! diff --git a/docs/dev/logs/2013-08-12.release b/docs/dev/logs/2013-08-12.release new file mode 100644 index 00000000..1db05681 --- /dev/null +++ b/docs/dev/logs/2013-08-12.release @@ -0,0 +1,28 @@ +- already on 2.3.0-1 during release + - user bug: there should be no changes / commits during a release process + +hard linking docs/man/man7/cdist-type__user.7 -> cdist-2.3.0-1-g8192c2c/docs/man/man7 +hard linking docs/man/man7/cdist-type__user.html -> cdist-2.3.0-1-g8192c2c/docs/man/man7 +hard linking docs/man/man7/cdist-type__user_groups.7 -> cdist-2.3.0-1-g8192c2c/docs/man/man7 +hard linking docs/man/man7/cdist-type__user_groups.html -> cdist-2.3.0-1-g8192c2c/docs/man/man7 +hard linking scripts/cdist -> cdist-2.3.0-1-g8192c2c/scripts +creating dist +Creating tar archive +removing 'cdist-2.3.0-1-g8192c2c' (and everything under it) +running upload +Submitting dist/cdist-2.3.0-1-g8192c2c.tar.gz to http://pypi.python.org/pypi +Server response (200): OK +touch .lock-pypi +./PKGBUILD.in 2.3.0 +==> Retrieving sources... + -> Downloading cdist-2.3.0.tar.gz... + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +curl: (22) The requested URL returned error: 404 Not Found +==> ERROR: Failure while downloading cdist-2.3.0.tar.gz + Aborting... +make: *** [PKGBUILD] Error 1 +[12:38] bento:cdist% + diff --git a/docs/dev/logs/2013-08-18.cache-enhancement b/docs/dev/logs/2013-08-18.cache-enhancement new file mode 100644 index 00000000..95052dfe --- /dev/null +++ b/docs/dev/logs/2013-08-18.cache-enhancement @@ -0,0 +1,17 @@ +- always save cache = outdir + - even if run aborts (for debugging) + - add a state flag +- save cache in a date based directory +- also add support for a per-host pidfile +- allow user to specify cache dir - to give + full flexibility +- drop context - it is a very small unecessary wrapper + - maye introduce cdist.log instead! +- replace out_path with out_base + - directory under which all the subdirectories are + created + -> by default ~/.cdist/run + -> out_base_path +- drop support for deprecated environment variables + __cdist_out_dir + __cdist_remote_out_dir diff --git a/docs/dev/logs/2013-08-28.release b/docs/dev/logs/2013-08-28.release new file mode 100644 index 00000000..23e23d88 --- /dev/null +++ b/docs/dev/logs/2013-08-28.release @@ -0,0 +1,5 @@ +- release process releases pypi from something + that is git describe based, not changelog based... + + - git describe should equal changelog, but may be + inconsistent due to branch merging! diff --git a/docs/dev/logs/2013-09-05.test-cp b/docs/dev/logs/2013-09-05.test-cp new file mode 100644 index 00000000..1f02ee5d --- /dev/null +++ b/docs/dev/logs/2013-09-05.test-cp @@ -0,0 +1,34 @@ +Test copy copys symlinks - making real files would be better + +Test how to use cp: + +[12:54] bento:~% cd test +[12:54] bento:test% ln -s /etc/passwd +[12:54] bento:test% cd .. +[12:54] bento:~% cp -r test test2 +[12:54] bento:~% ls -lh test2/ +total 4.0K +lrwxrwxrwx 1 nico nico 11 Sep 5 12:54 passwd -> /etc/passwd +[12:54] bento:~% rm -rf test2/ + +-------------------------------------------------------------------------------- +[12:54] bento:~% ls -lh test2/ +total 4.0K +lrwxrwxrwx 1 nico nico 11 Sep 5 12:54 passwd -> /etc/passwd +[12:54] bento:~% rm -rf test2/ +[12:54] bento:~% cp -r --dereference test test2 +[12:56] bento:~% ls -l test2/ +total 4 +-rw------- 1 nico nico 960 Sep 5 12:56 passwd +[12:56] bento:~% + +-------------------------------------------------------------------------------- +[13:04] bento:cdist% git describe +2.3.2 +[13:09] bento:cdist% vi MANIFEST.in +[13:09] bento:cdist% vi MANIFEST +[13:09] bento:cdist% vi setup.py +[13:09] bento:cdist% cat cdist/version.py +VERSION = "2.3.1-34-g7acf041" +[13:10] bento:cdist% + diff --git a/docs/dev/logs/2013-10-03.ossawards/infos b/docs/dev/logs/2013-10-03.ossawards/infos new file mode 100644 index 00000000..a18c686f --- /dev/null +++ b/docs/dev/logs/2013-10-03.ossawards/infos @@ -0,0 +1,13 @@ +Required for the ossawards until 2013-10-06: + - all source code + - licenses GPLv3 + - installation instructions, + - On Linux do the following: + - pip install + - + - necessary documents and + - a demo video onto our web hard. + - installation + - cdist via cdist + - presentation + - build from existing ones (?) diff --git a/docs/dev/logs/2013-10-29.__line b/docs/dev/logs/2013-10-29.__line new file mode 100644 index 00000000..bc15de2c --- /dev/null +++ b/docs/dev/logs/2013-10-29.__line @@ -0,0 +1,6 @@ +- fix handling of fixed strings + - ensure special characters are not interpreted +[12:18] bento:~% cat /etc/bash.bashrc +cat: /etc/bash.bashrc: Permission denied +[12:19] bento:~% + diff --git a/docs/gfx/cdist-and-sexy-white.png b/docs/gfx/cdist-and-sexy-white.png new file mode 100644 index 00000000..1366b9c2 Binary files /dev/null and b/docs/gfx/cdist-and-sexy-white.png differ diff --git a/docs/gfx/label-cdist-ngcm.odt b/docs/gfx/label-cdist-ngcm.odt new file mode 100644 index 00000000..5e73b332 Binary files /dev/null and b/docs/gfx/label-cdist-ngcm.odt differ diff --git a/docs/man/cdist-reference.text.sh b/docs/man/cdist-reference.text.sh index 225d647f..b41be801 100755 --- a/docs/man/cdist-reference.text.sh +++ b/docs/man/cdist-reference.text.sh @@ -116,8 +116,13 @@ confdir/type//parameter/required:: confdir/type//parameter/optional:: Parameters optionally accepted by type, \n seperated list. +confdir/type//parameter/default/*:: + Default values for optional parameters. + Assuming an optional parameter name of 'foo', it's default value would + be read from the file confdir/type//parameter/default/foo. + confdir/type//parameter/boolean:: - Boolean parameters accepted by type, \n seperated list. + Boolean parameters accepted by type, \n seperated list. confdir/type//explorer:: Location of the type specific explorers. @@ -179,13 +184,13 @@ ENVIRONMENT VARIABLES --------------------- __explorer:: Directory that contains all global explorers. - Available for: explorer, type explorer + Available for: initial manifest, explorer, type explorer, shell __manifest:: Directory that contains the initial manifest. - Available for: initial manifest, type manifest + Available for: initial manifest, type manifest, shell __global:: Directory that contains generic output like explorer. - Available for: initial manifest, type manifest, type gencode + Available for: initial manifest, type manifest, type gencode, shell __object:: Directory that contains the current object. Available for: type manifest, type explorer, type gencode @@ -200,7 +205,7 @@ __object_name:: Available for: type manifest, type explorer, type gencode __target_host:: The host we are deploying to. - Available for: explorer, initial manifest, type explorer, type manifest, type gencode + Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell __type:: Path to the current type. Available for: type manifest, type gencode @@ -216,6 +221,6 @@ SEE ALSO COPYING ------- -Copyright \(C) 2011-2012 Nico Schottelius. Free use of this software is +Copyright \(C) 2011-2013 Nico Schottelius. Free use of this software is granted under the terms of the GNU General Public License version 3 (GPLv3). eof diff --git a/docs/man/man1/cdist.text b/docs/man/man1/cdist.text index 113454a7..de50a4ce 100644 --- a/docs/man/man1/cdist.text +++ b/docs/man/man1/cdist.text @@ -5,40 +5,51 @@ Nico Schottelius NAME ---- -cdist - Configuration management +cdist - Usable Configuration Management SYNOPSIS -------- -cdist [-h] [-V] +cdist [-h] [-d] [-v] [-V] {banner,config,shell} ... -cdist banner +cdist banner [-h] [-d] [-v] cdist config [-h] [-d] [-V] [-c CONF_DIR] [-i MANIFEST] [-p] [-s] host [host ...] +cdist shell [-h] [-d] [-v] [-s SHELL] DESCRIPTION ----------- cdist is the frontend executable to the cdist configuration management. -cdist supports different as explained below. The options to the main -program are: +cdist supports different subcommands as explained below. + +GENERAL +------- +All commands except the following options: + +-d, --debug:: + Set log level to debug -h, --help:: Show the help screen +-v, --verbose: + Set log level to info, be more verbose + -V, --version:: Show version and exit BANNER -------- -Displays the cdist banner. +------ +Displays the cdist banner. Useful for printing +cdist posters - a must have for every office. CONFIG ------ -Configure a system +Configure one or more hosts -h, --help:: Show the help screen @@ -52,9 +63,6 @@ Configure a system --conf-dir argument have higher precedence over those set through the environment variable. --d, --debug:: - Enable debug output - -i MANIFEST, --initial-manifest MANIFEST:: Path to a cdist manifest or - to read from stdin @@ -70,20 +78,30 @@ Configure a system --remote-exec REMOTE_EXEC: Command to use for remote execution (should behave like ssh) +SHELL +----- +This command allows you to spawn a shell that enables access +to the types as commands. It can be thought as an +"interactive manifest" environment. See below for example +usage. Its primary use is for debugging type parameters. + +-s/--shell:: + Select shell to use, defaults to current shell + EXAMPLES -------- -------------------------------------------------------------------------------- # Configure ikq05.ethz.ch with debug enabled -cdist config -d ikq05.ethz.ch +% cdist config -d ikq05.ethz.ch # Configure hosts in parallel and use a different configuration directory -cdist config -c ~/p/cdist-nutzung \ +% cdist config -c ~/p/cdist-nutzung \ -p ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch # Use custom remote exec / copy commands -cdist config --remote-exec /path/to/my/remote/exec \ +% cdist config --remote-exec /path/to/my/remote/exec \ --remote-copy /path/to/my/remote/copy \ -p ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch @@ -91,10 +109,18 @@ cdist config --remote-exec /path/to/my/remote/exec \ cdist banner # Show help -cdist --help +% cdist --help # Show Version -cdist --version +% cdist --version + +# Enter a shell that has access to emulated types +% cdist shell +% __git +usage: __git --source SOURCE [--state STATE] [--branch BRANCH] + [--group GROUP] [--owner OWNER] [--mode MODE] object_id + + -------------------------------------------------------------------------------- @@ -125,5 +151,5 @@ SEE ALSO COPYING ------- -Copyright \(C) 2011-2012 Nico Schottelius. Free use of this software is +Copyright \(C) 2011-2013 Nico Schottelius. Free use of this software is granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/docs/man/man7/cdist-best-practice.text b/docs/man/man7/cdist-best-practice.text index 818c423a..e6685cad 100644 --- a/docs/man/man7/cdist-best-practice.text +++ b/docs/man/man7/cdist-best-practice.text @@ -118,7 +118,7 @@ The following **.git/config** is taken from a a real world scenario: url = git://git.schottelius.org/cdist fetch = +refs/heads/*:refs/remotes/upstream/* -# Same as upstream, but works when being offline +# Same as upstream, but works when being offline [remote "local"] fetch = +refs/heads/*:refs/remotes/local/* url = /home/users/nico/p/cdist @@ -167,7 +167,7 @@ TEMPLATING * create directory templates/ in your type (convention) * create the template as an executable file like templates/basic.conf.sh, it will output text using shell variables for the values --------------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- #!/bin/sh # in the template, use cat << eof (here document) to output the text # and use standard shell variables in the template @@ -182,19 +182,58 @@ server { error_log /var/log/nginx/$SERVERNAME_error.log } EOF --------------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- * in the manifest, export the relevant variables and add the following lines in your manifest: --------------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- # export variables needed for the template export SERVERNAME='test" export ROOT='/var/www/test' # render the template mkdir -p "$__object/files" "$__type/templates/basic.conf.sh" > "$__object/files/basic.conf" -# send the rendered template - __file /etc/nginx/sites-available/test.conf --state present --source "$__object/files/basic.conf" --------------------------------------------------------------------------------------- +# send the rendered template + __file /etc/nginx/sites-available/test.conf \ + --state present + --source "$__object/files/basic.conf" +-------------------------------------------------------------------------------- + + +TESTING A NEW TYPE +------------------ +If you want to test a new type on a node, you can tell cdist to only use an +object of this type: Use the '--initial-manifest' parameter +with - (stdin) as argument and feed object into stdin +of cdist: + +-------------------------------------------------------------------------------- +# Singleton type without parameter +echo __ungleich_munin_server | cdist --initial-manifest - munin.panter.ch + +# Singleton type with parameter +echo __ungleich_munin_node --allow 1.2.3.4 | \ + cdist --initial-manifest - rails-19.panter.ch + +# Normal type +echo __file /tmp/stdintest --mode 0644 | \ + cdist --initial-manifest - cdist-dev-01.ungleich.ch +-------------------------------------------------------------------------------- + + +OTHER CONTENT IN CDIST REPOSITORY +--------------------------------- +Usually the cdist repository contains all configuration +items. Sometimes you may have additional resources that +you would like to store in your central configuration +repositiory (like password files from KeepassX, +Libreoffice diagrams, etc.). + +It is recommended to use a subfolder named "non-cdist" +in the repository for such content: It allows you to +easily distinguish what is used by cdist and what not +and also to store all important files in one +repository. + SEE ALSO -------- @@ -204,5 +243,5 @@ SEE ALSO COPYING ------- -Copyright \(C) 2011-2012 Nico Schottelius. Free use of this software is +Copyright \(C) 2011-2013 Nico Schottelius. Free use of this software is granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/docs/man/man7/cdist-bootstrap.text b/docs/man/man7/cdist-bootstrap.text index 7e1bff0f..985d0f53 100644 --- a/docs/man/man7/cdist-bootstrap.text +++ b/docs/man/man7/cdist-bootstrap.text @@ -64,11 +64,11 @@ So **2.0** is the latest version branch in this example. All versions (2.0.x) within one version branch (2.0) are compatible to each other and won't break your configuration when updating. -It's up to you decide on which branch you want to base your own work: +It's up to you to decide which branch you want to base your own work on: master contains more recent changes, newer types, but may also break. -The versions branches are stable, but thus may miss the latest features. +The version branches are stable, but may lack the latest features. Your decision can be changed later on, but may result in merge conflicts, -which you'd have to solve. +which you will need to solve. Let's assume you want latest stuff and select the master branch as base for your own work. Now it's time to create your branch, which contains your diff --git a/docs/man/man7/cdist-hacker.text b/docs/man/man7/cdist-hacker.text index d0f9a399..9dd52d35 100644 --- a/docs/man/man7/cdist-hacker.text +++ b/docs/man/man7/cdist-hacker.text @@ -51,7 +51,7 @@ work nor kill the authors brain: - On a merge request, always name the branch I should pull from - Always ensure **all** manpages build. Use **./build man** to test. - If you developed more than **one** feature, consider submitting them in - seperate branches. This way one feature can already be included, even if + separate branches. This way one feature can already be included, even if the other needs to be improved. As soon as your work meets these requirements, write a mail diff --git a/docs/man/man7/cdist-manifest.text b/docs/man/man7/cdist-manifest.text index 19f6053e..92d0b897 100644 --- a/docs/man/man7/cdist-manifest.text +++ b/docs/man/man7/cdist-manifest.text @@ -11,7 +11,7 @@ cdist-manifest - (Re-)Use types DESCRIPTION ----------- Manifests are used to define which objects to create. -Objects are instances of **types**, like in object orientated programming languages. +Objects are instances of **types**, like in object oriented programming languages. An object is represented by the combination of **type + slash + object name**: **__file/etc/cdist-configured** is an object of the type ***__file*** with the name ***etc/cdist-configured***. @@ -25,8 +25,8 @@ the reference with pointers to the manpages. Types in manifests are used like normal command line tools. Let's have a look at an example: -------------------------------------------------------------------------------- -# Create object of type __package with the parameter state = removed -__package apache2 --state removed +# Create object of type __package with the parameter state = absent +__package apache2 --state absent # Same with the __directory type __directory /tmp/cdist --state present @@ -57,9 +57,9 @@ DEFINE STATE IN THE INITIAL MANIFEST ------------------------------------ The **initial manifest** is the entry point for cdist to find out, which **objects** to configure on the selected host. -Cdist searches for the initial manifest at **cdist/conf/manifest/init**. +Cdist expects the initial manifest at **cdist/conf/manifest/init**. -Within this initial manifest, you define, which objects should be +Within this initial manifest you define, which objects should be created on which host. To distinguish between hosts, you can use the environment variable **__target_host**. Let's have a look at a simple example: @@ -107,7 +107,7 @@ DEPENDENCIES ------------ If you want to describe that something requires something else, just setup the variable "require" to contain the requirements. Multiple -requirements can be added white space seperated. +requirements can be added white space separated. -------------------------------------------------------------------------------- # No dependency @@ -135,12 +135,12 @@ The initial manifest may for instance contain the following code: -------------------------------------------------------------------------------- # Always create this file, so other sysadmins know cdist is used. -__file /etc/cdist-configured --type file +__file /etc/cdist-configured case "$__target_host" in my.server.name) - __file /root/bin/ --type directory - __file /etc/issue.net --type file --source "$__manifest/issue.net + __directory /root/bin/ + __file /etc/issue.net --source "$__manifest/issue.net ;; esac -------------------------------------------------------------------------------- @@ -148,9 +148,20 @@ esac The manifest of the type "nologin" may look like this: -------------------------------------------------------------------------------- -__file /etc/nologin --type file --source "$__type/files/default.nologin" +__file /etc/nologin --source "$__type/files/default.nologin" -------------------------------------------------------------------------------- +This example makes use of dependencies: + +-------------------------------------------------------------------------------- +# Ensure that lighttpd is installed +__package lighttpd --state present +# Ensure that munin makes use of lighttpd instead of the default webserver +# package as decided by the package manager +require="__package/lighttpd" __package munin --state present +-------------------------------------------------------------------------------- + + SEE ALSO -------- diff --git a/docs/man/man7/cdist-quickstart.text b/docs/man/man7/cdist-quickstart.text index b718da64..8b754650 100644 --- a/docs/man/man7/cdist-quickstart.text +++ b/docs/man/man7/cdist-quickstart.text @@ -72,7 +72,9 @@ As soon as you are able to login without password to localhost, we can use cdist to configure it. You can copy and paste the following code into your shell to get started and configure localhost: -------------------------------------------------------------------------------- -# Get cdist +# Get cdist +# Mirrors can be found on +# http://www.nico.schottelius.org/software/cdist/install/#index2h4 git clone git://git.schottelius.org/cdist # Create manifest (maps configuration to host(s) diff --git a/docs/man/man7/cdist-stages.text b/docs/man/man7/cdist-stages.text index fa5e28d1..5f2d2e4c 100644 --- a/docs/man/man7/cdist-stages.text +++ b/docs/man/man7/cdist-stages.text @@ -33,7 +33,7 @@ be created, if it has different parameters. STAGE 3: OBJECT INFORMATION RETRIEVAL ------------------------------------- Every object is checked whether its type has explorers and if so, these are -executed on the target host. The results are transfered back +executed on the target host. The results are transferred back and can be used in the following stages to decide what changes need to be made on the target to implement the desired state. diff --git a/docs/man/man7/cdist-troubleshooting.text b/docs/man/man7/cdist-troubleshooting.text new file mode 100644 index 00000000..7c5e7612 --- /dev/null +++ b/docs/man/man7/cdist-troubleshooting.text @@ -0,0 +1,63 @@ +cdist-troubleshooting(7) +======================== +Nico Schottelius + + +NAME +---- +cdist-troubleshooting - common problems and their solutions + + +ERROR IN MANIFEST IS NOT CONSIDERED AN ERROR BY CDIST +----------------------------------------------------- +Situation: You are executing other scripts from a manifest. +This script fails, but cdist does not recognise the error. +An example script would be something like this: + +-------------------------------------------------------------------------------- +% cat ~/.cdist/manifest/init +"$__manifest/special" +% cat ~/.cdist/manifest/special +#!/bin/sh +echo "Here is an unclean exiting script" +somecommandthatdoesnotexist +echo "I continue here although previous command failed" +-------------------------------------------------------------------------------- + +We can clearly see that **somecommandthatdoesnotexist** +will fail in ~/.cdist/manifest/special. But as the custom +script is not called with the -e flag (exit on failure) of shell, +it does not lead to an error. And thus cdist sees the exit 0 +code of the last echo line instead of the failing command. + +All scripts executed by cdist carry the -e flag. +To prevent the above from happening, there are three solutions available, +two of which can be used in the calling script: +-------------------------------------------------------------------------------- +# Execute as before, but abort on failure +sh -e "$__manifest/special" + +# Source the script in our namespace, runs in a set -e environment: +. "$__manifest/special" +-------------------------------------------------------------------------------- + +The third solution is to include a shebang header in every script +you write to use the -e flag: + +-------------------------------------------------------------------------------- +% cat ~/.cdist/manifest/special +#!/bin/sh -e +... +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist(1) +- cdist-tutorial(7) + + +COPYING +------- +Copyright \(C) 2013 Nico Schottelius. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/docs/man/man7/cdist-type.text b/docs/man/man7/cdist-type.text index 54b67be5..8415f991 100644 --- a/docs/man/man7/cdist-type.text +++ b/docs/man/man7/cdist-type.text @@ -67,25 +67,31 @@ A type consists of Types are stored below cdist/conf/type/. Their name should always be prefixed with two underscores (__) to prevent collisions with other executables in $PATH. -To begin a new type, just create the directory **cdist/conf/type/__NAME**. +To implement a new type, create the directory **cdist/conf/type/__NAME**. DEFINING PARAMETERS ------------------- Every type consists of required, optional and boolean parameters, which must -be created in a newline seperated file in ***parameter/required***, +each be declared in a newline separated file in ***parameter/required***, ***parameter/required_multiple***, ***parameter/optional***, ***parameter/optional_multiple*** and ***parameter/boolean***. Parameters which are allowed multiple times should be listed in -required_multiple or optional_multiple respectively. For all other parameters -the standard unix behaviour of the last given wins is applied. +required_multiple or optional_multiple respectively. All other parameters +follow the standard unix behaviour "the last given wins". If either is missing, the type will have no required, no optional, no boolean or no parameters at all. +Default values for optional parameters can be predefined in +***parameter/default/***. + Example: -------------------------------------------------------------------------------- echo servername >> cdist/conf/type/__nginx_vhost/parameter/required echo logdirectory >> cdist/conf/type/__nginx_vhost/parameter/optional +echo loglevel >> cdist/conf/type/__nginx_vhost/parameter/optional +mkdir cdist/conf/type/__nginx_vhost/parameter/default +echo warning > cdist/conf/type/__nginx_vhost/parameter/default/loglevel echo server_alias >> cdist/conf/type/__nginx_vhost/parameter/optional_multiple echo use_ssl >> cdist/conf/type/__nginx_vhost/parameter/boolean -------------------------------------------------------------------------------- @@ -108,6 +114,9 @@ if [ -f "$__object/parameter/logdirectory" ]; then logdirectory="$(cat "$__object/parameter/logdirectory")" fi +# optional parameter with predefined default +loglevel="$(cat "$__object/parameter/loglevel")" + # boolean parameter if [ -f "$__object/parameter/use_ssl" ]; then # file exists -> True @@ -125,7 +134,7 @@ fi INPUT FROM STDIN ------------------ +---------------- Every type can access what has been written on stdin when it has been called. The result is saved into the ***stdin*** file in the object directory. @@ -141,6 +150,7 @@ If you have not seen this syntax (<< eof) before, it may help you to read about "here documents". In the __file type, stdin is used as source for the file, if - is used for source: + -------------------------------------------------------------------------------- if [ -f "$__object/parameter/source" ]; then source="$(cat "$__object/parameter/source")" @@ -229,7 +239,7 @@ the output of gencode-remote is executed on the target. The gencode scripts can make use of the parameters, the global explorers and the type specific explorers. -If the gencode scripts encounter an error, it should print diagnostic +If the gencode scripts encounters an error, it should print diagnostic messages to stderr and exit non-zero. If you need to debug the gencode script, you can write to stderr: diff --git a/docs/speeches/2012-12-11_lisa.odp b/docs/speeches/2012-12-11_lisa.odp new file mode 100644 index 00000000..45c78955 Binary files /dev/null and b/docs/speeches/2012-12-11_lisa.odp differ diff --git a/docs/speeches/2013-01-23_panter.notes b/docs/speeches/2013-01-23_panter.notes new file mode 100644 index 00000000..d223c7bc --- /dev/null +++ b/docs/speeches/2013-01-23_panter.notes @@ -0,0 +1,10 @@ +sexy & sexy: ein glückliches Paar + +inhalt vom vortrag + +ziele von sexy und cdist + + systemadministration hochgradig zu automatisieren + effizientes (tägliches) arbeiten + + diff --git a/docs/speeches/2013-01-23_panter.odp b/docs/speeches/2013-01-23_panter.odp new file mode 100644 index 00000000..7fa0e4c7 Binary files /dev/null and b/docs/speeches/2013-01-23_panter.odp differ diff --git a/docs/speeches/2013-03-25_ad_novum.odp b/docs/speeches/2013-03-25_ad_novum.odp new file mode 100644 index 00000000..305f0a5e Binary files /dev/null and b/docs/speeches/2013-03-25_ad_novum.odp differ diff --git a/docs/speeches/2013-10-05_ossawards.odp b/docs/speeches/2013-10-05_ossawards.odp new file mode 100644 index 00000000..d0e4511c Binary files /dev/null and b/docs/speeches/2013-10-05_ossawards.odp differ diff --git a/docs/speeches/2013-10-05_ossawards/ossawards-2013-1sys.png b/docs/speeches/2013-10-05_ossawards/ossawards-2013-1sys.png new file mode 100644 index 00000000..d5b9586a Binary files /dev/null and b/docs/speeches/2013-10-05_ossawards/ossawards-2013-1sys.png differ diff --git a/docs/speeches/2013-10-05_ossawards/ossawards-2013-1sys1comp.png b/docs/speeches/2013-10-05_ossawards/ossawards-2013-1sys1comp.png new file mode 100644 index 00000000..f61d232c Binary files /dev/null and b/docs/speeches/2013-10-05_ossawards/ossawards-2013-1sys1comp.png differ diff --git a/docs/speeches/2013-10-05_ossawards/ossawards-2013-1sys4comp.png b/docs/speeches/2013-10-05_ossawards/ossawards-2013-1sys4comp.png new file mode 100644 index 00000000..5b37e094 Binary files /dev/null and b/docs/speeches/2013-10-05_ossawards/ossawards-2013-1sys4comp.png differ diff --git a/docs/speeches/2013-10-05_ossawards/ossawards-2013-4comp.png b/docs/speeches/2013-10-05_ossawards/ossawards-2013-4comp.png new file mode 100644 index 00000000..392eeeb8 Binary files /dev/null and b/docs/speeches/2013-10-05_ossawards/ossawards-2013-4comp.png differ diff --git a/docs/speeches/2013-10-05_ossawards/ossawards-2013-4sys.png b/docs/speeches/2013-10-05_ossawards/ossawards-2013-4sys.png new file mode 100644 index 00000000..5da1785a Binary files /dev/null and b/docs/speeches/2013-10-05_ossawards/ossawards-2013-4sys.png differ diff --git a/docs/speeches/2013-10-05_ossawards/ossawards-2013-sys-cdist.png b/docs/speeches/2013-10-05_ossawards/ossawards-2013-sys-cdist.png new file mode 100644 index 00000000..5fa66641 Binary files /dev/null and b/docs/speeches/2013-10-05_ossawards/ossawards-2013-sys-cdist.png differ diff --git a/docs/speeches/2013-10-05_ossawards/ossawards-2013-sys-chairman.png b/docs/speeches/2013-10-05_ossawards/ossawards-2013-sys-chairman.png new file mode 100644 index 00000000..01071950 Binary files /dev/null and b/docs/speeches/2013-10-05_ossawards/ossawards-2013-sys-chairman.png differ diff --git a/docs/speeches/2013-10-05_ossawards/ossawards-2013.xoj b/docs/speeches/2013-10-05_ossawards/ossawards-2013.xoj new file mode 100644 index 00000000..ce55c8b1 Binary files /dev/null and b/docs/speeches/2013-10-05_ossawards/ossawards-2013.xoj differ diff --git a/docs/speeches/2013-11-22_eth_linux_erfa.odp b/docs/speeches/2013-11-22_eth_linux_erfa.odp new file mode 100644 index 00000000..71d4719b Binary files /dev/null and b/docs/speeches/2013-11-22_eth_linux_erfa.odp differ diff --git a/docs/speeches/2013-11-22_eth_linux_erfa.pdf b/docs/speeches/2013-11-22_eth_linux_erfa.pdf new file mode 100644 index 00000000..3c2430af Binary files /dev/null and b/docs/speeches/2013-11-22_eth_linux_erfa.pdf differ diff --git a/docs/web/cdist/documentation.mdwn b/docs/web/cdist/documentation.mdwn index e5fd9bc9..db25b566 100644 --- a/docs/web/cdist/documentation.mdwn +++ b/docs/web/cdist/documentation.mdwn @@ -4,4 +4,6 @@ You can browse the latest [latest version of the manpages](/software/cdist/man/latest) or have a look at [all versions](/software/cdist/man). +You can also view [speeches about cdist](/software/cdist/speeches). + [[!tag cdist unix]] diff --git a/docs/web/cdist/install.mdwn b/docs/web/cdist/install.mdwn index ad97cd2b..c81354f0 100644 --- a/docs/web/cdist/install.mdwn +++ b/docs/web/cdist/install.mdwn @@ -10,7 +10,7 @@ This is the machine you use to configure the target hosts. * /bin/sh: A posix like shell (for instance bash, dash, zsh) * Python >= 3.2 * SSH client - * Asciidoc (for building the manpages) + * Asciidoc and xsltproc (for building the manpages) ### Target Hosts @@ -45,21 +45,25 @@ For Debian **wheezy** or newer: On **squeeze** you can add following line in **/etc/apt/sources.list** - deb http://ftp.debian.org/debian wheezy main + deb http://ftp.debian.org/debian wheezy main And add pinning entry in **/etc/apt/preferences.d/wheezy**: - Package: * - Pin: release n=wheezy - Pin-Priority: 1 + Package: * + Pin: release n=wheezy + Pin-Priority: 1 Please be aware that both **openssh-server** and **openssh-client** might be removed on **python3.2** installation. You surely want to reinstall them: - apt-get install -t wheezy openssh-server openssh-client + apt-get install -t wheezy openssh-server openssh-client For older Debian versions, installing python 3.2 from source is required. +If you want to build the cdist manpages: + + aptitude install --without-recommends asciidoc xsltproc + ### Fedora Fedora 15 and newer includes a recent python. @@ -139,7 +143,7 @@ To install cdist, execute the following commands: If you want to build and use the manpages, run: - ./build man + make man export MANPATH=$MANPATH:$(pwd -P)/doc/man #### Available versions in git diff --git a/docs/web/cdist/update.mdwn b/docs/web/cdist/update.mdwn index e486dff9..9e47fccc 100644 --- a/docs/web/cdist/update.mdwn +++ b/docs/web/cdist/update.mdwn @@ -14,13 +14,67 @@ If you stay on a version branche (i.e. 1.0, 1.1., ...), nothing should break. The master branch on the other hand is the development branch and may not be working, break your setup or eat the tree in your garden. +### Safely upgrading to new versions + +To upgrade to **any** further cdist version, you can take the +following procedure to do a safe upgrade: + + # Create new branch to try out the update + git checkout -b upgrade_cdist + + # Get latest cdist version in git database + git fetch -v + + # see what will happen on merge - replace + # master with the branch you plan to merge + git diff upgrade_cdist..origin/master + + # Merge the new version + git merge origin/master + +Now you can ensure all custom types work with the new version. +Assume that you need to go back to an older version during +the migration/update, you can do so as follows: + + # commit changes + git commit -m ... + + # go back to original branch + git checkout master + +After that, you can go back and continue the upgrade: + + # git checkout upgrade_cdist + + ## Update The Python Package To upgrade to the lastet version do pip install --upgrade cdist -## Update Instructions +## General Update Instructions + +### Updating from 2.2 to 2.3 + +No incompatiblities. + +### Updating from 2.1 to 2.2 + +Starting with 2.2, the syntax for requiring a singleton type changed: +Old format: + + require="__singleton_type/singleton" ... + +New format: + + require="__singleton_type" ... + +Internally the "singleton" object id was dropped to make life more easy. +You can probably fix your configuration by running the following code +snippet (currently untested, please report back if it works for you): + + find ~/.cdist/* -type f -exec sed -i 's,/singleton,,' {} \; ### Updating from 2.0 to 2.1 @@ -46,7 +100,6 @@ Have a look at the update guide for [[2.0 to 2.1|2.0-to-2.1]]. * Type **\_\_user**: Parameter --groups removed (use the new \_\_user_groups type) * Type **\_\_ssh_authorized_key** has been replaced by more flexible type **\_\_ssh_authorized_keys** - * require="" is deprecated: Use --after and --before as parameters instead ### Updating from 1.7 to 2.0 diff --git a/docs/web/cdist/why.mdwn b/docs/web/cdist/why.mdwn index 6dcfd441..f571555c 100644 --- a/docs/web/cdist/why.mdwn +++ b/docs/web/cdist/why.mdwn @@ -42,7 +42,8 @@ in almost all cases all dependencies are usually fulfilled. Cdist does not require an agent or a high level programming languages on the target host: it will run on any host that has a **ssh server running** and a posix compatible shell -(**/bin/sh**). +(**/bin/sh**). Compared to other configuration management systems, +it does not require to open up an additional port. ## Push based distribution diff --git a/other/examples/remote/local/copy b/other/examples/remote/local/copy index 644fee15..0cec3643 100755 --- a/other/examples/remote/local/copy +++ b/other/examples/remote/local/copy @@ -1,6 +1,7 @@ #!/bin/sh # # 2012 Nico Schottelius (nico-cdist schottelius.org) +# 2013 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -19,9 +20,5 @@ # # -recursive=$1; shift -src=$1; shift -dst=$1; shift - -dst=$(echo $dst | sed "s/^${__target_host}://") -cp "$recursive" "$src" "$dst" +code="$(echo "$@" | sed "s|\([[:space:]]\)$__target_host:|\1|g")" +cp --dereference $code diff --git a/other/examples/remote/sudo/copy b/other/examples/remote/sudo/copy new file mode 100755 index 00000000..577f270e --- /dev/null +++ b/other/examples/remote/sudo/copy @@ -0,0 +1,51 @@ +#!/bin/sh +# +# 2012 Matt Coddington (mcoddington at gmail.com) +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# 2013 Chase Allen James (nx-cdist at nu-ex.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Use rsync over ssh to copy files. Uses the "--rsync-path" option +# to run the remote rsync instance with sudo. +# +# This command assumes your ssh configuration is already set up +# in ~/.ssh/config. +# +# Usage: +# cdist config --remote-copy /path/to/this/script target_host +# + +# For rsync to do the right thing, the source has to end with "/" if it is +# a directory. The below preprocessor loop takes care of that. + +# second last argument is the source +source_index=$(($#-1)) +index=0 +for arg in $@; do + if [ $index -eq 0 ]; then + # reset $@ + set -- + fi + index=$((index+=1)) + if [ $index -eq $source_index -a -d "$arg" ]; then + arg="${arg%/}/" + fi + set -- "$@" "$arg" +done + +rsync --copy-links --rsync-path="sudo rsync" -e 'ssh' "$@" diff --git a/other/examples/remote/sudo/exec b/other/examples/remote/sudo/exec new file mode 100755 index 00000000..90eae2da --- /dev/null +++ b/other/examples/remote/sudo/exec @@ -0,0 +1,30 @@ +#!/bin/sh +# +# 2013 Chase Allen James (nx-cdist at nu-ex.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Prefixes all remote commands with sudo. +# +# This command assumes your ssh configuration is already set up +# in ~/.ssh/config. +# +# Usage: +# cdist config --remote-exec "/path/to/this/script" target_host +# + +host="$1"; shift +ssh -q "$host" sudo sh -c \""$@"\" diff --git a/scripts/cdist b/scripts/cdist index fd18933a..39449666 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# 2010-2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -26,7 +26,7 @@ def commandline(): import cdist.banner import cdist.config - import cdist.install + import cdist.shell # Construct parser others can reuse parser = {} @@ -52,42 +52,44 @@ def commandline(): parents=[parser['loglevel']]) parser['banner'].set_defaults(func=cdist.banner.banner) - # Config and install (common stuff) - parser['configinstall'] = argparse.ArgumentParser(add_help=False) - parser['configinstall'].add_argument('host', nargs='+', - help='one or more hosts to operate on') - parser['configinstall'].add_argument('-c', '--conf-dir', - help='Add configuration directory (can be repeated, last one wins)', - action='append') - parser['configinstall'].add_argument('-i', '--initial-manifest', - help='Path to a cdist manifest or \'-\' to read from stdin.', - dest='manifest', required=False) - parser['configinstall'].add_argument('-p', '--parallel', - help='Operate on multiple hosts in parallel', - action='store_true', dest='parallel') - parser['configinstall'].add_argument('-s', '--sequential', - help='Operate on multiple hosts sequentially (default)', - action='store_false', dest='parallel') - - parser['configinstall'].add_argument('--remote-copy', - help='Command to use for remote copy (should behave like scp)', - action='store', dest='remote_copy', - default="scp -o User=root -q") - parser['configinstall'].add_argument('--remote-exec', - help='Command to use for remote execution (should behave like ssh)', - action='store', dest='remote_exec', - default="ssh -o User=root -q") - # Config parser['config'] = parser['sub'].add_parser('config', - parents=[parser['loglevel'], parser['configinstall']]) - parser['config'].set_defaults(func=config) + parents=[parser['loglevel']]) + parser['config'].add_argument('host', nargs='+', + help='one or more hosts to operate on') + parser['config'].add_argument('-c', '--conf-dir', + help='Add configuration directory (can be repeated, last one wins)', + action='append') + parser['config'].add_argument('-i', '--initial-manifest', + help='Path to a cdist manifest or \'-\' to read from stdin.', + dest='manifest', required=False) + parser['config'].add_argument('-n', '--dry-run', + help='Do not execute code', action='store_true') + parser['config'].add_argument('-o', '--out-dir', + help='Directory to save cdist output in', dest="out_path") + parser['config'].add_argument('-p', '--parallel', + help='Operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser['config'].add_argument('-s', '--sequential', + help='Operate on multiple hosts sequentially (default)', + action='store_false', dest='parallel') + parser['config'].add_argument('--remote-copy', + help='Command to use for remote copy (should behave like scp)', + action='store', dest='remote_copy', + default=cdist.REMOTE_COPY) + parser['config'].add_argument('--remote-exec', + help='Command to use for remote execution (should behave like ssh)', + action='store', dest='remote_exec', + default=cdist.REMOTE_EXEC) + parser['config'].set_defaults(func=cdist.config.Config.commandline) + + # Shell + parser['shell'] = parser['sub'].add_parser('shell', + parents=[parser['loglevel']]) + parser['shell'].add_argument('-s', '--shell', + help='Select shell to use, defaults to current shell') + parser['shell'].set_defaults(func=cdist.shell.Shell.commandline) - # Install - # 20120525/sar: commented until it actually does something - #parser['install'] = parser['sub'].add_parser('install', - # parents=[parser['loglevel'], parser['configinstall']]) - #parser['install'].set_defaults(func=install) for p in parser: parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" @@ -101,106 +103,24 @@ def commandline(): logging.root.setLevel(logging.DEBUG) log.debug(args) - args.func(args) + log.info("version %s" % cdist.VERSION) -def config(args): - configinstall(args, mode=cdist.config.Config) + # Work around python 3.3 bug: + # http://bugs.python.org/issue16308 + # http://bugs.python.org/issue9253 -def install(args): - configinstall(args, mode=cdist.install.Install) - -def configinstall(args, mode): - """Configure or install remote system""" - import multiprocessing - import time - - initial_manifest_tempfile = None - if args.manifest == '-': - # read initial manifest from stdin - import tempfile - try: - handle, initial_manifest_temp_path = tempfile.mkstemp(prefix='cdist.stdin.') - with os.fdopen(handle, 'w') as fd: - fd.write(sys.stdin.read()) - except (IOError, OSError) as e: - raise cdist.Error("Creating tempfile for stdin data failed: %s" % e) - - args.manifest = initial_manifest_temp_path - import atexit - atexit.register(lambda: os.remove(initial_manifest_temp_path)) - - process = {} - failed_hosts = [] - time_start = time.time() - - for host in args.host: - if args.parallel: - log.debug("Creating child process for %s", host) - process[host] = multiprocessing.Process(target=configinstall_onehost, args=(host, args, mode, True)) - process[host].start() - else: - try: - configinstall_onehost(host, args, mode, parallel=False) - except cdist.Error as e: - failed_hosts.append(host) - - # Catch errors in parallel mode when joining - if args.parallel: - for host in process.keys(): - log.debug("Joining process %s", host) - process[host].join() - - if not process[host].exitcode == 0: - failed_hosts.append(host) - - time_end = time.time() - log.info("Total processing time for %s host(s): %s", len(args.host), - (time_end - time_start)) - - if len(failed_hosts) > 0: - raise cdist.Error("Failed to deploy to the following hosts: " + - " ".join(failed_hosts)) - -def configinstall_onehost(host, args, mode, parallel): - """Configure or install ONE remote system""" + # FIXME: catching AttributeError also hides + # real problems.. try a different way + # FIXME: we always print main help, not + # the help of the actual parser being used! try: - import cdist.context + getattr(args, "func") + except AttributeError: + parser['main'].print_help() + sys.exit(0) - context = cdist.context.Context( - target_host=host, - remote_copy=args.remote_copy, - remote_exec=args.remote_exec, - initial_manifest=args.manifest, - add_conf_dirs=args.conf_dir, - exec_path=sys.argv[0], - debug=args.debug) - - c = mode(context) - c.deploy_and_cleanup() - context.cleanup() - - except cdist.Error as e: - context.log.error(e) - # We are running in our own process here, need to sys.exit! - if parallel: - sys.exit(1) - else: - raise - - except KeyboardInterrupt: - # Ignore in parallel mode, we are existing anyway - if parallel: - sys.exit(0) - # Pass back to controlling code in sequential mode - else: - raise - -def emulator(): - """Prepare and run emulator""" - import cdist.emulator - emulator = cdist.emulator.Emulator(sys.argv) - return emulator.run() + args.func(args) if __name__ == "__main__": # Sys is needed for sys.exit() @@ -208,8 +128,8 @@ if __name__ == "__main__": cdistpythonversion = '3.2' if sys.version < cdistpythonversion: - print('Cdist requires Python >= ' + cdistpythonversion + - ' on the source host.', file=sys.stderr) + print('Python >= ' + cdistpythonversion + + ' is required on the source host.', file=sys.stderr) sys.exit(1) @@ -220,22 +140,24 @@ if __name__ == "__main__": import os import re import cdist + import cdist.log - log = logging.getLogger("cdist") + logging.setLoggerClass(cdist.log.Log) logging.basicConfig(format='%(levelname)s: %(message)s') + log = logging.getLogger("cdist") if re.match("__", os.path.basename(sys.argv[0])): - emulator() + import cdist.emulator + emulator = cdist.emulator.Emulator(sys.argv) + emulator.run() else: commandline() except KeyboardInterrupt: - pass + exit_code = 2 except cdist.Error as e: log.error(e) exit_code = 1 - # Determine exit code by return value of function - sys.exit(exit_code)