Release 5.1.3

__docker_swarm: Fix for Docker 19.03

See merge request ungleich-public/cdist!789

.gitattributes

@@ -4,3 +4,5 @@
 docs/speeches export-ignore
 docs/video export-ignore
 docs/src/man7 export-ignore
+bin/build-helper export-ignore
+README-maintainers export-ignore

.gitignore

@@ -12,6 +12,7 @@ Session.vim
 # Temporary
 .netrwhist
 *~
+*.tmp
 # Auto-generated tag files
 tags
 # Persistent undo

Makefile

@@ -18,36 +18,30 @@
 #
 #
 
-helper=./bin/build-helper
+.PHONY: help
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "man             build only man user documentation"
+	@echo "html            build only html user documentation"
+	@echo "docs            build both man and html user documentation"
+	@echo "dotman          build man pages for types in your ~/.cdist directory"
+	@echo "speeches        build speeches pdf files"
+	@echo "install         install in the system site-packages directory"
+	@echo "install-user    install in the user site-packages directory"
+	@echo "docs-clean      clean documentation"
+	@echo "clean           clean"
 
-DOCS_SRC_DIR=docs/src
-SPEECHDIR=docs/speeches
-TYPEDIR=cdist/conf/type
-
-WEBSRCDIR=docs/web
-
-WEBDIR=$$HOME/vcs/www.nico.schottelius.org
-WEBBLOG=$(WEBDIR)/blog
-WEBBASE=$(WEBDIR)/software/cdist
-WEBPAGE=$(WEBBASE).mdwn
-
-CHANGELOG_VERSION=$(shell $(helper) changelog-version)
-CHANGELOG_FILE=docs/changelog
-
-PYTHON_VERSION=cdist/version.py
+DOCS_SRC_DIR=./docs/src
+SPEECHDIR=./docs/speeches
+TYPEDIR=./cdist/conf/type
 
 SPHINXM=make -C $(DOCS_SRC_DIR) man
 SPHINXH=make -C $(DOCS_SRC_DIR) html
 SPHINXC=make -C $(DOCS_SRC_DIR) clean
 
-SHELLCHECKCMD=shellcheck -s sh -f gcc -x
-# Skip SC2154 for variables starting with __ since such variables are cdist
-# environment variables.
-SHELLCHECK_SKIP=grep -v ': __.*is referenced but not assigned.*\[SC2154\]'
 ################################################################################
 # Manpages
 #
-MAN1DSTDIR=$(DOCS_SRC_DIR)/man1
 MAN7DSTDIR=$(DOCS_SRC_DIR)/man7
 
 # Manpages #1: Types
@@ -69,11 +63,16 @@ DOCSREFSH=$(DOCS_SRC_DIR)/cdist-reference.rst.sh
 $(DOCSREF): $(DOCSREFSH)
 	$(DOCSREFSH)
 
+version:
+	@[ -f "cdist/version.py" ] || { \
+		printf "Missing 'cdist/version.py', please generate it first.\n" && exit 1; \
+	}
+
 # Manpages #3: generic part
-man: $(MANTYPES) $(DOCSREF) $(PYTHON_VERSION)
+man: version $(MANTYPES) $(DOCSREF)
 	$(SPHINXM)
 
-html: $(MANTYPES) $(DOCSREF) $(PYTHON_VERSION)
+html: version $(MANTYPES) $(DOCSREF)
 	$(SPHINXH)
 
 docs: man html
@@ -81,24 +80,6 @@ docs: man html
 docs-clean:
 	$(SPHINXC)
 
-# Manpages #5: release part
-MANWEBDIR=$(WEBBASE)/man/$(CHANGELOG_VERSION)
-HTMLBUILDDIR=docs/dist/html
-
-docs-dist: html
-	rm -rf "${MANWEBDIR}"
-	mkdir -p "${MANWEBDIR}"
-	# mkdir -p "${MANWEBDIR}/man1" "${MANWEBDIR}/man7"
-	# cp ${MAN1DSTDIR}/*.html ${MAN1DSTDIR}/*.css ${MANWEBDIR}/man1
-	# cp ${MAN7DSTDIR}/*.html ${MAN7DSTDIR}/*.css ${MANWEBDIR}/man7
-	cp -R ${HTMLBUILDDIR}/* ${MANWEBDIR}
-	cd ${MANWEBDIR} && git add . && git commit -m "cdist manpages update: $(CHANGELOG_VERSION)" || true
-
-man-latest-link: web-pub
-	# Fix ikiwiki, which does not like symlinks for pseudo security
-	ssh staticweb.ungleich.ch \
-		"cd /home/services/www/nico/nico.schottelius.org/www/software/cdist/man/ && rm -f latest && ln -sf "$(CHANGELOG_VERSION)" latest"
-
 # Manpages: .cdist Types
 DOT_CDIST_PATH=${HOME}/.cdist
 DOTMAN7DSTDIR=$(MAN7DSTDIR)
@@ -111,8 +92,7 @@ DOTMANTYPES=$(subst /man.rst,.rst,$(DOTMANTYPEPREFIX))
 $(DOTMAN7DSTDIR)/cdist-type%.rst: $(DOTTYPEDIR)/%/man.rst
 	ln -sf "$^" $@
 
-# Manpages #3: generic part
-dotman: $(DOTMANTYPES)
+dotman: version $(DOTMANTYPES)
 	$(SPHINXM)
 
 ################################################################################
@@ -120,7 +100,6 @@ dotman: $(DOTMANTYPES)
 #
 SPEECHESOURCES=$(SPEECHDIR)/*.tex
 SPEECHES=$(SPEECHESOURCES:.tex=.pdf)
-SPEECHESWEBDIR=$(WEBBASE)/speeches
 
 # Create speeches and ensure Toc is up-to-date
 $(SPEECHDIR)/%.pdf: $(SPEECHDIR)/%.tex
@@ -130,160 +109,26 @@ $(SPEECHDIR)/%.pdf: $(SPEECHDIR)/%.tex
 
 speeches: $(SPEECHES)
 
-speeches-dist: speeches
-	rm -rf "${SPEECHESWEBDIR}"
-	mkdir -p "${SPEECHESWEBDIR}"
-	cp ${SPEECHES} "${SPEECHESWEBDIR}"
-	cd ${SPEECHESWEBDIR} && git add . && git commit -m "cdist speeches updated" || true
-
 ################################################################################
-# Website
+# Misc
 #
-
-BLOGFILE=$(WEBBLOG)/cdist-$(CHANGELOG_VERSION)-released.mdwn
-
-$(BLOGFILE): $(CHANGELOG_FILE)
-	$(helper) blog $(CHANGELOG_VERSION) $(BLOGFILE)
-
-web-blog: $(BLOGFILE)
-
-web-doc:
-	# Go to top level, because of cdist.mdwn
-	rsync -av "$(WEBSRCDIR)/" "${WEBBASE}/.."
-	cd "${WEBBASE}/.." && git add cdist* && git commit -m "cdist doc update" cdist* || true
-
-web-dist: web-blog web-doc
-
-web-pub: web-dist docs-dist speeches-dist
-	cd "${WEBDIR}" && make pub
-
-web-release-all: man-latest-link
-web-release-all-no-latest: web-pub
-
-################################################################################
-# Release: Mailinglist
-#
-ML_FILE=.lock-ml
-
-# Only send mail once - lock until new changelog things happened
-$(ML_FILE): $(CHANGELOG_FILE)
-	$(helper) ml-release $(CHANGELOG_VERSION)
-	touch $@
-
-ml-release: $(ML_FILE)
-
-
-################################################################################
-# pypi
-#
-PYPI_FILE=.pypi-release
-$(PYPI_FILE): man $(PYTHON_VERSION)
-	python3 setup.py sdist upload
-	touch $@
-
-pypi-release: $(PYPI_FILE)
-################################################################################
-# archlinux
-#
-ARCHLINUX_FILE=.lock-archlinux
-ARCHLINUXTAR=cdist-$(CHANGELOG_VERSION)-1.src.tar.gz
-
-$(ARCHLINUXTAR): PKGBUILD
-	umask 022; mkaurball
-
-PKGBUILD: PKGBUILD.in $(PYTHON_VERSION)
-	./PKGBUILD.in $(CHANGELOG_VERSION)
-
-$(ARCHLINUX_FILE): $(ARCHLINUXTAR) $(PYTHON_VERSION)
-	burp -c system $(ARCHLINUXTAR)
-	touch $@
-
-archlinux-release: $(ARCHLINUX_FILE)
-
-################################################################################
-# Release
-#
-
-$(PYTHON_VERSION) version: .git/refs/heads/master
-	$(helper) version
-
-# Code that is better handled in a shell script
-check-%:
-	$(helper) $@
-
-release:
-	$(helper) $@
-
-################################################################################
-# Cleanup
-#
-
-clean:
+clean: docs-clean
 	rm -f $(DOCS_SRC_DIR)/cdist-reference.rst
 
 	find "$(DOCS_SRC_DIR)" -mindepth 2 -type l \
 	| xargs rm -f
 
-	make -C $(DOCS_SRC_DIR) clean
-
 	find * -name __pycache__  | xargs rm -rf
 
-	# Archlinux
-	rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz
-	rm -rf pkg/ src/
-
-	rm -f MANIFEST PKGBUILD
-	rm -rf dist/
-
-	# Signed release
-	rm -f cdist-*.tar.gz
-	rm -f cdist-*.tar.gz.asc
-
-distclean: clean
-	rm -f cdist/version.py
+	# distutils
+	rm -rf ./build
 
 ################################################################################
-# Misc
+# install
 #
 
-# The pub is Nico's "push to all git remotes" way ("make pub")
-pub:
-	git push --mirror
+install:
+	python3 setup.py install
 
-test:
-	$(helper) $@
-
-test-remote:
-	$(helper) $@
-
-pycodestyle pep8:
-	$(helper) $@
-
-shellcheck-global-explorers:
-	@find cdist/conf/explorer -type f -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0
-
-shellcheck-type-explorers:
-	@find cdist/conf/type -type f -path "*/explorer/*" -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0
-
-shellcheck-manifests:
-	@find cdist/conf/type -type f -name manifest -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0
-
-shellcheck-local-gencodes:
-	@find cdist/conf/type -type f -name gencode-local -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0
-
-shellcheck-remote-gencodes:
-	@find cdist/conf/type -type f -name gencode-remote -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0
-
-shellcheck-scripts:
-	@$(SHELLCHECKCMD) scripts/cdist-dump || exit 0
-
-shellcheck-gencodes: shellcheck-local-gencodes shellcheck-remote-gencodes
-
-shellcheck-types: shellcheck-type-explorers shellcheck-manifests shellcheck-gencodes
-
-shellcheck: shellcheck-global-explorers shellcheck-types shellcheck-scripts
-
-shellcheck-type-files:
-	@find cdist/conf/type -type f -path "*/files/*" -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0
-
-shellcheck-with-files: shellcheck shellcheck-type-files
+install-user:
+	python3 setup.py install --user

PKGBUILD.in

@@ -9,7 +9,7 @@ pkgver=$version
 pkgrel=1
 pkgdesc='A Usable Configuration Management System"'
 arch=('any')
-url='http://www.nico.schottelius.org/software/cdist/'
+url='https://www.cdi.st/'
 license=('GPL3')
 depends=('python>=3.2.0')
 source=("http://pypi.python.org/packages/source/c/cdist/cdist-\${pkgver}.tar.gz")

README

@@ -3,4 +3,5 @@ cdist
 
 cdist is a usable configuration management system.
 
-For the web documentation have a look at docs/web/.
+For the web documentation have a look at https://www.cdi.st/
+or at docs/src for reStructuredText manual.

README-maintainers

@@ -0,0 +1,4 @@
+Maintainers should use ./bin/build-helper script.
+
+Makefile is intended for end users. It can be used for non-maintaining
+targets that can be run from pure source (without git repository).

bin/build-helper

@@ -1,6 +1,7 @@
 #!/bin/sh
 #
 # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2016-2019 Darko Poljak (darko.poljak at gmail.com)
 #
 # This file is part of cdist.
 #
@@ -18,17 +19,66 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# This file contains the heavy lifting found usually in the Makefile
+# This file contains the heavy lifting found usually in the Makefile.
 #
 
-basedir=${0%/*}/../
-# Change to checkout directory
-cd "$basedir"
+usage() {
+    printf "usage: %s TARGET [TARGET-ARGS...]
+    Available targets:
+        changelog-changes
+        changelog-version
+        check-date
+        check-unittest
+        ml-release
+        archlinux-release
+        pypi-release
+        release-git-tag
+        sign-git-release
+        release
+        test
+        test-remote
+        pycodestyle
+        pep8
+        check-pycodestyle
+        shellcheck-global-explorers
+        shellcheck-type-explorers
+        shellcheck-manifests
+        shellcheck-local-gencodes
+        shellcheck-remote-gencodes
+        shellcheck-scripts
+        shellcheck-gencodes
+        shellcheck-types
+        shellcheck
+        shellcheck-type-files
+        shellcheck-with-files
+        shellcheck-build-helper
+        check-shellcheck
+        version-branch
+        version
+        target-version
+        clean
+        distclean\n" "$1"
+}
 
-version=$(git describe)
+basename="${0##*/}"
+
+if [ $# -lt 1 ]
+then
+    usage "${basename}"
+    exit 1
+fi
 
 option=$1; shift
 
+SHELLCHECKCMD="shellcheck -s sh -f gcc -x"
+# Skip SC2154 for variables starting with __ since such variables are cdist
+# environment variables.
+SHELLCHECK_SKIP=': __.*is referenced but not assigned.*\[SC2154\]'
+
+# Change to checkout directory
+basedir="${0%/*}/../"
+cd "$basedir"
+
 case "$option" in
     changelog-changes)
         if [ "$#" -eq 1 ]; then
@@ -66,8 +116,8 @@ case "$option" in
         date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //')
 
         if [ "$date_today" != "$date_changelog" ]; then
-            echo "Date in changelog is not today"
-            echo "Changelog: $date_changelog"
+            printf "Date in changelog is not today\n"
+            printf "Changelog date: %s\n" "${date_changelog}"
             exit 1
         fi
     ;;
@@ -76,54 +126,17 @@ case "$option" in
         "$0" test
     ;;
 
-    blog)
-        version=$1; shift
-        blogfile=$1; shift
-        dir=${blogfile%/*}
-        file=${blogfile##*/}
-
-
-        cat << eof > "$blogfile"
-[[!meta title="Cdist $version released"]]
-
-Here's a short overview about the changes found in version ${version}:
-
-eof
-
-        $0 changelog-changes "$version" >> "$blogfile"
-
-        cat << eof >> "$blogfile"
-For more information visit the [[cdist homepage|software/cdist]].
-
-[[!tag cdist config unix]]
-eof
-        cd "$dir"
-        git add "$file"
-        # Allow git commit to fail if there are no changes
-        git commit -m "cdist blog update: $version" "$blogfile" || true
-    ;;
-
     ml-release)
         if [ $# -ne 1 ]; then
-            echo "$0 ml-release version" >&2
+            printf "%s ml-release version\n" "$0" >&2
             exit 1
         fi
 
         version=$1; shift
 
-        to_a=cdist
-        to_d=l.schottelius.org
-        to=${to_a}@${to_d}
-
-        from_a=nico-cdist
-        from_d=schottelius.org
-        from=${from_a}@${from_d}
-
         ( 
         cat << eof
-From: Nico -telmich- Schottelius <$from>
-To: cdist mailing list <$to>
-Subject: cdist $version released
+Subject: cdist $version has been released
 
 Hello .*,
 
@@ -134,25 +147,41 @@ eof
         "$0" changelog-changes "$version"
         cat << eof
 
-Cheers,
-
-Nico
-
--- 
-Automatisation at its best level. With cdist.
 eof
-        ) | /usr/sbin/sendmail -f "$from" "$to"
+        ) > mailinglist.tmp
     ;;
 
+    archlinux-release)
+        if [ $# -ne 1 ]; then
+            printf "%s archlinux-release version\n" "$0" >&2
+            exit 1
+        fi
+        version=$1; shift
+
+        ARCHLINUXTAR="cdist-${version}-1.src.tar.gz"
+        ./PKGBUILD.in "${version}"
+        umask 022
+        mkaurball
+        burp -c system "${ARCHLINUXTAR}"
+    ;;
+
+    pypi-release)
+        # Ensure that pypi release has the right version
+        "$0" version
+
+        make docs-clean
+        make docs
+        python3 setup.py sdist upload
+    ;;
 
     release-git-tag)
         target_version=$($0 changelog-version)
-        if git rev-parse --verify refs/tags/$target_version 2>/dev/null; then
-            echo "Tag for $target_version exists, aborting"
+        if git rev-parse --verify "refs/tags/${target_version}" 2>/dev/null; then
+            printf "Tag for %s exists, aborting\n" "${target_version}"
             exit 1
         fi
-        printf "Enter tag description for ${target_version}: "
-        read tagmessage
+        printf "Enter tag description for %s: " "${target_version}"
+        read -r tagmessage
 
         # setup for signed tags:
         # gpg --fulL-gen-key
@@ -170,7 +199,8 @@ eof
         # gpg --verify <asc-file> <file>
         # gpg --no-default-keyring --keyring <pubkey.gpg> --verify <asc-file> <file>
         # Ensure gpg-agent is running.
-        export GPG_TTY=$(tty)
+        GPG_TTY=$(tty)
+        export GPG_TTY
         gpg-agent
 
         git tag -s "$target_version" -m "$tagmessage"
@@ -180,14 +210,14 @@ eof
     sign-git-release)
         if [ $# -lt 2 ]
         then
-            printf "usage: $0 sign-git-release TAG TOKEN [ARCHIVE]\n"
+            printf "usage: %s sign-git-release TAG TOKEN [ARCHIVE]\n" "$0"
             printf "    if ARCHIVE is not specified then it is created\n"
             exit 1
         fi
         tag="$1"
         if ! git rev-parse -q --verify "${tag}" >/dev/null 2>&1
         then
-            printf "Tag \"${tag}\" not found.\n"
+            printf "Tag \"%s\" not found.\n" "${tag}"
             exit 1
         fi
         token="$2"
@@ -209,34 +239,35 @@ eof
         fi
         gpg --armor --detach-sign "${archivename}" || exit 1
 
-        # make github release
-        curl -H "Authorization: token ${token}" \
-            --request POST \
-            --data "{ \"tag_name\":\"${tag}\", \
-                      \"target_commitish\":\"master\", \
-                      \"name\": \"${tag}\", \
-                      \"body\":\"${tag}\", \
-                      \"draft\":false, \
-                      \"prerelease\": false}" \
-            "https://api.github.com/repos/ungleich/cdist/releases" || exit 1
+        project="ungleich-public%2Fcdist"
+        sed_cmd='s/^.*"markdown":"\([^"]*\)".*$/\1/'
 
-        # get release ID
-        repoid=$(curl "https://api.github.com/repos/ungleich/cdist/releases/tags/${tag}" \
-            | python3 -c 'import json; import sys; print(json.loads(sys.stdin.read())["id"])') \
-            || exit 1
+        # upload archive
+        response_archive=$(curl -f -X POST \
+             --http1.1 \
+             -H "PRIVATE-TOKEN: ${token}" \
+             -F "file=@${archivename}" \
+             "https://code.ungleich.ch/api/v4/projects/${project}/uploads" \
+             | sed "${sed_cmd}") || exit 1
 
-        # upload archive and then signature
-        curl -H "Authorization: token ${token}" \
-             -H "Accept: application/vnd.github.manifold-preview" \
-             -H "Content-Type: application/x-gtar" \
-             --data-binary @${archivename} \
-            "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}" \
-            || exit 1
-        curl -H "Authorization: token ${token}" \
-             -H "Accept: application/vnd.github.manifold-preview" \
-             -H "Content-Type: application/pgp-signature" \
-             --data-binary @${archivename}.asc \
-            "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}.asc" \
+        # upload archive signature
+        response_archive_sig=$(curl -f -X POST \
+             --http1.1 \
+             -H "PRIVATE-TOKEN: ${token}" \
+             -F "file=@${archivename}.asc" \
+            "https://code.ungleich.ch/api/v4/projects/${project}/uploads" \
+             | sed "${sed_cmd}") || exit 1
+
+        # make release
+        changelog=$("$0" changelog-changes "$1" | sed 's/^[[:space:]]*//')
+        release_notes=$(
+            printf "%s\n\n%s\n\n**Changelog**\n\n%s\n" \
+                "${response_archive}" "${response_archive_sig}" "${changelog}"
+        )
+        curl -f -X POST \
+             -H "PRIVATE-TOKEN: ${token}" \
+             -F "description=${release_notes}" \
+            "https://code.ungleich.ch/api/v4/projects/${project}/repository/tags/${tag}/release" \
             || exit 1
 
         # remove generated files (archive and asc)
@@ -252,30 +283,30 @@ eof
         target_version=$($0 changelog-version)
         target_branch=$($0 version-branch)
 
-        echo "Beginning release process for $target_version"
+        printf "Beginning release process for %s\n" "${target_version}"
 
         # First check everything is sane
         "$0" check-date
         "$0" check-unittest
         "$0" check-pycodestyle
-        "$0" shellcheck
+        "$0" check-shellcheck
 
         # Generate version file to be included in packaging
         "$0" target-version
 
         # Ensure the git status is clean, else abort
         if ! git diff-index --name-only --exit-code HEAD ; then
-            echo "Unclean tree, see files above, aborting"
+            printf "Unclean tree, see files above, aborting.\n"
             exit 1
         fi
 
         # Ensure we are on the master branch
         masterbranch=yes
         if [ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then
-            echo "Releases are happening from the master branch, aborting"
+            printf "Releases are happening from the master branch, aborting.\n"
 
-            echo "Enter the magic word to release anyway"
-            read magicword
+            printf "Enter the magic word to release anyway:"
+            read -r magicword
 
             if [ "$magicword" = "iknowwhatido" ]; then
                 masterbranch=no
@@ -286,7 +317,7 @@ eof
 
         if [ "$masterbranch" = yes ]; then
             # Ensure version branch exists
-            if ! git rev-parse --verify refs/heads/$target_branch 2>/dev/null; then
+            if ! git rev-parse --verify "refs/heads/${target_branch}" 2>/dev/null; then
                 git branch "$target_branch"
             fi
 
@@ -304,20 +335,12 @@ eof
         make docs-clean
         make docs
 
-        # Generate speeches (indirect check if they build)
-        make speeches
-
         ############################################################# 
         # Everything green, let's do the release
 
         # Tag the current commit
         "$0" release-git-tag
 
-        # sign git tag
-        printf "Enter github authentication token: "
-        read token
-        "$0" sign-git-release "${target_version}" "${token}"
-
         # Also merge back the version branch
         if [ "$masterbranch" = yes ]; then
             git checkout master
@@ -325,41 +348,41 @@ eof
         fi
 
         # Publish git changes
-        make pub
-
-        # publish man, speeches, website
-        if [ "$masterbranch" = yes ]; then
-            make web-release-all
-        else
-            make web-release-all-no-latest
-        fi
-
-        # Ensure that pypi release has the right version
-        "$0" version
+        # if you want to have mirror locally then uncomment this and comment below
+        #     git push --mirror
+            git push
+            # push also new branch and set up tracking
+            git push -u origin "${target_branch}"
+        # fi
 
         # Create and publish package for pypi
-        make pypi-release
+        "$0" pypi-release
 
-        # Archlinux release is based on pypi
-        make archlinux-release
+        # sign git tag
+        printf "Enter upstream repository authentication token: "
+        read -r token
+        "$0" sign-git-release "${target_version}" "${token}"
 
         # Announce change on ML
-        make ml-release
+        "$0" ml-release "${target_version}"
 
         cat << eof
 Manual steps post release:
-
-    - linkedin
-    - hackernews
-    - reddit
+    - cdist-web
+    - send mail body generated in mailinglist.tmp and inform Dmitry for deb
     - twitter
-
 eof
-
     ;;
 
     test)
-        export PYTHONPATH="$(pwd -P)"
+        if [ ! -f "cdist/version.py" ]
+        then
+            printf "cdist/version.py is missing, generate it first.\n"
+            exit 1
+        fi
+
+        PYTHONPATH="$(pwd -P)"
+        export PYTHONPATH
 
         if [ $# -lt 1 ]; then
             python3 -m cdist.test
@@ -369,7 +392,15 @@ eof
     ;;
 
     test-remote)
-        export PYTHONPATH="$(pwd -P)"
+        if [ ! -f "cdist/version.py" ]
+        then
+            printf "cdist/version.py is missing, generate it first.\n"
+            exit 1
+        fi
+
+        PYTHONPATH="$(pwd -P)"
+        export PYTHONPATH
+
         python3 -m cdist.test.exec.remote
     ;;
 
@@ -382,9 +413,9 @@ eof
         printf "\\nPlease review pycodestyle report.\\n"
         while true
         do
-            echo "Continue (yes/no)?"
+            printf "Continue (yes/no)?\n"
             any=
-            read any
+            read -r any
             case "$any" in
                 yes)
                     break
@@ -393,20 +424,74 @@ eof
                     exit 1
                 ;;
                 *)
-                    echo "Please answer with 'yes' or 'no' explicitly."
+                    printf "Please answer with 'yes' or 'no' explicitly.\n"
                 ;;
         esac
         done
     ;;
 
+    shellcheck-global-explorers)
+        find cdist/conf/explorer -type f -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-type-explorers)
+        find cdist/conf/type -type f -path "*/explorer/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-manifests)
+        find cdist/conf/type -type f -name manifest -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-local-gencodes)
+        find cdist/conf/type -type f -name gencode-local -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-remote-gencodes)
+        find cdist/conf/type -type f -name gencode-remote -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-scripts)
+        ${SHELLCHECKCMD} scripts/cdist-dump scripts/cdist-new-type || exit 0
+    ;;
+
+    shellcheck-gencodes)
+        "$0" shellcheck-local-gencodes
+        "$0" shellcheck-remote-gencodes
+    ;;
+
+    shellcheck-types)
+        "$0" shellcheck-type-explorers
+        "$0" shellcheck-manifests
+        "$0" shellcheck-gencodes
+    ;;
+
     shellcheck)
-        make helper=${helper} WEBDIR=${WEBDIR} shellcheck
+        "$0" shellcheck-global-explorers
+        "$0" shellcheck-types
+        "$0" shellcheck-scripts
+    ;;
+
+    shellcheck-type-files)
+        find cdist/conf/type -type f -path "*/files/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-with-files)
+        "$0" shellcheck
+        "$0" shellcheck-type-files
+    ;;
+
+    shellcheck-build-helper)
+        ${SHELLCHECKCMD} ./bin/build-helper
+    ;;
+
+    check-shellcheck)
+        "$0" shellcheck
         printf "\\nPlease review shellcheck report.\\n"
         while true
         do
-            echo "Continue (yes/no)?"
+            printf "Continue (yes/no)?\n"
             any=
-            read any
+            read -r any
             case "$any" in
                 yes)
                     break
@@ -415,7 +500,7 @@ eof
                     exit 1
                 ;;
                 *)
-                    echo "Please answer with 'yes' or 'no' explicitly."
+                    printf "Please answer with 'yes' or 'no' explicitly.\n"
                 ;;
         esac
         done
@@ -426,16 +511,39 @@ eof
     ;;
 
     version)
-        echo "VERSION = \"$(git describe)\"" > cdist/version.py
+        printf "VERSION = \"%s\"\n" "$(git describe)" > cdist/version.py
     ;;
 
     target-version)
         target_version=$($0 changelog-version)
-        echo "VERSION = \"${target_version}\"" > cdist/version.py
+        printf "VERSION = \"%s\"\n" "${target_version}" > cdist/version.py
     ;;
 
+    clean)
+        make clean
+
+        # Archlinux
+        rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz
+        rm -rf pkg/ src/
+
+        rm -f MANIFEST PKGBUILD
+        rm -rf dist/
+
+        # Signed release
+        rm -f cdist-*.tar.gz
+        rm -f cdist-*.tar.gz.asc
+
+        # Temp files
+        rm -f ./*.tmp
+    ;;
+
+    distclean)
+        "$0" clean
+        rm -f cdist/version.py
+    ;;
     *)
-        echo "Unknown helper target $@ - aborting"
+        printf "Unknown target: '%s'.\n" "${option}" >&2
+        usage "${basename}"
         exit 1
     ;;
 

bin/build-helper.freebsd

@@ -1,482 +0,0 @@
-#!/bin/sh
-#
-# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
-# 2016 Darko Poljak (darko.poljak at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# This file contains the heavy lifting found usually in the Makefile
-#
-
-# vars for make
-helper=$0
-
-basedir=${0%/*}/../
-# run_as is used to check how the script is called (by $0 value)
-# currently supported sufixes for $0 are:
-# .freebsd - run as freebsd
-basename=${0##*/}
-run_as=${basename#*.}
-case "$run_as" in
-    freebsd)
-        to_a=cdist-configuration-management
-        to_d=googlegroups.com
-        from_a=darko.poljak
-        from_d=gmail.com
-        ml_name="Darko Poljak"
-        ml_sig_name="Darko"
-
-        # vars for make
-        WEBDIR=../vcs/www.nico.schottelius.org
-    ;;
-    *)
-        to_a=cdist
-        to_d=l.schottelius.org
-        from_a=nico-cdist
-        from_d=schottelius.org
-        ml_name="Nico -telmich- Schottelius"
-        ml_sig_name="Nico"
-
-        # vars for make
-        WEBDIR=$$HOME/vcs/www.nico.schottelius.org
-    ;;
-esac
-
-# Change to checkout directory
-cd "$basedir"
-
-version=$(git describe)
-
-option=$1; shift
-
-case "$option" in
-    print-make-vars)
-        printf "helper: ${helper}\n"
-        printf "WEBDIR: ${WEBDIR}\n"
-    ;;
-    print-runas)
-        printf "run_as: $run_as\n"
-    ;;
-    changelog-changes)
-        if [ "$#" -eq 1 ]; then
-            start=$1
-        else
-            start="[[:digit:]]"
-        fi
-
-        end="[[:digit:]]"
-
-        awk -F: "BEGIN { start=0 }
-            {
-                if(start == 0) {
-                    if (\$0 ~ /^$start/) {
-                        start = 1
-                    }
-                } else {
-                    if (\$0 ~ /^$end/) {
-                        exit
-                    } else {
-                        print \$0 
-                    }
-                }
-            }" "$basedir/docs/changelog"
-    ;;
-
-    changelog-version)
-        # get version from changelog
-        grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/:.*//'
-    ;;
-
-    check-date)
-        # verify date in changelog is today
-        date_today="$(date +%Y-%m-%d)"
-        date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //')
-
-        if [ "$date_today" != "$date_changelog" ]; then
-            echo "Date in changelog is not today"
-            echo "Changelog: $date_changelog"
-            exit 1
-        fi
-    ;;
-
-    check-unittest)
-        "$0" test
-    ;;
-
-    blog)
-        version=$1; shift
-        blogfile=$1; shift
-        dir=${blogfile%/*}
-        file=${blogfile##*/}
-
-
-        cat << eof > "$blogfile"
-[[!meta title="Cdist $version released"]]
-
-Here's a short overview about the changes found in version ${version}:
-
-eof
-
-        $0 changelog-changes "$version" >> "$blogfile"
-
-        cat << eof >> "$blogfile"
-For more information visit the [[cdist homepage|software/cdist]].
-
-[[!tag cdist config unix]]
-eof
-        cd "$dir"
-        git add "$file"
-        # Allow git commit to fail if there are no changes
-        git commit -m "cdist blog update: $version" "$blogfile" || true
-    ;;
-
-    ml-release)
-        if [ $# -ne 1 ]; then
-            echo "$0 ml-release version" >&2
-            exit 1
-        fi
-
-        version=$1; shift
-
-        to=${to_a}@${to_d}
-        from=${from_a}@${from_d}
-
-        ( 
-        cat << eof
-From: ${ml_name} <$from>
-To: cdist mailing list <$to>
-Subject: cdist $version released
-
-Hello .*,
-
-cdist $version has been released with the following changes:
-
-eof
-
-        "$0" changelog-changes "$version"
-        cat << eof
-
-Cheers,
-
-${ml_sig_name}
-
--- 
-Automatisation at its best level. With cdist.
-eof
-        ) | /usr/sbin/sendmail -f "$from" "$to"
-    ;;
-
-    release-git-tag)
-        target_version=$($0 changelog-version)
-        if git rev-parse --verify refs/tags/$target_version 2>/dev/null; then
-            echo "Tag for $target_version exists, aborting"
-            exit 1
-        fi
-        printf "Enter tag description for ${target_version}: "
-        read tagmessage
-
-        # setup for signed tags:
-        # gpg --fulL-gen-key
-        # gpg --list-secret-keys --keyid-format LONG
-        # git config --local user.signingkey <id>
-        # for exporting pub key:
-        #     gpg --armor --export <id> > pubkey.asc
-        #     gpg --output pubkey.gpg --export <id>
-        # show tag with signature
-        # git show <tag>
-        # verify tag signature
-        # git tag -v <tag>
-        #
-        # gpg verify signature
-        # gpg --verify <asc-file> <file>
-        # gpg --no-default-keyring --keyring <pubkey.gpg> --verify <asc-file> <file>
-        # Ensure gpg-agent is running.
-        export GPG_TTY=$(tty)
-        gpg-agent
-
-        git tag -s "$target_version" -m "$tagmessage"
-        git push --tags
-    ;;
-
-    sign-git-release)
-        if [ $# -lt 2 ]
-        then
-            printf "usage: $0 sign-git-release TAG TOKEN [ARCHIVE]\n"
-            printf "    if ARCHIVE is not specified then it is created\n"
-            exit 1
-        fi
-        tag="$1"
-        if ! git rev-parse -q --verify "${tag}" >/dev/null 2>&1
-        then
-            printf "Tag \"${tag}\" not found.\n"
-            exit 1
-        fi
-        token="$2"
-        if [ $# -gt 2 ]
-        then
-            archivename="$3"
-        else
-            archivename="cdist-${tag}.tar"
-            git archive --prefix="cdist-${tag}/" -o "${archivename}" "${tag}" \
-                || exit 1
-            # make sure target version is generated
-            "$0" target-version
-            tar -x -f "${archivename}" || exit 1
-            cp cdist/version.py "cdist-${tag}/cdist/version.py" || exit 1
-            tar -c -f "${archivename}" "cdist-${tag}/" || exit 1
-            rm -r -f "cdist-${tag}/"
-            gzip "${archivename}" || exit 1
-            archivename="${archivename}.gz"
-        fi
-        gpg --armor --detach-sign "${archivename}" || exit 1
-
-        # make github release
-        curl -H "Authorization: token ${token}" \
-            --request POST \
-            --data "{ \"tag_name\":\"${tag}\", \
-                      \"target_commitish\":\"master\", \
-                      \"name\": \"${tag}\", \
-                      \"body\":\"${tag}\", \
-                      \"draft\":false, \
-                      \"prerelease\": false}" \
-            "https://api.github.com/repos/ungleich/cdist/releases" || exit 1
-
-        # get release ID
-        repoid=$(curl "https://api.github.com/repos/ungleich/cdist/releases/tags/${tag}" \
-            | python3 -c 'import json; import sys; print(json.loads(sys.stdin.read())["id"])') \
-            || exit 1
-
-        # upload archive and then signature
-        curl -H "Authorization: token ${token}" \
-             -H "Accept: application/vnd.github.manifold-preview" \
-             -H "Content-Type: application/x-gtar" \
-             --data-binary @${archivename} \
-            "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}" \
-            || exit 1
-        curl -H "Authorization: token ${token}" \
-             -H "Accept: application/vnd.github.manifold-preview" \
-             -H "Content-Type: application/pgp-signature" \
-             --data-binary @${archivename}.asc \
-            "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}.asc" \
-            || exit 1
-
-        # remove generated files (archive and asc)
-        if [ $# -eq 2 ]
-        then
-            rm -f "${archivename}"
-        fi
-        rm -f "${archivename}.asc"
-    ;;
-
-    release)
-        set -e
-        target_version=$($0 changelog-version)
-        target_branch=$($0 version-branch)
-
-        echo "Beginning release process for $target_version"
-
-        # First check everything is sane
-        "$0" check-date
-        "$0" check-unittest
-        "$0" check-pycodestyle
-        "$0" shellcheck
-
-        # Generate version file to be included in packaging
-        "$0" target-version
-
-        # Ensure the git status is clean, else abort
-        if ! git diff-index --name-only --exit-code HEAD ; then
-            echo "Unclean tree, see files above, aborting"
-            exit 1
-        fi
-
-        # Ensure we are on the master branch
-        masterbranch=yes
-        if [ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then
-            echo "Releases are happening from the master branch, aborting"
-
-            echo "Enter the magic word to release anyway"
-            read magicword
-
-            if [ "$magicword" = "iknowwhatido" ]; then
-                masterbranch=no
-            else
-                exit 1
-            fi
-        fi
-
-        if [ "$masterbranch" = yes ]; then
-            # Ensure version branch exists
-            if ! git rev-parse --verify refs/heads/$target_branch 2>/dev/null; then
-                git branch "$target_branch"
-            fi
-
-            # Merge master branch into version branch
-            git checkout "$target_branch"
-            git merge master
-        fi
-
-        # Verify that after the merge everything works
-        "$0" check-date
-        "$0" check-unittest
-
-        # Generate documentation (man and html)
-        # First, clean old generated docs
-        make helper=${helper} WEBDIR=${WEBDIR} docs-clean
-        make helper=${helper} WEBDIR=${WEBDIR} docs
-
-        # Generate speeches (indirect check if they build)
-        make helper=${helper} WEBDIR=${WEBDIR} speeches
-
-        ############################################################# 
-        # Everything green, let's do the release
-
-        # Tag the current commit
-        "$0" release-git-tag
-
-        # sign git tag
-        printf "Enter github authentication token: "
-        read token
-        "$0" sign-git-release "${target_version}" "${token}"
-
-        # Also merge back the version branch
-        if [ "$masterbranch" = yes ]; then
-            git checkout master
-            git merge "$target_branch"
-        fi
-
-        # Publish git changes
-        case "$run_as" in
-            freebsd)
-                # if we are not Nico :) then just push, no mirror
-                git push
-                # push also new branch and set up tracking
-                git push -u origin "${target_branch}"
-            ;;
-            *)
-                make helper=${helper} WEBDIR=${WEBDIR} pub
-            ;;
-        esac
-
-        # Ensure that pypi release has the right version
-        "$0" version
-
-        # Create and publish package for pypi
-        make helper=${helper} WEBDIR=${WEBDIR} pypi-release
-
-        # publish man, speeches, website
-        if [ "$masterbranch" = yes ]; then
-            make helper=${helper} WEBDIR=${WEBDIR} web-release-all
-        else
-            make helper=${helper} WEBDIR=${WEBDIR} web-release-all-no-latest
-        fi
-
-        case "$run_as" in
-            freebsd)
-            ;;
-            *)
-                # Archlinux release is based on pypi
-                make archlinux-release
-            ;;
-        esac
-
-        # Announce change on ML
-        make helper=${helper} WEBDIR=${WEBDIR} ml-release
-    ;;
-
-    test)
-        export PYTHONPATH="$(pwd -P)"
-
-        if [ $# -lt 1 ]; then
-            python3 -m cdist.test
-        else
-            python3 -m unittest "$@"
-        fi
-    ;;
-
-    test-remote)
-        export PYTHONPATH="$(pwd -P)"
-        python3 -m cdist.test.exec.remote
-    ;;
-
-    pycodestyle|pep8)
-        pycodestyle "${basedir}" "${basedir}/scripts/cdist" | less
-    ;;
-
-    check-pycodestyle)
-        "$0" pycodestyle
-        printf "\\nPlease review pycodestyle report.\\n"
-        while true
-        do
-            echo "Continue (yes/no)?"
-            any=
-            read any
-            case "$any" in
-                yes)
-                    break
-                ;;
-                no)
-                    exit 1
-                ;;
-                *)
-                    echo "Please answer with 'yes' or 'no' explicitly."
-                ;;
-        esac
-        done
-    ;;
-
-    shellcheck)
-        make helper=${helper} WEBDIR=${WEBDIR} shellcheck
-        printf "\\nPlease review shellcheck report.\\n"
-        while true
-        do
-            echo "Continue (yes/no)?"
-            any=
-            read any
-            case "$any" in
-                yes)
-                    break
-                ;;
-                no)
-                    exit 1
-                ;;
-                *)
-                    echo "Please answer with 'yes' or 'no' explicitly."
-                ;;
-        esac
-        done
-    ;;
-
-    version-branch)
-        "$0" changelog-version | cut -d. -f '1,2'
-    ;;
-
-    version)
-        echo "VERSION = \"$(git describe)\"" > cdist/version.py
-    ;;
-
-    target-version)
-        target_version=$($0 changelog-version)
-        echo "VERSION = \"${target_version}\"" > cdist/version.py
-    ;;
-
-    *)
-        echo "Unknown helper target $@ - aborting"
-        exit 1
-    ;;
-
-esac

cdist/argparse.py

@@ -11,9 +11,9 @@ import cdist.configuration
 BETA_COMMANDS = set(('install', 'inventory', ))
 # set of beta arguments for sub-commands
 BETA_ARGS = {
-    'config': set(('jobs', 'tag', 'all_tagged_hosts', 'use_archiving', )),
+    'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
 }
-EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
+EPILOG = "Get cdist at https://code.ungleich.ch/ungleich-public/cdist"
 # Parser others can reuse
 parser = None
 
@@ -191,8 +191,7 @@ def get_parsers():
                                   name="positive int"),
            help=('Operate in parallel in specified maximum number of jobs. '
                  'Global explorers, object prepare and object run are '
-                 'supported. Without argument CPU count is used by default. '
-                 'Currently in beta.'),
+                 'supported. Without argument CPU count is used by default. '),
            action='store', dest='jobs',
            const=multiprocessing.cpu_count())
     parser['config_main'].add_argument(

cdist/conf/explorer/init

@@ -29,7 +29,7 @@ case "$uname_s" in
     Linux)
         (pgrep -P0 -l | awk '/^1[ \t]/ {print $2;}') || true
     ;;
-    FreeBSD)
+    FreeBSD|OpenBSD)
         ps -o comm= -p 1 || true
     ;;
     *)

cdist/conf/type/__acl/explorer/checks

@@ -18,30 +18,22 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-[ ! -e "/$__object_id" ] && exit 0
+# TODO check if filesystem has ACL turned on etc
 
-for parameter in user group
-do
-    if [ ! -f "$__object/parameter/$parameter" ]
-    then
-        continue
-    fi
-
-    while read -r acl
+if [ -f "$__object/parameter/acl" ]
+then
+    grep -E '^(default:)?(user|group):' "$__object/parameter/acl" \
+    | while read -r acl
     do
-        check="$( echo "$acl" | awk -F: '{print $1}' )"
+        param="$( echo "$acl" | awk -F: '{print $(NF-2)}' )"
+        check="$( echo "$acl" | awk -F: '{print $(NF-1)}' )"
 
-        if [ "$parameter" = 'user' ]
-        then
-            getent_db=passwd
-        else
-            getent_db="$parameter"
-        fi
+        [ "$param" = 'user' ] && db=passwd || db="$param"
 
-        if ! getent "$getent_db" "$check" > /dev/null
+        if ! getent "$db" "$check" > /dev/null
         then
-            echo "missing $parameter '$check'"
+            echo "missing $param '$check'" >&2
+            exit 1
         fi
-    done \
-        < "$__object/parameter/$parameter"
-done
+    done
+fi

cdist/conf/type/__acl/gencode-remote

@@ -20,59 +20,65 @@
 
 file_is="$( cat "$__object/explorer/file_is" )"
 
-[ "$file_is" = 'missing' ] && exit 0
-
-missing_users_groups="$( cat "$__object/explorer/missing_users_groups" )"
-
-if [ -n "$missing_users_groups" ]
-then
-    echo "$missing_users_groups" >&2
-    exit 1
-fi
+[ "$file_is" = 'missing' ] && [ -z "$__cdist_dry_run" ] && exit 0
 
 os="$( cat "$__global/explorer/os" )"
 
-acl_is="$( cat "$__object/explorer/acl_is" )"
-
 acl_path="/$__object_id"
 
-if [ -f "$__object/parameter/default" ] && [ "$file_is" = 'directory' ]
+acl_is="$( cat "$__object/explorer/acl_is" )"
+
+if [ -f "$__object/parameter/acl" ]
 then
-    set_default=1
+    acl_should="$( cat "$__object/parameter/acl" )"
+elif
+    [ -f "$__object/parameter/user" ] \
+        || [ -f "$__object/parameter/group" ] \
+        || [ -f "$__object/parameter/mask" ] \
+        || [ -f "$__object/parameter/other" ]
+then
+    acl_should="$( for param in user group mask other
+    do
+        [ ! -f "$__object/parameter/$param" ] && continue
+
+        echo "$param" | grep -Eq 'mask|other' && sep=:: || sep=:
+
+        echo "$param$sep$( cat "$__object/parameter/$param" )"
+    done )"
 else
-    set_default=0
+    echo 'no parameters set' >&2
+    exit 1
 fi
 
-acl_should="$( for parameter in user group mask other
-do
-    if [ ! -f "$__object/parameter/$parameter" ]
-    then
-        continue
-    fi
+if [ -f "$__object/parameter/default" ]
+then
+    acl_should="$( echo "$acl_should" \
+        | sed 's/^default://' \
+        | sort -u \
+        | sed 's/\(.*\)/default:\1\n\1/' )"
+fi
 
-    while read -r acl
-    do
-        if echo "$acl" | awk -F: '{ print $NF }' | grep -Fq 'X'
-        then
-            [ "$file_is" = 'directory' ] && rep=x || rep=-
+if [ "$file_is" = 'regular' ] \
+    && echo "$acl_should" | grep -Eq '^default:'
+then
+    # only directories can have default ACLs,
+    # but instead of error,
+    # let's just remove default entries
+    acl_should="$( echo "$acl_should" | grep -Ev '^default:' )"
+fi
 
-            acl="$( echo "$acl" | sed "s/\(.*\)X/\1$rep/" )"
-        fi
+if echo "$acl_should" | awk -F: '{ print $NF }' | grep -Fq 'X'
+then
+    [ "$file_is" = 'directory' ] && rep=x || rep=-
 
-        echo "$parameter" | grep -Eq '(mask|other)' && sep=:: || sep=:
-
-        echo "$parameter$sep$acl"
-
-        [ "$set_default" = '1' ] && echo "default:$parameter$sep$acl"
-    done \
-        < "$__object/parameter/$parameter"
-done )"
+    acl_should="$( echo "$acl_should" | sed "s/\\(.*\\)X/\\1$rep/" )"
+fi
 
 setfacl_exec='setfacl'
 
 if [ -f "$__object/parameter/recursive" ]
 then
-    if echo "$os" | grep -Eq 'macosx|freebsd'
+    if echo "$os" | grep -Fq 'freebsd'
     then
         echo "$os setfacl do not support recursive operations" >&2
     else
@@ -82,44 +88,36 @@ fi
 
 if [ -f "$__object/parameter/remove" ]
 then
-    if echo "$os" | grep -Fq 'solaris'
-    then
-        # Solaris setfacl behaves differently.
-        # We will not support Solaris for now, because no way to test it.
-        # But adding support should be easy (use -s instead of -m on modify).
-        echo "$os setfacl do not support -x flag for ACL remove" >&2
-    else
-        echo "$acl_is" | while read -r acl
-        do
-            # Skip wanted ACL entries which already exist
-            # and skip mask and other entries, because we
-            # can't actually remove them, but only change.
-            if echo "$acl_should" | grep -Eq "^$acl" \
-                || echo "$acl" | grep -Eq '^(default:)?(mask|other)'
-            then continue
-            fi
+    echo "$acl_is" | while read -r acl
+    do
+        # skip wanted ACL entries which already exist
+        # and skip mask and other entries, because we
+        # can't actually remove them, but only change.
+        if echo "$acl_should" | grep -Eq "^$acl" \
+            || echo "$acl" | grep -Eq '^(default:)?(mask|other)'
+        then continue
+        fi
 
-            if echo "$os" | grep -Eq 'macosx|freebsd'
-            then
-                remove="$acl"
-            else
-                remove="$( echo "$acl" | sed 's/:...$//' )"
-            fi
+        if echo "$os" | grep -Fq 'freebsd'
+        then
+            remove="$acl"
+        else
+            remove="$( echo "$acl" | sed 's/:...$//' )"
+        fi
 
-            echo "$setfacl_exec -x \"$remove\" \"$acl_path\""
-            echo "removed '$remove'" >> "$__messages_out"
-        done
-    fi
+        echo "$setfacl_exec -x \"$remove\" \"$acl_path\""
+        echo "removed '$remove'" >> "$__messages_out"
+    done
 fi
 
 for acl in $acl_should
 do
     if ! echo "$acl_is" | grep -Eq "^$acl"
     then
-        if echo "$os" | grep -Eq 'macosx|freebsd' \
+        if echo "$os" | grep -Fq 'freebsd' \
             && echo "$acl" | grep -Eq '^default:'
         then
-            echo "setting default ACL in $os is currently not supported. sorry :(" >&2
+            echo "setting default ACL in $os is currently not supported" >&2
         else
             echo "$setfacl_exec -m \"$acl\" \"$acl_path\""
             echo "added '$acl'" >> "$__messages_out"

cdist/conf/type/__acl/man.rst

@@ -8,46 +8,36 @@ cdist-type__acl - Set ACL entries
 
 DESCRIPTION
 -----------
-ACL must be defined as 3-symbol combination, using ``r``, ``w``, ``x`` and ``-``.
-
-Fully supported on Linux (tested on Debian and CentOS).
-
-Partial support for FreeBSD, OSX and Solaris.
-
-OpenBSD and NetBSD support is not possible.
+Fully supported and tested on Linux (ext4 filesystem), partial support for FreeBSD.
 
 See ``setfacl`` and ``acl`` manpages for more details.
 
 
-OPTIONAL MULTIPLE PARAMETERS
+REQUIRED MULTIPLE PARAMETERS
 ----------------------------
-user
-   Add user ACL entry.
-
-group
-   Add group ACL entry.
-
-
-OPTIONAL PARAMETERS
--------------------
-mask
-   Add mask ACL entry.
-
-other
-   Add other ACL entry.
+acl
+   Set ACL entry following ``getfacl`` output syntax.
 
 
 BOOLEAN PARAMETERS
 ------------------
+default
+   Set all ACL entries as default too.
+   Only directories can have default ACLs.
+   Setting default ACL in FreeBSD is currently not supported.
+
 recursive
    Make ``setfacl`` recursive (Linux only), but not ``getfacl`` in explorer.
 
-default
-   Add default ACL entries (FreeBSD not supported).
-
 remove
-   Remove undefined ACL entries (Solaris not supported).
-   ACL entries for ``mask`` and ``other`` can't be removed.
+   Remove undefined ACL entries.
+   ``mask`` and ``other`` entries can't be removed, but only changed.
+
+
+DEPRECATED PARAMETERS
+---------------------
+Parameters ``user``, ``group``, ``mask`` and ``other`` are deprecated and they
+will be removed in future versions. Please use ``acl`` parameter instead.
 
 
 EXAMPLES
@@ -56,15 +46,30 @@ EXAMPLES
 .. code-block:: sh
 
     __acl /srv/project \
+        --default \
         --recursive \
+        --remove \
+        --acl user:alice:rwx \
+        --acl user:bob:r-x \
+        --acl group:project-group:rwx \
+        --acl group:some-other-group:r-x \
+        --acl mask::r-x \
+        --acl other::r-x
+
+    # give Alice read-only access to subdir,
+    # but don't allow her to see parent content.
+
+    __acl /srv/project2 \
+        --remove \
+        --acl default:group:secret-project:rwx \
+        --acl group:secret-project:rwx \
+        --acl user:alice:--x
+
+    __acl /srv/project2/subdir \
         --default \
         --remove \
-        --user alice:rwx \
-        --user bob:r-x \
-        --group project-group:rwx \
-        --group some-other-group:r-x \
-        --mask r-x \
-        --other r-x
+        --acl group:secret-project:rwx \
+        --acl user:alice:r-x
 
 
 AUTHORS

cdist/conf/type/__acl/parameter/deprecated/group

@@ -0,0 +1 @@
+see manual for details

cdist/conf/type/__acl/parameter/deprecated/mask

@@ -0,0 +1 @@
+see manual for details

cdist/conf/type/__acl/parameter/deprecated/other

@@ -0,0 +1 @@
+see manual for details

cdist/conf/type/__acl/parameter/deprecated/user

@@ -0,0 +1 @@
+see manual for details

cdist/conf/type/__acl/parameter/optional_multiple

@@ -1,2 +1,3 @@
+acl
 user
 group

cdist/conf/type/__apt_key/explorer/state

@@ -27,6 +27,18 @@ else
    keyid="$__object_id"
 fi
 
-apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \
-   && echo present \
-   || echo absent
+keydir="$(cat "$__object/parameter/keydir")"
+keyfile="$keydir/$__object_id.gpg"
+
+if [ -d "$keydir" ]
+then
+   if [ -f "$keyfile" ]
+   then echo present
+   else echo absent
+   fi
+else
+   # fallback to deprecated apt-key
+   apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \
+      && echo present \
+      || echo absent
+fi

cdist/conf/type/__apt_key/gencode-remote

@@ -31,12 +31,84 @@ if [ "$state_should" = "$state_is" ]; then
    exit 0
 fi
 
+keydir="$(cat "$__object/parameter/keydir")"
+keyfile="$keydir/$__object_id.gpg"
+
 case "$state_should" in
    present)
       keyserver="$(cat "$__object/parameter/keyserver")"
-      echo "apt-key adv --keyserver \"$keyserver\" --recv-keys \"$keyid\""
+
+      if [ -f "$__object/parameter/uri" ]; then
+         uri="$(cat "$__object/parameter/uri")"
+
+         if [ -d "$keydir" ]; then
+            cat << EOF
+
+curl -s -L \\
+    -o "$keyfile" \\
+    "$uri"
+
+if grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK' \\
+    "$keyfile"
+then
+    cat "$keyfile" \\
+        | gpg --export > "$keyfile"
+fi
+
+EOF
+         else
+            # fallback to deprecated apt-key
+            echo "curl -s -L '$uri' | apt-key add -"
+         fi
+      elif [ -d "$keydir" ]; then
+         tmp='/tmp/cdist_apt_key_tmp'
+
+         # we need to kill gpg after 30 seconds, because gpg
+         # can get stuck if keyserver is not responding.
+         # exporting env var and not exit 1,
+         # because we need to clean up and kill dirmngr.
+         cat << EOF
+
+mkdir -m 700 -p "$tmp"
+
+if timeout 30s \\
+    gpg --homedir "$tmp" \\
+        --keyserver "$keyserver" \\
+        --recv-keys "$keyid"
+then
+    gpg --homedir "$tmp" \\
+        --export "$keyid" \\
+        > "$keyfile"
+else
+    export GPG_GOT_STUCK=1
+fi
+
+GNUPGHOME="$tmp" gpgconf --kill dirmngr
+
+rm -rf "$tmp"
+
+if [ -n "\$GPG_GOT_STUCK" ]
+then
+    echo "GPG GOT STUCK - no response from keyserver after 30 seconds" >&2
+    exit 1
+fi
+
+EOF
+      else
+         # fallback to deprecated apt-key
+         echo "apt-key adv --keyserver \"$keyserver\" --recv-keys \"$keyid\""
+      fi
+
+      echo "added '$keyid'" >> "$__messages_out"
    ;;
    absent)
-      echo "apt-key del \"$keyid\""
+      if [ -f "$keyfile" ]; then
+         echo "rm '$keyfile'"
+      else
+         # fallback to deprecated apt-key
+         echo "apt-key del \"$keyid\""
+      fi
+
+      echo "removed '$keyid'" >> "$__messages_out"
    ;;
 esac

cdist/conf/type/__apt_key/man.rst

@@ -28,6 +28,12 @@ keyserver
    the keyserver from which to fetch the key. If omitted the default set
    in ./parameter/default/keyserver is used.
 
+keydir
+   key save location, defaults to ``/etc/apt/trusted.pgp.d``
+
+uri
+   the URI from which to download the key
+
 
 EXAMPLES
 --------
@@ -47,15 +53,20 @@ EXAMPLES
     # same thing with other keyserver
     __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com
 
+    # download key from the internet
+    __apt_key rabbitmq \
+       --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc
+
 
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
+Ander Punnar <ander-at-kvlt-dot-ee>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2014 Steven Armstrong. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
+Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can
+redistribute it and/or modify it under the terms of the GNU General Public
+License as published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.

cdist/conf/type/__apt_key/manifest

@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+__package gnupg
+
+if [ -f "$__object/parameter/uri" ]
+then __package curl
+else __package dirmngr
+fi

cdist/conf/type/__apt_key/parameter/default/keydir

@@ -0,0 +1 @@
+/etc/apt/trusted.gpg.d

cdist/conf/type/__apt_key/parameter/optional

@@ -1,3 +1,5 @@
 state
 keyid
 keyserver
+keydir
+uri

cdist/conf/type/__block/gencode-remote

@@ -18,6 +18,11 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+# quote function from http://www.etalabs.net/sh_tricks.html
+quote() {
+   printf '%s\n' "$1" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
+}
+
 file="$(cat "$__object/parameter/file" 2>/dev/null || echo "/$__object_id")"
 state_should=$(cat "$__object/parameter/state")
 prefix=$(cat "$__object/parameter/prefix" 2>/dev/null || echo "#cdist:__block/$__object_id")
@@ -46,7 +51,7 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
 if [ -f "$file" ]; then
    cp -p "$file" "\$tmpfile"
 fi
-awk -v prefix="^$prefix\$" -v suffix="^$suffix\$" '
+awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
 {
    if (match(\$0,prefix)) {
       triggered=1

cdist/conf/type/__cdist/man.rst

@@ -30,7 +30,7 @@ username
 
 source
     Select the source from which to clone cdist from.
-    Defaults to "git://github.com/ungleich/cdist.git".
+    Defaults to "git@code.ungleich.ch:ungleich-public/cdist.git".
 
 
 branch
@@ -47,7 +47,7 @@ EXAMPLES
     __cdist /home/cdist/cdist
 
     # Use alternative source
-    __cdist --source "git://github.com/ungleich/cdist" /home/cdist/cdist
+    __cdist --source "git@code.ungleich.ch:ungleich-public/cdist.git" /home/cdist/cdist
 
 
 AUTHORS

cdist/conf/type/__cdist/parameter/default/source

@@ -1 +1 @@
-git://github.com/ungleich/cdist.git
+git@code.ungleich.ch:ungleich-public/cdist.git

cdist/conf/type/__consul/files/versions/1.5.0/cksum

@@ -0,0 +1 @@
+886614099 103959898 consul

cdist/conf/type/__consul/files/versions/1.5.0/source

@@ -0,0 +1 @@
+https://releases.hashicorp.com/consul/1.5.0/consul_1.5.0_linux_amd64.zip

cdist/conf/type/__consul/gencode-remote

@@ -42,7 +42,7 @@ source_file_name="${source##*/}"
 cksum_should=$(cut -d' ' -f1,2 "$version_dir/cksum")
 
 cat << eof
-    tmpdir=\$(mktemp -d --tmpdir="/tmp" "${__type##*/}.XXXXXXXXXX")
+    tmpdir=\$(mktemp -d -p /tmp "${__type##*/}.XXXXXXXXXX")
     curl -s -L "$source" > "\$tmpdir/$source_file_name"
     unzip -p "\$tmpdir/$source_file_name" > "${destination}.tmp"
     rm -rf "\$tmpdir"

cdist/conf/type/__consul/manifest

@@ -24,7 +24,7 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-   scientific|centos|redhat|ubuntu|debian|devuan|archlinux|gentoo)
+   alpine|scientific|centos|redhat|ubuntu|debian|devuan|archlinux|gentoo)
       # any linux should work
       :
    ;;
@@ -47,6 +47,7 @@ fi
 
 if [ -f "$__object/parameter/direct" ]; then
     __package unzip
+    __package curl
 else
     __staged_file /usr/local/bin/consul \
        --source "$(cat "$version_dir/source")" \

cdist/conf/type/__consul_agent/files/consul.sys-openrc

@@ -0,0 +1,38 @@
+#!/sbin/openrc-run
+# 2019 Nico Schottelius (nico-cdist at schottelius.org)
+
+description="consul agent"
+
+pidfile="${CONSUL_PIDFILE:-"/var/run/$RC_SVCNAME/pidfile"}"
+command="${CONSUL_BINARY:-"/usr/local/bin/consul"}"
+
+
+checkconfig() {
+	if [ ! -d /var/run/consul ] ; then
+		mkdir -p /var/run/consul || return 1
+        chown consul:consul /var/run/$NAME || return 1
+        chmod 2770 /var/run/$NAME || return 1
+	fi
+}
+
+start() {
+    need net
+
+    start-stop-daemon --start --quiet --oknodo \
+            --pidfile "$pidfile" --background \
+            --exec $command -- agent -pid-file="$pidfile" -config-dir /etc/consul/conf.d
+}
+start_pre() {
+	checkconfig
+}
+
+stop() {
+	if [ "${RC_CMD}" = "restart" ] ; then
+		checkconfig || return 1
+	fi
+
+	ebegin "Stopping $RC_SVCNAME"
+	start-stop-daemon --stop --exec "$command" \
+		--pidfile "$pidfile" --quiet
+	eend $?
+}

cdist/conf/type/__consul_agent/manifest

@@ -1,7 +1,7 @@
 #!/bin/sh -e
 #
 # 2015 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2015 Nico Schottelius (nico-cdist at schottelius.org)
+# 2015-2019 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -23,7 +23,7 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-   scientific|centos|debian|devuan|redhat|ubuntu)
+   alpine|scientific|centos|debian|devuan|redhat|ubuntu)
       # whitelist safeguard
       :
    ;;
@@ -181,22 +181,25 @@ init_upstart()
 
 # Install init script to start on boot
 case "$os" in
-   centos|redhat)
-      os_version="$(sed 's/[^0-9.]//g' "$__global/explorer/os_version")"
-      major_version="${os_version%%.*}"
-      case "$major_version" in
-         [456])
-            init_sysvinit redhat
-         ;;
-         7)
-            init_systemd
-         ;;
-         *)
-            echo "Unsupported CentOS/Redhat version: $os_version" >&2
-            exit 1
-         ;;
-      esac
-   ;;
+    alpine|devuan)
+        init_sysvinit debian
+        ;;
+    centos|redhat)
+        os_version="$(sed 's/[^0-9.]//g' "$__global/explorer/os_version")"
+        major_version="${os_version%%.*}"
+        case "$major_version" in
+            [456])
+                init_sysvinit redhat
+                ;;
+            7)
+                init_systemd
+                ;;
+            *)
+                echo "Unsupported CentOS/Redhat version: $os_version" >&2
+                exit 1
+                ;;
+        esac
+        ;;
 
     debian)
         os_version=$(cat "$__global/explorer/os_version")
@@ -214,13 +217,9 @@ case "$os" in
                 exit 1
             ;;
         esac
-    ;;
-
-    devuan)
-        init_sysvinit debian
-    ;;
+        ;;
 
     ubuntu)
         init_upstart
-    ;;
+        ;;
 esac

cdist/conf/type/__docker/manifest

@@ -64,6 +64,43 @@ case "$os" in
         require="__apt_source/docker" __package docker-ce --state "${state}"
       fi
    ;;
+   devuan)
+      os_version="$(cat "$__global/explorer/os_version")"
+
+      case "$os_version" in
+        ascii)
+          distribution="stretch"
+        ;;
+        jessie)
+          distribution="jessie"
+        ;;
+        *)
+          echo "Your devuan release ($os_version) is currently not supported by this type (${__type##*/}).">&2
+          echo "Please contribute an implementation for it if you can." >&2
+          exit 1
+        ;;
+      esac
+
+      if [ "${state}" = "present" ]; then
+        __package apt-transport-https
+        __package ca-certificates
+        __package gnupg2
+      fi
+      __apt_key_uri docker --name "Docker Release (CE deb) <docker@docker.com>" \
+        --uri "https://download.docker.com/linux/${os}/gpg" --state "${state}"
+
+      require="__apt_key_uri/docker" __apt_source docker \
+         --uri "https://download.docker.com/linux/${os}" \
+         --distribution "${distribution}" \
+         --state "${state}" \
+         --component "stable"
+      if [ "$version" != "latest" ]; then
+        require="__apt_source/docker" __package docker-ce --version "${version}" --state "${state}"
+      else
+        require="__apt_source/docker" __package docker-ce --state "${state}"
+      fi
+
+   ;;
    *)
       echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
       echo "Please contribute an implementation for it if you can." >&2

cdist/conf/type/__docker_swarm/explorer/swarm-state

@@ -18,4 +18,4 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-docker info 2>/dev/null | grep "^Swarm: " | cut -d " " -f 2-
+docker info 2>/dev/null | grep '^ *Swarm: ' | awk '{print $2}'

cdist/conf/type/__git/man.rst

@@ -44,7 +44,7 @@ EXAMPLES
     __git /home/services/dokuwiki --source git://github.com/splitbrain/dokuwiki.git
 
     # Checkout cdist, stay on branch 2.1
-    __git /home/nico/cdist --source git://github.com/ungleich/cdist.git --branch 2.1
+    __git /home/nico/cdist --source git@code.ungleich.ch:ungleich-public/cdist.git --branch 2.1
 
 
 AUTHORS

cdist/conf/type/__grafana_dashboard/manifest

@@ -8,10 +8,12 @@ case $os in
     debian|devuan)
         case $os_version in
             8*|jessie)
-                apt_source_distribution=jessie
+                # Differntation not needed anymore
+                apt_source_distribution=stable
                 ;;
             9*|ascii/ceres|ascii)
-                apt_source_distribution=stretch
+                # Differntation not needed anymore
+                apt_source_distribution=stable
                 ;;
             *)
                 echo "Don't know how to install Grafana on $os $os_version. Send us a pull request!" >&2
@@ -21,10 +23,10 @@ case $os in
 
         __apt_key_uri grafana \
             --name 'Grafana Release Signing Key' \
-            --uri https://packagecloud.io/gpg.key
+            --uri https://packages.grafana.com/gpg.key
 
         require="$require __apt_key_uri/grafana" __apt_source grafana \
-                    --uri https://packagecloud.io/grafana/stable/debian/ \
+                    --uri https://packages.grafana.com/oss/deb \
                     --distribution $apt_source_distribution \
                     --component main
 

cdist/conf/type/__letsencrypt_cert/manifest

@@ -62,11 +62,12 @@ if [ -z "${certbot_fullpath}" ]; then
 								 --distribution ascii-backports \
 								 --component main
 
-					require="__apt_source/ascii-backports" __package_apt python-certbot \
-						--target-release ascii-backports
 					require="__apt_source/ascii-backports" __package_apt certbot \
 						--target-release ascii-backports
 				;;
+				bewoulf*)
+					__package_apt certbot
+				;;
 				*)
 					echo "Unsupported OS version: $os_version" >&2
 					exit 1

cdist/conf/type/__package_apk/explorer/state

@@ -27,6 +27,10 @@ else
    name="$__object_id"
 fi
 
+# Remove the @.. repo tag for finding out whether it is installed
+# f.i. pass@testing => pass
+name="$(echo "$name" | sed 's/@.*//')"
+
 if [ "$(apk list -I "$name")" ]; then
     echo present
 else

cdist/conf/type/__postfix/manifest

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2012-2014 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2019 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -22,7 +23,7 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-   ubuntu|debian|archlinux|suse|scientific|centos|devuan)
+   alpine|ubuntu|debian|archlinux|suse|scientific|centos|devuan)
       __package postfix --state present
    ;;
    *)

cdist/conf/type/__postfix_postconf/explorer/value

@@ -22,7 +22,7 @@
 os=$("$__explorer/os")
 
 case "$os" in
-   ubuntu|debian|archlinux|suse|scientific|centos|devuan)
+   alpine|ubuntu|debian|archlinux|suse|scientific|centos|devuan)
       :
    ;;
    *)

cdist/conf/type/__postfix_postconf/gencode-remote

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2012-2014 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2019 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -21,7 +22,7 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-   ubuntu|debian|archlinux|suse|scientific|centos|devuan)
+   alpine|archlinux|centos|debian|devuan|suse|scientific|ubuntu)
       :
    ;;
    *)

cdist/conf/type/__postgres_database/explorer/state

@@ -34,7 +34,7 @@ esac
 
 name="$__object_id"
 
-if test -n "$(su - "$postgres_user" -c "psql postgres -tAc \"SELECT 1 FROM pg_database WHERE datname='$name'\"")"
+if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='$name'\"")"
 then
     echo 'present'
 else

cdist/conf/type/__postgres_role/explorer/state

@@ -34,7 +34,7 @@ esac
 
 name="$__object_id"
 
-if test -n "$(su - "$postgres_user" -c "psql postgres -tAc \"SELECT 1 FROM pg_roles WHERE rolname='$name'\"")"
+if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='$name'\"")"
 then
     echo 'present'
 else

cdist/conf/type/__postgres_role/gencode-remote

@@ -55,7 +55,7 @@ case "$state_should" in
         [ -n "$password" ] && password="PASSWORD '$password'"
 
         cmd="CREATE ROLE $name WITH $password $booleans"
-        echo "su - '$postgres_user' -c \"psql postgres -c \\\"$cmd\\\"\""
+        echo "su - '$postgres_user' -c \"psql postgres -wc \\\"$cmd\\\"\""
     ;;
     absent)
         echo "su - '$postgres_user' -c \"dropuser \\\"$name\\\"\""

cdist/conf/type/__user/manifest

@@ -0,0 +1,32 @@
+#!/bin/sh -e
+#
+# 2019 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Manage users.
+
+os=$(cat "$__global/explorer/os")
+
+case "$os" in
+    alpine)
+        __package shadow
+        ;;
+    *)
+        :
+        ;;
+esac

cdist/conf/type/__zypper_service/man.rst

@@ -46,10 +46,10 @@ EXAMPLES
     # Ensure that internal SLES11 SP3 RIS is in installed and all other services and repos are discarded
     __zypper_service INTERNAL_SLES11_SP3 --service_desc "Internal SLES11 SP3 RIS" --uri "http://path/to/your/ris/dir" --remove-all-other-services --remove-all-repos
 
-    # Ensure that internal SLES11 SP3 RIS is in installed, no changes to ohter services or repos
+    # Ensure that internal SLES11 SP3 RIS is in installed, no changes to other services or repos
     __zypper_service INTERNAL_SLES11_SP3 --service_desc "Internal SLES11 SP3 RIS" --uri "http://path/to/your/ris/dir"
 
-    # Drop service by uri, no changes to ohter services or repos
+    # Drop service by uri, no changes to other services or repos
     __zypper_service INTERNAL_SLES11_SP3 --state absent --uri "http://path/to/your/ris/dir"
 
 

cdist/config.py

@@ -43,6 +43,33 @@ from cdist import core, inventory
 from cdist.util.remoteutil import inspect_ssh_mux_opts
 
 
+def graph_check_cycle(graph):
+    # Start from each node in the graph and check for cycle starting from it.
+    for node in graph:
+        # Cycle path.
+        path = [node]
+        has_cycle = _graph_dfs_cycle(graph, node, path)
+        if has_cycle:
+            return has_cycle, path
+    return False, None
+
+
+def _graph_dfs_cycle(graph, node, path):
+    for neighbour in graph.get(node, ()):
+        # If node is already in path then this is cycle.
+        if neighbour in path:
+            path.append(neighbour)
+            return True
+        path.append(neighbour)
+        rv = _graph_dfs_cycle(graph, neighbour, path)
+        if rv:
+            return True
+        # Remove last item from list - neighbour whose DFS path we have have
+        # just checked.
+        del path[-1]
+    return False
+
+
 class Config(object):
     """Cdist main class to hold arbitrary data"""
 
@@ -77,9 +104,12 @@ class Config(object):
         self.remove_remote_files_dirs = remove_remote_files_dirs
 
         self.explorer = core.Explorer(self.local.target_host, self.local,
-                                      self.remote, jobs=self.jobs)
-        self.manifest = core.Manifest(self.local.target_host, self.local)
-        self.code = core.Code(self.local.target_host, self.local, self.remote)
+                                      self.remote, jobs=self.jobs,
+                                      dry_run=self.dry_run)
+        self.manifest = core.Manifest(self.local.target_host, self.local,
+                                      dry_run=self.dry_run)
+        self.code = core.Code(self.local.target_host, self.local, self.remote,
+                              dry_run=self.dry_run)
 
     def _init_files_dirs(self):
         """Prepare files and directories for the run"""
@@ -254,14 +284,14 @@ class Config(object):
                     cls.onehost(host, host_tags, host_base_path, hostdir,
                                 args, parallel=False,
                                 configuration=configuration)
-                except cdist.Error as e:
+                except cdist.Error:
                     failed_hosts.append(host)
         if args.parallel and len(process_args) == 1:
             log.debug("Only 1 host for parallel processing, doing it "
                       "sequentially")
             try:
                 cls.onehost(*process_args[0])
-            except cdist.Error as e:
+            except cdist.Error:
                 failed_hosts.append(host)
         elif args.parallel:
             log.trace("Multiprocessing start method is {}".format(
@@ -653,6 +683,28 @@ class Config(object):
         self.__dict__.update(state)
         self._open_logger()
 
+    def _validate_dependencies(self):
+        '''
+            Build dependency graph for unfinished objects and
+            check for cycles.
+        '''
+        graph = {}
+        for cdist_object in self.object_list():
+            obj_name = cdist_object.name
+            if obj_name not in graph:
+                graph[obj_name] = []
+            if cdist_object.state == cdist_object.STATE_DONE:
+                continue
+
+            for requirement in cdist_object.requirements_unfinished(
+                    cdist_object.requirements):
+                graph[obj_name].append(requirement.name)
+
+            for requirement in cdist_object.requirements_unfinished(
+                    cdist_object.autorequire):
+                graph[obj_name].append(requirement.name)
+        return graph_check_cycle(graph)
+
     def iterate_until_finished(self):
         """
             Go through all objects and solve them
@@ -662,6 +714,12 @@ class Config(object):
         objects_changed = True
 
         while objects_changed:
+            # Check for cycles as early as possible.
+            has_cycle, path = self._validate_dependencies()
+            if has_cycle:
+                raise cdist.UnresolvableRequirementsError(
+                    "Cycle detected in object dependencies:\n{}!".format(
+                        " -> ".join(path)))
             objects_changed = self.iterate_once()
 
         # Check whether all objects have been finished
@@ -700,8 +758,29 @@ class Config(object):
                     ("The requirements of the following objects could not be "
                      "resolved:\n%s") % ("\n".join(info_string)))
 
+    def _handle_deprecation(self, cdist_object):
+        cdist_type = cdist_object.cdist_type
+        deprecated = cdist_type.deprecated
+        if deprecated is not None:
+            if deprecated:
+                self.log.warning("Type %s is deprecated: %s", cdist_type.name,
+                                 deprecated)
+            else:
+                self.log.warning("Type %s is deprecated.", cdist_type.name)
+        for param in cdist_object.parameters:
+            if param in cdist_type.deprecated_parameters:
+                msg = cdist_type.deprecated_parameters[param]
+                if msg:
+                    format = "%s parameter of type %s is deprecated: %s"
+                    args = [param, cdist_type.name, msg]
+                else:
+                    format = "%s parameter of type %s is deprecated."
+                    args = [param, cdist_type.name]
+                self.log.warning(format, *args)
+
     def object_prepare(self, cdist_object, transfer_type_explorers=True):
         """Prepare object: Run type explorer + manifest"""
+        self._handle_deprecation(cdist_object)
         self.log.verbose("Preparing object {}".format(cdist_object.name))
         self.log.verbose(
             "Running manifest and explorers for " + cdist_object.name)

cdist/core/cdist_type.py

@@ -69,6 +69,7 @@ class CdistType(object):
         self.__optional_multiple_parameters = None
         self.__boolean_parameters = None
         self.__parameter_defaults = None
+        self.__deprecated_parameters = None
 
     def __hash__(self):
         return hash(self.name)
@@ -133,6 +134,17 @@ class CdistType(object):
            cannot run in parallel."""
         return os.path.isfile(os.path.join(self.absolute_path, "nonparallel"))
 
+    @property
+    def deprecated(self):
+        """Get type deprecation message. If message is None then type
+        is not deprecated."""
+        deprecated_path = os.path.join(self.absolute_path, "deprecated")
+        try:
+            with open(deprecated_path, 'r') as f:
+                return f.read()
+        except FileNotFoundError:
+            return None
+
     @property
     def explorers(self):
         """Return a list of available explorers"""
@@ -264,3 +276,23 @@ class CdistType(object):
             finally:
                 self.__parameter_defaults = defaults
         return self.__parameter_defaults
+
+    @property
+    def deprecated_parameters(self):
+        if not self.__deprecated_parameters:
+            deprecated = {}
+            try:
+                deprecated_dir = os.path.join(self.absolute_path,
+                                              "parameter",
+                                              "deprecated")
+                for name in cdist.core.listdir(deprecated_dir):
+                    try:
+                        with open(os.path.join(deprecated_dir, name)) as fd:
+                            deprecated[name] = fd.read().strip()
+                    except EnvironmentError:
+                        pass  # Swallow errors raised by open() or read()
+            except EnvironmentError:
+                pass  # Swallow error raised by os.listdir()
+            finally:
+                self.__deprecated_parameters = deprecated
+        return self.__deprecated_parameters

cdist/core/code.py

@@ -97,7 +97,7 @@ class Code(object):
 
     """
     # target_host is tuple (target_host, target_hostname, target_fqdn)
-    def __init__(self, target_host, local, remote):
+    def __init__(self, target_host, local, remote, dry_run=False):
         self.target_host = target_host
         self.local = local
         self.remote = remote
@@ -113,6 +113,9 @@ class Code(object):
                 local.log),
         }
 
+        if dry_run:
+            self.env['__cdist_dry_run'] = '1'
+
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.cdist_type
         script = os.path.join(self.local.type_path,

cdist/core/explorer.py

@@ -67,7 +67,7 @@ class Explorer(object):
     """Executes cdist explorers.
 
     """
-    def __init__(self, target_host, local, remote, jobs=None):
+    def __init__(self, target_host, local, remote, jobs=None, dry_run=False):
         self.target_host = target_host
 
         self._open_logger()
@@ -84,6 +84,10 @@ class Explorer(object):
             '__cdist_log_level_name': util.log_level_name_env_var_val(
                 self.log),
         }
+
+        if dry_run:
+            self.env['__cdist_dry_run'] = '1'
+
         self._type_explorers_transferred = []
         self.jobs = jobs
 

cdist/core/manifest.py

@@ -96,7 +96,7 @@ class Manifest(object):
     """Executes cdist manifests.
 
     """
-    def __init__(self, target_host, local):
+    def __init__(self, target_host, local, dry_run=False):
         self.target_host = target_host
         self.local = local
 
@@ -117,6 +117,9 @@ class Manifest(object):
                 self.log),
         }
 
+        if dry_run:
+            self.env['__cdist_dry_run'] = '1'
+
     def _open_logger(self):
         self.log = logging.getLogger(self.target_host[0])
 

cdist/emulator.py

@@ -319,7 +319,9 @@ class Emulator(object):
                                        lastcreatedtype)
                     else:
                         if 'require' in self.env:
-                            self.env['require'] += " " + lastcreatedtype
+                            appendix = " " + lastcreatedtype
+                            if appendix not in self.env['require']:
+                                self.env['require'] += appendix
                         else:
                             self.env['require'] = lastcreatedtype
                         self.log.debug(("Injecting require for "

cdist/inventory.py

@@ -315,7 +315,7 @@ class InventoryHost(Inventory):
         hostpath = self._host_path(host)
         self.log.trace("hostpath: {}".format(hostpath))
         if self.action == "add" and not os.path.exists(hostpath):
-                self._new_hostpath(hostpath)
+            self._new_hostpath(hostpath)
         else:
             if not os.path.isfile(hostpath):
                 raise cdist.Error(("Host path \'{}\' is"

cdist/test/cdist_type/__init__.py

@@ -123,6 +123,16 @@ class TypeTestCase(test.CdistTestCase):
         cdist_type = core.CdistType(base_path, '__not_nonparallel')
         self.assertFalse(cdist_type.is_nonparallel)
 
+    def test_deprecated(self):
+        base_path = fixtures
+        cdist_type = core.CdistType(base_path, '__deprecated')
+        self.assertIsNotNone(cdist_type.deprecated)
+
+    def test_not_deprecated(self):
+        base_path = fixtures
+        cdist_type = core.CdistType(base_path, '__not_deprecated')
+        self.assertIsNone(cdist_type.deprecated)
+
     def test_install_is_install(self):
         base_path = fixtures
         cdist_type = core.CdistType(base_path, '__install')
@@ -190,3 +200,18 @@ class TypeTestCase(test.CdistTestCase):
         self.assertEqual(
             list(sorted(cdist_type.parameter_defaults.keys())),
             ['bar', 'foo'])
+
+    def test_without_deprecated_parameters(self):
+        base_path = fixtures
+        cdist_type = core.CdistType(base_path,
+                                    '__without_deprecated_parameters')
+        self.assertEqual(cdist_type.deprecated_parameters, {})
+
+    def test_with_deprecated_parameters(self):
+        base_path = fixtures
+        cdist_type = core.CdistType(base_path, '__with_deprecated_parameters')
+        self.assertTrue('eggs' in cdist_type.deprecated_parameters)
+        self.assertTrue('spam' in cdist_type.deprecated_parameters)
+        self.assertEqual(cdist_type.deprecated_parameters['eggs'],
+                         'Deprecated')
+        self.assertEqual(cdist_type.deprecated_parameters['spam'], '')

cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated/eggs

@@ -0,0 +1 @@
+Deprecated

cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/optional

@@ -0,0 +1,3 @@
+spam
+eggs
+sausage

cdist/test/config/__init__.py

@@ -23,7 +23,6 @@
 
 import os
 import shutil
-import tempfile
 
 from cdist import test
 from cdist import core
@@ -212,7 +211,7 @@ class ConfigRunTestCase(test.CdistTestCase):
         dryrun.run()
         # if we are here, dryrun works like expected
 
-    def test_desp_resolver(self):
+    def test_deps_resolver(self):
         """Test to show dependency resolver warning message."""
         local = cdist.exec.local.Local(
             target_host=self.target_host,
@@ -229,6 +228,71 @@ class ConfigRunTestCase(test.CdistTestCase):
         config = cdist.config.Config(local, self.remote, dry_run=True)
         config.run()
 
+    def test_graph_check_cycle_empty(self):
+        graph = {}
+        has_cycle, path = cdist.config.graph_check_cycle(graph)
+        self.assertFalse(has_cycle)
+
+    def test_graph_check_cycle_1(self):
+        #
+        # a -> b -> c
+        #      |
+        #      +--> d -> e
+        graph = {
+            'a': ['b', ],
+            'b': ['c', 'd', ],
+            'd': ['e', ],
+        }
+        has_cycle, path = cdist.config.graph_check_cycle(graph)
+        self.assertFalse(has_cycle)
+
+    def test_graph_check_cycle_2(self):
+        #
+        # a -> b -> c
+        # /\        |
+        #  \        |
+        #   +-------+
+        graph = {
+            'a': ['b', ],
+            'b': ['c', ],
+            'c': ['a', ],
+        }
+        has_cycle, path = cdist.config.graph_check_cycle(graph)
+        self.assertTrue(has_cycle)
+        self.assertGreater(path.count(path[-1]), 1)
+
+    def test_graph_check_cycle_3(self):
+        #
+        # a -> b -> c
+        #  \        \
+        #   \        +--> g
+        #    \            /\
+        #     \           /|
+        #      +-> d -> e  |
+        #           \      |
+        #            + --> f
+        #
+        # h -> i --> j
+        # |    /\    |
+        # \/    |    \/
+        # n     m <- k
+        graph = {
+            'a': ['b', 'd', ],
+            'b': ['c', ],
+            'c': ['g', ],
+            'd': ['e', 'f', ],
+            'e': ['g', ],
+            'f': ['g', ],
+            'h': ['i', 'n', ],
+            'i': ['j', ],
+            'j': ['k', ],
+            'k': ['m', ],
+            'm': ['i', ],
+        }
+        has_cycle, path = cdist.config.graph_check_cycle(graph)
+        self.assertTrue(has_cycle)
+        self.assertGreater(path.count(path[-1]), 1)
+
 
 # Currently the resolving code will simply detect that this object does
 # not exist. It should probably check if the type is a singleton as well

docs/changelog

@@ -1,6 +1,57 @@
 Changelog
 ---------
 
+5.1.3: 2019-08-30
+	* Build: Overcome bash CDPATH when building docs (Dmitry Bogatov)
+	* Type __grafana_dashboard: Update distribution name, package signing key URI and repository URI (Dominique Roux)
+	* Type __letsencrypt_cert: Add Devuan Beowulf support (Nico Schottelius)
+	* Type __letsencrypt_cert: Fix Devuan Ascii: support (Nico Schottelius)
+	* Type __docker: Add devuan support (Dominique Roux)
+	* Type __docker_swarm: Fix for Docker 19.03 (Ľubomír Kučera)
+
+5.1.2: 2019-06-21
+	* Core: Add support for type parameters deprecation (Darko Poljak)
+	* Type __acl: Rewrite and improve (Ander Punnar)
+
+5.1.1: 2019-05-28
+	* Type __apt_key: Use gpg key, fallback to deprecated apt-key (Ander Punnar)
+	* Type __acl: Fix and improve (Ander Punnar)
+	* Documentation: Document type stdin inside loop caveats (Darko Poljak)
+
+5.1.0: 2019-05-22
+	* Type __consul: Add alpine support (Nico Schottelius)
+	* Type __consul: Add version 1.5.0 (Nico Schottelius)
+	* Type __consul_agent: Add alpine support (Nico Schottelius)
+	* New helper script: cdist-new-type (Steven Armstrong, Darko Poljak)
+	* Core: Add support for deprecated type marker (Darko Poljak)
+
+5.0.2: 2019-05-17
+	* Type __package_apk: Fix @repo handling in explorer (Nico Schottelius)
+	* Type __postfix: Add alpine support (Nico Schottelius)
+	* Type __postfix_postconf: Add alpine support (Nico Schottelius)
+	* Type __user: Add alpine support (Nico Schottelius)
+	* Core: Set __cdist_dry_run env var (Ander Punnar)
+
+5.0.1: 2019-05-09
+	* Documentation: Add 'Perils of CDIST_ORDER_DEPENDENCY' sub-section (Darko Poljak)
+	* Build: Clean and separate end user targets into Makefile and maintainer targets into build-helper (Darko Poljak)
+	* Core: Update residual references to old cdist homepage (Darko Poljak)
+	* Documentation: Update residual references to old cdist homepage and git source (Darko Poljak)
+	* Type __cdist: Fix non working 'git://' protocol source (Darko Poljak)
+
+5.0.0: 2019-05-05
+	* Type __zypper_service: Fix spelling error in manpage (Dmitry Bogatov)
+	* Explorer init: Add support for OpenBSD (sideeffect42)
+	* Type __postgres_database: Run psql with -w (no-password) (sideeffect42)
+	* Type __postgres_role: Run psql with -w (no-password) (sideeffect42)
+	* Type __block: Quote prefix/suffix - fix when prefix/suffix contains quotes (sideeffect42)
+	* Build: Update due to migration to code.ungleich.ch (Darko Poljak)
+	* Documentation: Update due to migration to code.ungleich.ch (Darko Poljak)
+	* Core: Detect and report dependency cycle as soon as possible (Darko Poljak)
+	* Core, documentation: Release -j/--jobs option, i.e. make it non-beta (Darko Poljak)
+	* Documentation: Update due to new cdist website (Darko Poljak)
+	* Build: Update due to new cdist website (Darko Poljak)
+
 4.11.1: 2019-04-22
 	* Core: Improve explorer error reporting (Darko Poljak)
 	* Type __directory: explorer stat: add support for Solaris (Ander Punnar)

docs/dev/github-gitlab-migration/release

@@ -0,0 +1,50 @@
+#!/bin/sh -e
+
+set -x
+
+printf "Enter tag name: "
+read tag
+printf "Enter repository authentication token: "
+read token
+
+git tag -d "${tag}" || :
+
+git tag "${tag}" -m "Release ${tag}"
+git push origin "${tag}"
+
+echo 'foo' > foo
+echo 'foo signature' > foo.asc
+
+archivename="foo"
+
+project="poljakowski%2Fmy-cdist-testing"
+sed_cmd='s/^.*"markdown":"\([^"]*\)".*$/\1/'
+
+# upload archive
+response_archive=$(curl -f -X POST \
+     -H "PRIVATE-TOKEN: ${token}" \
+     -F "file=@${archivename}" \
+     "https://code.ungleich.ch/api/v4/projects/${project}/uploads" \
+     | sed "${sed_cmd}") || exit 1
+
+# upload archive signature
+response_archive_sig=$(curl -f -X POST \
+     -H "PRIVATE-TOKEN: ${token}" \
+     -F "file=@${archivename}.asc" \
+    "https://code.ungleich.ch/api/v4/projects/${project}/uploads" \
+     | sed "${sed_cmd}") || exit 1
+
+# make release
+curl -f -X POST \
+     -H "PRIVATE-TOKEN: ${token}" \
+     -F "description=Release ${tag}<br/>${response_archive}<br/>${response_archive_sig}" \
+    "https://code.ungleich.ch/api/v4/projects/${project}/repository/tags/${tag}/release" \
+    || exit 1
+
+# get tag
+curl -f -X GET \
+     -H "PRIVATE-TOKEN: ${token}" \
+    "https://code.ungleich.ch/api/v4/projects/${project}/repository/tags/${tag}" \
+    || exit 1
+
+rm -f foo foo.asc

docs/src/cdist-best-practice.rst

@@ -224,3 +224,140 @@ in the repository for such content: It allows you to
 easily distinguish what is used by cdist and what is not
 and also to store all important files in one
 repository.
+
+
+Perils of CDIST_ORDER_DEPENDENCY
+--------------------------------
+With CDIST_ORDER_DEPENDENCY all types are executed in the order in which they
+are created in the manifest. The current created object automatically depends
+on the previously created object.
+
+It essentially helps you to build up blocks of code that build upon each other
+(like first creating the directory xyz than the file below the directory).
+
+This can be helpful, but it can also be the source of *evil*.
+
+
+CDIST_ORDER_DEPENDENCY easily causes unobvious dependency cycles
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Let's see an example. Suppose you have special init manifest where among other
+things you are assuring that remote host has packages `sudo` and `curl`
+installed.
+
+**init1**
+
+.. code-block:: sh
+
+   CDIST_ORDER_DEPENDENCY=1
+   export CDIST_ORDER_DEPENDENCY
+
+   for p in sudo curl
+   do
+      __package "${p}"
+   done
+
+Then you have some other special init manifest where among other things you are
+assuring `sudo` package is installed.
+
+**init2**
+
+.. code-block:: sh
+
+   CDIST_ORDER_DEPENDENCY=1
+   export CDIST_ORDER_DEPENDENCY
+
+   __package sudo
+
+Then you have third init manifest where you combine those two init manifests,
+by including them:
+
+**init**
+
+.. code-block:: sh
+
+   sh -e "$__manifest/init1"
+   sh -e "$__manifest/init2"
+
+The resulting init manifest is then equal to:
+
+.. code-block:: sh
+
+   CDIST_ORDER_DEPENDENCY=1
+   export CDIST_ORDER_DEPENDENCY
+
+   for p in sudo curl
+   do
+      __package "${p}"
+   done
+
+   CDIST_ORDER_DEPENDENCY=1
+   export CDIST_ORDER_DEPENDENCY
+
+   __package sudo
+
+In the end you get the following dependencies:
+
+* `__package/curl` depends on `__package/sudo`
+* `__package/sudo` depends on `__package/curl`
+
+And here you have a circular dependency!
+
+In the real world manifest can be quite complex, dependencies can become
+complicated and circual dependencies are not so obvious. Resolving it can
+become cumbersome.
+
+**Practical solution?**
+
+Instead of managing complex init manifests you can write custom types.
+Each custom type can do one thing, it has well defined dependencies that will
+not leak into init manifest. In custom type you can also add special explorers
+and gencode.
+
+Then, in init manifest you combine your complex types. It is:  
+
+* cleaner
+* easier to follow
+* easier to maintain
+* easier to debug.
+
+
+CDIST_ORDER_DEPENDENCY kills parallelization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Suppose you have defined CDIST_ORDER_DEPENDENCY and then, among other things,
+you specify creation of three, by nature independent, files.
+
+**init**
+
+.. code-block:: sh
+
+   CDIST_ORDER_DEPENDENCY=1
+   export CDIST_ORDER_DEPENDENCY
+
+   ...
+   __file /tmp/file1
+   __file /tmp/file2
+   __file /tmp/file3
+   ...
+
+Due to defined CDIST_ORDER_DEPENDENCY cdist will execute them in specified order.
+It is better to use CDIST_ORDER_DEPENDENCY in well defined blocks:
+
+**init**
+
+.. code-block:: sh
+
+   CDIST_ORDER_DEPENDENCY=1
+   export CDIST_ORDER_DEPENDENCY
+   ...
+   unset CDIST_ORDER_DEPENDENCY
+
+   __file /tmp/file1
+   __file /tmp/file2
+   __file /tmp/file3
+
+   CDIST_ORDER_DEPENDENCY=1
+   export CDIST_ORDER_DEPENDENCY
+   ...
+   unset CDIST_ORDER_DEPENDENCY

docs/src/cdist-hacker.rst

@@ -16,7 +16,7 @@ Reporting bugs
 --------------
 If you believe you've found a bug and verified that it is
 in the latest version, drop a mail to the cdist mailing list,
-subject prefixed with "[BUG] " or create an issue on github.
+subject prefixed with "[BUG] " or create an issue on code.ungleich.ch.
 
 
 Coding conventions (everywhere)
@@ -51,7 +51,7 @@ work nor kill the authors brain:
 
 As soon as your work meets these requirements, write a mail
 for inclusion to the mailinglist **cdist-configuration-management at googlegroups.com**
-or open a pull request at http://github.com/ungleich/cdist.
+or open a merge request at https://code.ungleich.ch/ungleich-public/cdist.
 
 
 How to submit a new type
@@ -76,7 +76,7 @@ The following workflow works fine for most developers
 .. code-block:: sh
 
     # get latest upstream master branch
-    git clone https://github.com/ungleich/cdist.git
+    git clone https://code.ungleich.ch/ungleich-public/cdist.git
 
     # update if already existing
     cd cdist; git fetch -v; git merge origin/master
@@ -88,22 +88,22 @@ The following workflow works fine for most developers
     # *hack*
     *hack*
 
-    # clone the cdist repository on github if you haven't done so
+    # clone the cdist repository on code.ungleich.ch if you haven't done so
 
     # configure your repo to know about your clone (only once)
-    git remote add github git@github.com:YOURUSERNAME/cdist.git
+    git remote add ungleich git@code.ungleich.ch:YOURUSERNAME/cdist.git
 
-    # push the new branch to github 
-    git push github documentation_cleanup
+    # push the new branch to ungleich gitlab
+    git push ungleich documentation_cleanup
 
     # (or everything)
-    git push --mirror github
+    git push --mirror ungleich
 
-    # create a pull request at github (use a browser)
+    # create a merge request at ungleich gitlab (use a browser)
     # *fixthingsbecausequalityassurancefoundissuesinourpatch*
     *hack*
 
-    # push code to github again
+    # push code to ungleich gitlab again
     git push ... # like above
 
     # add comment that everything should be green now (use a browser)

docs/src/cdist-install.rst

@@ -36,23 +36,43 @@ To install cdist, execute the following commands:
 
 .. code-block:: sh
 
-    git clone https://github.com/ungleich/cdist.git
+    git clone https://code.ungleich.ch/ungleich-public/cdist.git
     cd cdist
     export PATH=$PATH:$(pwd -P)/bin
 
+From version 4.2.0 cdist tags and releases are signed.
+You can get GPG public key used for signing `here <_static/pgp-key-EFD2AE4EC36B6901.asc>`_.
+
+You can also get cdist from `github mirror <https://github.com/ungleich/cdist>`_.
+
 To install cdist with distutils from cloned repository, first you have to
 create version.py:
 
 .. code-block:: sh
 
-    make version
+    ./bin/build-helper version
 
-Then, as usual, you execute the following command:
+Then you install it with:
+
+.. code-block:: sh
+
+   make install
+
+or with:
+
+.. code-block:: sh
+
+   make install-user
+
+to install it into user *site-packages* directory.
+Or directly with distutils:
 
 .. code-block:: sh
 
     python setup.py install
 
+Note that `bin/build-helper` script is intended for cdist maintainers.
+
 
 Available versions in git
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -74,14 +94,6 @@ So for instance if you want to use and stay with version 4.1, you can use
 
     git checkout -b 4.1 origin/4.1
 
-Git mirrors
-^^^^^^^^^^^
-
-If the main site is down, you can acquire cdist from one of the following sites:
-
- * git://github.com/telmich/cdist.git `github <https://github.com/telmich/cdist>`_
- * git://git.code.sf.net/p/cdist/code `sourceforge <https://sourceforge.net/p/cdist/code>`_
-
 Building and using documentation (man and html)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

docs/src/cdist-intro.rst

@@ -1,15 +0,0 @@
-cdist - usable configuration management
-=======================================
-
-.. image:: cdist-logo.png
-    :alt: cdist-logo
-
-cdist is a usable configuration management system.
-It adheres to the KISS principle and 
-is being used in small up to enterprise grade environments.
-cdist is an alternative to other configuration management systems like
-
-* `bcfg2 <http://trac.mcs.anl.gov/projects/bcfg2>`_
-* `chef <http://wiki.opscode.com/display/chef/>`_
-* `cfengine <http://www.cfengine.org/>`_
-* `puppet <http://www.puppetlabs.com/>`_.

docs/src/cdist-manifest.rst

@@ -163,6 +163,8 @@ automatically depends on the previously created object.
 It essentially helps you to build up blocks of code that build upon each other
 (like first creating the directory xyz than the file below the directory).
 
+Read also about `perils of CDIST_ORDER_DEPENDENCY <cdist-best-practice.html#perils-of-cdist-order-dependency>`_.
+
 
 Overrides
 ---------

docs/src/cdist-os.rst

@@ -1,16 +1,19 @@
-Supported Operating Systems
+Supported operating systems
 ===========================
 
 cdist was tested or is know to run on at least
 
-* `Archlinux <http://www.archlinux.org/>`_
-* `Debian <http://www.debian.org/>`_
-* `CentOS <http://www.centos.org/>`_
-* `Fedora <http://fedoraproject.org/>`_
+* `Alpine Linux <https://alpinelinux.org>`_
+* `Archlinux <http://www.archlinux.org>`_
+* `CentOS <http://www.centos.org>`_
+* `Debian <http://www.debian.org>`_
+* `Devuan <https://devuan.org>`_
+* `Fedora <http://fedoraproject.org>`_
 * `FreeBSD <http://www.freebsd.org>`_
-* `Gentoo <http://www.gentoo.org/>`_
-* `Mac OS X <http://www.apple.com/macosx/>`_
+* `Gentoo <http://www.gentoo.org>`_
+* `Mac OS X <http://www.apple.com/macosx>`_
+* `NetBSD <https://www.netbsd.org>`_
 * `OpenBSD <http://www.openbsd.org>`_
-* `Redhat <http://www.redhat.com/>`_
-* `Ubuntu <http://www.ubuntu.com/>`_
-* `XenServer <http://www.citrix.com/xenserver/>`_
+* `Redhat <http://www.redhat.com>`_
+* `Ubuntu <http://www.ubuntu.com>`_
+* `XenServer <http://www.citrix.com/xenserver>`_

docs/src/cdist-parallelization.rst

@@ -12,8 +12,7 @@ The other way is to operate in parallel within one host where you specify
 the number of jobs. This is enabled with :strong:`-j/--jobs` option where you
 can specify the number of parallel jobs. By default,
 :strong:`multiprocessing.cpu_count()` is used. For this mode global explorers,
-object preparation and object run are supported and this option is still in
-:strong:`beta`.
+object preparation and object run are supported.
 
 You can, of course, use those two options together. This means that each host
 will be processed by its own process. Within each process cdist will operate
@@ -32,11 +31,11 @@ Examples
 
     # Configure hosts read from file hosts.file sequentially but using default
     # number of parallel jobs
-    $ cdist config -b -j -f hosts.file
+    $ cdist config -j -f hosts.file
 
     # Configure hosts read from file hosts.file in parallel using 16
     # parallel jobs
-    $ cdist config -b -j 16 -p -f hosts.file
+    $ cdist config -j 16 -p -f hosts.file
 
 
 Caveats

docs/src/cdist-quickstart.rst

@@ -54,9 +54,7 @@ we can use cdist to configure it. You can copy and paste the following
 code into your shell to get started and configure localhost::
 
     # Get cdist 
-    # Mirrors can be found on
-    # http://www.nico.schottelius.org/software/cdist/install/#index2h4
-    git clone git://github.com/ungleich/cdist
+    git clone git@code.ungleich.ch:ungleich-public/cdist.git
 
     # Create manifest (maps configuration to host(s)
     cd cdist

docs/src/cdist-reference.rst.sh

@@ -219,6 +219,10 @@ __cdist_log_level, __cdist_log_level_name
     | TRACE          | 5               |
     +----------------+-----------------+
 
+    Available for: initial manifest, explorer, type manifest, type explorer,
+    type gencode.
+__cdist_dry_run
+    Is set only when doing dry run (``-n`` flag).
     Available for: initial manifest, explorer, type manifest, type explorer,
     type gencode.
 __explorer
@@ -323,6 +327,7 @@ CDIST_OVERRIDE
 
 CDIST_ORDER_DEPENDENCY
     Create dependencies based on the execution order (see  \`cdist manifest <cdist-manifest.html>\`_).
+    Read also about \`perils of CDIST_ORDER_DEPENDENCY <cdist-best-practice.html#perils-of-cdist-order-dependency>\`_.
 
 CDIST_REMOTE_EXEC
     Use this command for remote execution (should behave like ssh).

docs/src/cdist-support.rst

@@ -1,11 +1,9 @@
 Support
 -------
 
-IRC
-~~~
-
-You can join the development ***IRC channel***
-`#cstar on irc.freenode.net <irc://irc.freenode.org/#cstar>`_.
+Chat
+~~~~
+Chat with us: `ungleich chat <https://chat.ungleich.ch/ungleich/channels/cdist>`_.
 
 Mailing list
 ~~~~~~~~~~~~
@@ -25,4 +23,4 @@ Commercial support
 ~~~~~~~~~~~~~~~~~~
 
 You can request commercial support for cdist from
-`my company <http://www.ungleich.ch/>`_.
+`ungleich <http://www.ungleich.ch/>`_.

docs/src/cdist-troubleshooting.rst

@@ -47,7 +47,8 @@ you write to use the -e flag:
 Using debug dump helper script
 ------------------------------
 Since cdist stores data to local cache that can be used for debugging there
-is a helper script that dumps data from local cache.
+is a helper script that dumps data from local cache,
+`cdist-dump <man1/cdist-dump.html>`_.
 
 For more info see:
 

docs/src/cdist-type.rst

@@ -71,6 +71,31 @@ when using -j option. Example of such a type is __package_dpkg type where dpkg i
 prevents to be run in more than one instance.
 
 
+Deprecated types
+-----------------
+If a type is flagged with 'deprecated' marker then it is considered deprecated.
+When it is used cdist writes warning line. If 'deprecated' marker has content
+then this content is printed as a deprecation messages, e.g.:
+
+.. code-block:: sh
+
+    $ ls -l deprecated 
+    -rw-r--r--  1 darko  darko  71 May 20 18:30 deprecated
+    $ cat deprecated 
+    This type is deprecated. It will be removed in the next minor release.
+    $ echo '__foo foo' | ./bin/cdist config -i - 185.203.112.26
+    WARNING: 185.203.112.26: Type __foo is deprecated: This type is deprecated. It will be removed in the next minor release.
+
+If 'deprecated' marker has no content then general message is printed, e.g.:
+
+.. code-block:: sh
+
+    $ ls -l deprecated 
+    -rw-r--r--  1 darko  darko  0 May 20 18:36 deprecated
+    $ echo '__bar foo' | ./bin/cdist config -i - 185.203.112.26
+    WARNING: 185.203.112.26: Type __bar is deprecated.
+
+
 How to write a new type
 -----------------------
 A type consists of
@@ -93,6 +118,9 @@ they are written in shell so they are executed using '/bin/sh -e' or 'CDIST_LOCA
 
 For executable shell code it is suggested that shebang is '#!/bin/sh -e'.
 
+For creating type skeleton you can use helper script
+`cdist-new-type <man1/cdist-new-type.html>`_.
+
 
 Defining parameters
 -------------------
@@ -158,6 +186,31 @@ Example: (e.g. in cdist/conf/type/__nginx_vhost/manifest)
     fi
 
 
+Deprecated parameters
+---------------------
+To deprecate type parameters one can declare a file for each deprecated
+parameter under **parameter/deprecated** directory.
+
+When such parameter is used cdist writes warning line with deprecation message.
+If such file has content then this content is printed as deprecation message.
+If there is no content then generic parameter deprecation message is printed.
+
+Example:
+
+.. code-block:: sh
+
+    $ ls parameter/deprecated/
+    eggs    spam
+    $ cat parameter/deprecated/eggs
+    eggs parameter is deprecated, please use multiple egg parameter.
+    $ cat parameter/deprecated/spam
+    $ echo '__foo foo --foo foo --eggs eggs' | ./bin/cdist config -i - 185.203.112.26
+    WARNING: 185.203.112.26: eggs parameter of type __foo is deprecated: eggs parameter is deprecated, please use multiple egg parameter.
+    $ echo '__foo foo --foo foo --eggs eggs --spam spam' | ./bin/cdist config -i - 185.203.112.26
+    WARNING: 185.203.112.26: spam parameter of type __foo is deprecated.
+    WARNING: 185.203.112.26: eggs parameter of type __foo is deprecated: eggs parameter is deprecated, please use multiple egg parameter.
+
+
 Input from stdin
 ----------------
 Every type can access what has been written on stdin when it has been called.
@@ -188,6 +241,73 @@ In the __file type, stdin is used as source for the file, if - is used for sourc
     ....
 
 
+Stdin inside a loop
+~~~~~~~~~~~~~~~~~~~
+Since cdist saves type's stdin content in the object as **$__object/stdin**,
+so it can be accessed in manifest and gencode-* scripts, this can lead to
+unexpected behavior. For example, suppose you have some type with the following
+in its manifest:
+
+.. code-block:: sh
+
+    if [ -f "$__object/parameter/foo" ]
+    then
+        while read -r l
+        do
+            __file "$l"
+            echo "$l" >&2
+        done < "$__object/parameter/foo"
+    fi
+
+and init manifest:
+
+.. code-block:: sh
+
+    __foo foo --foo a --foo b --foo c
+
+You expect that manifest stderr content is:
+
+.. code-block:: sh
+
+    a
+    b
+    c
+
+and that files *a*, *b* and *c* are created. But all you get in manifest stderr
+is:
+
+.. code-block:: sh
+
+    a
+
+and only *a* file is created.
+
+When redirecting parameter *foo* file content to while's stdin that means that all
+commands in while body have this same stdin. So when *__file* type gets executed,
+cdist saves its stdin which means it gets the remaining content of parameter *foo*
+file, i.e.:
+
+.. code-block:: sh
+
+    b
+    c
+
+The solution is to make sure that your types inside such loops get their stdin
+from somewhere else, e.g. for the above problem *__file* type can get empty
+stdin from */dev/null*:
+
+.. code-block:: sh
+
+    if [ -f "$__object/parameter/foo" ]
+    then
+        while read -r l
+        do
+            __file "$l" < /dev/null
+            echo "$l" >&2
+        done < "$__object/parameter/foo"
+    fi
+
+
 Writing the manifest
 --------------------
 In the manifest of a type you can use other types, so your type extends
@@ -371,6 +491,15 @@ It is available for initial manifest, explorer, type manifest,
 type explorer, type gencode.
 
 
+Detecting dry run
+-----------------
+
+If ``$__cdist_dry_run`` environment variable is set, then it's dry run.
+
+It is available for initial manifest, explorer, type manifest,
+type explorer, type gencode.
+
+
 Hints for typewriters
 ----------------------
 It must be assumed that the target is pretty dumb and thus does not have high

docs/src/cdist-upgrade.rst

@@ -1,5 +1,5 @@
-How to update cdist
-===================
+How to upgrade cdist
+====================
 
 Update the git installation
 ---------------------------

docs/src/conf.py

@@ -56,7 +56,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'cdist'
-# copyright = '2016, Darko Poljak'
+copyright = 'ungleich GmbH 2019'
 # author = 'Darko Poljak'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -138,7 +138,7 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-# html_logo = None
+html_logo = '_static/cdist-logo.jpeg'
 
 # The name of an image file (relative to this directory) to use as a favicon of
 # the docs.  This file should be a Windows icon file (.ico)
@@ -150,6 +150,7 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 # html_static_path = ['_static']
+html_static_path = ['_static']
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied

docs/src/index.rst

@@ -1,24 +1,28 @@
-Welcome to cdist documentation
-==============================
+cdist - usable configuration management
+=======================================
+
+cdist is a usable configuration management system.
+It adheres to the KISS principle and 
+is being used in small up to enterprise grade environments.
 
-Contents:
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 3
    :glob:
    :numbered:
+   :hidden:
 
-   cdist-intro
    cdist-why
+   cdist-features
    cdist-os
    cdist-install
-   cdist-update
+   cdist-upgrade
    cdist-support
-   cdist-features
    cdist-quickstart
    cdist-real-world
    man1/cdist
    man1/cdist-dump
+   man1/cdist-new-type
    cdist-bootstrap
    cdist-configuration
    cdist-manifest

docs/src/man1/cdist-new-type.rst

@@ -0,0 +1,74 @@
+cdist-new-type(1)
+=================
+
+NAME
+----
+cdist-new-type - Create new type skeleton
+
+
+SYNOPSIS
+--------
+
+::
+
+    cdist-new-type TYPE-NAME AUTHOR-NAME AUTHOR-EMAIL [TYPE-BASE-PATH]
+
+
+
+DESCRIPTION
+-----------
+cdist-new-type is a helper script that creates new type skeleton.
+It is then up to the type author to finish the type.
+
+It creates skeletons for the following files:
+
+* man.rst
+* manifest
+* gencode-remote.
+
+Upon creation it prints the path to the newly created type directory.
+
+
+ARGUMENTS
+---------
+**TYPE-NAME**
+   Name of the new type.
+
+**AUTHOR-NAME**
+   Type author's full name.
+
+**AUTHOR-NAME**
+   Type author's email.
+
+**TYPE-BASE-PATH**
+    Path to the base directory of the type. If not set it defaults
+    to '$PWD/type'.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Create new type __foo in ~/.cdist directory.
+    $ cd ~/.cdist
+    $ cdist-new-type '__foo' 'Foo Bar' 'foo.bar at foobar.org'
+    /home/foo/.cdist/type/__foo
+
+
+SEE ALSO
+--------
+:strong:`cdist`\ (1)
+
+
+AUTHORS
+-------
+
+| Steven Armstrong <steven-cdist--@--armstrong.cc>
+| Darko Poljak <darko.poljak--@--ungleich.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2019 Steven Armstrong, Darko Poljak. Free use of this software is
+granted under the terms of the GNU General Public License v3 or later (GPLv3+).

docs/src/man1/cdist.rst

@@ -165,7 +165,7 @@ Install command is currently in beta.
     Operate in parallel in specified maximum number of
     jobs. Global explorers, object prepare and object run
     are supported. Without argument CPU count is used by
-    default. Currently in beta.
+    default.
 
 **-n, --dry-run**
     Do not execute code.

docs/web/cdist.mdwn

@@ -1,21 +0,0 @@
-[[!meta title="cdist - usable configuration management"]]
-
-![cdist-logo](cdist-logo.png "cdist logo")
-
-cdist is a usable configuration management system.
-It adheres to the KISS principle and 
-is being used in small up to enterprise grade environments.
-cdist is an alternative to other configuration management systems like
-[bcfg2](http://bcfg2.org/),
-[chef](https://www.chef.sh/),
-[cfengine](https://cfengine.com/)
-and [puppet](https://puppet.com/).
-
- * [[Why should I use cdist?|why]]
- * [[Documentation|documentation]]
- * [[Supported Operating Systems|os]]
- * [[Installation|install]]
- * [[Update|update]]
- * [[Support|support]]
-
-[[!tag cdist unix]]

docs/web/cdist/documentation.mdwn

@@ -1,12 +0,0 @@
-[[!meta title="Documentation"]]
-
-You can browse the
-[latest version of the documentation](/software/cdist/man/latest) or
-have a look at [all versions](/software/cdist/man).
-
-You can also view [speeches about cdist](/software/cdist/speeches).
-
-Checking out beta? Find the docs here:
-[beta documentation](/software/cdist/man/beta).
-
-[[!tag cdist unix]]

docs/web/cdist/features.mdwn

@@ -1,26 +0,0 @@
-But cdist ticks differently, here is the feature set that makes it unique:
-
-[[!table  data="""
-Keywords | Description
-Simplicity | There is only one type to extend cdist called ***type***
-Design | Type and core cleanly separated
-Design | Sticks completly to the KISS (keep it simple and stupid) paradigma
-Design | Meaningful error messages - do not lose time debugging error messages
-Design | Consistency in behaviour, naming and documentation
-Design | No surprise factor: Only do what is obviously clear, no magic
-Design | Define target state, do not focus on methods or scripts
-Design | Push architecture: Instantly apply your changes
-Small core | cdist's core is very small - less code, less bugs
-Fast development | Focus on straightforwardness of type creation is a main development objective
-Fast development | Batteries included: A lot of requirements can be solved using standard types
-Modern Programming Language | cdist is written in Python
-Requirements, Scalability | No central server needed, cdist operates in push mode and can be run from any computer
-Requirements, Scalability, Upgrade | cdist only needs to be updated on the master, not on the target hosts
-Requirements, Security | Uses well-know [SSH](http://www.openssh.com/) as transport protocol
-Requirements, Simplicity | Requires only shell and SSH server on the target
-UNIX | Reuse of existing tools like cat, find, mv, ... 
-UNIX, familar environment, documentation | Is available as manpages and HTML
-UNIX, simplicity, familar environment | cdist is configured in POSIX shell
-"""]]
-
-[[!tag cdist unix]]

docs/web/cdist/install.mdwn

@@ -1,103 +0,0 @@
-[[!meta title="How to install cdist"]]
-[[!toc levels=3]]
-
-## Requirements
-
-### Source Host
-
-This is the machine you use to configure the target hosts.
-
- * /bin/sh: A posix like shell (for instance bash, dash, zsh)
- * Python >= 3.2
- * SSH client
- * Asciidoc and xsltproc (for building the manpages)
-
-### Target Hosts
-
- * /bin/sh: A posix like shell (for instance bash, dash, zsh)
- * SSH server
-
-## Install cdist
-
-You can install cdist either from git or as a python package.
-
-### From git
-
-Cloning cdist from git gives you the advantage of having
-a version control in place for development of your own stuff
-immediately.
-
-To install cdist, execute the following commands:
-
-    git clone https://github.com/ungleich/cdist.git
-    cd cdist
-    export PATH=$PATH:$(pwd -P)/bin
-
-From version 4.2.0 cdist tags and github releases are signed.
-You can get GPG public key used for signing [here](/software/cdist/pgp-key-EFD2AE4EC36B6901.asc).
-
-#### Available versions in git
-
- * The active development takes place in the **master** branch
- * The current stable version can be found in the **2.0** branch
- * The upcoming stable version can be found in the **2.1** branch
-
-Other branches may be available for features or bugfixes, but they
-may vanish at any point. To select a specific branch use
-
-    # Generic code
-    git checkout -b <localbranchname> origin/<branchname>
-
-So for instance if you want to use and stay with version 2.0, you can use
-
-    git checkout -b 2.0 origin/2.0
-
-#### Git Mirrors
-
-If the main site is down, you can acquire cdist from one of the following sites:
-
- * git://github.com/telmich/cdist.git ([github](https://github.com/telmich/cdist))
- * git://git.code.sf.net/p/cdist/code ([sourceforge](https://sourceforge.net/p/cdist/code))
-
-#### Building and using documentation (man and html)
-
-If you want to build and use the documentation, run:
-
-    make docs
-
-Documentation comes in two formats, man pages and full HTML
-documentation. Documentation is built into distribution's
-docs/dist directory. man pages are in docs/dist/man and
-HTML documentation in docs/dist/html.
-
-If you want to use man pages, run:
-
-    export MANPATH=$MANPATH:$(pwd -P)/docs/dist/man
-
-Or you can move manpages from docs/dist/man directory to some
-other directory and add it to MANPATH.
-
-Full HTML documentation can be accessed at docs/dist/html/index.html.
-
-You can also build manpages for types in your ~/.cdist directory:
-
-    make dotman
-
-Built manpages are now in docs/dist/man directory. If you have
-some other custom .cdist directory, e.g. /opt/cdist then use:
-
-    DOT_CDIST_PATH=/opt/cdist make dotman
-
-
-### Python Package
-
-Cdist is available as a python package at
-[PyPi](http://pypi.python.org/pypi/cdist/). You can install it using
-
-    pip install cdist
-
-## Use cdist
-
-[[Dig into the documentation|documentation]] to get started with cdist!
-
-[[!tag cdist unix]]

docs/web/cdist/os.mdwn

@@ -1,18 +0,0 @@
-[[!meta title="Supported Operating Systems"]]
-
-cdist was tested or is know to run on at least
-
- * [Archlinux](http://www.archlinux.org/)
- * [Debian](http://www.debian.org/)
- * [CentOS](http://www.centos.org/)
- * [Scientific](https://www.scientificlinux.org/)
- * [Fedora](http://fedoraproject.org/)
- * [FreeBSD](http://www.freebsd.org)
- * [Gentoo](http://www.gentoo.org/)
- * [Mac OS X](http://www.apple.com/macosx/)
- * [OpenBSD](http://www.openbsd.org)
- * [Redhat](http://www.redhat.com/)
- * [Ubuntu](http://www.ubuntu.com/)
- * [XenServer](http://www.citrix.com/xenserver/)
-
-[[!tag cdist unix]]

docs/web/cdist/support.mdwn

@@ -1,28 +0,0 @@
-## Support
-
-### IRC
-
-You can join the development ***IRC channel***
-[#cstar on irc.freenode.net](irc://irc.freenode.org/#cstar).
-
-### Mailing list
-
-Bug reports, questions, patches, etc. should be send to the
-[cdist mailing list](https://groups.google.com/forum/#!forum/cdist-configuration-management).
-
-### Linkedin
-
-If you have an account
-at [Linked in](http://www.linkedin.com/),
-you can join the
-[cdist group](http://www.linkedin.com/groups/cdist-configuration-management-3952797).
-
-### Chat
-Chat with us: [ungleich chat](https://chat.ungleich.ch/channel/cdist).
-
-### Commercial support
-
-You can request commercial support for cdist from
-[my company](http://www.ungleich.ch/english/).
-
-[[!tag cdist unix]]

docs/web/cdist/update.mdwn

@@ -1,158 +0,0 @@
-[[!meta title="How to update cdist"]]
-
-## Update The Git Installation
-
-To upgrade cdist in the current branch use
-
-    git pull
-
-    # Also update the manpages
-    ./build man
-    export MANPATH=$MANPATH:$(pwd -P)/doc/man
-
-If you stay on a version branche (i.e. 1.0, 1.1., ...), nothing should break.
-The master branch on the other hand is the development branch and may not be
-working, break your setup or eat the tree in your garden.
-
-### Safely upgrading to new versions
-
-To upgrade to **any** further cdist version, you can take the
-following procedure to do a safe upgrade:
-
-    # Create new branch to try out the update
-    git checkout -b upgrade_cdist
-
-    # Get latest cdist version in git database
-    git fetch -v
-
-    # see what will happen on merge - replace
-    # master with the branch you plan to merge
-    git diff upgrade_cdist..origin/master
-
-    # Merge the new version
-    git merge origin/master
-
-Now you can ensure all custom types work with the new version.
-Assume that you need to go back to an older version during
-the migration/update, you can do so as follows:
-
-    # commit changes
-    git commit -m ...
-
-    # go back to original branch
-    git checkout master
-
-After that, you can go back and continue the upgrade:
-
-    # git checkout upgrade_cdist
-
-
-## Update The Python Package
-
-To upgrade to the lastet version do
-
-    pip install --upgrade cdist
-
-## General Update Instructions
-
-### Updating from 3.0 to 3.1
-
-The type **\_\_ssh_authorized_keys** now also manages existing keys, 
-not only the ones added by cdist.
-
-### Updating from 2.3 to 3.0
-
-The **changed** attribute of objects has been removed.
-Use [messaging](/software/cdist/man/3.0.0/man7/cdist-messaging.html) instead.
-
-### Updating from 2.2 to 2.3
-
-No incompatibilities.
-
-### Updating from 2.1 to 2.2
-
-Starting with 2.2, the syntax for requiring a singleton type changed:
-Old format:
-
-    require="__singleton_type/singleton" ...
-
-New format:
-
-    require="__singleton_type" ...
-
-Internally the "singleton" object id was dropped to make life more easy.
-You can probably fix your configuration by running the following code
-snippet (currently untested, please report back if it works for you):
-
-    find ~/.cdist/* -type f -exec sed -i 's,/singleton,,' {} \;
-
-### Updating from 2.0 to 2.1
-
-Have a look at the update guide for [[2.0 to 2.1|2.0-to-2.1]].
-
- * Type **\_\_package* and \_\_process** use --state **present** or **absent**.
-   The states **removed/installed** and **stopped/running** have been removed.
-   Support for the new states is already present in 2.0.
- * Type **\_\_directory**: Parameter --parents and --recursive are now boolean
-   The old "yes/no" values need to be removed.
- * Type **\_\_rvm_ruby**: Parameter --default is now boolean
-   The old "yes/no" values need to be removed.
- * Type **\_\_rvm_gemset**: Parameter --default is now boolean
-   The old "yes/no" values need to be removed.
- * Type **\_\_addifnosuchline** and **\_\_removeline** have been replaced by **\_\_line**
- * The **conf** directory is now located at **cdist/conf**.
-   You need to migrate your types, explorers and manifests
-   manually to the new location.
- * Replace the variable **\_\_self** by **\_\_object_name**
-   Support for the variable **\_\_object_name** is already present in 2.0.
- * The types **\_\_autofs**, **\_\_autofs_map** and **\_\_autofs_reload** have been removed
-   (no maintainer, no users)
- * Type **\_\_user**: Parameter --groups removed (use the new \_\_user_groups type)
- * Type **\_\_ssh_authorized_key** has been replaced by more flexible type 
-    **\_\_ssh_authorized_keys**
-
-### Updating from 1.7 to 2.0
-
-* Ensure python (>= 3.2) is installed on the source host
-* Use "cdist config host" instead of "cdist-deploy-to host"
-* Use "cdist config -p host1 host2" instead of "cdist-mass-deploy"
-* Use "cdist banner" for fun
-* Use **\_\_object_name** instead of **\_\_self** in manifests
-
-### Updating from 1.6 to 1.7
-
-* If you used the global explorer **hardware_type**, you need to change
-  your code to use **machine** instead.
-
-### Updating from 1.5 to 1.6
-
-* If you used **\_\_package_apt --preseed**, you need to use the new
-  type **\_\_debconf_set_selections** instead.
-* The **\_\_package** types accepted either --state deinstalled or
-  --state uninstaaled. Starting with 1.6, it was made consistently
-  to --state removed.
-
-### Updating from 1.3 to 1.5
-
-No incompatibilities.
-
-### Updating from 1.2 to 1.3
-
-Rename **gencode** of every type to **gencode-remote**.
-
-### Updating from 1.1 to 1.2
-
-No incompatibilities.
-
-### Updating from 1.0 to 1.1
-
-In 1.1 the type **\_\_file** was split into **\_\_directory**, **\_\_file** and
-**\_\_link**. The parameter **--type** was removed from **\_\_file**. Thus you
-need to replace **\_\_file** calls in your manifests:
-
- * Remove --type from all \_\_file calls
- * If type was symlink, use \_\_link and --type symbolic
- * If type was directory, use \_\_directory
-
-
-[[!tag cdist unix]]

docs/web/cdist/update/2.0-to-2.1.mdwn

@@ -1,118 +0,0 @@
-[[!meta title="Update Guide for 2.0 to 2.1"]]
-
-## Introduction
-
-When changing your installation from 2.0 to 2.1, there are
-a lot of changes coming up. 2.1 is mainly a cleanup release,
-which removes long time deprecated behaviour, but also makes
-a lot of things more consistent and allows you to split off your types,
-explorers and manifest to custom directories.
-
-This document will guide you to a successful update.
-
-## Preparation
-
-As for every software and system you use in production, you should first of 
-all make a backup of your data. To prevent any breakage, it is
-recommended to create a new git branch to do the update on:
-
-    % git checkout -b update_to_2.1
-
-This also ensure that whenever you need to do a change in your
-2.0 based tree, you can simply go back to that branch, apply the change
-and configure your systems - independently of your update progress!
-
-Next fetch the latest upstream changes, I assume that
-origin refers to one of the upstream mirrors (change origin if you use
-another remote name for upstream cdist):
-
-    % git fetch -v origin
-
-## Merge the changes
-
-Now try to merge upstream into the new branch.
-
-    % git merge origin/2.1
-
-Fix any conflicts that may have been occurred due to local changes
-and then **git add** and *git commit** those changes. This should seldom 
-occur and if, it's mostly for people hacking on the cdist core.
-
-## Move "conf" directory
-
-One of the biggest changes in cdist 2.1 is that you can have multiple
-**conf** directories: Indeed, the new default behaviour of cdist is to
-search for conf directories
-
- * below the python module (cdist/conf in the source tree or in the installed location)
- * at ~/.cdist/ (on conf suffix there)
-
-So you can now choose, where to store your types.
-
-### Integrate your conf/ back into the tree
-
-If you choose to store your types together with the upstream types,
-you can just move all your stuff below **cdist/conf**:
-
-    % git mv conf/type/* cdist/conf/type
-    % git mv conf/manifest/* cdist/conf/manifest
-    % git mv conf/explorer/* cdist/conf/explorer
-    % git commit -m "Re-Integrate my conf directory into cdist 2.1 tree"
-
-### Move your conf/ directory to ~/.cdist
-
-If you want to store your site specific
-configuration outside of the cdist tree, you
-can move your conf/ directory to your homedirectory ($HOME) under ~/.cdist:
-
-    % mv conf ~/.cdist
-    % git rm -r conf
-    % git commit -m "Move my conf directory to ~/.cdist"
-
-It it still recommended to use a version control system like git in it:
-
-    % cd ~/.cdist
-    % git init
-    % git add .
-    % git commit -m "Create new git repository containing my cdist configuration"
-
-## Test the migration
-
-Some of the types shipped with upstream were changed, so you may want to test 
-the result by running cdist on one of your staging target hosts:
-
-    % ./bin/cdist config -v staging-host
-
-All incompatibilities are listed on the [[cdist update page|software/cdist/update]],
-so you can browse through the list and update your configuration.
-
-## Final Cleanups
-
-When everything is tested, there are some cleanups to be done to finalise the update.
-
-### When continuing to keep conf/ in the tree
-
-You can then merge back your changes into the master tree and continue to work
-as normal.
-
-### When using ~/.cdist
-
-If you decided to move your site specific code to ~/.cdist, you can now switch your
-**master** branch or version branch to upstream directly. Assumnig you are in the
-cdist directory, having your previous branch checked out, you can create a clean
-state using the following commands:
-
-    % upstream_branch=2.1
-    % current_branch=$(git rev-parse --abbrev-ref HEAD)
-    % git checkout -b archive_my_own_tree
-    % git branch -D "$current_branch"
-    % git checkout -b "$current_branch" "origin/$upstream_branch"
-
-Afther these commands, your previous main branch is accessible at
-**archive_my_own_tree** and your branch is now tracking upstream.
-
-## Questions? Critics? Hints?
-
-If you think this manual helped or misses some information, do not 
-hesitate to contact us on any of the usual ways (irc, mailinglist,
-github issue tracker, ...).

docs/web/cdist/why.mdwn

@@ -1,69 +0,0 @@
-[[!meta title="Why should I use cdist?"]]
-
-[[!toc]]
-
-There are several motivations to use cdist, these
-are probably the most popular ones.
-
-## Known language
-
-Cdist is being configured in
-[shell script](https://en.wikipedia.org/wiki/Shell_script).
-Shell script is used by UNIX system engineers for decades.
-So when cdist is introduced, your staff does not need to learn a new
-[DSL](https://en.wikipedia.org/wiki/Domain-specific_language)
-or programming language.
-
-## Powerful language
-
-Not only is shell scripting widely known by system engineers,
-but it is also a very powerful language. Here are some features
-which make daily work easy:
-
- * Configuration can react dynamicly on explored values
- * High level string manipulation (using sed, awk, grep)
- * Conditional support (**if, case**)
- * Loop support (**for, while**)
- * Support for dependencies between cdist types
-
-## More than shell scripting
-
-If you compare regular shell scripting with cdist, there is one major
-difference: When using cdist types,
-the results are 
-[idempotent](https://en.wikipedia.org/wiki/Idempotence). 
-In practise that means it does not matter in which order you 
-call cdist types, the result is always the same.
-
-## Zero dependency configuration management
-
-Cdist requires very litte on a target system. Even better,
-in almost all cases all dependencies are usually fulfilled.
-Cdist does not require an agent or a high level programming
-languages on the target host: it will run on any host that
-has a **ssh server running** and a posix compatible shell
-(**/bin/sh**). Compared to other configuration management systems,
-it does not require to open up an additional port.
-
-## Push based distribution
-
-Cdist uses the push based model for configuration. In this
-scenario, one (or more) computers connect the target hosts
-and apply the configuration. That way the source host has
-very little requirements: Cdist can even run on a sysadmin
-notebook that is loosely connected to the network and has
-limited amount of resources.
-
-Furthermore, from a security point of view, only one machine 
-needs access to the target hosts. No target hosts will ever
-need to connect back to the source host, which contains the
-full configuration.
-
-## Highly scalable
-
-If at some point you manage more hosts than can be handled from
-a single source host, you can simply add more resources: Either
-add more cores to one host or add hosts.
-Cdist will utilise the given resources in parallel.
-
-[[!tag cdist unix]]

scripts/cdist-new-type

@@ -0,0 +1,159 @@
+#!/bin/sh
+
+basename="${0##*/}"
+
+if [ $# -lt 3 ]
+then
+    printf "usage: %s TYPE-NAME AUTHOR-NAME AUTHOR-EMAIL [TYPE-BASE-PATH]
+    TYPE-NAME       Name of the type.
+    AUTHOR-NAME     Type author's full name.
+    AUTHOR-EMAIL    Type author's email.
+    TYPE-BASE-PATH  Path to the base directory of the type. If not set it defaults
+                    to '\$PWD/type'.\n" "${basename}"
+    exit 1
+fi
+
+type_name="$1"
+shift
+author_name="$1"
+shift
+author_email="$1"
+shift
+
+if [ $# -ge 1 ]
+then
+    type_base_path="$1"
+    shift
+else
+    #type_base_path=~/.cdist/type
+    type_base_path="$PWD/type"
+fi
+
+error() {
+   printf "%s\n" "$*" >&2
+}
+
+die() {
+   error "$@"
+   exit 1
+}
+
+cd "$type_base_path" || die "Could not change to type directory: $type_base_path.
+You have to specify type base path or run me from within a cdist conf directory,
+e.g. ~/.cdist."
+
+year=$(date +%Y)
+copyright="# $year $author_name ($author_email)"
+
+license="# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+"
+
+set -e
+
+mkdir "$type_name"
+cd "$type_name"
+
+### man page
+header="cdist-type${type_name}(7)"
+header_length="${#header}"
+cat >> man.rst << DONE
+$header
+$(while [ "${header_length}" -gt 0 ]; do printf "="; header_length=$((header_length - 1)); done; printf "\n";)
+
+NAME
+----
+cdist-type${type_name} - TODO
+
+
+DESCRIPTION
+-----------
+This space intentionally left blank.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # TODO
+    ${type_name}
+
+
+SEE ALSO
+--------
+:strong:\`TODO\`\\ (7)
+
+
+AUTHORS
+-------
+$author_name <$author_email>
+
+
+COPYING
+-------
+Copyright \(C) $year $author_name. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+DONE
+
+### manifest
+cat >> manifest << DONE
+#!/bin/sh -e
+#
+${copyright}
+#
+${license}
+
+os=\$(cat "\$__global/explorer/os")
+
+case "\$os" in
+   *)
+      printf "Your operating system (%s) is currently not supported by this type (%s)\n" "\$os" "\${__type##*/}" >&2
+      printf "Please contribute an implementation for it if you can.\n" >&2
+      exit 1
+   ;;
+esac
+DONE
+chmod +x manifest
+
+# gencode-remote
+cat >> gencode-remote << DONE
+#!/bin/sh -e
+#
+${copyright}
+#
+${license}
+DONE
+chmod +x gencode-remote
+
+printf "%s/%s\n" "$type_base_path" "$type_name"

setup.py

@@ -36,12 +36,12 @@ setup(
     name="cdist",
     packages=["cdist", "cdist.core", "cdist.exec", "cdist.util", ],
     package_data={'cdist': package_data},
-    scripts=["scripts/cdist", "scripts/cdist-dump"],
+    scripts=["scripts/cdist", "scripts/cdist-dump", "scripts/cdist-new-type"],
     version=cdist.version.VERSION,
     description="A Usable Configuration Management System",
     author="Nico Schottelius",
     author_email="nico-cdist-pypi@schottelius.org",
-    url="http://www.nico.schottelius.org/software/cdist/",
+    url="https://www.cdi.st/",
     classifiers=[
         "Development Status :: 6 - Mature",
         "Environment :: Console",