Support Added for Debian 10* in __letsencrypt_cert

[explorer/os] Unquote values from /etc/os-release

See merge request ungleich-public/cdist!803

.gitattributes

@@ -0,0 +1,8 @@
+.gitignore export-ignore
+.gitattributes export-ignore
+.gitkeep export-ignore
+docs/speeches export-ignore
+docs/video export-ignore
+docs/src/man7 export-ignore
+bin/build-helper export-ignore
+README-maintainers export-ignore

.gitignore

@@ -1,5 +1,22 @@
 # -vim
-.*.swp
+# Swap
+[._]*.s[a-v][a-z]
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+
+# Temporary
+.netrwhist
+*~
+*.tmp
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
 
 # Ignore generated manpages
 docs/src/.marker
@@ -11,18 +28,23 @@ docs/src/cdist-reference.rst
 # Ignore cdist cache for version control
 /cache/
 
+# Ignore inventory basedir
+cdist/inventory/
+
 # Python: cache, distutils, distribution in general
 __pycache__/
 *.pyc
 MANIFEST
 dist/
 cdist/version.py
+cdist.egg-info/
 
 # sphinx build dirs, cache
 _build/
 docs/dist
 
 # Ignore temp files used for signing
+cdist-*.tar
 cdist-*.tar.gz
 cdist-*.tar.gz.asc
 

Makefile

@@ -18,31 +18,30 @@
 #
 #
 
-helper=./bin/build-helper
+.PHONY: help
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "man             build only man user documentation"
+	@echo "html            build only html user documentation"
+	@echo "docs            build both man and html user documentation"
+	@echo "dotman          build man pages for types in your ~/.cdist directory"
+	@echo "speeches        build speeches pdf files"
+	@echo "install         install in the system site-packages directory"
+	@echo "install-user    install in the user site-packages directory"
+	@echo "docs-clean      clean documentation"
+	@echo "clean           clean"
 
-DOCS_SRC_DIR=docs/src
-SPEECHDIR=docs/speeches
-TYPEDIR=cdist/conf/type
-
-WEBSRCDIR=docs/web
-
-WEBDIR=$$HOME/vcs/www.nico.schottelius.org
-WEBBLOG=$(WEBDIR)/blog
-WEBBASE=$(WEBDIR)/software/cdist
-WEBPAGE=$(WEBBASE).mdwn
-
-CHANGELOG_VERSION=$(shell $(helper) changelog-version)
-CHANGELOG_FILE=docs/changelog
-
-PYTHON_VERSION=cdist/version.py
+DOCS_SRC_DIR=./docs/src
+SPEECHDIR=./docs/speeches
+TYPEDIR=./cdist/conf/type
 
 SPHINXM=make -C $(DOCS_SRC_DIR) man
 SPHINXH=make -C $(DOCS_SRC_DIR) html
 SPHINXC=make -C $(DOCS_SRC_DIR) clean
+
 ################################################################################
 # Manpages
 #
-MAN1DSTDIR=$(DOCS_SRC_DIR)/man1
 MAN7DSTDIR=$(DOCS_SRC_DIR)/man7
 
 # Manpages #1: Types
@@ -54,6 +53,7 @@ MANTYPES=$(subst /man.rst,.rst,$(MANTYPEPREFIX))
 
 # Link manpage: do not create man.html but correct named file
 $(MAN7DSTDIR)/cdist-type%.rst: $(TYPEDIR)/%/man.rst
+	mkdir -p $(MAN7DSTDIR)
 	ln -sf "../../../$^" $@
 
 # Manpages #2: reference
@@ -63,11 +63,16 @@ DOCSREFSH=$(DOCS_SRC_DIR)/cdist-reference.rst.sh
 $(DOCSREF): $(DOCSREFSH)
 	$(DOCSREFSH)
 
+version:
+	@[ -f "cdist/version.py" ] || { \
+		printf "Missing 'cdist/version.py', please generate it first.\n" && exit 1; \
+	}
+
 # Manpages #3: generic part
-man: $(MANTYPES) $(DOCSREF) $(PYTHON_VERSION)
+man: version $(MANTYPES) $(DOCSREF)
 	$(SPHINXM)
 
-html: $(MANTYPES) $(DOCSREF) $(PYTHON_VERSION)
+html: version $(MANTYPES) $(DOCSREF)
 	$(SPHINXH)
 
 docs: man html
@@ -75,24 +80,6 @@ docs: man html
 docs-clean:
 	$(SPHINXC)
 
-# Manpages #5: release part
-MANWEBDIR=$(WEBBASE)/man/$(CHANGELOG_VERSION)
-HTMLBUILDDIR=docs/dist/html
-
-docs-dist: html
-	rm -rf "${MANWEBDIR}"
-	mkdir -p "${MANWEBDIR}"
-	# mkdir -p "${MANWEBDIR}/man1" "${MANWEBDIR}/man7"
-	# cp ${MAN1DSTDIR}/*.html ${MAN1DSTDIR}/*.css ${MANWEBDIR}/man1
-	# cp ${MAN7DSTDIR}/*.html ${MAN7DSTDIR}/*.css ${MANWEBDIR}/man7
-	cp -R ${HTMLBUILDDIR}/* ${MANWEBDIR}
-	cd ${MANWEBDIR} && git add . && git commit -m "cdist manpages update: $(CHANGELOG_VERSION)" || true
-
-man-latest-link: web-pub
-	# Fix ikiwiki, which does not like symlinks for pseudo security
-	ssh staticweb.ungleich.ch \
-		"cd /home/services/www/nico/nico.schottelius.org/www/software/cdist/man/ && rm -f latest && ln -sf "$(CHANGELOG_VERSION)" latest"
-
 # Manpages: .cdist Types
 DOT_CDIST_PATH=${HOME}/.cdist
 DOTMAN7DSTDIR=$(MAN7DSTDIR)
@@ -105,8 +92,7 @@ DOTMANTYPES=$(subst /man.rst,.rst,$(DOTMANTYPEPREFIX))
 $(DOTMAN7DSTDIR)/cdist-type%.rst: $(DOTTYPEDIR)/%/man.rst
 	ln -sf "$^" $@
 
-# Manpages #3: generic part
-dotman: $(DOTMANTYPES)
+dotman: version $(DOTMANTYPES)
 	$(SPHINXM)
 
 ################################################################################
@@ -114,7 +100,6 @@ dotman: $(DOTMANTYPES)
 #
 SPEECHESOURCES=$(SPEECHDIR)/*.tex
 SPEECHES=$(SPEECHESOURCES:.tex=.pdf)
-SPEECHESWEBDIR=$(WEBBASE)/speeches
 
 # Create speeches and ensure Toc is up-to-date
 $(SPEECHDIR)/%.pdf: $(SPEECHDIR)/%.tex
@@ -124,128 +109,26 @@ $(SPEECHDIR)/%.pdf: $(SPEECHDIR)/%.tex
 
 speeches: $(SPEECHES)
 
-speeches-dist: speeches
-	rm -rf "${SPEECHESWEBDIR}"
-	mkdir -p "${SPEECHESWEBDIR}"
-	cp ${SPEECHES} "${SPEECHESWEBDIR}"
-	cd ${SPEECHESWEBDIR} && git add . && git commit -m "cdist speeches updated" || true
-
 ################################################################################
-# Website
+# Misc
 #
-
-BLOGFILE=$(WEBBLOG)/cdist-$(CHANGELOG_VERSION)-released.mdwn
-
-$(BLOGFILE): $(CHANGELOG_FILE)
-	$(helper) blog $(CHANGELOG_VERSION) $(BLOGFILE)
-
-web-blog: $(BLOGFILE)
-
-web-doc:
-	# Go to top level, because of cdist.mdwn
-	rsync -av "$(WEBSRCDIR)/" "${WEBBASE}/.."
-	cd "${WEBBASE}/.." && git add cdist* && git commit -m "cdist doc update" cdist* || true
-
-web-dist: web-blog web-doc
-
-web-pub: web-dist docs-dist speeches-dist
-	cd "${WEBDIR}" && make pub
-
-web-release-all: man-latest-link
-web-release-all-no-latest: web-pub
-
-################################################################################
-# Release: Mailinglist
-#
-ML_FILE=.lock-ml
-
-# Only send mail once - lock until new changelog things happened
-$(ML_FILE): $(CHANGELOG_FILE)
-	$(helper) ml-release $(CHANGELOG_VERSION)
-	touch $@
-
-ml-release: $(ML_FILE)
-
-
-################################################################################
-# pypi
-#
-PYPI_FILE=.pypi-release
-$(PYPI_FILE): man $(PYTHON_VERSION)
-	python3 setup.py sdist upload
-	touch $@
-
-pypi-release: $(PYPI_FILE)
-################################################################################
-# archlinux
-#
-ARCHLINUX_FILE=.lock-archlinux
-ARCHLINUXTAR=cdist-$(CHANGELOG_VERSION)-1.src.tar.gz
-
-$(ARCHLINUXTAR): PKGBUILD
-	umask 022; mkaurball
-
-PKGBUILD: PKGBUILD.in $(PYTHON_VERSION)
-	./PKGBUILD.in $(CHANGELOG_VERSION)
-
-$(ARCHLINUX_FILE): $(ARCHLINUXTAR) $(PYTHON_VERSION)
-	burp -c system $(ARCHLINUXTAR)
-	touch $@
-
-archlinux-release: $(ARCHLINUX_FILE)
-
-################################################################################
-# Release
-#
-
-$(PYTHON_VERSION): .git/refs/heads/master
-	$(helper) version
-
-# Code that is better handled in a shell script
-check-%:
-	$(helper) $@
-
-release:
-	$(helper) $@
-
-################################################################################
-# Cleanup
-#
-
-clean:
+clean: docs-clean
 	rm -f $(DOCS_SRC_DIR)/cdist-reference.rst
 
 	find "$(DOCS_SRC_DIR)" -mindepth 2 -type l \
 	| xargs rm -f
 
-	make -C $(DOCS_SRC_DIR) clean
-
 	find * -name __pycache__  | xargs rm -rf
 
-	# Archlinux
-	rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz
-	rm -rf pkg/ src/
-
-	rm -f MANIFEST PKGBUILD
-	rm -rf dist/
-
-	# Signed release
-	rm -f cdist-*.tar.gz
-	rm -f cdist-*.tar.gz.asc
-
-distclean: clean
-	rm -f cdist/version.py
+	# distutils
+	rm -rf ./build
 
 ################################################################################
-# Misc
+# install
 #
 
-# The pub is Nico's "push to all git remotes" way ("make pub")
-pub:
-	git push --mirror
+install:
+	python3 setup.py install
 
-test:
-	$(helper) $@
-
-pep8:
-	$(helper) $@
+install-user:
+	python3 setup.py install --user

PKGBUILD.in

@@ -9,7 +9,7 @@ pkgver=$version
 pkgrel=1
 pkgdesc='A Usable Configuration Management System"'
 arch=('any')
-url='http://www.nico.schottelius.org/software/cdist/'
+url='https://www.cdi.st/'
 license=('GPL3')
 depends=('python>=3.2.0')
 source=("http://pypi.python.org/packages/source/c/cdist/cdist-\${pkgver}.tar.gz")

README

@@ -3,4 +3,5 @@ cdist
 
 cdist is a usable configuration management system.
 
-For the web documentation have a look at docs/web/.
+For the web documentation have a look at https://www.cdi.st/
+or at docs/src for reStructuredText manual.

README-maintainers

@@ -0,0 +1,4 @@
+Maintainers should use ./bin/build-helper script.
+
+Makefile is intended for end users. It can be used for non-maintaining
+targets that can be run from pure source (without git repository).

bin/build-helper

@@ -1,6 +1,7 @@
 #!/bin/sh
 #
 # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2016-2019 Darko Poljak (darko.poljak at gmail.com)
 #
 # This file is part of cdist.
 #
@@ -18,17 +19,66 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# This file contains the heavy lifting found usually in the Makefile
+# This file contains the heavy lifting found usually in the Makefile.
 #
 
-basedir=${0%/*}/../
-# Change to checkout directory
-cd "$basedir"
+usage() {
+    printf "usage: %s TARGET [TARGET-ARGS...]
+    Available targets:
+        changelog-changes
+        changelog-version
+        check-date
+        check-unittest
+        ml-release
+        archlinux-release
+        pypi-release
+        release-git-tag
+        sign-git-release
+        release
+        test
+        test-remote
+        pycodestyle
+        pep8
+        check-pycodestyle
+        shellcheck-global-explorers
+        shellcheck-type-explorers
+        shellcheck-manifests
+        shellcheck-local-gencodes
+        shellcheck-remote-gencodes
+        shellcheck-scripts
+        shellcheck-gencodes
+        shellcheck-types
+        shellcheck
+        shellcheck-type-files
+        shellcheck-with-files
+        shellcheck-build-helper
+        check-shellcheck
+        version-branch
+        version
+        target-version
+        clean
+        distclean\n" "$1"
+}
 
-version=$(git describe)
+basename="${0##*/}"
+
+if [ $# -lt 1 ]
+then
+    usage "${basename}"
+    exit 1
+fi
 
 option=$1; shift
 
+SHELLCHECKCMD="shellcheck -s sh -f gcc -x"
+# Skip SC2154 for variables starting with __ since such variables are cdist
+# environment variables.
+SHELLCHECK_SKIP=': __.*is referenced but not assigned.*\[SC2154\]'
+
+# Change to checkout directory
+basedir="${0%/*}/../"
+cd "$basedir"
+
 case "$option" in
     changelog-changes)
         if [ "$#" -eq 1 ]; then
@@ -66,8 +116,8 @@ case "$option" in
         date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //')
 
         if [ "$date_today" != "$date_changelog" ]; then
-            echo "Date in changelog is not today"
-            echo "Changelog: $date_changelog"
+            printf "Date in changelog is not today\n"
+            printf "Changelog date: %s\n" "${date_changelog}"
             exit 1
         fi
     ;;
@@ -76,54 +126,17 @@ case "$option" in
         "$0" test
     ;;
 
-    blog)
-        version=$1; shift
-        blogfile=$1; shift
-        dir=${blogfile%/*}
-        file=${blogfile##*/}
-
-
-        cat << eof > "$blogfile"
-[[!meta title="Cdist $version released"]]
-
-Here's a short overview about the changes found in version ${version}:
-
-eof
-
-        $0 changelog-changes "$version" >> "$blogfile"
-
-        cat << eof >> "$blogfile"
-For more information visit the [[cdist homepage|software/cdist]].
-
-[[!tag cdist config unix]]
-eof
-        cd "$dir"
-        git add "$file"
-        # Allow git commit to fail if there are no changes
-        git commit -m "cdist blog update: $version" "$blogfile" || true
-    ;;
-
     ml-release)
         if [ $# -ne 1 ]; then
-            echo "$0 ml-release version" >&2
+            printf "%s ml-release version\n" "$0" >&2
             exit 1
         fi
 
         version=$1; shift
 
-        to_a=cdist
-        to_d=l.schottelius.org
-        to=${to_a}@${to_d}
-
-        from_a=nico-cdist
-        from_d=schottelius.org
-        from=${from_a}@${from_d}
-
         ( 
         cat << eof
-From: Nico -telmich- Schottelius <$from>
-To: cdist mailing list <$to>
-Subject: cdist $version released
+Subject: cdist $version has been released
 
 Hello .*,
 
@@ -134,25 +147,41 @@ eof
         "$0" changelog-changes "$version"
         cat << eof
 
-Cheers,
-
-Nico
-
--- 
-Automatisation at its best level. With cdist.
 eof
-        ) | /usr/sbin/sendmail -f "$from" "$to"
+        ) > mailinglist.tmp
     ;;
 
+    archlinux-release)
+        if [ $# -ne 1 ]; then
+            printf "%s archlinux-release version\n" "$0" >&2
+            exit 1
+        fi
+        version=$1; shift
+
+        ARCHLINUXTAR="cdist-${version}-1.src.tar.gz"
+        ./PKGBUILD.in "${version}"
+        umask 022
+        mkaurball
+        burp -c system "${ARCHLINUXTAR}"
+    ;;
+
+    pypi-release)
+        # Ensure that pypi release has the right version
+        "$0" version
+
+        make docs-clean
+        make docs
+        python3 setup.py sdist upload
+    ;;
 
     release-git-tag)
         target_version=$($0 changelog-version)
-        if git rev-parse --verify refs/tags/$target_version 2>/dev/null; then
-            echo "Tag for $target_version exists, aborting"
+        if git rev-parse --verify "refs/tags/${target_version}" 2>/dev/null; then
+            printf "Tag for %s exists, aborting\n" "${target_version}"
             exit 1
         fi
-        printf "Enter tag description for ${target_version}: "
-        read tagmessage
+        printf "Enter tag description for %s: " "${target_version}"
+        read -r tagmessage
 
         # setup for signed tags:
         # gpg --fulL-gen-key
@@ -170,7 +199,8 @@ eof
         # gpg --verify <asc-file> <file>
         # gpg --no-default-keyring --keyring <pubkey.gpg> --verify <asc-file> <file>
         # Ensure gpg-agent is running.
-        export GPG_TTY=$(tty)
+        GPG_TTY=$(tty)
+        export GPG_TTY
         gpg-agent
 
         git tag -s "$target_version" -m "$tagmessage"
@@ -180,14 +210,14 @@ eof
     sign-git-release)
         if [ $# -lt 2 ]
         then
-            printf "usage: $0 sign-git-release TAG TOKEN [ARCHIVE]\n"
+            printf "usage: %s sign-git-release TAG TOKEN [ARCHIVE]\n" "$0"
             printf "    if ARCHIVE is not specified then it is created\n"
             exit 1
         fi
         tag="$1"
         if ! git rev-parse -q --verify "${tag}" >/dev/null 2>&1
         then
-            printf "Tag \"${tag}\" not found.\n"
+            printf "Tag \"%s\" not found.\n" "${tag}"
             exit 1
         fi
         token="$2"
@@ -195,44 +225,53 @@ eof
         then
             archivename="$3"
         else
-            archivename="cdist-${tag}.tar.gz"
+            archivename="cdist-${tag}.tar"
             git archive --prefix="cdist-${tag}/" -o "${archivename}" "${tag}" \
                 || exit 1
+            # make sure target version is generated
+            "$0" target-version
+            tar -x -f "${archivename}" || exit 1
+            cp cdist/version.py "cdist-${tag}/cdist/version.py" || exit 1
+            tar -c -f "${archivename}" "cdist-${tag}/" || exit 1
+            rm -r -f "cdist-${tag}/"
+            gzip "${archivename}" || exit 1
+            archivename="${archivename}.gz"
         fi
         gpg --armor --detach-sign "${archivename}" || exit 1
 
-        # make github release
-        curl -H "Authorization: token ${token}" \
-            --request POST \
-            --data "{ \"tag_name\":\"${tag}\", \
-                      \"target_commitish\":\"master\", \
-                      \"name\": \"${tag}\", \
-                      \"body\":\"${tag}\", \
-                      \"draft\":false, \
-                      \"prerelease\": false}" \
-            "https://api.github.com/repos/ungleich/cdist/releases" || exit 1
+        project="ungleich-public%2Fcdist"
+        sed_cmd='s/^.*"markdown":"\([^"]*\)".*$/\1/'
 
-        # get release ID
-        repoid=$(curl "https://api.github.com/repos/ungleich/cdist/releases/tags/${tag}" \
-            | python3 -c 'import json; import sys; print(json.loads(sys.stdin.read())["id"])') \
-            || exit 1
+        # upload archive
+        response_archive=$(curl -f -X POST \
+             --http1.1 \
+             -H "PRIVATE-TOKEN: ${token}" \
+             -F "file=@${archivename}" \
+             "https://code.ungleich.ch/api/v4/projects/${project}/uploads" \
+             | sed "${sed_cmd}") || exit 1
 
-        # upload archive and then signature
-        curl -H "Authorization: token ${token}" \
-             -H "Accept: application/vnd.github.manifold-preview" \
-             -H "Content-Type: application/x-gtar" \
-             --data-binary @${archivename} \
-            "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}" \
-            || exit 1
-        curl -H "Authorization: token ${token}" \
-             -H "Accept: application/vnd.github.manifold-preview" \
-             -H "Content-Type: application/pgp-signature" \
-             --data-binary @${archivename}.asc \
-            "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}.asc" \
+        # upload archive signature
+        response_archive_sig=$(curl -f -X POST \
+             --http1.1 \
+             -H "PRIVATE-TOKEN: ${token}" \
+             -F "file=@${archivename}.asc" \
+            "https://code.ungleich.ch/api/v4/projects/${project}/uploads" \
+             | sed "${sed_cmd}") || exit 1
+
+        # make release
+        changelog=$("$0" changelog-changes "$1" | sed 's/^[[:space:]]*//')
+        release_notes=$(
+            printf "%s\n\n%s\n\n**Changelog**\n\n%s\n" \
+                "${response_archive}" "${response_archive_sig}" "${changelog}"
+        )
+        curl -f -X POST \
+             -H "PRIVATE-TOKEN: ${token}" \
+             -F "description=${release_notes}" \
+            "https://code.ungleich.ch/api/v4/projects/${project}/repository/tags/${tag}/release" \
             || exit 1
 
         # remove generated files (archive and asc)
-        if [ $# -eq 2]
+        if [ $# -eq 2 ]
         then
             rm -f "${archivename}"
         fi
@@ -244,29 +283,30 @@ eof
         target_version=$($0 changelog-version)
         target_branch=$($0 version-branch)
 
-        echo "Beginning release process for $target_version"
+        printf "Beginning release process for %s\n" "${target_version}"
 
         # First check everything is sane
         "$0" check-date
         "$0" check-unittest
-        "$0" check-pep8
+        "$0" check-pycodestyle
+        "$0" check-shellcheck
 
         # Generate version file to be included in packaging
         "$0" target-version
 
         # Ensure the git status is clean, else abort
         if ! git diff-index --name-only --exit-code HEAD ; then
-            echo "Unclean tree, see files above, aborting"
+            printf "Unclean tree, see files above, aborting.\n"
             exit 1
         fi
 
         # Ensure we are on the master branch
         masterbranch=yes
         if [ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then
-            echo "Releases are happening from the master branch, aborting"
+            printf "Releases are happening from the master branch, aborting.\n"
 
-            echo "Enter the magic word to release anyway"
-            read magicword
+            printf "Enter the magic word to release anyway:"
+            read -r magicword
 
             if [ "$magicword" = "iknowwhatido" ]; then
                 masterbranch=no
@@ -277,7 +317,7 @@ eof
 
         if [ "$masterbranch" = yes ]; then
             # Ensure version branch exists
-            if ! git rev-parse --verify refs/heads/$target_branch 2>/dev/null; then
+            if ! git rev-parse --verify "refs/heads/${target_branch}" 2>/dev/null; then
                 git branch "$target_branch"
             fi
 
@@ -295,20 +335,12 @@ eof
         make docs-clean
         make docs
 
-        # Generate speeches (indirect check if they build)
-        make speeches
-
         ############################################################# 
         # Everything green, let's do the release
 
         # Tag the current commit
         "$0" release-git-tag
 
-        # sign git tag
-        printf "Enter github authentication token: "
-        read token
-        "$0" sign-git-release "${target_version}" "${token}"
-
         # Also merge back the version branch
         if [ "$masterbranch" = yes ]; then
             git checkout master
@@ -316,41 +348,41 @@ eof
         fi
 
         # Publish git changes
-        make pub
-
-        # publish man, speeches, website
-        if [ "$masterbranch" = yes ]; then
-            make web-release-all
-        else
-            make web-release-all-no-latest
-        fi
-
-        # Ensure that pypi release has the right version
-        "$0" version
+        # if you want to have mirror locally then uncomment this and comment below
+        #     git push --mirror
+            git push
+            # push also new branch and set up tracking
+            git push -u origin "${target_branch}"
+        # fi
 
         # Create and publish package for pypi
-        make pypi-release
+        "$0" pypi-release
 
-        # Archlinux release is based on pypi
-        make archlinux-release
+        # sign git tag
+        printf "Enter upstream repository authentication token: "
+        read -r token
+        "$0" sign-git-release "${target_version}" "${token}"
 
         # Announce change on ML
-        make ml-release
+        "$0" ml-release "${target_version}"
 
         cat << eof
 Manual steps post release:
-
-    - linkedin
-    - hackernews
-    - reddit
+    - cdist-web
+    - send mail body generated in mailinglist.tmp and inform Dmitry for deb
     - twitter
-
 eof
-
     ;;
 
     test)
-        export PYTHONPATH="$(pwd -P)"
+        if [ ! -f "cdist/version.py" ]
+        then
+            printf "cdist/version.py is missing, generate it first.\n"
+            exit 1
+        fi
+
+        PYTHONPATH="$(pwd -P)"
+        export PYTHONPATH
 
         if [ $# -lt 1 ]; then
             python3 -m cdist.test
@@ -359,18 +391,31 @@ eof
         fi
     ;;
 
-    pep8)
-        pep8 "${basedir}" "${basedir}/scripts/cdist" | less
+    test-remote)
+        if [ ! -f "cdist/version.py" ]
+        then
+            printf "cdist/version.py is missing, generate it first.\n"
+            exit 1
+        fi
+
+        PYTHONPATH="$(pwd -P)"
+        export PYTHONPATH
+
+        python3 -m cdist.test.exec.remote
     ;;
 
-    check-pep8)
-        "$0" pep8
-        echo "Please review pep8 report."
+    pycodestyle|pep8)
+        pycodestyle "${basedir}" "${basedir}/scripts/cdist" | less
+    ;;
+
+    check-pycodestyle)
+        "$0" pycodestyle
+        printf "\\nPlease review pycodestyle report.\\n"
         while true
         do
-            echo "Continue (yes/no)?"
+            printf "Continue (yes/no)?\n"
             any=
-            read any
+            read -r any
             case "$any" in
                 yes)
                     break
@@ -379,7 +424,83 @@ eof
                     exit 1
                 ;;
                 *)
-                    echo "Please answer with 'yes' or 'no' explicitly."
+                    printf "Please answer with 'yes' or 'no' explicitly.\n"
+                ;;
+        esac
+        done
+    ;;
+
+    shellcheck-global-explorers)
+        find cdist/conf/explorer -type f -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-type-explorers)
+        find cdist/conf/type -type f -path "*/explorer/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-manifests)
+        find cdist/conf/type -type f -name manifest -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-local-gencodes)
+        find cdist/conf/type -type f -name gencode-local -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-remote-gencodes)
+        find cdist/conf/type -type f -name gencode-remote -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-scripts)
+        ${SHELLCHECKCMD} scripts/cdist-dump scripts/cdist-new-type || exit 0
+    ;;
+
+    shellcheck-gencodes)
+        "$0" shellcheck-local-gencodes
+        "$0" shellcheck-remote-gencodes
+    ;;
+
+    shellcheck-types)
+        "$0" shellcheck-type-explorers
+        "$0" shellcheck-manifests
+        "$0" shellcheck-gencodes
+    ;;
+
+    shellcheck)
+        "$0" shellcheck-global-explorers
+        "$0" shellcheck-types
+        "$0" shellcheck-scripts
+    ;;
+
+    shellcheck-type-files)
+        find cdist/conf/type -type f -path "*/files/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-with-files)
+        "$0" shellcheck
+        "$0" shellcheck-type-files
+    ;;
+
+    shellcheck-build-helper)
+        ${SHELLCHECKCMD} ./bin/build-helper
+    ;;
+
+    check-shellcheck)
+        "$0" shellcheck
+        printf "\\nPlease review shellcheck report.\\n"
+        while true
+        do
+            printf "Continue (yes/no)?\n"
+            any=
+            read -r any
+            case "$any" in
+                yes)
+                    break
+                ;;
+                no)
+                    exit 1
+                ;;
+                *)
+                    printf "Please answer with 'yes' or 'no' explicitly.\n"
                 ;;
         esac
         done
@@ -390,16 +511,39 @@ eof
     ;;
 
     version)
-        echo "VERSION = \"$(git describe)\"" > cdist/version.py
+        printf "VERSION = \"%s\"\n" "$(git describe)" > cdist/version.py
     ;;
 
     target-version)
         target_version=$($0 changelog-version)
-        echo "VERSION = \"${target_version}\"" > cdist/version.py
+        printf "VERSION = \"%s\"\n" "${target_version}" > cdist/version.py
     ;;
 
+    clean)
+        make clean
+
+        # Archlinux
+        rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz
+        rm -rf pkg/ src/
+
+        rm -f MANIFEST PKGBUILD
+        rm -rf dist/
+
+        # Signed release
+        rm -f cdist-*.tar.gz
+        rm -f cdist-*.tar.gz.asc
+
+        # Temp files
+        rm -f ./*.tmp
+    ;;
+
+    distclean)
+        "$0" clean
+        rm -f cdist/version.py
+    ;;
     *)
-        echo "Unknown helper target $@ - aborting"
+        printf "Unknown target: '%s'.\n" "${option}" >&2
+        usage "${basename}"
         exit 1
     ;;
 

bin/build-helper.freebsd

@@ -1,468 +0,0 @@
-#!/bin/sh
-#
-# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
-# 2016 Darko Poljak (darko.poljak at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# This file contains the heavy lifting found usually in the Makefile
-#
-
-# vars for make
-helper=$0
-
-basedir=${0%/*}/../
-# run_as is used to check how the script is called (by $0 value)
-# currently supported sufixes for $0 are:
-# .freebsd - run as freebsd
-basename=${0##*/}
-run_as=${basename#*.}
-case "$run_as" in
-    freebsd)
-        to_a=cdist-configuration-management
-        to_d=googlegroups.com
-        from_a=darko.poljak
-        from_d=gmail.com
-        ml_name="Darko Poljak"
-        ml_sig_name="Darko"
-
-        # vars for make
-        WEBDIR=../vcs/www.nico.schottelius.org
-    ;;
-    *)
-        to_a=cdist
-        to_d=l.schottelius.org
-        from_a=nico-cdist
-        from_d=schottelius.org
-        ml_name="Nico -telmich- Schottelius"
-        ml_sig_name="Nico"
-
-        # vars for make
-        WEBDIR=$$HOME/vcs/www.nico.schottelius.org
-    ;;
-esac
-
-# Change to checkout directory
-cd "$basedir"
-
-version=$(git describe)
-
-option=$1; shift
-
-case "$option" in
-    print-make-vars)
-        printf "helper: ${helper}\n"
-        printf "WEBDIR: ${WEBDIR}\n"
-    ;;
-    print-runas)
-        printf "run_as: $run_as\n"
-    ;;
-    changelog-changes)
-        if [ "$#" -eq 1 ]; then
-            start=$1
-        else
-            start="[[:digit:]]"
-        fi
-
-        end="[[:digit:]]"
-
-        awk -F: "BEGIN { start=0 }
-            {
-                if(start == 0) {
-                    if (\$0 ~ /^$start/) {
-                        start = 1
-                    }
-                } else {
-                    if (\$0 ~ /^$end/) {
-                        exit
-                    } else {
-                        print \$0 
-                    }
-                }
-            }" "$basedir/docs/changelog"
-    ;;
-
-    changelog-version)
-        # get version from changelog
-        grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/:.*//'
-    ;;
-
-    check-date)
-        # verify date in changelog is today
-        date_today="$(date +%Y-%m-%d)"
-        date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //')
-
-        if [ "$date_today" != "$date_changelog" ]; then
-            echo "Date in changelog is not today"
-            echo "Changelog: $date_changelog"
-            exit 1
-        fi
-    ;;
-
-    check-unittest)
-        "$0" test
-    ;;
-
-    blog)
-        version=$1; shift
-        blogfile=$1; shift
-        dir=${blogfile%/*}
-        file=${blogfile##*/}
-
-
-        cat << eof > "$blogfile"
-[[!meta title="Cdist $version released"]]
-
-Here's a short overview about the changes found in version ${version}:
-
-eof
-
-        $0 changelog-changes "$version" >> "$blogfile"
-
-        cat << eof >> "$blogfile"
-For more information visit the [[cdist homepage|software/cdist]].
-
-[[!tag cdist config unix]]
-eof
-        cd "$dir"
-        git add "$file"
-        # Allow git commit to fail if there are no changes
-        git commit -m "cdist blog update: $version" "$blogfile" || true
-    ;;
-
-    ml-release)
-        if [ $# -ne 1 ]; then
-            echo "$0 ml-release version" >&2
-            exit 1
-        fi
-
-        version=$1; shift
-
-        to=${to_a}@${to_d}
-        from=${from_a}@${from_d}
-
-        ( 
-        cat << eof
-From: ${ml_name} <$from>
-To: cdist mailing list <$to>
-Subject: cdist $version released
-
-Hello .*,
-
-cdist $version has been released with the following changes:
-
-eof
-
-        "$0" changelog-changes "$version"
-        cat << eof
-
-Cheers,
-
-${ml_sig_name}
-
--- 
-Automatisation at its best level. With cdist.
-eof
-        ) | /usr/sbin/sendmail -f "$from" "$to"
-    ;;
-
-    release-git-tag)
-        target_version=$($0 changelog-version)
-        if git rev-parse --verify refs/tags/$target_version 2>/dev/null; then
-            echo "Tag for $target_version exists, aborting"
-            exit 1
-        fi
-        printf "Enter tag description for ${target_version}: "
-        read tagmessage
-
-        # setup for signed tags:
-        # gpg --fulL-gen-key
-        # gpg --list-secret-keys --keyid-format LONG
-        # git config --local user.signingkey <id>
-        # for exporting pub key:
-        #     gpg --armor --export <id> > pubkey.asc
-        #     gpg --output pubkey.gpg --export <id>
-        # show tag with signature
-        # git show <tag>
-        # verify tag signature
-        # git tag -v <tag>
-        #
-        # gpg verify signature
-        # gpg --verify <asc-file> <file>
-        # gpg --no-default-keyring --keyring <pubkey.gpg> --verify <asc-file> <file>
-        # Ensure gpg-agent is running.
-        export GPG_TTY=$(tty)
-        gpg-agent
-
-        git tag -s "$target_version" -m "$tagmessage"
-        git push --tags
-    ;;
-
-    sign-git-release)
-        if [ $# -lt 2 ]
-        then
-            printf "usage: $0 sign-git-release TAG TOKEN [ARCHIVE]\n"
-            printf "    if ARCHIVE is not specified then it is created\n"
-            exit 1
-        fi
-        tag="$1"
-        if ! git rev-parse -q --verify "${tag}" >/dev/null 2>&1
-        then
-            printf "Tag \"${tag}\" not found.\n"
-            exit 1
-        fi
-        token="$2"
-        if [ $# -gt 2 ]
-        then
-            archivename="$3"
-        else
-            archivename="cdist-${tag}.tar.gz"
-            git archive --prefix="cdist-${tag}/" -o "${archivename}" "${tag}" \
-                || exit 1
-        fi
-        gpg --armor --detach-sign "${archivename}" || exit 1
-
-        # make github release
-        curl -H "Authorization: token ${token}" \
-            --request POST \
-            --data "{ \"tag_name\":\"${tag}\", \
-                      \"target_commitish\":\"master\", \
-                      \"name\": \"${tag}\", \
-                      \"body\":\"${tag}\", \
-                      \"draft\":false, \
-                      \"prerelease\": false}" \
-            "https://api.github.com/repos/ungleich/cdist/releases" || exit 1
-
-        # get release ID
-        repoid=$(curl "https://api.github.com/repos/ungleich/cdist/releases/tags/${tag}" \
-            | python3 -c 'import json; import sys; print(json.loads(sys.stdin.read())["id"])') \
-            || exit 1
-
-        # upload archive and then signature
-        curl -H "Authorization: token ${token}" \
-             -H "Accept: application/vnd.github.manifold-preview" \
-             -H "Content-Type: application/x-gtar" \
-             --data-binary @${archivename} \
-            "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}" \
-            || exit 1
-        curl -H "Authorization: token ${token}" \
-             -H "Accept: application/vnd.github.manifold-preview" \
-             -H "Content-Type: application/pgp-signature" \
-             --data-binary @${archivename}.asc \
-            "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}.asc" \
-            || exit 1
-
-        # remove generated files (archive and asc)
-        if [ $# -eq 2]
-        then
-            rm -f "${archivename}"
-        fi
-        rm -f "${archivename}.asc"
-    ;;
-
-    release)
-        set -e
-        target_version=$($0 changelog-version)
-        target_branch=$($0 version-branch)
-
-        echo "Beginning release process for $target_version"
-
-        # First check everything is sane
-        "$0" check-date
-        "$0" check-unittest
-        "$0" check-pep8
-
-        # Generate version file to be included in packaging
-        "$0" target-version
-
-        # Ensure the git status is clean, else abort
-        if ! git diff-index --name-only --exit-code HEAD ; then
-            echo "Unclean tree, see files above, aborting"
-            exit 1
-        fi
-
-        # Ensure we are on the master branch
-        masterbranch=yes
-        if [ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then
-            echo "Releases are happening from the master branch, aborting"
-
-            echo "Enter the magic word to release anyway"
-            read magicword
-
-            if [ "$magicword" = "iknowwhatido" ]; then
-                masterbranch=no
-            else
-                exit 1
-            fi
-        fi
-
-        if [ "$masterbranch" = yes ]; then
-            # Ensure version branch exists
-            if ! git rev-parse --verify refs/heads/$target_branch 2>/dev/null; then
-                git branch "$target_branch"
-            fi
-
-            # Merge master branch into version branch
-            git checkout "$target_branch"
-            git merge master
-        fi
-
-        # Verify that after the merge everything works
-        "$0" check-date
-        "$0" check-unittest
-
-        # Generate documentation (man and html)
-        # First, clean old generated docs
-        make helper=${helper} WEBDIR=${WEBDIR} docs-clean
-        make helper=${helper} WEBDIR=${WEBDIR} docs
-
-        # Generate speeches (indirect check if they build)
-        make helper=${helper} WEBDIR=${WEBDIR} speeches
-
-        ############################################################# 
-        # Everything green, let's do the release
-
-        # Tag the current commit
-        "$0" release-git-tag
-
-        # sign git tag
-        printf "Enter github authentication token: "
-        read token
-        "$0" sign-git-release "${target_version}" "${token}"
-
-        # Also merge back the version branch
-        if [ "$masterbranch" = yes ]; then
-            git checkout master
-            git merge "$target_branch"
-        fi
-
-        # Publish git changes
-        case "$run_as" in
-            freebsd)
-                # if we are not Nico :) then just push, no mirror
-                git push
-                # push also new branch and set up tracking
-                git push -u origin "${target_branch}"
-            ;;
-            *)
-                make helper=${helper} WEBDIR=${WEBDIR} pub
-            ;;
-        esac
-
-        # publish man, speeches, website
-        if [ "$masterbranch" = yes ]; then
-            make helper=${helper} WEBDIR=${WEBDIR} web-release-all
-        else
-            make helper=${helper} WEBDIR=${WEBDIR} web-release-all-no-latest
-        fi
-
-        # Ensure that pypi release has the right version
-        "$0" version
-
-        # Create and publish package for pypi
-        make helper=${helper} WEBDIR=${WEBDIR} pypi-release
-
-        case "$run_as" in
-            freebsd)
-            ;;
-            *)
-                # Archlinux release is based on pypi
-                make archlinux-release
-            ;;
-        esac
-
-        # Announce change on ML
-        make helper=${helper} WEBDIR=${WEBDIR} ml-release
-
-        cat << eof
-Manual steps post release:
-
-    - linkedin
-    - hackernews
-    - reddit
-    - twitter
-
-eof
-
-        case "$run_as" in
-            freebsd)
-                cat <<eof
-Additional steps post release:
-    - archlinux release
-eof
-            ;;
-            *)
-            ;;
-        esac
-
-    ;;
-
-    test)
-        export PYTHONPATH="$(pwd -P)"
-
-        if [ $# -lt 1 ]; then
-            python3 -m cdist.test
-        else
-            python3 -m unittest "$@"
-        fi
-    ;;
-
-    pep8)
-        pep8 "${basedir}" "${basedir}/scripts/cdist" | less
-    ;;
-
-    check-pep8)
-        "$0" pep8
-        echo "Please review pep8 report."
-        while true
-        do
-            echo "Continue (yes/no)?"
-            any=
-            read any
-            case "$any" in
-                yes)
-                    break
-                ;;
-                no)
-                    exit 1
-                ;;
-                *)
-                    echo "Please answer with 'yes' or 'no' explicitly."
-                ;;
-        esac
-        done
-    ;;
-
-    version-branch)
-        "$0" changelog-version | cut -d. -f '1,2'
-    ;;
-
-    version)
-        echo "VERSION = \"$(git describe)\"" > cdist/version.py
-    ;;
-
-    target-version)
-        target_version=$($0 changelog-version)
-        echo "VERSION = \"${target_version}\"" > cdist/version.py
-    ;;
-
-    *)
-        echo "Unknown helper target $@ - aborting"
-        exit 1
-    ;;
-
-esac

cdist/__init__.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
+# 2012-2017 Steven Armstrong (steven-cdist at armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -20,9 +21,9 @@
 #
 
 import os
-import subprocess
 import hashlib
 
+import cdist.log
 import cdist.version
 
 VERSION = cdist.version.VERSION
@@ -42,8 +43,9 @@ BANNER = """
    "P'        ""         ""
 """
 
-REMOTE_COPY = "scp -o User=root"
+REMOTE_COPY = "scp -o User=root -q"
 REMOTE_EXEC = "ssh -o User=root"
+REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
 
 
 class Error(Exception):
@@ -79,18 +81,159 @@ class CdistBetaRequired(cdist.Error):
         return err_msg.format(*fmt_args)
 
 
-class CdistObjectError(Error):
-    """Something went wrong with an object"""
+class CdistEntityError(Error):
+    """Something went wrong while executing cdist entity"""
+    def __init__(self, entity_name, entity_params, stdout_paths,
+                 stderr_paths, subject=''):
+        self.entity_name = entity_name
+        self.entity_params = entity_params
+        self.stderr_paths = stderr_paths
+        self.stdout_paths = stdout_paths
+        if isinstance(subject, Error):
+            self.original_error = subject
+        else:
+            self.original_error = None
+        self.message = str(subject)
 
-    def __init__(self, cdist_object, message):
-        self.name = cdist_object.name
-        self.source = " ".join(cdist_object.source)
-        self.message = message
+    def _stdpath(self, stdpaths, header_name):
+        result = {}
+        for name, path in stdpaths:
+            if name not in result:
+                result[name] = []
+            try:
+                if os.path.exists(path) and os.path.getsize(path) > 0:
+                    output = []
+                    label_begin = name + ":" + header_name
+                    output.append(label_begin)
+                    output.append('\n')
+                    output.append('-' * len(label_begin))
+                    output.append('\n')
+                    with open(path, 'r') as fd:
+                        output.append(fd.read())
+                    output.append('\n')
+                    result[name].append(''.join(output))
+            except UnicodeError as ue:
+                result[name].append(('Cannot output {}:{} due to: {}.\n'
+                                     'You can try to read the error file "{}"'
+                                     ' yourself.').format(
+                                         name, header_name, ue, path))
+        return result
+
+    def _stderr(self):
+        return self._stdpath(self.stderr_paths, 'stderr')
+
+    def _stdout(self):
+        return self._stdpath(self.stdout_paths, 'stdout')
+
+    def _update_dict_list(self, target, source):
+        for x in source:
+            if x not in target:
+                target[x] = []
+            target[x].extend(source[x])
+
+    @property
+    def std_streams(self):
+        std_dict = {}
+        self._update_dict_list(std_dict, self._stdout())
+        self._update_dict_list(std_dict, self._stderr())
+        return std_dict
 
     def __str__(self):
-        return '%s: %s (defined at %s)' % (self.name,
-                                           self.message,
-                                           self.source)
+        output = []
+        output.append(self.message)
+        output.append('\n\n')
+        header = "Error processing " + self.entity_name
+        under_header = '=' * len(header)
+        output.append(header)
+        output.append('\n')
+        output.append(under_header)
+        output.append('\n')
+        for param_name, param_value in self.entity_params:
+            output.append(param_name + ': ' + str(param_value))
+            output.append('\n')
+        output.append('\n')
+        for x in self.std_streams:
+            output.append(''.join(self.std_streams[x]))
+        return ''.join(output)
+
+
+class CdistObjectError(CdistEntityError):
+    """Something went wrong while working on a specific cdist object"""
+    def __init__(self, cdist_object, subject=''):
+        params = [
+            ('name', cdist_object.name, ),
+            ('path', cdist_object.absolute_path, ),
+            ('source', " ".join(cdist_object.source), ),
+            ('type', os.path.realpath(
+                cdist_object.cdist_type.absolute_path), ),
+        ]
+        stderr_paths = []
+        for stderr_name in os.listdir(cdist_object.stderr_path):
+            stderr_path = os.path.join(cdist_object.stderr_path,
+                                       stderr_name)
+            stderr_paths.append((stderr_name, stderr_path, ))
+        stdout_paths = []
+        for stdout_name in os.listdir(cdist_object.stdout_path):
+            stdout_path = os.path.join(cdist_object.stdout_path,
+                                       stdout_name)
+            stdout_paths.append((stdout_name, stdout_path, ))
+        super().__init__("object '{}'".format(cdist_object.name),
+                         params, stdout_paths, stderr_paths, subject)
+
+
+class CdistObjectExplorerError(CdistEntityError):
+    """
+    Something went wrong while working on a specific
+    cdist object explorer
+    """
+    def __init__(self, cdist_object, explorer_name, explorer_path,
+                 stderr_path, subject=''):
+        params = [
+            ('object name', cdist_object.name, ),
+            ('object path', cdist_object.absolute_path, ),
+            ('object source', " ".join(cdist_object.source), ),
+            ('object type', os.path.realpath(
+                cdist_object.cdist_type.absolute_path), ),
+            ('explorer name', explorer_name, ),
+            ('explorer path', explorer_path, ),
+        ]
+        stdout_paths = []
+        stderr_paths = [
+            ('remote', stderr_path, ),
+        ]
+        super().__init__("explorer '{}' of object '{}'".format(
+            explorer_name, cdist_object.name), params, stdout_paths,
+            stderr_paths, subject)
+
+
+class InitialManifestError(CdistEntityError):
+    """Something went wrong while executing initial manifest"""
+    def __init__(self, initial_manifest, stdout_path, stderr_path, subject=''):
+        params = [
+            ('path', initial_manifest, ),
+        ]
+        stdout_paths = [
+            ('init', stdout_path, ),
+        ]
+        stderr_paths = [
+            ('init', stderr_path, ),
+        ]
+        super().__init__('initial manifest', params, stdout_paths,
+                         stderr_paths, subject)
+
+
+class GlobalExplorerError(CdistEntityError):
+    """Something went wrong while executing global explorer"""
+    def __init__(self, name, path, stderr_path, subject=''):
+        params = [
+            ('name', name, ),
+            ('path', path, ),
+        ]
+        stderr_paths = [
+            ('remote', stderr_path, ),
+        ]
+        super().__init__("global explorer '{}'".format(name),
+                         params, [], stderr_paths, subject)
 
 
 def file_to_list(filename):
@@ -114,3 +257,15 @@ def str_hash(s):
         return hashlib.md5(s.encode('utf-8')).hexdigest()
     else:
         raise Error("Param should be string")
+
+
+def home_dir():
+    if 'HOME' in os.environ:
+        home = os.environ['HOME']
+        if home:
+            rv = os.path.join(home, ".cdist")
+        else:
+            rv = None
+    else:
+        rv = None
+    return rv

cdist/argparse.py

@@ -1,29 +1,49 @@
 import argparse
 import cdist
 import multiprocessing
-import os
 import logging
 import collections
+import functools
+import cdist.configuration
+import cdist.preos
 
 
 # set of beta sub-commands
-BETA_COMMANDS = set(('install', ))
+BETA_COMMANDS = set(('install', 'inventory', ))
 # set of beta arguments for sub-commands
 BETA_ARGS = {
-    'config': set(('jobs', )),
+    'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
 }
-EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
+EPILOG = "Get cdist at https://code.ungleich.ch/ungleich-public/cdist"
 # Parser others can reuse
 parser = None
 
 
+_verbosity_level_off = -2
 _verbosity_level = {
-    0: logging.ERROR,
-    1: logging.WARNING,
-    2: logging.INFO,
+    None: logging.WARNING,
+    _verbosity_level_off: logging.OFF,
+    -1: logging.ERROR,
+    0: logging.WARNING,
+    1: logging.INFO,
+    2: logging.VERBOSE,
+    3: logging.DEBUG,
+    4: logging.TRACE,
 }
+
+
+# Generate verbosity level constants:
+# VERBOSE_OFF, VERBOSE_ERROR, VERBOSE_WARNING, VERBOSE_INFO, VERBOSE_VERBOSE,
+# VERBOSE_DEBUG, VERBOSE_TRACE.
+this_globals = globals()
+for level in _verbosity_level:
+    const = 'VERBOSE_' + logging.getLevelName(_verbosity_level[level])
+    this_globals[const] = level
+
+
+# All verbosity levels above 4 are TRACE.
 _verbosity_level = collections.defaultdict(
-    lambda: logging.DEBUG, _verbosity_level)
+    lambda: logging.TRACE, _verbosity_level)
 
 
 def add_beta_command(cmd):
@@ -55,17 +75,15 @@ def check_beta(args_dict):
                     raise cdist.CdistBetaRequired(cmd, arg)
 
 
-def check_positive_int(value):
-    import argparse
-
+def check_lower_bounded_int(value, lower_bound, name):
     try:
         val = int(value)
     except ValueError:
         raise argparse.ArgumentTypeError(
                 "{} is invalid int value".format(value))
-    if val <= 0:
+    if val < lower_bound:
         raise argparse.ArgumentTypeError(
-                "{} is invalid positive int value".format(val))
+                "{} is invalid {} value".format(val, name))
     return val
 
 
@@ -80,31 +98,43 @@ def get_parsers():
     # Options _all_ parsers have in common
     parser['loglevel'] = argparse.ArgumentParser(add_help=False)
     parser['loglevel'].add_argument(
-            '-d', '--debug',
-            help=('Set log level to debug (deprecated, use -vvv instead)'),
+            '-l', '--log-level', metavar='LOGLEVEL',
+            type=functools.partial(check_lower_bounded_int, lower_bound=-1,
+                                   name="log level"),
+            help=('Set the specified verbosity level. '
+                  'The levels, in order from the lowest to the highest, are: '
+                  'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) '
+                  'TRACE (4 or higher). If used along with -v then -v '
+                  'increases last set value and -l overwrites last set '
+                  'value.'),
+            action='store', dest='verbose', required=False)
+    parser['loglevel'].add_argument(
+            '-q', '--quiet',
+            help='Quiet mode: disables logging, including WARNING and ERROR.',
             action='store_true', default=False)
     parser['loglevel'].add_argument(
             '-v', '--verbose',
             help=('Increase the verbosity level. Every instance of -v '
                   'increments the verbosity level by one. Its default value '
-                  'is 0. There are 4 levels of verbosity. The order of levels '
-                  'from the lowest to the highest are: ERROR (0), '
-                  'WARNING (1), INFO (2) and DEBUG (3 or higher).'),
-            action='count', default=0)
+                  'is 0 which includes ERROR and WARNING levels. '
+                  'The levels, in order from the lowest to the highest, are: '
+                  'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) '
+                  'TRACE (4 or higher). If used along with -l then -l '
+                  'overwrites last set value and -v increases last set '
+                  'value.'),
+            action='count', default=None)
 
     parser['beta'] = argparse.ArgumentParser(add_help=False)
     parser['beta'].add_argument(
            '-b', '--beta',
-           help=('Enable beta functionalities. '
-                 'Can also be enabled using CDIST_BETA env var.'),
-           action='store_true', dest='beta',
-           default='CDIST_BETA' in os.environ)
+           help=('Enable beta functionality. '),
+           action='store_true', dest='beta', default=None)
 
     # Main subcommand parser
     parser['main'] = argparse.ArgumentParser(
-            description='cdist ' + cdist.VERSION, parents=[parser['loglevel']])
+            description='cdist ' + cdist.VERSION)
     parser['main'].add_argument(
-            '-V', '--version', help='Show version', action='version',
+            '-V', '--version', help='Show version.', action='version',
             version='%(prog)s ' + cdist.VERSION)
     parser['sub'] = parser['main'].add_subparsers(
             title="Commands", dest="command")
@@ -114,67 +144,147 @@ def get_parsers():
             'banner', parents=[parser['loglevel']])
     parser['banner'].set_defaults(func=cdist.banner.banner)
 
+    parser['inventory_common'] = argparse.ArgumentParser(add_help=False)
+    parser['inventory_common'].add_argument(
+           '-I', '--inventory',
+           help=('Use specified custom inventory directory. '
+                 'Inventory directory is set up by the following rules: '
+                 'if cdist configuration resolves this value then specified '
+                 'directory is used, '
+                 'if HOME env var is set then ~/.cdist/inventory is '
+                 'used, otherwise distribution inventory directory is used.'),
+           dest="inventory_dir", required=False)
+
+    parser['common'] = argparse.ArgumentParser(add_help=False)
+    parser['common'].add_argument(
+           '-g', '--config-file',
+           help=('Use specified custom configuration file.'),
+           dest="config_file", required=False)
+
     # Config
     parser['config_main'] = argparse.ArgumentParser(add_help=False)
+    parser['config_main'].add_argument(
+           '-4', '--force-ipv4',
+           help=('Force to use IPv4 addresses only. No influence for custom'
+                 ' remote commands.'),
+           action='store_const', dest='force_ipv', const=4)
+    parser['config_main'].add_argument(
+           '-6', '--force-ipv6',
+           help=('Force to use IPv6 addresses only. No influence for custom'
+                 ' remote commands.'),
+           action='store_const', dest='force_ipv', const=6)
+    parser['config_main'].add_argument(
+            '-C', '--cache-path-pattern',
+            help=('Specify custom cache path pattern. If '
+                  'it is not set then default hostdir is used.'),
+            dest='cache_path_pattern',
+            default=None)
     parser['config_main'].add_argument(
             '-c', '--conf-dir',
             help=('Add configuration directory (can be repeated, '
-                  'last one wins)'), action='append')
+                  'last one wins).'), action='append')
     parser['config_main'].add_argument(
            '-i', '--initial-manifest',
-           help='path to a cdist manifest or \'-\' to read from stdin.',
+           help='Path to a cdist manifest or \'-\' to read from stdin.',
            dest='manifest', required=False)
     parser['config_main'].add_argument(
            '-j', '--jobs', nargs='?',
-           type=check_positive_int,
-           help=('Specify the maximum number of parallel jobs, currently '
-                 'only global explorers are supported'),
+           type=functools.partial(check_lower_bounded_int, lower_bound=1,
+                                  name="positive int"),
+           help=('Operate in parallel in specified maximum number of jobs. '
+                 'Global explorers, object prepare and object run are '
+                 'supported. Without argument CPU count is used by default. '),
            action='store', dest='jobs',
            const=multiprocessing.cpu_count())
     parser['config_main'].add_argument(
            '-n', '--dry-run',
-           help='do not execute code', action='store_true')
+           help='Do not execute code.', action='store_true')
     parser['config_main'].add_argument(
            '-o', '--out-dir',
-           help='directory to save cdist output in', dest="out_path")
+           help='Directory to save cdist output in.', dest="out_path")
+    parser['config_main'].add_argument(
+           '-P', '--timestamp',
+           help=('Timestamp log messages with the current local date and time '
+                 'in the format: YYYYMMDDHHMMSS.us.'),
+           action='store_true', dest='timestamp')
+    parser['config_main'].add_argument(
+           '-R', '--use-archiving', nargs='?',
+           choices=('tar', 'tgz', 'tbz2', 'txz',),
+           help=('Operate by using archiving with compression where '
+                 'appropriate. Supported values are: tar - tar archive, '
+                 'tgz - gzip tar archive (the default), '
+                 'tbz2 - bzip2 tar archive and txz - lzma tar archive. '
+                 'Currently in beta.'),
+           action='store', dest='use_archiving',
+           const='tgz')
 
     # remote-copy and remote-exec defaults are environment variables
     # if set; if not then None - these will be futher handled after
     # parsing to determine implementation default
+    parser['config_main'].add_argument(
+           '-r', '--remote-out-dir',
+           help='Directory to save cdist output in on the target host.',
+           dest="remote_out_path")
     parser['config_main'].add_argument(
            '--remote-copy',
-           help='Command to use for remote copy (should behave like scp)',
+           help='Command to use for remote copy (should behave like scp).',
            action='store', dest='remote_copy',
-           default=os.environ.get('CDIST_REMOTE_COPY'))
+           default=None)
     parser['config_main'].add_argument(
            '--remote-exec',
            help=('Command to use for remote execution '
-                 '(should behave like ssh)'),
+                 '(should behave like ssh).'),
            action='store', dest='remote_exec',
-           default=os.environ.get('CDIST_REMOTE_EXEC'))
+           default=None)
+    parser['config_main'].add_argument(
+           '-S', '--disable-saving-output-streams',
+           help='Disable saving output streams.',
+           action='store_false', dest='save_output_streams', default=True)
 
     # Config
     parser['config_args'] = argparse.ArgumentParser(add_help=False)
     parser['config_args'].add_argument(
-            'host', nargs='*', help='host(s) to operate on')
+             '-A', '--all-tagged',
+             help=('Use all hosts present in tags db. Currently in beta.'),
+             action="store_true", dest="all_tagged_hosts", default=False)
+    parser['config_args'].add_argument(
+             '-a', '--all',
+             help=('List hosts that have all specified tags, '
+                   'if -t/--tag is specified.'),
+             action="store_true", dest="has_all_tags", default=False)
     parser['config_args'].add_argument(
             '-f', '--file',
-            help=('Read additional hosts to operate on from specified file '
-                  'or from stdin if \'-\' (each host on separate line). '
-                  'If no host or host file is specified then, by default, '
-                  'read hosts from stdin.'),
+            help=('Read specified file for a list of additional hosts to '
+                  'operate on or if \'-\' is given, read stdin (one host per '
+                  'line). If no host or host file is specified then, by '
+                  'default, read hosts from stdin.'),
             dest='hostfile', required=False)
     parser['config_args'].add_argument(
-           '-p', '--parallel',
-           help='operate on multiple hosts in parallel',
-           action='store_true', dest='parallel')
+           '-p', '--parallel', nargs='?', metavar='HOST_MAX',
+           type=functools.partial(check_lower_bounded_int, lower_bound=1,
+                                  name="positive int"),
+           help=('Operate on multiple hosts in parallel for specified maximum '
+                 'hosts at a time. Without argument CPU count is used by '
+                 'default.'),
+           action='store', dest='parallel',
+           const=multiprocessing.cpu_count())
     parser['config_args'].add_argument(
            '-s', '--sequential',
-           help='operate on multiple hosts sequentially (default)',
-           action='store_false', dest='parallel')
+           help='Operate on multiple hosts sequentially (default).',
+           action='store_const', dest='parallel', const=0)
+    parser['config_args'].add_argument(
+             '-t', '--tag',
+             help=('Host is specified by tag, not hostname/address; '
+                   'list all hosts that contain any of specified tags. '
+                   'Currently in beta.'),
+             dest='tag', required=False, action="store_true", default=False)
+    parser['config_args'].add_argument(
+            'host', nargs='*', help='Host(s) to operate on.')
     parser['config'] = parser['sub'].add_parser(
             'config', parents=[parser['loglevel'], parser['beta'],
+                               parser['common'],
                                parser['config_main'],
+                               parser['inventory_common'],
                                parser['config_args']])
     parser['config'].set_defaults(func=cdist.config.Config.commandline)
 
@@ -183,6 +293,140 @@ def get_parsers():
                                                  parents=[parser['config']])
     parser['install'].set_defaults(func=cdist.install.Install.commandline)
 
+    # Inventory
+    parser['inventory'] = parser['sub'].add_parser('inventory')
+    parser['invsub'] = parser['inventory'].add_subparsers(
+            title="Inventory commands", dest="subcommand")
+
+    parser['add-host'] = parser['invsub'].add_parser(
+            'add-host', parents=[parser['loglevel'], parser['beta'],
+                                 parser['common'],
+                                 parser['inventory_common']])
+    parser['add-host'].add_argument(
+            'host', nargs='*', help='Host(s) to add.')
+    parser['add-host'].add_argument(
+           '-f', '--file',
+           help=('Read additional hosts to add from specified file '
+                 'or from stdin if \'-\' (each host on separate line). '
+                 'If no host or host file is specified then, by default, '
+                 'read from stdin.'),
+           dest='hostfile', required=False)
+
+    parser['add-tag'] = parser['invsub'].add_parser(
+            'add-tag', parents=[parser['loglevel'], parser['beta'],
+                                parser['common'],
+                                parser['inventory_common']])
+    parser['add-tag'].add_argument(
+           'host', nargs='*',
+           help='List of host(s) for which tags are added.')
+    parser['add-tag'].add_argument(
+           '-f', '--file',
+           help=('Read additional hosts to add tags from specified file '
+                 'or from stdin if \'-\' (each host on separate line). '
+                 'If no host or host file is specified then, by default, '
+                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
+                 ' are specified then tags are read from stdin and are'
+                 ' added to all hosts.'),
+           dest='hostfile', required=False)
+    parser['add-tag'].add_argument(
+           '-T', '--tag-file',
+           help=('Read additional tags to add from specified file '
+                 'or from stdin if \'-\' (each tag on separate line). '
+                 'If no tag or tag file is specified then, by default, '
+                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
+                 ' are specified then tags are read from stdin and are'
+                 ' added to all hosts.'),
+           dest='tagfile', required=False)
+    parser['add-tag'].add_argument(
+           '-t', '--taglist',
+           help=("Tag list to be added for specified host(s), comma separated"
+                 " values."),
+           dest="taglist", required=False)
+
+    parser['del-host'] = parser['invsub'].add_parser(
+            'del-host', parents=[parser['loglevel'], parser['beta'],
+                                 parser['common'],
+                                 parser['inventory_common']])
+    parser['del-host'].add_argument(
+            'host', nargs='*', help='Host(s) to delete.')
+    parser['del-host'].add_argument(
+            '-a', '--all', help=('Delete all hosts.'),
+            dest='all', required=False, action="store_true", default=False)
+    parser['del-host'].add_argument(
+            '-f', '--file',
+            help=('Read additional hosts to delete from specified file '
+                  'or from stdin if \'-\' (each host on separate line). '
+                  'If no host or host file is specified then, by default, '
+                  'read from stdin.'),
+            dest='hostfile', required=False)
+
+    parser['del-tag'] = parser['invsub'].add_parser(
+            'del-tag', parents=[parser['loglevel'], parser['beta'],
+                                parser['common'],
+                                parser['inventory_common']])
+    parser['del-tag'].add_argument(
+            'host', nargs='*',
+            help='List of host(s) for which tags are deleted.')
+    parser['del-tag'].add_argument(
+            '-a', '--all',
+            help=('Delete all tags for specified host(s).'),
+            dest='all', required=False, action="store_true", default=False)
+    parser['del-tag'].add_argument(
+            '-f', '--file',
+            help=('Read additional hosts to delete tags for from specified '
+                  'file or from stdin if \'-\' (each host on separate line). '
+                  'If no host or host file is specified then, by default, '
+                  'read from stdin. If no tags/tagfile nor hosts/hostfile'
+                  ' are specified then tags are read from stdin and are'
+                  ' deleted from all hosts.'),
+            dest='hostfile', required=False)
+    parser['del-tag'].add_argument(
+            '-T', '--tag-file',
+            help=('Read additional tags from specified file '
+                  'or from stdin if \'-\' (each tag on separate line). '
+                  'If no tag or tag file is specified then, by default, '
+                  'read from stdin. If no tags/tagfile nor'
+                  ' hosts/hostfile are specified then tags are read from'
+                  ' stdin and are added to all hosts.'),
+            dest='tagfile', required=False)
+    parser['del-tag'].add_argument(
+            '-t', '--taglist',
+            help=("Tag list to be deleted for specified host(s), "
+                  "comma separated values."),
+            dest="taglist", required=False)
+
+    parser['list'] = parser['invsub'].add_parser(
+            'list', parents=[parser['loglevel'], parser['beta'],
+                             parser['common'],
+                             parser['inventory_common']])
+    parser['list'].add_argument(
+            'host', nargs='*', help='Host(s) to list.')
+    parser['list'].add_argument(
+            '-a', '--all',
+            help=('List hosts that have all specified tags, '
+                  'if -t/--tag is specified.'),
+            action="store_true", dest="has_all_tags", default=False)
+    parser['list'].add_argument(
+            '-f', '--file',
+            help=('Read additional hosts to list from specified file '
+                  'or from stdin if \'-\' (each host on separate line). '
+                  'If no host or host file is specified then, by default, '
+                  'list all.'), dest='hostfile', required=False)
+    parser['list'].add_argument(
+            '-H', '--host-only', help=('Suppress tags listing.'),
+            action="store_true", dest="list_only_host", default=False)
+    parser['list'].add_argument(
+            '-t', '--tag',
+            help=('Host is specified by tag, not hostname/address; '
+                  'list all hosts that contain any of specified tags.'),
+            action="store_true", default=False)
+
+    parser['inventory'].set_defaults(
+            func=cdist.inventory.Inventory.commandline)
+
+    # PreOs
+    parser['preos'] = parser['sub'].add_parser('preos', add_help=False)
+
     # Shell
     parser['shell'] = parser['sub'].add_parser(
             'shell', parents=[parser['loglevel']])
@@ -199,12 +443,31 @@ def get_parsers():
 
 
 def handle_loglevel(args):
-    if args.debug:
-        retval = "-d/--debug is deprecated, use -vvv instead"
-        args.verbose = 3
-    else:
-        retval = None
+    if hasattr(args, 'quiet') and args.quiet:
+        args.verbose = _verbosity_level_off
 
     logging.root.setLevel(_verbosity_level[args.verbose])
 
-    return retval
+
+def parse_and_configure(argv, singleton=True):
+    parser = get_parsers()
+    parser_args = parser['main'].parse_args(argv)
+    try:
+        cfg = cdist.configuration.Configuration(parser_args,
+                                                singleton=singleton)
+        args = cfg.get_args()
+    except ValueError as e:
+        raise cdist.Error(str(e))
+    # Loglevels are handled globally in here
+    handle_loglevel(args)
+
+    log = logging.getLogger("cdist")
+
+    log.verbose("version %s" % cdist.VERSION)
+    log.trace('command line args: {}'.format(cfg.command_line_args))
+    log.trace('configuration: {}'.format(cfg.get_config()))
+    log.trace('configured args: {}'.format(args))
+
+    check_beta(vars(args))
+
+    return parser, cfg

cdist/autil.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+#
+# 2017 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+
+import cdist
+import tarfile
+import os
+import glob
+import tempfile
+
+
+_ARCHIVING_MODES = {
+    'tar': '',
+    'tgz': 'gz',
+    'tbz2': 'bz2',
+    'txz': 'xz',
+}
+
+
+_UNARCHIVE_OPT = {
+    'tar': None,
+    'tgz': '-z',
+    'tbz2': '-j',
+    'txz': '-J',
+}
+
+
+# Archiving will be enabled if directory contains more than FILES_LIMIT files.
+FILES_LIMIT = 1
+
+
+def get_extract_option(mode):
+    return _UNARCHIVE_OPT[mode]
+
+
+def tar(source, mode="tgz"):
+    if mode not in _ARCHIVING_MODES:
+        raise cdist.Error("Unsupported archiving mode {}.".format(mode))
+
+    files = glob.glob1(source, '*')
+    fcnt = len(files)
+    if fcnt <= FILES_LIMIT:
+        return None, fcnt
+
+    tarmode = 'w:{}'.format(_ARCHIVING_MODES[mode])
+    _, tarpath = tempfile.mkstemp(suffix='.' + mode)
+    with tarfile.open(tarpath, tarmode, dereference=True) as tar:
+        if os.path.isdir(source):
+            for f in files:
+                tar.add(os.path.join(source, f), arcname=f)
+        else:
+            tar.add(source)
+    return tarpath, fcnt

cdist/banner.py

@@ -20,8 +20,6 @@
 #
 
 import logging
-import sys
-
 import cdist
 
 log = logging.getLogger(__name__)

cdist/conf/explorer/cpu_cores

@@ -25,13 +25,17 @@
 os=$("$__explorer/os")
 case "$os" in
     "macosx")
-        echo "$(sysctl -n hw.physicalcpu)"
+        sysctl -n hw.physicalcpu
+    ;;
+
+    "openbsd")
+        sysctl -n hw.ncpuonline
     ;;
 
     *)
         if [ -r /proc/cpuinfo ]; then
             cores="$(grep "core id" /proc/cpuinfo | sort | uniq | wc -l)"
-            if [ ${cores} -eq 0 ]; then
+            if [ "${cores}" -eq 0 ]; then
                 cores="1"
             fi
             echo "$cores"

cdist/conf/explorer/cpu_sockets

@@ -25,14 +25,14 @@
 os=$("$__explorer/os")
 case "$os" in
     "macosx")
-        echo "$(system_profiler SPHardwareDataType | grep "Number of Processors" | awk -F': ' '{print $2}')"
+        system_profiler SPHardwareDataType | grep "Number of Processors" | awk -F': ' '{print $2}'
     ;;
 
     *)
     if [ -r /proc/cpuinfo ]; then
-        sockets="$(grep "physical id" /proc/cpuinfo | sort | uniq | wc -l)"
-        if [ ${sockets} -eq 0 ]; then
-            sockets="$(cat /proc/cpuinfo | grep "processor" | wc -l)"
+        sockets="$(grep "physical id" /proc/cpuinfo | sort -u | wc -l)"
+        if [ "${sockets}" -eq 0 ]; then
+            sockets="$(grep -c "processor" /proc/cpuinfo)"
         fi
         echo "${sockets}"
     fi

cdist/conf/explorer/disks

@@ -1,2 +1,27 @@
-cd /dev
-echo sd? hd? vd?
+#!/bin/sh
+
+uname_s="$(uname -s)"
+
+case "${uname_s}" in
+    FreeBSD)
+        sysctl -n kern.disks
+    ;;
+    OpenBSD|NetBSD)
+        sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+' | xargs
+    ;;
+    Linux)
+        if command -v lsblk > /dev/null
+        then
+            # exclude ram disks, floppies and cdroms
+            # https://www.kernel.org/doc/Documentation/admin-guide/devices.txt
+            lsblk -e 1,2,11 -dno name | xargs
+        else
+            printf "Don't know how to list disks for %s operating system without lsblk, if you can please submit a patch\n" "${uname_s}" >&2
+        fi
+    ;;
+    *)
+        printf "Don't know how to list disks for %s operating system, if you can please submit a patch\n" "${uname_s}" >&2
+    ;;
+esac
+
+exit 0

cdist/conf/explorer/init

@@ -1,6 +1,7 @@
 #!/bin/sh
 #
 # 2016 Daniel Heule (hda at sfs.biz)
+# Copyright 2017, Philippe Gregoire <pg@pgregoire.xyz>
 #
 # This file is part of cdist.
 #
@@ -25,7 +26,10 @@
 uname_s="$(uname -s)"
 
 case "$uname_s" in
-    Linux|FreeBSD)
+    Linux)
+        (pgrep -P0 -l | awk '/^1[ \t]/ {print $2;}') || true
+    ;;
+    FreeBSD|OpenBSD)
         ps -o comm= -p 1 || true
     ;;
     *)

cdist/conf/explorer/interfaces

@@ -1,6 +1,6 @@
-#!/bin/sh
+#!/bin/sh -e
 #
-# 2012 Sébastien Gross <seb•ɑƬ•chezwam•ɖɵʈ•org>
+# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
 #
 # This file is part of cdist.
 #
@@ -17,35 +17,12 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# List all network interfaces in explorer/ifaces. One interface per line.
-#
-# If your OS is not supported please provide a ifconfig output
-#
 
-# Use ip, if available
-if command -v ip >/dev/null; then
-    ip -o link show | sed -n 's/^[0-9]\+: \(.\+\): <.*/\1/p'
-    exit 0
-fi
-
-if ! command -v ifconfig >/dev/null; then
-   # no ifconfig, nothing we could do
-   exit 0
-fi
-
-uname_s="$(uname -s)"
-REGEXP='s/^(.*)(:[[:space:]]*flags=|Link encap).*/\1/p'
-
-case "$uname_s" in
-    Darwin)
-	ifconfig -a | sed -n -E "$REGEXP"
-	;;
-    Linux|*BSD)
-	ifconfig -a | sed -n -r "$REGEXP"
-	;;
-    *)
-	echo "Unsupported ifconfig output for $uname_s" >&2
-	exit 1
-	;;
-esac
+if command -v ip >/dev/null
+then
+	ip -o link show | sed -n 's/^[0-9]\+: \(.\+\): <.*/\1/p'
+elif command -v ifconfig >/dev/null
+then
+	ifconfig -a | sed -n -E 's/^(.*)(:[[:space:]]*flags=|Link encap).*/\1/p'
+fi \
+ | sort -u

cdist/conf/explorer/is-freebsd-jail

@@ -0,0 +1,2 @@
+#!/bin/sh
+sysctl -n security.jail.jailed 2>/dev/null | grep "1" || true

cdist/conf/explorer/kernel_name

@@ -1 +1,2 @@
+#!/bin/sh
 uname -s

cdist/conf/explorer/lsb_codename

@@ -20,8 +20,9 @@
 #
 
 set +e
-case "$($__explorer/os)" in
+case "$("$__explorer/os")" in
    openwrt)
+      # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_CODENAME")
    ;;
    *)

cdist/conf/explorer/lsb_description

@@ -20,8 +20,9 @@
 #
 
 set +e
-case "$($__explorer/os)" in
+case "$("$__explorer/os")" in
    openwrt)
+      # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION")
    ;;
    *)

cdist/conf/explorer/lsb_id

@@ -20,8 +20,9 @@
 #
 
 set +e
-case "$($__explorer/os)" in
+case "$("$__explorer/os")" in
    openwrt)
+      # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_ID")
    ;;
    *)

cdist/conf/explorer/lsb_release

@@ -20,8 +20,9 @@
 #
 
 set +e
-case "$($__explorer/os)" in
+case "$("$__explorer/os")" in
    openwrt)
+      # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_RELEASE")
    ;;
    *)

cdist/conf/explorer/machine

@@ -22,6 +22,6 @@
 #
 #
 
-if command -v uname 2>&1 >/dev/null; then
+if command -v uname >/dev/null 2>&1 ; then
    uname -m
 fi

cdist/conf/explorer/machine_type

@@ -22,13 +22,13 @@
 
 # FIXME: other system types (not linux ...)
 
-if [ -d "/proc/vz" -a ! -d "/proc/bc" ]; then
+if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then
     echo openvz
     exit
 fi
 
 if [ -e "/proc/1/environ" ] &&
-    cat "/proc/1/environ" | tr '\000' '\n' | grep -Eiq '^container='; then
+    tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then
     echo lxc
     exit
 fi
@@ -56,6 +56,20 @@ if [ -r /proc/cpuinfo ]; then
                     exit 
                 fi
             fi
+
+            if [ -r /sys/class/dmi/id/sys_vendor ]; then
+                if grep -q -i 'qemu' /sys/class/dmi/id/sys_vendor; then
+                    echo "virtual_by_kvm"
+                    exit
+                fi
+            fi
+
+            if [ -r /sys/class/dmi/id/chassis_vendor ]; then
+                if grep -q -i 'qemu' /sys/class/dmi/id/chassis_vendor; then
+                    echo "virtual_by_kvm"
+                    exit
+                fi
+            fi                        
         fi
         echo "virtual_by_unknown"
     else

cdist/conf/explorer/memory

@@ -2,6 +2,7 @@
 #
 # 2014 Daniel Heule  (hda at sfs.biz)
 # 2014 Thomas Oettli (otho at sfs.biz)
+# Copyright 2017, Philippe Gregoire <pg@pgregoire.xyz>
 #
 # This file is part of cdist.
 #
@@ -28,6 +29,10 @@ case "$os" in
         echo "$(sysctl -n hw.memsize)/1024" | bc
     ;;
 
+    "openbsd")
+        echo "$(sysctl -n hw.physmem) / 1048576" | bc
+    ;;
+
     *)
     if [ -r /proc/meminfo ]; then
         grep "MemTotal:" /proc/meminfo | awk '{print $2}'

cdist/conf/explorer/os

@@ -1,6 +1,7 @@
 #!/bin/sh
 #
 # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+# Copyright 2017, Philippe Gregoire <pg@pgregoire.xyz>
 #
 # This file is part of cdist.
 #
@@ -50,15 +51,17 @@ if grep -q ^DISTRIB_ID=Ubuntu /etc/lsb-release 2>/dev/null; then
    exit 0
 fi
 
+# devuan ascii has both devuan_version and debian_version, so we need to check devuan_version first!
+if [ -f /etc/devuan_version ]; then
+   echo devuan
+   exit 0
+fi
+
 if [ -f /etc/debian_version ]; then
    echo debian
    exit 0
 fi
 
-if [ -f /etc/devuan_version ]; then
-   echo devuan
-   exit 0
-fi
 ###
 
 if [ -f /etc/gentoo-release ]; then
@@ -139,5 +142,12 @@ case "$uname_s" in
    ;;
 esac
 
+if [ -f /etc/os-release ]; then
+   # already lowercase, according to:
+   # https://www.freedesktop.org/software/systemd/man/os-release.html
+   awk -F= '/^ID=/ { if ($2 ~ /^'"'"'(.*)'"'"'$/ || $2 ~ /^"(.*)"$/) { print substr($2, 2, length($2) - 2) } else { print $2 } }' /etc/os-release
+   exit 0
+fi
+
 echo "Unknown OS" >&2
 exit 1

cdist/conf/explorer/os_release

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2018 Adam Dej (dejko.a at gmail.com)
 #
 # This file is part of cdist.
 #
@@ -17,7 +17,10 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+#
 
-# set defaults
-options="$(cat "$__object/parameter/options" 2>/dev/null \
-   || echo "" | tee "$__object/parameter/options")"
+# See os-release(5) and http://0pointer.de/blog/projects/os-release
+
+set +e
+
+cat /etc/os-release || cat /usr/lib/os-release || true

cdist/conf/explorer/os_version

@@ -22,7 +22,7 @@
 #
 #
 
-case "$($__explorer/os)" in
+case "$("$__explorer/os")" in
    amazon)
       cat /etc/system-release
    ;;

cdist/conf/type/__acl/explorer/acl_is

@@ -0,0 +1,31 @@
+#!/bin/sh -e
+#
+# 2018 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+[ ! -e "/$__object_id" ] && exit 0
+
+if ! command -v getfacl > /dev/null
+then
+    echo 'getfacl not available' >&2
+    exit 1
+fi
+
+getfacl "/$__object_id" 2>/dev/null \
+    | grep -Eo '^(default:)?(user|group|(mask|other):):[^:][[:graph:]]+' \
+    || true

cdist/conf/type/__acl/explorer/checks

@@ -0,0 +1,39 @@
+#!/bin/sh -e
+#
+# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# TODO check if filesystem has ACL turned on etc
+
+if [ -f "$__object/parameter/acl" ]
+then
+    grep -E '^(default:)?(user|group):' "$__object/parameter/acl" \
+    | while read -r acl
+    do
+        param="$( echo "$acl" | awk -F: '{print $(NF-2)}' )"
+        check="$( echo "$acl" | awk -F: '{print $(NF-1)}' )"
+
+        [ "$param" = 'user' ] && db=passwd || db="$param"
+
+        if ! getent "$db" "$check" > /dev/null
+        then
+            echo "missing $param '$check'" >&2
+            exit 1
+        fi
+    done
+fi

cdist/conf/type/__acl/explorer/file_is

@@ -0,0 +1,31 @@
+#!/bin/sh -e
+#
+# 2018 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+if [ -e "/$__object_id" ]
+then
+    if [ -d "/$__object_id" ]
+    then echo directory
+    elif [ -f "/$__object_id" ]
+    then echo regular
+    else echo other
+    fi
+else
+    echo missing
+fi

cdist/conf/type/__acl/gencode-remote

@@ -0,0 +1,126 @@
+#!/bin/sh -e
+#
+# 2018 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+file_is="$( cat "$__object/explorer/file_is" )"
+
+[ "$file_is" = 'missing' ] && [ -z "$__cdist_dry_run" ] && exit 0
+
+os="$( cat "$__global/explorer/os" )"
+
+acl_path="/$__object_id"
+
+acl_is="$( cat "$__object/explorer/acl_is" )"
+
+if [ -f "$__object/parameter/acl" ]
+then
+    acl_should="$( cat "$__object/parameter/acl" )"
+elif
+    [ -f "$__object/parameter/user" ] \
+        || [ -f "$__object/parameter/group" ] \
+        || [ -f "$__object/parameter/mask" ] \
+        || [ -f "$__object/parameter/other" ]
+then
+    acl_should="$( for param in user group mask other
+    do
+        [ ! -f "$__object/parameter/$param" ] && continue
+
+        echo "$param" | grep -Eq 'mask|other' && sep=:: || sep=:
+
+        echo "$param$sep$( cat "$__object/parameter/$param" )"
+    done )"
+else
+    echo 'no parameters set' >&2
+    exit 1
+fi
+
+if [ -f "$__object/parameter/default" ]
+then
+    acl_should="$( echo "$acl_should" \
+        | sed 's/^default://' \
+        | sort -u \
+        | sed 's/\(.*\)/default:\1\n\1/' )"
+fi
+
+if [ "$file_is" = 'regular' ] \
+    && echo "$acl_should" | grep -Eq '^default:'
+then
+    # only directories can have default ACLs,
+    # but instead of error,
+    # let's just remove default entries
+    acl_should="$( echo "$acl_should" | grep -Ev '^default:' )"
+fi
+
+if echo "$acl_should" | awk -F: '{ print $NF }' | grep -Fq 'X'
+then
+    [ "$file_is" = 'directory' ] && rep=x || rep=-
+
+    acl_should="$( echo "$acl_should" | sed "s/\\(.*\\)X/\\1$rep/" )"
+fi
+
+setfacl_exec='setfacl'
+
+if [ -f "$__object/parameter/recursive" ]
+then
+    if echo "$os" | grep -Fq 'freebsd'
+    then
+        echo "$os setfacl do not support recursive operations" >&2
+    else
+        setfacl_exec="$setfacl_exec -R"
+    fi
+fi
+
+if [ -f "$__object/parameter/remove" ]
+then
+    echo "$acl_is" | while read -r acl
+    do
+        # skip wanted ACL entries which already exist
+        # and skip mask and other entries, because we
+        # can't actually remove them, but only change.
+        if echo "$acl_should" | grep -Eq "^$acl" \
+            || echo "$acl" | grep -Eq '^(default:)?(mask|other)'
+        then continue
+        fi
+
+        if echo "$os" | grep -Fq 'freebsd'
+        then
+            remove="$acl"
+        else
+            remove="$( echo "$acl" | sed 's/:...$//' )"
+        fi
+
+        echo "$setfacl_exec -x \"$remove\" \"$acl_path\""
+        echo "removed '$remove'" >> "$__messages_out"
+    done
+fi
+
+for acl in $acl_should
+do
+    if ! echo "$acl_is" | grep -Eq "^$acl"
+    then
+        if echo "$os" | grep -Fq 'freebsd' \
+            && echo "$acl" | grep -Eq '^default:'
+        then
+            echo "setting default ACL in $os is currently not supported" >&2
+        else
+            echo "$setfacl_exec -m \"$acl\" \"$acl_path\""
+            echo "added '$acl'" >> "$__messages_out"
+        fi
+    fi
+done

cdist/conf/type/__acl/man.rst

@@ -0,0 +1,85 @@
+cdist-type__acl(7)
+==================
+
+NAME
+----
+cdist-type__acl - Set ACL entries
+
+
+DESCRIPTION
+-----------
+Fully supported and tested on Linux (ext4 filesystem), partial support for FreeBSD.
+
+See ``setfacl`` and ``acl`` manpages for more details.
+
+
+REQUIRED MULTIPLE PARAMETERS
+----------------------------
+acl
+   Set ACL entry following ``getfacl`` output syntax.
+
+
+BOOLEAN PARAMETERS
+------------------
+default
+   Set all ACL entries as default too.
+   Only directories can have default ACLs.
+   Setting default ACL in FreeBSD is currently not supported.
+
+recursive
+   Make ``setfacl`` recursive (Linux only), but not ``getfacl`` in explorer.
+
+remove
+   Remove undefined ACL entries.
+   ``mask`` and ``other`` entries can't be removed, but only changed.
+
+
+DEPRECATED PARAMETERS
+---------------------
+Parameters ``user``, ``group``, ``mask`` and ``other`` are deprecated and they
+will be removed in future versions. Please use ``acl`` parameter instead.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __acl /srv/project \
+        --default \
+        --recursive \
+        --remove \
+        --acl user:alice:rwx \
+        --acl user:bob:r-x \
+        --acl group:project-group:rwx \
+        --acl group:some-other-group:r-x \
+        --acl mask::r-x \
+        --acl other::r-x
+
+    # give Alice read-only access to subdir,
+    # but don't allow her to see parent content.
+
+    __acl /srv/project2 \
+        --remove \
+        --acl default:group:secret-project:rwx \
+        --acl group:secret-project:rwx \
+        --acl user:alice:--x
+
+    __acl /srv/project2/subdir \
+        --default \
+        --remove \
+        --acl group:secret-project:rwx \
+        --acl user:alice:r-x
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2018 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__acl/parameter/boolean

@@ -0,0 +1,3 @@
+recursive
+default
+remove

cdist/conf/type/__acl/parameter/deprecated/group

@@ -0,0 +1 @@
+see manual for details

cdist/conf/type/__acl/parameter/deprecated/mask

@@ -0,0 +1 @@
+see manual for details

cdist/conf/type/__acl/parameter/deprecated/other

@@ -0,0 +1 @@
+see manual for details

cdist/conf/type/__acl/parameter/deprecated/user

@@ -0,0 +1 @@
+see manual for details

cdist/conf/type/__acl/parameter/optional

@@ -0,0 +1,2 @@
+mask
+other

cdist/conf/type/__acl/parameter/optional_multiple

@@ -0,0 +1,3 @@
+acl
+user
+group

cdist/conf/type/__apt_default_release/man.rst

@@ -0,0 +1,46 @@
+cdist-type__apt_default_release(7)
+==================================
+
+NAME
+----
+cdist-type__apt_default_release - Configure the default release for apt
+
+
+DESCRIPTION
+-----------
+Configure the default release for apt, using the APT::Default-Release
+configuration value.
+
+REQUIRED PARAMETERS
+-------------------
+release
+   The value to set APT::Default-Release to.
+
+   This can contain release name, codename or release version. Examples:
+   'stable', 'testing', 'unstable', 'stretch', 'buster', '4.0', '5.0*'.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __apt_default_release --release stretch
+
+
+AUTHORS
+-------
+Matthijs Kooijman <matthijs--@--stdin.nl>
+
+
+COPYING
+-------
+Copyright \(C) 2017 Matthijs Kooijman. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__apt_default_release/manifest

@@ -0,0 +1,41 @@
+#!/bin/sh -e
+#
+# 2014 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2017 Matthijs Kooijman (matthijs at stdin.nl)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+os=$(cat "$__global/explorer/os")
+release="$(cat "$__object/parameter/release")"
+
+case "$os" in
+   ubuntu|debian|devuan)
+      __file /etc/apt/apt.conf.d/99-default-release \
+         --owner root --group root --mode 644 \
+         --source - << DONE
+APT::Default-Release "$release";
+DONE
+   ;;
+   *)
+      cat >&2 << DONE
+The developer of this type (${__type##*/}) did not think your operating system
+($os) would have any use for it. If you think otherwise please submit a patch.
+DONE
+      exit 1
+   ;;
+esac

cdist/conf/type/__apt_default_release/parameter/required

@@ -0,0 +1 @@
+release

cdist/conf/type/__apt_key/explorer/state

@@ -27,6 +27,18 @@ else
    keyid="$__object_id"
 fi
 
-apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \
-   && echo present \
-   || echo absent
+keydir="$(cat "$__object/parameter/keydir")"
+keyfile="$keydir/$__object_id.gpg"
+
+if [ -d "$keydir" ]
+then
+   if [ -f "$keyfile" ]
+   then echo present
+   else echo absent
+   fi
+else
+   # fallback to deprecated apt-key
+   apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \
+      && echo present \
+      || echo absent
+fi

cdist/conf/type/__apt_key/gencode-remote

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011-2014 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -31,12 +31,84 @@ if [ "$state_should" = "$state_is" ]; then
    exit 0
 fi
 
+keydir="$(cat "$__object/parameter/keydir")"
+keyfile="$keydir/$__object_id.gpg"
+
 case "$state_should" in
    present)
       keyserver="$(cat "$__object/parameter/keyserver")"
-      echo "apt-key adv --keyserver \"$keyserver\" --recv-keys \"$keyid\""
+
+      if [ -f "$__object/parameter/uri" ]; then
+         uri="$(cat "$__object/parameter/uri")"
+
+         if [ -d "$keydir" ]; then
+            cat << EOF
+
+curl -s -L \\
+    -o "$keyfile" \\
+    "$uri"
+
+key="\$( cat "$keyfile" )"
+
+if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK'
+then
+    echo "\$key" | gpg --dearmor > "$keyfile"
+fi
+
+EOF
+         else
+            # fallback to deprecated apt-key
+            echo "curl -s -L '$uri' | apt-key add -"
+         fi
+      elif [ -d "$keydir" ]; then
+         tmp='/tmp/cdist_apt_key_tmp'
+
+         # we need to kill gpg after 30 seconds, because gpg
+         # can get stuck if keyserver is not responding.
+         # exporting env var and not exit 1,
+         # because we need to clean up and kill dirmngr.
+         cat << EOF
+
+mkdir -m 700 -p "$tmp"
+
+if timeout 30s \\
+    gpg --homedir "$tmp" \\
+        --keyserver "$keyserver" \\
+        --recv-keys "$keyid"
+then
+    gpg --homedir "$tmp" \\
+        --export "$keyid" \\
+        > "$keyfile"
+else
+    export GPG_GOT_STUCK=1
+fi
+
+GNUPGHOME="$tmp" gpgconf --kill dirmngr
+
+rm -rf "$tmp"
+
+if [ -n "\$GPG_GOT_STUCK" ]
+then
+    echo "GPG GOT STUCK - no response from keyserver after 30 seconds" >&2
+    exit 1
+fi
+
+EOF
+      else
+         # fallback to deprecated apt-key
+         echo "apt-key adv --keyserver \"$keyserver\" --recv-keys \"$keyid\""
+      fi
+
+      echo "added '$keyid'" >> "$__messages_out"
    ;;
    absent)
-      echo "apt-key del \"$keyid\""
+      if [ -f "$keyfile" ]; then
+         echo "rm '$keyfile'"
+      else
+         # fallback to deprecated apt-key
+         echo "apt-key del \"$keyid\""
+      fi
+
+      echo "removed '$keyid'" >> "$__messages_out"
    ;;
 esac

cdist/conf/type/__apt_key/man.rst

@@ -28,6 +28,12 @@ keyserver
    the keyserver from which to fetch the key. If omitted the default set
    in ./parameter/default/keyserver is used.
 
+keydir
+   key save location, defaults to ``/etc/apt/trusted.pgp.d``
+
+uri
+   the URI from which to download the key
+
 
 EXAMPLES
 --------
@@ -47,15 +53,20 @@ EXAMPLES
     # same thing with other keyserver
     __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com
 
+    # download key from the internet
+    __apt_key rabbitmq \
+       --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc
+
 
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
+Ander Punnar <ander-at-kvlt-dot-ee>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2014 Steven Armstrong. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
+Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can
+redistribute it and/or modify it under the terms of the GNU General Public
+License as published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.

cdist/conf/type/__apt_key/manifest

@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+__package gnupg
+
+if [ -f "$__object/parameter/uri" ]
+then __package curl
+else __package dirmngr
+fi

cdist/conf/type/__apt_key/parameter/default/keydir

@@ -0,0 +1 @@
+/etc/apt/trusted.gpg.d

cdist/conf/type/__apt_key/parameter/optional

@@ -1,3 +1,5 @@
 state
 keyid
 keyserver
+keydir
+uri

cdist/conf/type/__apt_key_uri/explorer/state

@@ -27,6 +27,6 @@ else
    name="$__object_id"
 fi
 
-apt-key list | grep -Fqe "$name" \
+apt-key list 2> /dev/null | grep -Fqe "$name" \
    && echo present \
    || echo absent

cdist/conf/type/__apt_key_uri/gencode-remote

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011-2014 Steven Armstrong (steven-cdist at armstrong.cc)
 #

cdist/conf/type/__apt_key_uri/manifest

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2013-2014 Steven Armstrong (steven-cdist at armstrong.cc)
 #

cdist/conf/type/__apt_mark/explorer/apt_version

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2016 Ander Punnar (cdist at kvlt.ee)
 #
@@ -26,6 +26,6 @@ apt_version_is=$(dpkg-query --show --showformat '${Version}' apt)
 
 apt_version_should=0.8.14.2
 
-dpkg --compare-versions $apt_version_should le $apt_version_is \
+dpkg --compare-versions "$apt_version_should" le "$apt_version_is" \
     && echo 0 \
     || echo 1

cdist/conf/type/__apt_mark/explorer/package_installed

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2016 Ander Punnar (cdist at kvlt.ee)
 #
@@ -24,7 +24,7 @@ else
     name="$__object_id"
 fi
 
-dpkg-query --show --showformat '${Status}' $name 2>/dev/null \
-    | grep -q 'ok installed' \
+dpkg-query --show --showformat '${Status}' "$name" 2>/dev/null \
+    | grep -Fq 'ok installed' \
     && echo 0 \
     || echo 1

cdist/conf/type/__apt_mark/explorer/state

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2016 Ander Punnar (cdist at kvlt.ee)
 #
@@ -24,4 +24,4 @@ else
     name="$__object_id"
 fi
 
-apt-mark showhold | grep -q $name && echo hold || echo unhold
+apt-mark showhold | grep -Fq "$name" && echo hold || echo unhold

cdist/conf/type/__apt_mark/gencode-remote

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2016 Ander Punnar (cdist at kvlt.ee)
 #

cdist/conf/type/__apt_norecommends/manifest

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
 #

cdist/conf/type/__apt_ppa/explorer/state

@@ -23,10 +23,11 @@
 
 name="$__object_id"
 
+# shellcheck disable=SC1091
 . /etc/lsb-release
 
 repo_name="${name#ppa:}"
-repo_file_name="$(echo "$repo_name" | sed -e "s|[/:]|-|" -e "s|\.|_|")-${DISTRIB_CODENAME}.list"
+repo_file_name="$(echo "$repo_name" | sed -e 's|[/:]|-|' -e 's|\.|_|')-${DISTRIB_CODENAME}.list"
 
 [ -s "/etc/apt/sources.list.d/${repo_file_name}" ] \
    && echo present || echo absent

cdist/conf/type/__apt_ppa/gencode-remote

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -29,9 +29,9 @@ fi
 
 case "$state_should" in
    present)
-      echo add-apt-repository \"$name\"
+      echo "add-apt-repository '$name'"
    ;;
    absent)
-      echo remove-apt-repository \"$name\"
+      echo "remove-apt-repository '$name'"
    ;;
 esac

cdist/conf/type/__apt_ppa/manifest

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011-2016 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -18,8 +18,6 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-name="$__object_id"
-
 __package software-properties-common
 
 require="__package/software-properties-common" \

cdist/conf/type/__apt_source/gencode-remote

@@ -0,0 +1,28 @@
+#!/bin/sh -e
+#
+# 2018 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+name="$__object_id"
+destination="/etc/apt/sources.list.d/${name}.list"
+
+if grep -q "^__file${destination}" "$__messages_in"; then
+   printf 'apt-get update || apt-get update\n'
+fi
+

cdist/conf/type/__apt_source/man.rst

@@ -8,7 +8,8 @@ cdist-type__apt_source - Manage apt sources
 
 DESCRIPTION
 -----------
-This cdist type allows you to manage apt sources.
+This cdist type allows you to manage apt sources. It invokes index update
+internally when needed so call of index updating type is not needed.
 
 
 REQUIRED PARAMETERS
@@ -63,7 +64,7 @@ Steven Armstrong <steven-cdist--@--armstrong.cc>
 
 COPYING
 -------
-Copyright \(C) 2011-2014 Steven Armstrong. You can redistribute it
+Copyright \(C) 2011-2018 Steven Armstrong. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.

cdist/conf/type/__apt_source/manifest

@@ -1,6 +1,6 @@
-#!/bin/sh
+#!/bin/sh -e
 #
-# 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2011-2018 Steven Armstrong (steven-cdist at armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -50,5 +50,3 @@ __file "/etc/apt/sources.list.d/${name}.list" \
    --source "$__object/files/source.list" \
    --owner root --group root --mode 0644 \
    --state "$state"
-
-require="$__object_name" __apt_update_index

cdist/conf/type/__apt_update_index/gencode-remote

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 #

cdist/conf/type/__block/gencode-remote

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2013 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -18,6 +18,11 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+# quote function from http://www.etalabs.net/sh_tricks.html
+quote() {
+   printf '%s\n' "$1" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
+}
+
 file="$(cat "$__object/parameter/file" 2>/dev/null || echo "/$__object_id")"
 state_should=$(cat "$__object/parameter/state")
 prefix=$(cat "$__object/parameter/prefix" 2>/dev/null || echo "#cdist:__block/$__object_id")
@@ -46,7 +51,7 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
 if [ -f "$file" ]; then
    cp -p "$file" "\$tmpfile"
 fi
-awk -v prefix="^$prefix\$" -v suffix="^$suffix\$" '
+awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
 {
    if (match(\$0,prefix)) {
       triggered=1

cdist/conf/type/__block/manifest

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2013-2014 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -18,8 +18,6 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-
-file="$(cat "$__object/parameter/file" 2>/dev/null || echo "/$__object_id")"
 prefix=$(cat "$__object/parameter/prefix" 2>/dev/null || echo "#cdist:__block/$__object_id")
 suffix=$(cat "$__object/parameter/suffix" 2>/dev/null || echo "#/cdist:__block/$__object_id")
 text=$(cat "$__object/parameter/text")

cdist/conf/type/__ccollect_source/gencode-remote

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2014 Nico Schottelius (nico-cdist at schottelius.org)
 #
@@ -42,21 +42,20 @@ get_current_value() {
 }
 
 set_group() {
-    echo chgrp \"$1\" \"$destination\"
-    echo chgrp $1 >> "$__messages_out"
+    echo "chgrp '$1' '$destination'"
+    echo "chgrp '$1'" >> "$__messages_out"
 }
 
 set_owner() {
-    echo chown \"$1\" \"$destination\"
-    echo chown $1 >> "$__messages_out"
+    echo "chown '$1' '$destination'"
+    echo "chown '$1'" >> "$__messages_out"
 }
 
 set_mode() {
-   echo chmod \"$1\" \"$destination\"
-   echo chmod $1 >> "$__messages_out"
+   echo "chmod '$1' '$destination'"
+   echo "chmod '$1'" >> "$__messages_out"
 }
 
-set_attributes=
 case "$state_should" in
     present|exists)
     # Note: Mode - needs to happen last as a chown/chgrp can alter mode by
@@ -67,11 +66,11 @@ case "$state_should" in
 
             # change 0xxx format to xxx format => same as stat returns
             if [ "$attribute" = mode ]; then
-                value_should="$(echo $value_should | sed 's/^0\(...\)/\1/')"
+                value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')"
             fi
             
             value_is="$(get_current_value "$attribute" "$value_should")"
-            if [ -f "$__object/files/set-attributes" -o "$value_should" != "$value_is" ]; then
+            if [ -f "$__object/files/set-attributes" ] || [ "$value_should" != "$value_is" ]; then
                 "set_$attribute" "$value_should"
             fi
         fi
@@ -81,7 +80,7 @@ case "$state_should" in
 
     absent)
         if [ "$type" = "file" ]; then
-            echo rm -f \"$destination\"
+            echo "rm -f '$destination'"
             echo remove >> "$__messages_out"
         fi
     ;;

cdist/conf/type/__ccollect_source/man.rst

@@ -38,6 +38,8 @@ BOOLEAN PARAMETERS
 verbose
     Whether to report backup verbosely
 
+create-destination
+    Create the directory specified in the destination parameter on the remote host
 
 EXAMPLES
 --------
@@ -50,6 +52,13 @@ EXAMPLES
         --exclude '/proc/*' --exclude '/sys/*' \
         --verbose
 
+    __ccollect_source doc.ungleich.ch \
+        --source doc.ungleich.ch:/ \
+        --destination /backup/doc.ungleich.ch \
+        --exclude '/proc/*' --exclude '/sys/*' \
+        --verbose \ 
+        --create-destination
+
 
 SEE ALSO
 --------

cdist/conf/type/__ccollect_source/manifest

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2014 Nico Schottelius (nico-cdist at schottelius.org)
 #
@@ -22,7 +22,7 @@ name="$__object_id"
 state="$(cat "$__object/parameter/state")"
 source="$(cat "$__object/parameter/source")"
 destination="$(cat "$__object/parameter/destination")"
-ccollectconf="$(cat "$__object/parameter/ccollectconf" | sed 's,/$,,')"
+ccollectconf="$(sed 's,/$,,' "$__object/parameter/ccollectconf")"
 
 sourcedir="$ccollectconf/sources"
 basedir="$sourcedir/$name"
@@ -53,3 +53,7 @@ if [ -f "$__object/parameter/exclude" ]; then
     __file "$exclude_file" --source - --state "$state" \
         < "$__object/parameter/exclude"
 fi
+
+if [ -f "$__object/parameter/create-destination" ]; then
+    __directory "${destination}" --parents --state "${state}"
+fi

cdist/conf/type/__ccollect_source/parameter/boolean

@@ -1 +1,2 @@
 verbose
+create-destination

cdist/conf/type/__cdist/man.rst

@@ -30,7 +30,7 @@ username
 
 source
     Select the source from which to clone cdist from.
-    Defaults to "git://github.com/ungleich/cdist.git".
+    Defaults to "git@code.ungleich.ch:ungleich-public/cdist.git".
 
 
 branch
@@ -47,7 +47,7 @@ EXAMPLES
     __cdist /home/cdist/cdist
 
     # Use alternative source
-    __cdist --source "git://github.com/ungleich/cdist" /home/cdist/cdist
+    __cdist --source "git@code.ungleich.ch:ungleich-public/cdist.git" /home/cdist/cdist
 
 
 AUTHORS

cdist/conf/type/__cdist/manifest

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2013 Nico Schottelius (nico-cdist at schottelius.org)
 #

cdist/conf/type/__cdist/parameter/default/source

@@ -1 +1 @@
-git://github.com/ungleich/cdist.git
+git@code.ungleich.ch:ungleich-public/cdist.git

cdist/conf/type/__cdistmarker/gencode-remote

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # Copyright (C) 2011 Daniel Maher (phrawzty+cdist at gmail.com)
 #

cdist/conf/type/__check_messages/gencode-remote

@@ -0,0 +1,26 @@
+#!/bin/sh -e
+#
+# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+if grep -Eq \
+    "$( cat "$__object/parameter/pattern" )" \
+    "$__messages_in"
+then
+    tee "$__messages_out" < "$__object/parameter/execute"
+fi

cdist/conf/type/__check_messages/man.rst

@@ -0,0 +1,52 @@
+cdist-type__check_messages(7)
+=============================
+
+NAME
+----
+cdist-type__check_messages - Check messages for pattern and execute command on match.
+
+
+DESCRIPTION
+-----------
+Check messages for pattern and execute command on match.
+
+This type is useful if you chain together multiple related types using
+dependencies and want to restart service if at least one type changes
+something.
+
+For more information about messages see `cdist messaging <cdist-messaging.html>`_.
+
+For more information about dependencies and execution order see
+`cdist manifest <cdist-manifest.html#dependencies>`_ documentation.
+
+
+REQUIRED PARAMETERS
+-------------------
+pattern
+   Extended regular expression pattern for search (passed to ``grep -E``).
+
+execute
+   Command to execute on pattern match.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __check_messages munin \
+        --pattern '^__(file|link|line)/etc/munin/' \
+        --execute 'service munin-node restart'
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2019 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__check_messages/parameter/required

@@ -0,0 +1,2 @@
+pattern
+execute

cdist/conf/type/__chroot_mount/gencode-local

@@ -0,0 +1,36 @@
+#!/bin/sh -e
+#
+# 2016 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+chroot="/$__object_id"
+
+if [ -f "$__object/parameter/manage-resolv-conf" ]; then
+   suffix="$(cat "$__object/parameter/manage-resolv-conf")"
+   resolv_conf="${chroot}/etc/resolv.conf"
+   original_resolv_conf="${resolv_conf}.${suffix}"
+   cat << DONE
+$__remote_exec $__target_host << EOSSH
+if [ -f "${resolv_conf}" ]; then
+   mv "${resolv_conf}" "${original_resolv_conf}"
+fi
+# copy hosts resolv.conf into chroot
+cp /etc/resolv.conf "${resolv_conf}"
+EOSSH
+DONE
+fi

cdist/conf/type/__chroot_mount/gencode-remote

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2012 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -41,8 +41,4 @@ mountpoint -q "${chroot}/dev/pts" \
 [ -d "${chroot}/tmp" ] || mkdir -m 1777 "${chroot}/tmp"
 mountpoint -q "${chroot}/tmp" \
    || mount -t tmpfs -o mode=1777,strictatime,nodev,nosuid tmpfs "${chroot}/tmp"
-
-if [ ! -f "${chroot}/etc/resolv.conf" ]; then
-   cp /etc/resolv.conf "${chroot}/etc/"
-fi
 DONE

cdist/conf/type/__chroot_mount/man.rst

@@ -1,5 +1,5 @@
 cdist-type__chroot_mount(7)
-===================================
+===========================
 
 NAME
 ----
@@ -18,7 +18,17 @@ None
 
 OPTIONAL PARAMETERS
 -------------------
-None
+manage-resolv-conf
+    manage /etc/resolv.conf inside the chroot.
+    Use the value of this parameter as the suffix to save a copy
+    of the current /etc/resolv.conf to /etc/resolv.conf.$suffix.
+    This is used by the __chroot_umount type to restore the initial
+    file content when unmounting the chroot.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
 
 
 EXAMPLES
@@ -28,6 +38,9 @@ EXAMPLES
 
     __chroot_mount /path/to/chroot
 
+    __chroot_mount /path/to/chroot \
+      --manage-resolv-conf "some-known-string"
+
 
 AUTHORS
 -------
@@ -36,7 +49,7 @@ Steven Armstrong <steven-cdist--@--armstrong.cc>
 
 COPYING
 -------
-Copyright \(C) 2012 Steven Armstrong. You can redistribute it
+Copyright \(C) 2012-2017 Steven Armstrong. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.

cdist/conf/type/__chroot_mount/parameter/optional

@@ -0,0 +1 @@
+manage-resolv-conf

cdist/conf/type/__chroot_umount/gencode-local

@@ -0,0 +1,36 @@
+#!/bin/sh -e
+#
+# 2016 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+chroot="/$__object_id"
+
+if [ -f "$__object/parameter/manage-resolv-conf" ]; then
+   suffix="$(cat "$__object/parameter/manage-resolv-conf")"
+   resolv_conf="${chroot}/etc/resolv.conf"
+   original_resolv_conf="${resolv_conf}.${suffix}"
+cat << DONE
+$__remote_exec $__target_host << EOSSH
+if [ -f "${original_resolv_conf}" ]; then
+   # restore original /etc/resolv.conf that we moved out of the way
+   # in __chroot_mount/gencode-local
+   mv -f "${original_resolv_conf}" "${resolv_conf}"
+fi
+EOSSH
+DONE
+fi

cdist/conf/type/__chroot_umount/gencode-remote

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2012 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -26,7 +26,6 @@ umount -l "${chroot}/dev/pts"
 umount -l "${chroot}/dev"
 umount -l "${chroot}/sys"
 umount -l "${chroot}/proc"
-rm -f "${chroot}/etc/resolv.conf"
 if [ -d "${chroot}/etc/resolvconf/resolv.conf.d" ]; then
    # ensure /etc/resolvconf/resolv.conf.d/tail is not linked to \
    # e.g. /etc/resolvconf/resolv.conf.d/original

cdist/conf/type/__chroot_umount/man.rst

@@ -18,7 +18,17 @@ None
 
 OPTIONAL PARAMETERS
 -------------------
-None
+manage-resolv-conf
+    manage /etc/resolv.conf inside the chroot.
+    Use the value of this parameter as the suffix to find the backup file
+    that was saved by the __chroot_mount.
+    This is used by the to restore the initial file content when unmounting
+    the chroot.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
 
 
 EXAMPLES
@@ -28,6 +38,9 @@ EXAMPLES
 
     __chroot_umount /path/to/chroot
 
+    __chroot_umount /path/to/chroot \
+      --manage-resolv-conf "some-known-string"
+
 
 SEE ALSO
 --------
@@ -41,7 +54,7 @@ Steven Armstrong <steven-cdist--@--armstrong.cc>
 
 COPYING
 -------
-Copyright \(C) 2012 Steven Armstrong. You can redistribute it
+Copyright \(C) 2012-2017 Steven Armstrong. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.

cdist/conf/type/__chroot_umount/manifest

@@ -0,0 +1,36 @@
+#!/bin/sh -e
+#
+# 2016 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+chroot="/$__object_id"
+
+if [ -f "$__object/parameter/manage-resolv-conf" ]; then
+   suffix="$(cat "$__object/parameter/manage-resolv-conf")"
+   resolv_conf="${chroot}/etc/resolv.conf"
+   original_resolv_conf="${resolv_conf}.${suffix}"
+cat << DONE
+$__remote_exec $__target_host << EOSSH
+if [ -f "${original_resolv_conf}" ]; then
+   # restore original /etc/resolv.conf that we moved out of the way
+   # in __chroot_mount/gencode-local
+   mv -f "${original_resolv_conf}" "${resolv_conf}"
+fi
+EOSSH
+DONE
+fi

cdist/conf/type/__chroot_umount/parameter/optional

@@ -0,0 +1 @@
+manage-resolv-conf

cdist/conf/type/__clean_path/explorer/list

@@ -0,0 +1,35 @@
+#!/bin/sh -e
+#
+# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+path="/$__object_id"
+
+[ ! -d "$path" ] && exit 0
+
+pattern="$( cat "$__object/parameter/pattern" )"
+
+if [ -f "$__object/parameter/exclude" ]
+then
+    exclude="$( cat "$__object/parameter/exclude" )"
+
+    find "$path" -mindepth 1 -maxdepth 1 -regex "$pattern" \
+        -and -not -regex "$exclude"
+else
+    find "$path" -mindepth 1 -maxdepth 1 -regex "$pattern"
+fi

cdist/conf/type/__clean_path/gencode-remote

@@ -0,0 +1,48 @@
+#!/bin/sh -e
+#
+# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+[ ! -s "$__object/explorer/list" ] && exit 0
+
+path="/$__object_id"
+
+pattern="$( cat "$__object/parameter/pattern" )"
+
+if [ -f "$__object/parameter/exclude" ]
+then
+    exclude="$( cat "$__object/parameter/exclude" )"
+
+    echo "find '$path' -mindepth 1 -maxdepth 1 -regex '$pattern'" \
+        "-and -not -regex '$exclude'" \
+        '-exec rm -rf {} \;'
+else
+    echo "find '$path' -mindepth 1 -maxdepth 1 -regex '$pattern'" \
+        '-exec rm -rf {} \;'
+fi
+
+while read -r f
+do
+    echo "removed '$f'" >> "$__messages_out"
+done \
+< "$__object/explorer/list"
+
+if [ -f "$__object/parameter/onchange" ]
+then
+    cat "$__object/parameter/onchange"
+fi

cdist/conf/type/__clean_path/man.rst

@@ -0,0 +1,60 @@
+cdist-type__clean_path(7)
+=========================
+
+NAME
+----
+cdist-type__clean_path - Remove files and directories which match the pattern.
+
+
+DESCRIPTION
+-----------
+Remove files and directories which match the pattern.
+
+Provided path (as __object_id) must be a directory.
+
+Patterns are passed to ``find``'s ``-regex`` - see ``find(1)`` for more details.
+
+Look up of files and directories is non-recursive (``-maxdepth 1``).
+
+Parent directory is excluded (``-mindepth 1``).
+
+This type is not POSIX compatible (sorry, Solaris users).
+
+
+REQUIRED PARAMETERS
+-------------------
+pattern
+   Pattern of files which are removed from path.
+
+
+OPTIONAL PARAMETERS
+-------------------
+exclude
+   Pattern of files which are excluded from removal.
+
+onchange
+   The code to run if files or directories were removed.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __clean_path /etc/apache2/conf-enabled \
+        --pattern '.+' \
+        --exclude '.+\(charset\.conf\|security\.conf\)' \
+        --onchange 'service apache2 restart'
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2019 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__clean_path/parameter/optional

@@ -0,0 +1,2 @@
+exclude
+onchange

cdist/conf/type/__clean_path/parameter/required

@@ -0,0 +1 @@
+pattern

cdist/conf/type/__config_file/gencode-remote

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2015 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -19,16 +19,9 @@
 #
 
 destination="$__object_id"
-state="$(cat "$__object/parameter/state")"
-
-if [ "$state" = "absent" ]; then
-   # nothing to do
-   exit 0
-fi
 
 if [ -f "$__object/parameter/onchange" ]; then
    if grep -q "^__file/${destination}" "$__messages_in"; then
       cat "$__object/parameter/onchange"
    fi
 fi
-

cdist/conf/type/__config_file/manifest

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2015 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -19,7 +19,8 @@
 #
 
 set -- "/${__object_id}"
-for param in $(ls "$__object/parameter/"); do
+cd "$__object/parameter/"
+for param in *; do
    case "$param" in
       source)
          source="$(cat "$__object/parameter/source")"

cdist/conf/type/__consul/files/versions/1.0.6/cksum

@@ -0,0 +1 @@
+4120550353 48801129 consul

cdist/conf/type/__consul/files/versions/1.0.6/source

@@ -0,0 +1 @@
+https://releases.hashicorp.com/consul/1.0.6/consul_1.0.6_linux_amd64.zip

cdist/conf/type/__consul/files/versions/1.2.3/cksum

@@ -0,0 +1 @@
+191982 110369685

cdist/conf/type/__consul/files/versions/1.2.3/source

@@ -0,0 +1 @@
+https://releases.hashicorp.com/consul/1.2.3/consul_1.2.3_linux_amd64.zip