Release 6.1.0

[__hostname] Support more operating systems

See merge request ungleich-public/cdist!802

.gitattributes

@@ -0,0 +1,8 @@
+.gitignore export-ignore
+.gitattributes export-ignore
+.gitkeep export-ignore
+docs/speeches export-ignore
+docs/video export-ignore
+docs/src/man7 export-ignore
+bin/build-helper export-ignore
+README-maintainers export-ignore

.gitignore

@@ -1,18 +1,61 @@
 # -vim
-.*.swp
+# Swap
+[._]*.s[a-v][a-z]
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+
+# Temporary
+.netrwhist
+*~
+*.tmp
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
 
 # Ignore generated manpages
-doc/man/.marker
-doc/man/man1/*.1
-doc/man/man7/*.7
-doc/man/man*/*.html
-doc/man/man*/*.xml
-doc/man/man7/cdist-type__*.text
-doc/man/man7/cdist-reference.text
-doc/man/man*/docbook-xsl.css
+docs/src/.marker
+docs/src/man1/*.1
+docs/src/man7/*.7
+docs/src/man7/cdist-type__*.rst
+docs/src/cdist-reference.rst
 
 # Ignore cdist cache for version control
 /cache/
 
-# Python / cache
+# Ignore inventory basedir
+cdist/inventory/
+
+# Python: cache, distutils, distribution in general
 __pycache__/
+*.pyc
+/MANIFEST
+dist/
+cdist/version.py
+cdist.egg-info/
+
+# sphinx build dirs, cache
+_build/
+docs/dist
+
+# Ignore temp files used for signing
+cdist-*.tar
+cdist-*.tar.gz
+cdist-*.tar.gz.asc
+
+# Packaging: Archlinux
+/PKGBUILD
+/cdist-*.pkg.tar.xz
+/cdist-*.tar.gz
+/pkg
+/src
+build
+.lock-*
+.git-current-branch
+.lock*
+.pypi-release

MANIFEST.in

@@ -0,0 +1,3 @@
+include docs/changelog
+recursive-include docs/gfx *.png *.text
+recursive-include docs *.text *.html *.1 *.7

Makefile

@@ -0,0 +1,134 @@
+#
+# 2013 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+.PHONY: help
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "man             build only man user documentation"
+	@echo "html            build only html user documentation"
+	@echo "docs            build both man and html user documentation"
+	@echo "dotman          build man pages for types in your ~/.cdist directory"
+	@echo "speeches        build speeches pdf files"
+	@echo "install         install in the system site-packages directory"
+	@echo "install-user    install in the user site-packages directory"
+	@echo "docs-clean      clean documentation"
+	@echo "clean           clean"
+
+DOCS_SRC_DIR=./docs/src
+SPEECHDIR=./docs/speeches
+TYPEDIR=./cdist/conf/type
+
+SPHINXM=make -C $(DOCS_SRC_DIR) man
+SPHINXH=make -C $(DOCS_SRC_DIR) html
+SPHINXC=make -C $(DOCS_SRC_DIR) clean
+
+################################################################################
+# Manpages
+#
+MAN7DSTDIR=$(DOCS_SRC_DIR)/man7
+
+# Manpages #1: Types
+# Use shell / ls to get complete list - $(TYPEDIR)/*/man.rst does not work
+# Using ls does not work if no file with given pattern exist, so use wildcard
+MANTYPESRC=$(wildcard $(TYPEDIR)/*/man.rst)
+MANTYPEPREFIX=$(subst $(TYPEDIR)/,$(MAN7DSTDIR)/cdist-type,$(MANTYPESRC))
+MANTYPES=$(subst /man.rst,.rst,$(MANTYPEPREFIX))
+
+# Link manpage: do not create man.html but correct named file
+$(MAN7DSTDIR)/cdist-type%.rst: $(TYPEDIR)/%/man.rst
+	mkdir -p $(MAN7DSTDIR)
+	ln -sf "../../../$^" $@
+
+# Manpages #2: reference
+DOCSREF=$(MAN7DSTDIR)/cdist-reference.rst
+DOCSREFSH=$(DOCS_SRC_DIR)/cdist-reference.rst.sh
+
+$(DOCSREF): $(DOCSREFSH)
+	$(DOCSREFSH)
+
+version:
+	@[ -f "cdist/version.py" ] || { \
+		printf "Missing 'cdist/version.py', please generate it first.\n" && exit 1; \
+	}
+
+# Manpages #3: generic part
+man: version $(MANTYPES) $(DOCSREF)
+	$(SPHINXM)
+
+html: version $(MANTYPES) $(DOCSREF)
+	$(SPHINXH)
+
+docs: man html
+
+docs-clean:
+	$(SPHINXC)
+
+# Manpages: .cdist Types
+DOT_CDIST_PATH=${HOME}/.cdist
+DOTMAN7DSTDIR=$(MAN7DSTDIR)
+DOTTYPEDIR=$(DOT_CDIST_PATH)/type
+DOTMANTYPESRC=$(wildcard $(DOTTYPEDIR)/*/man.rst)
+DOTMANTYPEPREFIX=$(subst $(DOTTYPEDIR)/,$(DOTMAN7DSTDIR)/cdist-type,$(DOTMANTYPESRC))
+DOTMANTYPES=$(subst /man.rst,.rst,$(DOTMANTYPEPREFIX))
+
+# Link manpage: do not create man.html but correct named file
+$(DOTMAN7DSTDIR)/cdist-type%.rst: $(DOTTYPEDIR)/%/man.rst
+	ln -sf "$^" $@
+
+dotman: version $(DOTMANTYPES)
+	$(SPHINXM)
+
+################################################################################
+# Speeches
+#
+SPEECHESOURCES=$(SPEECHDIR)/*.tex
+SPEECHES=$(SPEECHESOURCES:.tex=.pdf)
+
+# Create speeches and ensure Toc is up-to-date
+$(SPEECHDIR)/%.pdf: $(SPEECHDIR)/%.tex
+	pdflatex -output-directory $(SPEECHDIR) $^
+	pdflatex -output-directory $(SPEECHDIR) $^
+	pdflatex -output-directory $(SPEECHDIR) $^
+
+speeches: $(SPEECHES)
+
+################################################################################
+# Misc
+#
+clean: docs-clean
+	rm -f $(DOCS_SRC_DIR)/cdist-reference.rst
+
+	find "$(DOCS_SRC_DIR)" -mindepth 2 -type l \
+	| xargs rm -f
+
+	find * -name __pycache__  | xargs rm -rf
+
+	# distutils
+	rm -rf ./build
+
+################################################################################
+# install
+#
+
+install:
+	python3 setup.py install
+
+install-user:
+	python3 setup.py install --user

PKGBUILD.in

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+version="$1"
+outfile=${0%.in}
+
+cat << eof > "${outfile}"
+pkgname=cdist
+pkgver=$version
+pkgrel=1
+pkgdesc='A Usable Configuration Management System"'
+arch=('any')
+url='https://www.cdi.st/'
+license=('GPL3')
+depends=('python>=3.2.0')
+source=("http://pypi.python.org/packages/source/c/cdist/cdist-\${pkgver}.tar.gz")
+
+package() {
+    cd cdist-\${pkgver}
+    python3 setup.py build install --root="\${pkgdir}"
+    find "\$pkgdir" -type d -exec chmod 0755 {} \;
+    find "\$pkgdir" -type f -exec chmod a+r {} \;
+}
+eof
+
+makepkg -g >> "${outfile}"
+
+# Fix this issue:
+# error: failed to upload cdist-3.1.6-1.src.tar.gz: Error - all files must have permissions of 644 or 755.
+chmod a+r "${outfile}"

README

@@ -1,313 +1,7 @@
-[[!meta title="cdist - usable configuration management"]]
-
-     
-                 ..          .       .x+=:.        s   
-               dF           @88>    z`    ^%      :8  
-              '88bu.        %8P        .   <k    .88 
-          .   '*88888bu      .       .@8Ned8"   :888ooo
-     .udR88N    ^"*8888N   .@88u   .@^%8888"  -*8888888
-    <888'888k  beWE "888L ''888E` x88:  `)8b.   8888
-    9888 'Y"   888E  888E   888E  8888N=*8888   8888
-    9888       888E  888E   888E   %8"    R88   8888
-    9888       888E  888F   888E    @8Wou 9%   .8888Lu=
-    ?8888u../ .888N..888    888&  .888888P`    ^%888*
-     "8888P'   `"888*""     R888" `   ^"F        'Y" 
-       "P'        ""         ""  
-     
-
-[[!toc levels=3]]
-
-## Introduction
-
-cdist is an alternative to other configuration management systems like
-[cfengine](http://www.cfengine.org/),
-[bcfg2](http://trac.mcs.anl.gov/projects/bcfg2),
-[chef](http://wiki.opscode.com/display/chef/)
-and [puppet](http://www.puppetlabs.com/).
-But cdist ticks differently, here is the feature set that makes it unique: 
-
-[[!table  data="""
-Keywords | Description
-Simplicity | There is only one type to extend cdist called ***type***
-Design | Type and core cleanly seperated
-Design | Sticks completly to the KISS (keep it simple and stupid) paradigma
-Design | Meaningful error messages - do not lose time debugging error messages
-Design | Consistency in behaviour, naming and documentation
-Design | No surprise factor: Only do what is obviously clear, no magic
-Design | Define target state, do not focus on methods or scripts
-Design | Push architecture: Instantly apply your changes
-Small core | cdist's core is very small - less code, less bugs
-Fast development | Focus on straightforwardness of type creation is a main development objective
-Modern Programming Language | cdist is written in Python
-Requirements, Scalability | No central server needed, cdist operates in push mode and can be run from any computer
-Requirements, Scalability, Upgrade | cdist only needs to be updated on the master, not on the target hosts
-Requirements, Security | Uses well-know [SSH](http://www.openssh.com/) as transport protocol
-Requirements, Simplicity | Requires only shell and SSH server on the target
-UNIX | Reuse of existing tools like cat, find, mv, ...
-UNIX, familar environment, documentation | Is available as manpages and HTML
-UNIX, simplicity, familar environment | cdist is configured in POSIX shell
-"""]]
-
-### Documentation
-
-The cdist documentation is included as manpages in the distribution.  
-You can browse the documentation online as well:
-
- * [latest version](man/latest)
- * [all versions (>= 2.0.4)](man)
-
-### OS support
-
-cdist was tested or is know to run on at least
-
- * [Archlinux](http://www.archlinux.org/)
- * [Debian](http://www.debian.org/)
- * [CentOS](http://www.centos.org/)
- * [Fedora](http://fedoraproject.org/)
- * [Gentoo](http://www.gentoo.org/)
- * [Mac OS X](http://www.apple.com/macosx/)
- * [OpenBSD](http://www.openbsd.org)
- * [Redhat](http://www.redhat.com/)
- * [Ubuntu](http://www.ubuntu.com/)
- * [XenServer](http://www.citrix.com/xenserver/)
-
-
-## Requirements
-
-### Server
-
- * A posix like shell
- * Python (>= 3.2 required)
- * SSH client
- * Asciidoc (for building the manpages)
-
-### Client ("target host")
-
- * A posix like shell
- * SSH server
-
-
-## Installation
-
-### Preparation
-
-Ensure you have Python 3.2 installed on the machine you use to **deploy to the targets**
-(the ***source host***).
-
-#### Archlinux
-
-Archlinux already has python >= 3.2, so you only need to do:
-
-    pacman -S python
-
-#### Debian
-
-For Debian >= wheezy:
-
-    aptitude install python3
-
-For older Debian versions, installing python 3.2 manually is required.
-
-#### Fedora
-
-For Fedora >= 15:
-
-    yum install python3
-
-#### FreeBSD
-
-For the port:
-
-    cd /usr/ports/lang/python32/ && make install clean
-
-For the package:
-
-    pkg_add -r python32
-
-#### Gentoo
-
-Gentoo only provides python 3.2 in testing packages (http://www.gentoo.org/doc/en/handbook/handbook-x86.xml?part=3&chap=3).
-If you want to ensure nothing breaks you must set back the python version to what was default before.
-
-    emerge -av =python-3.2.2 --autounmask-write
-    emerge -av =python-3.2.2
-    eselect python list
-    eselect python list set python3.2
-
-#### Max OS X
-
-You can choose between Homebrew and Macports, either way works:
-
-[Homebrew](http://mxcl.github.com/homebrew/) variant:
-
-    brew install python3
-
-[Macports](http://www.macports.org/install.php) variant:
-
-    port install python32
-    ln -s /opt/local/bin/python3.2 /opt/local/bin/python3
-
-### Get cdist
-
-You can clone cdist from git, which gives you the advantage of having
-a version control in place for development of your own stuff as well.
-To install cdist, execute the following commands:
-
-    git clone git://git.schottelius.org/cdist
-    cd cdist
-    export PATH=$PATH:$(pwd -P)/bin
-
-    # If you want the manpages
-    ./build man
-    export MANPATH=$MANPATH:$(pwd -P)/doc/man
-
-
-### Available versions
-
-There are at least the following branches available:
-
- * Development: master
- * 2.0: Python rewrite of cdist core [stable branch]
-
-Old versions:
-
- * 1.7: Bugfixes, cleanups, new type and explorer rename
- * 1.6: New types, cleaned up \_\_package* types, internal cleanup
- * 1.5: Focus on object orientation instead of global stage orientation
- * 1.4: Support for redefiniton of objects (if equal)
- * 1.3: Support for local and remote code execution (current stable)
- * 1.2: Dependencies supported
- * 1.1: \_\_file to \_\_file, \_\_directory, \_\_link migration
- * 1.0: First official release
-
-Other branches may be available for features or bugfixes, but they
-may vanish at any point. To select a specific branch use
-
-    # Generic code
-    git checkout -b <name> origin/<name>
-    
-    # Stay on a specific version
-    version=2.0
-    git checkout -b $version origin/$version
-
-### Mirrors
-
- * git://github.com/telmich/cdist.git ([github](https://github.com/telmich/cdist))
- * git://git.code.sf.net/p/cdist/code ([sourceforge](https://sourceforge.net/p/cdist/code))
-
-## Update
-
-To upgrade cdist in the current branch use
-
-    git pull
-
-    # Also update the manpages
-    ./build man
-    export MANPATH=$MANPATH:$(pwd -P)/doc/man
-
-If you stay on a version branche (i.e. 1.0, 1.1., ...), nothing should break.
-The master branch on the other hand is the development branch and may not be
-working, break your setup or eat the tree in your garden.
-
-### Upgrading from 1.7 to 2.0
-
-* Ensure python (>= 3.2) is installed on the server
-* Use "cdist config host" instead of "cdist-deploy-to host"
-* Use "cdist config -p host1 host2" instead of "cdist-mass-deploy"
-* Use "cdist banner" for fun
-* Use **\_\_object_fq** instead of **\_\_self** in manifests
-
-### Upgrading from 1.6 to 1.7
-
-* If you used the global explorer **hardware_type**, you need to change
-  your code to use **machine** instead.
-
-### Upgrading from 1.5 to 1.6
-
-* If you used **\_\_package_apt --preseed**, you need to use the new
-  type **\_\_debconf_set_selections** instead.
-* The **\_\_package** types accepted either --state deinstalled or
-  --state uninstaaled. Starting with 1.6, it was made consistently
-  to --state removed.
-
-### Upgrading from 1.3 to 1.5
-
-No incompatiblities.
-
-### Upgrading from 1.2 to 1.3
-
-Rename **gencode** of every type to **gencode-remote**.
-
-### Upgrading from 1.1 to 1.2
-
-No incompatiblities.
-
-### Upgrading from 1.0 to 1.1
-
-In 1.1 the type **\_\_file** was split into **\_\_directory**, **\_\_file** and
-**\_\_link**. The parameter **--type** was removed from **\_\_file**. Thus you
-need to replace **\_\_file** calls in your manifests:
-
- * Remove --type from all \_\_file calls
- * If type was symlink, use \_\_link and --type symbolic
- * If type was directory, use \_\_directory
-
-
-## Support
-
-### IRC
-
-You can join the development ***IRC channel***
-[#cstar on irc.freenode.org](irc://irc.freenode.org/#cstar).
-
-### Mailing list
-
-Bug reports, questions, patches, etc. should be send to the
-[cdist mailing list](http://l.schottelius.org/mailman/listinfo/cdist).
-
-### Linkedin
-
-If you have an account
-at [Linked in](http://www.linkedin.com/),
-you can join the
-[cdist group](http://www.linkedin.com/groups/cdist-configuration-management-3952797).
-
-### Commercial support
-
-You can request commercial support for cdist from
-[my company](http://firma.schottelius.org/english/).
-
-## Used by
-
-If you're using cdist, feel free to send a report to the mailing list.
-Interesting information are for instance
-
- * Which services do you manage?
- * How many machines do you manage?
- * What are the pros/cons you see in cdist?
- * General comments/critics
-
-### Nico Schottelius, Systems Group ETH Zurich and privately
-
-Yes, I'm actually eating my own dogfood and currently managing
-
- * [plone](http://plone.org/) (cms)
- * [moinmoin](http://moinmo.in/) (wiki)
- * [apache](http://httpd.apache.org/) (webserver)
- * [kerberos (mit)](http://web.mit.edu/kerberos/) (authentication)
- * [nss-pam-ldapd](http://arthurdejong.org/nss-pam-ldapd/) (authentication)
- * [ircd-hybrid](http://www.ircd-hybrid.org/) (chat)
- * [stunnel](http://stunnel.mirt.net/) (SSL tunnel)
- * [mercurial-server](http://www.lshift.net/mercurial-server.html) (version control)
- * [xfce](http://www.xfce.org/) (lightweight desktop environment)
- * [slim](http://slim.berlios.de/) (graphical login manager for X11)
-
-with cdist on more than **60** production machines of the
-[Systems Group](http://www.systems.ethz.ch) at the
-[ETH Zurich](http://www.ethz.ch) as well at home.
-
-### Steven Armstrong, CBRG ETH Zurich
-
-The CBRG is managing most of their compute clusters with cdist.
+cdist
+-----
 
+cdist is a usable configuration management system.
 
+For the web documentation have a look at https://www.cdi.st/
+or at docs/src for reStructuredText manual.

README-maintainers

@@ -0,0 +1,4 @@
+Maintainers should use ./bin/build-helper script.
+
+Makefile is intended for end users. It can be used for non-maintaining
+targets that can be run from pure source (without git repository).

bin/build-helper

@@ -0,0 +1,550 @@
+#!/bin/sh
+#
+# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2016-2019 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# This file contains the heavy lifting found usually in the Makefile.
+#
+
+usage() {
+    printf "usage: %s TARGET [TARGET-ARGS...]
+    Available targets:
+        changelog-changes
+        changelog-version
+        check-date
+        check-unittest
+        ml-release
+        archlinux-release
+        pypi-release
+        release-git-tag
+        sign-git-release
+        release
+        test
+        test-remote
+        pycodestyle
+        pep8
+        check-pycodestyle
+        shellcheck-global-explorers
+        shellcheck-type-explorers
+        shellcheck-manifests
+        shellcheck-local-gencodes
+        shellcheck-remote-gencodes
+        shellcheck-scripts
+        shellcheck-gencodes
+        shellcheck-types
+        shellcheck
+        shellcheck-type-files
+        shellcheck-with-files
+        shellcheck-build-helper
+        check-shellcheck
+        version-branch
+        version
+        target-version
+        clean
+        distclean\n" "$1"
+}
+
+basename="${0##*/}"
+
+if [ $# -lt 1 ]
+then
+    usage "${basename}"
+    exit 1
+fi
+
+option=$1; shift
+
+SHELLCHECKCMD="shellcheck -s sh -f gcc -x"
+# Skip SC2154 for variables starting with __ since such variables are cdist
+# environment variables.
+SHELLCHECK_SKIP=': __.*is referenced but not assigned.*\[SC2154\]'
+
+# Change to checkout directory
+basedir="${0%/*}/../"
+cd "$basedir"
+
+case "$option" in
+    changelog-changes)
+        if [ "$#" -eq 1 ]; then
+            start=$1
+        else
+            start="[[:digit:]]"
+        fi
+
+        end="[[:digit:]]"
+
+        awk -F: "BEGIN { start=0 }
+            {
+                if(start == 0) {
+                    if (\$0 ~ /^$start/) {
+                        start = 1
+                    }
+                } else {
+                    if (\$0 ~ /^$end/) {
+                        exit
+                    } else {
+                        print \$0 
+                    }
+                }
+            }" "$basedir/docs/changelog"
+    ;;
+
+    changelog-version)
+        # get version from changelog
+        grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/:.*//'
+    ;;
+
+    check-date)
+        # verify date in changelog is today
+        date_today="$(date +%Y-%m-%d)"
+        date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //')
+
+        if [ "$date_today" != "$date_changelog" ]; then
+            printf "Date in changelog is not today\n"
+            printf "Changelog date: %s\n" "${date_changelog}"
+            exit 1
+        fi
+    ;;
+
+    check-unittest)
+        "$0" test
+    ;;
+
+    ml-release)
+        if [ $# -ne 1 ]; then
+            printf "%s ml-release version\n" "$0" >&2
+            exit 1
+        fi
+
+        version=$1; shift
+
+        ( 
+        cat << eof
+Subject: cdist $version has been released
+
+Hello .*,
+
+cdist $version has been released with the following changes:
+
+eof
+
+        "$0" changelog-changes "$version"
+        cat << eof
+
+eof
+        ) > mailinglist.tmp
+    ;;
+
+    archlinux-release)
+        if [ $# -ne 1 ]; then
+            printf "%s archlinux-release version\n" "$0" >&2
+            exit 1
+        fi
+        version=$1; shift
+
+        ARCHLINUXTAR="cdist-${version}-1.src.tar.gz"
+        ./PKGBUILD.in "${version}"
+        umask 022
+        mkaurball
+        burp -c system "${ARCHLINUXTAR}"
+    ;;
+
+    pypi-release)
+        # Ensure that pypi release has the right version
+        "$0" version
+
+        make docs-clean
+        make docs
+        python3 setup.py sdist upload
+    ;;
+
+    release-git-tag)
+        target_version=$($0 changelog-version)
+        if git rev-parse --verify "refs/tags/${target_version}" 2>/dev/null; then
+            printf "Tag for %s exists, aborting\n" "${target_version}"
+            exit 1
+        fi
+        printf "Enter tag description for %s: " "${target_version}"
+        read -r tagmessage
+
+        # setup for signed tags:
+        # gpg --fulL-gen-key
+        # gpg --list-secret-keys --keyid-format LONG
+        # git config --local user.signingkey <id>
+        # for exporting pub key:
+        #     gpg --armor --export <id> > pubkey.asc
+        #     gpg --output pubkey.gpg --export <id>
+        # show tag with signature
+        # git show <tag>
+        # verify tag signature
+        # git tag -v <tag>
+        #
+        # gpg verify signature
+        # gpg --verify <asc-file> <file>
+        # gpg --no-default-keyring --keyring <pubkey.gpg> --verify <asc-file> <file>
+        # Ensure gpg-agent is running.
+        GPG_TTY=$(tty)
+        export GPG_TTY
+        gpg-agent
+
+        git tag -s "$target_version" -m "$tagmessage"
+        git push --tags
+    ;;
+
+    sign-git-release)
+        if [ $# -lt 2 ]
+        then
+            printf "usage: %s sign-git-release TAG TOKEN [ARCHIVE]\n" "$0"
+            printf "    if ARCHIVE is not specified then it is created\n"
+            exit 1
+        fi
+        tag="$1"
+        if ! git rev-parse -q --verify "${tag}" >/dev/null 2>&1
+        then
+            printf "Tag \"%s\" not found.\n" "${tag}"
+            exit 1
+        fi
+        token="$2"
+        if [ $# -gt 2 ]
+        then
+            archivename="$3"
+        else
+            archivename="cdist-${tag}.tar"
+            git archive --prefix="cdist-${tag}/" -o "${archivename}" "${tag}" \
+                || exit 1
+            # make sure target version is generated
+            "$0" target-version
+            tar -x -f "${archivename}" || exit 1
+            cp cdist/version.py "cdist-${tag}/cdist/version.py" || exit 1
+            tar -c -f "${archivename}" "cdist-${tag}/" || exit 1
+            rm -r -f "cdist-${tag}/"
+            gzip "${archivename}" || exit 1
+            archivename="${archivename}.gz"
+        fi
+        gpg --armor --detach-sign "${archivename}" || exit 1
+
+        project="ungleich-public%2Fcdist"
+        sed_cmd='s/^.*"markdown":"\([^"]*\)".*$/\1/'
+
+        # upload archive
+        response_archive=$(curl -f -X POST \
+             --http1.1 \
+             -H "PRIVATE-TOKEN: ${token}" \
+             -F "file=@${archivename}" \
+             "https://code.ungleich.ch/api/v4/projects/${project}/uploads" \
+             | sed "${sed_cmd}") || exit 1
+
+        # upload archive signature
+        response_archive_sig=$(curl -f -X POST \
+             --http1.1 \
+             -H "PRIVATE-TOKEN: ${token}" \
+             -F "file=@${archivename}.asc" \
+            "https://code.ungleich.ch/api/v4/projects/${project}/uploads" \
+             | sed "${sed_cmd}") || exit 1
+
+        # make release
+        changelog=$("$0" changelog-changes "$1" | sed 's/^[[:space:]]*//')
+        release_notes=$(
+            printf "%s\n\n%s\n\n**Changelog**\n\n%s\n" \
+                "${response_archive}" "${response_archive_sig}" "${changelog}"
+        )
+        curl -f -X POST \
+             -H "PRIVATE-TOKEN: ${token}" \
+             -F "description=${release_notes}" \
+            "https://code.ungleich.ch/api/v4/projects/${project}/repository/tags/${tag}/release" \
+            || exit 1
+
+        # remove generated files (archive and asc)
+        if [ $# -eq 2 ]
+        then
+            rm -f "${archivename}"
+        fi
+        rm -f "${archivename}.asc"
+    ;;
+
+    release)
+        set -e
+        target_version=$($0 changelog-version)
+        target_branch=$($0 version-branch)
+
+        printf "Beginning release process for %s\n" "${target_version}"
+
+        # First check everything is sane
+        "$0" check-date
+        "$0" check-unittest
+        "$0" check-pycodestyle
+        "$0" check-shellcheck
+
+        # Generate version file to be included in packaging
+        "$0" target-version
+
+        # Ensure the git status is clean, else abort
+        if ! git diff-index --name-only --exit-code HEAD ; then
+            printf "Unclean tree, see files above, aborting.\n"
+            exit 1
+        fi
+
+        # Ensure we are on the master branch
+        masterbranch=yes
+        if [ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then
+            printf "Releases are happening from the master branch, aborting.\n"
+
+            printf "Enter the magic word to release anyway:"
+            read -r magicword
+
+            if [ "$magicword" = "iknowwhatido" ]; then
+                masterbranch=no
+            else
+                exit 1
+            fi
+        fi
+
+        if [ "$masterbranch" = yes ]; then
+            # Ensure version branch exists
+            if ! git rev-parse --verify "refs/heads/${target_branch}" 2>/dev/null; then
+                git branch "$target_branch"
+            fi
+
+            # Merge master branch into version branch
+            git checkout "$target_branch"
+            git merge master
+        fi
+
+        # Verify that after the merge everything works
+        "$0" check-date
+        "$0" check-unittest
+
+        # Generate documentation (man and html)
+        # First, clean old generated docs
+        make docs-clean
+        make docs
+
+        ############################################################# 
+        # Everything green, let's do the release
+
+        # Tag the current commit
+        "$0" release-git-tag
+
+        # Also merge back the version branch
+        if [ "$masterbranch" = yes ]; then
+            git checkout master
+            git merge "$target_branch"
+        fi
+
+        # Publish git changes
+        # if you want to have mirror locally then uncomment this and comment below
+        #     git push --mirror
+            git push
+            # push also new branch and set up tracking
+            git push -u origin "${target_branch}"
+        # fi
+
+        # Create and publish package for pypi
+        "$0" pypi-release
+
+        # sign git tag
+        printf "Enter upstream repository authentication token: "
+        read -r token
+        "$0" sign-git-release "${target_version}" "${token}"
+
+        # Announce change on ML
+        "$0" ml-release "${target_version}"
+
+        cat << eof
+Manual steps post release:
+    - cdist-web
+    - send mail body generated in mailinglist.tmp and inform Dmitry for deb
+    - twitter
+eof
+    ;;
+
+    test)
+        if [ ! -f "cdist/version.py" ]
+        then
+            printf "cdist/version.py is missing, generate it first.\n"
+            exit 1
+        fi
+
+        PYTHONPATH="$(pwd -P)"
+        export PYTHONPATH
+
+        if [ $# -lt 1 ]; then
+            python3 -m cdist.test
+        else
+            python3 -m unittest "$@"
+        fi
+    ;;
+
+    test-remote)
+        if [ ! -f "cdist/version.py" ]
+        then
+            printf "cdist/version.py is missing, generate it first.\n"
+            exit 1
+        fi
+
+        PYTHONPATH="$(pwd -P)"
+        export PYTHONPATH
+
+        python3 -m cdist.test.exec.remote
+    ;;
+
+    pycodestyle|pep8)
+        pycodestyle "${basedir}" "${basedir}/scripts/cdist" | less
+    ;;
+
+    check-pycodestyle)
+        "$0" pycodestyle
+        printf "\\nPlease review pycodestyle report.\\n"
+        while true
+        do
+            printf "Continue (yes/no)?\n"
+            any=
+            read -r any
+            case "$any" in
+                yes)
+                    break
+                ;;
+                no)
+                    exit 1
+                ;;
+                *)
+                    printf "Please answer with 'yes' or 'no' explicitly.\n"
+                ;;
+        esac
+        done
+    ;;
+
+    shellcheck-global-explorers)
+        find cdist/conf/explorer -type f -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-type-explorers)
+        find cdist/conf/type -type f -path "*/explorer/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-manifests)
+        find cdist/conf/type -type f -name manifest -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-local-gencodes)
+        find cdist/conf/type -type f -name gencode-local -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-remote-gencodes)
+        find cdist/conf/type -type f -name gencode-remote -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-scripts)
+        ${SHELLCHECKCMD} scripts/cdist-dump scripts/cdist-new-type || exit 0
+    ;;
+
+    shellcheck-gencodes)
+        "$0" shellcheck-local-gencodes
+        "$0" shellcheck-remote-gencodes
+    ;;
+
+    shellcheck-types)
+        "$0" shellcheck-type-explorers
+        "$0" shellcheck-manifests
+        "$0" shellcheck-gencodes
+    ;;
+
+    shellcheck)
+        "$0" shellcheck-global-explorers
+        "$0" shellcheck-types
+        "$0" shellcheck-scripts
+    ;;
+
+    shellcheck-type-files)
+        find cdist/conf/type -type f -path "*/files/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+    ;;
+
+    shellcheck-with-files)
+        "$0" shellcheck
+        "$0" shellcheck-type-files
+    ;;
+
+    shellcheck-build-helper)
+        ${SHELLCHECKCMD} ./bin/build-helper
+    ;;
+
+    check-shellcheck)
+        "$0" shellcheck
+        printf "\\nPlease review shellcheck report.\\n"
+        while true
+        do
+            printf "Continue (yes/no)?\n"
+            any=
+            read -r any
+            case "$any" in
+                yes)
+                    break
+                ;;
+                no)
+                    exit 1
+                ;;
+                *)
+                    printf "Please answer with 'yes' or 'no' explicitly.\n"
+                ;;
+        esac
+        done
+    ;;
+
+    version-branch)
+        "$0" changelog-version | cut -d. -f '1,2'
+    ;;
+
+    version)
+        printf "VERSION = \"%s\"\n" "$(git describe)" > cdist/version.py
+    ;;
+
+    target-version)
+        target_version=$($0 changelog-version)
+        printf "VERSION = \"%s\"\n" "${target_version}" > cdist/version.py
+    ;;
+
+    clean)
+        make clean
+
+        # Archlinux
+        rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz
+        rm -rf pkg/ src/
+
+        rm -f MANIFEST PKGBUILD
+        rm -rf dist/
+
+        # Signed release
+        rm -f cdist-*.tar.gz
+        rm -f cdist-*.tar.gz.asc
+
+        # Temp files
+        rm -f ./*.tmp
+    ;;
+
+    distclean)
+        "$0" clean
+        rm -f cdist/version.py
+    ;;
+    *)
+        printf "Unknown target: '%s'.\n" "${option}" >&2
+        usage "${basename}"
+        exit 1
+    ;;
+
+esac

bin/cdist

@@ -1,7 +1,7 @@
-#!/usr/bin/env python3
+#!/bin/sh
 # -*- coding: utf-8 -*-
 #
-# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2012 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -20,215 +20,14 @@
 #
 #
 
-def commandline():
-    """Parse command line"""
-    import argparse
+# Wrapper for real script to allow execution from checkout
+dir=${0%/*}
 
-    import cdist.banner
-    import cdist.config
-    import cdist.install
+# Ensure version is present - the bundled/shipped version contains a static version,
+# the git version contains a dynamic version
+"$dir/build-helper" version
 
-    # Construct parser others can reuse
-    parser = {}
-    # Options _all_ parsers have in common
-    parser['loglevel'] = argparse.ArgumentParser(add_help=False)
-    parser['loglevel'].add_argument('-d', '--debug',
-        help='Set log level to debug', action='store_true',
-        default=False)
-    parser['loglevel'].add_argument('-v', '--verbose',
-        help='Set log level to info, be more verbose',
-        action='store_true', default=False)
+libdir=$(cd "${dir}/../" && pwd -P)
+export PYTHONPATH="${libdir}"
 
-    # Main subcommand parser
-    parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION,
-        parents=[parser['loglevel']])
-    parser['main'].add_argument('-V', '--version',
-        help='Show version', action='version',
-        version='%(prog)s ' + cdist.VERSION)
-    parser['sub'] = parser['main'].add_subparsers(title="Commands")
-
-    # Banner
-    parser['banner'] = parser['sub'].add_parser('banner', 
-        parents=[parser['loglevel']])
-    parser['banner'].set_defaults(func=cdist.banner.banner)
-
-    # Config and install (common stuff)
-    parser['configinstall'] = argparse.ArgumentParser(add_help=False)
-    parser['configinstall'].add_argument('host', nargs='+',
-        help='one or more hosts to operate on')
-    parser['configinstall'].add_argument('-c', '--cdist-home',
-         help='Change cdist home (default: .. from bin directory)',
-         action='store')
-    parser['configinstall'].add_argument('-i', '--initial-manifest', 
-         help='Path to a cdist manifest or \'-\' to read from stdin.',
-         dest='manifest', required=False)
-    parser['configinstall'].add_argument('-p', '--parallel',
-         help='Operate on multiple hosts in parallel',
-         action='store_true', dest='parallel')
-    parser['configinstall'].add_argument('-s', '--sequential',
-         help='Operate on multiple hosts sequentially (default)',
-         action='store_false', dest='parallel')
-
-    # Config
-    parser['config'] = parser['sub'].add_parser('config',
-        parents=[parser['loglevel'], parser['configinstall']])
-    parser['config'].set_defaults(func=config)
-
-    # Install
-    parser['install'] = parser['sub'].add_parser('install',
-        parents=[parser['loglevel'], parser['configinstall']])
-    parser['install'].set_defaults(func=install)
-
-    for p in parser:
-        parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
-
-    args = parser['main'].parse_args(sys.argv[1:])
-
-    # Loglevels are handled globally in here and debug wins over verbose
-    if args.verbose:
-        logging.root.setLevel(logging.INFO)
-    if args.debug:
-        logging.root.setLevel(logging.DEBUG)
-
-    log.debug(args)
-    args.func(args)
-
-def config(args):
-    configinstall(args, mode=cdist.config.Config)
-
-def install(args):
-    configinstall(args, mode=cdist.install.Install)
-
-def configinstall(args, mode):
-    """Configure or install remote system"""
-    import multiprocessing
-    import time
-
-    initial_manifest_tempfile = None
-    if args.manifest == '-':
-        # read initial manifest from stdin
-        import tempfile
-        handle, initial_manifest_temp_path = tempfile.mkstemp(prefix='cdist.stdin.')
-        with os.fdopen(handle, 'w') as fd:
-            fd.write(sys.stdin.read())
-        args.manifest = initial_manifest_temp_path
-        import atexit
-        atexit.register(lambda: os.remove(initial_manifest_temp_path))
-
-    try:
-        process = {}
-        failed_hosts = []
-        time_start = time.time()
-
-        for host in args.host:
-            if args.parallel:
-                log.debug("Creating child process for %s", host)
-                process[host] = multiprocessing.Process(target=configinstall_onehost, args=(host, args, mode, True))
-                process[host].start()
-            else:
-                if not configinstall_onehost(host, args, mode, parallel=False):
-                    failed_hosts.append(host)
-
-        if args.parallel:
-            for p in process.keys():
-                log.debug("Joining process %s", p)
-                process[p].join()
-
-                if not process[p].exitcode == 0:
-                    failed_hosts.append(p)
-
-        if len(failed_hosts) > 0:
-            log.warn("Failed to deploy to the following hosts: " + 
-                " ".join(failed_hosts))
-
-        time_end = time.time()
-        log.info("Total processing time for %s host(s): %s", len(args.host),
-                    (time_end - time_start))
-
-    except KeyboardInterrupt:
-        if args.parallel:
-            for p in process.keys():
-                # NOT needed: KeyBoardInterrupet (aka SIGINT)
-                # is forwarded to processes spawned by multiprocess!
-                # pid = process[p].pid.__str__()
-                #log.warn("Terminating deploy " + p + " (" + pid + ")")
-                # process[p].terminate()
-                pass
-
-        sys.exit(0)
-
-
-def configinstall_onehost(host, args, mode, parallel):
-    """Configure or install ONE remote system"""
-
-    try:
-        import cdist.context
-
-        context = cdist.context.Context(
-            target_host=host,
-            initial_manifest=args.manifest,
-            base_path=args.cdist_home,
-            exec_path=sys.argv[0],
-            debug=args.debug)
-
-        c = mode(context)
-        c.deploy_and_cleanup()
-        context.cleanup()
-
-    except cdist.Error as e:
-        log.error(e)
-        return False
-    except KeyboardInterrupt:
-        # Do not care in sequential mode, catch in parallel mode
-        if not parallel:
-            raise
-        else:
-            # Catch here, above does not need to know about our errors
-            return False
-
-    return True
-
-def emulator():
-    """Prepare and run emulator"""
-    try:
-        import cdist.emulator
-        emulator = cdist.emulator.Emulator(sys.argv)
-        emulator.run()
-
-    except cdist.Error as e:
-        log.error(e)
-        sys.exit(1)
-
-if __name__ == "__main__":
-    try:
-        import logging
-        import os
-        import re
-        import sys
-
-        cdistpythonversion = '3.2'
-        if sys.version < cdistpythonversion:
-            print('Cdist requires Python >= ' + cdistpythonversion +
-                ' on the source host.')
-            sys.exit(1)
-
-        # Ensure our /lib/ is included into PYTHON_PATH
-        sys.path.insert(0, os.path.abspath(
-                os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')))
-
-        # And now import our stuff
-        import cdist
-
-        log = logging.getLogger("cdist")
-
-        logging.basicConfig(format='%(levelname)s: %(message)s')
-
-        if re.match("__", os.path.basename(sys.argv[0])):
-            emulator()
-        else:
-            commandline()
-
-    except KeyboardInterrupt:
-        sys.exit(0)
-
-    sys.exit(0)
+"$dir/../scripts/cdist" "$@"

build

@@ -1,170 +0,0 @@
-#!/bin/sh
-#
-# 2011 Nico Schottelius (nico-cdist at schottelius.org)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Push a directory to a target, both sides have the same name (i.e. explorers)
-# or
-# Pull a directory from a target, both sides have the same name (i.e. explorers)
-#
-
-# exit on any error
-#set -e
-
-version=$(git describe)
-
-# Manpage and HTML
-A2XM="a2x -f manpage --no-xmllint -a encoding=UTF-8"
-A2XH="a2x -f xhtml --no-xmllint -a encoding=UTF-8"
-
-# Developer webbase
-WEBDIR=$HOME/niconetz
-WEBBASE=$WEBDIR/software/cdist
-WEBMAN=$WEBBASE/man/$version
-WEBPAGE=${WEBBASE}.mdwn
-
-# Documentation
-MANDIR=doc/man
-MAN1DSTDIR=${MANDIR}/man1
-MAN7DSTDIR=${MANDIR}/man7
-SPEECHESDIR=doc/speeches
-
-case "$1" in
-   man)
-      set -e
-      "$0" mangen
-      "$0" mantype
-      "$0" manbuild
-   ;;
-
-   manbuild)
-      trap abort INT
-      abort() {
-         kill 0
-      }
-      for section in 1 7; do
-         for src in ${MANDIR}/man${section}/*.text; do
-            manpage="${src%.text}.$section"
-            if [ ! -f "$manpage" -o "$manpage" -ot "$src" ]; then
-               echo "Compiling man page for $src"
-               $A2XM "$src"
-            fi
-            htmlpage="${src%.text}.html"
-            if [ ! -f "$htmlpage" -o "$htmlpage" -ot "$src" ]; then
-               echo "Compiling html page for $src"
-               $A2XH "$src"
-            fi
-         done
-      done
-   ;;
-
-   mantype)
-	   for mansrc in conf/type/*/man.text; do
-         dst="$(echo $mansrc | sed -e 's;conf/;cdist-;'  -e 's;/;;' -e 's;/man;;' -e 's;^;doc/man/man7/;')"
-         ln -sf "../../../$mansrc" "$dst"
-      done
-   ;;
-
-   mangen)
-      ${MANDIR}/cdist-reference.text.sh
-   ;;
-
-   release)
-        ./doc/dev/releasechecklist
-   ;;
-
-   speeches)
-      cd "$SPEECHESDIR"
-      for speech in *tex; do
-         pdflatex "$speech"
-         pdflatex "$speech"
-         pdflatex "$speech"
-      done
-   ;;
-      
-   webmain)
-      cp README ${WEBPAGE}
-      cd ${WEBDIR} && git commit -m "cdist main update" ${WEBPAGE}
-      cd ${WEBDIR} && make pub
-   ;;
-
-   web)
-      cp README ${WEBPAGE}
-      rm -rf ${WEBMAN}
-      mkdir -p ${WEBMAN}/man1 ${WEBMAN}/man7
-
-      # old stuff
-      # rm -rf ${WEBDIR}/${WEBBASE}/speeches && mkdir ${WEBDIR}/${WEBBASE}/speeches
-      # cp ${SPEECHESDIR}/*.pdf ${WEBDIR}/${WEBBASE}/speeches
-      # git describe > ${WEBDIR}/${WEBBASE}/man/VERSION
-
-      cp ${MAN1DSTDIR}/*.html ${WEBMAN}/man1 
-      cp ${MAN7DSTDIR}/*.html ${WEBMAN}/man7
-      
-      cd ${WEBDIR} && git add ${WEBBASE}
-      cd ${WEBDIR} && git commit -m "cdist update" ${WEBBASE} ${WEBPAGE}
-      cd ${WEBDIR} && make pub
-
-      # Fix ikiwiki, which does not like symlinks for pseudo security
-      ssh tee.schottelius.org \
-        "cd /home/services/www/nico/www.nico.schottelius.org/www/software/cdist/man &&
-            ln -sf "$version" latest"
-   ;;
-
-   p|pu|pub)
-      git push --mirror
-      git push --mirror github
-      git push --mirror sf
-   ;;
-
-   clean)
-      rm -f ${MAN7DSTDIR}/cdist-reference.text
-      find "${MANDIR}" -mindepth 2 -type l \
-         -o -name "*.1" \
-         -o -name "*.7" \
-         -o -name "*.html" \
-         -o -name "*.xml" \
-      | xargs rm -f
-   ;;
-
-   test)
-      shift # skip t
-      export PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib
-
-      if [ $# -lt 1 ]; then
-         python3 -m cdist.test
-      else
-         python3 -m unittest "$@"
-      fi
-   ;;
-
-   *)
-      echo ''
-      echo 'Welcome to cdist!'
-      echo ''
-      echo 'Here are the possible targets:'
-      echo ''
-      echo '	clean: Remove build stuff'
-      echo '	man: Build manpages (requires Asciidoc)'
-      echo '	test: Run tests'
-      echo ''
-      echo ''
-      echo "Unknown target, \"$1\"" >&2
-      exit 1
-   ;;
-esac

cdist/__init__.py

@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+#
+# 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
+# 2012-2017 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import os
+import hashlib
+
+import cdist.log
+import cdist.version
+
+VERSION = cdist.version.VERSION
+
+BANNER = """
+             ..          .       .x+=:.        s
+           dF           @88>    z`    ^%      :8
+          '88bu.        %8P        .   <k    .88
+      .   '*88888bu      .       .@8Ned8"   :888ooo
+ .udR88N    ^"*8888N   .@88u   .@^%8888"  -*8888888
+<888'888k  beWE "888L ''888E` x88:  `)8b.   8888
+9888 'Y"   888E  888E   888E  8888N=*8888   8888
+9888       888E  888E   888E   %8"    R88   8888
+9888       888E  888F   888E    @8Wou 9%   .8888Lu=
+?8888u../ .888N..888    888&  .888888P`    ^%888*
+ "8888P'   `"888*""     R888" `   ^"F        'Y"
+   "P'        ""         ""
+"""
+
+REMOTE_COPY = "scp -o User=root -q"
+REMOTE_EXEC = "ssh -o User=root"
+REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
+
+
+class Error(Exception):
+    """Base exception class for this project"""
+    pass
+
+
+class UnresolvableRequirementsError(cdist.Error):
+    """Resolving requirements failed"""
+    pass
+
+
+class CdistBetaRequired(cdist.Error):
+    """Beta functionality is used but beta is not enabled"""
+
+    def __init__(self, command, arg=None):
+        self.command = command
+        self.arg = arg
+
+    def __str__(self):
+        if self.arg is None:
+            err_msg = ("\'{}\' command is beta, but beta is "
+                       "not enabled. If you want to use it please enable beta "
+                       "functionalities by using the -b/--beta command "
+                       "line flag or setting CDIST_BETA env var.")
+            fmt_args = [self.command, ]
+        else:
+            err_msg = ("\'{}\' argument of \'{}\' command is beta, but beta "
+                       "is not enabled. If you want to use it please enable "
+                       "beta functionalities by using the -b/--beta "
+                       "command line flag or setting CDIST_BETA env var.")
+            fmt_args = [self.arg, self.command, ]
+        return err_msg.format(*fmt_args)
+
+
+class CdistEntityError(Error):
+    """Something went wrong while executing cdist entity"""
+    def __init__(self, entity_name, entity_params, stdout_paths,
+                 stderr_paths, subject=''):
+        self.entity_name = entity_name
+        self.entity_params = entity_params
+        self.stderr_paths = stderr_paths
+        self.stdout_paths = stdout_paths
+        if isinstance(subject, Error):
+            self.original_error = subject
+        else:
+            self.original_error = None
+        self.message = str(subject)
+
+    def _stdpath(self, stdpaths, header_name):
+        result = {}
+        for name, path in stdpaths:
+            if name not in result:
+                result[name] = []
+            try:
+                if os.path.exists(path) and os.path.getsize(path) > 0:
+                    output = []
+                    label_begin = name + ":" + header_name
+                    output.append(label_begin)
+                    output.append('\n')
+                    output.append('-' * len(label_begin))
+                    output.append('\n')
+                    with open(path, 'r') as fd:
+                        output.append(fd.read())
+                    output.append('\n')
+                    result[name].append(''.join(output))
+            except UnicodeError as ue:
+                result[name].append(('Cannot output {}:{} due to: {}.\n'
+                                     'You can try to read the error file "{}"'
+                                     ' yourself.').format(
+                                         name, header_name, ue, path))
+        return result
+
+    def _stderr(self):
+        return self._stdpath(self.stderr_paths, 'stderr')
+
+    def _stdout(self):
+        return self._stdpath(self.stdout_paths, 'stdout')
+
+    def _update_dict_list(self, target, source):
+        for x in source:
+            if x not in target:
+                target[x] = []
+            target[x].extend(source[x])
+
+    @property
+    def std_streams(self):
+        std_dict = {}
+        self._update_dict_list(std_dict, self._stdout())
+        self._update_dict_list(std_dict, self._stderr())
+        return std_dict
+
+    def __str__(self):
+        output = []
+        output.append(self.message)
+        output.append('\n\n')
+        header = "Error processing " + self.entity_name
+        under_header = '=' * len(header)
+        output.append(header)
+        output.append('\n')
+        output.append(under_header)
+        output.append('\n')
+        for param_name, param_value in self.entity_params:
+            output.append(param_name + ': ' + str(param_value))
+            output.append('\n')
+        output.append('\n')
+        for x in self.std_streams:
+            output.append(''.join(self.std_streams[x]))
+        return ''.join(output)
+
+
+class CdistObjectError(CdistEntityError):
+    """Something went wrong while working on a specific cdist object"""
+    def __init__(self, cdist_object, subject=''):
+        params = [
+            ('name', cdist_object.name, ),
+            ('path', cdist_object.absolute_path, ),
+            ('source', " ".join(cdist_object.source), ),
+            ('type', os.path.realpath(
+                cdist_object.cdist_type.absolute_path), ),
+        ]
+        stderr_paths = []
+        for stderr_name in os.listdir(cdist_object.stderr_path):
+            stderr_path = os.path.join(cdist_object.stderr_path,
+                                       stderr_name)
+            stderr_paths.append((stderr_name, stderr_path, ))
+        stdout_paths = []
+        for stdout_name in os.listdir(cdist_object.stdout_path):
+            stdout_path = os.path.join(cdist_object.stdout_path,
+                                       stdout_name)
+            stdout_paths.append((stdout_name, stdout_path, ))
+        super().__init__("object '{}'".format(cdist_object.name),
+                         params, stdout_paths, stderr_paths, subject)
+
+
+class CdistObjectExplorerError(CdistEntityError):
+    """
+    Something went wrong while working on a specific
+    cdist object explorer
+    """
+    def __init__(self, cdist_object, explorer_name, explorer_path,
+                 stderr_path, subject=''):
+        params = [
+            ('object name', cdist_object.name, ),
+            ('object path', cdist_object.absolute_path, ),
+            ('object source', " ".join(cdist_object.source), ),
+            ('object type', os.path.realpath(
+                cdist_object.cdist_type.absolute_path), ),
+            ('explorer name', explorer_name, ),
+            ('explorer path', explorer_path, ),
+        ]
+        stdout_paths = []
+        stderr_paths = [
+            ('remote', stderr_path, ),
+        ]
+        super().__init__("explorer '{}' of object '{}'".format(
+            explorer_name, cdist_object.name), params, stdout_paths,
+            stderr_paths, subject)
+
+
+class InitialManifestError(CdistEntityError):
+    """Something went wrong while executing initial manifest"""
+    def __init__(self, initial_manifest, stdout_path, stderr_path, subject=''):
+        params = [
+            ('path', initial_manifest, ),
+        ]
+        stdout_paths = [
+            ('init', stdout_path, ),
+        ]
+        stderr_paths = [
+            ('init', stderr_path, ),
+        ]
+        super().__init__('initial manifest', params, stdout_paths,
+                         stderr_paths, subject)
+
+
+class GlobalExplorerError(CdistEntityError):
+    """Something went wrong while executing global explorer"""
+    def __init__(self, name, path, stderr_path, subject=''):
+        params = [
+            ('name', name, ),
+            ('path', path, ),
+        ]
+        stderr_paths = [
+            ('remote', stderr_path, ),
+        ]
+        super().__init__("global explorer '{}'".format(name),
+                         params, [], stderr_paths, subject)
+
+
+def file_to_list(filename):
+    """Return list from \n seperated file"""
+    if os.path.isfile(filename):
+        file_fd = open(filename, "r")
+        lines = file_fd.readlines()
+        file_fd.close()
+
+        # Remove \n from all lines
+        lines = map(lambda s: s.strip(), lines)
+    else:
+        lines = []
+
+    return lines
+
+
+def str_hash(s):
+    """Return hash of string s"""
+    if isinstance(s, str):
+        return hashlib.md5(s.encode('utf-8')).hexdigest()
+    else:
+        raise Error("Param should be string")
+
+
+def home_dir():
+    if 'HOME' in os.environ:
+        home = os.environ['HOME']
+        if home:
+            rv = os.path.join(home, ".cdist")
+        else:
+            rv = None
+    else:
+        rv = None
+    return rv

cdist/argparse.py

@@ -0,0 +1,473 @@
+import argparse
+import cdist
+import multiprocessing
+import logging
+import collections
+import functools
+import cdist.configuration
+import cdist.preos
+
+
+# set of beta sub-commands
+BETA_COMMANDS = set(('install', 'inventory', ))
+# set of beta arguments for sub-commands
+BETA_ARGS = {
+    'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
+}
+EPILOG = "Get cdist at https://code.ungleich.ch/ungleich-public/cdist"
+# Parser others can reuse
+parser = None
+
+
+_verbosity_level_off = -2
+_verbosity_level = {
+    None: logging.WARNING,
+    _verbosity_level_off: logging.OFF,
+    -1: logging.ERROR,
+    0: logging.WARNING,
+    1: logging.INFO,
+    2: logging.VERBOSE,
+    3: logging.DEBUG,
+    4: logging.TRACE,
+}
+
+
+# Generate verbosity level constants:
+# VERBOSE_OFF, VERBOSE_ERROR, VERBOSE_WARNING, VERBOSE_INFO, VERBOSE_VERBOSE,
+# VERBOSE_DEBUG, VERBOSE_TRACE.
+this_globals = globals()
+for level in _verbosity_level:
+    const = 'VERBOSE_' + logging.getLevelName(_verbosity_level[level])
+    this_globals[const] = level
+
+
+# All verbosity levels above 4 are TRACE.
+_verbosity_level = collections.defaultdict(
+    lambda: logging.TRACE, _verbosity_level)
+
+
+def add_beta_command(cmd):
+    BETA_COMMANDS.add(cmd)
+
+
+def add_beta_arg(cmd, arg):
+    if cmd in BETA_ARGS:
+        if arg not in BETA_ARGS[cmd]:
+            BETA_ARGS[cmd].append(arg)
+    else:
+        BETA_ARGS[cmd] = set((arg, ))
+
+
+def check_beta(args_dict):
+    if 'beta' not in args_dict:
+        args_dict['beta'] = False
+    # Check only if beta is not enabled: if beta option is specified then
+    # raise error.
+    if not args_dict['beta']:
+        cmd = args_dict['command']
+        # first check if command is beta
+        if cmd in BETA_COMMANDS:
+            raise cdist.CdistBetaRequired(cmd)
+        # then check if some command's argument is beta
+        if cmd in BETA_ARGS:
+            for arg in BETA_ARGS[cmd]:
+                if arg in args_dict and args_dict[arg]:
+                    raise cdist.CdistBetaRequired(cmd, arg)
+
+
+def check_lower_bounded_int(value, lower_bound, name):
+    try:
+        val = int(value)
+    except ValueError:
+        raise argparse.ArgumentTypeError(
+                "{} is invalid int value".format(value))
+    if val < lower_bound:
+        raise argparse.ArgumentTypeError(
+                "{} is invalid {} value".format(val, name))
+    return val
+
+
+def get_parsers():
+    global parser
+
+    # Construct parser others can reuse
+    if parser:
+        return parser
+    else:
+        parser = {}
+    # Options _all_ parsers have in common
+    parser['loglevel'] = argparse.ArgumentParser(add_help=False)
+    parser['loglevel'].add_argument(
+            '-l', '--log-level', metavar='LOGLEVEL',
+            type=functools.partial(check_lower_bounded_int, lower_bound=-1,
+                                   name="log level"),
+            help=('Set the specified verbosity level. '
+                  'The levels, in order from the lowest to the highest, are: '
+                  'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) '
+                  'TRACE (4 or higher). If used along with -v then -v '
+                  'increases last set value and -l overwrites last set '
+                  'value.'),
+            action='store', dest='verbose', required=False)
+    parser['loglevel'].add_argument(
+            '-q', '--quiet',
+            help='Quiet mode: disables logging, including WARNING and ERROR.',
+            action='store_true', default=False)
+    parser['loglevel'].add_argument(
+            '-v', '--verbose',
+            help=('Increase the verbosity level. Every instance of -v '
+                  'increments the verbosity level by one. Its default value '
+                  'is 0 which includes ERROR and WARNING levels. '
+                  'The levels, in order from the lowest to the highest, are: '
+                  'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) '
+                  'TRACE (4 or higher). If used along with -l then -l '
+                  'overwrites last set value and -v increases last set '
+                  'value.'),
+            action='count', default=None)
+
+    parser['beta'] = argparse.ArgumentParser(add_help=False)
+    parser['beta'].add_argument(
+           '-b', '--beta',
+           help=('Enable beta functionality. '),
+           action='store_true', dest='beta', default=None)
+
+    # Main subcommand parser
+    parser['main'] = argparse.ArgumentParser(
+            description='cdist ' + cdist.VERSION)
+    parser['main'].add_argument(
+            '-V', '--version', help='Show version.', action='version',
+            version='%(prog)s ' + cdist.VERSION)
+    parser['sub'] = parser['main'].add_subparsers(
+            title="Commands", dest="command")
+
+    # Banner
+    parser['banner'] = parser['sub'].add_parser(
+            'banner', parents=[parser['loglevel']])
+    parser['banner'].set_defaults(func=cdist.banner.banner)
+
+    parser['inventory_common'] = argparse.ArgumentParser(add_help=False)
+    parser['inventory_common'].add_argument(
+           '-I', '--inventory',
+           help=('Use specified custom inventory directory. '
+                 'Inventory directory is set up by the following rules: '
+                 'if cdist configuration resolves this value then specified '
+                 'directory is used, '
+                 'if HOME env var is set then ~/.cdist/inventory is '
+                 'used, otherwise distribution inventory directory is used.'),
+           dest="inventory_dir", required=False)
+
+    parser['common'] = argparse.ArgumentParser(add_help=False)
+    parser['common'].add_argument(
+           '-g', '--config-file',
+           help=('Use specified custom configuration file.'),
+           dest="config_file", required=False)
+
+    # Config
+    parser['config_main'] = argparse.ArgumentParser(add_help=False)
+    parser['config_main'].add_argument(
+           '-4', '--force-ipv4',
+           help=('Force to use IPv4 addresses only. No influence for custom'
+                 ' remote commands.'),
+           action='store_const', dest='force_ipv', const=4)
+    parser['config_main'].add_argument(
+           '-6', '--force-ipv6',
+           help=('Force to use IPv6 addresses only. No influence for custom'
+                 ' remote commands.'),
+           action='store_const', dest='force_ipv', const=6)
+    parser['config_main'].add_argument(
+            '-C', '--cache-path-pattern',
+            help=('Specify custom cache path pattern. If '
+                  'it is not set then default hostdir is used.'),
+            dest='cache_path_pattern',
+            default=None)
+    parser['config_main'].add_argument(
+            '-c', '--conf-dir',
+            help=('Add configuration directory (can be repeated, '
+                  'last one wins).'), action='append')
+    parser['config_main'].add_argument(
+           '-i', '--initial-manifest',
+           help='Path to a cdist manifest or \'-\' to read from stdin.',
+           dest='manifest', required=False)
+    parser['config_main'].add_argument(
+           '-j', '--jobs', nargs='?',
+           type=functools.partial(check_lower_bounded_int, lower_bound=1,
+                                  name="positive int"),
+           help=('Operate in parallel in specified maximum number of jobs. '
+                 'Global explorers, object prepare and object run are '
+                 'supported. Without argument CPU count is used by default. '),
+           action='store', dest='jobs',
+           const=multiprocessing.cpu_count())
+    parser['config_main'].add_argument(
+           '-n', '--dry-run',
+           help='Do not execute code.', action='store_true')
+    parser['config_main'].add_argument(
+           '-o', '--out-dir',
+           help='Directory to save cdist output in.', dest="out_path")
+    parser['config_main'].add_argument(
+           '-P', '--timestamp',
+           help=('Timestamp log messages with the current local date and time '
+                 'in the format: YYYYMMDDHHMMSS.us.'),
+           action='store_true', dest='timestamp')
+    parser['config_main'].add_argument(
+           '-R', '--use-archiving', nargs='?',
+           choices=('tar', 'tgz', 'tbz2', 'txz',),
+           help=('Operate by using archiving with compression where '
+                 'appropriate. Supported values are: tar - tar archive, '
+                 'tgz - gzip tar archive (the default), '
+                 'tbz2 - bzip2 tar archive and txz - lzma tar archive. '
+                 'Currently in beta.'),
+           action='store', dest='use_archiving',
+           const='tgz')
+
+    # remote-copy and remote-exec defaults are environment variables
+    # if set; if not then None - these will be futher handled after
+    # parsing to determine implementation default
+    parser['config_main'].add_argument(
+           '-r', '--remote-out-dir',
+           help='Directory to save cdist output in on the target host.',
+           dest="remote_out_path")
+    parser['config_main'].add_argument(
+           '--remote-copy',
+           help='Command to use for remote copy (should behave like scp).',
+           action='store', dest='remote_copy',
+           default=None)
+    parser['config_main'].add_argument(
+           '--remote-exec',
+           help=('Command to use for remote execution '
+                 '(should behave like ssh).'),
+           action='store', dest='remote_exec',
+           default=None)
+    parser['config_main'].add_argument(
+           '-S', '--disable-saving-output-streams',
+           help='Disable saving output streams.',
+           action='store_false', dest='save_output_streams', default=True)
+
+    # Config
+    parser['config_args'] = argparse.ArgumentParser(add_help=False)
+    parser['config_args'].add_argument(
+             '-A', '--all-tagged',
+             help=('Use all hosts present in tags db. Currently in beta.'),
+             action="store_true", dest="all_tagged_hosts", default=False)
+    parser['config_args'].add_argument(
+             '-a', '--all',
+             help=('List hosts that have all specified tags, '
+                   'if -t/--tag is specified.'),
+             action="store_true", dest="has_all_tags", default=False)
+    parser['config_args'].add_argument(
+            '-f', '--file',
+            help=('Read specified file for a list of additional hosts to '
+                  'operate on or if \'-\' is given, read stdin (one host per '
+                  'line). If no host or host file is specified then, by '
+                  'default, read hosts from stdin.'),
+            dest='hostfile', required=False)
+    parser['config_args'].add_argument(
+           '-p', '--parallel', nargs='?', metavar='HOST_MAX',
+           type=functools.partial(check_lower_bounded_int, lower_bound=1,
+                                  name="positive int"),
+           help=('Operate on multiple hosts in parallel for specified maximum '
+                 'hosts at a time. Without argument CPU count is used by '
+                 'default.'),
+           action='store', dest='parallel',
+           const=multiprocessing.cpu_count())
+    parser['config_args'].add_argument(
+           '-s', '--sequential',
+           help='Operate on multiple hosts sequentially (default).',
+           action='store_const', dest='parallel', const=0)
+    parser['config_args'].add_argument(
+             '-t', '--tag',
+             help=('Host is specified by tag, not hostname/address; '
+                   'list all hosts that contain any of specified tags. '
+                   'Currently in beta.'),
+             dest='tag', required=False, action="store_true", default=False)
+    parser['config_args'].add_argument(
+            'host', nargs='*', help='Host(s) to operate on.')
+    parser['config'] = parser['sub'].add_parser(
+            'config', parents=[parser['loglevel'], parser['beta'],
+                               parser['common'],
+                               parser['config_main'],
+                               parser['inventory_common'],
+                               parser['config_args']])
+    parser['config'].set_defaults(func=cdist.config.Config.commandline)
+
+    # Install
+    parser['install'] = parser['sub'].add_parser('install', add_help=False,
+                                                 parents=[parser['config']])
+    parser['install'].set_defaults(func=cdist.install.Install.commandline)
+
+    # Inventory
+    parser['inventory'] = parser['sub'].add_parser('inventory')
+    parser['invsub'] = parser['inventory'].add_subparsers(
+            title="Inventory commands", dest="subcommand")
+
+    parser['add-host'] = parser['invsub'].add_parser(
+            'add-host', parents=[parser['loglevel'], parser['beta'],
+                                 parser['common'],
+                                 parser['inventory_common']])
+    parser['add-host'].add_argument(
+            'host', nargs='*', help='Host(s) to add.')
+    parser['add-host'].add_argument(
+           '-f', '--file',
+           help=('Read additional hosts to add from specified file '
+                 'or from stdin if \'-\' (each host on separate line). '
+                 'If no host or host file is specified then, by default, '
+                 'read from stdin.'),
+           dest='hostfile', required=False)
+
+    parser['add-tag'] = parser['invsub'].add_parser(
+            'add-tag', parents=[parser['loglevel'], parser['beta'],
+                                parser['common'],
+                                parser['inventory_common']])
+    parser['add-tag'].add_argument(
+           'host', nargs='*',
+           help='List of host(s) for which tags are added.')
+    parser['add-tag'].add_argument(
+           '-f', '--file',
+           help=('Read additional hosts to add tags from specified file '
+                 'or from stdin if \'-\' (each host on separate line). '
+                 'If no host or host file is specified then, by default, '
+                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
+                 ' are specified then tags are read from stdin and are'
+                 ' added to all hosts.'),
+           dest='hostfile', required=False)
+    parser['add-tag'].add_argument(
+           '-T', '--tag-file',
+           help=('Read additional tags to add from specified file '
+                 'or from stdin if \'-\' (each tag on separate line). '
+                 'If no tag or tag file is specified then, by default, '
+                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
+                 ' are specified then tags are read from stdin and are'
+                 ' added to all hosts.'),
+           dest='tagfile', required=False)
+    parser['add-tag'].add_argument(
+           '-t', '--taglist',
+           help=("Tag list to be added for specified host(s), comma separated"
+                 " values."),
+           dest="taglist", required=False)
+
+    parser['del-host'] = parser['invsub'].add_parser(
+            'del-host', parents=[parser['loglevel'], parser['beta'],
+                                 parser['common'],
+                                 parser['inventory_common']])
+    parser['del-host'].add_argument(
+            'host', nargs='*', help='Host(s) to delete.')
+    parser['del-host'].add_argument(
+            '-a', '--all', help=('Delete all hosts.'),
+            dest='all', required=False, action="store_true", default=False)
+    parser['del-host'].add_argument(
+            '-f', '--file',
+            help=('Read additional hosts to delete from specified file '
+                  'or from stdin if \'-\' (each host on separate line). '
+                  'If no host or host file is specified then, by default, '
+                  'read from stdin.'),
+            dest='hostfile', required=False)
+
+    parser['del-tag'] = parser['invsub'].add_parser(
+            'del-tag', parents=[parser['loglevel'], parser['beta'],
+                                parser['common'],
+                                parser['inventory_common']])
+    parser['del-tag'].add_argument(
+            'host', nargs='*',
+            help='List of host(s) for which tags are deleted.')
+    parser['del-tag'].add_argument(
+            '-a', '--all',
+            help=('Delete all tags for specified host(s).'),
+            dest='all', required=False, action="store_true", default=False)
+    parser['del-tag'].add_argument(
+            '-f', '--file',
+            help=('Read additional hosts to delete tags for from specified '
+                  'file or from stdin if \'-\' (each host on separate line). '
+                  'If no host or host file is specified then, by default, '
+                  'read from stdin. If no tags/tagfile nor hosts/hostfile'
+                  ' are specified then tags are read from stdin and are'
+                  ' deleted from all hosts.'),
+            dest='hostfile', required=False)
+    parser['del-tag'].add_argument(
+            '-T', '--tag-file',
+            help=('Read additional tags from specified file '
+                  'or from stdin if \'-\' (each tag on separate line). '
+                  'If no tag or tag file is specified then, by default, '
+                  'read from stdin. If no tags/tagfile nor'
+                  ' hosts/hostfile are specified then tags are read from'
+                  ' stdin and are added to all hosts.'),
+            dest='tagfile', required=False)
+    parser['del-tag'].add_argument(
+            '-t', '--taglist',
+            help=("Tag list to be deleted for specified host(s), "
+                  "comma separated values."),
+            dest="taglist", required=False)
+
+    parser['list'] = parser['invsub'].add_parser(
+            'list', parents=[parser['loglevel'], parser['beta'],
+                             parser['common'],
+                             parser['inventory_common']])
+    parser['list'].add_argument(
+            'host', nargs='*', help='Host(s) to list.')
+    parser['list'].add_argument(
+            '-a', '--all',
+            help=('List hosts that have all specified tags, '
+                  'if -t/--tag is specified.'),
+            action="store_true", dest="has_all_tags", default=False)
+    parser['list'].add_argument(
+            '-f', '--file',
+            help=('Read additional hosts to list from specified file '
+                  'or from stdin if \'-\' (each host on separate line). '
+                  'If no host or host file is specified then, by default, '
+                  'list all.'), dest='hostfile', required=False)
+    parser['list'].add_argument(
+            '-H', '--host-only', help=('Suppress tags listing.'),
+            action="store_true", dest="list_only_host", default=False)
+    parser['list'].add_argument(
+            '-t', '--tag',
+            help=('Host is specified by tag, not hostname/address; '
+                  'list all hosts that contain any of specified tags.'),
+            action="store_true", default=False)
+
+    parser['inventory'].set_defaults(
+            func=cdist.inventory.Inventory.commandline)
+
+    # PreOs
+    parser['preos'] = parser['sub'].add_parser('preos', add_help=False)
+
+    # Shell
+    parser['shell'] = parser['sub'].add_parser(
+            'shell', parents=[parser['loglevel']])
+    parser['shell'].add_argument(
+            '-s', '--shell',
+            help=('Select shell to use, defaults to current shell. Used shell'
+                  ' should be POSIX compatible shell.'))
+    parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
+
+    for p in parser:
+        parser[p].epilog = EPILOG
+
+    return parser
+
+
+def handle_loglevel(args):
+    if hasattr(args, 'quiet') and args.quiet:
+        args.verbose = _verbosity_level_off
+
+    logging.root.setLevel(_verbosity_level[args.verbose])
+
+
+def parse_and_configure(argv, singleton=True):
+    parser = get_parsers()
+    parser_args = parser['main'].parse_args(argv)
+    try:
+        cfg = cdist.configuration.Configuration(parser_args,
+                                                singleton=singleton)
+        args = cfg.get_args()
+    except ValueError as e:
+        raise cdist.Error(str(e))
+    # Loglevels are handled globally in here
+    handle_loglevel(args)
+
+    log = logging.getLogger("cdist")
+
+    log.verbose("version %s" % cdist.VERSION)
+    log.trace('command line args: {}'.format(cfg.command_line_args))
+    log.trace('configuration: {}'.format(cfg.get_config()))
+    log.trace('configured args: {}'.format(args))
+
+    check_beta(vars(args))
+
+    return parser, cfg

cdist/autil.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+#
+# 2017 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+
+import cdist
+import tarfile
+import os
+import glob
+import tempfile
+
+
+_ARCHIVING_MODES = {
+    'tar': '',
+    'tgz': 'gz',
+    'tbz2': 'bz2',
+    'txz': 'xz',
+}
+
+
+_UNARCHIVE_OPT = {
+    'tar': None,
+    'tgz': '-z',
+    'tbz2': '-j',
+    'txz': '-J',
+}
+
+
+# Archiving will be enabled if directory contains more than FILES_LIMIT files.
+FILES_LIMIT = 1
+
+
+def get_extract_option(mode):
+    return _UNARCHIVE_OPT[mode]
+
+
+def tar(source, mode="tgz"):
+    if mode not in _ARCHIVING_MODES:
+        raise cdist.Error("Unsupported archiving mode {}.".format(mode))
+
+    files = glob.glob1(source, '*')
+    fcnt = len(files)
+    if fcnt <= FILES_LIMIT:
+        return None, fcnt
+
+    tarmode = 'w:{}'.format(_ARCHIVING_MODES[mode])
+    _, tarpath = tempfile.mkstemp(suffix='.' + mode)
+    with tarfile.open(tarpath, tarmode, dereference=True) as tar:
+        if os.path.isdir(source):
+            for f in files:
+                tar.add(os.path.join(source, f), arcname=f)
+        else:
+            tar.add(source)
+    return tarpath, fcnt

cdist/banner.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -20,8 +20,6 @@
 #
 
 import logging
-import sys
-
 import cdist
 
 log = logging.getLogger(__name__)

cdist/conf/explorer/cpu_cores

@@ -0,0 +1,44 @@
+#!/bin/sh
+#
+# 2014 Daniel Heule  (hda at sfs.biz)
+# 2014 Thomas Oettli (otho at sfs.biz)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+# FIXME: other system types (not linux ...)
+
+os=$("$__explorer/os")
+case "$os" in
+    "macosx")
+        sysctl -n hw.physicalcpu
+    ;;
+
+    "openbsd")
+        sysctl -n hw.ncpuonline
+    ;;
+
+    *)
+        if [ -r /proc/cpuinfo ]; then
+            cores="$(grep "core id" /proc/cpuinfo | sort | uniq | wc -l)"
+            if [ "${cores}" -eq 0 ]; then
+                cores="1"
+            fi
+            echo "$cores"
+        fi
+    ;;
+esac

cdist/conf/explorer/cpu_sockets

@@ -0,0 +1,40 @@
+#!/bin/sh
+#
+# 2014 Daniel Heule  (hda at sfs.biz)
+# 2014 Thomas Oettli (otho at sfs.biz)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+# FIXME: other system types (not linux ...)
+
+os=$("$__explorer/os")
+case "$os" in
+    "macosx")
+        system_profiler SPHardwareDataType | grep "Number of Processors" | awk -F': ' '{print $2}'
+    ;;
+
+    *)
+    if [ -r /proc/cpuinfo ]; then
+        sockets="$(grep "physical id" /proc/cpuinfo | sort -u | wc -l)"
+        if [ "${sockets}" -eq 0 ]; then
+            sockets="$(grep -c "processor" /proc/cpuinfo)"
+        fi
+        echo "${sockets}"
+    fi
+    ;;
+esac

cdist/conf/explorer/disks

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+uname_s="$(uname -s)"
+
+case "${uname_s}" in
+    FreeBSD)
+        sysctl -n kern.disks
+    ;;
+    OpenBSD|NetBSD)
+        sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+' | xargs
+    ;;
+    Linux)
+        if command -v lsblk > /dev/null
+        then
+            # exclude ram disks, floppies and cdroms
+            # https://www.kernel.org/doc/Documentation/admin-guide/devices.txt
+            lsblk -e 1,2,11 -dno name | xargs
+        else
+            printf "Don't know how to list disks for %s operating system without lsblk, if you can please submit a patch\n" "${uname_s}" >&2
+        fi
+    ;;
+    *)
+        printf "Don't know how to list disks for %s operating system, if you can please submit a patch\n" "${uname_s}" >&2
+    ;;
+esac
+
+exit 0

cdist/conf/explorer/hostname

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# 2012 Evax Software <contact@evax.fr>
+# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,10 +17,13 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+#
+# Retrieve the running hostname
+#
 
-user="$__object_id"
-if su - $user -c "[ -d \"\$HOME/.rvm\" ]" ; then
-   echo "installed"
+if command -v hostname >/dev/null
+then
+	hostname
 else
-   echo "removed"
+	uname -n
 fi

cdist/conf/explorer/init

@@ -0,0 +1,39 @@
+#!/bin/sh
+#
+# 2016 Daniel Heule (hda at sfs.biz)
+# Copyright 2017, Philippe Gregoire <pg@pgregoire.xyz>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Returns the process name of pid 1 ( normaly the init system )
+# for example at linux this value is "init" or "systemd" in most cases
+#
+
+uname_s="$(uname -s)"
+
+case "$uname_s" in
+    Linux)
+        (pgrep -P0 -l | awk '/^1[ \t]/ {print $2;}') || true
+    ;;
+    FreeBSD|OpenBSD)
+        ps -o comm= -p 1 || true
+    ;;
+    *)
+        # return a empty string as unknown value
+        echo ""
+    ;;
+esac

cdist/conf/explorer/interfaces

@@ -0,0 +1,28 @@
+#!/bin/sh -e
+#
+# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+if command -v ip >/dev/null
+then
+	ip -o link show | sed -n 's/^[0-9]\+: \(.\+\): <.*/\1/p'
+elif command -v ifconfig >/dev/null
+then
+	ifconfig -a | sed -n -E 's/^(.*)(:[[:space:]]*flags=|Link encap).*/\1/p'
+fi \
+ | sort -u

cdist/conf/explorer/is-freebsd-jail

@@ -0,0 +1,2 @@
+#!/bin/sh
+sysctl -n security.jail.jailed 2>/dev/null | grep "1" || true

cdist/conf/explorer/kernel_name

@@ -0,0 +1,2 @@
+#!/bin/sh
+uname -s

cdist/conf/explorer/lsb_codename

@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+set +e
+case "$("$__explorer/os")" in
+   openwrt)
+      # shellcheck disable=SC1091
+      (. /etc/openwrt_release && echo "$DISTRIB_CODENAME")
+   ;;
+   *)
+      lsb_release=$(command -v lsb_release)
+      if [ -x "$lsb_release" ]; then
+         $lsb_release --short --codename
+      fi
+   ;;
+esac

cdist/conf/explorer/lsb_description

@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+set +e
+case "$("$__explorer/os")" in
+   openwrt)
+      # shellcheck disable=SC1091
+      (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION")
+   ;;
+   *)
+      lsb_release=$(command -v lsb_release)
+      if [ -x "$lsb_release" ]; then
+         $lsb_release --short --description
+      fi
+   ;;
+esac

cdist/conf/explorer/lsb_id

@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+set +e
+case "$("$__explorer/os")" in
+   openwrt)
+      # shellcheck disable=SC1091
+      (. /etc/openwrt_release && echo "$DISTRIB_ID")
+   ;;
+   *)
+      lsb_release=$(command -v lsb_release)
+      if [ -x "$lsb_release" ]; then
+         $lsb_release --short --id
+      fi
+   ;;
+esac

cdist/conf/explorer/lsb_release

@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+set +e
+case "$("$__explorer/os")" in
+   openwrt)
+      # shellcheck disable=SC1091
+      (. /etc/openwrt_release && echo "$DISTRIB_RELEASE")
+   ;;
+   *)
+      lsb_release=$(command -v lsb_release)
+      if [ -x "$lsb_release" ]; then
+         $lsb_release --short --release
+      fi
+   ;;
+esac

cdist/conf/explorer/machine

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2010-2011 Andi Brönnimann (andi-cdist at v-net.ch)
 #
 # This file is part of cdist.
 #
@@ -18,10 +18,10 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Get an existing groups gshadow entry.
+# All os variables are lower case
+#
 #
 
-name=$__object_id
-
-getent gshadow "$name" || true
-
+if command -v uname >/dev/null 2>&1 ; then
+   uname -m
+fi

cdist/conf/explorer/machine_type

@@ -0,0 +1,80 @@
+#!/bin/sh
+#
+# 2014 Daniel Heule  (hda at sfs.biz)
+# 2014 Thomas Oettli (otho at sfs.biz)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+# FIXME: other system types (not linux ...)
+
+if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then
+    echo openvz
+    exit
+fi
+
+if [ -e "/proc/1/environ" ] &&
+    tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then
+    echo lxc
+    exit
+fi
+
+if [ -r /proc/cpuinfo ]; then
+    # this should only exist on virtual guest machines,
+    # tested on vmware, xen, kvm
+    if grep -q "hypervisor" /proc/cpuinfo; then
+        # this file is aviable in xen guest systems
+        if [ -r /sys/hypervisor/type ]; then
+            if grep -q -i "xen" /sys/hypervisor/type; then
+                echo virtual_by_xen
+                exit 
+            fi
+        else
+            if [ -r /sys/class/dmi/id/product_name ]; then
+                if grep -q -i 'vmware' /sys/class/dmi/id/product_name; then
+                    echo "virtual_by_vmware"
+                    exit
+                elif grep -q -i 'bochs' /sys/class/dmi/id/product_name; then
+                    echo "virtual_by_kvm"
+                    exit 
+                elif grep -q -i 'virtualbox' /sys/class/dmi/id/product_name; then
+                    echo "virtual_by_virtualbox"
+                    exit 
+                fi
+            fi
+
+            if [ -r /sys/class/dmi/id/sys_vendor ]; then
+                if grep -q -i 'qemu' /sys/class/dmi/id/sys_vendor; then
+                    echo "virtual_by_kvm"
+                    exit
+                fi
+            fi
+
+            if [ -r /sys/class/dmi/id/chassis_vendor ]; then
+                if grep -q -i 'qemu' /sys/class/dmi/id/chassis_vendor; then
+                    echo "virtual_by_kvm"
+                    exit
+                fi
+            fi                        
+        fi
+        echo "virtual_by_unknown"
+    else
+        echo "physical"
+    fi
+else
+    echo "unknown"
+fi

cdist/conf/explorer/memory

@@ -0,0 +1,41 @@
+#!/bin/sh
+#
+# 2014 Daniel Heule  (hda at sfs.biz)
+# 2014 Thomas Oettli (otho at sfs.biz)
+# Copyright 2017, Philippe Gregoire <pg@pgregoire.xyz>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+# FIXME: other system types (not linux ...)
+
+os=$("$__explorer/os")
+case "$os" in
+    "macosx")
+        echo "$(sysctl -n hw.memsize)/1024" | bc
+    ;;
+
+    "openbsd")
+        echo "$(sysctl -n hw.physmem) / 1048576" | bc
+    ;;
+
+    *)
+    if [ -r /proc/meminfo ]; then
+        grep "MemTotal:" /proc/meminfo | awk '{print $2}'
+    fi
+    ;;
+esac

cdist/conf/explorer/os

@@ -0,0 +1,153 @@
+#!/bin/sh
+#
+# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+# Copyright 2017, Philippe Gregoire <pg@pgregoire.xyz>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# All os variables are lower case.  Keep this file in alphabetical
+# order by os variable except in cases where order otherwise matters,
+# in which case keep the primary os and its derivatives together in
+# a block (see Debian and Redhat examples below).
+#
+
+if grep -q ^Amazon /etc/system-release 2>/dev/null; then
+   echo amazon
+   exit 0
+fi
+
+if [ -f /etc/arch-release ]; then
+   echo archlinux
+   exit 0
+fi
+
+if [ -f /etc/cdist-preos ]; then
+   echo cdist-preos
+   exit 0
+fi
+
+if [ -d /gnu/store ]; then
+   echo guixsd
+   exit 0
+fi
+
+### Debian and derivatives
+if grep -q ^DISTRIB_ID=Ubuntu /etc/lsb-release 2>/dev/null; then
+   echo ubuntu
+   exit 0
+fi
+
+# devuan ascii has both devuan_version and debian_version, so we need to check devuan_version first!
+if [ -f /etc/devuan_version ]; then
+   echo devuan
+   exit 0
+fi
+
+if [ -f /etc/debian_version ]; then
+   echo debian
+   exit 0
+fi
+
+###
+
+if [ -f /etc/gentoo-release ]; then
+   echo gentoo
+   exit 0
+fi
+
+if [ -f /etc/openwrt_version ]; then
+    echo openwrt
+    exit 0
+fi
+
+if [ -f /etc/owl-release ]; then
+   echo owl
+   exit 0
+fi
+
+### Redhat and derivatives
+if grep -q ^Scientific /etc/redhat-release 2>/dev/null; then
+    echo scientific
+    exit 0
+fi
+
+if grep -q ^CentOS /etc/redhat-release 2>/dev/null; then
+    echo centos
+    exit 0
+fi
+
+if grep -q ^Fedora /etc/redhat-release 2>/dev/null; then
+   echo fedora
+   exit 0
+fi
+
+if grep -q ^Mitel /etc/redhat-release 2>/dev/null; then
+   echo mitel
+   exit 0
+fi
+
+if [ -f /etc/redhat-release ]; then
+   echo redhat
+   exit 0
+fi
+###
+
+if [ -f /etc/SuSE-release ]; then
+   echo suse
+   exit 0
+fi
+
+if [ -f /etc/slackware-version ]; then
+   echo slackware
+   exit 0
+fi
+
+uname_s="$(uname -s)"
+
+# Assume there is no tr on the client -> do lower case ourselves
+case "$uname_s" in
+   Darwin)
+      echo macosx
+      exit 0
+   ;;
+   NetBSD)
+      echo netbsd
+      exit 0
+   ;;
+   FreeBSD)
+      echo freebsd
+      exit 0
+   ;;
+   OpenBSD)
+      echo openbsd
+      exit 0
+   ;;
+   SunOS)
+      echo solaris
+      exit 0
+   ;;
+esac
+
+if [ -f /etc/os-release ]; then
+   # already lowercase, according to:
+   # https://www.freedesktop.org/software/systemd/man/os-release.html
+   awk -F= '/^ID=/ { if ($2 ~ /^'"'"'(.*)'"'"'$/ || $2 ~ /^"(.*)"$/) { print substr($2, 2, length($2) - 2) } else { print $2 } }' /etc/os-release
+   exit 0
+fi
+
+echo "Unknown OS" >&2
+exit 1

cdist/conf/explorer/os_release

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2018 Adam Dej (dejko.a at gmail.com)
 #
 # This file is part of cdist.
 #
@@ -19,8 +19,8 @@
 #
 #
 
+# See os-release(5) and http://0pointer.de/blog/projects/os-release
+
 set +e
-lsb_release=$(which lsb_release 2>/dev/null)
-if [ -x "$lsb_release" ]; then
-   $lsb_release --short --codename
-fi
+
+cat /etc/os-release || cat /usr/lib/os-release || true

cdist/conf/explorer/os_version

@@ -0,0 +1,73 @@
+#!/bin/sh
+#
+# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# All os variables are lower case
+#
+#
+
+case "$("$__explorer/os")" in
+   amazon)
+      cat /etc/system-release
+   ;;
+   archlinux)
+      # empty, but well...
+      cat /etc/arch-release
+   ;;
+   debian)
+      cat /etc/debian_version
+   ;;
+   devuan)
+      cat /etc/devuan_version
+   ;;
+   fedora)
+      cat /etc/fedora-release
+   ;;
+   gentoo)
+      cat /etc/gentoo-release
+   ;;
+   macosx)
+      sw_vers -productVersion
+   ;;
+   *bsd|solaris)
+      uname -r
+   ;;
+   openwrt)
+      cat /etc/openwrt_version
+   ;;
+   owl)
+      cat /etc/owl-release
+   ;;
+   redhat|centos|mitel|scientific)
+      cat /etc/redhat-release
+   ;;
+   slackware)
+      cat /etc/slackware-version
+   ;;
+   suse)
+      if [ -f /etc/os-release ]; then
+        cat /etc/os-release
+      else
+        cat /etc/SuSE-release
+      fi
+   ;;
+   ubuntu)
+      lsb_release -sr
+   ;;
+esac

cdist/conf/explorer/runlevel

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2012 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -20,7 +20,7 @@
 #
 
 set +e
-lsb_release=$(which lsb_release 2>/dev/null)
-if [ -x "$lsb_release" ]; then
-   $lsb_release --short --id
+executable=$(command -v runlevel)
+if [ -x "$executable" ]; then
+   "$executable" | awk '{ print $2 }'
 fi

cdist/conf/manifest/sample-from-distribution

@@ -0,0 +1,62 @@
+## #
+## # Sample manifest from cdist distribution
+## #
+## 
+## # Every machine becomes a marker, so sysadmins know that automatic
+## # configurations are happening
+## __file /etc/cdist-configured
+## __cdistmarker
+## 
+## case "$__target_host" in
+##    # Everybody has this
+##    localhost)
+##       require="__file/etc/cdist-configured" __link /tmp/cdist-testfile \
+##          --source /etc/cdist-configured  --type symbolic
+##       require="__directory/tmp/cdist-test-dir" __file /tmp/cdist-test-dir/test-file \
+##          --mode 0750 --owner nobody --group root
+##       __directory /tmp/cdist-test-dir --mode 4777
+## 
+##       require="__file/etc/cdist-configured __link/tmp/cdist-testfile" \
+##          __file /tmp/cdist-another-testfile
+## 
+##    ;;
+## 
+##    #
+##    # Use an alias in /etc/hosts for localhost to use these hosts:
+##    # 
+##    # 127.0.0.1    localhost.localdomain   localhost cdist-archlinux
+##    #
+##    cdist-archlinux)
+##       # This is the specific package type for pacman
+##       __package_pacman zsh --state installed
+##       
+##       # The __package type autoselect the right type based on the os
+##       __package vim --state installed
+## 
+##       # If the type is a singleton, it does not take an object id
+##       __issue
+##    ;;
+##    # This is how it would look like on gentoo
+##    cdist-gentoo)
+##       # Same stuff for gentoo
+##       __package tree --state installed
+##    ;;
+## 
+##    cdist-debian)
+##       __package_apt atop --state installed
+##       __package apache2 --state removed
+##    ;;
+## 
+##    cdist-redhat)
+##       __issue
+##       __motd
+##    ;;
+## 
+##    # Real machines may be used with their hostname or fqdn,
+##    # depending on how you call cdist
+##    # ...
+##    # ;;
+##    # machine.example.org)
+##    # ...
+##    # ;;
+## esac

cdist/conf/type/__acl/explorer/acl_is

@@ -0,0 +1,31 @@
+#!/bin/sh -e
+#
+# 2018 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+[ ! -e "/$__object_id" ] && exit 0
+
+if ! command -v getfacl > /dev/null
+then
+    echo 'getfacl not available' >&2
+    exit 1
+fi
+
+getfacl "/$__object_id" 2>/dev/null \
+    | grep -Eo '^(default:)?(user|group|(mask|other):):[^:][[:graph:]]+' \
+    || true

cdist/conf/type/__acl/explorer/checks

@@ -0,0 +1,39 @@
+#!/bin/sh -e
+#
+# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# TODO check if filesystem has ACL turned on etc
+
+if [ -f "$__object/parameter/acl" ]
+then
+    grep -E '^(default:)?(user|group):' "$__object/parameter/acl" \
+    | while read -r acl
+    do
+        param="$( echo "$acl" | awk -F: '{print $(NF-2)}' )"
+        check="$( echo "$acl" | awk -F: '{print $(NF-1)}' )"
+
+        [ "$param" = 'user' ] && db=passwd || db="$param"
+
+        if ! getent "$db" "$check" > /dev/null
+        then
+            echo "missing $param '$check'" >&2
+            exit 1
+        fi
+    done
+fi

cdist/conf/type/__acl/explorer/file_is

@@ -0,0 +1,31 @@
+#!/bin/sh -e
+#
+# 2018 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+if [ -e "/$__object_id" ]
+then
+    if [ -d "/$__object_id" ]
+    then echo directory
+    elif [ -f "/$__object_id" ]
+    then echo regular
+    else echo other
+    fi
+else
+    echo missing
+fi

cdist/conf/type/__acl/gencode-remote

@@ -0,0 +1,126 @@
+#!/bin/sh -e
+#
+# 2018 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+file_is="$( cat "$__object/explorer/file_is" )"
+
+[ "$file_is" = 'missing' ] && [ -z "$__cdist_dry_run" ] && exit 0
+
+os="$( cat "$__global/explorer/os" )"
+
+acl_path="/$__object_id"
+
+acl_is="$( cat "$__object/explorer/acl_is" )"
+
+if [ -f "$__object/parameter/acl" ]
+then
+    acl_should="$( cat "$__object/parameter/acl" )"
+elif
+    [ -f "$__object/parameter/user" ] \
+        || [ -f "$__object/parameter/group" ] \
+        || [ -f "$__object/parameter/mask" ] \
+        || [ -f "$__object/parameter/other" ]
+then
+    acl_should="$( for param in user group mask other
+    do
+        [ ! -f "$__object/parameter/$param" ] && continue
+
+        echo "$param" | grep -Eq 'mask|other' && sep=:: || sep=:
+
+        echo "$param$sep$( cat "$__object/parameter/$param" )"
+    done )"
+else
+    echo 'no parameters set' >&2
+    exit 1
+fi
+
+if [ -f "$__object/parameter/default" ]
+then
+    acl_should="$( echo "$acl_should" \
+        | sed 's/^default://' \
+        | sort -u \
+        | sed 's/\(.*\)/default:\1\n\1/' )"
+fi
+
+if [ "$file_is" = 'regular' ] \
+    && echo "$acl_should" | grep -Eq '^default:'
+then
+    # only directories can have default ACLs,
+    # but instead of error,
+    # let's just remove default entries
+    acl_should="$( echo "$acl_should" | grep -Ev '^default:' )"
+fi
+
+if echo "$acl_should" | awk -F: '{ print $NF }' | grep -Fq 'X'
+then
+    [ "$file_is" = 'directory' ] && rep=x || rep=-
+
+    acl_should="$( echo "$acl_should" | sed "s/\\(.*\\)X/\\1$rep/" )"
+fi
+
+setfacl_exec='setfacl'
+
+if [ -f "$__object/parameter/recursive" ]
+then
+    if echo "$os" | grep -Fq 'freebsd'
+    then
+        echo "$os setfacl do not support recursive operations" >&2
+    else
+        setfacl_exec="$setfacl_exec -R"
+    fi
+fi
+
+if [ -f "$__object/parameter/remove" ]
+then
+    echo "$acl_is" | while read -r acl
+    do
+        # skip wanted ACL entries which already exist
+        # and skip mask and other entries, because we
+        # can't actually remove them, but only change.
+        if echo "$acl_should" | grep -Eq "^$acl" \
+            || echo "$acl" | grep -Eq '^(default:)?(mask|other)'
+        then continue
+        fi
+
+        if echo "$os" | grep -Fq 'freebsd'
+        then
+            remove="$acl"
+        else
+            remove="$( echo "$acl" | sed 's/:...$//' )"
+        fi
+
+        echo "$setfacl_exec -x \"$remove\" \"$acl_path\""
+        echo "removed '$remove'" >> "$__messages_out"
+    done
+fi
+
+for acl in $acl_should
+do
+    if ! echo "$acl_is" | grep -Eq "^$acl"
+    then
+        if echo "$os" | grep -Fq 'freebsd' \
+            && echo "$acl" | grep -Eq '^default:'
+        then
+            echo "setting default ACL in $os is currently not supported" >&2
+        else
+            echo "$setfacl_exec -m \"$acl\" \"$acl_path\""
+            echo "added '$acl'" >> "$__messages_out"
+        fi
+    fi
+done

cdist/conf/type/__acl/man.rst

@@ -0,0 +1,85 @@
+cdist-type__acl(7)
+==================
+
+NAME
+----
+cdist-type__acl - Set ACL entries
+
+
+DESCRIPTION
+-----------
+Fully supported and tested on Linux (ext4 filesystem), partial support for FreeBSD.
+
+See ``setfacl`` and ``acl`` manpages for more details.
+
+
+REQUIRED MULTIPLE PARAMETERS
+----------------------------
+acl
+   Set ACL entry following ``getfacl`` output syntax.
+
+
+BOOLEAN PARAMETERS
+------------------
+default
+   Set all ACL entries as default too.
+   Only directories can have default ACLs.
+   Setting default ACL in FreeBSD is currently not supported.
+
+recursive
+   Make ``setfacl`` recursive (Linux only), but not ``getfacl`` in explorer.
+
+remove
+   Remove undefined ACL entries.
+   ``mask`` and ``other`` entries can't be removed, but only changed.
+
+
+DEPRECATED PARAMETERS
+---------------------
+Parameters ``user``, ``group``, ``mask`` and ``other`` are deprecated and they
+will be removed in future versions. Please use ``acl`` parameter instead.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __acl /srv/project \
+        --default \
+        --recursive \
+        --remove \
+        --acl user:alice:rwx \
+        --acl user:bob:r-x \
+        --acl group:project-group:rwx \
+        --acl group:some-other-group:r-x \
+        --acl mask::r-x \
+        --acl other::r-x
+
+    # give Alice read-only access to subdir,
+    # but don't allow her to see parent content.
+
+    __acl /srv/project2 \
+        --remove \
+        --acl default:group:secret-project:rwx \
+        --acl group:secret-project:rwx \
+        --acl user:alice:--x
+
+    __acl /srv/project2/subdir \
+        --default \
+        --remove \
+        --acl group:secret-project:rwx \
+        --acl user:alice:r-x
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2018 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__acl/parameter/boolean

@@ -0,0 +1,3 @@
+recursive
+default
+remove

cdist/conf/type/__acl/parameter/deprecated/group

@@ -0,0 +1 @@
+see manual for details

cdist/conf/type/__acl/parameter/deprecated/mask

@@ -0,0 +1 @@
+see manual for details

cdist/conf/type/__acl/parameter/deprecated/other

@@ -0,0 +1 @@
+see manual for details

cdist/conf/type/__acl/parameter/deprecated/user

@@ -0,0 +1 @@
+see manual for details

cdist/conf/type/__acl/parameter/optional

@@ -0,0 +1,2 @@
+mask
+other

cdist/conf/type/__acl/parameter/optional_multiple

@@ -0,0 +1,3 @@
+acl
+user
+group

cdist/conf/type/__apt_default_release/man.rst

@@ -0,0 +1,46 @@
+cdist-type__apt_default_release(7)
+==================================
+
+NAME
+----
+cdist-type__apt_default_release - Configure the default release for apt
+
+
+DESCRIPTION
+-----------
+Configure the default release for apt, using the APT::Default-Release
+configuration value.
+
+REQUIRED PARAMETERS
+-------------------
+release
+   The value to set APT::Default-Release to.
+
+   This can contain release name, codename or release version. Examples:
+   'stable', 'testing', 'unstable', 'stretch', 'buster', '4.0', '5.0*'.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __apt_default_release --release stretch
+
+
+AUTHORS
+-------
+Matthijs Kooijman <matthijs--@--stdin.nl>
+
+
+COPYING
+-------
+Copyright \(C) 2017 Matthijs Kooijman. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__apt_default_release/manifest

@@ -0,0 +1,41 @@
+#!/bin/sh -e
+#
+# 2014 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2017 Matthijs Kooijman (matthijs at stdin.nl)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+os=$(cat "$__global/explorer/os")
+release="$(cat "$__object/parameter/release")"
+
+case "$os" in
+   ubuntu|debian|devuan)
+      __file /etc/apt/apt.conf.d/99-default-release \
+         --owner root --group root --mode 644 \
+         --source - << DONE
+APT::Default-Release "$release";
+DONE
+   ;;
+   *)
+      cat >&2 << DONE
+The developer of this type (${__type##*/}) did not think your operating system
+($os) would have any use for it. If you think otherwise please submit a patch.
+DONE
+      exit 1
+   ;;
+esac

cdist/conf/type/__apt_default_release/parameter/required

@@ -0,0 +1 @@
+release

cdist/conf/type/__apt_key/explorer/state

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2011-2014 Steven Armstrong (steven-cdist at armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -17,22 +17,28 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+#
+# Get the current state of the apt key.
+#
 
-device="$(cat "$__object/parameter/device")"
-type="$(cat "$__object/parameter/type")"
-
-if [ "$type" = "swap" ]; then
-   echo "mkswap $device"
+if [ -f "$__object/parameter/keyid" ]; then
+   keyid="$(cat "$__object/parameter/keyid")"
 else
-   command="mkfs -t $type -q"
-   if [ -f "$__object/parameter/options" ]; then
-      options="$(cat "$__object/parameter/options")"
-      command="$command $options"
-   fi
-   command="$command $device"
-   if [ -f "$__object/parameter/blocks" ]; then
-      blocks="$(cat "$__object/parameter/blocks")"
-      command="$command $blocks"
-   fi
-   echo "$command"
+   keyid="$__object_id"
+fi
+
+keydir="$(cat "$__object/parameter/keydir")"
+keyfile="$keydir/$__object_id.gpg"
+
+if [ -d "$keydir" ]
+then
+   if [ -f "$keyfile" ]
+   then echo present
+   else echo absent
+   fi
+else
+   # fallback to deprecated apt-key
+   apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \
+      && echo present \
+      || echo absent
 fi

cdist/conf/type/__apt_key/gencode-remote

@@ -0,0 +1,114 @@
+#!/bin/sh -e
+#
+# 2011-2014 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+if [ -f "$__object/parameter/keyid" ]; then
+   keyid="$(cat "$__object/parameter/keyid")"
+else
+   keyid="$__object_id"
+fi
+state_should="$(cat "$__object/parameter/state")"
+state_is="$(cat "$__object/explorer/state")"
+
+if [ "$state_should" = "$state_is" ]; then
+   # nothing to do
+   exit 0
+fi
+
+keydir="$(cat "$__object/parameter/keydir")"
+keyfile="$keydir/$__object_id.gpg"
+
+case "$state_should" in
+   present)
+      keyserver="$(cat "$__object/parameter/keyserver")"
+
+      if [ -f "$__object/parameter/uri" ]; then
+         uri="$(cat "$__object/parameter/uri")"
+
+         if [ -d "$keydir" ]; then
+            cat << EOF
+
+curl -s -L \\
+    -o "$keyfile" \\
+    "$uri"
+
+key="\$( cat "$keyfile" )"
+
+if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK'
+then
+    echo "\$key" | gpg --dearmor > "$keyfile"
+fi
+
+EOF
+         else
+            # fallback to deprecated apt-key
+            echo "curl -s -L '$uri' | apt-key add -"
+         fi
+      elif [ -d "$keydir" ]; then
+         tmp='/tmp/cdist_apt_key_tmp'
+
+         # we need to kill gpg after 30 seconds, because gpg
+         # can get stuck if keyserver is not responding.
+         # exporting env var and not exit 1,
+         # because we need to clean up and kill dirmngr.
+         cat << EOF
+
+mkdir -m 700 -p "$tmp"
+
+if timeout 30s \\
+    gpg --homedir "$tmp" \\
+        --keyserver "$keyserver" \\
+        --recv-keys "$keyid"
+then
+    gpg --homedir "$tmp" \\
+        --export "$keyid" \\
+        > "$keyfile"
+else
+    export GPG_GOT_STUCK=1
+fi
+
+GNUPGHOME="$tmp" gpgconf --kill dirmngr
+
+rm -rf "$tmp"
+
+if [ -n "\$GPG_GOT_STUCK" ]
+then
+    echo "GPG GOT STUCK - no response from keyserver after 30 seconds" >&2
+    exit 1
+fi
+
+EOF
+      else
+         # fallback to deprecated apt-key
+         echo "apt-key adv --keyserver \"$keyserver\" --recv-keys \"$keyid\""
+      fi
+
+      echo "added '$keyid'" >> "$__messages_out"
+   ;;
+   absent)
+      if [ -f "$keyfile" ]; then
+         echo "rm '$keyfile'"
+      else
+         # fallback to deprecated apt-key
+         echo "apt-key del \"$keyid\""
+      fi
+
+      echo "removed '$keyid'" >> "$__messages_out"
+   ;;
+esac

cdist/conf/type/__apt_key/man.rst

@@ -0,0 +1,72 @@
+cdist-type__apt_key(7)
+======================
+
+NAME
+----
+cdist-type__apt_key - Manage the list of keys used by apt
+
+
+DESCRIPTION
+-----------
+Manages the list of keys used by apt to authenticate packages.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+   'present' or 'absent'. Defaults to 'present'
+
+keyid
+   the id of the key to add. Defaults to __object_id
+
+keyserver
+   the keyserver from which to fetch the key. If omitted the default set
+   in ./parameter/default/keyserver is used.
+
+keydir
+   key save location, defaults to ``/etc/apt/trusted.pgp.d``
+
+uri
+   the URI from which to download the key
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Add Ubuntu Archive Automatic Signing Key
+    __apt_key 437D05B5
+    # Same thing
+    __apt_key 437D05B5 --state present
+    # Get rid of it
+    __apt_key 437D05B5 --state absent
+
+    # same thing with human readable name and explicit keyid
+    __apt_key UbuntuArchiveKey --keyid 437D05B5
+
+    # same thing with other keyserver
+    __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com
+
+    # download key from the internet
+    __apt_key rabbitmq \
+       --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc
+
+
+AUTHORS
+-------
+Steven Armstrong <steven-cdist--@--armstrong.cc>
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can
+redistribute it and/or modify it under the terms of the GNU General Public
+License as published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__apt_key/manifest

@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+__package gnupg
+
+if [ -f "$__object/parameter/uri" ]
+then __package curl
+else __package dirmngr
+fi

cdist/conf/type/__apt_key/parameter/default/keydir

@@ -0,0 +1 @@
+/etc/apt/trusted.gpg.d

cdist/conf/type/__apt_key/parameter/default/keyserver

@@ -0,0 +1 @@
+pool.sks-keyservers.net

cdist/conf/type/__apt_key/parameter/default/state

@@ -0,0 +1 @@
+present

cdist/conf/type/__apt_key/parameter/optional

@@ -0,0 +1,5 @@
+state
+keyid
+keyserver
+keydir
+uri

cdist/conf/type/__apt_key_uri/explorer/state

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011-2014 Steven Armstrong (steven-cdist at armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Retrieve the status of a package - parsed dpkg output
+# Get the current state of the apt key.
 #
 
 if [ -f "$__object/parameter/name" ]; then
@@ -27,4 +27,6 @@ else
    name="$__object_id"
 fi
 
-rpm -q --whatprovides "$name" 2>/dev/null || true
+apt-key list 2> /dev/null | grep -Fqe "$name" \
+   && echo present \
+   || echo absent

cdist/conf/type/__apt_key_uri/gencode-remote

@@ -1,6 +1,6 @@
-#!/bin/sh
+#!/bin/sh -e
 #
-# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011-2014 Steven Armstrong (steven-cdist at armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -17,30 +17,14 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# Mostly a wrapper for ln
-#
 
-destination="/$__object_id"
-
-type="$(cat "$__object/parameter/type")"
-source="$(cat "$__object/parameter/source")"
-
-case "$type" in
-   symbolic)
-      lnopt="-s"
-   ;;
-   hard)
-      lnopt=""
-   ;;
-   *)
-      echo "Unknown type: $type" >&2
-      exit 1
-   ;;
-esac
-
-state_is="$(cat "$__object/explorer/state")"
+if [ -f "$__object/parameter/name" ]; then
+   name="$(cat "$__object/parameter/name")"
+else
+   name="$__object_id"
+fi
 state_should="$(cat "$__object/parameter/state")"
+state_is="$(cat "$__object/explorer/state")"
 
 if [ "$state_should" = "$state_is" ]; then
    # nothing to do
@@ -49,13 +33,13 @@ fi
 
 case "$state_should" in
    present)
-      echo ln ${lnopt} -f \"$source\" \"$destination\"
+      uri="$(cat "$__object/parameter/uri")"
+      printf 'curl -s -L "%s" | apt-key add -\n' "$uri"
    ;;
    absent)
-      echo rm -f \"$destination\"
-   ;;
-   *)
-      echo "Unknown state: $state_should" >&2
-      exit 1
+      cat << DONE
+keyid=\$(apt-key list | grep -B1 "$name" | awk '/pub/ { print \$2 }' | cut -d'/' -f 2)
+apt-key del \$keyid
+DONE
    ;;
 esac

cdist/conf/type/__apt_key_uri/man.rst

@@ -0,0 +1,51 @@
+cdist-type__apt_key_uri(7)
+==========================
+
+NAME
+----
+cdist-type__apt_key_uri - Add apt key from uri
+
+
+DESCRIPTION
+-----------
+Download a key from an uri and add it to the apt keyring.
+
+
+REQUIRED PARAMETERS
+-------------------
+uri
+   the uri from which to download the key
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+   'present' or 'absent', defaults to 'present'
+
+name
+   a name for this key, used when testing if it is already installed.
+   Defaults to __object_id
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __apt_key_uri rabbitmq \
+       --name 'RabbitMQ Release Signing Key <info@rabbitmq.com>' \
+       --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc \
+       --state present
+
+
+AUTHORS
+-------
+Steven Armstrong <steven-cdist--@--armstrong.cc>
+
+
+COPYING
+-------
+Copyright \(C) 2011-2014 Steven Armstrong. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__apt_key_uri/manifest

@@ -0,0 +1,21 @@
+#!/bin/sh -e
+#
+# 2013-2014 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+__package curl

cdist/conf/type/__apt_key_uri/parameter/default/state

@@ -0,0 +1 @@
+present

cdist/conf/type/__apt_key_uri/parameter/optional

@@ -1,2 +1,2 @@
 state
-user
+name

cdist/conf/type/__apt_key_uri/parameter/required

@@ -0,0 +1 @@
+uri

cdist/conf/type/__apt_mark/explorer/apt_version

@@ -0,0 +1,31 @@
+#!/bin/sh -e
+#
+# 2016 Ander Punnar (cdist at kvlt.ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+apt_version_is=$(dpkg-query --show --showformat '${Version}' apt)
+
+# from APT changelog:
+#   apt (0.8.14.2) UNRELEASED; urgency=low
+#   provide a 'dpkg --set-selections' wrapper to set/release holds
+
+apt_version_should=0.8.14.2
+
+dpkg --compare-versions "$apt_version_should" le "$apt_version_is" \
+    && echo 0 \
+    || echo 1

cdist/conf/type/__apt_mark/explorer/package_installed

@@ -0,0 +1,30 @@
+#!/bin/sh -e
+#
+# 2016 Ander Punnar (cdist at kvlt.ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+if [ -f "$__object/parameter/name" ]; then
+    name="$(cat "$__object/parameter/name")"
+else
+    name="$__object_id"
+fi
+
+dpkg-query --show --showformat '${Status}' "$name" 2>/dev/null \
+    | grep -Fq 'ok installed' \
+    && echo 0 \
+    || echo 1

cdist/conf/type/__apt_mark/explorer/state

@@ -0,0 +1,27 @@
+#!/bin/sh -e
+#
+# 2016 Ander Punnar (cdist at kvlt.ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+if [ -f "$__object/parameter/name" ]; then
+    name="$(cat "$__object/parameter/name")"
+else
+    name="$__object_id"
+fi
+
+apt-mark showhold | grep -Fq "$name" && echo hold || echo unhold

cdist/conf/type/__apt_mark/gencode-remote

@@ -0,0 +1,56 @@
+#!/bin/sh -e
+#
+# 2016 Ander Punnar (cdist at kvlt.ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+if [ -f "$__object/parameter/name" ]; then
+    name="$(cat "$__object/parameter/name")"
+else
+    name="$__object_id"
+fi
+
+apt_version="$(cat "$__object/explorer/apt_version")"
+
+if [ "$apt_version" != '0' ]; then
+    echo 'APT version not supported' >&2
+    exit 1
+fi
+
+package_installed="$(cat "$__object/explorer/package_installed")"
+
+if [ "$package_installed" != '0' ]; then
+    exit 0
+fi
+
+state_should="$(cat "$__object/parameter/state")"
+
+state_is="$(cat "$__object/explorer/state")"
+
+if [ "$state_should" = "$state_is" ]; then
+    exit 0
+fi
+
+case "$state_should" in
+    hold|unhold)
+        echo "apt-mark $state_should $name > /dev/null"
+    ;;
+    *)
+        echo "Unknown state: $state_should" >&2
+        exit 1
+    ;;
+esac

cdist/conf/type/__apt_mark/man.rst

@@ -0,0 +1,47 @@
+cdist-type__apt_mark(7)
+=======================
+
+NAME
+----
+cdist-type__apt_mark - set package state as 'hold' or 'unhold'
+
+
+DESCRIPTION
+-----------
+See apt-mark(8) for details.
+
+
+REQUIRED PARAMETERS
+-------------------
+state
+   Either "hold" or "unhold".
+
+
+OPTIONAL PARAMETERS
+-------------------
+name
+   If supplied, use the name and not the object id as the package name.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # hold package
+    __apt_mark quagga --state hold
+    # unhold package
+    __apt_mark quagga --state unhold
+
+
+AUTHORS
+-------
+Ander Punnar <cdist--@--kvlt.ee>
+
+
+COPYING
+-------
+Copyright \(C) 2016 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__apt_norecommends/man.rst

@@ -0,0 +1,42 @@
+cdist-type__apt_norecommends(7)
+===============================
+
+NAME
+----
+cdist-type__apt_norecommends - Configure apt to not install recommended packages
+
+
+DESCRIPTION
+-----------
+Configure apt to not install any recommended or suggested packages.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __apt_norecommends
+
+
+AUTHORS
+-------
+Steven Armstrong <steven-cdist--@--armstrong.cc>
+
+
+COPYING
+-------
+Copyright \(C) 2014 Steven Armstrong. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__apt_norecommends/manifest

@@ -0,0 +1,44 @@
+#!/bin/sh -e
+#
+# 2014 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+os=$(cat "$__global/explorer/os")
+
+case "$os" in
+   ubuntu|debian|devuan)
+      # No stinking recommends thank you very much.
+      # If I want something installed I will do so myself.
+      __file /etc/apt/apt.conf.d/99-no-recommends \
+         --owner root --group root --mode 644 \
+         --source - << DONE
+APT::Install-Recommends "0";
+APT::Install-Suggests "0";
+APT::AutoRemove::RecommendsImportant "0";
+APT::AutoRemove::SuggestsImportant "0";
+DONE
+   ;;
+   *)
+      cat >&2 << DONE
+The developer of this type (${__type##*/}) did not think your operating system
+($os) would have any use for it. If you think otherwise please submit a patch.
+DONE
+      exit 1
+   ;;
+esac

cdist/conf/type/__apt_ppa/explorer/state

@@ -23,11 +23,12 @@
 
 name="$__object_id"
 
+# shellcheck disable=SC1091
 . /etc/lsb-release
 
 repo_name="${name#ppa:}"
-repo_file_name="$(echo "$repo_name" | sed "s:\/:\-:")-${DISTRIB_CODENAME}.list"
+repo_file_name="$(echo "$repo_name" | sed -e 's|[/:]|-|' -e 's|\.|_|')-${DISTRIB_CODENAME}.list"
 
 [ -s "/etc/apt/sources.list.d/${repo_file_name}" ] \
-   && echo enabled || echo disabled
+   && echo present || echo absent
 

cdist/conf/type/__apt_ppa/gencode-remote

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -22,14 +22,16 @@ name="$__object_id"
 state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
 
-if [ "$state_should" != "$state_is" ]; then
-   case "$state_should" in
-      enabled)
-         echo add-apt-repository \"$name\"
-      ;;
-      disabled)
-         echo remove-apt-repository \"$name\"
-      ;;
-   esac
+if [ "$state_should" = "$state_is" ]; then
+   # Nothing to do, move along
+   exit 0
 fi
 
+case "$state_should" in
+   present)
+      echo "add-apt-repository '$name'"
+   ;;
+   absent)
+      echo "remove-apt-repository '$name'"
+   ;;
+esac

cdist/conf/type/__apt_ppa/man.rst

@@ -0,0 +1,50 @@
+cdist-type__apt_ppa(7)
+======================
+
+NAME
+----
+cdist-type__apt_ppa - Manage ppa repositories
+
+
+DESCRIPTION
+-----------
+This cdist type allows manage ubuntu ppa repositories.
+
+
+REQUIRED PARAMETERS
+-------------------
+state
+   The state the ppa should be in, either 'present' or 'absent'.
+   Defaults to 'present'
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Enable a ppa repository
+    __apt_ppa ppa:sans-intern/missing-bits
+    # same as
+    __apt_ppa ppa:sans-intern/missing-bits --state present
+
+    # Disable a ppa repository
+    __apt_ppa ppa:sans-intern/missing-bits --state absent
+
+
+AUTHORS
+-------
+Steven Armstrong <steven-cdist--@--armstrong.cc>
+
+
+COPYING
+-------
+Copyright \(C) 2011-2014 Steven Armstrong. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__apt_ppa/manifest

@@ -1,6 +1,6 @@
-#!/bin/sh
+#!/bin/sh -e
 #
-# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2011-2016 Steven Armstrong (steven-cdist at armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -18,11 +18,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-name="$__object_id"
+__package software-properties-common
 
-__package python-software-properties --state installed
-
-require="__package/python-software-properties" \
+require="__package/software-properties-common" \
    __file /usr/local/bin/remove-apt-repository \
    --source "$__type/files/remove-apt-repository" \
    --mode 0755

cdist/conf/type/__apt_ppa/parameter/default/state

@@ -0,0 +1 @@
+present

cdist/conf/type/__apt_source/files/source.list.template

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -u
+
+entry="$uri $distribution $component"
+cat << DONE
+# Created by cdist ${__type##*/}
+# Do not change. Changes will be overwritten.
+#
+
+# $name
+deb ${forcedarch} $entry
+DONE
+if [ -f "$__object/parameter/include-src" ]; then
+   echo "deb-src $entry"
+fi

cdist/conf/type/__apt_source/gencode-remote

@@ -1,6 +1,6 @@
-#!/bin/sh
+#!/bin/sh -e
 #
-# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2018 Steven Armstrong (steven-cdist at armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -17,11 +17,12 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+#
 
 name="$__object_id"
+destination="/etc/apt/sources.list.d/${name}.list"
 
-if su - postgres -c "echo '\q' | psql '$name'" 2>/dev/null; then
-   echo "present"
-else
-   echo "absent"
+if grep -q "^__file${destination}" "$__messages_in"; then
+   printf 'apt-get update || apt-get update\n'
 fi
+

cdist/conf/type/__apt_source/man.rst

@@ -0,0 +1,70 @@
+cdist-type__apt_source(7)
+=========================
+
+NAME
+----
+cdist-type__apt_source - Manage apt sources
+
+
+DESCRIPTION
+-----------
+This cdist type allows you to manage apt sources. It invokes index update
+internally when needed so call of index updating type is not needed.
+
+
+REQUIRED PARAMETERS
+-------------------
+uri
+   the uri to the apt repository
+
+
+OPTIONAL PARAMETERS
+-------------------
+arch
+   set this if you need to force and specific arch (ubuntu specific)
+
+state
+   'present' or 'absent', defaults to 'present'
+
+distribution
+   the distribution codename to use. Defaults to DISTRIB_CODENAME from
+   the targets /etc/lsb-release
+
+component
+   space delimited list of components to enable. Defaults to an empty string.
+
+
+BOOLEAN PARAMETERS
+------------------
+include-src
+   include deb-src entries
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __apt_source rabbitmq \
+       --uri http://www.rabbitmq.com/debian/ \
+       --distribution testing \
+       --component main \
+       --include-src \
+       --state present
+
+    __apt_source canonical_partner \
+       --uri http://archive.canonical.com/ \
+       --component partner --state present
+
+
+AUTHORS
+-------
+Steven Armstrong <steven-cdist--@--armstrong.cc>
+
+
+COPYING
+-------
+Copyright \(C) 2011-2018 Steven Armstrong. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__apt_source/manifest

@@ -0,0 +1,52 @@
+#!/bin/sh -e
+#
+# 2011-2018 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+name="$__object_id"
+state="$(cat "$__object/parameter/state")"
+uri="$(cat "$__object/parameter/uri")"
+
+if [ -f "$__object/parameter/distribution" ]; then
+   distribution="$(cat "$__object/parameter/distribution")"
+else
+   distribution="$(cat "$__global/explorer/lsb_codename")"
+fi
+
+component="$(cat "$__object/parameter/component")"
+
+if [ -f "$__object/parameter/arch" ]; then
+   forcedarch="[arch=$(cat "$__object/parameter/arch")]"
+else
+   forcedarch=""
+fi
+
+# export variables for use in template
+export name
+export uri
+export distribution
+export component
+export forcedarch
+
+# generate file from template
+mkdir "$__object/files"
+"$__type/files/source.list.template" > "$__object/files/source.list"
+__file "/etc/apt/sources.list.d/${name}.list" \
+   --source "$__object/files/source.list" \
+   --owner root --group root --mode 0644 \
+   --state "$state"

cdist/conf/type/__apt_source/parameter/boolean

@@ -0,0 +1 @@
+include-src

cdist/conf/type/__apt_source/parameter/default/state

@@ -0,0 +1 @@
+present

cdist/conf/type/__apt_source/parameter/optional

@@ -0,0 +1,4 @@
+state
+distribution
+component
+arch

cdist/conf/type/__apt_source/parameter/required

@@ -0,0 +1 @@
+uri

cdist/conf/type/__apt_update_index/gencode-remote

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -20,7 +20,7 @@
 
 # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists
 cat << DONE
-if find /etc/apt -cnewer /var/lib/apt/lists | grep . > /dev/null; then
+if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then
    apt-get update || apt-get update
 fi
 DONE

cdist/conf/type/__apt_update_index/man.rst

@@ -0,0 +1,41 @@
+cdist-type__apt_update_index(7)
+===============================
+
+NAME
+----
+cdist-type__apt_update_index - Update apt's package index
+
+
+DESCRIPTION
+-----------
+This cdist type runs apt-get update whenever any apt sources have changed.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __apt_update_index
+
+
+AUTHORS
+-------
+Steven Armstrong <steven-cdist--@--armstrong.cc>
+
+
+COPYING
+-------
+Copyright \(C) 2011 Steven Armstrong. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__block/explorer/block

@@ -0,0 +1,40 @@
+#!/bin/sh
+#
+# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+# 2014 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+file="$(cat "$__object/parameter/file" 2>/dev/null || echo "/$__object_id")"
+
+# file does not exist, nothing we could do
+[ -f "$file" ] || exit 0
+
+prefix=$(cat "$__object/parameter/prefix" 2>/dev/null || echo "#cdist:__block/$__object_id")
+suffix=$(cat "$__object/parameter/suffix" 2>/dev/null || echo "#/cdist:__block/$__object_id")
+awk -v prefix="^$prefix\$" -v suffix="^$suffix\$" '{
+   if (match($0,prefix)) {
+      triggered=1
+   }
+   if (triggered) {
+      if (match($0,suffix)) {
+            triggered=0
+      }
+      print
+   }
+}' "$file"

cdist/conf/type/__block/gencode-remote

@@ -0,0 +1,89 @@
+#!/bin/sh -e
+#
+# 2013 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# quote function from http://www.etalabs.net/sh_tricks.html
+quote() {
+   printf '%s\n' "$1" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
+}
+
+file="$(cat "$__object/parameter/file" 2>/dev/null || echo "/$__object_id")"
+state_should=$(cat "$__object/parameter/state")
+prefix=$(cat "$__object/parameter/prefix" 2>/dev/null || echo "#cdist:__block/$__object_id")
+suffix=$(cat "$__object/parameter/suffix" 2>/dev/null || echo "#/cdist:__block/$__object_id")
+
+block="$__object/files/block"
+if [ ! -s "$__object/explorer/block" ]; then
+   state_is='absent'
+else
+   state_is=$(diff -q "$block" "$__object/explorer/block" >/dev/null \
+      && echo present \
+      || echo changed
+   )
+fi
+
+state_should="$(cat "$__object/parameter/state")"
+if [ "$state_should" = "$state_is" ]; then
+   # Nothing to do, move along
+   exit 0
+fi
+
+remove_block() {
+   cat << DONE
+tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
+# preserve ownership and permissions of existing file
+if [ -f "$file" ]; then
+   cp -p "$file" "\$tmpfile"
+fi
+awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
+{
+   if (match(\$0,prefix)) {
+      triggered=1
+   }
+   if (triggered) {
+      if (match(\$0,suffix)) {
+         triggered=0
+      }
+   } else {
+      print
+   }
+}' "$file" > "\$tmpfile"
+mv -f "\$tmpfile" "$file"
+DONE
+}
+
+case "$state_should" in
+   present)
+      if [ "$state_is" = "changed" ]; then
+         echo update >> "$__messages_out"
+         remove_block
+      else
+         echo add >> "$__messages_out"
+      fi
+      cat << DONE
+cat >> "$file" << ${__type##*/}_DONE
+$(cat "$block")
+${__type##*/}_DONE
+DONE
+   ;;
+   absent)
+      echo remove >> "$__messages_out"
+      remove_block
+   ;;
+esac

cdist/conf/type/__block/man.rst

@@ -0,0 +1,82 @@
+cdist-type__block(7)
+====================
+
+NAME
+----
+cdist-type__block - Manage blocks of text in files
+
+
+DESCRIPTION
+-----------
+Manage a block of text in an existing file.
+The block is identified using the prefix and suffix parameters.
+Everything between prefix and suffix is considered to be a managed block
+of text.
+
+
+REQUIRED PARAMETERS
+-------------------
+text
+   the text to manage.
+   If text is '-' (dash), take what was written to stdin as the text.
+
+
+OPTIONAL PARAMETERS
+-------------------
+file
+   the file in which to manage the text block.
+   Defaults to object_id.
+
+prefix
+   the prefix to add before the text.
+   Defaults to #cdist:__block/$__object_id
+
+suffix
+   the suffix to add after the text.
+   Defaults to #/cdist:__block/$__object_id
+
+state
+   'present' or 'absent', defaults to 'present'
+
+
+MESSAGES
+--------
+add
+   block was added
+update
+   block was updated/changed
+remove
+   block was removed
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # text from argument
+    __block /path/to/file \
+       --prefix '#start' \
+       --suffix '#end' \
+       --text 'some\nblock of\ntext'
+
+    # text from stdin
+    __block some-id \
+       --file /path/to/file \
+       --text - << DONE
+    here some block
+    of text
+    DONE
+
+
+AUTHORS
+-------
+Steven Armstrong <steven-cdist--@--armstrong.cc>
+
+
+COPYING
+-------
+Copyright \(C) 2013 Steven Armstrong. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.