Implement python types

Makefile

@@ -31,9 +31,9 @@ help:
 	@echo "docs-clean      clean documentation"
 	@echo "clean           clean"
 
-DOCS_SRC_DIR=./docs/src
-SPEECHDIR=./docs/speeches
-TYPEDIR=./cdist/conf/type
+DOCS_SRC_DIR=docs/src
+SPEECHDIR=docs/speeches
+TYPEDIR=cdist/conf/type
 
 SPHINXM=make -C $(DOCS_SRC_DIR) man
 SPHINXH=make -C $(DOCS_SRC_DIR) html

bin/build-helper

@@ -23,8 +23,9 @@
 #
 
 usage() {
-    printf "usage: %s TARGET [TARGET-ARGS...]
+    printf "usage: %s TARGET RUN-AS
     Available targets:
+        print-runas
         changelog-changes
         changelog-version
         check-date
@@ -57,29 +58,64 @@ usage() {
         version
         target-version
         clean
-        distclean\n" "$1"
+        distclean
+    Run as:
+        nico
+        darko - default, if empty string specified\n" "$1"
 }
 
 basename="${0##*/}"
 
-if [ $# -lt 1 ]
+if [ $# -lt 2 ]
 then
     usage "${basename}"
     exit 1
 fi
 
 option=$1; shift
+run_as="$1"; shift
+
+case "$run_as" in
+    nico)
+        from_a=nico.schottelius
+        from_d=ungleich.ch
+        ml_name="Nico Schottelius"
+        ml_sig_name="Nico"
+    ;;
+    darko|'')
+        from_a=darko.poljak
+        from_d=gmail.com
+        ml_name="Darko Poljak"
+        ml_sig_name="Darko"
+        if [ -z "${run_as}" ]
+        then
+            run_as="darko"
+        fi
+    ;;
+    *)
+        printf "Unsupported RUN-AS value: '%s'.\n" "${run_as}" >&2
+        usage "${basename}"
+        exit 1
+    ;;
+esac
 
 SHELLCHECKCMD="shellcheck -s sh -f gcc -x"
 # Skip SC2154 for variables starting with __ since such variables are cdist
 # environment variables.
 SHELLCHECK_SKIP=': __.*is referenced but not assigned.*\[SC2154\]'
 
+to_a="cdist-configuration-management"
+to_d="googlegroups.com"
+
 # Change to checkout directory
 basedir="${0%/*}/../"
 cd "$basedir"
 
 case "$option" in
+    print-runas)
+        printf "run_as: '%s'\n" "$run_as"
+    ;;
+
     changelog-changes)
         if [ "$#" -eq 1 ]; then
             start=$1
@@ -123,7 +159,7 @@ case "$option" in
     ;;
 
     check-unittest)
-        "$0" test
+        "$0" test "${run_as}"
     ;;
 
     ml-release)
@@ -132,10 +168,20 @@ case "$option" in
             exit 1
         fi
 
+        # Send mail only once - lock until new changelog things happened.
+        [ ! -f .lock-ml ] && touch .lock-ml
+        x=$(find 'docs' -name changelog -type f -newer .lock-ml)
+        [ -z "${x}" ] && exit 0
+
         version=$1; shift
 
+        to=${to_a}@${to_d}
+        from=${from_a}@${from_d}
+
         ( 
         cat << eof
+From: ${ml_name} <$from>
+To: cdist mailing list <$to>
 Subject: cdist $version has been released
 
 Hello .*,
@@ -144,11 +190,23 @@ cdist $version has been released with the following changes:
 
 eof
 
-        "$0" changelog-changes "$version"
+        "$0" changelog-changes "${run_as}" "$version"
         cat << eof
 
+Cheers,
+${ml_sig_name}
+
+-- 
+Automatisation at its best level. With cdist.
 eof
         ) > mailinglist.tmp
+
+        if [ "$run_as" = "nico" ]
+        then
+            /usr/sbin/sendmail -f "$from" "$to" < mailinglist.tmp && rm -f mailinglist.tmp
+        fi
+
+        touch .lock-ml
     ;;
 
     archlinux-release)
@@ -167,7 +225,7 @@ eof
 
     pypi-release)
         # Ensure that pypi release has the right version
-        "$0" version
+        "$0" version "${run_as}"
 
         make docs-clean
         make docs
@@ -175,7 +233,7 @@ eof
     ;;
 
     release-git-tag)
-        target_version=$($0 changelog-version)
+        target_version=$($0 changelog-version "${run_as}")
         if git rev-parse --verify "refs/tags/${target_version}" 2>/dev/null; then
             printf "Tag for %s exists, aborting\n" "${target_version}"
             exit 1
@@ -229,7 +287,7 @@ eof
             git archive --prefix="cdist-${tag}/" -o "${archivename}" "${tag}" \
                 || exit 1
             # make sure target version is generated
-            "$0" target-version
+            "$0" target-version "${run_as}"
             tar -x -f "${archivename}" || exit 1
             cp cdist/version.py "cdist-${tag}/cdist/version.py" || exit 1
             tar -c -f "${archivename}" "cdist-${tag}/" || exit 1
@@ -259,7 +317,7 @@ eof
              | sed "${sed_cmd}") || exit 1
 
         # make release
-        changelog=$("$0" changelog-changes "$1" | sed 's/^[[:space:]]*//')
+        changelog=$("$0" changelog-changes "${run_as}" "$1" | sed 's/^[[:space:]]*//')
         release_notes=$(
             printf "%s\n\n%s\n\n**Changelog**\n\n%s\n" \
                 "${response_archive}" "${response_archive_sig}" "${changelog}"
@@ -280,19 +338,19 @@ eof
 
     release)
         set -e
-        target_version=$($0 changelog-version)
-        target_branch=$($0 version-branch)
+        target_version=$($0 changelog-version "${run_as}")
+        target_branch=$($0 version-branch "${run_as}")
 
         printf "Beginning release process for %s\n" "${target_version}"
 
         # First check everything is sane
-        "$0" check-date
-        "$0" check-unittest
-        "$0" check-pycodestyle
-        "$0" check-shellcheck
+        "$0" check-date "${run_as}"
+        "$0" check-unittest "${run_as}"
+        "$0" check-pycodestyle "${run_as}"
+        "$0" check-shellcheck "${run_as}"
 
         # Generate version file to be included in packaging
-        "$0" target-version
+        "$0" target-version "${run_as}"
 
         # Ensure the git status is clean, else abort
         if ! git diff-index --name-only --exit-code HEAD ; then
@@ -327,8 +385,8 @@ eof
         fi
 
         # Verify that after the merge everything works
-        "$0" check-date
-        "$0" check-unittest
+        "$0" check-date "${run_as}"
+        "$0" check-unittest "${run_as}"
 
         # Generate documentation (man and html)
         # First, clean old generated docs
@@ -339,7 +397,7 @@ eof
         # Everything green, let's do the release
 
         # Tag the current commit
-        "$0" release-git-tag
+        "$0" release-git-tag "${run_as}"
 
         # Also merge back the version branch
         if [ "$masterbranch" = yes ]; then
@@ -348,28 +406,37 @@ eof
         fi
 
         # Publish git changes
-        # if you want to have mirror locally then uncomment this and comment below
+        # if you want to have mirror locally then uncomment this support
+        # if [ "$run_as" = "nico" ]
+        # then
         #     git push --mirror
+        # else
+            # if we are not Nico :) then just push, no mirror
             git push
             # push also new branch and set up tracking
             git push -u origin "${target_branch}"
         # fi
 
         # Create and publish package for pypi
-        "$0" pypi-release
+        "$0" pypi-release "${run_as}"
+
+        if [ "$run_as" = "nico" ]
+        then
+            # Archlinux release is based on pypi
+            "$0" archlinux-release "${run_as}"
+        fi
 
         # sign git tag
         printf "Enter upstream repository authentication token: "
         read -r token
-        "$0" sign-git-release "${target_version}" "${token}"
+        "$0" sign-git-release "${run_as}" "${target_version}" "${token}"
 
         # Announce change on ML
-        "$0" ml-release "${target_version}"
+        "$0" ml-release "${run_as}" "${target_version}"
 
         cat << eof
 Manual steps post release:
     - cdist-web
-    - send mail body generated in mailinglist.tmp and inform Dmitry for deb
     - twitter
 eof
     ;;
@@ -409,7 +476,7 @@ eof
     ;;
 
     check-pycodestyle)
-        "$0" pycodestyle
+        "$0" pycodestyle "${run_as}"
         printf "\\nPlease review pycodestyle report.\\n"
         while true
         do
@@ -451,24 +518,24 @@ eof
     ;;
 
     shellcheck-scripts)
-        ${SHELLCHECKCMD} scripts/cdist-dump scripts/cdist-new-type || exit 0
+        ${SHELLCHECKCMD} scripts/cdist-dump || exit 0
     ;;
 
     shellcheck-gencodes)
-        "$0" shellcheck-local-gencodes
-        "$0" shellcheck-remote-gencodes
+        "$0" shellcheck-local-gencodes "${run_as}"
+        "$0" shellcheck-remote-gencodes "${run_as}"
     ;;
 
     shellcheck-types)
-        "$0" shellcheck-type-explorers
-        "$0" shellcheck-manifests
-        "$0" shellcheck-gencodes
+        "$0" shellcheck-type-explorers "${run_as}"
+        "$0" shellcheck-manifests "${run_as}"
+        "$0" shellcheck-gencodes "${run_as}"
     ;;
 
     shellcheck)
-        "$0" shellcheck-global-explorers
-        "$0" shellcheck-types
-        "$0" shellcheck-scripts
+        "$0" shellcheck-global-explorers "${run_as}"
+        "$0" shellcheck-types "${run_as}"
+        "$0" shellcheck-scripts "${run_as}"
     ;;
 
     shellcheck-type-files)
@@ -476,8 +543,8 @@ eof
     ;;
 
     shellcheck-with-files)
-        "$0" shellcheck
-        "$0" shellcheck-type-files
+        "$0" shellcheck "${run_as}"
+        "$0" shellcheck-type-files "${run_as}"
     ;;
 
     shellcheck-build-helper)
@@ -485,7 +552,7 @@ eof
     ;;
 
     check-shellcheck)
-        "$0" shellcheck
+        "$0" shellcheck "${run_as}"
         printf "\\nPlease review shellcheck report.\\n"
         while true
         do
@@ -507,7 +574,7 @@ eof
     ;;
 
     version-branch)
-        "$0" changelog-version | cut -d. -f '1,2'
+        "$0" changelog-version "${run_as}" | cut -d. -f '1,2'
     ;;
 
     version)
@@ -515,7 +582,7 @@ eof
     ;;
 
     target-version)
-        target_version=$($0 changelog-version)
+        target_version=$($0 changelog-version "${run_as}")
         printf "VERSION = \"%s\"\n" "${target_version}" > cdist/version.py
     ;;
 
@@ -538,7 +605,7 @@ eof
     ;;
 
     distclean)
-        "$0" clean
+        "$0" clean "${run_as}"
         rm -f cdist/version.py
     ;;
     *)

cdist/argparse.py

@@ -5,11 +5,12 @@ import logging
 import collections
 import functools
 import cdist.configuration
+import cdist.trigger
 import cdist.preos
 
 
 # set of beta sub-commands
-BETA_COMMANDS = set(('install', 'inventory', ))
+BETA_COMMANDS = set(('install', 'inventory', 'preos', 'trigger', ))
 # set of beta arguments for sub-commands
 BETA_ARGS = {
     'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
@@ -21,7 +22,6 @@ parser = None
 
 _verbosity_level_off = -2
 _verbosity_level = {
-    None: logging.WARNING,
     _verbosity_level_off: logging.OFF,
     -1: logging.ERROR,
     0: logging.WARNING,
@@ -436,6 +436,28 @@ def get_parsers():
                   ' should be POSIX compatible shell.'))
     parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
 
+    # Trigger
+    parser['trigger'] = parser['sub'].add_parser(
+            'trigger', parents=[parser['loglevel'],
+                                parser['beta'],
+                                parser['common'],
+                                parser['config_main']])
+    parser['trigger'].add_argument(
+            '-D', '--directory', action='store', required=False,
+            help=('Where to create local files'))
+    parser['trigger'].add_argument(
+            '-H', '--http-port', action='store', default=3000, required=False,
+            help=('Create trigger listener via http on specified port'))
+    parser['trigger'].add_argument(
+            '--ipv6', default=False,
+            help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'),
+            action='store_true')
+    parser['trigger'].add_argument(
+            '-O', '--source', action='store', required=False,
+            help=('Which file to copy for creation'))
+
+    parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline)
+
     for p in parser:
         parser[p].epilog = EPILOG
 

cdist/conf/explorer/interfaces

@@ -18,11 +18,13 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-if command -v ip >/dev/null
+if command -v ip > /dev/null
 then
-	ip -o link show | sed -n 's/^[0-9]\+: \(.\+\): <.*/\1/p'
-elif command -v ifconfig >/dev/null
+    ip -o link show | sed -n 's/^[0-9]\+: \(.\+\): <.*/\1/p'
+
+elif command -v ifconfig > /dev/null
 then
-	ifconfig -a | sed -n -E 's/^(.*)(:[[:space:]]*flags=|Link encap).*/\1/p'
-fi \
- | sort -u
+    ifconfig -a \
+        | sed -n -E 's/^(.*)(:[[:space:]]*flags=|Link encap).*/\1/p' \
+        | sort -u
+fi

cdist/conf/explorer/os

@@ -145,7 +145,7 @@ esac
 if [ -f /etc/os-release ]; then
    # already lowercase, according to:
    # https://www.freedesktop.org/software/systemd/man/os-release.html
-   awk -F= '/^ID=/ { if ($2 ~ /^'"'"'(.*)'"'"'$/ || $2 ~ /^"(.*)"$/) { print substr($2, 2, length($2) - 2) } else { print $2 } }' /etc/os-release
+   awk -F= '/^ID=/ {print $2;}' /etc/os-release
    exit 0
 fi
 

cdist/conf/type/__acl/explorer/missing_users_groups

@@ -18,22 +18,30 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# TODO check if filesystem has ACL turned on etc
+[ ! -e "/$__object_id" ] && exit 0
 
-if [ -f "$__object/parameter/acl" ]
-then
-    grep -E '^(default:)?(user|group):' "$__object/parameter/acl" \
-    | while read -r acl
+for parameter in user group
+do
+    if [ ! -f "$__object/parameter/$parameter" ]
+    then
+        continue
+    fi
+
+    while read -r acl
     do
-        param="$( echo "$acl" | awk -F: '{print $(NF-2)}' )"
-        check="$( echo "$acl" | awk -F: '{print $(NF-1)}' )"
+        check="$( echo "$acl" | awk -F: '{print $1}' )"
 
-        [ "$param" = 'user' ] && db=passwd || db="$param"
-
-        if ! getent "$db" "$check" > /dev/null
+        if [ "$parameter" = 'user' ]
         then
-            echo "missing $param '$check'" >&2
-            exit 1
+            getent_db=passwd
+        else
+            getent_db="$parameter"
         fi
-    done
-fi
+
+        if ! getent "$getent_db" "$check" > /dev/null
+        then
+            echo "missing $parameter '$check'"
+        fi
+    done \
+        < "$__object/parameter/$parameter"
+done

cdist/conf/type/__acl/gencode-remote

@@ -20,65 +20,59 @@
 
 file_is="$( cat "$__object/explorer/file_is" )"
 
-[ "$file_is" = 'missing' ] && [ -z "$__cdist_dry_run" ] && exit 0
+[ "$file_is" = 'missing' ] && exit 0
 
-os="$( cat "$__global/explorer/os" )"
+missing_users_groups="$( cat "$__object/explorer/missing_users_groups" )"
 
-acl_path="/$__object_id"
-
-acl_is="$( cat "$__object/explorer/acl_is" )"
-
-if [ -f "$__object/parameter/acl" ]
+if [ -n "$missing_users_groups" ]
 then
-    acl_should="$( cat "$__object/parameter/acl" )"
-elif
-    [ -f "$__object/parameter/user" ] \
-        || [ -f "$__object/parameter/group" ] \
-        || [ -f "$__object/parameter/mask" ] \
-        || [ -f "$__object/parameter/other" ]
-then
-    acl_should="$( for param in user group mask other
-    do
-        [ ! -f "$__object/parameter/$param" ] && continue
-
-        echo "$param" | grep -Eq 'mask|other' && sep=:: || sep=:
-
-        echo "$param$sep$( cat "$__object/parameter/$param" )"
-    done )"
-else
-    echo 'no parameters set' >&2
+    echo "$missing_users_groups" >&2
     exit 1
 fi
 
-if [ -f "$__object/parameter/default" ]
+os="$( cat "$__global/explorer/os" )"
+
+acl_is="$( cat "$__object/explorer/acl_is" )"
+
+acl_path="/$__object_id"
+
+if [ -f "$__object/parameter/default" ] && [ "$file_is" = 'directory' ]
 then
-    acl_should="$( echo "$acl_should" \
-        | sed 's/^default://' \
-        | sort -u \
-        | sed 's/\(.*\)/default:\1\n\1/' )"
+    set_default=1
+else
+    set_default=0
 fi
 
-if [ "$file_is" = 'regular' ] \
-    && echo "$acl_should" | grep -Eq '^default:'
-then
-    # only directories can have default ACLs,
-    # but instead of error,
-    # let's just remove default entries
-    acl_should="$( echo "$acl_should" | grep -Ev '^default:' )"
-fi
+acl_should="$( for parameter in user group mask other
+do
+    if [ ! -f "$__object/parameter/$parameter" ]
+    then
+        continue
+    fi
 
-if echo "$acl_should" | awk -F: '{ print $NF }' | grep -Fq 'X'
-then
-    [ "$file_is" = 'directory' ] && rep=x || rep=-
+    while read -r acl
+    do
+        if echo "$acl" | awk -F: '{ print $NF }' | grep -Fq 'X'
+        then
+            [ "$file_is" = 'directory' ] && rep=x || rep=-
 
-    acl_should="$( echo "$acl_should" | sed "s/\\(.*\\)X/\\1$rep/" )"
-fi
+            acl="$( echo "$acl" | sed "s/\(.*\)X/\1$rep/" )"
+        fi
+
+        echo "$parameter" | grep -Eq '(mask|other)' && sep=:: || sep=:
+
+        echo "$parameter$sep$acl"
+
+        [ "$set_default" = '1' ] && echo "default:$parameter$sep$acl"
+    done \
+        < "$__object/parameter/$parameter"
+done )"
 
 setfacl_exec='setfacl'
 
 if [ -f "$__object/parameter/recursive" ]
 then
-    if echo "$os" | grep -Fq 'freebsd'
+    if echo "$os" | grep -Eq 'macosx|freebsd'
     then
         echo "$os setfacl do not support recursive operations" >&2
     else
@@ -88,36 +82,44 @@ fi
 
 if [ -f "$__object/parameter/remove" ]
 then
-    echo "$acl_is" | while read -r acl
-    do
-        # skip wanted ACL entries which already exist
-        # and skip mask and other entries, because we
-        # can't actually remove them, but only change.
-        if echo "$acl_should" | grep -Eq "^$acl" \
-            || echo "$acl" | grep -Eq '^(default:)?(mask|other)'
-        then continue
-        fi
+    if echo "$os" | grep -Fq 'solaris'
+    then
+        # Solaris setfacl behaves differently.
+        # We will not support Solaris for now, because no way to test it.
+        # But adding support should be easy (use -s instead of -m on modify).
+        echo "$os setfacl do not support -x flag for ACL remove" >&2
+    else
+        echo "$acl_is" | while read -r acl
+        do
+            # Skip wanted ACL entries which already exist
+            # and skip mask and other entries, because we
+            # can't actually remove them, but only change.
+            if echo "$acl_should" | grep -Eq "^$acl" \
+                || echo "$acl" | grep -Eq '^(default:)?(mask|other)'
+            then continue
+            fi
 
-        if echo "$os" | grep -Fq 'freebsd'
-        then
-            remove="$acl"
-        else
-            remove="$( echo "$acl" | sed 's/:...$//' )"
-        fi
+            if echo "$os" | grep -Eq 'macosx|freebsd'
+            then
+                remove="$acl"
+            else
+                remove="$( echo "$acl" | sed 's/:...$//' )"
+            fi
 
-        echo "$setfacl_exec -x \"$remove\" \"$acl_path\""
-        echo "removed '$remove'" >> "$__messages_out"
-    done
+            echo "$setfacl_exec -x \"$remove\" \"$acl_path\""
+            echo "removed '$remove'" >> "$__messages_out"
+        done
+    fi
 fi
 
 for acl in $acl_should
 do
     if ! echo "$acl_is" | grep -Eq "^$acl"
     then
-        if echo "$os" | grep -Fq 'freebsd' \
+        if echo "$os" | grep -Eq 'macosx|freebsd' \
             && echo "$acl" | grep -Eq '^default:'
         then
-            echo "setting default ACL in $os is currently not supported" >&2
+            echo "setting default ACL in $os is currently not supported. sorry :(" >&2
         else
             echo "$setfacl_exec -m \"$acl\" \"$acl_path\""
             echo "added '$acl'" >> "$__messages_out"

cdist/conf/type/__acl/man.rst

@@ -8,36 +8,46 @@ cdist-type__acl - Set ACL entries
 
 DESCRIPTION
 -----------
-Fully supported and tested on Linux (ext4 filesystem), partial support for FreeBSD.
+ACL must be defined as 3-symbol combination, using ``r``, ``w``, ``x`` and ``-``.
+
+Fully supported on Linux (tested on Debian and CentOS).
+
+Partial support for FreeBSD, OSX and Solaris.
+
+OpenBSD and NetBSD support is not possible.
 
 See ``setfacl`` and ``acl`` manpages for more details.
 
 
-REQUIRED MULTIPLE PARAMETERS
+OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
-acl
-   Set ACL entry following ``getfacl`` output syntax.
+user
+   Add user ACL entry.
+
+group
+   Add group ACL entry.
+
+
+OPTIONAL PARAMETERS
+-------------------
+mask
+   Add mask ACL entry.
+
+other
+   Add other ACL entry.
 
 
 BOOLEAN PARAMETERS
 ------------------
-default
-   Set all ACL entries as default too.
-   Only directories can have default ACLs.
-   Setting default ACL in FreeBSD is currently not supported.
-
 recursive
    Make ``setfacl`` recursive (Linux only), but not ``getfacl`` in explorer.
 
+default
+   Add default ACL entries (FreeBSD not supported).
+
 remove
-   Remove undefined ACL entries.
-   ``mask`` and ``other`` entries can't be removed, but only changed.
-
-
-DEPRECATED PARAMETERS
----------------------
-Parameters ``user``, ``group``, ``mask`` and ``other`` are deprecated and they
-will be removed in future versions. Please use ``acl`` parameter instead.
+   Remove undefined ACL entries (Solaris not supported).
+   ACL entries for ``mask`` and ``other`` can't be removed.
 
 
 EXAMPLES
@@ -46,30 +56,15 @@ EXAMPLES
 .. code-block:: sh
 
     __acl /srv/project \
-        --default \
         --recursive \
-        --remove \
-        --acl user:alice:rwx \
-        --acl user:bob:r-x \
-        --acl group:project-group:rwx \
-        --acl group:some-other-group:r-x \
-        --acl mask::r-x \
-        --acl other::r-x
-
-    # give Alice read-only access to subdir,
-    # but don't allow her to see parent content.
-
-    __acl /srv/project2 \
-        --remove \
-        --acl default:group:secret-project:rwx \
-        --acl group:secret-project:rwx \
-        --acl user:alice:--x
-
-    __acl /srv/project2/subdir \
         --default \
         --remove \
-        --acl group:secret-project:rwx \
-        --acl user:alice:r-x
+        --user alice:rwx \
+        --user bob:r-x \
+        --group project-group:rwx \
+        --group some-other-group:r-x \
+        --mask r-x \
+        --other r-x
 
 
 AUTHORS

cdist/conf/type/__acl/parameter/deprecated/group

@@ -1 +0,0 @@
-see manual for details

cdist/conf/type/__acl/parameter/deprecated/mask

@@ -1 +0,0 @@
-see manual for details

cdist/conf/type/__acl/parameter/deprecated/other

@@ -1 +0,0 @@
-see manual for details

cdist/conf/type/__acl/parameter/deprecated/user

@@ -1 +0,0 @@
-see manual for details

cdist/conf/type/__acl/parameter/optional_multiple

@@ -1,3 +1,2 @@
-acl
 user
 group

cdist/conf/type/__apt_key/explorer/state

@@ -27,18 +27,6 @@ else
    keyid="$__object_id"
 fi
 
-keydir="$(cat "$__object/parameter/keydir")"
-keyfile="$keydir/$__object_id.gpg"
-
-if [ -d "$keydir" ]
-then
-   if [ -f "$keyfile" ]
-   then echo present
-   else echo absent
-   fi
-else
-   # fallback to deprecated apt-key
-   apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \
-      && echo present \
-      || echo absent
-fi
+apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \
+   && echo present \
+   || echo absent

cdist/conf/type/__apt_key/gencode-remote

@@ -31,84 +31,12 @@ if [ "$state_should" = "$state_is" ]; then
    exit 0
 fi
 
-keydir="$(cat "$__object/parameter/keydir")"
-keyfile="$keydir/$__object_id.gpg"
-
 case "$state_should" in
    present)
       keyserver="$(cat "$__object/parameter/keyserver")"
-
-      if [ -f "$__object/parameter/uri" ]; then
-         uri="$(cat "$__object/parameter/uri")"
-
-         if [ -d "$keydir" ]; then
-            cat << EOF
-
-curl -s -L \\
-    -o "$keyfile" \\
-    "$uri"
-
-key="\$( cat "$keyfile" )"
-
-if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK'
-then
-    echo "\$key" | gpg --dearmor > "$keyfile"
-fi
-
-EOF
-         else
-            # fallback to deprecated apt-key
-            echo "curl -s -L '$uri' | apt-key add -"
-         fi
-      elif [ -d "$keydir" ]; then
-         tmp='/tmp/cdist_apt_key_tmp'
-
-         # we need to kill gpg after 30 seconds, because gpg
-         # can get stuck if keyserver is not responding.
-         # exporting env var and not exit 1,
-         # because we need to clean up and kill dirmngr.
-         cat << EOF
-
-mkdir -m 700 -p "$tmp"
-
-if timeout 30s \\
-    gpg --homedir "$tmp" \\
-        --keyserver "$keyserver" \\
-        --recv-keys "$keyid"
-then
-    gpg --homedir "$tmp" \\
-        --export "$keyid" \\
-        > "$keyfile"
-else
-    export GPG_GOT_STUCK=1
-fi
-
-GNUPGHOME="$tmp" gpgconf --kill dirmngr
-
-rm -rf "$tmp"
-
-if [ -n "\$GPG_GOT_STUCK" ]
-then
-    echo "GPG GOT STUCK - no response from keyserver after 30 seconds" >&2
-    exit 1
-fi
-
-EOF
-      else
-         # fallback to deprecated apt-key
-         echo "apt-key adv --keyserver \"$keyserver\" --recv-keys \"$keyid\""
-      fi
-
-      echo "added '$keyid'" >> "$__messages_out"
+      echo "apt-key adv --keyserver \"$keyserver\" --recv-keys \"$keyid\""
    ;;
    absent)
-      if [ -f "$keyfile" ]; then
-         echo "rm '$keyfile'"
-      else
-         # fallback to deprecated apt-key
-         echo "apt-key del \"$keyid\""
-      fi
-
-      echo "removed '$keyid'" >> "$__messages_out"
+      echo "apt-key del \"$keyid\""
    ;;
 esac

cdist/conf/type/__apt_key/man.rst

@@ -28,12 +28,6 @@ keyserver
    the keyserver from which to fetch the key. If omitted the default set
    in ./parameter/default/keyserver is used.
 
-keydir
-   key save location, defaults to ``/etc/apt/trusted.pgp.d``
-
-uri
-   the URI from which to download the key
-
 
 EXAMPLES
 --------
@@ -53,20 +47,15 @@ EXAMPLES
     # same thing with other keyserver
     __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com
 
-    # download key from the internet
-    __apt_key rabbitmq \
-       --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc
-
 
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
-Ander Punnar <ander-at-kvlt-dot-ee>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can
-redistribute it and/or modify it under the terms of the GNU General Public
-License as published by the Free Software Foundation, either version 3 of the
+Copyright \(C) 2011-2014 Steven Armstrong. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.

cdist/conf/type/__apt_key/manifest

@@ -1,8 +0,0 @@
-#!/bin/sh -e
-
-__package gnupg
-
-if [ -f "$__object/parameter/uri" ]
-then __package curl
-else __package dirmngr
-fi

cdist/conf/type/__apt_key/parameter/default/keydir

@@ -1 +0,0 @@
-/etc/apt/trusted.gpg.d

cdist/conf/type/__apt_key/parameter/optional

@@ -1,5 +1,3 @@
 state
 keyid
 keyserver
-keydir
-uri

cdist/conf/type/__cdist_preos_trigger/gencode-remote

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+os=$(cat "$__global/explorer/os")
+
+case "$os" in
+    devuan)
+        echo "update-rc.d cdist-preos-trigger defaults > /dev/null"
+    ;;
+    *)
+    ;;
+esac
+

cdist/conf/type/__cdist_preos_trigger/man.rst

@@ -0,0 +1,45 @@
+cdist-type__cdist_preos_trigger(7)
+==================================
+
+NAME
+----
+cdist-type__cdist_preos_trigger - configure cdist preos trigger
+
+
+DESCRIPTION
+-----------
+Create cdist PreOS trigger by creating systemd unit file that will be started
+at boot and will execute trigger command - connect to specified host and port.
+
+
+REQUIRED PARAMETERS
+-------------------
+trigger-command
+    Command that will be executed as a PreOS cdist trigger.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Configure default curl trigger for host cdist.ungleich.ch at port 80.
+    __cdist_preos_trigger http --trigger-command '/usr/bin/curl cdist.ungleich.ch:80'
+
+
+AUTHORS
+-------
+Darko Poljak <darko.poljak--@--ungleich.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2016 Darko Poljak. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__cdist_preos_trigger/manifest

@@ -0,0 +1,67 @@
+#!/bin/sh
+
+os="$(cat "$__global/explorer/os")"
+trigger_command=$(cat "$__object/parameter/trigger-command")
+
+case "$os" in
+    devuan)
+        __file /etc/init.d/cdist-preos-trigger --owner root \
+            --group root \
+            --mode 755 \
+            --source - << EOF
+#!/bin/sh
+# /etc/init.d/cdist-preos-trigger
+
+### BEGIN INIT INFO
+# Provides:          cdist-preos-trigger
+# Required-Start:    \$all
+# Required-Stop:
+# Default-Start:     2 3 4 5 S
+# Default-Stop:      0 1 6
+# Short-Description: Execute cdist preos trigger command
+# Description:       Execute cdist preos trigger commnad.
+### END INIT INFO
+
+case "\$1" in
+    start)
+        echo "Starting cdist-preos-trigger command"
+        ${trigger_command} &
+        ;;
+    stop)
+        # no-op
+        ;;
+    *)
+        echo "Usage: /etc/init.d/cdist-preos-trigger {start|stop}"
+        exit 1
+        ;;
+esac
+
+exit 0
+EOF
+    ;;
+    *)
+        __file /etc/systemd/system/cdist-preos-trigger.service --owner root \
+            --group root \
+            --mode 644 \
+            --source - << EOF
+[Unit]
+Description=preos trigger
+Wants=network-online.target
+After=network.target network-online.target
+
+[Service]
+Type=simple
+Restart=no
+# Broken systemd
+ExecStartPre=/bin/sleep 5
+ExecStart=${trigger_command}
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+        require="__file/etc/systemd/system/cdist-preos-trigger.service" \
+            __start_on_boot cdist-preos-trigger
+    ;;
+esac
+

cdist/conf/type/__cdist_preos_trigger/parameter/required

@@ -0,0 +1 @@
+trigger-command

cdist/conf/type/__consul/files/versions/1.5.0/cksum

@@ -1 +0,0 @@
-886614099 103959898 consul

cdist/conf/type/__consul/files/versions/1.5.0/source

@@ -1 +0,0 @@
-https://releases.hashicorp.com/consul/1.5.0/consul_1.5.0_linux_amd64.zip

cdist/conf/type/__consul/gencode-remote

@@ -42,7 +42,7 @@ source_file_name="${source##*/}"
 cksum_should=$(cut -d' ' -f1,2 "$version_dir/cksum")
 
 cat << eof
-    tmpdir=\$(mktemp -d -p /tmp "${__type##*/}.XXXXXXXXXX")
+    tmpdir=\$(mktemp -d --tmpdir="/tmp" "${__type##*/}.XXXXXXXXXX")
     curl -s -L "$source" > "\$tmpdir/$source_file_name"
     unzip -p "\$tmpdir/$source_file_name" > "${destination}.tmp"
     rm -rf "\$tmpdir"

cdist/conf/type/__consul/manifest

@@ -24,7 +24,7 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-   alpine|scientific|centos|redhat|ubuntu|debian|devuan|archlinux|gentoo)
+   scientific|centos|redhat|ubuntu|debian|devuan|archlinux|gentoo)
       # any linux should work
       :
    ;;
@@ -47,7 +47,6 @@ fi
 
 if [ -f "$__object/parameter/direct" ]; then
     __package unzip
-    __package curl
 else
     __staged_file /usr/local/bin/consul \
        --source "$(cat "$version_dir/source")" \

cdist/conf/type/__consul_agent/files/consul.sys-openrc

@@ -1,38 +0,0 @@
-#!/sbin/openrc-run
-# 2019 Nico Schottelius (nico-cdist at schottelius.org)
-
-description="consul agent"
-
-pidfile="${CONSUL_PIDFILE:-"/var/run/$RC_SVCNAME/pidfile"}"
-command="${CONSUL_BINARY:-"/usr/local/bin/consul"}"
-
-
-checkconfig() {
-	if [ ! -d /var/run/consul ] ; then
-		mkdir -p /var/run/consul || return 1
-        chown consul:consul /var/run/$NAME || return 1
-        chmod 2770 /var/run/$NAME || return 1
-	fi
-}
-
-start() {
-    need net
-
-    start-stop-daemon --start --quiet --oknodo \
-            --pidfile "$pidfile" --background \
-            --exec $command -- agent -pid-file="$pidfile" -config-dir /etc/consul/conf.d
-}
-start_pre() {
-	checkconfig
-}
-
-stop() {
-	if [ "${RC_CMD}" = "restart" ] ; then
-		checkconfig || return 1
-	fi
-
-	ebegin "Stopping $RC_SVCNAME"
-	start-stop-daemon --stop --exec "$command" \
-		--pidfile "$pidfile" --quiet
-	eend $?
-}

cdist/conf/type/__consul_agent/manifest

@@ -1,7 +1,7 @@
 #!/bin/sh -e
 #
 # 2015 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2015-2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2015 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -23,7 +23,7 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-   alpine|scientific|centos|debian|devuan|redhat|ubuntu)
+   scientific|centos|debian|devuan|redhat|ubuntu)
       # whitelist safeguard
       :
    ;;
@@ -181,25 +181,22 @@ init_upstart()
 
 # Install init script to start on boot
 case "$os" in
-    alpine|devuan)
-        init_sysvinit debian
-        ;;
-    centos|redhat)
-        os_version="$(sed 's/[^0-9.]//g' "$__global/explorer/os_version")"
-        major_version="${os_version%%.*}"
-        case "$major_version" in
-            [456])
-                init_sysvinit redhat
-                ;;
-            7)
-                init_systemd
-                ;;
-            *)
-                echo "Unsupported CentOS/Redhat version: $os_version" >&2
-                exit 1
-                ;;
-        esac
-        ;;
+   centos|redhat)
+      os_version="$(sed 's/[^0-9.]//g' "$__global/explorer/os_version")"
+      major_version="${os_version%%.*}"
+      case "$major_version" in
+         [456])
+            init_sysvinit redhat
+         ;;
+         7)
+            init_systemd
+         ;;
+         *)
+            echo "Unsupported CentOS/Redhat version: $os_version" >&2
+            exit 1
+         ;;
+      esac
+   ;;
 
     debian)
         os_version=$(cat "$__global/explorer/os_version")
@@ -217,9 +214,13 @@ case "$os" in
                 exit 1
             ;;
         esac
-        ;;
+    ;;
+
+    devuan)
+        init_sysvinit debian
+    ;;
 
     ubuntu)
         init_upstart
-        ;;
+    ;;
 esac

cdist/conf/type/__docker/manifest

@@ -64,43 +64,6 @@ case "$os" in
         require="__apt_source/docker" __package docker-ce --state "${state}"
       fi
    ;;
-   devuan)
-      os_version="$(cat "$__global/explorer/os_version")"
-
-      case "$os_version" in
-        ascii)
-          distribution="stretch"
-        ;;
-        jessie)
-          distribution="jessie"
-        ;;
-        *)
-          echo "Your devuan release ($os_version) is currently not supported by this type (${__type##*/}).">&2
-          echo "Please contribute an implementation for it if you can." >&2
-          exit 1
-        ;;
-      esac
-
-      if [ "${state}" = "present" ]; then
-        __package apt-transport-https
-        __package ca-certificates
-        __package gnupg2
-      fi
-      __apt_key_uri docker --name "Docker Release (CE deb) <docker@docker.com>" \
-        --uri "https://download.docker.com/linux/${os}/gpg" --state "${state}"
-
-      require="__apt_key_uri/docker" __apt_source docker \
-         --uri "https://download.docker.com/linux/${os}" \
-         --distribution "${distribution}" \
-         --state "${state}" \
-         --component "stable"
-      if [ "$version" != "latest" ]; then
-        require="__apt_source/docker" __package docker-ce --version "${version}" --state "${state}"
-      else
-        require="__apt_source/docker" __package docker-ce --state "${state}"
-      fi
-
-   ;;
    *)
       echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
       echo "Please contribute an implementation for it if you can." >&2

cdist/conf/type/__docker_swarm/explorer/swarm-state

@@ -18,4 +18,4 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-docker info 2>/dev/null | grep '^ *Swarm: ' | awk '{print $2}'
+docker info 2>/dev/null | grep "^Swarm: " | cut -d " " -f 2-

cdist/conf/type/__file_py/__init__.py

@@ -0,0 +1,103 @@
+import os
+import re
+import sys
+from cdist.core import PythonType
+
+
+class FileType(PythonType):
+    def get_attribute(self, stat_file, attribute, value_should):
+        if os.path.exists(stat_file):
+            if re.match('[0-9]', value_should):
+                index = 1
+            else:
+                index = 2
+            with open(stat_file, 'r') as f:
+                for line in f:
+                    if re.match(attribute + ":", line):
+                        fields = line.split()
+                        return fields[index]
+            return None
+
+    def set_attribute(self, attribute, value_should, destination):
+        cmd = {
+            'group': 'chgrp',
+            'owner': 'chown',
+            'mode': 'chmod',
+        }
+        self.send_message("{} '{}'".format(cmd[attribute], value_should))
+        return "{} '{}' '{}'".format(cmd[attribute], value_should, destination)
+
+    def type_manifest(self):
+        yield from ()
+
+    def type_gencode(self):
+        typeis = self.get_explorer('type')
+        state_should = self.get_parameter('state')
+
+        if state_should == 'exists' and typeis == 'file':
+            return
+
+        source = self.get_parameter('source')
+        if source == '-':
+            source = self.stdin_path
+        destination = '/' + self.object_id
+        if state_should == 'pre-exists':
+            if source is not None:
+                self.die('--source cannot be used with --state pre-exists')
+            if typeis == 'file':
+                return None
+            else:
+                self.die('File {} does not exist'.format(destination))
+
+        create_file = False
+        upload_file = False
+        set_attributes = False
+        code = []
+        if state_should == 'present' or state_should == 'exists':
+            if source is None:
+                remote_stat = self.get_explorer('stat')
+                if not remote_stat:
+                    create_file = True
+            else:
+                if os.path.exists(source):
+                    if typeis == 'file':
+                        local_cksum = self.run_local(['cksum', source, ])
+                        local_cksum = local_cksum.split()[0]
+                        remote_cksum = self.get_explorer('cksum')
+                        remote_cksum = remote_cksum.split()[0]
+                        upload_file = local_cksum != remote_cksum
+                    else:
+                        upload_file = True
+                else:
+                    self.die('Source {} does not exist'.format(source))
+            if create_file or upload_file:
+                set_attributes = True
+                tempfile_template = '{}.cdist.XXXXXXXXXX'.format(destination)
+                destination_upload = self.run_remote(
+                    ["mktemp", tempfile_template, ])
+                if upload_file:
+                    self.transfer(source, destination_upload)
+                code.append('rm -rf {}'.format(destination))
+                code.append('mv {} {}'.format(destination_upload, destination))
+
+        if state_should in ('present', 'exists', 'pre-exists', ):
+            for attribute in ('group', 'owner', 'mode', ):
+                if attribute in self.parameters:
+                    value_should = self.get_parameter(attribute)
+                    if attribute == 'mode':
+                        value_should = re.sub('^0', '', value_should)
+                    stat_file = self.get_explorer_file('stat')
+                    value_is = self.get_attribute(stat_file, attribute,
+                                                  value_should)
+                    if set_attributes or value_should != value_is:
+                        code.append(self.set_attribute(attribute,
+                                                       value_should,
+                                                       destination))
+        elif state_should == 'absent':
+            if typeis == 'file':
+                code.append('rm -f {}'.format(destination))
+                self.send_message('remove')
+        else:
+            self.die('Unknown state {}'.format(state_should))
+
+        return "\n".join(code)

cdist/conf/type/__file_py/explorer/cksum

@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Retrieve the md5sum of a file to be created, if it is already existing.
+#
+
+destination="/$__object_id"
+
+if [ -e "$destination" ]; then
+    if [ -f  "$destination" ]; then
+        cksum < "$destination"
+    else
+        echo "NO REGULAR FILE"
+    fi
+else
+    echo "NO FILE FOUND, NO CHECKSUM CALCULATED."
+fi

cdist/conf/type/__file_py/explorer/stat

@@ -0,0 +1,56 @@
+#!/bin/sh
+#
+# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+destination="/$__object_id"
+
+# nothing to work with, nothing we could do
+[ -e "$destination" ] || exit 0
+
+os=$("$__explorer/os")
+case "$os" in
+   "freebsd"|"netbsd"|"openbsd")
+      # FIXME: should be something like this based on man page, but can not test
+      stat -f "type: %ST
+owner: %Du %Su
+group: %Dg %Sg
+mode: %Op %Sp
+size: %Dz
+links: %Dl
+" "$destination"
+   ;;
+   "macosx")
+     stat -f "type: %HT
+owner: %Du %Su
+group: %Dg %Sg
+mode: %Lp %Sp
+size: %Dz
+links: %Dl
+" "$destination"
+   ;;
+   *)
+      stat --printf="type: %F
+owner: %u %U
+group: %g %G
+mode: %a %A
+size: %s
+links: %h
+" "$destination"
+   ;;
+esac

cdist/conf/type/__file_py/explorer/type

@@ -1,6 +1,6 @@
-#!/bin/sh -e
+#!/bin/sh
 #
-# 2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2013 Steven Armstrong (steven-cdist armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -17,16 +17,17 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# Manage users.
 
-os=$(cat "$__global/explorer/os")
+destination="/$__object_id"
 
-case "$os" in
-    alpine)
-        __package shadow
-        ;;
-    *)
-        :
-        ;;
-esac
+if [ ! -e "$destination" ]; then
+   echo none
+elif [ -h "$destination" ]; then
+   echo symlink
+elif [ -f "$destination" ]; then
+   echo file
+elif [ -d "$destination" ]; then
+   echo directory
+else
+   echo unknown
+fi

cdist/conf/type/__file_py/parameter/default/state

@@ -0,0 +1 @@
+present

cdist/conf/type/__file_py/parameter/optional

@@ -0,0 +1,5 @@
+state
+group
+mode
+owner
+source

cdist/conf/type/__git/gencode-remote

@@ -19,33 +19,32 @@
 #
 #
 
-state_is=$(cat "$__object/explorer/state")
-owner_is=$(cat "$__object/explorer/owner")
-group_is=$(cat "$__object/explorer/group")
+state_is="$(cat "$__object/explorer/state")"
+owner_is="$(cat "$__object/explorer/owner")"
+group_is="$(cat "$__object/explorer/group")"
 
-state_should=$(cat "$__object/parameter/state")
+state_should="$(cat "$__object/parameter/state")"
 
-branch=$(cat "$__object/parameter/branch")
+branch="$(cat "$__object/parameter/branch")"
 
-source=$(cat "$__object/parameter/source")
+source="$(cat "$__object/parameter/source")"
 
 destination="/$__object_id"
 
-owner=$(cat "$__object/parameter/owner")
-group=$(cat "$__object/parameter/group")
-mode=$(cat "$__object/parameter/mode")
+owner="$(cat "$__object/parameter/owner")"
+group="$(cat "$__object/parameter/group")"
+mode="$(cat "$__object/parameter/mode")"
 
-[ -f "$__object/parameter/recursive" ] && recursive='--recursive' || recursive=''
-
-[ "$state_should" = "$state_is" ] \
- && [ "$owner" = "$owner_is" ] \
- && [ "$group" = "$group_is" ] \
- && [ -n "$mode" ] && exit 0
+[  "$state_should" = "$state_is" ] && \
+[  "$owner" = "$owner_is" ] && \
+[  "$group" = "$group_is" ] && \
+[  -n "$mode" ] && exit 0
 
 case $state_should in
     present)
+
         if [ "$state_should" != "$state_is" ]; then
-            echo git clone --quiet "$recursive" --branch "$branch" "$source" "$destination"
+            echo git clone --quiet --branch "$branch" "$source" "$destination"
         fi
         if { [ -n "$owner" ] && [ "$owner_is" != "$owner" ]; } || \
            { [ -n "$group" ] && [ "$group_is" != "$group" ]; }; then
@@ -55,9 +54,8 @@ case $state_should in
             echo chmod -R "$mode" "$destination"
         fi
     ;;
-
+    # Handled in manifest
     absent)
-        # Handled in manifest
     ;;
 
     *)

cdist/conf/type/__git/man.rst

@@ -35,8 +35,6 @@ mode
 owner
    User to chown to.
 
-recursive
-   Passes the --recursive flag to git when cloning the repository.
 
 EXAMPLES
 --------

cdist/conf/type/__git/parameter/boolean

@@ -1 +0,0 @@
-recursive

cdist/conf/type/__grafana_dashboard/manifest

@@ -8,12 +8,10 @@ case $os in
     debian|devuan)
         case $os_version in
             8*|jessie)
-                # Differntation not needed anymore
-                apt_source_distribution=stable
+                apt_source_distribution=jessie
                 ;;
             9*|ascii/ceres|ascii)
-                # Differntation not needed anymore
-                apt_source_distribution=stable
+                apt_source_distribution=stretch
                 ;;
             *)
                 echo "Don't know how to install Grafana on $os $os_version. Send us a pull request!" >&2
@@ -23,10 +21,10 @@ case $os in
 
         __apt_key_uri grafana \
             --name 'Grafana Release Signing Key' \
-            --uri https://packages.grafana.com/gpg.key
+            --uri https://packagecloud.io/gpg.key
 
         require="$require __apt_key_uri/grafana" __apt_source grafana \
-                    --uri https://packages.grafana.com/oss/deb \
+                    --uri https://packagecloud.io/grafana/stable/debian/ \
                     --distribution $apt_source_distribution \
                     --component main
 

cdist/conf/type/__group/explorer/group

@@ -1,7 +1,6 @@
 #!/bin/sh
 #
 # 2011-2015 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -22,21 +21,7 @@
 # Get an existing groups group entry.
 #
 
-not_supported() {
-	echo "Your operating system ($("$__explorer/os")) is currently not supported." >&2
-	echo "Cannot extract group information." >&2
-	echo "Please contribute an implementation for it if you can." >&2
-	exit 1
-}
-
 name=$__object_id
 
-if command -v getent >/dev/null
-then
-	getent group "$name" || true
-elif [ -f /etc/group ]
-then
-	grep "^${name}:" /etc/group || true
-else
-	not_supported
-fi
+getent group "$name" || true
+

cdist/conf/type/__group/explorer/gshadow

@@ -1,7 +1,6 @@
 #!/bin/sh
 #
 # 2011-2015 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -23,28 +22,13 @@
 #
 
 name=$__object_id
-os=$("$__explorer/os")
+os="$("$__explorer/os")"
 
-not_supported() {
-	echo "Your operating system ($os) is currently not supported." >&2
-	echo "Cannot extract group information." >&2
-	echo "Please contribute an implementation for it if you can." >&2
-	exit 1
-}
-
-case $os in
-	"freebsd"|"netbsd")
-		echo "$os does not have getent gshadow" >&2
-		exit 0
-	;;
+case "$os" in
+    "freebsd"|"netbsd")
+	echo "$os does not have getent gshadow"
+	exit 0
+    ;;
 esac
 
-if command -v getent >/dev/null
-then
-	getent gshadow "$name" || true
-elif [ -f /etc/gshadow ]
-then
-	grep "^${name}:" /etc/gshadow || true
-else
-	not_supported
-fi
+getent gshadow "$name" || true

cdist/conf/type/__letsencrypt_cert/manifest

@@ -33,9 +33,6 @@ if [ -z "${certbot_fullpath}" ]; then
 					require="__apt_source/stretch-backports" __package_apt certbot \
 						--target-release stretch-backports
 				;;
-				10*)
-					__package_apt certbot
-				;;
 				*)
 					echo "Unsupported OS version: $os_version" >&2
 					exit 1
@@ -65,12 +62,11 @@ if [ -z "${certbot_fullpath}" ]; then
 								 --distribution ascii-backports \
 								 --component main
 
+					require="__apt_source/ascii-backports" __package_apt python-certbot \
+						--target-release ascii-backports
 					require="__apt_source/ascii-backports" __package_apt certbot \
 						--target-release ascii-backports
 				;;
-				beowulf*)
-					__package_apt certbot
-				;;
 				*)
 					echo "Unsupported OS version: $os_version" >&2
 					exit 1

cdist/conf/type/__package_apk/explorer/state

@@ -27,10 +27,6 @@ else
    name="$__object_id"
 fi
 
-# Remove the @.. repo tag for finding out whether it is installed
-# f.i. pass@testing => pass
-name="$(echo "$name" | sed 's/@.*//')"
-
 if [ "$(apk list -I "$name")" ]; then
     echo present
 else

cdist/conf/type/__postfix/manifest

@@ -1,7 +1,6 @@
 #!/bin/sh -e
 #
 # 2012-2014 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2019 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -23,7 +22,7 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-   alpine|ubuntu|debian|archlinux|suse|scientific|centos|devuan)
+   ubuntu|debian|archlinux|suse|scientific|centos|devuan)
       __package postfix --state present
    ;;
    *)

cdist/conf/type/__postfix_postconf/explorer/value

@@ -22,7 +22,7 @@
 os=$("$__explorer/os")
 
 case "$os" in
-   alpine|ubuntu|debian|archlinux|suse|scientific|centos|devuan)
+   ubuntu|debian|archlinux|suse|scientific|centos|devuan)
       :
    ;;
    *)

cdist/conf/type/__postfix_postconf/gencode-remote

@@ -1,7 +1,6 @@
 #!/bin/sh -e
 #
 # 2012-2014 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2019 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -22,7 +21,7 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-   alpine|archlinux|centos|debian|devuan|suse|scientific|ubuntu)
+   ubuntu|debian|archlinux|suse|scientific|centos|devuan)
       :
    ;;
    *)

cdist/conf/type/__ssh_authorized_keys/explorer/file

@@ -1,7 +1,6 @@
 #!/bin/sh
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -20,42 +19,9 @@
 #
 
 if [ -f "$__object/parameter/file" ]; then
-	cat "$__object/parameter/file"
+   cat "$__object/parameter/file"
 else
-	if [ -s "$__object/parameter/owner" ]
-	then
-		owner=$(cat "$__object/parameter/owner")
-	else
-		owner="$__object_id"
-	fi
-
-	if command -v getent >/dev/null
-	then
-		owner_line=$(getent passwd "$owner")
-	elif [ -f /etc/passwd ]
-	then
-		case $owner
-		in
-			[0-9][0-9]*)
-				owner_line=$(awk -F: "\$3 == \"${owner}\" { print }" /etc/passwd)
-				;;
-			*)
-				owner_line=$(awk -F: "\$1 == \"${owner}\" { print }" /etc/passwd)
-				;;
-		esac
-	fi
-
-	if [ "$owner_line" ]
-	then
-		home=$(echo "$owner_line" | cut -d':' -f6)
-	fi
-
-	if [ ! -d "$home" ]
-	then
-		# Don't know how to determine user's home directory, fall back to ~
-		home="~$owner"
-		command -v realpath >/dev/null && home=$(realpath "$home")
-	fi
-
-	[ -d "$home" ] && echo "$home/.ssh/authorized_keys"
+   owner="$(cat "$__object/parameter/owner" 2>/dev/null || echo "$__object_id")"
+   home=$(getent passwd "$owner" | cut -d':' -f 6)
+   echo "$home/.ssh/authorized_keys"
 fi

cdist/conf/type/__ssh_authorized_keys/explorer/group

@@ -1,7 +1,6 @@
 #!/bin/sh
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,28 +18,6 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-if [ -s "$__object/parameter/owner" ]
-then
-	owner=$(cat "$__object/parameter/owner")
-else
-	owner="$__object_id"
-fi
-
-if command -v getent >/dev/null
-then
-	gid=$(getent passwd "$owner" | cut -d':' -f4)
-	getent group "$gid" || true
-else
-	# Fallback to local file scanning
-	case $owner
-	in
-		[0-9][0-9]*)
-			gid=$(awk -F: "\$3 == \"${owner}\" { print \$4 }" /etc/passwd)
-			;;
-		*)
-			gid=$(awk -F: "\$1 == \"${owner}\" { print \$4 }" /etc/passwd)
-			;;
-	esac
-
-	awk -F: "\$3 == \"$gid\" { print }" /etc/group
-fi
+owner="$(cat "$__object/parameter/owner" 2>/dev/null || echo "$__object_id")"
+gid="$(getent passwd "$owner" | cut -d':' -f 4)"
+getent group "$gid" || true

cdist/conf/type/__ssh_authorized_keys/manifest

@@ -23,12 +23,6 @@ owner="$(cat "$__object/parameter/owner" 2>/dev/null || echo "$__object_id")"
 state="$(cat "$__object/parameter/state" 2>/dev/null)"
 file="$(cat "$__object/explorer/file")"
 
-if [ ! -f "$__object/parameter/nofile" ] && [ -z "$file" ]
-then
-	echo "Cannot determine path of authorized_keys file" >&2
-	exit 1
-fi
-
 if [ ! -f "$__object/parameter/noparent" ] || [ ! -f "$__object/parameter/nofile" ]; then
    group="$(cut -d':' -f 1 "$__object/explorer/group")"
    if [ -z "$group" ]; then

cdist/conf/type/__ssh_dot_ssh/explorer/group

@@ -1,7 +1,6 @@
 #!/bin/sh
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,11 +18,5 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-gid=$("$__type_explorer/passwd" | cut -d':' -f4)
-
-if command -v getent >/dev/null
-then
-	getent group "$gid" || true
-else
-	awk -F: "\$3 == \"$gid\" { print }" /etc/group
-fi
+gid="$("$__type_explorer/passwd" | cut -d':' -f 4)"
+getent group "$gid" || true

cdist/conf/type/__ssh_dot_ssh/explorer/passwd

@@ -2,7 +2,6 @@
 #
 # 2012 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2014 Nico Schottelius (nico-cdist at schottelius.org)
-# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -22,16 +21,4 @@
 
 owner="$__object_id"
 
-if command -v getent >/dev/null
-then
-    getent passwd "$owner" || true
-else
-    case $owner in
-        [0-9][0-9]*)
-            awk -F: "\$3 == \"$owner\" { print }" /etc/passwd
-            ;;
-        *)
-            grep "^$owner:" /etc/passwd || true
-            ;;
-    esac
-fi
+getent passwd "$owner" || true

cdist/conf/type/__sysctl/manifest

@@ -2,7 +2,6 @@
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2018 Takashi Yoshi (takashi at yoshi.email)
-# 2019 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -25,7 +24,7 @@ os=$(cat "$__global/explorer/os")
 
 case "$os" in
    # Linux
-   alpine|redhat|centos|ubuntu|debian|devuan|archlinux|coreos)
+   redhat|centos|ubuntu|debian|devuan|archlinux|coreos)
       :
    ;;
    # BSD

cdist/conf/type/__user/explorer/group

@@ -23,9 +23,11 @@
 
 if [ -f "$__object/parameter/gid" ]; then
    gid=$(cat "$__object/parameter/gid")
-   if command -v getent >/dev/null; then
-      getent group "$gid" || true
+   getent=$(command -v getent)
+   if [ X != X"${getent}" ]; then
+      "${getent}" group "$gid" || true
    elif [ -f /etc/group ]; then
       grep -E "^(${gid}|([^:]+:){2}${gid}):" /etc/group || true
    fi
 fi
+

cdist/conf/type/__user/explorer/passwd

@@ -23,8 +23,9 @@
 
 name=$__object_id
 
-if command -v getent >/dev/null; then
-  getent passwd "$name" || true
+getent=$(command -v getent)
+if [ X != X"${getent}" ]; then
+  "${getent}" passwd "$name" || true
 elif [ -f /etc/passwd ]; then
   grep "^${name}:" /etc/passwd || true
 fi

cdist/conf/type/__user/explorer/shadow

@@ -1,4 +1,4 @@
-#!/bin/sh -e
+#!/bin/sh
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -22,19 +22,18 @@
 #
 
 name=$__object_id
+os="$("$__explorer/os")"
+# Default to using shadow passwords
+database="shadow"
 
-case $("$__explorer/os") in
-  'freebsd'|'netbsd'|'openbsd')
-    database='passwd'
-    ;;
-  # Default to using shadow passwords
-  *)
-    database='shadow'
-    ;;
+case "$os" in
+  "freebsd"|"netbsd"|"openbsd")  database="passwd";;
 esac
+  
 
-if command -v getent >/dev/null; then
-  getent "$database" "$name" || true
+getent=$(command -v getent)
+if [ X != X"${getent}" ]; then
+  "${getent}" "$database" "$name" || true
 elif [ -f /etc/shadow ]; then
   grep "^${name}:" /etc/shadow || true
 fi

cdist/config.py

@@ -90,13 +90,15 @@ class Config(object):
                 shutil.rmtree(path)
 
     def __init__(self, local, remote, dry_run=False, jobs=None,
-                 cleanup_cmds=None, remove_remote_files_dirs=False):
+                 cleanup_cmds=None, remove_remote_files_dirs=False,
+                 timestamp=False):
 
         self.local = local
         self.remote = remote
         self._open_logger()
         self.dry_run = dry_run
         self.jobs = jobs
+        self.timestamp = timestamp
         if cleanup_cmds:
             self.cleanup_cmds = cleanup_cmds
         else:
@@ -104,12 +106,9 @@ class Config(object):
         self.remove_remote_files_dirs = remove_remote_files_dirs
 
         self.explorer = core.Explorer(self.local.target_host, self.local,
-                                      self.remote, jobs=self.jobs,
-                                      dry_run=self.dry_run)
-        self.manifest = core.Manifest(self.local.target_host, self.local,
-                                      dry_run=self.dry_run)
-        self.code = core.Code(self.local.target_host, self.local, self.remote,
-                              dry_run=self.dry_run)
+                                      self.remote, jobs=self.jobs)
+        self.manifest = core.Manifest(self.local.target_host, self.local)
+        self.code = core.Code(self.local.target_host, self.local, self.remote)
 
     def _init_files_dirs(self):
         """Prepare files and directories for the run"""
@@ -427,7 +426,8 @@ class Config(object):
                 cleanup_cmds.append(cleanup_cmd)
             c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs,
                     cleanup_cmds=cleanup_cmds,
-                    remove_remote_files_dirs=remove_remote_files_dirs)
+                    remove_remote_files_dirs=remove_remote_files_dirs,
+                    timestamp=args.timestamp)
             c.run()
             cls._remove_paths()
 
@@ -758,39 +758,50 @@ class Config(object):
                     ("The requirements of the following objects could not be "
                      "resolved:\n%s") % ("\n".join(info_string)))
 
-    def _handle_deprecation(self, cdist_object):
-        cdist_type = cdist_object.cdist_type
-        deprecated = cdist_type.deprecated
-        if deprecated is not None:
-            if deprecated:
-                self.log.warning("Type %s is deprecated: %s", cdist_type.name,
-                                 deprecated)
+    def _timeit(self, func, msg_prefix):
+        def wrapper_func(*args, **kwargs):
+            loglevel = self.log.getEffectiveLevel()
+            if loglevel >= logging.VERBOSE and self.timestamp:
+                start_time = time.time()
+                rv = func(*args, **kwargs)
+                end_time = time.time()
+                duration = end_time - start_time
+                self.log.verbose("%s duration: %.6f seconds",
+                                 msg_prefix, duration)
             else:
-                self.log.warning("Type %s is deprecated.", cdist_type.name)
-        for param in cdist_object.parameters:
-            if param in cdist_type.deprecated_parameters:
-                msg = cdist_type.deprecated_parameters[param]
-                if msg:
-                    format = "%s parameter of type %s is deprecated: %s"
-                    args = [param, cdist_type.name, msg]
-                else:
-                    format = "%s parameter of type %s is deprecated."
-                    args = [param, cdist_type.name]
-                self.log.warning(format, *args)
+                rv = func(*args, **kwargs)
+            return rv
+        return wrapper_func
 
     def object_prepare(self, cdist_object, transfer_type_explorers=True):
         """Prepare object: Run type explorer + manifest"""
-        self._handle_deprecation(cdist_object)
         self.log.verbose("Preparing object {}".format(cdist_object.name))
         self.log.verbose(
             "Running manifest and explorers for " + cdist_object.name)
         self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
         try:
-            self.manifest.run_type_manifest(cdist_object)
+            self.log.verbose("Preparing object {}".format(cdist_object.name))
+            self.log.verbose(
+                    "Running manifest and explorers for " + cdist_object.name)
+            self.explorer.run_type_explorers(cdist_object,
+                                             transfer_type_explorers)
+            if self.is_py_type(cdist_object):
+                self._timeit(self.manifest.run_py_type_manifest,
+                             "Python type manifest for {}".format(
+                                 cdist_object.name))(cdist_object)
+            else:
+                self._timeit(self.manifest.run_type_manifest,
+                             "Type manifest for {}".format(
+                                 cdist_object.name))(cdist_object)
             cdist_object.state = core.CdistObject.STATE_PREPARED
         except cdist.Error as e:
             raise cdist.CdistObjectError(cdist_object, e)
 
+    def is_py_type(self, cdist_object):
+        cdist_type = cdist_object.cdist_type
+        init_path = os.path.join(cdist_type.absolute_path, '__init__.py')
+        return os.path.exists(init_path)
+
     def object_run(self, cdist_object):
         """Run gencode and code for an object"""
         try:
@@ -801,9 +812,20 @@ class Config(object):
 
             # Generate
             self.log.debug("Generating code for %s" % (cdist_object.name))
-            cdist_object.code_local = self.code.run_gencode_local(cdist_object)
-            cdist_object.code_remote = self.code.run_gencode_remote(
-                cdist_object)
+            if self.is_py_type(cdist_object):
+                cdist_object.code_local = ''
+                cdist_object.code_remote = self._timeit(self.code.run_py,
+                             "Python type generate code for {}".format(
+                                 cdist_object.name))(cdist_object)
+            else:
+                cdist_object.code_local = self._timeit(
+                    self.code.run_gencode_local,
+                             "Type generate code local for {}".format(
+                                 cdist_object.name))(cdist_object)
+                cdist_object.code_remote = self._timeit(
+                    self.code.run_gencode_remote,
+                             "Type generate code remote for {}".format(
+                                 cdist_object.name))(cdist_object)
             if cdist_object.code_local or cdist_object.code_remote:
                 cdist_object.changed = True
 
@@ -814,12 +836,16 @@ class Config(object):
                 if cdist_object.code_local:
                     self.log.trace("Executing local code for %s"
                                    % (cdist_object.name))
-                    self.code.run_code_local(cdist_object)
+                    self._timeit(self.code.run_code_local,
+                                "Type run code local for {}".format(
+                                    cdist_object.name))(cdist_object)
                 if cdist_object.code_remote:
                     self.log.trace("Executing remote code for %s"
                                    % (cdist_object.name))
                     self.code.transfer_code_remote(cdist_object)
-                    self.code.run_code_remote(cdist_object)
+                    self._timeit(self.code.run_code_remote,
+                                "Type run code remote for {}".format(
+                                    cdist_object.name))(cdist_object)
 
             # Mark this object as done
             self.log.trace("Finishing run of " + cdist_object.name)

cdist/core/__init__.py

@@ -21,6 +21,7 @@
 #
 
 from cdist.core.cdist_type import CdistType
+from cdist.core.python_type import PythonType, ManifestEntry
 from cdist.core.cdist_type import InvalidTypeError
 from cdist.core.cdist_object import CdistObject
 from cdist.core.cdist_object import IllegalObjectIdError

cdist/core/cdist_type.py

@@ -69,7 +69,6 @@ class CdistType(object):
         self.__optional_multiple_parameters = None
         self.__boolean_parameters = None
         self.__parameter_defaults = None
-        self.__deprecated_parameters = None
 
     def __hash__(self):
         return hash(self.name)
@@ -134,17 +133,6 @@ class CdistType(object):
            cannot run in parallel."""
         return os.path.isfile(os.path.join(self.absolute_path, "nonparallel"))
 
-    @property
-    def deprecated(self):
-        """Get type deprecation message. If message is None then type
-        is not deprecated."""
-        deprecated_path = os.path.join(self.absolute_path, "deprecated")
-        try:
-            with open(deprecated_path, 'r') as f:
-                return f.read()
-        except FileNotFoundError:
-            return None
-
     @property
     def explorers(self):
         """Return a list of available explorers"""
@@ -276,23 +264,3 @@ class CdistType(object):
             finally:
                 self.__parameter_defaults = defaults
         return self.__parameter_defaults
-
-    @property
-    def deprecated_parameters(self):
-        if not self.__deprecated_parameters:
-            deprecated = {}
-            try:
-                deprecated_dir = os.path.join(self.absolute_path,
-                                              "parameter",
-                                              "deprecated")
-                for name in cdist.core.listdir(deprecated_dir):
-                    try:
-                        with open(os.path.join(deprecated_dir, name)) as fd:
-                            deprecated[name] = fd.read().strip()
-                    except EnvironmentError:
-                        pass  # Swallow errors raised by open() or read()
-            except EnvironmentError:
-                pass  # Swallow error raised by os.listdir()
-            finally:
-                self.__deprecated_parameters = deprecated
-        return self.__deprecated_parameters

cdist/core/code.py

@@ -22,6 +22,10 @@
 #
 
 import os
+import importlib.util
+import inspect
+import cdist
+from cdist.core import PythonType
 from . import util
 
 
@@ -97,7 +101,7 @@ class Code(object):
 
     """
     # target_host is tuple (target_host, target_hostname, target_fqdn)
-    def __init__(self, target_host, local, remote, dry_run=False):
+    def __init__(self, target_host, local, remote):
         self.target_host = target_host
         self.local = local
         self.remote = remote
@@ -113,8 +117,43 @@ class Code(object):
                 local.log),
         }
 
-        if dry_run:
-            self.env['__cdist_dry_run'] = '1'
+    def run_py(self, cdist_object):
+        cdist_type = cdist_object.cdist_type
+        module_name = cdist_type.name
+        file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
+
+        if os.path.isfile(file_path):
+            spec = importlib.util.spec_from_file_location(module_name,
+                                                          file_path)
+            m = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(m)
+            classes = inspect.getmembers(m, inspect.isclass)
+            type_class = None
+            for _, cl in classes:
+                if cl != PythonType and issubclass(cl, PythonType):
+                    if type_class:
+                        raise cdist.Error("Only one python type class is "
+                                          "supported, but at least two "
+                                          "found: {}".format((type_class,
+                                                              cl, )))
+                    else:
+                        type_class = cl
+            env = os.environ.copy()
+            env.update(self.env)
+            message_prefix = cdist_object.name
+            type_obj = type_class(env=env, cdist_object=cdist_object,
+                                  local=self.local, remote=self.remote,
+                                  message_prefix=message_prefix)
+            if hasattr(type_obj, 'run') and inspect.ismethod(type_obj.run):
+                if self.local.save_output_streams:
+                    which = 'gencode-py'
+                    stderr_path = os.path.join(cdist_object.stderr_path, which)
+                    stdout_path = os.path.join(cdist_object.stdout_path, which)
+                    with open(stderr_path, 'a+') as stderr, \
+                            open(stdout_path, 'a+') as stdout:
+                        return type_obj.run(stdout=stdout, stderr=stderr)
+                else:
+                    return type_obj.run()
 
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.cdist_type

cdist/core/explorer.py

@@ -67,7 +67,7 @@ class Explorer(object):
     """Executes cdist explorers.
 
     """
-    def __init__(self, target_host, local, remote, jobs=None, dry_run=False):
+    def __init__(self, target_host, local, remote, jobs=None):
         self.target_host = target_host
 
         self._open_logger()
@@ -84,10 +84,6 @@ class Explorer(object):
             '__cdist_log_level_name': util.log_level_name_env_var_val(
                 self.log),
         }
-
-        if dry_run:
-            self.env['__cdist_dry_run'] = '1'
-
         self._type_explorers_transferred = []
         self.jobs = jobs
 

cdist/core/manifest.py

@@ -22,9 +22,13 @@
 
 import logging
 import os
-
+import importlib.util
+import inspect
 import cdist
+import cdist.emulator
 from . import util
+from cdist.core import PythonType, ManifestEntry
+
 
 '''
 common:
@@ -96,7 +100,7 @@ class Manifest(object):
     """Executes cdist manifests.
 
     """
-    def __init__(self, target_host, local, dry_run=False):
+    def __init__(self, target_host, local):
         self.target_host = target_host
         self.local = local
 
@@ -117,9 +121,6 @@ class Manifest(object):
                 self.log),
         }
 
-        if dry_run:
-            self.env['__cdist_dry_run'] = '1'
-
     def _open_logger(self):
         self.log = logging.getLogger(self.target_host[0])
 
@@ -212,3 +213,73 @@ class Manifest(object):
                     type_manifest,
                     env=self.env_type_manifest(cdist_object),
                     message_prefix=message_prefix)
+
+    def env_py_type_manifest(self, cdist_object):
+        env = os.environ.copy()
+        env.update(self.env)
+        env.update({
+            '__cdist_object_marker': self.local.object_marker_name,
+            '__cdist_manifest': cdist_object.cdist_type,
+            '__manifest': self.local.manifest_path,
+            '__object': cdist_object.absolute_path,
+            '__object_id': cdist_object.object_id,
+            '__object_name': cdist_object.name,
+            '__type': cdist_object.cdist_type.absolute_path,
+        })
+
+        return env
+
+    def run_py_type_manifest(self, cdist_object):
+        cdist_type = cdist_object.cdist_type
+        module_name = cdist_type.name
+        file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
+        message_prefix = cdist_object.name
+        if os.path.isfile(file_path):
+            self.log.verbose("Running python type manifest for object %s",
+                             cdist_object.name)
+            spec = importlib.util.spec_from_file_location(module_name,
+                                                          file_path)
+            m = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(m)
+            classes = inspect.getmembers(m, inspect.isclass)
+            type_class = None
+            for _, cl in classes:
+                if cl != PythonType and issubclass(cl, PythonType):
+                    if type_class:
+                        raise cdist.Error("Only one python type class is "
+                                          "supported, but at least two "
+                                          "found: {}".format((type_class,
+                                                              cl, )))
+                    else:
+                        type_class = cl
+            env = self.env_py_type_manifest(cdist_object)
+            type_obj = type_class(env=env, cdist_object=cdist_object,
+                                  local=self.local, remote=None,
+                                  message_prefix=message_prefix)
+            if self.local.save_output_streams:
+                which = 'manifest'
+                stderr_path = os.path.join(cdist_object.stderr_path, which)
+                stdout_path = os.path.join(cdist_object.stdout_path, which)
+                with open(stderr_path, 'a+') as stderr, \
+                        open(stdout_path, 'a+') as stdout:
+                    self._process_py_type_manifest_entries(
+                        type_obj, env, stdout=stdout, stderr=stderr)
+            else:
+                self._process_py_type_manifest_entries(type_obj, env)
+
+    def _process_py_type_manifest_entries(self, type_obj, env, stdout=None,
+                                          stderr=None):
+        if hasattr(type_obj, 'manifest') and \
+                inspect.ismethod(type_obj.manifest):
+            for entry in type_obj.manifest(stdout=stdout, stderr=stderr):
+                if not isinstance(entry, ManifestEntry):
+                    raise TypeError("Manifest entry must be of "
+                                    "type ManifestEntry")
+                kwargs = {
+                    'argv': entry.cmd_line(),
+                    'env': env,
+                }
+                if entry.stdin:
+                    kwargs['stdin'] = entry.stdin
+                emulator = cdist.emulator.Emulator(**kwargs)
+                emulator.run()

cdist/core/python_type.py

@@ -0,0 +1,160 @@
+import logging
+import os
+import io
+import sys
+import re
+from cdist import message, Error
+
+
+class PythonType:
+    def __init__(self, env, cdist_object, local, remote, message_prefix=None):
+        self.env = env
+        self.cdist_object = cdist_object
+        self.object_id = cdist_object.object_id
+        self.object_name = cdist_object.name
+        self.cdist_type = cdist_object.cdist_type
+        self.local = local
+        self.remote = remote
+        self.object_path = cdist_object.absolute_path
+        self.type_path = cdist_object.cdist_type.absolute_path
+        self.explorer_path = os.path.join(self.object_path, 'explorer')
+        self.parameters = cdist_object.parameters
+        self.stdin_path = os.path.join(self.object_path, 'stdin')
+        self.log = logging.getLogger(
+            self.local.target_host[0] + ':' + self.object_name)
+
+        self.message_prefix = message_prefix
+        self.message = None
+
+    def get_parameter(self, name):
+        return self.parameters.get(name)
+
+    def get_explorer_file(self, name):
+        path = os.path.join(self.explorer_path, name)
+        return path
+
+    def get_explorer(self, name):
+        path = self.get_explorer_file(name)
+        with open(path, 'r') as f:
+            value = f.read()
+            if value:
+                value = value.strip()
+            return value
+
+    def run_local(self, command, env=None):
+        rv = self.local.run(command, env=env, return_output=True)
+        if rv:
+            rv = rv.rstrip('\n')
+        return rv
+
+    def run_remote(self, command, env=None):
+        rv = self.remote.run(command, env=env, return_output=True)
+        if rv:
+            rv = rv.rstrip('\n')
+        return rv
+
+    def transfer(self, source, destination):
+        self.remote.transfer(source, destination)
+
+    def die(self, msg):
+        raise Error("{}: {}".format(self.cdist_object, msg))
+
+    def type_manifest(self):
+        pass
+
+    def type_gencode(self):
+        pass
+
+    def manifest(self, stdout=None, stderr=None):
+        try:
+            if self.message_prefix:
+                self.message = message.Message(self.message_prefix,
+                                               self.local.messages_path)
+                self.env.update(self.message.env)
+            if stdout is not None:
+                stdout_save = sys.stdout
+                sys.stdout = stdout
+            if stderr is not None:
+                stderr_save = sys.stderr
+                sys.stderr = stderr
+            yield from self.type_manifest()
+        finally:
+            if self.message:
+                self.message.merge_messages()
+            if stdout is not None:
+                sys.stdout = stdout_save
+            if stderr is not None:
+                sys.stderr = stderr_save
+
+    def run(self, stdout=None, stderr=None):
+        try:
+            if self.message_prefix:
+                self.message = message.Message(self.message_prefix,
+                                               self.local.messages_path)
+            if stdout is not None:
+                stdout_save = sys.stdout
+                sys.stdout = stdout
+            if stderr is not None:
+                stderr_save = sys.stderr
+                sys.stderr = stderr
+            return self.type_gencode()
+        finally:
+            if self.message:
+                self.message.merge_messages()
+            if stdout is not None:
+                sys.stdout = stdout_save
+            if stderr is not None:
+                sys.stderr = stderr_save
+
+    def send_message(self, msg):
+        if self.message:
+            with open(self.message.messages_out, 'a') as f:
+                print(msg, file=f)
+
+    def receive_message(self, pattern):
+        if self.message:
+            with open(self.message.messages_in, 'r') as f:
+                for line in f:
+                    match = re.search(pattern, line)
+                    if match:
+                        return match
+        return None
+
+
+class ManifestEntry:
+    def __init__(self, name, stdin=None, parameters=None):
+        self.name = name
+        if parameters is None:
+            self.parameters = {}
+        else:
+            self.parameters = parameters
+        self.set_stdin(stdin)
+
+    def set_stdin(self, value):
+        # If file-like object then read its value.
+        if value is not None and isinstance(value, io.IOBase):
+            value = value.read()
+
+        # Convert to bytes file-like object.
+        if value is None:
+            self.stdin = None
+        elif isinstance(value, str):
+            self.stdin = io.BytesIO(value.encode('utf-8'))
+        elif isinstance(value, bytes) or isinstance(value, bytearray):
+            self.stdin = io.BytesIO(value)
+        else:
+            raise TypeError("value must be str, bytes, bytearray, file-like "
+                            "object or None")
+
+    def cmd_line(self):
+        argv = [self.name, ]
+        for param in self.parameters:
+            argv.append(param)
+            val = self.parameters[param]
+            if val:
+                argv.append(val)
+        return argv
+
+    def __repr__(self):
+        return '<ManifestEntry name={}, parameters={}, stdin={}>'.format(
+            self.name, self.parameters, self.stdin)

cdist/inventory.py

@@ -315,7 +315,7 @@ class InventoryHost(Inventory):
         hostpath = self._host_path(host)
         self.log.trace("hostpath: {}".format(hostpath))
         if self.action == "add" and not os.path.exists(hostpath):
-            self._new_hostpath(hostpath)
+                self._new_hostpath(hostpath)
         else:
             if not os.path.isfile(hostpath):
                 raise cdist.Error(("Host path \'{}\' is"

cdist/preos/debootstrap/debootstrap.py

@@ -127,6 +127,12 @@ class Debian(object):
                             help="suite used for debootstrap, "
                                  "by default '{}'".format(defargs.suite),
                             dest='suite', default=defargs.suite)
+        parser.add_argument(
+            '-t', '--trigger-command',
+            help=("trigger command that will be added to cdist config; "
+                  "'__cdist_preos_trigger http ...' type is appended to "
+                  "initial manifest"),
+            dest='trigger_command')
         parser.add_argument(
             '-y', '--remote-copy',
             help=("remote copy that cdist config will use, by default "
@@ -145,6 +151,7 @@ class Debian(object):
         log = logging.getLogger(cls.__name__)
 
         parser = cls.get_parser()
+        cdist.argparse.add_beta_command(cls._preos_name)
         args = parser.parse_args(argv)
         if args.script and not args.mirror:
             raise cdist.Error("script option cannot be used without "
@@ -172,7 +179,7 @@ class Debian(object):
             env = vars(args)
             new_env = {}
             for key in env:
-                if key == 'verbose' and env[key]:
+                if key == 'verbose':
                     if env[key] >= 3:
                         new_env['debug'] = "yes"
                     elif env[key] == 2:

cdist/preos/debootstrap/files/code

@@ -127,6 +127,13 @@ then
         exit 1
     fi
 
+    if [ "${trigger_command}" ]
+    then
+        trigger_line="__cdist_preos_trigger http --trigger-command '${trigger_command}'\n"
+    else
+        trigger_line=""
+    fi
+
     if [ "${keyfile_cnt}" -a "${keyfile_cnt}" -gt 0 ]
     then
         i="$((keyfile_cnt - 1))"
@@ -174,7 +181,7 @@ then
     fi
     grub_lines="${grub_manifest_line}${grub_kern_params_line}"
 
-    printf "${ssh_auth_keys_line}${grub_lines}" \
+    printf "${trigger_line}${ssh_auth_keys_line}${grub_lines}" \
         | cat "${manifest}" - |\
         cdist config \
             ${cdist_params} -i - \

cdist/test/cdist_type/__init__.py

@@ -123,16 +123,6 @@ class TypeTestCase(test.CdistTestCase):
         cdist_type = core.CdistType(base_path, '__not_nonparallel')
         self.assertFalse(cdist_type.is_nonparallel)
 
-    def test_deprecated(self):
-        base_path = fixtures
-        cdist_type = core.CdistType(base_path, '__deprecated')
-        self.assertIsNotNone(cdist_type.deprecated)
-
-    def test_not_deprecated(self):
-        base_path = fixtures
-        cdist_type = core.CdistType(base_path, '__not_deprecated')
-        self.assertIsNone(cdist_type.deprecated)
-
     def test_install_is_install(self):
         base_path = fixtures
         cdist_type = core.CdistType(base_path, '__install')
@@ -200,18 +190,3 @@ class TypeTestCase(test.CdistTestCase):
         self.assertEqual(
             list(sorted(cdist_type.parameter_defaults.keys())),
             ['bar', 'foo'])
-
-    def test_without_deprecated_parameters(self):
-        base_path = fixtures
-        cdist_type = core.CdistType(base_path,
-                                    '__without_deprecated_parameters')
-        self.assertEqual(cdist_type.deprecated_parameters, {})
-
-    def test_with_deprecated_parameters(self):
-        base_path = fixtures
-        cdist_type = core.CdistType(base_path, '__with_deprecated_parameters')
-        self.assertTrue('eggs' in cdist_type.deprecated_parameters)
-        self.assertTrue('spam' in cdist_type.deprecated_parameters)
-        self.assertEqual(cdist_type.deprecated_parameters['eggs'],
-                         'Deprecated')
-        self.assertEqual(cdist_type.deprecated_parameters['spam'], '')

cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated/eggs

@@ -1 +0,0 @@
-Deprecated

cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/optional

@@ -1,3 +0,0 @@
-spam
-eggs
-sausage

cdist/trigger.py

@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2016 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import ipaddress
+import logging
+import re
+import socket
+import http.server
+import os
+import socketserver
+import shutil
+
+import cdist.config
+import cdist.log
+import cdist.util.ipaddr as ipaddr
+
+
+class Trigger():
+    """cdist trigger handling"""
+
+    # Arguments that are only trigger specific
+    triggers_args = ["http_port", "ipv6", "directory", "source", ]
+
+    def __init__(self, http_port=None, dry_run=False, ipv6=False,
+                 directory=None, source=None, cdistargs=None):
+        self.dry_run = dry_run
+        self.http_port = int(http_port)
+        self.ipv6 = ipv6
+        self.args = cdistargs
+
+        self.directory = directory
+        self.source = source
+
+        log.debug("IPv6: %s", self.ipv6)
+
+    def run_httpd(self):
+        server_address = ('', self.http_port)
+
+        if self.ipv6:
+            httpdcls = HTTPServerV6
+        else:
+            httpdcls = HTTPServerV4
+        httpd = httpdcls(self.args, self.directory, self.source,
+                         server_address, TriggerHttp)
+
+        log.debug("Starting server at port %d", self.http_port)
+        if self.dry_run:
+            log.debug("Running in dry run mode")
+        httpd.serve_forever()
+
+    def run(self):
+        if self.http_port:
+            self.run_httpd()
+
+    @classmethod
+    def commandline(cls, args):
+        global log
+
+        # remove root logger default cdist handler and configure trigger's own
+        logging.getLogger().handlers = []
+        logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s')
+
+        log = logging.getLogger("trigger")
+        ownargs = {}
+        for targ in cls.triggers_args:
+            arg = getattr(args, targ)
+            ownargs[targ] = arg
+
+            del arg
+
+        t = cls(dry_run=args.dry_run, cdistargs=args, **ownargs)
+        t.run()
+
+
+class TriggerHttp(http.server.BaseHTTPRequestHandler):
+    actions = {
+        "cdist": ["config", "install", ],
+        "file":  ["present", "absent", ],
+    }
+
+    def do_HEAD(self):
+        self.dispatch_request()
+
+    def do_POST(self):
+        self.dispatch_request()
+
+    def do_GET(self):
+        self.dispatch_request()
+
+    def _actions_regex(self):
+        regex = ["^/(?P<subsystem>", ]
+        regex.extend("|".join(self.actions.keys()))
+        regex.append(")/(?P<action>")
+        regex.extend("|".join("|".join(self.actions[x]) for x in self.actions))
+        regex.append(")/")
+
+        return "".join(regex)
+
+    def dispatch_request(self):
+        host = self.client_address[0]
+        code = 200
+        message = None
+
+        self.cdistargs = self.server.cdistargs
+
+        actions_regex = self._actions_regex()
+        m = re.match(actions_regex, self.path)
+
+        if m:
+            subsystem = m.group('subsystem')
+            action = m.group('action')
+            handler = getattr(self, "handler_" + subsystem)
+
+            if action not in self.actions[subsystem]:
+                code = 404
+        else:
+            code = 404
+
+        if code == 200:
+            log.debug("Calling {} -> {}".format(subsystem, action))
+            try:
+                handler(action, host)
+            except cdist.Error as e:
+                # cdist is not broken, cdist run is broken
+                code = 599  # use arbitrary unassigned error code
+                message = str(e)
+            except Exception as e:
+                # cdist/trigger server is broken
+                log.exception(e)
+                code = 500
+
+        self.send_response(code=code, message=message)
+        self.end_headers()
+
+    def handler_file(self, action, host):
+        if not self.server.directory or not self.server.source:
+            log.info("Cannot serve file request: directory or source "
+                     "not setup")
+            return
+
+        try:
+            ipaddress.ip_address(host)
+        except ValueError:
+            log.error("Host is not a valid IP address - aborting")
+            return
+
+        dst = os.path.join(self.server.directory, host)
+
+        if action == "present":
+            shutil.copyfile(self.server.source, dst)
+        if action == "absent":
+            if os.path.exists(dst):
+                os.remove(dst)
+
+    def handler_cdist(self, action, host):
+        log.debug("Running cdist action %s for %s", action, host)
+
+        if self.server.dry_run:
+            log.info("Dry run, skipping cdist execution")
+            return
+
+        cname = action.title()
+        module = getattr(cdist, action)
+        theclass = getattr(module, cname)
+
+        if hasattr(self.cdistargs, 'out_path'):
+            out_path = self.cdistargs.out_path
+        else:
+            out_path = None
+        host_base_path, hostdir = theclass.create_host_base_dirs(
+            host, theclass.create_base_root_path(out_path))
+        theclass.construct_remote_exec_copy_patterns(self.cdistargs)
+        host_tags = None
+        host_name = ipaddr.resolve_target_host_name(host)
+        log.debug('Resolved target host name: %s', host_name)
+        if host_name:
+            target_host = host_name
+        else:
+            target_host = host
+        log.debug('Using target_host: %s', target_host)
+        log.debug("Executing cdist onehost with params: %s, %s, %s, %s, %s, ",
+                  target_host, host_tags, host_base_path, hostdir,
+                  self.cdistargs)
+        cfg = cdist.configuration.Configuration(self.cdistargs)
+        configuration = cfg.get_config(section='GLOBAL')
+        theclass.onehost(target_host, host_tags, host_base_path, hostdir,
+                         self.cdistargs, parallel=False,
+                         configuration=configuration)
+
+
+class HTTPServerV6(socketserver.ForkingMixIn, http.server.HTTPServer):
+    """
+    Server that listens to both IPv4 and IPv6 requests.
+    """
+    address_family = socket.AF_INET6
+
+    def __init__(self, cdistargs, directory, source, *args, **kwargs):
+        self.cdistargs = cdistargs
+        self.dry_run = cdistargs.dry_run
+        self.directory = directory
+        self.source = source
+
+        http.server.HTTPServer.__init__(self, *args, **kwargs)
+
+
+class HTTPServerV4(HTTPServerV6):
+    """
+    Server that listens to IPv4 requests.
+    """
+    address_family = socket.AF_INET

completions/bash/cdist-completion.bash

@@ -6,7 +6,7 @@ _cdist()
     prev="${COMP_WORDS[COMP_CWORD-1]}"
     prevprev="${COMP_WORDS[COMP_CWORD-2]}"
     opts="-h --help -q --quiet -v --verbose -V --version"
-    cmds="banner config install inventory preos shell"
+    cmds="banner config install inventory preos shell trigger"
 
     case "${prevprev}" in
         shell)
@@ -80,6 +80,14 @@ _cdist()
             COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
             return 0
             ;;
+        trigger)
+            opts="-h --help -d --debug -v --verbose -b --beta \
+                -C --cache-path-pattern -c --conf-dir -i --initial-manifest \
+                -j --jobs -n --dry-run -o --out-dir --remote-copy \
+                --remote-exec -6 --ipv6 -H --http-port -D --directory -S --source"
+            COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+            return 0
+            ;;
         inventory)
             cmds="list add-host del-host add-tag del-tag"
             opts="-h --help -q --quiet -v --verbose"

completions/zsh/_cdist

@@ -11,7 +11,7 @@ _cdist()
 
     case $state in
         opts_cmds)
-            _arguments '1:Options and commands:(banner config install inventory preos shell -h --help -q --quiet -v --verbose -V --version)'
+            _arguments '1:Options and commands:(banner config install inventory preos shell trigger -h --help -q --quiet -v --verbose -V --version)'
             ;;
         *)
             case $words[2] in

docs/changelog

@@ -2,50 +2,8 @@ Changelog
 ---------
 
 next:
-	* Type __group: Support OSes without getent (Dennis Camera)
-	* Type __user: Support OSes without getent (Dennis Camera)
-	* Type __ssh_authorized_keys: Support OSes without getent (Dennis Camera)
-	* Type __ssh_dot_ssh: Support OSes without getent (Dennis Camera)
-	* Explorer interfaces: Always sort output (Dennis Camera)
-	* Explorer os: Unquote value from os-release file (Dennis Camera)
-
-6.0.0: 2019-10-01
-	* Type __letsencrypt_cert: Fix beowulf's spelling (Mondi Ravi)
 	* Core: Add preos functionality (Darko Poljak)
-	* Type __sysctl: Add alpine support (Nico Schottelius)
-	* Type __git: Add recursive option (Dennis Camera)
-	* Type __apt_key: Fix key dearmor (Ander Punnar)
-
-5.1.3: 2019-08-30
-	* Build: Overcome bash CDPATH when building docs (Dmitry Bogatov)
-	* Type __grafana_dashboard: Update distribution name, package signing key URI and repository URI (Dominique Roux)
-	* Type __letsencrypt_cert: Add Devuan Beowulf support (Nico Schottelius)
-	* Type __letsencrypt_cert: Fix Devuan Ascii: support (Nico Schottelius)
-	* Type __docker: Add devuan support (Dominique Roux)
-	* Type __docker_swarm: Fix for Docker 19.03 (Ľubomír Kučera)
-
-5.1.2: 2019-06-21
-	* Core: Add support for type parameters deprecation (Darko Poljak)
-	* Type __acl: Rewrite and improve (Ander Punnar)
-
-5.1.1: 2019-05-28
-	* Type __apt_key: Use gpg key, fallback to deprecated apt-key (Ander Punnar)
-	* Type __acl: Fix and improve (Ander Punnar)
-	* Documentation: Document type stdin inside loop caveats (Darko Poljak)
-
-5.1.0: 2019-05-22
-	* Type __consul: Add alpine support (Nico Schottelius)
-	* Type __consul: Add version 1.5.0 (Nico Schottelius)
-	* Type __consul_agent: Add alpine support (Nico Schottelius)
-	* New helper script: cdist-new-type (Steven Armstrong, Darko Poljak)
-	* Core: Add support for deprecated type marker (Darko Poljak)
-
-5.0.2: 2019-05-17
-	* Type __package_apk: Fix @repo handling in explorer (Nico Schottelius)
-	* Type __postfix: Add alpine support (Nico Schottelius)
-	* Type __postfix_postconf: Add alpine support (Nico Schottelius)
-	* Type __user: Add alpine support (Nico Schottelius)
-	* Core: Set __cdist_dry_run env var (Ander Punnar)
+	* Core: Add trigger functionality (Nico Schottelius, Darko Poljak)
 
 5.0.1: 2019-05-09
 	* Documentation: Add 'Perils of CDIST_ORDER_DEPENDENCY' sub-section (Darko Poljak)

docs/dev/python-types/benchmark

@@ -0,0 +1,376 @@
+# sh type, no file at remote
+echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121125154.045799] 185.203.112.26: Starting configuration run
+INFO: [20181121125237.029892] 185.203.112.26: Processing __file/root/foo0.bin
+INFO: [20181121125239.881529] 185.203.112.26: Processing __file/root/foo1.bin
+INFO: [20181121125243.265672] 185.203.112.26: Processing __file/root/foo2.bin
+INFO: [20181121125246.929903] 185.203.112.26: Processing __file/root/foo3.bin
+INFO: [20181121125251.811258] 185.203.112.26: Processing __file/root/foo4.bin
+INFO: [20181121125257.784416] 185.203.112.26: Processing __file/root/foo5.bin
+INFO: [20181121125302.686275] 185.203.112.26: Processing __file/root/foo6.bin
+INFO: [20181121125306.394030] 185.203.112.26: Processing __file/root/foo7.bin
+INFO: [20181121125308.610023] 185.203.112.26: Processing __file/root/foo8.bin
+INFO: [20181121125310.868538] 185.203.112.26: Processing __file/root/foo9.bin
+INFO: [20181121125313.017972] 185.203.112.26: Processing __file/root/foo10.bin
+INFO: [20181121125315.201342] 185.203.112.26: Processing __file/root/foo11.bin
+INFO: [20181121125317.333055] 185.203.112.26: Processing __file/root/foo12.bin
+INFO: [20181121125319.463929] 185.203.112.26: Processing __file/root/foo13.bin
+INFO: [20181121125321.595410] 185.203.112.26: Processing __file/root/foo14.bin
+INFO: [20181121125323.689697] 185.203.112.26: Processing __file/root/foo15.bin
+INFO: [20181121125325.768283] 185.203.112.26: Processing __file/root/foo16.bin
+INFO: [20181121125327.814793] 185.203.112.26: Processing __file/root/foo17.bin
+INFO: [20181121125329.873073] 185.203.112.26: Processing __file/root/foo18.bin
+INFO: [20181121125331.953886] 185.203.112.26: Processing __file/root/foo19.bin
+INFO: [20181121125334.118290] 185.203.112.26: Processing __file/root/foo20.bin
+INFO: [20181121125336.390849] 185.203.112.26: Processing __file/root/foo21.bin
+INFO: [20181121125338.576698] 185.203.112.26: Processing __file/root/foo22.bin
+INFO: [20181121125340.819044] 185.203.112.26: Processing __file/root/foo23.bin
+INFO: [20181121125343.680419] 185.203.112.26: Processing __file/root/foo24.bin
+INFO: [20181121125346.044907] 185.203.112.26: Processing __file/root/foo25.bin
+INFO: [20181121125348.179574] 185.203.112.26: Processing __file/root/foo26.bin
+INFO: [20181121125350.314970] 185.203.112.26: Processing __file/root/foo27.bin
+INFO: [20181121125352.447394] 185.203.112.26: Processing __file/root/foo28.bin
+INFO: [20181121125354.586637] 185.203.112.26: Processing __file/root/foo29.bin
+INFO: [20181121125356.722699] 185.203.112.26: Processing __file/root/foo30.bin
+INFO: [20181121125358.883538] 185.203.112.26: Processing __file/root/foo31.bin
+INFO: [20181121125401.020967] 185.203.112.26: Processing __file/root/foo32.bin
+INFO: [20181121125403.160146] 185.203.112.26: Processing __file/root/foo33.bin
+INFO: [20181121125405.289048] 185.203.112.26: Processing __file/root/foo34.bin
+INFO: [20181121125407.423994] 185.203.112.26: Processing __file/root/foo35.bin
+INFO: [20181121125409.530135] 185.203.112.26: Processing __file/root/foo36.bin
+INFO: [20181121125411.659683] 185.203.112.26: Processing __file/root/foo37.bin
+INFO: [20181121125413.786177] 185.203.112.26: Processing __file/root/foo38.bin
+INFO: [20181121125415.919152] 185.203.112.26: Processing __file/root/foo39.bin
+INFO: [20181121125418.051496] 185.203.112.26: Processing __file/root/foo40.bin
+INFO: [20181121125420.204577] 185.203.112.26: Processing __file/root/foo41.bin
+INFO: [20181121125422.339697] 185.203.112.26: Processing __file/root/foo42.bin
+INFO: [20181121125424.450966] 185.203.112.26: Processing __file/root/foo43.bin
+INFO: [20181121125426.487831] 185.203.112.26: Processing __file/root/foo44.bin
+INFO: [20181121125428.585516] 185.203.112.26: Processing __file/root/foo45.bin
+INFO: [20181121125430.749002] 185.203.112.26: Processing __file/root/foo46.bin
+INFO: [20181121125432.865290] 185.203.112.26: Processing __file/root/foo47.bin
+INFO: [20181121125435.004009] 185.203.112.26: Processing __file/root/foo48.bin
+INFO: [20181121125437.228566] 185.203.112.26: Processing __file/root/foo49.bin
+INFO: [20181121125439.429440] 185.203.112.26: Finished successful run in 165.38 seconds
+
+# sh type, files exist at remote but content changes
+echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121125529.952800] 185.203.112.26: Starting configuration run
+INFO: [20181121125541.175180] 185.203.112.26: Processing __file/root/foo0.bin
+INFO: [20181121125543.219561] 185.203.112.26: Processing __file/root/foo1.bin
+INFO: [20181121125545.116073] 185.203.112.26: Processing __file/root/foo2.bin
+INFO: [20181121125547.011359] 185.203.112.26: Processing __file/root/foo3.bin
+INFO: [20181121125548.916288] 185.203.112.26: Processing __file/root/foo4.bin
+INFO: [20181121125550.821351] 185.203.112.26: Processing __file/root/foo5.bin
+INFO: [20181121125552.723887] 185.203.112.26: Processing __file/root/foo6.bin
+INFO: [20181121125554.635662] 185.203.112.26: Processing __file/root/foo7.bin
+INFO: [20181121125556.568639] 185.203.112.26: Processing __file/root/foo8.bin
+INFO: [20181121125558.508852] 185.203.112.26: Processing __file/root/foo9.bin
+INFO: [20181121125600.464475] 185.203.112.26: Processing __file/root/foo10.bin
+INFO: [20181121125602.429261] 185.203.112.26: Processing __file/root/foo11.bin
+INFO: [20181121125604.428942] 185.203.112.26: Processing __file/root/foo12.bin
+INFO: [20181121125606.442193] 185.203.112.26: Processing __file/root/foo13.bin
+INFO: [20181121125608.474473] 185.203.112.26: Processing __file/root/foo14.bin
+INFO: [20181121125610.535252] 185.203.112.26: Processing __file/root/foo15.bin
+INFO: [20181121125612.609560] 185.203.112.26: Processing __file/root/foo16.bin
+INFO: [20181121125614.708507] 185.203.112.26: Processing __file/root/foo17.bin
+INFO: [20181121125616.824721] 185.203.112.26: Processing __file/root/foo18.bin
+INFO: [20181121125618.924521] 185.203.112.26: Processing __file/root/foo19.bin
+INFO: [20181121125621.007543] 185.203.112.26: Processing __file/root/foo20.bin
+INFO: [20181121125623.133204] 185.203.112.26: Processing __file/root/foo21.bin
+INFO: [20181121125625.333471] 185.203.112.26: Processing __file/root/foo22.bin
+INFO: [20181121125627.396334] 185.203.112.26: Processing __file/root/foo23.bin
+INFO: [20181121125629.526492] 185.203.112.26: Processing __file/root/foo24.bin
+INFO: [20181121125631.628454] 185.203.112.26: Processing __file/root/foo25.bin
+INFO: [20181121125633.743142] 185.203.112.26: Processing __file/root/foo26.bin
+INFO: [20181121125635.952547] 185.203.112.26: Processing __file/root/foo27.bin
+INFO: [20181121125637.986746] 185.203.112.26: Processing __file/root/foo28.bin
+INFO: [20181121125640.020415] 185.203.112.26: Processing __file/root/foo29.bin
+INFO: [20181121125642.081373] 185.203.112.26: Processing __file/root/foo30.bin
+INFO: [20181121125644.174744] 185.203.112.26: Processing __file/root/foo31.bin
+INFO: [20181121125646.286532] 185.203.112.26: Processing __file/root/foo32.bin
+INFO: [20181121125648.396447] 185.203.112.26: Processing __file/root/foo33.bin
+INFO: [20181121125650.460107] 185.203.112.26: Processing __file/root/foo34.bin
+INFO: [20181121125652.557125] 185.203.112.26: Processing __file/root/foo35.bin
+INFO: [20181121125654.667456] 185.203.112.26: Processing __file/root/foo36.bin
+INFO: [20181121125656.746960] 185.203.112.26: Processing __file/root/foo37.bin
+INFO: [20181121125658.854229] 185.203.112.26: Processing __file/root/foo38.bin
+INFO: [20181121125700.968145] 185.203.112.26: Processing __file/root/foo39.bin
+INFO: [20181121125703.109376] 185.203.112.26: Processing __file/root/foo40.bin
+INFO: [20181121125705.318163] 185.203.112.26: Processing __file/root/foo41.bin
+INFO: [20181121125707.440575] 185.203.112.26: Processing __file/root/foo42.bin
+INFO: [20181121125709.551261] 185.203.112.26: Processing __file/root/foo43.bin
+INFO: [20181121125711.657753] 185.203.112.26: Processing __file/root/foo44.bin
+INFO: [20181121125713.774819] 185.203.112.26: Processing __file/root/foo45.bin
+INFO: [20181121125715.887428] 185.203.112.26: Processing __file/root/foo46.bin
+INFO: [20181121125717.995104] 185.203.112.26: Processing __file/root/foo47.bin
+INFO: [20181121125720.110196] 185.203.112.26: Processing __file/root/foo48.bin
+INFO: [20181121125722.232932] 185.203.112.26: Processing __file/root/foo49.bin
+INFO: [20181121125724.451523] 185.203.112.26: Finished successful run in 114.50 seconds
+
+# py type, no file at remote
+echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file_py /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121125812.034197] 185.203.112.26: Starting configuration run
+INFO: [20181121125823.927353] 185.203.112.26: Processing __file_py/root/foo0.bin
+INFO: [20181121125825.715361] 185.203.112.26: Processing __file_py/root/foo1.bin
+INFO: [20181121125827.511296] 185.203.112.26: Processing __file_py/root/foo2.bin
+INFO: [20181121125829.293455] 185.203.112.26: Processing __file_py/root/foo3.bin
+INFO: [20181121125831.086696] 185.203.112.26: Processing __file_py/root/foo4.bin
+INFO: [20181121125832.867582] 185.203.112.26: Processing __file_py/root/foo5.bin
+INFO: [20181121125834.652511] 185.203.112.26: Processing __file_py/root/foo6.bin
+INFO: [20181121125836.450393] 185.203.112.26: Processing __file_py/root/foo7.bin
+INFO: [20181121125838.255152] 185.203.112.26: Processing __file_py/root/foo8.bin
+INFO: [20181121125840.065808] 185.203.112.26: Processing __file_py/root/foo9.bin
+INFO: [20181121125841.889049] 185.203.112.26: Processing __file_py/root/foo10.bin
+INFO: [20181121125843.719280] 185.203.112.26: Processing __file_py/root/foo11.bin
+INFO: [20181121125845.560165] 185.203.112.26: Processing __file_py/root/foo12.bin
+INFO: [20181121125847.416138] 185.203.112.26: Processing __file_py/root/foo13.bin
+INFO: [20181121125849.289851] 185.203.112.26: Processing __file_py/root/foo14.bin
+INFO: [20181121125851.180203] 185.203.112.26: Processing __file_py/root/foo15.bin
+INFO: [20181121125853.074978] 185.203.112.26: Processing __file_py/root/foo16.bin
+INFO: [20181121125855.086107] 185.203.112.26: Processing __file_py/root/foo17.bin
+INFO: [20181121125857.041100] 185.203.112.26: Processing __file_py/root/foo18.bin
+INFO: [20181121125859.025581] 185.203.112.26: Processing __file_py/root/foo19.bin
+INFO: [20181121125901.072067] 185.203.112.26: Processing __file_py/root/foo20.bin
+INFO: [20181121125903.026711] 185.203.112.26: Processing __file_py/root/foo21.bin
+INFO: [20181121125904.994824] 185.203.112.26: Processing __file_py/root/foo22.bin
+INFO: [20181121125906.956296] 185.203.112.26: Processing __file_py/root/foo23.bin
+INFO: [20181121125908.929231] 185.203.112.26: Processing __file_py/root/foo24.bin
+INFO: [20181121125910.882672] 185.203.112.26: Processing __file_py/root/foo25.bin
+INFO: [20181121125912.839834] 185.203.112.26: Processing __file_py/root/foo26.bin
+INFO: [20181121125914.789904] 185.203.112.26: Processing __file_py/root/foo27.bin
+INFO: [20181121125916.743930] 185.203.112.26: Processing __file_py/root/foo28.bin
+INFO: [20181121125918.698258] 185.203.112.26: Processing __file_py/root/foo29.bin
+INFO: [20181121125920.657118] 185.203.112.26: Processing __file_py/root/foo30.bin
+INFO: [20181121125922.618898] 185.203.112.26: Processing __file_py/root/foo31.bin
+INFO: [20181121125924.567847] 185.203.112.26: Processing __file_py/root/foo32.bin
+INFO: [20181121125926.524617] 185.203.112.26: Processing __file_py/root/foo33.bin
+INFO: [20181121125928.396400] 185.203.112.26: Processing __file_py/root/foo34.bin
+INFO: [20181121125930.209237] 185.203.112.26: Processing __file_py/root/foo35.bin
+INFO: [20181121125931.998377] 185.203.112.26: Processing __file_py/root/foo36.bin
+INFO: [20181121125933.786883] 185.203.112.26: Processing __file_py/root/foo37.bin
+INFO: [20181121125935.579348] 185.203.112.26: Processing __file_py/root/foo38.bin
+INFO: [20181121125937.366197] 185.203.112.26: Processing __file_py/root/foo39.bin
+INFO: [20181121125939.155643] 185.203.112.26: Processing __file_py/root/foo40.bin
+INFO: [20181121125941.052837] 185.203.112.26: Processing __file_py/root/foo41.bin
+INFO: [20181121125942.953670] 185.203.112.26: Processing __file_py/root/foo42.bin
+INFO: [20181121125944.781567] 185.203.112.26: Processing __file_py/root/foo43.bin
+INFO: [20181121125946.622485] 185.203.112.26: Processing __file_py/root/foo44.bin
+INFO: [20181121125948.470701] 185.203.112.26: Processing __file_py/root/foo45.bin
+INFO: [20181121125950.356949] 185.203.112.26: Processing __file_py/root/foo46.bin
+INFO: [20181121125952.232014] 185.203.112.26: Processing __file_py/root/foo47.bin
+INFO: [20181121125954.128887] 185.203.112.26: Processing __file_py/root/foo48.bin
+INFO: [20181121125956.037541] 185.203.112.26: Processing __file_py/root/foo49.bin
+INFO: [20181121125957.514738] 185.203.112.26: Finished successful run in 105.48 seconds
+
+# py type, files exist at remote but content changes
+echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file_py /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121130056.484643] 185.203.112.26: Starting configuration run
+INFO: [20181121130108.545059] 185.203.112.26: Processing __file_py/root/foo0.bin
+INFO: [20181121130110.339217] 185.203.112.26: Processing __file_py/root/foo1.bin
+INFO: [20181121130112.136448] 185.203.112.26: Processing __file_py/root/foo2.bin
+INFO: [20181121130113.923820] 185.203.112.26: Processing __file_py/root/foo3.bin
+INFO: [20181121130115.715667] 185.203.112.26: Processing __file_py/root/foo4.bin
+INFO: [20181121130117.508696] 185.203.112.26: Processing __file_py/root/foo5.bin
+INFO: [20181121130119.300839] 185.203.112.26: Processing __file_py/root/foo6.bin
+INFO: [20181121130124.296312] 185.203.112.26: Processing __file_py/root/foo7.bin
+INFO: [20181121130131.109195] 185.203.112.26: Processing __file_py/root/foo8.bin
+INFO: [20181121130133.303817] 185.203.112.26: Processing __file_py/root/foo9.bin
+INFO: [20181121130136.396440] 185.203.112.26: Processing __file_py/root/foo10.bin
+INFO: [20181121130138.443128] 185.203.112.26: Processing __file_py/root/foo11.bin
+INFO: [20181121130140.462868] 185.203.112.26: Processing __file_py/root/foo12.bin
+INFO: [20181121130142.476196] 185.203.112.26: Processing __file_py/root/foo13.bin
+INFO: [20181121130145.937900] 185.203.112.26: Processing __file_py/root/foo14.bin
+INFO: [20181121130148.013672] 185.203.112.26: Processing __file_py/root/foo15.bin
+INFO: [20181121130150.042588] 185.203.112.26: Processing __file_py/root/foo16.bin
+INFO: [20181121130152.050793] 185.203.112.26: Processing __file_py/root/foo17.bin
+INFO: [20181121130154.083089] 185.203.112.26: Processing __file_py/root/foo18.bin
+INFO: [20181121130156.100091] 185.203.112.26: Processing __file_py/root/foo19.bin
+INFO: [20181121130158.103005] 185.203.112.26: Processing __file_py/root/foo20.bin
+INFO: [20181121130200.188390] 185.203.112.26: Processing __file_py/root/foo21.bin
+INFO: [20181121130202.197574] 185.203.112.26: Processing __file_py/root/foo22.bin
+INFO: [20181121130205.269102] 185.203.112.26: Processing __file_py/root/foo23.bin
+INFO: [20181121130208.457011] 185.203.112.26: Processing __file_py/root/foo24.bin
+INFO: [20181121130211.574321] 185.203.112.26: Processing __file_py/root/foo25.bin
+INFO: [20181121130213.719894] 185.203.112.26: Processing __file_py/root/foo26.bin
+INFO: [20181121130215.762977] 185.203.112.26: Processing __file_py/root/foo27.bin
+INFO: [20181121130217.778624] 185.203.112.26: Processing __file_py/root/foo28.bin
+INFO: [20181121130219.840477] 185.203.112.26: Processing __file_py/root/foo29.bin
+INFO: [20181121130221.852389] 185.203.112.26: Processing __file_py/root/foo30.bin
+INFO: [20181121130223.850898] 185.203.112.26: Processing __file_py/root/foo31.bin
+INFO: [20181121130225.858812] 185.203.112.26: Processing __file_py/root/foo32.bin
+INFO: [20181121130227.855295] 185.203.112.26: Processing __file_py/root/foo33.bin
+INFO: [20181121130229.952673] 185.203.112.26: Processing __file_py/root/foo34.bin
+INFO: [20181121130231.956904] 185.203.112.26: Processing __file_py/root/foo35.bin
+INFO: [20181121130233.961954] 185.203.112.26: Processing __file_py/root/foo36.bin
+INFO: [20181121130236.012158] 185.203.112.26: Processing __file_py/root/foo37.bin
+INFO: [20181121130238.024422] 185.203.112.26: Processing __file_py/root/foo38.bin
+INFO: [20181121130241.238800] 185.203.112.26: Processing __file_py/root/foo39.bin
+INFO: [20181121130243.463237] 185.203.112.26: Processing __file_py/root/foo40.bin
+INFO: [20181121130245.610314] 185.203.112.26: Processing __file_py/root/foo41.bin
+INFO: [20181121130247.661385] 185.203.112.26: Processing __file_py/root/foo42.bin
+INFO: [20181121130250.399845] 185.203.112.26: Processing __file_py/root/foo43.bin
+INFO: [20181121130252.832133] 185.203.112.26: Processing __file_py/root/foo44.bin
+INFO: [20181121130254.955658] 185.203.112.26: Processing __file_py/root/foo45.bin
+INFO: [20181121130257.039587] 185.203.112.26: Processing __file_py/root/foo46.bin
+INFO: [20181121130259.178847] 185.203.112.26: Processing __file_py/root/foo47.bin
+INFO: [20181121130301.357922] 185.203.112.26: Processing __file_py/root/foo48.bin
+INFO: [20181121130303.356299] 185.203.112.26: Processing __file_py/root/foo49.bin
+INFO: [20181121130305.144393] 185.203.112.26: Finished successful run in 128.66 seconds
+
+
+
+# init test file content
+head -c 102400 /dev/random > /tmp/test.file
+
+# sh type, no file at remote
+echo 'x=0; while [ $x -lt 50 ]; do __file /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121130612.519698] 185.203.112.26: Starting configuration run
+INFO: [20181121130624.219344] 185.203.112.26: Processing __file/root/foo0.bin
+INFO: [20181121130626.980052] 185.203.112.26: Processing __file/root/foo1.bin
+INFO: [20181121130631.200669] 185.203.112.26: Processing __file/root/foo2.bin
+INFO: [20181121130642.790229] 185.203.112.26: Processing __file/root/foo3.bin
+INFO: [20181121130646.565599] 185.203.112.26: Processing __file/root/foo4.bin
+INFO: [20181121130648.724875] 185.203.112.26: Processing __file/root/foo5.bin
+INFO: [20181121130651.464686] 185.203.112.26: Processing __file/root/foo6.bin
+INFO: [20181121130653.639581] 185.203.112.26: Processing __file/root/foo7.bin
+INFO: [20181121130655.773987] 185.203.112.26: Processing __file/root/foo8.bin
+INFO: [20181121130657.933136] 185.203.112.26: Processing __file/root/foo9.bin
+INFO: [20181121130700.065158] 185.203.112.26: Processing __file/root/foo10.bin
+INFO: [20181121130702.216456] 185.203.112.26: Processing __file/root/foo11.bin
+INFO: [20181121130704.429030] 185.203.112.26: Processing __file/root/foo12.bin
+INFO: [20181121130706.562114] 185.203.112.26: Processing __file/root/foo13.bin
+INFO: [20181121130708.696584] 185.203.112.26: Processing __file/root/foo14.bin
+INFO: [20181121130710.830002] 185.203.112.26: Processing __file/root/foo15.bin
+INFO: [20181121130712.966631] 185.203.112.26: Processing __file/root/foo16.bin
+INFO: [20181121130715.151833] 185.203.112.26: Processing __file/root/foo17.bin
+INFO: [20181121130717.355196] 185.203.112.26: Processing __file/root/foo18.bin
+INFO: [20181121130719.486316] 185.203.112.26: Processing __file/root/foo19.bin
+INFO: [20181121130721.619933] 185.203.112.26: Processing __file/root/foo20.bin
+INFO: [20181121130723.786670] 185.203.112.26: Processing __file/root/foo21.bin
+INFO: [20181121130725.924736] 185.203.112.26: Processing __file/root/foo22.bin
+INFO: [20181121130728.060224] 185.203.112.26: Processing __file/root/foo23.bin
+INFO: [20181121130730.178729] 185.203.112.26: Processing __file/root/foo24.bin
+INFO: [20181121130732.309264] 185.203.112.26: Processing __file/root/foo25.bin
+INFO: [20181121130734.479895] 185.203.112.26: Processing __file/root/foo26.bin
+INFO: [20181121130736.653085] 185.203.112.26: Processing __file/root/foo27.bin
+INFO: [20181121130738.814291] 185.203.112.26: Processing __file/root/foo28.bin
+INFO: [20181121130741.029646] 185.203.112.26: Processing __file/root/foo29.bin
+INFO: [20181121130743.128717] 185.203.112.26: Processing __file/root/foo30.bin
+INFO: [20181121130745.233272] 185.203.112.26: Processing __file/root/foo31.bin
+INFO: [20181121130747.364681] 185.203.112.26: Processing __file/root/foo32.bin
+INFO: [20181121130749.491793] 185.203.112.26: Processing __file/root/foo33.bin
+INFO: [20181121130751.620492] 185.203.112.26: Processing __file/root/foo34.bin
+INFO: [20181121130753.743519] 185.203.112.26: Processing __file/root/foo35.bin
+INFO: [20181121130755.862169] 185.203.112.26: Processing __file/root/foo36.bin
+INFO: [20181121130758.000172] 185.203.112.26: Processing __file/root/foo37.bin
+INFO: [20181121130800.090405] 185.203.112.26: Processing __file/root/foo38.bin
+INFO: [20181121130802.211849] 185.203.112.26: Processing __file/root/foo39.bin
+INFO: [20181121130804.356363] 185.203.112.26: Processing __file/root/foo40.bin
+INFO: [20181121130806.548412] 185.203.112.26: Processing __file/root/foo41.bin
+INFO: [20181121130808.671279] 185.203.112.26: Processing __file/root/foo42.bin
+INFO: [20181121130810.752813] 185.203.112.26: Processing __file/root/foo43.bin
+INFO: [20181121130812.844502] 185.203.112.26: Processing __file/root/foo44.bin
+INFO: [20181121130814.950501] 185.203.112.26: Processing __file/root/foo45.bin
+INFO: [20181121130817.040587] 185.203.112.26: Processing __file/root/foo46.bin
+INFO: [20181121130819.175850] 185.203.112.26: Processing __file/root/foo47.bin
+INFO: [20181121130821.332900] 185.203.112.26: Processing __file/root/foo48.bin
+INFO: [20181121130823.543119] 185.203.112.26: Processing __file/root/foo49.bin
+INFO: [20181121130825.833163] 185.203.112.26: Finished successful run in 133.31 seconds
+
+# sh type, files exist at remote
+echo 'x=0; while [ $x -lt 50 ]; do __file /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121130854.980007] 185.203.112.26: Starting configuration run
+INFO: [20181121130957.927705] 185.203.112.26: Finished successful run in 62.95 seconds
+
+# py type, no file at remote
+echo 'x=0; while [ $x -lt 50 ]; do __file_py /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121131110.179480] 185.203.112.26: Starting configuration run
+INFO: [20181121131122.086849] 185.203.112.26: Processing __file_py/root/foo0.bin
+INFO: [20181121131123.876029] 185.203.112.26: Processing __file_py/root/foo1.bin
+INFO: [20181121131125.668664] 185.203.112.26: Processing __file_py/root/foo2.bin
+INFO: [20181121131127.460721] 185.203.112.26: Processing __file_py/root/foo3.bin
+INFO: [20181121131129.591229] 185.203.112.26: Processing __file_py/root/foo4.bin
+INFO: [20181121131131.390379] 185.203.112.26: Processing __file_py/root/foo5.bin
+INFO: [20181121131133.195275] 185.203.112.26: Processing __file_py/root/foo6.bin
+INFO: [20181121131135.006282] 185.203.112.26: Processing __file_py/root/foo7.bin
+INFO: [20181121131136.834448] 185.203.112.26: Processing __file_py/root/foo8.bin
+INFO: [20181121131138.659301] 185.203.112.26: Processing __file_py/root/foo9.bin
+INFO: [20181121131140.496856] 185.203.112.26: Processing __file_py/root/foo10.bin
+INFO: [20181121131142.367813] 185.203.112.26: Processing __file_py/root/foo11.bin
+INFO: [20181121131144.239817] 185.203.112.26: Processing __file_py/root/foo12.bin
+INFO: [20181121131146.133314] 185.203.112.26: Processing __file_py/root/foo13.bin
+INFO: [20181121131148.049380] 185.203.112.26: Processing __file_py/root/foo14.bin
+INFO: [20181121131149.974696] 185.203.112.26: Processing __file_py/root/foo15.bin
+INFO: [20181121131151.929083] 185.203.112.26: Processing __file_py/root/foo16.bin
+INFO: [20181121131153.923590] 185.203.112.26: Processing __file_py/root/foo17.bin
+INFO: [20181121131155.874910] 185.203.112.26: Processing __file_py/root/foo18.bin
+INFO: [20181121131157.857904] 185.203.112.26: Processing __file_py/root/foo19.bin
+INFO: [20181121131159.902006] 185.203.112.26: Processing __file_py/root/foo20.bin
+INFO: [20181121131201.859840] 185.203.112.26: Processing __file_py/root/foo21.bin
+INFO: [20181121131203.810875] 185.203.112.26: Processing __file_py/root/foo22.bin
+INFO: [20181121131205.763291] 185.203.112.26: Processing __file_py/root/foo23.bin
+INFO: [20181121131207.710932] 185.203.112.26: Processing __file_py/root/foo24.bin
+INFO: [20181121131209.658154] 185.203.112.26: Processing __file_py/root/foo25.bin
+INFO: [20181121131211.615374] 185.203.112.26: Processing __file_py/root/foo26.bin
+INFO: [20181121131213.569721] 185.203.112.26: Processing __file_py/root/foo27.bin
+INFO: [20181121131215.522624] 185.203.112.26: Processing __file_py/root/foo28.bin
+INFO: [20181121131217.471128] 185.203.112.26: Processing __file_py/root/foo29.bin
+INFO: [20181121131219.421712] 185.203.112.26: Processing __file_py/root/foo30.bin
+INFO: [20181121131221.375699] 185.203.112.26: Processing __file_py/root/foo31.bin
+INFO: [20181121131223.327672] 185.203.112.26: Processing __file_py/root/foo32.bin
+INFO: [20181121131225.281373] 185.203.112.26: Processing __file_py/root/foo33.bin
+INFO: [20181121131227.256711] 185.203.112.26: Processing __file_py/root/foo34.bin
+INFO: [20181121131229.209255] 185.203.112.26: Processing __file_py/root/foo35.bin
+INFO: [20181121131231.170170] 185.203.112.26: Processing __file_py/root/foo36.bin
+INFO: [20181121131233.123407] 185.203.112.26: Processing __file_py/root/foo37.bin
+INFO: [20181121131235.077713] 185.203.112.26: Processing __file_py/root/foo38.bin
+INFO: [20181121131237.017138] 185.203.112.26: Processing __file_py/root/foo39.bin
+INFO: [20181121131238.988189] 185.203.112.26: Processing __file_py/root/foo40.bin
+INFO: [20181121131241.026849] 185.203.112.26: Processing __file_py/root/foo41.bin
+INFO: [20181121131242.978335] 185.203.112.26: Processing __file_py/root/foo42.bin
+INFO: [20181121131244.934562] 185.203.112.26: Processing __file_py/root/foo43.bin
+INFO: [20181121131246.885320] 185.203.112.26: Processing __file_py/root/foo44.bin
+INFO: [20181121131248.835008] 185.203.112.26: Processing __file_py/root/foo45.bin
+INFO: [20181121131250.789727] 185.203.112.26: Processing __file_py/root/foo46.bin
+INFO: [20181121131252.738686] 185.203.112.26: Processing __file_py/root/foo47.bin
+INFO: [20181121131254.691465] 185.203.112.26: Processing __file_py/root/foo48.bin
+INFO: [20181121131256.640896] 185.203.112.26: Processing __file_py/root/foo49.bin
+INFO: [20181121131258.194372] 185.203.112.26: Finished successful run in 108.01 seconds
+
+# py type, files exist at remote
+echo 'x=0; while [ $x -lt 50 ]; do __file_py /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121131327.054523] 185.203.112.26: Starting configuration run
+INFO: [20181121131428.031761] 185.203.112.26: Finished successful run in 60.98 seconds
+
+
+# Summary
+
+# sh type, no file at remote
+INFO: [20181121125439.429440] 185.203.112.26: Finished successful run in 165.38 seconds
+# py type, no file at remote
+INFO: [20181121125957.514738] 185.203.112.26: Finished successful run in 105.48 seconds
+
+# sh type, files exist at remote but content changes
+INFO: [20181121125724.451523] 185.203.112.26: Finished successful run in 114.50 seconds
+# py type, files exist at remote but content changes
+INFO: [20181121130305.144393] 185.203.112.26: Finished successful run in 128.66 seconds
+
+
+# sh type, no file at remote
+INFO: [20181121130825.833163] 185.203.112.26: Finished successful run in 133.31 seconds
+# py type, no file at remote
+INFO: [20181121131258.194372] 185.203.112.26: Finished successful run in 108.01 seconds
+
+# sh type, files exist at remote
+INFO: [20181121130957.927705] 185.203.112.26: Finished successful run in 62.95 seconds
+# py type, files exist at remote
+INFO: [20181121131428.031761] 185.203.112.26: Finished successful run in 60.98 seconds

docs/dev/python-types/benchmark.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+# Addapt to your env.
+CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf"
+export CDIST_PATH
+TARGET_HOST=185.203.112.26
+
+if [ $# -eq 0 ]
+then
+    N=1
+else
+    N=$1
+fi
+
+manifest() {
+    bytes=$(echo "$1 * 1024" | bc)
+    echo "head -c ${bytes} /dev/random | __file$2 /root/foo$3.bin --source - --mode 0640 --owner root --group root"
+}
+
+verbosity="-vv" #"-vvv"
+i=0
+while [ "$i" -lt "$N" ]
+do
+    if [ "$N" -ne 1 ]
+    then
+        printf "iteration %d\\n" "$i"
+    fi
+    printf "shinit clean state...\\n"
+    ssh root@${TARGET_HOST} 'rm foo$i.bin;'
+    manifest 50 "" $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
+
+    printf "pyinit clean state...\\n"
+    ssh root@${TARGET_HOST} 'rm foo$i.bin;'
+    manifest 50 '_py' $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
+
+    printf "shinit present state...\\n"
+    manifest 50 "" $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
+
+    printf "pyinit present state...\\n"
+    manifest 50 '_py' $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
+
+    i=$((i + 1))
+done

docs/dev/python-types/conf/manifest/pyinit

@@ -0,0 +1,7 @@
+#for x in 1; do
+#    echo xxx${x} | __file_py /root/foobar${x} --source - --mode 0640 --owner root --group root;
+#done
+#__dummy_config_py test1
+
+echo xxx | __file_py /root/foobar --source - --mode 0640 --owner root --group root
+__dummy_config_py test1

docs/dev/python-types/conf/manifest/shinit

@@ -0,0 +1,7 @@
+#for x in 1; do
+#    echo xxx${x} | __file /root/foobar${x} --source - --mode 0640 --owner root --group root;
+#done
+#__dummy_config_sh test1
+
+echo xxx | __file /root/foobar --source - --mode 0640 --owner root --group root
+__dummy_config_sh test1

docs/dev/python-types/conf/type/__dummy_config/__init__.py

@@ -0,0 +1,52 @@
+import os
+import sys
+from cdist.core import PythonType, ManifestEntry
+
+
+class DummyConfig(PythonType):
+    def type_manifest(self):
+        print('dummy py manifest stdout')
+        print('dummy py manifest stderr', file=sys.stderr)
+        filepy = ManifestEntry(name='__file_py', stdin='dummy=py\n',
+                               parameters={
+                                    '/root/dummypy.conf': None,
+                                    '--mode': '0640',
+                                    '--owner': 'root',
+                                    '--group': 'root',
+                                    '--source': '-',
+                               })
+        self.log.info('Created manifest entry %s', filepy)
+        yield filepy
+
+        self_path = os.path.dirname(os.path.realpath(__file__))
+        conf_path = os.path.join(self_path, 'files', 'dummypy.conf')
+        filepy = ManifestEntry(name='__file_py',
+                               parameters={
+                                    '/root/dummypy2.conf': None,
+                                    '--mode': '0640',
+                                    '--owner': 'root',
+                                    '--group': 'root',
+                                    '--source': conf_path,
+                               })
+        yield filepy
+
+        self_path = os.path.dirname(os.path.realpath(__file__))
+        conf_path = os.path.join(self_path, 'files', 'dummysh.conf')
+        with open(conf_path, 'r') as f:
+            filepy = ManifestEntry(name='__file', stdin=f,
+                                   parameters={
+                                        '/root/dummysh.conf': None,
+                                        '--mode': '0600',
+                                        '--owner': 'root',
+                                        '--group': 'root',
+                                        '--source': '-',
+                                   })
+            yield filepy
+
+    def type_gencode(self):
+        print('__dummy_config test stdout')
+        print('__dummy_config test stderr', file=sys.stderr)
+        pattern = "__file_py/root/dummypy2.conf:chgrp 'root'"
+        match = self.receive_message(pattern)
+        print('Received message:', match.string if match else None)
+        return None

docs/dev/python-types/conf/type/__dummy_config/files/dummypy.conf

@@ -0,0 +1 @@
+dummy=py2

docs/dev/python-types/conf/type/__dummy_config/files/dummysh.conf

@@ -0,0 +1 @@
+dummy=sh

docs/dev/python-types/conf/type/__dummy_config_py/__init__.py

@@ -0,0 +1,30 @@
+import os
+import sys
+from cdist.core import PythonType, ManifestEntry
+
+
+class DummyConfig(PythonType):
+    def type_manifest(self):
+        print('dummy manifest stdout')
+        print('dummy manifest stderr\n', file=sys.stderr)
+        filepy = ManifestEntry(name='__file_py', stdin='dummy=1\n',
+                               parameters={
+                                    '/root/dummy1.conf': None,
+                                    '--mode': '0640',
+                                    '--owner': 'root',
+                                    '--group': 'root',
+                                    '--source': '-',
+                               })
+        yield filepy
+
+        self_path = os.path.dirname(os.path.realpath(__file__))
+        conf_path = os.path.join(self_path, 'files', 'dummy.conf')
+        filepy = ManifestEntry(name='__file_py',
+                               parameters={
+                                    '/root/dummy2.conf': None,
+                                    '--mode': '0600',
+                                    '--owner': 'root',
+                                    '--group': 'root',
+                                    '--source': conf_path,
+                               })
+        yield filepy

docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf

@@ -0,0 +1 @@
+dummy=2

docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf

@@ -0,0 +1 @@
+dummy=2

docs/dev/python-types/conf/type/__dummy_config_sh/manifest

@@ -0,0 +1,6 @@
+printf 'dummy manifest stdout\n'
+printf 'dummy manifest stderr\n' >&2
+
+printf "dummy=1\\n" | __file /root/dummy1.conf --mode 0640 --owner root --group root --source -
+
+__file /root/dummy2.conf --mode 0600 --owner root --group root --source  "$__type/files/dummy.conf"

docs/dev/python-types/test.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# Addapt to your env.
+CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf"
+export CDIST_PATH
+TARGET_HOST=185.203.112.26
+env | grep CDIST_PATH
+
+for streams in ' ' '-S'
+do
+    for x in sh py
+    do
+        printf "[%s] Removing old foobar* files\\n" "$x"
+        printf -- "----------------\\n"
+        ssh root@${TARGET_HOST} 'rm foobar*; rm dummy*'
+        printf "[%s] Listing foobar* files\\n" "$x"
+        printf -- "----------------\\n"
+        ssh root@${TARGET_HOST} 'ls foobar* dummy*'
+        printf "[%s] Running cdist config, streams: %s\\n" "$x" "$streams"
+        printf -- "----------------\\n"
+        ./bin/cdist config -P ${streams} -v -i ./docs/dev/python-types/conf/manifest/${x}init  -- ${TARGET_HOST}
+        printf "[%s] Listing foobar* files\\n" "$x"
+        printf -- "----------------\\n"
+        ssh root@${TARGET_HOST} 'ls foobar* dummy*'
+        ./bin/cdist config -P ${streams} -v -i ./docs/dev/python-types/conf/manifest/${x}init  -- ${TARGET_HOST}
+    done
+done

docs/dev/python-types/timeit.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+# Addapt to your env.
+CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf"
+export CDIST_PATH
+TARGET_HOST=185.203.112.26
+
+if [ $# -eq 0 ]
+then
+    N=1
+else
+    N=$1
+fi
+
+i=0
+while [ "$i" -lt "$N" ]
+do
+    if [ "$N" -ne 1 ]
+    then
+        printf "iteration %d\\n" "$i"
+    fi
+    printf "shinit clean state...\\n"
+    ssh root@${TARGET_HOST} 'rm foobar*; rm dummy*;'
+
+    time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/shinit ${TARGET_HOST}
+    printf "pyinit clean state...\\n"
+    ssh root@$${TARGET_HOST} 'rm foobar*; rm dummy*;'
+    time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/pyinit ${TARGET_HOST}
+
+    printf "shinit present state...\\n"
+    time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/shinit ${TARGET_HOST}
+
+    printf "pyinit present state...\\n"
+    time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/pyinit ${TARGET_HOST}
+    i=$((i + 1))
+done

docs/src/cdist-preos.rst

@@ -24,14 +24,15 @@ For example, to create ubuntu PreOS:
 
 .. code-block:: sh
 
-    $ cdist preos ubuntu /preos/preos-ubuntu -B -C \
-        -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu
+    $ cdist preos ubuntu /preos/preos-ubuntu -b -C \
+        -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \
+        -t "/usr/bin/curl 192.168.111.5:3000/cdist/install/"
 
 For more info about available options see cdist manual page.
 
-This will bootstrap (``-B``) ubuntu PreOS in ``/preos/preos-ubuntu`` directory, it
+This will bootstrap (``-b``) ubuntu PreOS in ``/preos/preos-ubuntu`` directory, it
 will be configured (``-C``) using default built-in initial manifest and with
-specified ssh authorized key (``-k``).
+specified ssh authorized key (``-k``) and with specified trigger command (``-t``).
 After bootstrapping and configuration PXE
 boot directory will be created (``-p``) in ``/preos/pxe-ubuntu``.
 
@@ -41,6 +42,15 @@ proper dhcp, tftp setting).
 Since PreOS is configured with ssh authorized key it can be accessed throguh
 ssh, i.e. it can be further installed and configured with cdist.
 
+When installing and configuring new machines using cdist's PreOS concept
+cdist can use triggering for host installation/configuration, which is described
+in the previous chapter.
+
+When new machine is booted with PreOS then trigger command is executed.
+Machine will connect to cdist trigger server. If the request is, for example,
+for installation then cdist trigger server will start install command for the
+client host using parameters specified at trigger server startup.
+
 Implementing new PreOS sub-command
 ----------------------------------
 preos command is implemented as a plugin system. This plugin system scans for
@@ -135,3 +145,32 @@ When you try to run this new preos you will get:
 
 In the ``commandline`` function/method you have all the freedom to actually create
 PreOS.
+
+Simple tipical use case for using PreOS and trigger
+---------------------------------------------------
+Tipical use case for using PreOS and trigger command include the following steps.
+
+#. Create PreOS PXE with ssh key and trigger command for installation.
+
+    .. code-block:: sh
+
+        $ cdist preos ubuntu /preos/ubuntu -b -C \
+            -k ~/.ssh/id_rsa.pub -p /preos/pxe \
+            -t "/usr/bin/curl 192.168.111.5:3000/cdist/install/"
+
+#. Configure dhcp server and tftp server.
+
+#. On cdist host (192.168.111.5 from above) start trigger command (it will use
+   default init manifest for installation).
+
+    .. code-block:: sh
+
+        $ cdist trigger -b -v
+
+#. After all is set up start new machines (PXE boot).
+
+#. New machine boots and executes trigger command, i.e. triggers installation.
+
+#. Cdist trigger server starts installing host that has triggered it.
+
+#. After cdist install is finished new host is installed.

docs/src/cdist-reference.rst.sh

@@ -222,10 +222,6 @@ __cdist_log_level, __cdist_log_level_name
     | TRACE          | 5               |
     +----------------+-----------------+
 
-    Available for: initial manifest, explorer, type manifest, type explorer,
-    type gencode.
-__cdist_dry_run
-    Is set only when doing dry run (``-n`` flag).
     Available for: initial manifest, explorer, type manifest, type explorer,
     type gencode.
 __explorer

docs/src/cdist-trigger.rst

@@ -0,0 +1,35 @@
+Trigger
+=======
+
+Description
+-----------
+cdist supports triggering for host installation/configuration using trigger command.
+This command starts trigger server at management node, for example:
+
+.. code-block:: sh
+
+    $ cdist trigger -b -v
+
+This will start cdist trigger server in verbose mode. cdist trigger server accepts
+simple requests for configuration and for installation:
+
+* :strong:`/cdist/install/` for installation
+* :strong:`/cdist/config/` for configuration.
+
+Other configuration parameters are the same as in like cdist config (See `cdist <man1/cdist.html>`_).
+
+Machines can then trigger cdist trigger server with appropriate requests.
+If the request is, for example, for installation (:strong:`/cdist/install/`)
+then cdist trigger server will start install command for the client host using
+parameters specified at trigger server startup. For the above example that means
+that client will be installed using default initial manifest.
+
+When triggered cdist will try to reverse DNS lookup for host name and if
+host name is dervied then it is used for running cdist config. If no
+host name is resolved then IP address is used.
+
+This command returns the following response codes to client requests:
+
+* 200 for success
+* 599 for cdist run errors
+* 500 for cdist/server errors.

docs/src/cdist-troubleshooting.rst

@@ -47,8 +47,7 @@ you write to use the -e flag:
 Using debug dump helper script
 ------------------------------
 Since cdist stores data to local cache that can be used for debugging there
-is a helper script that dumps data from local cache,
-`cdist-dump <man1/cdist-dump.html>`_.
+is a helper script that dumps data from local cache.
 
 For more info see:
 

docs/src/cdist-type.rst

@@ -71,31 +71,6 @@ when using -j option. Example of such a type is __package_dpkg type where dpkg i
 prevents to be run in more than one instance.
 
 
-Deprecated types
------------------
-If a type is flagged with 'deprecated' marker then it is considered deprecated.
-When it is used cdist writes warning line. If 'deprecated' marker has content
-then this content is printed as a deprecation messages, e.g.:
-
-.. code-block:: sh
-
-    $ ls -l deprecated 
-    -rw-r--r--  1 darko  darko  71 May 20 18:30 deprecated
-    $ cat deprecated 
-    This type is deprecated. It will be removed in the next minor release.
-    $ echo '__foo foo' | ./bin/cdist config -i - 185.203.112.26
-    WARNING: 185.203.112.26: Type __foo is deprecated: This type is deprecated. It will be removed in the next minor release.
-
-If 'deprecated' marker has no content then general message is printed, e.g.:
-
-.. code-block:: sh
-
-    $ ls -l deprecated 
-    -rw-r--r--  1 darko  darko  0 May 20 18:36 deprecated
-    $ echo '__bar foo' | ./bin/cdist config -i - 185.203.112.26
-    WARNING: 185.203.112.26: Type __bar is deprecated.
-
-
 How to write a new type
 -----------------------
 A type consists of
@@ -118,9 +93,6 @@ they are written in shell so they are executed using '/bin/sh -e' or 'CDIST_LOCA
 
 For executable shell code it is suggested that shebang is '#!/bin/sh -e'.
 
-For creating type skeleton you can use helper script
-`cdist-new-type <man1/cdist-new-type.html>`_.
-
 
 Defining parameters
 -------------------
@@ -186,31 +158,6 @@ Example: (e.g. in cdist/conf/type/__nginx_vhost/manifest)
     fi
 
 
-Deprecated parameters
----------------------
-To deprecate type parameters one can declare a file for each deprecated
-parameter under **parameter/deprecated** directory.
-
-When such parameter is used cdist writes warning line with deprecation message.
-If such file has content then this content is printed as deprecation message.
-If there is no content then generic parameter deprecation message is printed.
-
-Example:
-
-.. code-block:: sh
-
-    $ ls parameter/deprecated/
-    eggs    spam
-    $ cat parameter/deprecated/eggs
-    eggs parameter is deprecated, please use multiple egg parameter.
-    $ cat parameter/deprecated/spam
-    $ echo '__foo foo --foo foo --eggs eggs' | ./bin/cdist config -i - 185.203.112.26
-    WARNING: 185.203.112.26: eggs parameter of type __foo is deprecated: eggs parameter is deprecated, please use multiple egg parameter.
-    $ echo '__foo foo --foo foo --eggs eggs --spam spam' | ./bin/cdist config -i - 185.203.112.26
-    WARNING: 185.203.112.26: spam parameter of type __foo is deprecated.
-    WARNING: 185.203.112.26: eggs parameter of type __foo is deprecated: eggs parameter is deprecated, please use multiple egg parameter.
-
-
 Input from stdin
 ----------------
 Every type can access what has been written on stdin when it has been called.
@@ -241,73 +188,6 @@ In the __file type, stdin is used as source for the file, if - is used for sourc
     ....
 
 
-Stdin inside a loop
-~~~~~~~~~~~~~~~~~~~
-Since cdist saves type's stdin content in the object as **$__object/stdin**,
-so it can be accessed in manifest and gencode-* scripts, this can lead to
-unexpected behavior. For example, suppose you have some type with the following
-in its manifest:
-
-.. code-block:: sh
-
-    if [ -f "$__object/parameter/foo" ]
-    then
-        while read -r l
-        do
-            __file "$l"
-            echo "$l" >&2
-        done < "$__object/parameter/foo"
-    fi
-
-and init manifest:
-
-.. code-block:: sh
-
-    __foo foo --foo a --foo b --foo c
-
-You expect that manifest stderr content is:
-
-.. code-block:: sh
-
-    a
-    b
-    c
-
-and that files *a*, *b* and *c* are created. But all you get in manifest stderr
-is:
-
-.. code-block:: sh
-
-    a
-
-and only *a* file is created.
-
-When redirecting parameter *foo* file content to while's stdin that means that all
-commands in while body have this same stdin. So when *__file* type gets executed,
-cdist saves its stdin which means it gets the remaining content of parameter *foo*
-file, i.e.:
-
-.. code-block:: sh
-
-    b
-    c
-
-The solution is to make sure that your types inside such loops get their stdin
-from somewhere else, e.g. for the above problem *__file* type can get empty
-stdin from */dev/null*:
-
-.. code-block:: sh
-
-    if [ -f "$__object/parameter/foo" ]
-    then
-        while read -r l
-        do
-            __file "$l" < /dev/null
-            echo "$l" >&2
-        done < "$__object/parameter/foo"
-    fi
-
-
 Writing the manifest
 --------------------
 In the manifest of a type you can use other types, so your type extends
@@ -491,15 +371,6 @@ It is available for initial manifest, explorer, type manifest,
 type explorer, type gencode.
 
 
-Detecting dry run
------------------
-
-If ``$__cdist_dry_run`` environment variable is set, then it's dry run.
-
-It is available for initial manifest, explorer, type manifest,
-type explorer, type gencode.
-
-
 Hints for typewriters
 ----------------------
 It must be assumed that the target is pretty dumb and thus does not have high

docs/src/index.rst

@@ -22,7 +22,6 @@ is being used in small up to enterprise grade environments.
    cdist-real-world
    man1/cdist
    man1/cdist-dump
-   man1/cdist-new-type
    cdist-bootstrap
    cdist-configuration
    cdist-manifest
@@ -32,6 +31,7 @@ is being used in small up to enterprise grade environments.
    cdist-messaging
    cdist-parallelization
    cdist-inventory
+   cdist-trigger
    cdist-preos
    cdist-integration
    cdist-reference

docs/src/man1/cdist-new-type.rst

@@ -1,74 +0,0 @@
-cdist-new-type(1)
-=================
-
-NAME
-----
-cdist-new-type - Create new type skeleton
-
-
-SYNOPSIS
---------
-
-::
-
-    cdist-new-type TYPE-NAME AUTHOR-NAME AUTHOR-EMAIL [TYPE-BASE-PATH]
-
-
-
-DESCRIPTION
------------
-cdist-new-type is a helper script that creates new type skeleton.
-It is then up to the type author to finish the type.
-
-It creates skeletons for the following files:
-
-* man.rst
-* manifest
-* gencode-remote.
-
-Upon creation it prints the path to the newly created type directory.
-
-
-ARGUMENTS
----------
-**TYPE-NAME**
-   Name of the new type.
-
-**AUTHOR-NAME**
-   Type author's full name.
-
-**AUTHOR-NAME**
-   Type author's email.
-
-**TYPE-BASE-PATH**
-    Path to the base directory of the type. If not set it defaults
-    to '$PWD/type'.
-
-
-EXAMPLES
---------
-
-.. code-block:: sh
-
-    # Create new type __foo in ~/.cdist directory.
-    $ cd ~/.cdist
-    $ cdist-new-type '__foo' 'Foo Bar' 'foo.bar at foobar.org'
-    /home/foo/.cdist/type/__foo
-
-
-SEE ALSO
---------
-:strong:`cdist`\ (1)
-
-
-AUTHORS
--------
-
-| Steven Armstrong <steven-cdist--@--armstrong.cc>
-| Darko Poljak <darko.poljak--@--ungleich.ch>
-
-
-COPYING
--------
-Copyright \(C) 2019 Steven Armstrong, Darko Poljak. Free use of this software is
-granted under the terms of the GNU General Public License v3 or later (GPLv3+).

docs/src/man1/cdist.rst

@@ -11,7 +11,7 @@ SYNOPSIS
 
 ::
 
-    cdist [-h] [-V] {banner,config,install,inventory,preos,shell} ...
+    cdist [-h] [-V] {banner,config,install,inventory,preos,shell,trigger} ...
 
     cdist banner [-h] [-l LOGLEVEL] [-q] [-v]
 
@@ -65,25 +65,35 @@ SYNOPSIS
                        [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
                        [-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
                        [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
-                       [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
+                       [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND]
+                       [-y REMOTE_COPY]
                        target_dir
 
     cdist preos devuan [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
                        [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
                        [-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
                        [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
-                       [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
+                       [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND]
+                       [-y REMOTE_COPY]
                        target_dir
 
     cdist preos ubuntu [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
                        [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
                        [-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
                        [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
-                       [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
+                       [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND]
+                       [-y REMOTE_COPY]
                        target_dir
 
     cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL]
 
+    cdist trigger [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4]
+                  [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
+                  [-j [JOBS]] [-n] [-o OUT_PATH] [-P]
+                  [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
+                  [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
+                  [-S] [-D DIRECTORY] [-H HTTP_PORT] [--ipv6] [-O SOURCE]
+
 
 DESCRIPTION
 -----------
@@ -519,6 +529,10 @@ PREOS DEBIAN/DEVUAN
 **-s SUITE, --suite SUITE**
     suite used for debootstrap, by default 'stable'
 
+**-t TRIGGER_COMMAND, --trigger-command TRIGGER_COMMAND**
+    trigger command that will be added to cdist config;
+    '``__cdist_preos_trigger http ...``' type is appended to initial manifest
+
 **-y REMOTE_COPY, --remote-copy REMOTE_COPY**
     remote copy that cdist config will use, by default
     internal script is used
@@ -579,6 +593,10 @@ PREOS UBUNTU
 **-s SUITE, --suite SUITE**
     suite used for debootstrap, by default 'xenial'
 
+**-t TRIGGER_COMMAND, --trigger-command TRIGGER_COMMAND**
+    trigger command that will be added to cdist config;
+    '``__cdist_preos_trigger http ...``' type is appended to initial manifest
+
 **-y REMOTE_COPY, --remote-copy REMOTE_COPY**
     remote copy that cdist config will use, by default
     internal script is used
@@ -596,6 +614,96 @@ usage. Its primary use is for debugging type parameters.
     be POSIX compatible shell.
 
 
+TRIGGER
+-------
+Start trigger (simple http server) that waits for connections. When host
+connects then it triggers config or install command and then cdist
+config/install is executed which configures/installs host.
+When triggered cdist will try to reverse DNS lookup for host name and if
+host name is dervied then it is used for running cdist config. If no
+host name is resolved then IP address is used.
+Request path recognizes following requests:
+
+* :strong:`/cdist/config/.*` for config
+* :strong:`/cdist/install/.*` for install.
+
+This command returns the following response codes to client requests:
+
+* 200 for success
+* 599 for cdist run errors
+* 500 for cdist/server errors.
+
+**-4, -force-ipv4**
+    Force to use IPv4 addresses only. No influence for
+    custom remote commands.
+
+**-6, --force-ipv6**
+    Force to use IPv6 addresses only. No influence for
+    custom remote commands.
+
+**-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN**
+    Specify custom cache path pattern. If it is not set
+    then default hostdir is used.
+
+**-c CONF_DIR, --conf-dir CONF_DIR**
+    Add configuration directory (can be repeated, last one
+    wins).
+
+**-D DIRECTORY, --directory DIRECTORY**
+    Where to create local files
+
+**-g CONFIG_FILE, --config-file CONFIG_FILE**
+    Use specified custom configuration file.
+
+**-H HTTP_PORT, --http-port HTTP_PORT**
+    Create trigger listener via http on specified port
+
+**-i MANIFEST, --initial-manifest MANIFEST**
+    Path to a cdist manifest or '-' to read from stdin.
+
+**--ipv6**
+    Listen to both IPv4 and IPv6 (instead of only IPv4)
+
+**-j [JOBS], --jobs [JOBS]**
+    Operate in parallel in specified maximum number of
+    jobs. Global explorers, object prepare and object run
+    are supported. Without argument CPU count is used by
+    default. Currently in beta.
+
+**-n, --dry-run**
+    Do not execute code.
+
+**-O SOURCE, --source SOURCE**
+    Which file to copy for creation
+
+**-o OUT_PATH, --out-dir OUT_PATH**
+    Directory to save cdist output in.
+
+**-P, --timestamp**
+    Timestamp log messages with the current local date and
+    time in the format: YYYYMMDDHHMMSS.us.
+
+**-R [{tar,tgz,tbz2,txz}], --use-archiving [{tar,tgz,tbz2,txz}]**
+    Operate by using archiving with compression where
+    appropriate. Supported values are: tar - tar archive,
+    tgz - gzip tar archive (the default), tbz2 - bzip2 tar
+    archive and txz - lzma tar archive. Currently in beta.
+
+**-r REMOTE_OUT_PATH, --remote-out-dir REMOTE_OUT_PATH**
+    Directory to save cdist output in on the target host.
+
+**--remote-copy REMOTE_COPY**
+    Command to use for remote copy (should behave like
+    scp).
+
+**--remote-exec REMOTE_EXEC**
+    Command to use for remote execution (should behave
+    like ssh).
+
+**-S, --disable-saving-output-streams**
+    Disable saving output streams.
+
+
 CONFIGURATION
 -------------
 cdist obtains configuration data from the following sources in the following
@@ -734,7 +842,8 @@ EXAMPLES
     % cdist config -f loadbalancers
 
     # Configure hosts read from file web.hosts using 16 parallel jobs
-    % cdist config -j 16 -f web.hosts
+    # (beta functionality)
+    % cdist config -b -j 16 -f web.hosts
 
     # Display banner
     cdist banner
@@ -790,20 +899,28 @@ EXAMPLES
     # Configure all hosts from inventory db
     $ cdist config -b -A
 
-    # Create default debian PreOS in debug mode
-    $ cdist preos debian /preos/preos-debian -vvvv -C \
-        -k ~/.ssh/id_rsa.pub -p /preos/pxe-debian
+    # Create default debian PreOS in debug mode with config
+    # trigger command
+    $ cdist preos debian /preos/preos-debian -b -vvvv -C \
+        -k ~/.ssh/id_rsa.pub -p /preos/pxe-debian \
+        -t "/usr/bin/curl 192.168.111.5:3000/config/"
 
-    # Create ubuntu PreOS
-    $ cdist preos ubuntu /preos/preos-ubuntu -C \
-        -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu
+    # Create ubuntu PreOS with install trigger command
+    $ cdist preos ubuntu /preos/preos-ubuntu -b -C \
+        -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \
+        -t "/usr/bin/curl 192.168.111.5:3000/install/"
 
-    # Create ubuntu PreOS on drive /dev/sdb
+    # Create ubuntu PreOS on drive /dev/sdb with install trigger command
     # and set root password to 'password'.
-    $ cdist preos ubuntu /mnt -B -C \
+    $ cdist preos ubuntu /mnt -b -B -C \
         -k ~/.ssh/id_rsa.pub -D /dev/sdb \
+        -t "/usr/bin/curl 192.168.111.5:3000/install/" \
         -P password
 
+    # Start trigger in verbose mode that will configure host using specified
+    # init manifest
+    % cdist trigger -b -v -i ~/.cdist/manifest/init-for-triggered
+
 
 ENVIRONMENT
 -----------

scripts/cdist

@@ -60,7 +60,7 @@ def commandline():
 
 
 if __name__ == "__main__":
-    cdistpythonversion = '3.2'
+    cdistpythonversion = '3.5'
     if sys.version < cdistpythonversion:
         print('Python >= {} is required on the source host.'.format(
                 cdistpythonversion), file=sys.stderr)

scripts/cdist-new-type

@@ -1,159 +0,0 @@
-#!/bin/sh
-
-basename="${0##*/}"
-
-if [ $# -lt 3 ]
-then
-    printf "usage: %s TYPE-NAME AUTHOR-NAME AUTHOR-EMAIL [TYPE-BASE-PATH]
-    TYPE-NAME       Name of the type.
-    AUTHOR-NAME     Type author's full name.
-    AUTHOR-EMAIL    Type author's email.
-    TYPE-BASE-PATH  Path to the base directory of the type. If not set it defaults
-                    to '\$PWD/type'.\n" "${basename}"
-    exit 1
-fi
-
-type_name="$1"
-shift
-author_name="$1"
-shift
-author_email="$1"
-shift
-
-if [ $# -ge 1 ]
-then
-    type_base_path="$1"
-    shift
-else
-    #type_base_path=~/.cdist/type
-    type_base_path="$PWD/type"
-fi
-
-error() {
-   printf "%s\n" "$*" >&2
-}
-
-die() {
-   error "$@"
-   exit 1
-}
-
-cd "$type_base_path" || die "Could not change to type directory: $type_base_path.
-You have to specify type base path or run me from within a cdist conf directory,
-e.g. ~/.cdist."
-
-year=$(date +%Y)
-copyright="# $year $author_name ($author_email)"
-
-license="# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-"
-
-set -e
-
-mkdir "$type_name"
-cd "$type_name"
-
-### man page
-header="cdist-type${type_name}(7)"
-header_length="${#header}"
-cat >> man.rst << DONE
-$header
-$(while [ "${header_length}" -gt 0 ]; do printf "="; header_length=$((header_length - 1)); done; printf "\n";)
-
-NAME
-----
-cdist-type${type_name} - TODO
-
-
-DESCRIPTION
------------
-This space intentionally left blank.
-
-
-REQUIRED PARAMETERS
--------------------
-None.
-
-
-OPTIONAL PARAMETERS
--------------------
-None.
-
-
-BOOLEAN PARAMETERS
-------------------
-None.
-
-
-EXAMPLES
---------
-
-.. code-block:: sh
-
-    # TODO
-    ${type_name}
-
-
-SEE ALSO
---------
-:strong:\`TODO\`\\ (7)
-
-
-AUTHORS
--------
-$author_name <$author_email>
-
-
-COPYING
--------
-Copyright \(C) $year $author_name. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
-DONE
-
-### manifest
-cat >> manifest << DONE
-#!/bin/sh -e
-#
-${copyright}
-#
-${license}
-
-os=\$(cat "\$__global/explorer/os")
-
-case "\$os" in
-   *)
-      printf "Your operating system (%s) is currently not supported by this type (%s)\n" "\$os" "\${__type##*/}" >&2
-      printf "Please contribute an implementation for it if you can.\n" >&2
-      exit 1
-   ;;
-esac
-DONE
-chmod +x manifest
-
-# gencode-remote
-cat >> gencode-remote << DONE
-#!/bin/sh -e
-#
-${copyright}
-#
-${license}
-DONE
-chmod +x gencode-remote
-
-printf "%s/%s\n" "$type_base_path" "$type_name"

setup.py

@@ -36,7 +36,7 @@ setup(
     name="cdist",
     packages=["cdist", "cdist.core", "cdist.exec", "cdist.util", ],
     package_data={'cdist': package_data},
-    scripts=["scripts/cdist", "scripts/cdist-dump", "scripts/cdist-new-type"],
+    scripts=["scripts/cdist", "scripts/cdist-dump"],
     version=cdist.version.VERSION,
     description="A Usable Configuration Management System",
     author="Nico Schottelius",