Merge remote-tracking branch 'origin/master' into new/__ini_value

Makefile

@@ -35,9 +35,9 @@ DOCS_SRC_DIR=./docs/src
 SPEECHDIR=./docs/speeches
 TYPEDIR=./cdist/conf/type
 
-SPHINXM=make -C $(DOCS_SRC_DIR) man
+SPHINXM=$(MAKE) -C $(DOCS_SRC_DIR) man
-SPHINXH=make -C $(DOCS_SRC_DIR) html
+SPHINXH=$(MAKE) -C $(DOCS_SRC_DIR) html
-SPHINXC=make -C $(DOCS_SRC_DIR) clean
+SPHINXC=$(MAKE) -C $(DOCS_SRC_DIR) clean
 
 ################################################################################
 # Manpages

README.md

@@ -24,8 +24,8 @@ For community-maintained types there is
 
 ## Participating
 
-IRC: ``#cdist`` @ freenode
+IRC: ``#cdist`` @ [libera](https://libera.chat)
 
 Matrix: ``#cdist:ungleich.ch``
 
-Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist
+Matrix and IRC are bridged.

bin/cdist

@@ -72,9 +72,11 @@ def commandline():
 
 
 if __name__ == "__main__":
-    if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION:
+    if sys.version_info[:3] < cdist.MIN_SUPPORTED_PYTHON_VERSION:
-        print('Python >= {} is required on the source host.'.format(
+        print(
-                cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr)
+            'Python >= {} is required on the source host.'.format(
+                ".".join(map(str, cdist.MIN_SUPPORTED_PYTHON_VERSION))),
+            file=sys.stderr)
         sys.exit(1)
 
     exit_code = 0

cdist/__init__.py

@@ -64,7 +64,7 @@ REMOTE_EXEC = "ssh -o User=root"
 REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
 
 
-MIN_SUPPORTED_PYTHON_VERSION = '3.5'
+MIN_SUPPORTED_PYTHON_VERSION = (3, 5)
 
 
 class Error(Exception):

cdist/argparse.py

@@ -485,19 +485,31 @@ def get_parsers():
     parser['scan'].add_argument(
         '-m', '--mode', help='Which modes should run',
         action='append', default=[],
-        choices=['scan', 'trigger'])
+        choices=['scan', 'trigger', 'config'])
+    parser['scan'].add_argument(
+        '--list',
+        action='store_true',
+        help='List the known hosts and exit')
     parser['scan'].add_argument(
         '--config',
         action='store_true',
         help='Try to configure detected hosts')
     parser['scan'].add_argument(
-        '-I', '--interfaces',
+        '-I', '--interface',
-        action='append',  default=[],
+        action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
     parser['scan'].add_argument(
-        '-d', '--delay',
+        '--name-mapper',
-        action='store',  default=3600,
+        action='store',  default=None,
-        help='How long to wait before reconfiguring after last try')
+        help='Map addresses to names, required for config mode')
+    parser['scan'].add_argument(
+        '-d', '--config-delay',
+        action='store',  default=3600, type=int,
+        help='How long (seconds) to wait before reconfiguring after last try')
+    parser['scan'].add_argument(
+        '-t', '--trigger-delay',
+        action='store',  default=5, type=int,
+        help='How long (seconds) to wait between ICMPv6 echo requests')
     parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
 
     for p in parser:
@@ -533,10 +545,10 @@ def parse_and_configure(argv, singleton=True):
 
     log = logging.getLogger("cdist")
 
-    log.verbose("version %s" % cdist.VERSION)
+    log.verbose("version %s", cdist.VERSION)
-    log.trace('command line args: {}'.format(cfg.command_line_args))
+    log.trace('command line args: %s', cfg.command_line_args)
-    log.trace('configuration: {}'.format(cfg.get_config()))
+    log.trace('configuration: %s', cfg.get_config())
-    log.trace('configured args: {}'.format(args))
+    log.trace('configured args: %s', args)
 
     check_beta(vars(args))
 

cdist/conf/explorer/memory

@@ -27,19 +27,18 @@
 str2bytes() {
 	awk -F' ' '
 	$2 ==   "B" || !$2 { print $1 }
-	$2 ==  "kB" { print $1 * 1000 }
+	$2 ==  "kB" { printf "%.f\n", ($1 * 1000) }
-	$2 ==  "MB" { print $1 * 1000 * 1000 }
+	$2 ==  "MB" { printf "%.f\n", ($1 * 1000 * 1000) }
-	$2 ==  "GB" { print $1 * 1000 * 1000 * 1000 }
+	$2 ==  "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) }
-	$2 ==  "TB" { print $1 * 1000 * 1000 * 1000 * 1000 }
+	$2 ==  "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) }
-	$2 == "kiB" { print $1 * 1024 }
+	$2 == "kiB" { printf "%.f\n", ($1 * 1024) }
-	$2 == "MiB" { print $1 * 1024 * 1024 }
+	$2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) }
-	$2 == "GiB" { print $1 * 1024 * 1024 * 1024 }
+	$2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) }
-	$2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }'
+	$2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }'
 }
 
 bytes2kib() {
-	set -- "$(cat)"
+	awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }'
-	test "$1" -gt 0 && echo $(($1 / 1024))
 }
 
 

cdist/conf/explorer/os_version

@@ -1,6 +1,7 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,12 +18,22 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
 # All os variables are lower case
 #
-#
 
-case "$("$__explorer/os")" in
+rc_getvar() {
+   awk -F= -v varname="$2" '
+      function unquote(s) {
+         if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/)
+            return substr(s, 2, length(s) - 2)
+         else
+            return s
+      }
+      $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1"
+}
+
+case $("${__explorer:?}/os")
+in
    amazon)
       cat /etc/system-release
    ;;
@@ -43,6 +54,8 @@ case "$("$__explorer/os")" in
               # sid versions don't have a number, so we decode by codename:
               case $(expr "$debian_version" : '\([a-z]\{1,\}\)/')
               in
+                  trixie) echo 12.99 ;;
+                  bookworm) echo 11.99 ;;
                   bullseye) echo 10.99 ;;
                   buster) echo 9.99 ;;
                   stretch) echo 8.99 ;;
@@ -50,7 +63,7 @@ case "$("$__explorer/os")" in
                   wheezy) echo 6.99 ;;
                   squeeze) echo 5.99 ;;
                   lenny) echo 4.99 ;;
-                  *) exit 1
+                  *) echo 99.99 ;;
               esac
               ;;
           *)
@@ -59,7 +72,23 @@ case "$("$__explorer/os")" in
       esac
    ;;
    devuan)
-      cat /etc/devuan_version
+      devuan_version=$(cat /etc/devuan_version)
+      case ${devuan_version}
+      in
+         (*/ceres)
+            # ceres versions don't have a number, so we decode by codename:
+            case ${devuan_version}
+            in
+               (chimaera/ceres) echo 3.99 ;;
+               (beowulf/ceres) echo 2.99 ;;
+               (ascii/ceres) echo 1.99 ;;
+               (*) exit 1
+            esac
+            ;;
+         (*)
+            echo "${devuan_version}"
+            ;;
+      esac
    ;;
    fedora)
       cat /etc/fedora-release
@@ -68,12 +97,20 @@ case "$("$__explorer/os")" in
       cat /etc/gentoo-release
    ;;
    macosx)
-      sw_vers -productVersion
+      # NOTE: Legacy versions (< 10.3) do not support options
+      sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }'
    ;;
    freebsd)
       # Apparently uname -r is not a reliable way to get the patch level.
       # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
-      freebsd-version
+      if command -v freebsd-version >/dev/null 2>&1
+      then
+         # get userland version
+         freebsd-version -u
+      else
+         # fallback to kernel release for FreeBSD < 10.0
+         uname -r
+      fi
    ;;
    *bsd|solaris)
       uname -r
@@ -98,7 +135,20 @@ case "$("$__explorer/os")" in
       fi
    ;;
    ubuntu)
-      lsb_release -sr
+      if command -v lsb_release >/dev/null 2>&1
+      then
+         lsb_release -sr
+      elif test -r /usr/lib/os-release
+      then
+         # fallback to /usr/lib/os-release if lsb_release is not present (like
+         # on minimized Ubuntu installations)
+         rc_getvar /usr/lib/os-release VERSION_ID
+      elif test -r /etc/lsb-release
+      then
+         # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old
+         # versions without /usr/lib/os-release.
+         rc_getvar /etc/lsb-release DISTRIB_RELEASE
+      fi
    ;;
    alpine)
        cat /etc/alpine-release

cdist/conf/type/__apt_backports/manifest

@@ -28,6 +28,7 @@
 #  lsb_release may not be given in all installations
 codename_os_release() {
     # shellcheck disable=SC1090
+    # shellcheck disable=SC1091
     . "$__global/explorer/os_release"
     printf "%s" "$VERSION_CODENAME"
 }

cdist/conf/type/__apt_key/explorer/state

@@ -27,18 +27,25 @@ else
    keyid="$__object_id"
 fi
 
+# From apt-key(8):
+#   Use of apt-key is deprecated, except for the use of apt-key del in
+#   maintainer scripts to remove existing keys from the main keyring.
+#   If such usage of apt-key is desired the additional installation of
+#   the GNU Privacy Guard suite (packaged in gnupg) is required.
+if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then
+   if apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK"
+      then echo present
+      else echo absent
+   fi
+   exit
+fi
+
 keydir="$(cat "$__object/parameter/keydir")"
 keyfile="$keydir/$__object_id.gpg"
 
-if [ -d "$keydir" ]
+if [ -f "$keyfile" ]
 then
-   if [ -f "$keyfile" ]
+   echo present
-   then echo present
+   exit
-   else echo absent
-   fi
-else
-   # fallback to deprecated apt-key
-   apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \
-      && echo present \
-      || echo absent
 fi
+echo absent

cdist/conf/type/__apt_key/gencode-remote

@@ -25,11 +25,7 @@ else
 fi
 state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
-
+method="$(cat "$__object/key_method")"
-if [ "$state_should" = "$state_is" ]; then
-   # nothing to do
-   exit 0
-fi
 
 keydir="$(cat "$__object/parameter/keydir")"
 keyfile="$keydir/$__object_id.gpg"
@@ -37,30 +33,18 @@ keyfile="$keydir/$__object_id.gpg"
 case "$state_should" in
    present)
       keyserver="$(cat "$__object/parameter/keyserver")"
-
+      # Using __download or __file as key source
-      if [ -f "$__object/parameter/uri" ]; then
+      # Propagate messages if needed
-         uri="$(cat "$__object/parameter/uri")"
+      if [ "${method}" = "uri" ] || [ "${method}" = "source" ]; then
-
+         if grep -Eq "^__(file|download)$keyfile" "$__messages_in"; then
-         if [ -d "$keydir" ]; then
+            echo "added '$keyid'" >> "$__messages_out"
-            cat << EOF
-
-curl -s -L \\
-    -o "$keyfile" \\
-    "$uri"
-
-key="\$( cat "$keyfile" )"
-
-if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK'
-then
-    echo "\$key" | gpg --dearmor > "$keyfile"
-fi
-
-EOF
-         else
-            # fallback to deprecated apt-key
-            echo "curl -s -L '$uri' | apt-key add -"
          fi
-      elif [ -d "$keydir" ]; then
+         exit 0
+      elif [ "${state_is}" = "present" ]; then
+         exit 0
+      fi
+      # Using key servers to fetch the key
+      if [ ! -f "$__object/parameter/use-deprecated-apt-key" ]; then
          # we need to kill gpg after 30 seconds, because gpg
          # can get stuck if keyserver is not responding.
          # exporting env var and not exit 1,
@@ -100,13 +84,16 @@ EOF
       echo "added '$keyid'" >> "$__messages_out"
    ;;
    absent)
-      if [ -f "$keyfile" ]; then
+      # Removal for keys added from a keyserver without this flag
-         echo "rm '$keyfile'"
+      # is done in the manifest
-      else
+      if [ "$state_is" != "absent" ] && \
+            [ -f "$__object/parameter/use-deprecated-apt-key" ]; then
          # fallback to deprecated apt-key
          echo "apt-key del \"$keyid\""
+         echo "removed '$keyid'" >> "$__messages_out"
+      # Propagate messages if needed
+      elif grep -Eq "^__file$keyfile" "$__messages_in"; then
+         echo "removed '$keyid'" >> "$__messages_out"
       fi
-
-      echo "removed '$keyid'" >> "$__messages_out"
    ;;
 esac

cdist/conf/type/__apt_key/man.rst

@@ -10,6 +10,14 @@ DESCRIPTION
 -----------
 Manages the list of keys used by apt to authenticate packages.
 
+This is done by placing the requested key in a file named
+``$__object_id.gpg`` in the ``keydir`` directory.
+
+This is supported by modern releases of Debian-based distributions.
+
+In order of preference, exactly one of: ``source``, ``uri`` or ``keyid``
+must be specified.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -18,21 +26,49 @@ None.
 
 OPTIONAL PARAMETERS
 -------------------
+keydir
+   keyring directory, defaults to ``/etc/apt/trusted.pgp.d``, which is
+   enabled system-wide by default.
+
+source
+   path to a file containing the GPG key of the repository.
+   Using this is recommended as it ensures that the manifest/type manintainer
+   has validated the key.
+   If ``-``, the GPG key is read from the type's stdin.
+
 state
    'present' or 'absent'. Defaults to 'present'
 
+uri
+   the URI from which to download the key.
+   It is highly recommended that you only use protocols with TLS like HTTPS.
+   This uses ``__download`` but does not use checksums, if you want to ensure
+   that the key doesn't change, you are better off downloading it and using
+   ``--source``.
+
+
+DEPRECATED OPTIONAL PARAMETERS
+------------------------------
 keyid
-   the id of the key to add. Defaults to __object_id
+   the id of the key to download from the ``keyserver``.
+   This is to be used in absence of ``--source`` and ``--uri`` or together
+   with ``--use-deprecated-apt-key`` for key removal.
+   Defaults to ``$__object_id``.
 
 keyserver
-   the keyserver from which to fetch the key. If omitted the default set
+   the keyserver from which to fetch the key.
-   in ./parameter/default/keyserver is used.
+   Defaults to ``pool.sks-keyservers.net``.
 
-keydir
-   key save location, defaults to ``/etc/apt/trusted.pgp.d``
 
-uri
+DEPRECATED BOOLEAN PARAMETERS
-   the URI from which to download the key
+-----------------------------
+use-deprecated-apt-key
+   ``apt-key(8)`` will last be available in Debian 11 and Ubuntu 22.04.
+   You can use this parameter to force usage of ``apt-key(8)``.
+   Please only use this parameter to *remove* keys from the keyring,
+   in order to prepare for removal of ``apt-key``.
+   Adding keys should be done without this parameter.
+   This parameter will be removed when Debian 11 stops being supported.
 
 
 EXAMPLES
@@ -40,33 +76,39 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # Add Ubuntu Archive Automatic Signing Key
+    # add a key that has been verified by a type maintainer
-    __apt_key 437D05B5
+    __apt_key jitsi_meet_2021 \
-    # Same thing
+       --source cdist-contrib/type/__jitsi_meet/files/apt_2021.gpg
-    __apt_key 437D05B5 --state present
-    # Get rid of it
-    __apt_key 437D05B5 --state absent
 
-    # same thing with human readable name and explicit keyid
+    # remove an old, deprecated or expired key
-    __apt_key UbuntuArchiveKey --keyid 437D05B5
+    __apt_key jitsi_meet_2016 --state absent
 
-    # same thing with other keyserver
+    # Get rid of a key that might have been added to
-    __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com
+    # /etc/apt/trusted.gpg with apt-key
+    __apt_key 0x40976EAF437D05B5 --use-deprecated-apt-key --state absent
 
-    # download key from the internet
+    # add a key that we define in-line
-    __apt_key rabbitmq \
+    __apt_key jitsi_meet_2021 --source '-' <<EOF
-       --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc
+    -----BEGIN PGP PUBLIC KEY BLOCK-----
+    [...]
+    -----END PGP PUBLIC KEY BLOCK-----
+    EOF
+
+    # download or update key from the internet
+    __apt_key rabbitmq_2007 \
+       --uri https://www.rabbitmq.com/rabbitmq-signing-key-public.asc
 
 
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
 Ander Punnar <ander-at-kvlt-dot-ee>
+Evilham <contact~~@~~evilham.com>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can
+Copyright \(C) 2011-2021 Steven Armstrong, Ander Punnar and Evilham. You can
 redistribute it and/or modify it under the terms of the GNU General Public
 License as published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.

cdist/conf/type/__apt_key/manifest

@@ -2,7 +2,105 @@
 
 __package gnupg
 
-if [ -f "$__object/parameter/uri" ]
+state_should="$(cat "${__object}/parameter/state")"
-then __package curl
+
-else __package dirmngr
+incompatible_args()
+{
+	cat >> /dev/stderr <<-EOF
+	This type does not support --${1} and --${method} simultaneously.
+	EOF
+	exit 1
+}
+
+if [ -f "${__object}/parameter/source" ]; then
+	method="source"
+	src="$(cat "${__object}/parameter/source")"
+	if [ "${src}" = "-" ]; then
+		src="${__object}/stdin"
+	fi
+fi
+if [ -f "${__object}/parameter/uri" ]; then
+	if [ -n "${method}" ]; then
+		incompatible_args uri
+	fi
+	method="uri"
+	src="$(cat "${__object}/parameter/uri")"
+fi
+if [ -f "${__object}/parameter/keyid" ]; then
+	if [ -n "${method}" ]; then
+		incompatible_args keyid
+	fi
+	method="keyid"
+fi
+# Keep old default
+if [ -z "${method}" ]; then
+	method="keyid"
+fi
+# Save this for later in gencode-remote
+echo "${method}" > "${__object}/key_method"
+
+# Required remotely (most likely already installed)
+__package dirmngr
+# We need this in case a key has to be dearmor'd
+__package gnupg
+export require="__package/gnupg"
+
+if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then
+	# This is required if apt-key(8) is to be used
+	if [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then
+		incompatible_args use-deprecated-apt-key
+	fi
+else
+	if [ "${state_should}" = "absent" ] && \
+			[ -f "${__object}/parameter/keyid" ]; then
+		cat >> /dev/stderr <<EOF
+You can't reliably remove by keyid without --use-deprecated-apt-key.
+This would very likely do something you do not intend.
+EOF
+		exit 1
+	fi
+fi
+
+keydir="$(cat "${__object}/parameter/keydir")"
+keyfile="${keydir}/${__object_id}.gpg"
+keyfilecdist="${keyfile}.cdist"
+if [ "${state_should}" != "absent" ]; then
+	# Ensure keydir exists
+	__directory "${keydir}" --state exists --mode 0755
+fi
+
+if [ "${state_should}" = "absent" ]; then
+	__file "${keyfile}" --state "absent"
+	__file "${keyfilecdist}" --state "absent"
+elif [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then
+	dearmor="$(cat <<-EOF
+	if [ '${state_should}' = 'present' ]; then
+		# Dearmor if necessary
+		if grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK' '${keyfilecdist}'; then
+			gpg --dearmor < '${keyfilecdist}' > '${keyfile}'
+		else
+			cp '${keyfilecdist}' '${keyfile}'
+		fi
+		# Ensure permissions
+		chown root '${keyfile}'
+		chmod 0444 '${keyfile}'
+	fi
+	EOF
+	)"
+
+	if [ "${method}" = "uri" ]; then
+		__download "${keyfilecdist}" \
+			--url "${src}" \
+			--onchange "${dearmor}"
+		require="__download${keyfilecdist}" \
+			__file "${keyfile}" \
+				--owner root \
+				--mode 0444 \
+				--state pre-exists
+	else
+		__file "${keyfilecdist}" --state "${state_should}" \
+			--mode 0444 \
+			--source "${src}" \
+			--onchange "${dearmor}"
+	fi
 fi

cdist/conf/type/__apt_key/parameter/boolean

@@ -0,0 +1 @@
+use-deprecated-apt-key

cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key

@@ -0,0 +1,3 @@
+apt-key(8) will last be available in Debian 11 and Ubuntu 22.04.
+Use this flag *only* to migrate to placing a keyring directly in the
+/etc/apt/trusted.gpg.d/ directory with a descriptive name.

cdist/conf/type/__apt_key/parameter/optional

@@ -1,5 +1,6 @@
-state
+keydir
 keyid
 keyserver
-keydir
+source
+state
 uri

cdist/conf/type/__apt_key_uri/deprecated

@@ -0,0 +1 @@
+Please migrate to using __apt_key key_id --uri URI.

cdist/conf/type/__apt_pin/man.rst

@@ -0,0 +1,79 @@
+cdist-type__apt_pin(7)
+======================
+
+NAME
+----
+cdist-type__apt_pin - Manage apt pinning rules
+
+
+DESCRIPTION
+-----------
+Adds/removes/edits rules to pin some packages to a specific distribution. Useful if using multiple debian repositories at the same time. (Useful, if one wants to use a few specific packages from backports or perhaps Debain testing... or even sid.)
+
+
+REQUIRED PARAMETERS
+-------------------
+distribution
+   Specifies what distribution the package should be pinned to. Accepts both codenames (buster/bullseye/sid) and suite names (stable/testing/...).
+
+
+OPTIONAL PARAMETERS
+-------------------
+package
+   Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used.
+
+priority
+   The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority)
+
+state
+   Will be passed to underlying `__file` type; see there for valid values and defaults.
+
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+   # Add the bullseye repo to buster, but do not install any packages by default,
+   # only if explicitely asked for (-1 means "never" for apt)
+    __apt_pin bullseye-default \
+       --package "*" \
+       --distribution bullseye \
+       --priority -1
+
+    require="__apt_pin/bullseye-default" __apt_source bullseye \
+       --uri http://deb.debian.org/debian/ \
+       --distribution bullseye \
+       --component main
+
+    __apt_pin foo --package "foo foo-*" --distribution bullseye
+
+    __foo # Assuming, this installs the `foo` package internally
+
+    __package foo-plugin-extras # Assuming we also need some extra stuff
+
+
+SEE ALSO
+--------
+:strong:`apt_preferences`\ (5)
+:strong:`cdist-type__apt_source`\ (7)
+:strong:`cdist-type__apt_backports`\ (7)
+:strong:`cdist-type__file`\ (7)
+
+AUTHORS
+-------
+Daniel Fancsali <fancsali@gmail.com>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Daniel Fancsali. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__apt_pin/manifest

@@ -0,0 +1,63 @@
+#!/bin/sh -e
+#
+# 2021 Daniel Fancsali (fancsali@gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+name="$__object_id"
+
+os=$(cat "$__global/explorer/os")
+state="$(cat "$__object/parameter/state")"
+
+if [ -f "$__object/parameter/package" ]; then
+    package="$(cat "$__object/parameter/package")"
+else
+    package=$name
+fi
+
+distribution="$(cat "$__object/parameter/distribution")"
+priority="$(cat "$__object/parameter/priority")"
+
+
+case "$os" in
+   debian|ubuntu|devuan)
+   ;;
+   *)
+      printf "This type is specific to Debian and it's derivatives" >&2
+      exit 1
+   ;;
+esac
+
+case $distribution in
+    stable|testing|unstable|experimental)
+        pin="release a=$distribution"
+        ;;
+    *)
+        pin="release n=$distribution"
+        ;;
+esac
+
+
+__file "/etc/apt/preferences.d/$name" \
+    --owner root --group root --mode 0644 \
+    --state "$state" \
+    --source - << EOF
+Package: $package
+Pin: $pin
+Pin-Priority: $priority
+EOF

cdist/conf/type/__apt_pin/parameter/default/state

@@ -0,0 +1 @@
+present

cdist/conf/type/__apt_pin/parameter/optional

@@ -0,0 +1,2 @@
+state
+package

cdist/conf/type/__apt_pin/parameter/required

@@ -0,0 +1,2 @@
+distribution
+priority

cdist/conf/type/__debconf_set_selections/explorer/state

@@ -0,0 +1,142 @@
+#!/bin/sh -e
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Determine current debconf selections' state.
+# Prints one of:
+#   present: all selections are already set as they should.
+#   different: one or more of the selections have a different value.
+#   absent: one or more of the selections are not (currently) defined.
+#
+
+test -x /usr/bin/perl || {
+	# cannot find perl (no perl ~ no debconf)
+	echo 'absent'
+	exit 0
+}
+
+linesfile="${__object:?}/parameter/line"
+test -s "${linesfile}" || {
+	if test -s "${__object:?}/parameter/file"
+	then
+		echo absent
+	else
+		echo present
+	fi
+	exit 0
+}
+
+# assert __type_explorer is set (because it is used by the Perl script)
+: "${__type_explorer:?}"
+
+/usr/bin/perl -- - "${linesfile}" <<'EOF'
+use strict;
+use warnings "all";
+
+use Fcntl qw(:DEFAULT :flock);
+
+use Debconf::Db;
+use Debconf::Question;
+
+# Extract @known... arrays from debconf-set-selections
+# These values are required to distinguish flags and values in the given lines.
+# DC: I couldn't think of a more ugly solution to the problem…
+my @knownflags;
+my @knowntypes;
+my $debconf_set_selections = '/usr/bin/debconf-set-selections';
+if (-e $debconf_set_selections) {
+	my $sed_known = 's/^my \(@known\(flags\|types\) = qw([a-z ]*);\).*$/\1/p';
+	eval `sed -n '$sed_known' '$debconf_set_selections'`;
+}
+
+sub mungeline ($) {
+	my $line = shift;
+	chomp $line;
+	$line =~ s/\r$//;
+	return $line;
+}
+
+sub fatal { printf STDERR @_; exit 1; }
+
+my $state = 'present';
+
+sub state {
+	my $new = shift;
+	if ($state eq 'present'
+	    or ($state eq 'different' and $new eq 'absent')) {
+		$state = $new;
+	}
+}
+
+
+# Load Debconf DB but manually lock on the state explorer script,
+# because Debconf aborts immediately if executed concurrently.
+# This is not really an ideal solution because the Debconf DB could be locked by
+# another process (e.g. apt-get), but no way to achieve this could be found.
+# If you know how to, please provide a patch.
+my $lockfile = "%ENV{'__type_explorer'}/state";
+if (open my $lock_fh, '+<', $lockfile) {
+   flock $lock_fh, LOCK_EX or die "Cannot lock $lockfile";
+}
+{
+	Debconf::Db->load(readonly => 'true');
+}
+
+
+while (<>) {
+	# Read and process lines (taken from debconf-set-selections)
+	$_ = mungeline($_);
+	while (/\\$/ && ! eof) {
+		s/\\$//;
+		$_ .= mungeline(<>);
+	}
+	next if /^\s*$/ || /^\s*\#/;
+
+	my ($owner, $label, $type, $content) = /^\s*(\S+)\s+(\S+)\s+(\S+)(?:\s(.*))?/
+		or fatal "invalid line: %s\n", $_;
+	$content = '' unless defined $content;
+
+
+	# Compare is and should state
+	my $q = Debconf::Question->get($label);
+
+	unless (defined $q) {
+		# probably a preseed
+		state 'absent';
+		next;
+	}
+
+	if (grep { $_ eq $q->type } @knownflags) {
+		# This line wants to set a flag, presumably.
+		if ($q->flag($q->type) ne $content) {
+			state 'different';
+		}
+	} else {
+		# Otherwise, it's probably a value…
+		if ($q->value ne $content) {
+			state 'different';
+		}
+
+		unless (grep { $_ eq $owner } (split /, /, $q->owners)) {
+			state 'different';
+		}
+	}
+}
+
+printf "%s\n", $state;
+EOF

cdist/conf/type/__debconf_set_selections/gencode-remote

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,16 +18,37 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# Setup selections
-#
 
-filename="$(cat "$__object/parameter/file")"
+if test -f "${__object:?}/parameter/line"
-
+then
-if [ "$filename" = "-" ]; then
+	filename="${__object:?}/parameter/line"
-    filename="$__object/stdin"
+elif test -s "${__object:?}/parameter/file"
+then
+	filename=$(cat "${__object:?}/parameter/file")
+	if test "${filename}" = '-'
+	then
+		filename="${__object:?}/stdin"
+	fi
+else
+	printf 'Neither --line nor --file set.\n' >&2
+	exit 1
 fi
 
-echo "debconf-set-selections << __file-eof"
+# setting no lines makes no sense
-cat "$filename"
+test -s "${filename}" || exit 0
-echo "__file-eof"
+
+state_is=$(cat "${__object:?}/explorer/state")
+
+if test "${state_is}" != 'present'
+then
+	cat <<-CODE
+	debconf-set-selections <<'EOF'
+	$(cat "${filename}")
+	EOF
+	CODE
+
+	awk '
+		{
+			printf "set %s %s %s %s\n", $1, $2, $3, $4
+		}' "${filename}" >>"${__messages_out:?}"
+fi

cdist/conf/type/__debconf_set_selections/man.rst

@@ -8,15 +8,33 @@ cdist-type__debconf_set_selections - Setup debconf selections
 
 DESCRIPTION
 -----------
-On Debian and alike systems debconf-set-selections(1) can be used
+On Debian and alike systems :strong:`debconf-set-selections`\ (1) can be used
 to setup configuration parameters.
 
 
 REQUIRED PARAMETERS
 -------------------
+cf. ``--line``.
+
+
+OPTIONAL PARAMETERS
+-------------------
 file
-   Use the given filename as input for debconf-set-selections(1)
+   Use the given filename as input for :strong:`debconf-set-selections`\ (1)
-   If filename is "-", read from stdin.
+   If filename is ``-``, read from stdin.
+
+   **This parameter is deprecated, because it doesn't work with state detection.**
+line
+   A line in :strong:`debconf-set-selections`\ (1) compatible format.
+   This parameter can be used multiple times to set multiple options.
+
+   (This parameter is actually required, but marked optional because the
+   deprecated ``--file`` is still accepted.)
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
 
 
 EXAMPLES
@@ -24,30 +42,29 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # Setup configuration for nslcd
+    # Setup gitolite's gituser
-    __debconf_set_selections nslcd --file /path/to/file
+    __debconf_set_selections nslcd --line 'gitolite gitolite/gituser string git'
 
-    # Setup configuration for nslcd from another type
+    # Setup configuration for nslcd from a file.
-    __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd"
+    # NB: Multiple lines can be passed to --line, although this can be considered a hack.
-
+    __debconf_set_selections nslcd --line "$(cat "${__files:?}/preseed/nslcd.debconf")"
-    __debconf_set_selections nslcd --file - << eof
-    gitolite gitolite/gituser string git
-    eof
 
 
 SEE ALSO
 --------
-:strong:`debconf-set-selections`\ (1), :strong:`cdist-type__update_alternatives`\ (7)
+- :strong:`cdist-type__update_alternatives`\ (7)
+- :strong:`debconf-set-selections`\ (1)
 
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2014 Nico Schottelius. You can redistribute it
+Copyright \(C) 2011-2014 Nico Schottelius, 2021 Dennis Camera.
-and/or modify it under the terms of the GNU General Public License as
+You can redistribute it and/or modify it under the terms of the GNU General
-published by the Free Software Foundation, either version 3 of the
+Public License as published by the Free Software Foundation, either version 3 of
-License, or (at your option) any later version.
+the License, or (at your option) any later version.

cdist/conf/type/__debconf_set_selections/manifest

@@ -1,6 +1,6 @@
 #!/bin/sh -e
 #
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,20 +18,4 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-if [ -f "$__object/parameter/destination" ]; then
+__package_apt debconf
-    destination=$(cat "$__object/parameter/destination")
-else
-    destination="/$__object_id"
-fi
-
-ownergroup=""
-if [ -f "$__object/parameter/owner" ]; then
-    ownergroup=$(cat "$__object/parameter/owner")
-fi
-if [ -f "$__object/parameter/group" ]; then
-    ownergroup="${ownergroup}:$(cat "$__object/parameter/group")"
-fi
-
-if [ "$ownergroup" ]; then
-    echo chown -R "$ownergroup" "$destination"
-fi

cdist/conf/type/__debconf_set_selections/parameter/optional_multiple

@@ -0,0 +1 @@
+line

cdist/conf/type/__download/explorer/remote_cmd

@@ -1,19 +0,0 @@
-#!/bin/sh -e
-
-if [ -f "$__object/parameter/cmd-get" ]
-then
-    cmd="$( cat "$__object/parameter/cmd-get" )"
-
-elif command -v curl > /dev/null
-then
-    cmd="curl -L -o - '%s'"
-
-elif command -v fetch > /dev/null
-then
-    cmd="fetch -o - '%s'"
-
-else
-    cmd="wget -O - '%s'"
-fi
-
-echo "$cmd"

cdist/conf/type/__download/explorer/remote_cmd_get

@@ -0,0 +1,16 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cat "$__object/parameter/cmd-get"
+elif
+    command -v curl > /dev/null
+then
+    echo "curl -sSL -o - '%s'"
+elif
+    command -v fetch > /dev/null
+then
+    echo "fetch -o - '%s'"
+else
+    echo "wget -O - '%s'"
+fi

cdist/conf/type/__download/explorer/remote_cmd_sum

@@ -0,0 +1,82 @@
+#!/bin/sh -e
+
+if [ ! -f "$__object/parameter/sum" ]
+then
+    exit 0
+fi
+
+if [ -f "$__object/parameter/cmd-sum" ]
+then
+    cat "$__object/parameter/cmd-sum"
+    exit 0
+fi
+
+sum_should="$( cat "$__object/parameter/sum" )"
+
+if echo "$sum_should" | grep -Fq ':'
+then
+    sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
+else
+    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+    then
+        sum_hash='cksum'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
+    then
+        sum_hash='md5'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
+    then
+        sum_hash='sha1'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
+    then
+        sum_hash='sha256'
+    else
+        echo 'hash format detection failed' >&2
+        exit 1
+    fi
+fi
+
+os="$( "$__explorer/os" )"
+
+case "$sum_hash" in
+    cksum)
+        echo "cksum %s | awk '{print \$1\" \"\$2}'"
+    ;;
+    md5)
+        case "$os" in
+            freebsd)
+                echo "md5 -q %s"
+            ;;
+            *)
+                echo "md5sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    sha1)
+        case "$os" in
+            freebsd)
+                echo "sha1 -q %s"
+            ;;
+            *)
+                echo "sha1sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    sha256)
+        case "$os" in
+            freebsd)
+                echo "sha256 -q %s"
+            ;;
+            *)
+                echo "sha256sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    *)
+        # we arrive here only if --sum is given with unknown format prefix
+        echo "unknown hash format: $sum_hash" >&2
+        exit 1
+    ;;
+esac

cdist/conf/type/__download/explorer/state

@@ -1,6 +1,11 @@
 #!/bin/sh -e
 
-dst="/$__object_id"
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
+else
+    dst="/$__object_id"
+fi
 
 if [ ! -f "$dst" ]
 then
@@ -8,59 +13,27 @@ then
     exit 0
 fi
 
+if [ ! -f "$__object/parameter/sum" ]
+then
+    echo 'present'
+    exit 0
+fi
+
 sum_should="$( cat "$__object/parameter/sum" )"
 
-if [ -f "$__object/parameter/cmd-sum" ]
+if echo "$sum_should" | grep -Fq ':'
 then
-    # shellcheck disable=SC2059
+    sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
-    sum_is="$( eval "$( printf \
-        "$( cat "$__object/parameter/cmd-sum" )" \
-        "$dst" )" )"
-else
-    os="$( "$__explorer/os" )"
-
-    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
-    then
-        sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
-
-    elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="md5:$( md5 -q "$dst" )"
-            ;;
-            *)
-                sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-
-    elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="sha1:$( sha1 -q "$dst" )"
-            ;;
-            *)
-                sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-
-    elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="sha256:$( sha256 -q "$dst" )"
-            ;;
-            *)
-                sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-    fi
 fi
 
+sum_cmd="$( "$__type_explorer/remote_cmd_sum" )"
+
+# shellcheck disable=SC2059
+sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )"
+
 if [ -z "$sum_is" ]
 then
-    echo 'no checksum from target' >&2
+    echo 'existing destination checksum failed' >&2
     exit 1
 fi
 

cdist/conf/type/__download/gencode-local

@@ -11,34 +11,133 @@ fi
 
 url="$( cat "$__object/parameter/url" )"
 
-tmp="$( mktemp )"
+if [ -f "$__object/parameter/destination" ]
-
+then
-dst="/$__object_id"
+    dst="$( cat "$__object/parameter/destination" )"
+else
+    dst="/$__object_id"
+fi
 
 if [ -f "$__object/parameter/cmd-get" ]
 then
     cmd="$( cat "$__object/parameter/cmd-get" )"
 
-elif command -v wget > /dev/null
-then
-    cmd="wget -O - '%s'"
-
 elif command -v curl > /dev/null
 then
-    cmd="curl -L -o - '%s'"
+    cmd="curl -sSL -o - '%s'"
 
 elif command -v fetch > /dev/null
 then
     cmd="fetch -o - '%s'"
 
+elif command -v wget > /dev/null
+then
+    cmd="wget -O - '%s'"
+
 else
-    echo 'no usable locally installed utility for downloading' >&2
+    echo 'local download failed, no usable utility' >&2
     exit 1
 fi
 
-printf "$cmd > %s\n" \
+echo "download_tmp=\"\$( mktemp )\""
-    "$url" \
+
-    "$tmp"
+# shellcheck disable=SC2059
+printf "$cmd > \"\$download_tmp\"\n" "$url"
+
+if [ -f "$__object/parameter/sum" ]
+then
+    sum_should="$( cat "$__object/parameter/sum" )"
+
+    if [ -f "$__object/parameter/cmd-sum" ]
+    then
+        local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
+    else
+        if echo "$sum_should" | grep -Fq ':'
+        then
+            sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
+
+            sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
+        else
+            if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+            then
+                sum_hash='cksum'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
+            then
+                sum_hash='md5'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
+            then
+                sum_hash='sha1'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
+            then
+                sum_hash='sha256'
+            else
+                echo 'hash format detection failed' >&2
+                exit 1
+            fi
+        fi
+
+        case "$sum_hash" in
+            cksum)
+                local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'"
+            ;;
+            md5)
+                if command -v md5 > /dev/null
+                then
+                    local_cmd_sum="md5 -q %s"
+                elif
+                    command -v md5sum > /dev/null
+                then
+                    local_cmd_sum="md5sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            sha1)
+                if command -v sha1 > /dev/null
+                then
+                    local_cmd_sum="sha1 -q %s"
+                elif
+                    command -v sha1sum > /dev/null
+                then
+                    local_cmd_sum="sha1sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            sha256)
+                if command -v sha256 > /dev/null
+                then
+                    local_cmd_sum="sha256 -q %s"
+                elif
+                    command -v sha256sum > /dev/null
+                then
+                    local_cmd_sum="sha256sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            *)
+                # we arrive here only if --sum is given with unknown format prefix
+                echo "unknown hash format: $sum_hash" >&2
+                exit 1
+            ;;
+        esac
+
+        if [ -z "$local_cmd_sum" ]
+        then
+            echo 'local checksum verification failed, no usable utility' >&2
+            exit 1
+        fi
+    fi
+
+    # shellcheck disable=SC2059
+    echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\""
+
+    echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
+
+    echo "echo 'local download checksum mismatch' >&2"
+
+    echo "rm -f \"\$download_tmp\""
+
+    echo 'exit 1; fi'
+fi
 
 if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$'
 then
@@ -47,12 +146,10 @@ else
     target_host="$__target_host"
 fi
 
-printf '%s %s %s:%s\n' \
+# shellcheck disable=SC2016
+printf '%s "$download_tmp" %s:%s\n' \
     "$__remote_copy" \
-    "$tmp" \
     "$target_host" \
     "$dst"
 
-echo "rm -f '$tmp'"
+echo "rm -f \"\$download_tmp\""
-
-echo 'downloaded' > "$__messages_out"

cdist/conf/type/__download/gencode-remote

@@ -6,17 +6,51 @@ state_is="$( cat "$__object/explorer/state" )"
 
 if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
 then
-    cmd="$( cat "$__object/explorer/remote_cmd" )"
+    cmd_get="$( cat "$__object/explorer/remote_cmd_get" )"
 
     url="$( cat "$__object/parameter/url" )"
 
-    dst="/$__object_id"
+    if [ -f "$__object/parameter/destination" ]
+    then
+        dst="$( cat "$__object/parameter/destination" )"
+    else
+        dst="/$__object_id"
+    fi
 
-    printf "$cmd > %s\n" \
+    echo "download_tmp=\"\$( mktemp )\""
-        "$url" \
-        "$dst"
 
-    echo 'downloaded' > "$__messages_out"
+    # shellcheck disable=SC2059
+    printf "$cmd_get > \"\$download_tmp\"\n" "$url"
+
+    if [ -f "$__object/parameter/sum" ]
+    then
+        sum_should="$( cat "$__object/parameter/sum" )"
+
+        if [ -f "$__object/parameter/cmd-sum" ]
+        then
+            remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
+        else
+            remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )"
+
+            if echo "$sum_should" | grep -Fq ':'
+            then
+                sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
+            fi
+        fi
+
+        # shellcheck disable=SC2059
+        echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\""
+
+        echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
+
+        echo "echo 'remote download checksum mismatch' >&2"
+
+        echo "rm -f \"\$download_tmp\""
+
+        echo 'exit 1; fi'
+    fi
+
+    echo "mv \"\$download_tmp\" '$dst'"
 fi
 
 if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ]

cdist/conf/type/__download/man.rst

@@ -8,10 +8,7 @@ cdist-type__download - Download a file
 
 DESCRIPTION
 -----------
-Destination (``$__object_id``) in target host must be persistent storage
+By default type will try to use ``curl``, ``fetch`` or ``wget``.
-in order to calculate checksum and decide if file must be (re-)downloaded.
-
-By default type will try to use ``wget``, ``curl`` or ``fetch``.
 If download happens in target (see ``--download``) then type will
 fallback to (and install) ``wget``.
 
@@ -19,23 +16,40 @@ If download happens in local machine, then environment variables like
 ``{http,https,ftp}_proxy`` etc can be used on cdist execution
 (``http_proxy=foo cdist config ...``).
 
+To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``.
+
 
 REQUIRED PARAMETERS
 -------------------
 url
    File's URL.
 
-sum
-   Checksum of file going to be downloaded.
-   By default output of ``cksum`` without filename is expected.
-   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
-
 
 OPTIONAL PARAMETERS
 -------------------
+destination
+   Downloaded file's destination in target. If unset, ``$__object_id`` is used.
+
+sum
+   Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256.
+
+   Type tries to detect hash format with regexes, but prefixes
+   ``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported.
+
+   Checksum have two purposes - state check and post-download verification.
+   In state check, if destination checksum mismatches, then content of URL
+   will be downloaded to temporary file. If downloaded temporary file's
+   checksum matches, then it will be moved to destination (overwritten).
+
+   For local downloads it is expected that usable utilities for checksum
+   calculation exist in the system.
+
 download
-   If ``local`` (default), then download file to local storage and copy
+   If ``local`` (default), then file is downloaded to local storage and copied
-   it to target host. If ``remote``, then download happens in target.
+   to target host. If ``remote``, then download happens in target.
+
+   For local downloads it is expected that usable utilities for downloading
+   exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``.
 
 cmd-get
    Command used for downloading.
@@ -65,7 +79,7 @@ EXAMPLES
     require='__directory/opt/cpma' \
         __download /opt/cpma/cnq3.zip \
             --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
-            --sum md5:46da3021ca9eace277115ec9106c5b46
+            --sum 46da3021ca9eace277115ec9106c5b46
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
@@ -81,7 +95,7 @@ Ander Punnar <ander-at-kvlt-dot-ee>
 
 COPYING
 -------
-Copyright \(C) 2020 Ander Punnar. You can redistribute it
+Copyright \(C) 2021 Ander Punnar. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.

cdist/conf/type/__download/manifest

@@ -1,6 +1,6 @@
 #!/bin/sh -e
 
-if grep -Eq '^wget' "$__object/explorer/remote_cmd"
+if grep -Eq '^wget' "$__object/explorer/remote_cmd_get"
 then
     __package wget
 fi

cdist/conf/type/__download/parameter/optional

@@ -1,4 +1,6 @@
 cmd-get
 cmd-sum
+destination
 download
 onchange
+sum

cdist/conf/type/__download/parameter/required

@@ -1,2 +1 @@
 url
-sum

cdist/conf/type/__filesystem/explorer/lsblk

@@ -27,7 +27,7 @@ else
 fi
 
 case "$os" in
-    alpine|centos|fedora|redhat|suse|gentoo)
+    alpine|centos|fedora|gentoo|redhat|suse|ubuntu)
         if [ ! -x "$(command -v lsblk)" ]; then
             echo "lsblk is required for __filesystem type" >&2
             exit 1

cdist/conf/type/__git/explorer/group

@@ -1,5 +1,24 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id/.git"
+destination="/${__object_id:?}/.git"
 
-stat --print "%G" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((group_gid + 1)) -ge 0
+then
+	group_should=$(cat "${__object:?}/parameter/group")
+
+	if expr "${group_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${group_gid}"
+	else
+		if command -v getent > /dev/null
+		then
+			getent group "${group_gid}" | cut -d : -f 1
+		else
+			awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group
+		fi
+	fi
+fi

cdist/conf/type/__git/explorer/owner

@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id/.git"
+destination="/${__object_id:?}/.git"
 
-stat --print "%U" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((owner_uid + 1)) -ge 0
+then
+	owner_should=$(cat "${__object:?}/parameter/owner")
+
+	if expr "${owner_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${owner_uid}"
+	else
+		printf '%s\n' "$(id -u -n "${owner_uid}")"
+	fi
+fi

cdist/conf/type/__letsencrypt_cert/explorer/certbot-path

@@ -1,3 +0,0 @@
-#!/bin/sh -e
-
-command -v certbot 2>/dev/null || true

cdist/conf/type/__letsencrypt_cert/explorer/certificate-data

@@ -0,0 +1,78 @@
+#!/bin/sh -e
+certbot_path="$(command -v certbot 2>/dev/null || true)"
+# Defaults
+certificate_exists="no"
+certificate_is_test="no"
+
+if [ -n "${certbot_path}" ]; then
+	# Find python executable that has access to certbot's module
+	python_path=$(sed -n '1s/^#! *//p' "${certbot_path}")
+
+	# Use a lock for cdist due to certbot not exiting with failure
+	# or having any flags for concurrent use.
+	_certbot() {
+		${python_path} - 2>/dev/null <<EOF
+from certbot.main import main
+import fcntl
+lock_file = "/tmp/certbot.cdist.lock"
+timeout=60
+with open(lock_file, 'w') as fd:
+    for i in range(timeout):
+        try:
+            # Get exclusive lock
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            break
+        except:
+            # Wait if that fails
+            import time
+            time.sleep(1)
+    else:
+        # Timed out, exit with failure
+        import sys
+        sys.exit(1)
+    # Do list certificates
+    main(["certificates", "--cert-name", "${__object_id:?}"])
+EOF
+	}
+
+
+	_certificate_exists() {
+		if grep -q "  Certificate Name: ${__object_id:?}$"; then
+			echo yes
+		else
+			echo no
+		fi
+	}
+
+	_certificate_is_test() {
+		if grep -q 'INVALID: TEST_CERT'; then
+			echo yes
+		else
+			echo no
+		fi
+	}
+
+	_certificate_domains() {
+		grep '    Domains: ' | cut -d ' ' -f 6- | tr ' ' '\n'
+	}
+
+	# Get data about all available certificates
+	certificates="$(_certbot)"
+
+	# Check whether or not the certificate exists
+	certificate_exists="$(echo "${certificates}" | _certificate_exists)"
+
+	# Check whether or not the certificate is for testing
+	certificate_is_test="$(echo "${certificates}" | _certificate_is_test)"
+
+	# Get domains for certificate
+	certificate_domains="$(echo "${certificates}" | _certificate_domains)"
+fi
+
+# Return received data
+cat <<EOF
+certbot_path:${certbot_path}
+certificate_exists:${certificate_exists}
+certificate_is_test:${certificate_is_test}
+${certificate_domains}
+EOF

cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains

@@ -1,8 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	certbot certificates --cert-name "${__object_id:?}" | grep '    Domains: ' | \
-		cut -d ' ' -f 6- | tr ' ' '\n'
-fi

cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists

@@ -1,13 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	if certbot certificates | grep -q "  Certificate Name: ${__object_id:?}$"; then
-		echo yes
-	else
-		echo no
-	fi
-else
-	echo no
-fi

cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test

@@ -1,14 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	if certbot certificates --cert-name "${__object_id:?}" | \
-		grep -q 'INVALID: TEST_CERT'; then
-		echo yes
-	else
-		echo no
-	fi
-else
-	echo no
-fi

cdist/conf/type/__letsencrypt_cert/gencode-remote

@@ -1,6 +1,10 @@
 #!/bin/sh -e
 
-certificate_exists=$(cat "${__object:?}/explorer/certificate-exists")
+_explorer_var() {
+	grep "^$1:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-
+}
+
+certificate_exists="$(_explorer_var certificate_exists)"
 name="${__object_id:?}"
 state=$(cat "${__object}/parameter/state")
 
@@ -29,8 +33,9 @@ case "${state}" in
 		fi
 
 		if [ "${certificate_exists}" = "yes" ]; then
-			existing_domains="${__object}/explorer/certificate-domains"
+			existing_domains=$(mktemp "${TMPDIR:-/tmp}/existing_domains.cdist.XXXXXXXXXX")
-			certificate_is_test=$(cat "${__object}/explorer/certificate-is-test")
+			tail -n +4 "${__object:?}/explorer/certificate-data" | grep -v '^$' > "${existing_domains}"
+			certificate_is_test="$(_explorer_var certificate_is_test)"
 
 			sort -uo "${requested_domains}" "${requested_domains}"
 			sort -uo "${existing_domains}" "${existing_domains}"

cdist/conf/type/__letsencrypt_cert/manifest

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")"
+certbot_fullpath="$(grep "^certbot_path:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-)"
 state=$(cat "${__object}/parameter/state")
 os="$(cat "${__global:?}/explorer/os")"
 

cdist/conf/type/__package_pkg_freebsd/gencode-remote

@@ -37,6 +37,7 @@ assert ()                 #  If condition false,
 	then
 		echo "Assertion failed:  \"$1\""
 		# shellcheck disable=SC2039
+		# shellcheck disable=SC3044
 		echo "File \"$0\", line $lineno, called by $(caller 0)"
 		exit $E_ASSERT_FAILED
 	fi

cdist/conf/type/__postgres_conf/explorer/postgres_user

@@ -0,0 +1,64 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$("${__explorer:?}/os")
+
+case ${os}
+in
+	(alpine)
+		echo 'postgres'
+		;;
+	(centos|rhel|scientific)
+		echo 'postgres'
+		;;
+	(debian|devuan|ubuntu)
+		echo 'postgres'
+		;;
+	(freebsd)
+		test -x /usr/local/etc/rc.d/postgresql || {
+			printf 'could not find postgresql rc script./n' >&2
+			exit 1
+		}
+		pg_status=$(/usr/local/etc/rc.d/postgresql onestatus) || {
+			printf 'postgresql daemon is not running.\n' >&2
+			exit 1
+		}
+		pg_pid=$(printf '%s\n' "${pg_status}" \
+			| sed -n 's/^pg_ctl:.*(PID: *\([0-9]*\))$/\1/p')
+
+		# PostgreSQL < 9.6: pgsql
+		# PostgreSQL >= 9.6: postgres
+		ps -o user -p "${pg_pid}" | sed -n '2p'
+		;;
+	(netbsd)
+		echo 'pgsql'
+		;;
+	(openbsd)
+		echo '_postgresql'
+		;;
+	(suse)
+		echo 'postgres'
+		;;
+	(*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac

cdist/conf/type/__postgres_conf/explorer/state

@@ -0,0 +1,223 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+postgres_user=$("${__type_explorer:?}/postgres_user")
+conf_name=${__object_id:?}
+
+tolower() { printf '%s' "$*" | tr '[:upper:]' '[:lower:]'; }
+
+tobytes() {
+	# NOTE: This function treats everything as base 2.
+	#       It is not compatible with SI units.
+	awk 'BEGIN { FS = "\n" }
+	/TB$/ { $0 = ($0 * 1024) "GB" }
+	/GB$/ { $0 = ($0 * 1024) "MB" }
+	/MB$/ { $0 = ($0 * 1024) "kB" }
+	/kB$/ { $0 = ($0 * 1024) "B" }
+	 /B?$/ { sub(/ *B?$/, "") }
+	($0*1) == $0  # is number
+	' <<-EOF
+	$1
+	EOF
+}
+
+tomillisecs() {
+	awk 'BEGIN { FS = "\n" }
+	    /d$/ { $0 = ($0 * 24) "h" }
+	    /h$/ { $0 = ($0 * 60) "min" }
+	  /min$/ { $0 = ($0 * 60) "s" }
+	/[^m]s$/ { $0 = ($0 * 1000) "ms" }
+	   /ms$/ { $0 *= 1 }
+	($0*1) == $0  # is number
+	' <<-EOF
+	$1
+	EOF
+}
+
+tobool() {
+	# prints either 'on' or 'off'
+	case $(tolower "$1")
+	in
+		(t|true|y|yes|on|1)
+			echo 'on' ;;
+		(f|false|n|no|off|0)
+			echo 'off' ;;
+		(*)
+			printf 'Inavlid bool value: %s\n' "$2" >&2
+			return 1
+			;;
+	esac
+	return 0
+}
+
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_exec() {
+	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
+}
+
+psql_conf_source() {
+	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
+	psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')"
+}
+psql_conf_cmp() (
+	IFS='|' read -r lower_name vartype setting unit <<-EOF
+	$(psql_exec "SELECT lower(name), vartype, setting, unit FROM pg_settings WHERE lower(name) = lower('$1')")
+	EOF
+
+	should_value=$2
+	is_value=${setting}
+
+	# The following case contains special cases for special settings.
+	case ${lower_name}
+	in
+		(archive_command)
+			if test "${setting}" = '(disabled)'
+			then
+				# DAFUQ PostgreSQL?!
+				# PostgreSQL returns (disabled) if the feature is inactive.
+				# We cannot compare the values unless it is enabled, first.
+				return 0
+			fi
+			;;
+		(archive_mode|backslash_quote|constraint_exclusion|force_parallel_mode|huge_pages|synchronous_commit)
+			# Although only 'on', 'off' are documented, PostgreSQL accepts all
+			# the "likely" variants of "on" and "off".
+			case $(tolower "${should_value}")
+			in
+				(on|off|true|false|yes|no|1|0)
+					should_value=$(tobool "${should_value}")
+					;;
+			esac
+			;;
+	esac
+
+	case ${vartype}
+	in
+		(bool)
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported for vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			should_value=$(tobool "${should_value}")
+
+			test "${is_value}" = "${should_value}"
+			;;
+		(enum)
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported with vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			# NOTE: All enums that are currently defined are lower case, but
+			#       PostgreSQL also accepts upper case spelling.
+			should_value=$(tolower "$2")
+
+			test "${is_value}" = "${should_value}"
+			;;
+		(integer)
+			# split multiples from unit, first (e.g. 8kB -> 8, kB)
+			case ${unit}
+			in
+				([0-9]*)
+					multiple=${unit%%[!0-9]*}
+					unit=${unit##*[0-9 ]}
+					;;
+				(*) multiple=1 ;;
+			esac
+
+			is_value=$((setting * multiple))${unit}
+
+			if expr "${should_value}" : '-\{0,1\}[0-9]*$' >/dev/null
+			then
+				# default unit
+				should_value=$((should_value * multiple))${unit}
+			fi
+
+			# then, do conversion
+			# NOTE: these conversions work for integers only!
+			case ${unit}
+			in
+				(B|[kMGT]B)
+					# bytes
+					is_bytes=$(tobytes "${is_value}")
+					should_bytes=$(tobytes "${should_value}")
+
+					test $((is_bytes)) -eq $((should_bytes))
+					;;
+				(ms|s|min|h|d)
+					# seconds
+					is_ms=$(tomillisecs "${is_value}")
+					should_ms=$(tomillisecs "${should_value}")
+
+					test $((is_ms)) -eq $((should_ms))
+					;;
+				('')
+					# no unit
+					is_int=${is_value}
+					should_int=${should_value}
+
+					test $((is_int)) -eq $((should_int))
+					;;
+			esac
+			;;
+		(real|string)
+			# NOTE: reals could possibly have units, but currently there none.
+
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported with vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			test "${is_value}" = "${should_value}"
+			;;
+	esac
+)
+
+psql_exec 'SELECT 1' >/dev/null || {
+	echo 'Connection to PostgreSQL server failed' >&2
+	exit 1
+}
+
+case $(psql_conf_source "${conf_name}")
+in
+	('')
+		printf 'Invalid configuration parameter: %s\n' "${conf_name}" >&2
+		exit 1
+		;;
+	(default)
+		echo absent
+		;;
+	(*)
+		if ! test -f "${__object:?}/parameter/value"
+		then
+			echo present
+		elif psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
+		then
+			echo present
+		else
+			echo different
+		fi
+		;;
+esac

cdist/conf/type/__postgres_conf/gencode-remote

@@ -0,0 +1,123 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2019-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+# 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
+
+conf_name=${__object_id:?}
+
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
+
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
+
+
+psql_cmd() {
+	printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")"
+}
+
+case ${state_should}
+in
+	(present)
+		test -n "${__object:?}/parameter/value" || {
+			echo 'Missing required parameter --value' >&2
+			exit 1
+		}
+
+		cat <<-EOF
+		exec 3< "\${__object:?}/parameter/value"
+		$(psql_cmd postgres -tAwq -o /dev/null -v ON_ERROR_STOP=on) <<'SQL'
+		\\set conf_value \`cat <&3\`
+		ALTER SYSTEM SET ${conf_name} = :'conf_value';
+		SELECT pg_reload_conf();
+		SQL
+		exec 3<&-
+		EOF
+		;;
+	(absent)
+		psql_cmd postgres -qwc "ALTER SYSTEM SET ${conf_name} TO DEFAULT"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		printf 'Only "present" and "absent" are acceptable.\n' >&2
+		exit 1
+		;;
+esac
+
+# Restart PostgreSQL server if required to apply new configuration value
+cat <<EOF
+
+if test 't' = "\$($(psql_cmd postgres -twAc "SELECT pending_restart FROM pg_settings WHERE lower(name) = lower('${conf_name}')"))"
+then
+	$(
+		init=$(cat "${__global:?}/explorer/init")
+		case ${init}
+		in
+			(systemd)
+				echo 'systemctl restart postgresql.service'
+				;;
+			(*openrc*)
+				echo 'rc-service postgresql restart'
+				;;
+			(sysvinit)
+				echo '/etc/init.d/postgresql restart'
+				;;
+			(init)
+				case $(cat "${__global:?}/explorer/kernel_name")
+				in
+					(FreeBSD)
+						echo '/usr/local/etc/rc.d/postgresql restart'
+						;;
+					(OpenBSD|NetBSD)
+						echo '/etc/rc.d/postgresql restart'
+						;;
+					(*)
+						echo "Unsupported operating system. Don't know how to restart services." >&2
+						exit 1
+				esac
+				;;
+			(*)
+				printf "Don't know how to restart services with your init (%s)\n" "${init}" >&2
+				exit 1
+		esac
+	)
+fi
+EOF

cdist/conf/type/__postgres_conf/man.rst

@@ -0,0 +1,60 @@
+cdist-type__postgres_conf(7)
+============================
+
+NAME
+----
+cdist-type__postgres_conf - Alter PostgreSQL configuration
+
+
+DESCRIPTION
+-----------
+Configure a running PostgreSQL server using ``ALTER SYSTEM``.
+
+
+REQUIRED PARAMETERS
+-------------------
+value
+   The value to set (can be omitted if ``--state`` is set to ``absent``).
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+   ``present`` or ``absent``.
+   Defaults to ``present``.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+   # set timezone
+   __postgres_conf timezone --value Europe/Zurich
+
+   # reset maximum number of concurrent connections to default (normally 100)
+   __postgres_conf max_connections --state absent
+
+
+SEE ALSO
+--------
+None.
+
+
+AUTHORS
+-------
+Beni Ruef (bernhard.ruef--@--ssrq-sds-fds.ch)
+Dennis Camera (dennis.camera--@--ssrq-sds-fds.ch)
+
+
+COPYING
+-------
+Copyright \(C) 2019-2021 SSRQ (www.ssrq-sds-fds.ch).
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.

cdist/conf/type/__postgres_conf/parameter/default/state

@@ -0,0 +1 @@
+present

cdist/conf/type/__postgres_conf/parameter/optional

@@ -0,0 +1,2 @@
+state
+value

cdist/conf/type/__postgres_database/explorer/postgres_user

@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user

cdist/conf/type/__postgres_database/explorer/state

@@ -1,6 +1,7 @@
 #!/bin/sh
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,25 +19,18 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$("${__explorer}/os")"
+postgres_user=$("${__type_explorer:?}/postgres_user")
-in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
-esac
 
+dbname=${__object_id:?}
 
-name="$__object_id"
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_exec() {
+	su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")"
+}
 
-if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='$name'\"")"
+if psql_exec postgres "SELECT datname FROM pg_database" | grep -qFx "${dbname}"
 then
-    echo 'present'
+	echo 'present'
 else
-    echo 'absent'
+	echo 'absent'
 fi

cdist/conf/type/__postgres_database/gencode-remote

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,60 +19,63 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
+quote() {
-in
+	for _arg
-   netbsd)
+	do
-      postgres_user='pgsql'
+		shift
-      ;;
+		if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
-   openbsd)
+		then
-      postgres_user='_postgresql'
+			# needs quoting
-      ;;
+			set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'"
-   *)
+		else
-      postgres_user='postgres'
+			set -- "$@" "${_arg}"
-      ;;
+		fi
-esac
+	done
+	unset _arg
 
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
 
-name="$__object_id"
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
-state_should="$(cat "$__object/parameter/state")"
-state_is="$(cat "$__object/explorer/state")"
 
-if [ "$state_should" != "$state_is" ]; then
+dbname=${__object_id:?}
-   case "$state_should" in
+state_should=$(cat "${__object:?}/parameter/state")
-      present)
+state_is=$(cat "${__object:?}/explorer/state")
-         owner=""
-         if [ -f "$__object/parameter/owner" ]; then
-            owner="-O \"$(cat "$__object/parameter/owner")\""
-         fi
 
-         template=""
+if test "${state_should}" = "$state_is"
-         if [ -f "$__object/parameter/template" ]; then
+then
-            template="--template \"$(cat "$__object/parameter/template")\""
+	exit 0
-         fi
-
-         encoding=""
-         if [ -f "$__object/parameter/encoding" ]; then
-            encoding="--encoding \"$(cat "$__object/parameter/encoding")\""
-         fi
-
-         lc_collate=""
-         if [ -f "$__object/parameter/lc-collate" ]; then
-            lc_collate="--lc-collate \"$(cat "$__object/parameter/lc-collate")\""
-         fi
-
-         lc_ctype=""
-         if [ -f "$__object/parameter/lc-ctype" ]; then
-            lc_ctype="--lc-ctype \"$(cat "$__object/parameter/lc-ctype")\""
-         fi
-
-         cat << EOF
-su - '$postgres_user' -c "createdb $owner \"$name\" $template $encoding $lc_collate $lc_ctype"
-EOF
-      ;;
-      absent)
-         cat << EOF
-su - '$postgres_user' -c "dropdb \"$name\""
-EOF
-      ;;
-   esac
 fi
+
+case ${state_should}
+in
+	(present)
+		set --
+
+		while read -r param_name opt
+		do
+			if test -f "${__object:?}/parameter/${param_name}"
+			then
+				set -- "$@" "${opt}" "$(cat "${__object:?}/parameter/${param_name}")"
+			fi
+		done <<-'EOF'
+		owner -O
+		template --template
+		encoding --encoding
+		lc_collate --lc-collate
+		lc_ctype --lc-ctype
+		EOF
+
+		set -- "$@" "${dbname}"
+
+		cat <<-EOF
+		su - $(quote "${postgres_user}") -c $(quote "$(quote createdb "$@")")
+		EOF
+		;;
+	(absent)
+		cat <<-EOF
+		su - $(quote "${postgres_user}") -c $(quote "$(quote dropdb "${dbname}")")
+		EOF
+		;;
+esac

cdist/conf/type/__postgres_extension/explorer/postgres_user

@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user

cdist/conf/type/__postgres_extension/explorer/state

@@ -0,0 +1,41 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Prints "present" if the extension is currently installed.
+# "absent" otherwise.
+
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+
+postgres_user=$("${__type_explorer:?}/postgres_user")
+
+IFS=: read -r dbname extname <<EOF
+${__object_id:?}
+EOF
+
+psql_exec() {
+	su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")"
+}
+
+if psql_exec "${dbname}" 'SELECT extname FROM pg_extension' | grep -qFx "${extname}"
+then
+	echo present
+else
+	echo absent
+fi

cdist/conf/type/__postgres_extension/gencode-remote

@@ -2,9 +2,10 @@
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2013 Tomas Pospisek (tpo_deb at sourcepole.ch)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This type was created by Tomas Pospisek based on the
-#__postgres_role type by Steven Armstrong
+# __postgres_role type by Steven Armstrong.
 #
 # This file is part of cdist.
 #
@@ -22,32 +23,38 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
+
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_cmd() {
+	printf 'su - %s -c %s\n' \
+		"$(quote "${postgres_user}")" \
+		"$(quote psql "$(quote "$1")" -c "$(quote "$2")")"
+}
+
+
+IFS=: read -r dbname extname <<EOF
+${__object_id:?}
+EOF
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
+
+case ${state_should}
 in
-   netbsd)
+	(present)
-      postgres_user='pgsql'
+		psql_cmd "${dbname}" "CREATE EXTENSION ${extname}"
-      ;;
+		;;
-   openbsd)
+	(absent)
-      postgres_user='_postgresql'
+		psql_cmd "${dbname}" "DROP EXTENSION ${extname}"
-      ;;
+		;;
-   *)
+	(*)
-      postgres_user='postgres'
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
-      ;;
+		exit 1
-esac
+		;;
-
-
-dbname=$(    echo "$__object_id" | cut -d":" -f1 )
-extension=$( echo "$__object_id" | cut -d":" -f2 )
-
-state_should=$( cat "$__object/parameter/state" )
-
-case "$state_should" in
-    present)
-         cmd="CREATE EXTENSION IF NOT EXISTS $extension"
-         echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'"
-    ;;
-    absent)
-         cmd="DROP EXTENSION IF EXISTS $extension"
-         echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'"
-    ;;
 esac

cdist/conf/type/__postgres_extension/man.rst

@@ -3,32 +3,36 @@ cdist-type__postgres_extension(7)
 
 NAME
 ----
-cdist-type__postgres_extension - manage postgres extensions
+cdist-type__postgres_extension - Manage PostgreSQL extensions
 
 
 DESCRIPTION
 -----------
-This cdist type allows you to create or drop postgres extensions.
+This cdist type allows you to manage PostgreSQL extensions.
 
-The object you need to pass to __postgres_extension consists of
+The ``__object_id`` to pass to ``__postgres_extension`` is of the form
-the database name and the extension name joined by a colon in the
+``dbname:extension``, e.g.:
-following form:
-
-.. code-block:: sh
-
-    dbname:extension
-
-f.ex.
 
 .. code-block:: sh
 
     rails_test:unaccent
 
 
+**CAUTION!** Be careful when installing extensions from (untrusted) third-party
+sources:
+
+   | Installing an extension as superuser requires trusting that the extension's
+     author wrote the extension installation script in a secure fashion. It is
+     not terribly difficult for a malicious user to create trojan-horse objects
+     that will compromise later execution of a carelessly-written extension
+     script, allowing that user to acquire superuser privileges.
+   | – `<https://www.postgresql.org/docs/13/sql-createextension.html#id-1.9.3.64.7>`_
+
+
 OPTIONAL PARAMETERS
 -------------------
 state
-    either "present" or "absent", defaults to "present"
+    either ``present`` or ``absent``, defaults to ``present``.
 
 
 EXAMPLES
@@ -36,24 +40,29 @@ EXAMPLES
 
 .. code-block:: sh
 
-    __postgres_extension           rails_test:unaccent
+   # Install extension unaccent into database rails_test
-    __postgres_extension --present rails_test:unaccent
+   __postgres_extension rails_test:unaccent
-    __postgres_extension --absent  rails_test:unaccent
+
+   # Drop extension unaccent from database fails_test
+   __postgres_extension rails_test:unaccent --state absent
 
 
 SEE ALSO
 --------
-:strong:`cdist-type__postgre_database`\ (7)
+- :strong:`cdist-type__postgres_database`\ (7)
+- PostgreSQL "CREATE EXTENSION" documentation at:
+  `<http://www.postgresql.org/docs/current/static/sql-createextension.html>`_.
 
-Postgres "Create Extension" documentation at: <http://www.postgresql.org/docs/current/static/sql-createextension.html>.
 
-AUTHOR
+AUTHORS
 -------
-Tomas Pospisek <tpo_deb--@--sourcepole.ch>
+| Tomas Pospisek <tpo_deb--@--sourcepole.ch>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+
 
 COPYING
 -------
-Copyright \(C) 2014 Tomas Pospisek. You can redistribute it
+Copyright \(C) 2014 Tomas Pospisek, 2021 Dennis Camera.
-and/or modify it under the terms of the GNU General Public License as
+You can redistribute it and/or modify it under the terms of the GNU General
-published by the Free Software Foundation, either version 3 of the
+Public License as published by the Free Software Foundation, either version 3 of
-License, or (at your option) any later version.
+the License, or (at your option) any later version.

cdist/conf/type/__postgres_role/explorer/postgres_user

@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user

cdist/conf/type/__postgres_role/explorer/state

@@ -19,19 +19,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case $("${__explorer:?}/os")
+postgres_user=$("${__type_explorer:?}/postgres_user")
-in
-	(netbsd)
-		postgres_user='pgsql'
-		;;
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(*)
-		postgres_user='postgres'
-		;;
-esac
-
 rolename=${__object_id:?}
 
 
@@ -55,8 +43,7 @@ role_properties=$(
 	  BEGIN { RS = "\036"; FS = "\034" }
 	  /^\([0-9]+ rows?\)/ { exit }
 	  NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next }
-	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }
+	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }'
-	  '
 )
 
 if test -n "${role_properties}"
@@ -90,12 +77,10 @@ then
 	# Check password
 	passwd_stored=$(
 		psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \
-		| awk 'BEGIN { RS = "\036" } NR == 2'
+		| awk 'BEGIN { RS = "\036" } NR == 2 { printf "%s.", $0 }')
-		printf .
+	passwd_stored=${passwd_stored%.}
-	)
-	passwd_stored=${passwd_stored%?.}
 
-	if test -f "${__object:?}/parameter/password"
+	if test -s "${__object:?}/parameter/password"
 	then
 		passwd_should=$(cat "${__object:?}/parameter/password"; printf .)
 	fi

cdist/conf/type/__postgres_role/gencode-remote

@@ -28,20 +28,7 @@ quote() {
 	fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
 }
 
-case $(cat "${__global:?}/explorer/os")
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
-in
-	(netbsd)
-		postgres_user='pgsql'
-		;;
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(*)
-		postgres_user='postgres'
-		;;
-esac
-
-
 rolename=${__object_id:?}
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
@@ -59,7 +46,7 @@ psql_query() {
 
 psql_set_password() {
 	# NOTE: Always make sure that the password does not end up in psql_history!
-	# NOTE: Never set an empty string as the password, because they can be
+	# NOTE: Never set an empty string as the password, because it can be
 	#       interpreted differently by different tooling.
 	if test -s "${__object:?}/parameter/password"
 	then

cdist/conf/type/__pyvenv/explorer/group

@@ -1,5 +1,24 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id"
+destination="/${__object_id:?}"
 
-stat --print "%G" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((group_gid + 1)) -ge 0
+then
+	group_should=$(cat "${__object:?}/parameter/group")
+
+	if expr "${group_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${group_gid}"
+	else
+		if command -v getent >/dev/null 2>&1
+		then
+			getent group "${group_gid}" | cut -d : -f 1
+		else
+			awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group
+		fi
+	fi
+fi

cdist/conf/type/__pyvenv/explorer/owner

@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id"
+destination="/${__object_id:?}"
 
-stat --print "%U" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((owner_uid + 1)) -ge 0
+then
+	owner_should=$(cat "${__object:?}/parameter/owner")
+
+	if expr "${owner_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${owner_uid}"
+	else
+		printf '%s\n' "$(id -u -n "${owner_uid}")"
+	fi
+fi

cdist/conf/type/__pyvenv/man.rst

@@ -61,7 +61,7 @@ EXAMPLES
     __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4
 
     # Create python virtualenv for user foo.
-    __pyvenv /home/foo/fooenv --group foo --user foo
+    __pyvenv /home/foo/fooenv --group foo --owner foo
 
     # Create python virtualenv with specific parameters.
     __pyvenv /home/services/djangoenv --venvparams "--copies --system-site-packages"

cdist/conf/type/__rsync/gencode-local

@@ -1,39 +1,104 @@
 #!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
-source=$(cat "$__object/parameter/source")
+if ! command -v rsync > /dev/null
-remote_user=$(cat "$__object/parameter/remote-user")
+then
+    echo 'rsync is missing in local machine' >&2
+    exit 1
+fi
 
-if [ -f "$__object/parameter/destination" ]; then
+src="$( cat "$__object/parameter/source" )"
-    destination=$(cat "$__object/parameter/destination")
+
+if [ ! -e "$src" ]
+then
+    echo "$src not found" >&2
+    exit 1
+fi
+
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
 else
-    destination="/$__object_id"
+    dst="/$__object_id"
 fi
 
-set --
+# if source is directory, then make sure that
-if [ -f "$__object/parameter/rsync-opts" ]; then
+# source and destination are ending with slash,
-    while read -r opts; do
+# because this is what you almost always want when
-        set -- "$@" "--$opts"
+# rsyncing two directories.
-    done < "$__object/parameter/rsync-opts"
+
+if [ -d "$src" ]
+then
+    if ! echo "$src" | grep -Eq '/$'
+    then
+        src="$src/"
+    fi
+
+    if ! echo "$dst" | grep -Eq '/$'
+    then
+        dst="$dst/"
+    fi
 fi
 
-echo rsync -a \
+remote_user="$( cat "$__object/parameter/remote-user" )"
-    --no-owner --no-group \
+
-    -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"
+options="$( cat "$__object/parameter/options" )"
+
+if [ -f "$__object/parameter/option" ]
+then
+    while read -r l
+    do
+        # there's a limitation in argparse: value can't begin with '-'.
+        # to workaround this, let's prefix opts with '\' in manifest and remove here.
+        # read more about argparse issue: https://bugs.python.org/issue9334
+
+        options="$options $( echo "$l" | sed 's/\\//g' )"
+    done \
+        < "$__object/parameter/option"
+fi
+
+if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ]
+then
+    options="$options --chown="
+
+    if [ -f "$__object/parameter/owner" ]
+    then
+        owner="$( cat "$__object/parameter/owner" )"
+        options="$options$owner"
+    fi
+
+    if [ -f "$__object/parameter/group" ]
+    then
+        group="$( cat "$__object/parameter/group" )"
+        options="$options:$group"
+    fi
+fi
+
+if [ -f "$__object/parameter/mode" ]
+then
+    mode="$( cat "$__object/parameter/mode" )"
+    options="$options --chmod=$mode"
+fi
+
+# IMPORTANT
+#
+# 1. we first dry-run rsync with change summary to find out
+#    if there are any changes and code generation is needed.
+# 2. normally, to get current state or target host, we run
+#    such operations in type explorers, but that's not
+#    possible due to how rsync works.
+# 3. redirecting output of dry-run to stderr to ease debugging.
+# 4. to understand how that cryptic regex works, please
+#    open rsync manpage and read about --itemize-changes.
+
+export RSYNC_RSH="$__remote_exec"
+
+# shellcheck disable=SC2086
+if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \
+    | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2
+then
+    exit 0
+fi
+
+echo "export RSYNC_RSH='$__remote_exec'"
+
+echo "rsync $options $src $remote_user@$__target_host:$dst"

cdist/conf/type/__rsync/man.rst

@@ -3,112 +3,73 @@ cdist-type__rsync(7)
 
 NAME
 ----
-cdist-type__rsync - Mirror directories using rsync
+cdist-type__rsync - Mirror directories using ``rsync``
 
 
 DESCRIPTION
 -----------
-WARNING: This type is of BETA quality:
+The purpose of this type is to bring power of ``rsync`` into ``cdist``.
-
-- it has not been tested widely
-- interfaces *may* change
-- if there is a better approach to solve the problem -> the type may even vanish
-
-If you are fine with these constraints, please read on.
-
-
-This cdist type allows you to mirror local directories to the
-target host using rsync. Rsync will be installed in the manifest of the type.
-If group or owner are giveng, a recursive chown will be executed on the 
-target host.
-
-A slash will be appended to the source directory so that only the contents
-of the directory are taken and not the directory name itself.
 
 
 REQUIRED PARAMETERS
 -------------------
 source
-    Where to take files from
+   Source directory in local machine.
+   If source is directory, slash (``/``) will be added to source and destination paths.
 
 
 OPTIONAL PARAMETERS
 -------------------
-group
+destination
-   Group to chgrp to.
+   Destination directory. Defaults to ``$__object_id``.
 
 owner
-   User to chown to.
+   Will be passed to ``rsync`` as ``--chown=OWNER``.
+   Read ``rsync(1)`` for more details.
 
-destination
+group
-    Use this as the base destination instead of the object id
+   Will be passed to ``rsync`` as ``--chown=:GROUP``.
+   Read ``rsync(1)`` for more details.
+
+mode
+   Will be passed to ``rsync`` as ``--chmod=MODE``.
+   Read ``rsync(1)`` for more details.
+
+options
+   Defaults to ``--recursive --links --perms --times``.
+   Due to `bug in Python's argparse<https://bugs.python.org/issue9334>`_, value must be prefixed with ``\``.
 
 remote-user
-    Use this user instead of the default "root" for rsync operations.
+   Defaults to ``root``.
 
 
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
-rsync-opts
+option
-    Use this option to give rsync options with.
+   Pass additional options to ``rsync``.
-    See rsync(1) for available options.
+   See ``rsync(1)`` for all possible options.
-    Only "--" options are supported.
+   Due to `bug in Python's argparse<https://bugs.python.org/issue9334>`_, value must be prefixed with ``\``.
-    Write the options without the beginning "--"
-    Can be specified multiple times.
-
-
-MESSAGES
---------
-NONE
 
 
 EXAMPLES
 --------
-
 .. code-block:: sh
 
-    # You can use any source directory
+    __rsync /var/www/example.com \
-    __rsync /tmp/testdir \
+        --owner root \
-        --source /etc
+        --group www-data \
-
+        --mode 'D750,F640' \
-    # Use source from type
+        --source "$__files/example.com/www"
-    __rsync /etc \
-        --source "$__type/files/package"
-
-    # Allow multiple __rsync objects to write to the same dir
-    __rsync mystuff \
-        --destination /usr/local/bin \
-        --source "$__type/files/package"
-
-    __rsync otherstuff \
-        --destination /usr/local/bin \
-        --source "$__type/files/package2"
-
-    # Use rsync option --exclude
-    __rsync /tmp/testdir \
-        --source /etc \
-        --rsync-opts exclude=sshd_conf
-
-    # Use rsync with multiple options --exclude --dry-run
-    __rsync /tmp/testing \
-        --source /home/tester \
-        --rsync-opts exclude=id_rsa \
-        --rsync-opts dry-run
-
-
-SEE ALSO
---------
-:strong:`rsync`\ (1)
 
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+Ander Punnar <ander-at-kvlt-dot-ee>
 
 
 COPYING
 -------
-Copyright \(C) 2015 Nico Schottelius. You can redistribute it
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
-and/or modify it under the terms of the GNU General Public License as
+under the terms of the GNU General Public License as published by the Free
-published by the Free Software Foundation, either version 3 of the
+Software Foundation, either version 3 of the License, or (at your option)
-License, or (at your option) any later version.
+any later version.

cdist/conf/type/__rsync/manifest

@@ -1,21 +1,3 @@
 #!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
 __package rsync

cdist/conf/type/__rsync/parameter/default/options

@@ -0,0 +1 @@
+--recursive --links --perms --times

cdist/conf/type/__rsync/parameter/optional

@@ -1,4 +1,6 @@
 destination
-owner
 group
+mode
+options
+owner
 remote-user

cdist/conf/type/__rsync/parameter/optional_multiple

@@ -1 +1 @@
-rsync-opts
+option

cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group

@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+if grep -Eq '^ssl-cert:' /etc/group
+then
+    echo 'present'
+else
+    echo 'absent'
+fi

cdist/conf/type/__snakeoil_cert/explorer/state

@@ -0,0 +1,24 @@
+#!/bin/sh -e
+
+key_path="$( cat "$__object/parameter/key-path" )"
+
+if echo "$key_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    key_path="$( printf "$key_path" "$__object_id" )"
+fi
+
+cert_path="$( cat "$__object/parameter/cert-path" )"
+
+if echo "$cert_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    cert_path="$( printf "$cert_path" "$__object_id" )"
+fi
+
+if [ ! -f "$key_path" ] || [ ! -f "$cert_path" ]
+then
+    echo 'absent'
+else
+    echo 'present'
+fi

cdist/conf/type/__snakeoil_cert/gencode-remote

@@ -0,0 +1,73 @@
+#!/bin/sh -e
+
+state="$( cat "$__object/explorer/state" )"
+
+if [ "$state" = 'present' ]
+then
+    exit 0
+fi
+
+if [ -f "$__object/parameter/common-name" ]
+then
+    common_name="$( cat "$__object/parameter/common-name" )"
+else
+    common_name="$__object_id"
+fi
+
+key_path="$( cat "$__object/parameter/key-path" )"
+
+if echo "$key_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    key_path="$( printf "$key_path" "$__object_id" )"
+fi
+
+cert_path="$( cat "$__object/parameter/cert-path" )"
+
+if echo "$cert_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    cert_path="$( printf "$cert_path" "$__object_id" )"
+fi
+
+key_type="$( cat "$__object/parameter/key-type" )"
+
+key_type_arg="$( echo "$key_type" | cut -d : -f 2 )"
+
+case "$key_type" in
+    rsa:*)
+        echo "openssl genrsa -out '$key_path' $key_type_arg"
+    ;;
+    ec:*)
+        echo "openssl ecparam -name $key_type_arg -genkey -noout -out '$key_path'"
+    ;;
+esac
+
+# shellcheck disable=SC2016
+echo 'csr_path="$( mktemp )"'
+
+echo "openssl req -new -subj '/CN=$common_name' -key '$key_path' -out \"\$csr_path\""
+
+echo "openssl x509 -req -sha256 -days 3650 -in \"\$csr_path\" -signkey '$key_path' -out '$cert_path'"
+
+# shellcheck disable=SC2016
+echo 'rm -f "$csr_path"'
+
+if [ "$( cat "$__object/explorer/ssl-cert-group" )" = 'present' ]
+then
+    key_group='ssl-cert'
+else
+    key_group='root'
+fi
+
+echo "chmod 640 '$key_path'"
+
+echo "chown root '$key_path'"
+
+echo "chgrp $key_group '$key_path'"
+
+echo "chmod 644 '$cert_path'"
+
+echo "chown root '$cert_path'"
+
+echo "chgrp root '$cert_path'"

cdist/conf/type/__snakeoil_cert/man.rst

@@ -0,0 +1,61 @@
+cdist-type__snakeoil_cert(7)
+============================
+
+NAME
+----
+cdist-type__snakeoil_cert - Generate self-signed certificate
+
+
+DESCRIPTION
+-----------
+The purpose of this type is to generate **self-signed** certificate and private key
+for **testing purposes**. Certificate will expire in 3650 days.
+
+Certificate's and key's access bits will be ``644`` and ``640`` respectively.
+If target system has ``ssl-cert`` group, then it will be used as key's group.
+Use ``require='__snakeoil_cert/...' __file ...`` to override.
+
+
+OPTIONAL PARAMETERS
+-------------------
+common-name
+   Defaults to ``$__object_id``.
+
+key-path
+   ``%s`` in path will be replaced with ``$__object_id``.
+   Defaults to ``/etc/ssl/private/%s.pem``.
+
+key-type
+   Possible values are ``rsa:$bits`` and ``ec:$name``.
+   For possible EC names see ``openssl ecparam -list_curves``.
+   Defaults to ``rsa:2048``.
+
+cert-path
+   ``%s`` in path will be replaced with ``$__object_id``.
+   Defaults to ``/etc/ssl/certs/%s.pem``.
+
+
+EXAMPLES
+--------
+.. code-block:: sh
+
+	__snakeoil_cert localhost-rsa \
+	    --common-name localhost \
+	    --key-type rsa:4096
+
+	__snakeoil_cert localhost-ec \
+	    --common-name localhost \
+	    --key-type ec:prime256v1
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.

cdist/conf/type/__snakeoil_cert/parameter/default/cert-path

@@ -0,0 +1 @@
+/etc/ssl/certs/%s.pem

cdist/conf/type/__snakeoil_cert/parameter/default/key-path

@@ -0,0 +1 @@
+/etc/ssl/private/%s.pem

cdist/conf/type/__snakeoil_cert/parameter/default/key-type

@@ -0,0 +1 @@
+rsa:2048

cdist/conf/type/__snakeoil_cert/parameter/optional

@@ -0,0 +1,4 @@
+common-name
+key-path
+key-type
+cert-path

cdist/conf/type/__ssh_authorized_key/explorer/entry

@@ -25,6 +25,7 @@ type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ]
 if [ -n "${type_and_key}" ]
 then
     file="$(cat "$__object/parameter/file")"
+    test -e "$file" || exit 0
 
     # get any entries that match the type and key
 

cdist/conf/type/__ssh_authorized_key/gencode-remote

@@ -37,9 +37,9 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
 # preserve ownership and permissions of existing file
 if [ -f "$file" ]; then
    cp -p "$file" "\$tmpfile"
+   grep -v -F -x '$line' '$file' >\$tmpfile
 fi
-grep -v -F -x '$line' '$file' > \$tmpfile || true
+cat "\$tmpfile" >"$file"
-mv -f "\$tmpfile" "$file"
 DONE
 }
 

cdist/conf/type/__ssh_authorized_keys/explorer/keys

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 
 # shellcheck disable=SC1090
+# shellcheck disable=SC1091
 file="$( . "$__type_explorer/file" )"
 
 if [ -f "$file" ]

cdist/conf/type/__sshd_config/manifest

@@ -39,7 +39,14 @@ in
 	(freebsd|netbsd|openbsd)
 		# whitelist
 		;;
+	(openbmc-phosphor)
+		# whitelist
+		# OpenBMC can be configured with dropbear and OpenSSH.
+		# If dropbear is used, the state explorer will already fail because it
+		# cannot find the sshd binary.
+		;;
 	(*)
+		: "${__type:?}"  # make shellcheck happy
 		printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \
 			"${os}" "${__type##*/}" >&2
 		printf 'Please contribute an implementation for it if you can.\n' >&2

cdist/conf/type/__update_alternatives/explorer/alternatives

@@ -1,4 +1,4 @@
 #!/bin/sh -e
 
-update-alternatives --display "$__object_id" 2>/dev/null \
+LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \
-    | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
+| awk -F ' - ' '/priority [0-9]+$/ { print $1 }'

cdist/conf/type/__update_alternatives/explorer/link

@@ -18,12 +18,12 @@ for altdir in \
     /var/lib/dpkg/alternatives \
     /var/lib/alternatives
 do
-    if [ ! -f "$altdir/$__object_id" ]
+    if [ ! -f "$altdir/${__object_id:?}" ]
     then
         continue
     fi
 
-    link="$( awk 'NR==2' "$altdir/$__object_id" )"
+    link="$( awk 'NR==2' "$altdir/${__object_id:?}" )"
 
     if [ -n "$link" ]
     then
@@ -31,9 +31,12 @@ do
     fi
 done
 
-if [ -z "$link" ]
+if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then
-    echo "unable to get link for $__object_id" >&2
+    # NOTE: ignore error for dry-runs because a package providing the link
+    #       might be managed by another cdist object (which wasn't executed,
+    #       because dry run…).
+    echo "unable to get link for ${__object_id:?}" >&2
     exit 1
 fi
 

cdist/conf/type/__update_alternatives/explorer/path_is

@@ -1,11 +1,15 @@
 #!/bin/sh -e
 
-path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \
+path_is=$(
-    | awk '/link currently points to/ {print $5}' )"
+    LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \
+    | awk '/link currently points to/ { print $5 }')
 
-if [ -z "$path_is" ]
+if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then
-    echo "unable to get current path for $__object_id" >&2
+    # NOTE: ignore error for dry-runs because a package providing the
+    #       alternative might be managed by another cdist object (which
+    #       wasn't executed, because dry run…).
+    echo "unable to get current path for ${__object_id:?}" >&2
     exit 1
 fi
 

cdist/conf/type/__update_alternatives/explorer/path_should_state

@@ -1,6 +1,6 @@
 #!/bin/sh -e
 
-if [ -f "$( cat "$__object/parameter/path" )" ]
+if [ -f "$( cat "${__object:?}/parameter/path" )" ]
 then
     echo 'present'
 else

cdist/conf/type/__update_alternatives/gencode-remote

@@ -18,37 +18,39 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 
-path_is="$( cat "$__object/explorer/path_is" )"
+path_is="$( cat "${__object:?}/explorer/path_is" )"
 
-path_should="$( cat "$__object/parameter/path" )"
+path_should="$( cat "${__object:?}/parameter/path" )"
 
 if [ "$path_is" = "$path_should" ]
 then
     exit 0
 fi
 
-if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ]
+if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \
+    && [ -z "${__cdist_dry_run+dry run}" ]
 then
     echo "$path_should does not exist in target" >&2
     exit 1
 fi
 
-name="$__object_id"
+name=${__object_id:?}
 
-alternatives="$( cat "$__object/explorer/alternatives" )"
+if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives"
-
-if ! echo "$alternatives" | grep -Fxq "$path_should"
 then
-    if [ ! -f "$__object/parameter/install" ]
+    if [ -f "${__object:?}/parameter/install" ]
     then
+        link="$( cat "${__object:?}/explorer/link" )"
+        echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
+    elif [ -z "${__cdist_dry_run+dry run}" ]
+    then
+        # NOTE: ignore error for dry-runs because a package providing the link
+        #       to be installed might be managed by another cdist object (which
+        #       wasn't executed, because dry run…).
         echo "$path_should is not in $name alternatives." >&2
         echo 'Please install missing packages or use --install to add path to alternatives.' >&2
         exit 1
     fi
-
-    link="$( cat "$__object/explorer/link" )"
-
-    echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
 fi
 
 echo "update-alternatives --set '$name' '$path_should'"

cdist/config.py

@@ -190,7 +190,7 @@ class Config:
                     fd.write(sys.stdin.read())
             except (IOError, OSError) as e:
                 raise cdist.Error(("Creating tempfile for stdin data "
-                                   "failed: %s" % e))
+                                   "failed: {}").format(e))
 
             args.manifest = initial_manifest_temp_path
             atexit.register(lambda: os.remove(initial_manifest_temp_path))
@@ -273,15 +273,15 @@ class Config:
                     host_tags = None
             host_base_path, hostdir = cls.create_host_base_dirs(
                 host, base_root_path)
-            log.debug("Base root path for target host \"{}\" is \"{}\"".format(
+            log.debug("Base root path for target host \"%s\" is \"%s\"",
-                host, host_base_path))
+                      host, host_base_path)
 
             hostcnt += 1
             if args.parallel:
                 pargs = (host, host_tags, host_base_path, hostdir, args, True,
                          configuration)
-                log.trace(("Args for multiprocessing operation "
+                log.trace("Args for multiprocessing operation for host %s: %s",
-                           "for host {}: {}".format(host, pargs)))
+                          host, pargs)
                 process_args.append(pargs)
             else:
                 try:
@@ -298,10 +298,10 @@ class Config:
             except cdist.Error:
                 failed_hosts.append(host)
         elif args.parallel:
-            log.trace("Multiprocessing start method is {}".format(
+            log.trace("Multiprocessing start method is %s",
-                multiprocessing.get_start_method()))
+                      multiprocessing.get_start_method())
-            log.trace(("Starting multiprocessing Pool for {} "
+            log.trace("Starting multiprocessing Pool for %d parallel host"
-                       "parallel host operation".format(args.parallel)))
+                      " operation", args.parallel)
 
             results = mp_pool_run(cls.onehost,
                                   process_args,
@@ -396,16 +396,13 @@ class Config:
 
             remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds(
                 args)
-            log.debug("remote_exec for host \"{}\": {}".format(
+            log.debug("remote_exec for host \"%s\": %s", host, remote_exec)
-                host, remote_exec))
+            log.debug("remote_copy for host \"%s\": %s", host, remote_copy)
-            log.debug("remote_copy for host \"{}\": {}".format(
-                host, remote_copy))
 
             family = cls._address_family(args)
-            log.debug("address family: {}".format(family))
+            log.debug("address family: %s", family)
             target_host = cls.resolve_target_addresses(host, family)
-            log.debug("target_host for host \"{}\": {}".format(
+            log.debug("target_host for host \"%s\": %s", host, target_host)
-                host, target_host))
 
             local = cdist.exec.local.Local(
                 target_host=target_host,
@@ -420,6 +417,9 @@ class Config:
                 exec_path=sys.argv[0],
                 save_output_streams=args.save_output_streams)
 
+            # Make __global state dir available to custom remote scripts.
+            os.environ['__global'] = local.base_path
+
             remote = cdist.exec.remote.Remote(
                 target_host=target_host,
                 remote_exec=remote_exec,
@@ -471,8 +471,8 @@ class Config:
         """Do what is most often done: deploy & cleanup"""
         start_time = time.time()
 
-        self.log.info("Starting {} run".format(
+        self.log.info("Starting %s run",
-            'dry' if self.dry_run else 'configuration'))
+                      'dry' if self.dry_run else 'configuration')
 
         self._init_files_dirs()
 
@@ -490,9 +490,9 @@ class Config:
         self._remove_files_dirs()
 
         self.local.save_cache(start_time)
-        self.log.info("Finished {} run in {:.2f} seconds".format(
+        self.log.info("Finished %s run in %.2f seconds",
                       'dry' if self.dry_run else 'successful',
-                      time.time() - start_time))
+                      time.time() - start_time)
 
     def cleanup(self):
         self.log.debug("Running cleanup commands")
@@ -516,8 +516,8 @@ class Config:
                 self.local.object_path, self.local.type_path,
                 self.local.object_marker_name):
             if cdist_object.cdist_type.is_install:
-                self.log.debug(("Running in config mode, ignoring install "
+                self.log.debug("Running in config mode, ignoring install "
-                                "object: {0}").format(cdist_object))
+                               "object: %s", cdist_object)
             else:
                 yield cdist_object
 
@@ -537,7 +537,7 @@ class Config:
         objects_changed = False
 
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(
+            if cdist_object.has_requirements_unfinished(
                     cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
@@ -548,7 +548,7 @@ class Config:
                 self.object_prepare(cdist_object)
                 objects_changed = True
 
-            if cdist_object.requirements_unfinished(
+            if cdist_object.has_requirements_unfinished(
                     cdist_object.autorequire):
                 """The previous step created objects we depend on -
                     wait for them
@@ -562,13 +562,13 @@ class Config:
         return objects_changed
 
     def _iterate_once_parallel(self):
-        self.log.debug("Iteration in parallel mode in {} jobs".format(
+        self.log.debug("Iteration in parallel mode in %d jobs", self.jobs)
-            self.jobs))
         objects_changed = False
 
         cargo = []
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(cdist_object.requirements):
+            if cdist_object.has_requirements_unfinished(
+                    cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
 
@@ -585,8 +585,8 @@ class Config:
             self.object_prepare(cargo[0])
             objects_changed = True
         elif cargo:
-            self.log.trace("Multiprocessing start method is {}".format(
+            self.log.trace("Multiprocessing start method is %s",
-                multiprocessing.get_start_method()))
+                           multiprocessing.get_start_method())
 
             self.log.trace("Multiprocessing cargo: %s", cargo)
 
@@ -600,9 +600,8 @@ class Config:
                                 "sequentially"))
                 self.explorer.transfer_type_explorers(cargo_types.pop())
             else:
-                self.log.trace(("Starting multiprocessing Pool for {} "
+                self.log.trace("Starting multiprocessing Pool for %d "
-                                "parallel types explorers transferring".format(
+                               "parallel types explorers transferring", nt)
-                                    nt)))
                 args = [
                     (ct, ) for ct in cargo_types
                 ]
@@ -611,8 +610,8 @@ class Config:
                 self.log.trace(("Multiprocessing for parallel transferring "
                                 "types' explorers finished"))
 
-            self.log.trace(("Starting multiprocessing Pool for {} parallel "
+            self.log.trace("Starting multiprocessing Pool for %d parallel "
-                            "objects preparation".format(n)))
+                           "objects preparation", n)
             args = [
                 (c, False, ) for c in cargo
             ]
@@ -623,12 +622,13 @@ class Config:
 
         del cargo[:]
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(cdist_object.requirements):
+            if cdist_object.has_requirements_unfinished(
+                    cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
 
             if cdist_object.state == core.CdistObject.STATE_PREPARED:
-                if cdist_object.requirements_unfinished(
+                if cdist_object.has_requirements_unfinished(
                         cdist_object.autorequire):
                     """The previous step created objects we depend on -
                     wait for them
@@ -664,10 +664,10 @@ class Config:
                 self.object_run(chunk[0])
                 objects_changed = True
             elif chunk:
-                self.log.trace("Multiprocessing start method is {}".format(
+                self.log.trace("Multiprocessing start method is %s",
-                    multiprocessing.get_start_method()))
+                               multiprocessing.get_start_method())
-                self.log.trace(("Starting multiprocessing Pool for {} "
+                self.log.trace("Starting multiprocessing Pool for %d "
-                               "parallel object run".format(n)))
+                               "parallel object run", n)
                 args = [
                     (c, ) for c in chunk
                 ]
@@ -699,20 +699,22 @@ class Config:
             check for cycles.
         '''
         graph = {}
-        for cdist_object in self.object_list():
+
+        def _add_requirements(cdist_object, requirements):
             obj_name = cdist_object.name
             if obj_name not in graph:
                 graph[obj_name] = []
+
+            for requirement in cdist_object.requirements_unfinished(
+                    requirements):
+                graph[obj_name].append(requirement.name)
+
+        for cdist_object in self.object_list():
             if cdist_object.state == cdist_object.STATE_DONE:
                 continue
 
-            for requirement in cdist_object.requirements_unfinished(
+            _add_requirements(cdist_object, cdist_object.requirements)
-                    cdist_object.requirements):
+            _add_requirements(cdist_object, cdist_object.autorequire)
-                graph[obj_name].append(requirement.name)
-
-            for requirement in cdist_object.requirements_unfinished(
-                    cdist_object.autorequire):
-                graph[obj_name].append(requirement.name)
         return graph_check_cycle(graph)
 
     def iterate_until_finished(self):
@@ -766,7 +768,7 @@ class Config:
 
             raise cdist.UnresolvableRequirementsError(
                     ("The requirements of the following objects could not be "
-                     "resolved:\n%s") % ("\n".join(info_string)))
+                     "resolved:\n{}").format("\n".join(info_string)))
 
     def _handle_deprecation(self, cdist_object):
         cdist_type = cdist_object.cdist_type
@@ -791,9 +793,9 @@ class Config:
     def object_prepare(self, cdist_object, transfer_type_explorers=True):
         """Prepare object: Run type explorer + manifest"""
         self._handle_deprecation(cdist_object)
-        self.log.verbose("Preparing object {}".format(cdist_object.name))
+        self.log.verbose("Preparing object %s", cdist_object.name)
-        self.log.verbose(
+        self.log.verbose("Running manifest and explorers for %s",
-            "Running manifest and explorers for " + cdist_object.name)
+                         cdist_object.name)
         self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
         try:
             self.manifest.run_type_manifest(cdist_object)
@@ -807,13 +809,13 @@ class Config:
     def object_run(self, cdist_object):
         """Run gencode and code for an object"""
         try:
-            self.log.verbose("Running object " + cdist_object.name)
+            self.log.verbose("Running object %s", cdist_object.name)
             if cdist_object.state == core.CdistObject.STATE_DONE:
                 raise cdist.Error(("Attempting to run an already finished "
-                                   "object: %s"), cdist_object)
+                                   "object: {}").format(cdist_object))
 
             # Generate
-            self.log.debug("Generating code for %s" % (cdist_object.name))
+            self.log.debug("Generating code for %s", cdist_object.name)
             cdist_object.code_local = self.code.run_gencode_local(cdist_object)
             cdist_object.code_remote = self.code.run_gencode_remote(
                 cdist_object)
@@ -822,20 +824,20 @@ class Config:
 
             # Execute
             if cdist_object.code_local or cdist_object.code_remote:
-                self.log.info("Processing %s" % (cdist_object.name))
+                self.log.info("Processing %s", cdist_object.name)
             if not self.dry_run:
                 if cdist_object.code_local:
-                    self.log.trace("Executing local code for %s"
+                    self.log.trace("Executing local code for %s",
-                                   % (cdist_object.name))
+                                   cdist_object.name)
                     self.code.run_code_local(cdist_object)
                 if cdist_object.code_remote:
-                    self.log.trace("Executing remote code for %s"
+                    self.log.trace("Executing remote code for %s",
-                                   % (cdist_object.name))
+                                   cdist_object.name)
                     self.code.transfer_code_remote(cdist_object)
                     self.code.run_code_remote(cdist_object)
 
             # Mark this object as done
-            self.log.trace("Finishing run of " + cdist_object.name)
+            self.log.trace("Finishing run of %s", cdist_object.name)
             cdist_object.state = core.CdistObject.STATE_DONE
         except cdist.Error as e:
             raise cdist.CdistObjectError(cdist_object, e)

cdist/core/cdist_object.py

@@ -34,17 +34,17 @@ class IllegalObjectIdError(cdist.Error):
         self.message = message or 'Illegal object id'
 
     def __str__(self):
-        return '%s: %s' % (self.message, self.object_id)
+        return '{}: {}'.format(self.message, self.object_id)
 
 
 class MissingObjectIdError(cdist.Error):
     def __init__(self, type_name):
         self.type_name = type_name
-        self.message = ("Type %s requires object id (is not a "
+        self.message = ("Type {} requires object id (is not a "
-                        "singleton type)") % self.type_name
+                        "singleton type)").format(self.type_name)
 
     def __str__(self):
-        return '%s' % (self.message)
+        return '{}'.format(self.message)
 
 
 class CdistObject:
@@ -142,7 +142,7 @@ class CdistObject:
             if self.object_marker in self.object_id.split(os.sep):
                 raise IllegalObjectIdError(
                         self.object_id, ('object_id may not contain '
-                                         '\'%s\'') % self.object_marker)
+                                         '\'{}\'').format(self.object_marker))
             if '//' in self.object_id:
                 raise IllegalObjectIdError(
                         self.object_id, 'object_id may not contain //')
@@ -189,7 +189,7 @@ class CdistObject:
                               object_id=object_id)
 
     def __repr__(self):
-        return '<CdistObject %s>' % self.name
+        return '<CdistObject {}>'.format(self.name)
 
     def __eq__(self, other):
         """define equality as 'name is the same'"""
@@ -247,6 +247,13 @@ class CdistObject:
             lambda obj: os.path.join(obj.absolute_path, 'typeorder'))
     typeorder_dep = fsproperty.FileListProperty(
             lambda obj: os.path.join(obj.absolute_path, 'typeorder_dep'))
+    # objects without parents are objects specified in init manifest
+    parents = fsproperty.FileListProperty(
+            lambda obj: os.path.join(obj.absolute_path, 'parents'))
+    # objects without children are object of types that do not reuse other
+    # types
+    children = fsproperty.FileListProperty(
+            lambda obj: os.path.join(obj.absolute_path, 'children'))
 
     def cleanup(self):
         try:
@@ -270,10 +277,10 @@ class CdistObject:
                 os.makedirs(path, exist_ok=allow_overwrite)
         except EnvironmentError as error:
             raise cdist.Error(('Error creating directories for cdist object: '
-                               '%s: %s') % (self, error))
+                               '{}: {}').format(self, error))
 
     def requirements_unfinished(self, requirements):
-        """Return state whether requirements are satisfied"""
+        """Return unsatisfied requirements"""
 
         object_list = []
 
@@ -284,3 +291,14 @@ class CdistObject:
                 object_list.append(cdist_object)
 
         return object_list
+
+    def has_requirements_unfinished(self, requirements):
+        """Return whether requirements are satisfied"""
+
+        for requirement in requirements:
+            cdist_object = self.object_from_name(requirement)
+
+            if cdist_object.state != self.STATE_DONE:
+                return True
+
+        return False

cdist/core/cdist_type.py

@@ -34,7 +34,7 @@ class InvalidTypeError(cdist.Error):
         self.source_path = os.path.realpath(self.type_absolute_path)
 
     def __str__(self):
-        return "Invalid type '%s' at '%s' defined at '%s'" % (
+        return "Invalid type '{}' at '{}' defined at '{}'".format(
                 self.type_path, self.type_absolute_path, self.source_path)
 
 
@@ -82,9 +82,9 @@ class CdistType:
                 yield cls(base_path, name)
             except InvalidTypeError as e:
                 # ignore invalid type, log warning and continue
-                msg = "Ignoring invalid type '%s' at '%s' defined at '%s'" % (
+                cls.log.warning("Ignoring invalid type '%s' at '%s' defined"
-                    e.type_path, e.type_absolute_path, e.source_path)
+                                " at '%s'", e.type_path, e.type_absolute_path,
-                cls.log.warning(msg)
+                                e.source_path)
                 # remove invalid from runtime conf dir
                 os.remove(e.type_absolute_path)
 
@@ -109,7 +109,7 @@ class CdistType:
         return cls._instances[name]
 
     def __repr__(self):
-        return '<CdistType %s>' % self.name
+        return '<CdistType {}>'.format(self.name)
 
     def __eq__(self, other):
         return isinstance(other, self.__class__) and self.name == other.name

cdist/core/code.py

@@ -122,8 +122,8 @@ class Code:
 
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.cdist_type
-        script = os.path.join(self.local.type_path,
+        gencode_attr = getattr(cdist_type, 'gencode_{}_path'.format(which))
-                              getattr(cdist_type, 'gencode_%s_path' % which))
+        script = os.path.join(self.local.type_path, gencode_attr)
         if os.path.isfile(script):
             env = os.environ.copy()
             env.update(self.env)
@@ -167,8 +167,8 @@ class Code:
 
     def _run_code(self, cdist_object, which, env=None):
         which_exec = getattr(self, which)
-        script = os.path.join(which_exec.object_path,
+        code_attr = getattr(cdist_object, 'code_{}_path'.format(which))
-                              getattr(cdist_object, 'code_%s_path' % which))
+        script = os.path.join(which_exec.object_path, code_attr)
         if which_exec.save_output_streams:
             stderr_path = os.path.join(cdist_object.stderr_path,
                                        'code-' + which)

cdist/core/explorer.py

@@ -131,18 +131,17 @@ class Explorer:
             self._run_global_explorer(explorer, out_path)
 
     def _run_global_explorers_parallel(self, out_path):
-        self.log.debug("Running global explorers in {} parallel jobs".format(
+        self.log.debug("Running global explorers in %s parallel jobs",
-            self.jobs))
+                       self.jobs)
-        self.log.trace("Multiprocessing start method is {}".format(
+        self.log.trace("Multiprocessing start method is %s",
-            multiprocessing.get_start_method()))
+                       multiprocessing.get_start_method())
-        self.log.trace(("Starting multiprocessing Pool for global "
+        self.log.trace("Starting multiprocessing Pool for global explorers"
-                       "explorers run"))
+                       " run")
         args = [
             (e, out_path, ) for e in self.list_global_explorer_names()
         ]
         mp_pool_run(self._run_global_explorer, args, jobs=self.jobs)
-        self.log.trace(("Multiprocessing run for global explorers "
+        self.log.trace("Multiprocessing run for global explorers finished")
-                        "finished"))
 
     # logger is not pickable, so remove it when we pickle
     def __getstate__(self):
@@ -161,8 +160,8 @@ class Explorer:
         self.remote.transfer(self.local.global_explorer_path,
                              self.remote.global_explorer_path,
                              self.jobs)
-        self.remote.run(["chmod", "0700",
+        self.remote.run(["chmod", "0700", "{}/*".format(
-                         "%s/*" % (self.remote.global_explorer_path)])
+            self.remote.global_explorer_path)])
 
     def run_global_explorer(self, explorer):
         """Run the given global explorer and return it's output."""
@@ -184,15 +183,14 @@ class Explorer:
         in the object.
 
         """
-        self.log.verbose("Running type explorers for {}".format(
+        self.log.verbose("Running type explorers for %s",
-            cdist_object.cdist_type))
+                         cdist_object.cdist_type)
         if transfer_type_explorers:
             self.log.trace("Transferring type explorers for type: %s",
                            cdist_object.cdist_type)
             self.transfer_type_explorers(cdist_object.cdist_type)
         else:
-            self.log.trace(("No need for transferring type explorers for "
+            self.log.trace("No need for transferring type explorers for %s",
-                            "type: %s"),
                            cdist_object.cdist_type)
         self.log.trace("Transferring object parameters for object: %s",
                        cdist_object.name)
@@ -236,15 +234,15 @@ class Explorer:
            remote side."""
         if cdist_type.explorers:
             if cdist_type.name in self._type_explorers_transferred:
-                self.log.trace(("Skipping retransfer of type explorers "
+                self.log.trace("Skipping retransfer of type explorers for: %s",
-                                "for: %s"), cdist_type)
+                               cdist_type)
             else:
                 source = os.path.join(self.local.type_path,
                                       cdist_type.explorer_path)
                 destination = os.path.join(self.remote.type_path,
                                            cdist_type.explorer_path)
                 self.remote.transfer(source, destination)
-                self.remote.run(["chmod", "0700", "%s/*" % (destination)])
+                self.remote.run(["chmod", "0700", "{}/*".format(destination)])
                 self._type_explorers_transferred.append(cdist_type.name)
 
     def transfer_object_parameters(self, cdist_object):

cdist/core/manifest.py

@@ -80,13 +80,12 @@ class NoInitialManifestError(cdist.Error):
 
         if user_supplied:
             if os.path.islink(manifest_path):
-                self.message = "%s: %s -> %s" % (
+                self.message = "{}: {} -> {}".format(
-                        msg_header, manifest_path,
+                    msg_header, manifest_path, os.path.realpath(manifest_path))
-                        os.path.realpath(manifest_path))
             else:
-                self.message = "%s: %s" % (msg_header, manifest_path)
+                self.message = "{}: {}".format(msg_header, manifest_path)
         else:
-            self.message = "%s" % (msg_header)
+            self.message = "{}".format(msg_header)
 
     def __str__(self):
         return repr(self.message)
@@ -107,7 +106,7 @@ class Manifest:
         self._open_logger()
 
         self.env = {
-            'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
+            'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']),
             # for use in type emulator
             '__cdist_type_base_path': self.local.type_path,
             '__global': self.local.base_path,
@@ -161,7 +160,7 @@ class Manifest:
             raise NoInitialManifestError(initial_manifest, user_supplied)
 
         message_prefix = "initialmanifest"
-        self.log.verbose("Running initial manifest " + initial_manifest)
+        self.log.verbose("Running initial manifest %s", initial_manifest)
         which = "init"
         if self.local.save_output_streams:
             stderr_path = os.path.join(self.local.stderr_base_path, which)

cdist/emulator.py

@@ -36,8 +36,8 @@ from cdist.core.manifest import Manifest
 class MissingRequiredEnvironmentVariableError(cdist.Error):
     def __init__(self, name):
         self.name = name
-        self.message = ("Emulator requires the environment variable %s to be "
+        self.message = ("Emulator requires the environment variable {} to be "
-                        "setup" % self.name)
+                        "setup").format(self.name)
 
     def __str__(self):
         return self.message
@@ -106,8 +106,9 @@ class Emulator:
             self.save_stdin()
             self.record_requirements()
             self.record_auto_requirements()
-            self.log.trace("Finished %s %s" % (
+            self.record_parent_child_relationships()
-                self.cdist_object.path, self.parameters))
+            self.log.trace("Finished %s %s", self.cdist_object.path,
+                           self.parameters)
 
     def __init_log(self):
         """Setup logging facility"""
@@ -169,7 +170,7 @@ class Emulator:
 
         # And finally parse/verify parameter
         self.args = parser.parse_args(self.argv[1:])
-        self.log.trace('Args: %s' % self.args)
+        self.log.trace('Args: %s', self.args)
 
     def init_object(self):
         # Initialize object - and ensure it is not in args
@@ -230,18 +231,18 @@ class Emulator:
         if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env:
             obj_params = self._object_params_in_context()
             if obj_params != self.parameters:
-                errmsg = ("Object %s already exists with conflicting "
+                errmsg = ("Object {} already exists with conflicting "
-                          "parameters:\n%s: %s\n%s: %s" % (
+                          "parameters:\n{}: {}\n{}: {}").format(
                               self.cdist_object.name,
                               " ".join(self.cdist_object.source),
                               obj_params,
                               self.object_source,
-                              self.parameters))
+                              self.parameters)
                 raise cdist.Error(errmsg)
         else:
             if self.cdist_object.exists:
-                self.log.debug(('Object %s override forced with '
+                self.log.debug('Object %s override forced with CDIST_OVERRIDE',
-                                'CDIST_OVERRIDE'), self.cdist_object.name)
+                               self.cdist_object.name)
                 self.cdist_object.create(True)
             else:
                 self.cdist_object.create()
@@ -259,8 +260,8 @@ class Emulator:
             parent = self.cdist_object.object_from_name(__object_name)
             parent.typeorder.append(self.cdist_object.name)
             if self._order_dep_on():
-                self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
+                self.log.trace('[ORDER_DEP] Adding %s to typeorder dep for %s',
-                                ' for %s'), depname, parent.name)
+                               depname, parent.name)
                 parent.typeorder_dep.append(depname)
         elif self._order_dep_on():
             self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
@@ -291,7 +292,7 @@ class Emulator:
                         fd.write(chunk)
                         chunk = self._read_stdin()
             except EnvironmentError as e:
-                raise cdist.Error('Failed to read from stdin: %s' % e)
+                raise cdist.Error('Failed to read from stdin: {}'.format(e))
 
     def record_requirement(self, requirement):
         """record requirement and return recorded requirement"""
@@ -300,16 +301,14 @@ class Emulator:
         try:
             cdist_object = self.cdist_object.object_from_name(requirement)
         except core.cdist_type.InvalidTypeError as e:
-            self.log.error(("%s requires object %s, but type %s does not"
+            self.log.error("%s requires object %s, but type %s does not"
-                            " exist. Defined at %s" % (
+                           " exist. Defined at %s", self.cdist_object.name,
-                                self.cdist_object.name,
+                           requirement, e.name, self.object_source)
-                                requirement, e.name, self.object_source)))
             raise
         except core.cdist_object.MissingObjectIdError:
-            self.log.error(("%s requires object %s without object id."
+            self.log.error("%s requires object %s without object id."
-                            " Defined at %s" % (self.cdist_object.name,
+                           " Defined at %s", self.cdist_object.name,
-                                                requirement,
+                           requirement, self.object_source)
-                                                self.object_source)))
             raise
 
         self.log.debug("Recording requirement %s for %s",
@@ -379,10 +378,9 @@ class Emulator:
                         self.env['require'] += " " + lastcreatedtype
                 else:
                     self.env['require'] = lastcreatedtype
-                self.log.debug(("Injecting require for "
+                self.log.debug("Injecting require for"
-                                "CDIST_ORDER_DEPENDENCY: %s for %s"),
+                               " CDIST_ORDER_DEPENDENCY: %s for %s",
-                               lastcreatedtype,
+                               lastcreatedtype, self.cdist_object.name)
-                               self.cdist_object.name)
             except IndexError:
                 # if no second last line, we are on the first type,
                 # so do not set a requirement
@@ -390,7 +388,7 @@ class Emulator:
 
         if "require" in self.env:
             requirements = self.env['require']
-            self.log.debug("reqs = " + requirements)
+            self.log.debug("reqs = %s", requirements)
             for requirement in self._parse_require(requirements):
                 # Ignore empty fields - probably the only field anyway
                 if len(requirement) == 0:
@@ -420,3 +418,21 @@ class Emulator:
                 self.log.debug("Recording autorequirement %s for %s",
                                current_object.name, parent.name)
                 parent.autorequire.append(current_object.name)
+
+    def record_parent_child_relationships(self):
+        # __object_name is the name of the object whose type manifest is
+        # currently executed
+        __object_name = self.env.get('__object_name', None)
+        if __object_name:
+            # The object whose type manifest is currently run
+            parent = self.cdist_object.object_from_name(__object_name)
+            # The object currently being defined
+            current_object = self.cdist_object
+            if current_object.name not in parent.children:
+                self.log.debug("Recording child %s for %s",
+                               current_object.name, parent.name)
+                parent.children.append(current_object.name)
+            if parent.name not in current_object.parents:
+                self.log.debug("Recording parent %s for %s",
+                               parent.name, current_object.name)
+                current_object.parents.append(parent.name)