From 64c247026a4fe02db1d31e2bd45839f577c7c1bb Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 30 Sep 2019 14:20:26 +0200
Subject: [PATCH 001/527] [__locale_system] Support Devuan

---
 cdist/conf/type/__locale_system/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 80f7401b..4a1fdeed 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -26,7 +26,7 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-    debian|ubuntu)
+    debian|devuan|ubuntu)
         locale_conf="/etc/default/locale"
         ;;
     archlinux)

From 2cf44c66d46435e859fe0528d4b414dafd8b468a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 15 Jan 2020 19:00:30 +0100
Subject: [PATCH 002/527] [__directory] Add --state exists and --state
 pre-exists

---
 cdist/conf/type/__directory/gencode-remote | 17 ++++++++++++++---
 cdist/conf/type/__directory/man.rst        | 15 +++++++++++++--
 2 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote
index 374db47a..e1ab69d7 100755
--- a/cdist/conf/type/__directory/gencode-remote
+++ b/cdist/conf/type/__directory/gencode-remote
@@ -3,6 +3,7 @@
 # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
 # 2013 Steven Armstrong (steven-cdist armstrong.cc)
 # 2014 Daniel Heule (hda at sfs.biz)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -21,8 +22,8 @@
 #
 
 destination="/$__object_id"
-state_should="$(cat "$__object/parameter/state")"
-type="$(cat "$__object/explorer/type")"
+state_should=$(cat "$__object/parameter/state")
+type=$(cat "$__object/explorer/type")
 stat_file="$__object/explorer/stat"
 
 # variable to keep track if we have to set directory attributes
@@ -72,7 +73,7 @@ set_mode() {
 }
 
 case "$state_should" in
-   present)
+   present|exists)
       if [ "$type" != "directory" ]; then
          set_attributes=1
          if [ "$type" != "none" ]; then
@@ -83,6 +84,10 @@ case "$state_should" in
          fi
          echo "mkdir $mkdiropt '$destination'"
          echo "create" >> "$__messages_out"
+      elif [ "$state_should" = 'exists' ]; then
+         # The type is directory and --state exists. We are done and do not
+         # check or set the attributes.
+         exit 0
       fi
 
       # Note: Mode - needs to happen last as a chown/chgrp can alter mode by
@@ -103,6 +108,12 @@ case "$state_should" in
          fi
       done
    ;;
+   pre-exists)
+      if [ "$type" != "directory" ]; then
+         echo "Directory \"$destination\" does not exist" >&2
+         exit 1
+      fi
+   ;;
    absent)
         if [ "$type" = "directory" ]; then
             echo "rm -rf '$destination'"
diff --git a/cdist/conf/type/__directory/man.rst b/cdist/conf/type/__directory/man.rst
index 74b00afe..7755334c 100644
--- a/cdist/conf/type/__directory/man.rst
+++ b/cdist/conf/type/__directory/man.rst
@@ -19,7 +19,18 @@ None.
 OPTIONAL PARAMETERS
 -------------------
 state
-   'present' or 'absent', defaults to 'present'
+   'present', 'absent', 'exists' or 'pre-exists', defaults to 'present' where:
+
+   present
+      the directory exists and the given attributes are set.
+   absent
+      the directory does not exist.
+   exists
+      the directory exists, but its attributes are not altered if it already
+      existed.
+   pre-exists
+      check that the directory exists and is indeed a directory, but do not
+      create or modify it.
 
 group
    Group to chgrp to.
@@ -36,7 +47,7 @@ BOOLEAN PARAMETERS
 parents
    Whether to create parents as well (mkdir -p behaviour).
    Warning: all intermediate directory permissions default
-   to whatever mkdir -p does. 
+   to whatever mkdir -p does.
 
    Usually this means root:root, 0700.
 

From c6aba8d189c2efc7c88f9c595acba7ceae5a4e00 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 23 Feb 2020 22:59:41 +0100
Subject: [PATCH 003/527] [explorer/disks] Fix for NetBSD

When connecting over SSH and running /bin/sh, the PATH is missing sbin locations.
sysctl is located at /sbin/sysctl on NetBSD.
---
 cdist/conf/explorer/disks | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks
index 87a6b5c6..08290bc7 100755
--- a/cdist/conf/explorer/disks
+++ b/cdist/conf/explorer/disks
@@ -1,14 +1,20 @@
-#!/bin/sh
+#!/bin/sh -e
 
 uname_s="$(uname -s)"
 
-case "${uname_s}" in
+case $uname_s in
     FreeBSD)
         sysctl -n kern.disks
     ;;
-    OpenBSD|NetBSD)
+    OpenBSD)
         sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+' | xargs
     ;;
+    NetBSD)
+        PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin"
+        sysctl -n hw.disknames \
+        | awk 'BEGIN { RS = " " } /^[lsw]d[0-9]+/' \
+        | xargs
+    ;;
     Linux)
         if command -v lsblk > /dev/null
         then
@@ -23,5 +29,3 @@ case "${uname_s}" in
         printf "Don't know how to list disks for %s operating system, if you can please submit a patch\n" "${uname_s}" >&2
     ;;
 esac
-
-exit 0

From d3bd2669ec49fb861016e614893dac280ed5fd35 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 23 Feb 2020 23:07:40 +0100
Subject: [PATCH 004/527] [explorer/disks] Support Linux without lsblk
 (fallback to sysfs)

---
 cdist/conf/explorer/disks | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks
index 08290bc7..0fabc95f 100755
--- a/cdist/conf/explorer/disks
+++ b/cdist/conf/explorer/disks
@@ -16,16 +16,33 @@ case $uname_s in
         | xargs
     ;;
     Linux)
-        if command -v lsblk > /dev/null
+        # list of major device numbers toexclude:
+        #  ram disks, floppies, cdroms
+        # https://www.kernel.org/doc/Documentation/admin-guide/devices.txt
+        ign_majors='1 2 11'
+
+        if command -v lsblk >/dev/null 2>&1
         then
-            # exclude ram disks, floppies and cdroms
-            # https://www.kernel.org/doc/Documentation/admin-guide/devices.txt
-            lsblk -e 1,2,11 -dno name | xargs
+            lsblk -e "$(echo "$ign_majors" | tr ' ' ',')" -dno name | xargs
+        elif test -d /sys/block/
+        then
+            # shellcheck disable=SC2012
+            ls -1 /sys/block/ \
+            | awk -v ign_majors="$(echo "$ign_majors" | tr ' ' '|')" '
+                {
+                  devfile = "/sys/block/" $0 "/dev"
+                  getline devno < devfile
+                  close(devfile)
+                  if (devno !~ "^(" ign_majors "):") print
+                }' \
+            | xargs
         else
-            printf "Don't know how to list disks for %s operating system without lsblk, if you can please submit a patch\n" "${uname_s}" >&2
+            echo "Don't know how to list disks on Linux without lsblk and sysfs." >&2
+            echo 'If you can, please submit a patch.'>&2
         fi
     ;;
     *)
-        printf "Don't know how to list disks for %s operating system, if you can please submit a patch\n" "${uname_s}" >&2
+        printf "Don't know how to list disks for %s operating system.\n" "${uname_s}" >&2
+        printf 'If you can please submit a patch\n' >&2
     ;;
 esac

From 1ef126e16f95e822562978abd895a3c036f7d5c4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 23 Feb 2020 23:08:40 +0100
Subject: [PATCH 005/527] [explorer/disks] Move xargs call to the bottom

---
 cdist/conf/explorer/disks | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks
index 0fabc95f..ed1afce4 100755
--- a/cdist/conf/explorer/disks
+++ b/cdist/conf/explorer/disks
@@ -7,13 +7,12 @@ case $uname_s in
         sysctl -n kern.disks
     ;;
     OpenBSD)
-        sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+' | xargs
+        sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+'
     ;;
     NetBSD)
         PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin"
         sysctl -n hw.disknames \
-        | awk 'BEGIN { RS = " " } /^[lsw]d[0-9]+/' \
-        | xargs
+        | awk 'BEGIN { RS = " " } /^[lsw]d[0-9]+/'
     ;;
     Linux)
         # list of major device numbers toexclude:
@@ -23,7 +22,7 @@ case $uname_s in
 
         if command -v lsblk >/dev/null 2>&1
         then
-            lsblk -e "$(echo "$ign_majors" | tr ' ' ',')" -dno name | xargs
+            lsblk -e "$(echo "$ign_majors" | tr ' ' ',')" -dno name
         elif test -d /sys/block/
         then
             # shellcheck disable=SC2012
@@ -34,8 +33,7 @@ case $uname_s in
                   getline devno < devfile
                   close(devfile)
                   if (devno !~ "^(" ign_majors "):") print
-                }' \
-            | xargs
+                }'
         else
             echo "Don't know how to list disks on Linux without lsblk and sysfs." >&2
             echo 'If you can, please submit a patch.'>&2
@@ -45,4 +43,5 @@ case $uname_s in
         printf "Don't know how to list disks for %s operating system.\n" "${uname_s}" >&2
         printf 'If you can please submit a patch\n' >&2
     ;;
-esac
+esac \
+| xargs

From 6db6dc4ac0950579ce13252dcca6d0f61f5533c6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 23 Feb 2020 23:14:14 +0100
Subject: [PATCH 006/527] [explorer/disks] Add license header

---
 cdist/conf/explorer/disks | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks
index ed1afce4..24540601 100755
--- a/cdist/conf/explorer/disks
+++ b/cdist/conf/explorer/disks
@@ -1,4 +1,24 @@
 #!/bin/sh -e
+#
+# based on previous work by other people, modified by:
+# 2020 Dennis Camera <dennis.camera at ssrq-sds-fds.ch>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Finds disks of the system (excl. ram disks, floppy, cdrom)
 
 uname_s="$(uname -s)"
 

From b2db864eaf95a50e7e0d31be52f86ab1d0bea480 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 26 Feb 2020 13:01:29 +0100
Subject: [PATCH 007/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8463bb89..35590655 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Explorer init: Rewrite and support more init systems (Dennis Camera)
 	* New type: __service (Timothée Floure)
 	* Types __consul_*: Add optional parameter for using distribution packages (Timothée Floure)
+	* Explorer disks: Fix NetBSD, support Linux w/o lsblk (Dennis Camera)
 
 6.5.1: 2020-02-15
 	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)

From 45cb56d22b2d4c6fc40618ae28cca5dd833e2387 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 Feb 2020 10:31:06 +0100
Subject: [PATCH 008/527] Import __matrix_riot type

---
 .../type/__matrix_riot/files/config.json.sh   | 56 ++++++++++++++
 cdist/conf/type/__matrix_riot/gencode-remote  | 74 +++++++++++++++++++
 cdist/conf/type/__matrix_riot/man.rst         | 72 ++++++++++++++++++
 cdist/conf/type/__matrix_riot/manifest        | 68 +++++++++++++++++
 .../__matrix_riot/parameter/default/brand     |  1 +
 .../parameter/default/cookie_policy_url       |  1 +
 .../parameter/default/default_country_code    |  1 +
 .../parameter/default/default_server_name     |  1 +
 .../parameter/default/default_server_url      |  1 +
 .../__matrix_riot/parameter/default/owner     |  1 +
 .../parameter/default/privacy_policy_url      |  1 +
 .../parameter/default/room_directory_servers  |  1 +
 .../type/__matrix_riot/parameter/optional     |  9 +++
 .../type/__matrix_riot/parameter/required     |  2 +
 14 files changed, 289 insertions(+)
 create mode 100755 cdist/conf/type/__matrix_riot/files/config.json.sh
 create mode 100755 cdist/conf/type/__matrix_riot/gencode-remote
 create mode 100644 cdist/conf/type/__matrix_riot/man.rst
 create mode 100755 cdist/conf/type/__matrix_riot/manifest
 create mode 100644 cdist/conf/type/__matrix_riot/parameter/default/brand
 create mode 100644 cdist/conf/type/__matrix_riot/parameter/default/cookie_policy_url
 create mode 100644 cdist/conf/type/__matrix_riot/parameter/default/default_country_code
 create mode 100644 cdist/conf/type/__matrix_riot/parameter/default/default_server_name
 create mode 100644 cdist/conf/type/__matrix_riot/parameter/default/default_server_url
 create mode 100644 cdist/conf/type/__matrix_riot/parameter/default/owner
 create mode 100644 cdist/conf/type/__matrix_riot/parameter/default/privacy_policy_url
 create mode 100644 cdist/conf/type/__matrix_riot/parameter/default/room_directory_servers
 create mode 100644 cdist/conf/type/__matrix_riot/parameter/optional
 create mode 100644 cdist/conf/type/__matrix_riot/parameter/required

diff --git a/cdist/conf/type/__matrix_riot/files/config.json.sh b/cdist/conf/type/__matrix_riot/files/config.json.sh
new file mode 100755
index 00000000..d15d4862
--- /dev/null
+++ b/cdist/conf/type/__matrix_riot/files/config.json.sh
@@ -0,0 +1,56 @@
+#!/bin/sh
+#
+# Upstream configuration guide/documentation:
+#   https://github.com/vector-im/riot-web/blob/develop/docs/config.md
+
+generate_embedded_pages () {
+  if [ $EMBED_HOMEPAGE ]; then
+    cat << EOF
+    "embeddedPages": {
+        "homeUrl": "home.html"
+      },
+EOF
+  fi
+}
+
+cat << EOF
+{
+    "default_server_config": {
+        "m.homeserver": {
+            "base_url": "$DEFAULT_SERVER_URL",
+            "server_name": "$DEFAULT_SERVER_NAME"
+        },
+        "m.identity_server": {
+            "base_url": "https://vector.im"
+        }
+    },
+    "brand": "$BRAND",
+    "defaultCountryCode": "$DEFAULT_COUNTRY_CODE",
+    "integrations_ui_url": "https://scalar.vector.im/",
+    "integrations_rest_url": "https://scalar.vector.im/api",
+    "integrations_widgets_urls": [
+        "https://scalar.vector.im/_matrix/integrations/v1",
+        "https://scalar.vector.im/api",
+        "https://scalar-staging.vector.im/_matrix/integrations/v1",
+        "https://scalar-staging.vector.im/api",
+        "https://scalar-staging.riot.im/scalar/api"
+    ],
+    "bug_report_endpoint_url": "https://riot.im/bugreports/submit",
+    "roomDirectory": {
+        "servers": [
+            $ROOM_DIRECTORY_SERVERS
+        ]
+    },
+    $(generate_embedded_pages)
+    "terms_and_conditions_links": [
+        {
+            "url": "$PRIVACY_POLICY_URL",
+            "text": "Privacy Policy"
+        },
+        {
+            "url": "$COOKIE_POLICY_URL",
+            "text": "Cookie Policy"
+        }
+    ]
+}
+EOF
diff --git a/cdist/conf/type/__matrix_riot/gencode-remote b/cdist/conf/type/__matrix_riot/gencode-remote
new file mode 100755
index 00000000..b8935588
--- /dev/null
+++ b/cdist/conf/type/__matrix_riot/gencode-remote
@@ -0,0 +1,74 @@
+#!/bin/sh -e
+#
+# 2019 Timothée Floure (timothee.floure@ungleich.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+VERSION=$(cat "$__object/parameter/version")
+INSTALL_DIR=$(cat "$__object/parameter/install_dir")
+OWNER=$(cat "$__object/parameter/owner")
+
+src="riot-v$VERSION"
+archive="$src.tar.gz"
+config='config.json'
+homepage='home.html'
+url="https://github.com/vector-im/riot-web/releases/download/v$VERSION/$archive"
+
+# tar and curl are installed by the __matrix-riot manifest. mktemp is usually
+# provided by coreutils and assumed installed.
+cat << EOF
+set -e
+
+# Ensure that coreutils is installed.
+if [ ! -x \$(which mktemp) ]; then
+  echo "mktemp is not available on the remote host." >&2
+  exit 1
+fi
+
+# Create temporary working directory.
+tmpdir=\$(mktemp -d)
+cd \$tmpdir
+
+# Download and extract sources.
+curl -L '$url' > $archive
+tar xf $archive
+
+# Backup configuration deployed by __matrix_riot.
+cp '$INSTALL_DIR/$config' '$config'
+
+# Backup homepage deployed by __matrix_riot
+if [ -f '$INSTALL_DIR/$homepage' ]; then
+  cp '$INSTALL_DIR/$homepage' '$homepage'
+fi
+
+# Deploy sources and restore configuration.
+rm -r '$INSTALL_DIR'
+mv '$src' '$INSTALL_DIR'
+cp '$config' '$INSTALL_DIR/$config'
+
+# Restore homepage if available.
+if [ -f '$homepage' ]; then
+  cp '$homepage' '$INSTALL_DIR/$homepage'
+fi
+
+# Chown deployed files to requested owner.
+chown -R '$OWNER' '$INSTALL_DIR'
+
+# Remove temporary working directory.
+cd /
+rm -r \$tmpdir
+EOF
diff --git a/cdist/conf/type/__matrix_riot/man.rst b/cdist/conf/type/__matrix_riot/man.rst
new file mode 100644
index 00000000..c8d80024
--- /dev/null
+++ b/cdist/conf/type/__matrix_riot/man.rst
@@ -0,0 +1,72 @@
+cdist-type__matrix_riot(7)
+======================
+
+NAME
+----
+cdist-type__matrix_riot - Install and configure Riot, a web Matrix client.
+
+
+DESCRIPTION
+-----------
+This type install and configure the Riot web client.
+
+
+REQUIRED PARAMETERS
+-------------------
+install_dir
+  Root directory of Riot's static files.
+
+version
+  Release of Riot to install.
+
+OPTIONAL PARAMETERS
+-------------------
+default_server_name
+  Name of matrix homeserver to connect to, defaults to 'matrix.org'.
+
+default_server_url
+  URL of matrix homeserver to connect to, defaults to 'https://matrix-client.matrix.org'.
+
+owner
+  Owner of the deployed files, passed to `chown`. Defaults to 'root'.
+
+brand
+  Web UI branding, defaults to 'Riot'.
+
+default_country_code
+  ISO 3166 alpha2 country code to use when showing country selectors, such as
+  phone number inputs. Defaults to GB.
+
+privacy_policy_url
+  Defaults to 'https://riot.im/privacy'.
+
+cookie_policy_url
+  Defaults to 'https://matrix.org/docs/guides/riot_im_cookie_policy'.
+
+homepage
+  Path to custom Riot homepage, displayed once logged in.
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __matrix_riot my-riot --install_dir /var/www/riot-web --version 1.5.6
+
+
+SEE ALSO
+--------
+- `cdist-type__matrix_synapse(7) <cdist-type__matrix_synapse.html>`_
+
+
+AUTHORS
+-------
+Timothée Floure <timothee.floure@ungleich.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2019 Timothée Floure. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__matrix_riot/manifest b/cdist/conf/type/__matrix_riot/manifest
new file mode 100755
index 00000000..703e8b6f
--- /dev/null
+++ b/cdist/conf/type/__matrix_riot/manifest
@@ -0,0 +1,68 @@
+#!/bin/sh -e
+#
+# 2019 Timothée Floure (timothee.floure@ungleich.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+INSTALL_DIR=$(cat "$__object/parameter/install_dir")
+
+default_server_name=$(cat "$__object/parameter/default_server_name")
+brand=$(cat "$__object/parameter/brand")
+default_country_code=$(cat "$__object/parameter/default_country_code")
+room_directory_servers=$(cat "$__object/parameter/room_directory_servers")
+privacy_policy_url=$(cat "$__object/parameter/privacy_policy_url")
+cookie_policy_url=$(cat "$__object/parameter/cookie_policy_url")
+
+export DEFAULT_SERVER_NAME=$default_server_name
+export BRAND=$brand
+export DEFAULT_COUNTRY_CODE=$default_country_code
+export ROOM_DIRECTORY_SERVERS=$room_directory_servers
+export PRIVACY_POLICY_URL=$privacy_policy_url
+export COOKIE_POLICY_URL=$cookie_policy_url
+
+if [ -f "$__object/parameter/homepage" ]; then
+    export EMBED_HOMEPAGE=1
+    homepage=$(cat "$__object/parameter/homepage")
+fi
+
+# Owner of the uploaded files.
+owner=$(cat "$__object/parameter/owner")
+
+# Ensure that curl and tar are installed, as they will be required by the
+# gencode-remote script.
+__package curl --state present
+__package tar --state present
+
+# Generate and deploy configuration file.
+mkdir -p "$__object/files"
+"$__type/files/config.json.sh" > "$__object/files/config.json"
+
+# Install the config.json configuration file. The application's sources are
+# downloaded and deployed by gencode-remote.
+__directory "$INSTALL_DIR" \
+    --owner "$owner" --mode 0755 --parents \
+    --state present
+require="__directory/$INSTALL_DIR" __file "$INSTALL_DIR/config.json" \
+  --source "$__object/files/config.json" \
+  --mode 0664 \
+  --state present
+if [ $EMBED_HOMEPAGE ]; then
+    require="__directory/$INSTALL_DIR" __file "$INSTALL_DIR/home.html" \
+      --source "$homepage" \
+      --mode 0664 \
+      --state present
+fi
diff --git a/cdist/conf/type/__matrix_riot/parameter/default/brand b/cdist/conf/type/__matrix_riot/parameter/default/brand
new file mode 100644
index 00000000..e8095bb8
--- /dev/null
+++ b/cdist/conf/type/__matrix_riot/parameter/default/brand
@@ -0,0 +1 @@
+Riot
diff --git a/cdist/conf/type/__matrix_riot/parameter/default/cookie_policy_url b/cdist/conf/type/__matrix_riot/parameter/default/cookie_policy_url
new file mode 100644
index 00000000..04e9c2b7
--- /dev/null
+++ b/cdist/conf/type/__matrix_riot/parameter/default/cookie_policy_url
@@ -0,0 +1 @@
+https://matrix.org/docs/guides/riot_im_cookie_policy
diff --git a/cdist/conf/type/__matrix_riot/parameter/default/default_country_code b/cdist/conf/type/__matrix_riot/parameter/default/default_country_code
new file mode 100644
index 00000000..30ac4a3b
--- /dev/null
+++ b/cdist/conf/type/__matrix_riot/parameter/default/default_country_code
@@ -0,0 +1 @@
+GB
diff --git a/cdist/conf/type/__matrix_riot/parameter/default/default_server_name b/cdist/conf/type/__matrix_riot/parameter/default/default_server_name
new file mode 100644
index 00000000..5528ffdd
--- /dev/null
+++ b/cdist/conf/type/__matrix_riot/parameter/default/default_server_name
@@ -0,0 +1 @@
+matrix.org
diff --git a/cdist/conf/type/__matrix_riot/parameter/default/default_server_url b/cdist/conf/type/__matrix_riot/parameter/default/default_server_url
new file mode 100644
index 00000000..2cb92277
--- /dev/null
+++ b/cdist/conf/type/__matrix_riot/parameter/default/default_server_url
@@ -0,0 +1 @@
+https://matrix-client.matrix.org
diff --git a/cdist/conf/type/__matrix_riot/parameter/default/owner b/cdist/conf/type/__matrix_riot/parameter/default/owner
new file mode 100644
index 00000000..d8649da3
--- /dev/null
+++ b/cdist/conf/type/__matrix_riot/parameter/default/owner
@@ -0,0 +1 @@
+root
diff --git a/cdist/conf/type/__matrix_riot/parameter/default/privacy_policy_url b/cdist/conf/type/__matrix_riot/parameter/default/privacy_policy_url
new file mode 100644
index 00000000..4cdd12c1
--- /dev/null
+++ b/cdist/conf/type/__matrix_riot/parameter/default/privacy_policy_url
@@ -0,0 +1 @@
+https://riot.im/privacy
diff --git a/cdist/conf/type/__matrix_riot/parameter/default/room_directory_servers b/cdist/conf/type/__matrix_riot/parameter/default/room_directory_servers
new file mode 100644
index 00000000..4ea73ad5
--- /dev/null
+++ b/cdist/conf/type/__matrix_riot/parameter/default/room_directory_servers
@@ -0,0 +1 @@
+"matrix.org"
diff --git a/cdist/conf/type/__matrix_riot/parameter/optional b/cdist/conf/type/__matrix_riot/parameter/optional
new file mode 100644
index 00000000..c5e949fe
--- /dev/null
+++ b/cdist/conf/type/__matrix_riot/parameter/optional
@@ -0,0 +1,9 @@
+default_server_url
+default_server_name
+brand
+default_country_code
+privacy_policy_url
+cookie_policy_url
+room_directory_servers
+owner
+homepage
diff --git a/cdist/conf/type/__matrix_riot/parameter/required b/cdist/conf/type/__matrix_riot/parameter/required
new file mode 100644
index 00000000..a76477e9
--- /dev/null
+++ b/cdist/conf/type/__matrix_riot/parameter/required
@@ -0,0 +1,2 @@
+version
+install_dir

From 80d2007ba93441d94d773031ee9b8c26ec48d6c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 Feb 2020 12:14:58 +0100
Subject: [PATCH 009/527] Import __matrix_synapse type

---
 .../__matrix_synapse/files/homeserver.yaml.sh | 1651 +++++++++++++++++
 .../type/__matrix_synapse/files/log.config.sh |   38 +
 cdist/conf/type/__matrix_synapse/man.rst      |  122 ++
 cdist/conf/type/__matrix_synapse/manifest     |  234 +++
 .../type/__matrix_synapse/parameter/boolean   |    5 +
 .../parameter/default/database_host           |    0
 .../parameter/default/database_password       |    0
 .../parameter/default/database_user           |    0
 .../parameter/default/ldap_base_dn            |    0
 .../parameter/default/ldap_bind_dn            |    0
 .../parameter/default/ldap_bind_password      |    0
 .../parameter/default/ldap_filter             |    1 +
 .../parameter/default/ldap_mail_attribute     |    1 +
 .../parameter/default/ldap_name_attribute     |    1 +
 .../parameter/default/ldap_uid_attribute      |    1 +
 .../parameter/default/ldap_uri                |    0
 .../parameter/default/max_upload_size         |    1 +
 .../parameter/default/turn_user_lifetime      |    1 +
 .../type/__matrix_synapse/parameter/optional  |   14 +
 .../parameter/optional_multiple               |    1 +
 .../type/__matrix_synapse/parameter/required  |    4 +
 cdist/conf/type/__matrix_synapse/singleton    |    0
 22 files changed, 2075 insertions(+)
 create mode 100755 cdist/conf/type/__matrix_synapse/files/homeserver.yaml.sh
 create mode 100755 cdist/conf/type/__matrix_synapse/files/log.config.sh
 create mode 100644 cdist/conf/type/__matrix_synapse/man.rst
 create mode 100755 cdist/conf/type/__matrix_synapse/manifest
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/boolean
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/default/database_host
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/default/database_password
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/default/database_user
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/default/ldap_base_dn
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/default/ldap_bind_dn
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/default/ldap_bind_password
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/default/ldap_filter
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/default/ldap_mail_attribute
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/default/ldap_name_attribute
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/default/ldap_uid_attribute
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/default/ldap_uri
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/default/max_upload_size
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/default/turn_user_lifetime
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/optional
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/optional_multiple
 create mode 100644 cdist/conf/type/__matrix_synapse/parameter/required
 create mode 100644 cdist/conf/type/__matrix_synapse/singleton

diff --git a/cdist/conf/type/__matrix_synapse/files/homeserver.yaml.sh b/cdist/conf/type/__matrix_synapse/files/homeserver.yaml.sh
new file mode 100755
index 00000000..01f328d9
--- /dev/null
+++ b/cdist/conf/type/__matrix_synapse/files/homeserver.yaml.sh
@@ -0,0 +1,1651 @@
+#!/bin/sh
+
+# NOTE: this template has been generated using the
+# matrix-synapse-1.5.1-1.fc31.noarch Fedora package for use with CDIST.
+
+generate_database () {
+  if [ "$DATABASE_ENGINE" = "sqlite3" ]; then
+    cat << EOF
+database:
+  # The database engine name
+  name: "$DATABASE_ENGINE"
+  # Arguments to pass to the engine
+  args:
+    # Path to the database
+    database: "$DATABASE_NAME"
+EOF
+  else
+cat << EOF
+database:
+  # The database engine name
+  name: "$DATABASE_ENGINE"
+  # Arguments to pass to the engine
+  args:
+    database: "$DATABASE_NAME"
+    host: "$DATABASE_HOST"
+    user: "$DATABASE_USER"
+    password: "$DATABASE_PASSWORD"
+EOF
+  fi
+}
+
+generate_password_providers () {
+  if [ "$ENABLE_LDAP_AUTH" = "true" ]; then
+    cat <<EOF
+password_providers:
+    - module: "ldap_auth_provider.LdapAuthProvider"
+      config:
+        enabled: $ENABLE_LDAP_AUTH
+        uri: "$LDAP_URI"
+        start_tls: true
+        base: "$LDAP_BASE_DN"
+        attributes:
+           uid: "$LDAP_UID_ATTRIBUTE"
+           mail: "$LDAP_MAIL_ATTRIBUTE"
+           name: "$LDAP_NAME_ATTRIBUTE"
+        filter: "$LDAP_FILTER"
+EOF
+    if [ $LDAP_SEARCH_MODE ]; then
+        cat <<EOF
+        mode: "search"
+        bind_dn: "$LDAP_BIND_DN"
+        bind_password: "$LDAP_BIND_PASSWORD"
+EOF
+    fi
+  else # LDAP auth is disabled.
+    echo "password_providers: []"
+  fi
+}
+
+generate_resources () {
+    if [ "$EXPOSE_METRICS" = "true" ]; then
+      echo "[client, federation, metrics]"
+    else
+      echo "[client, federation]"
+    fi
+}
+
+cat << EOF
+## Server ##
+
+# The domain name of the server, with optional explicit port.
+# This is used by remote servers to connect to this server,
+# e.g. matrix.org, localhost:8080, etc.
+# This is also the last part of your UserID.
+#
+server_name: "$SERVER_NAME"
+
+# When running as a daemon, the file to store the pid in
+#
+pid_file: "$PIDFILE"
+
+# The path to the web client which will be served at /_matrix/client/
+# if 'webclient' is configured under the 'listeners' configuration.
+#
+#web_client_location: "/path/to/web/root"
+
+# The public-facing base URL that clients use to access this HS
+# (not including _matrix/...). This is the same URL a user would
+# enter into the 'custom HS URL' field on their client. If you
+# use synapse with a reverse proxy, this should be the URL to reach
+# synapse via the proxy.
+#
+public_baseurl: "$BASE_URL"
+
+# Set the soft limit on the number of file descriptors synapse can use
+# Zero is used to indicate synapse should set the soft limit to the
+# hard limit.
+#
+#soft_file_limit: 0
+
+# Set to false to disable presence tracking on this homeserver.
+#
+#use_presence: false
+
+# Whether to require authentication to retrieve profile data (avatars,
+# display names) of other users through the client API. Defaults to
+# 'false'. Note that profile data is also available via the federation
+# API, so this setting is of limited value if federation is enabled on
+# the server.
+#
+#require_auth_for_profile_requests: true
+
+# If set to 'false', requires authentication to access the server's public rooms
+# directory through the client API. Defaults to 'true'.
+#
+#allow_public_rooms_without_auth: false
+
+# If set to 'false', forbids any other homeserver to fetch the server's public
+# rooms directory via federation. Defaults to 'true'.
+#
+#allow_public_rooms_over_federation: false
+
+# The default room version for newly created rooms.
+#
+# Known room versions are listed here:
+# https://matrix.org/docs/spec/#complete-list-of-room-versions
+#
+# For example, for room version 1, default_room_version should be set
+# to "1".
+#
+#default_room_version: "4"
+
+# The GC threshold parameters to pass to \`gc.set_threshold\`, if defined
+#
+#gc_thresholds: [700, 10, 10]
+
+# Set the limit on the returned events in the timeline in the get
+# and sync operations. The default value is -1, means no upper limit.
+#
+#filter_timeline_limit: 5000
+
+# Whether room invites to users on this server should be blocked
+# (except those sent by local server admins). The default is False.
+#
+#block_non_admin_invites: true
+
+# Room searching
+#
+# If disabled, new messages will not be indexed for searching and users
+# will receive errors when searching for messages. Defaults to enabled.
+#
+#enable_search: false
+
+# Restrict federation to the following whitelist of domains.
+# N.B. we recommend also firewalling your federation listener to limit
+# inbound federation traffic as early as possible, rather than relying
+# purely on this application-layer restriction.  If not specified, the
+# default is to whitelist everything.
+#
+#federation_domain_whitelist:
+#  - lon.example.com
+#  - nyc.example.com
+#  - syd.example.com
+
+# Prevent federation requests from being sent to the following
+# blacklist IP address CIDR ranges. If this option is not specified, or
+# specified with an empty list, no ip range blacklist will be enforced.
+#
+# As of Synapse v1.4.0 this option also affects any outbound requests to identity
+# servers provided by user input.
+#
+# (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly
+# listed here, since they correspond to unroutable addresses.)
+#
+federation_ip_range_blacklist:
+  - '127.0.0.0/8'
+  - '10.0.0.0/8'
+  - '172.16.0.0/12'
+  - '192.168.0.0/16'
+  - '100.64.0.0/10'
+  - '169.254.0.0/16'
+  - '::1/128'
+  - 'fe80::/64'
+  - 'fc00::/7'
+
+# List of ports that Synapse should listen on, their purpose and their
+# configuration.
+#
+# Options for each listener include:
+#
+#   port: the TCP port to bind to
+#
+#   bind_addresses: a list of local addresses to listen on. The default is
+#       'all local interfaces'.
+#
+#   type: the type of listener. Normally 'http', but other valid options are:
+#       'manhole' (see docs/manhole.md),
+#       'metrics' (see docs/metrics-howto.md),
+#       'replication' (see docs/workers.md).
+#
+#   tls: set to true to enable TLS for this listener. Will use the TLS
+#       key/cert specified in tls_private_key_path / tls_certificate_path.
+#
+#   x_forwarded: Only valid for an 'http' listener. Set to true to use the
+#       X-Forwarded-For header as the client IP. Useful when Synapse is
+#       behind a reverse-proxy.
+#
+#   resources: Only valid for an 'http' listener. A list of resources to host
+#       on this port. Options for each resource are:
+#
+#       names: a list of names of HTTP resources. See below for a list of
+#           valid resource names.
+#
+#       compress: set to true to enable HTTP comression for this resource.
+#
+#   additional_resources: Only valid for an 'http' listener. A map of
+#        additional endpoints which should be loaded via dynamic modules.
+#
+# Valid resource names are:
+#
+#   client: the client-server API (/_matrix/client), and the synapse admin
+#       API (/_synapse/admin). Also implies 'media' and 'static'.
+#
+#   consent: user consent forms (/_matrix/consent). See
+#       docs/consent_tracking.md.
+#
+#   federation: the server-server API (/_matrix/federation). Also implies
+#       'media', 'keys', 'openid'
+#
+#   keys: the key discovery API (/_matrix/keys).
+#
+#   media: the media API (/_matrix/media).
+#
+#   metrics: the metrics interface. See docs/metrics-howto.md.
+#
+#   openid: OpenID authentication.
+#
+#   replication: the HTTP replication API (/_synapse/replication). See
+#       docs/workers.md.
+#
+#   static: static resources under synapse/static (/_matrix/static). (Mostly
+#       useful for 'fallback authentication'.)
+#
+#   webclient: A web client. Requires web_client_location to be set.
+#
+listeners:
+  # TLS-enabled listener: for when matrix traffic is sent directly to synapse.
+  #
+  # Disabled by default. To enable it, uncomment the following. (Note that you
+  # will also need to give Synapse a TLS key and certificate: see the TLS section
+  # below.)
+  #
+  #- port: 8448
+  #  type: http
+  #  tls: true
+  #  resources:
+  #    - names: [client, federation]
+
+  # Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy
+  # that unwraps TLS.
+  #
+  # If you plan to use a reverse proxy, please see
+  # https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md.
+  #
+  - port: 8008
+    tls: false
+    type: http
+    x_forwarded: true
+    bind_addresses: ['::1', '127.0.0.1']
+
+    resources:
+      - names: $(generate_resources)
+        compress: false
+
+    # example additional_resources:
+    #
+    #additional_resources:
+    #  "/_matrix/my/custom/endpoint":
+    #    module: my_module.CustomRequestHandler
+    #    config: {}
+
+  # Turn on the twisted ssh manhole service on localhost on the given
+  # port.
+  #
+  #- port: 9000
+  #  bind_addresses: ['::1', '127.0.0.1']
+  #  type: manhole
+
+
+## Homeserver blocking ##
+
+# How to reach the server admin, used in ResourceLimitError
+#
+#admin_contact: 'mailto:admin@server.com'
+
+# Global blocking
+#
+#hs_disabled: false
+#hs_disabled_message: 'Human readable reason for why the HS is blocked'
+
+# Monthly Active User Blocking
+#
+# Used in cases where the admin or server owner wants to limit to the
+# number of monthly active users.
+#
+# 'limit_usage_by_mau' disables/enables monthly active user blocking. When
+# anabled and a limit is reached the server returns a 'ResourceLimitError'
+# with error type Codes.RESOURCE_LIMIT_EXCEEDED
+#
+# 'max_mau_value' is the hard limit of monthly active users above which
+# the server will start blocking user actions.
+#
+# 'mau_trial_days' is a means to add a grace period for active users. It
+# means that users must be active for this number of days before they
+# can be considered active and guards against the case where lots of users
+# sign up in a short space of time never to return after their initial
+# session.
+#
+# 'mau_limit_alerting' is a means of limiting client side alerting
+# should the mau limit be reached. This is useful for small instances
+# where the admin has 5 mau seats (say) for 5 specific people and no
+# interest increasing the mau limit further. Defaults to True, which
+# means that alerting is enabled
+#
+#limit_usage_by_mau: false
+#max_mau_value: 50
+#mau_trial_days: 2
+#mau_limit_alerting: false
+
+# If enabled, the metrics for the number of monthly active users will
+# be populated, however no one will be limited. If limit_usage_by_mau
+# is true, this is implied to be true.
+#
+#mau_stats_only: false
+
+# Sometimes the server admin will want to ensure certain accounts are
+# never blocked by mau checking. These accounts are specified here.
+#
+#mau_limit_reserved_threepids:
+#  - medium: 'email'
+#    address: 'reserved_user@example.com'
+
+# Used by phonehome stats to group together related servers.
+#server_context: context
+
+# Resource-constrained Homeserver Settings
+#
+# If limit_remote_rooms.enabled is True, the room complexity will be
+# checked before a user joins a new remote room. If it is above
+# limit_remote_rooms.complexity, it will disallow joining or
+# instantly leave.
+#
+# limit_remote_rooms.complexity_error can be set to customise the text
+# displayed to the user when a room above the complexity threshold has
+# its join cancelled.
+#
+# Uncomment the below lines to enable:
+#limit_remote_rooms:
+#  enabled: true
+#  complexity: 1.0
+#  complexity_error: "This room is too complex."
+
+# Whether to require a user to be in the room to add an alias to it.
+# Defaults to 'true'.
+#
+#require_membership_for_aliases: false
+
+# Whether to allow per-room membership profiles through the send of membership
+# events with profile information that differ from the target's global profile.
+# Defaults to 'true'.
+#
+#allow_per_room_profiles: false
+
+# How long to keep redacted events in unredacted form in the database. After
+# this period redacted events get replaced with their redacted form in the DB.
+#
+# Defaults to \`7d\`. Set to \`null\` to disable.
+#
+#redaction_retention_period: 28d
+
+# How long to track users' last seen time and IPs in the database.
+#
+# Defaults to \`28d\`. Set to \`null\` to disable clearing out of old rows.
+#
+#user_ips_max_age: 14d
+
+
+## TLS ##
+
+# PEM-encoded X509 certificate for TLS.
+# This certificate, as of Synapse 1.0, will need to be a valid and verifiable
+# certificate, signed by a recognised Certificate Authority.
+#
+# See 'ACME support' below to enable auto-provisioning this certificate via
+# Let's Encrypt.
+#
+# If supplying your own, be sure to use a \`.pem\` file that includes the
+# full certificate chain including any intermediate certificates (for
+# instance, if using certbot, use \`fullchain.pem\` as your certificate,
+# not \`cert.pem\`).
+#
+#tls_certificate_path: "CONFDIR/SERVERNAME.tls.crt"
+
+# PEM-encoded private key for TLS
+#
+#tls_private_key_path: "CONFDIR/SERVERNAME.tls.key"
+
+# Whether to verify TLS server certificates for outbound federation requests.
+#
+# Defaults to \`true\`. To disable certificate verification, uncomment the
+# following line.
+#
+#federation_verify_certificates: false
+
+# The minimum TLS version that will be used for outbound federation requests.
+#
+# Defaults to \`1\`. Configurable to \`1\`, \`1.1\`, \`1.2\`, or \`1.3\`. Note
+# that setting this value higher than \`1.2\` will prevent federation to most
+# of the public Matrix network: only configure it to \`1.3\` if you have an
+# entirely private federation setup and you can ensure TLS 1.3 support.
+#
+#federation_client_minimum_tls_version: 1.2
+
+# Skip federation certificate verification on the following whitelist
+# of domains.
+#
+# This setting should only be used in very specific cases, such as
+# federation over Tor hidden services and similar. For private networks
+# of homeservers, you likely want to use a private CA instead.
+#
+# Only effective if federation_verify_certicates is \`true\`.
+#
+#federation_certificate_verification_whitelist:
+#  - lon.example.com
+#  - *.domain.com
+#  - *.onion
+
+# List of custom certificate authorities for federation traffic.
+#
+# This setting should only normally be used within a private network of
+# homeservers.
+#
+# Note that this list will replace those that are provided by your
+# operating environment. Certificates must be in PEM format.
+#
+#federation_custom_ca_list:
+#  - myCA1.pem
+#  - myCA2.pem
+#  - myCA3.pem
+
+# ACME support: This will configure Synapse to request a valid TLS certificate
+# for your configured \`server_name\` via Let's Encrypt.
+#
+# Note that provisioning a certificate in this way requires port 80 to be
+# routed to Synapse so that it can complete the http-01 ACME challenge.
+# By default, if you enable ACME support, Synapse will attempt to listen on
+# port 80 for incoming http-01 challenges - however, this will likely fail
+# with 'Permission denied' or a similar error.
+#
+# There are a couple of potential solutions to this:
+#
+#  * If you already have an Apache, Nginx, or similar listening on port 80,
+#    you can configure Synapse to use an alternate port, and have your web
+#    server forward the requests. For example, assuming you set 'port: 8009'
+#    below, on Apache, you would write:
+#
+#    ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge
+#
+#  * Alternatively, you can use something like \`authbind\` to give Synapse
+#    permission to listen on port 80.
+#
+acme:
+    # ACME support is disabled by default. Set this to \`true\` and uncomment
+    # tls_certificate_path and tls_private_key_path above to enable it.
+    #
+    enabled: false
+
+    # Endpoint to use to request certificates. If you only want to test,
+    # use Let's Encrypt's staging url:
+    #     https://acme-staging.api.letsencrypt.org/directory
+    #
+    #url: https://acme-v01.api.letsencrypt.org/directory
+
+    # Port number to listen on for the HTTP-01 challenge. Change this if
+    # you are forwarding connections through Apache/Nginx/etc.
+    #
+    port: 80
+
+    # Local addresses to listen on for incoming connections.
+    # Again, you may want to change this if you are forwarding connections
+    # through Apache/Nginx/etc.
+    #
+    bind_addresses: ['::', '0.0.0.0']
+
+    # How many days remaining on a certificate before it is renewed.
+    #
+    reprovision_threshold: 30
+
+    # The domain that the certificate should be for. Normally this
+    # should be the same as your Matrix domain (i.e., 'server_name'), but,
+    # by putting a file at 'https://<server_name>/.well-known/matrix/server',
+    # you can delegate incoming traffic to another server. If you do that,
+    # you should give the target of the delegation here.
+    #
+    # For example: if your 'server_name' is 'example.com', but
+    # 'https://example.com/.well-known/matrix/server' delegates to
+    # 'matrix.example.com', you should put 'matrix.example.com' here.
+    #
+    # If not set, defaults to your 'server_name'.
+    #
+    domain: matrix.example.com
+
+    # file to use for the account key. This will be generated if it doesn't
+    # exist.
+    #
+    # If unspecified, we will use CONFDIR/client.key.
+    #
+    account_key_file: "$DATA_DIR/acme_account.key"
+
+# List of allowed TLS fingerprints for this server to publish along
+# with the signing keys for this server. Other matrix servers that
+# make HTTPS requests to this server will check that the TLS
+# certificates returned by this server match one of the fingerprints.
+#
+# Synapse automatically adds the fingerprint of its own certificate
+# to the list. So if federation traffic is handled directly by synapse
+# then no modification to the list is required.
+#
+# If synapse is run behind a load balancer that handles the TLS then it
+# will be necessary to add the fingerprints of the certificates used by
+# the loadbalancers to this list if they are different to the one
+# synapse is using.
+#
+# Homeservers are permitted to cache the list of TLS fingerprints
+# returned in the key responses up to the "valid_until_ts" returned in
+# key. It may be necessary to publish the fingerprints of a new
+# certificate and wait until the "valid_until_ts" of the previous key
+# responses have passed before deploying it.
+#
+# You can calculate a fingerprint from a given TLS listener via:
+# openssl s_client -connect $host:$port < /dev/null 2> /dev/null |
+#   openssl x509 -outform DER | openssl sha256 -binary | base64 | tr -d '='
+# or by checking matrix.org/federationtester/api/report?server_name=$host
+#
+#tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
+
+
+
+## Database ##
+
+$(generate_database)
+
+# Number of events to cache in memory.
+#
+#event_cache_size: 10K
+
+
+## Logging ##
+
+# A yaml python logging config file as described by
+# https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
+#
+log_config: "$LOG_CONFIG_PATH"
+
+
+## Ratelimiting ##
+
+# Ratelimiting settings for client actions (registration, login, messaging).
+#
+# Each ratelimiting configuration is made of two parameters:
+#   - per_second: number of requests a client can send per second.
+#   - burst_count: number of requests a client can send before being throttled.
+#
+# Synapse currently uses the following configurations:
+#   - one for messages that ratelimits sending based on the account the client
+#     is using
+#   - one for registration that ratelimits registration requests based on the
+#     client's IP address.
+#   - one for login that ratelimits login requests based on the client's IP
+#     address.
+#   - one for login that ratelimits login requests based on the account the
+#     client is attempting to log into.
+#   - one for login that ratelimits login requests based on the account the
+#     client is attempting to log into, based on the amount of failed login
+#     attempts for this account.
+#   - one for ratelimiting redactions by room admins. If this is not explicitly
+#     set then it uses the same ratelimiting as per rc_message. This is useful
+#     to allow room admins to deal with abuse quickly.
+#
+# The defaults are as shown below.
+#
+#rc_message:
+#  per_second: 0.2
+#  burst_count: 10
+#
+#rc_registration:
+#  per_second: 0.17
+#  burst_count: 3
+#
+#rc_login:
+#  address:
+#    per_second: 0.17
+#    burst_count: 3
+#  account:
+#    per_second: 0.17
+#    burst_count: 3
+#  failed_attempts:
+#    per_second: 0.17
+#    burst_count: 3
+#
+#rc_admin_redaction:
+#  per_second: 1
+#  burst_count: 50
+
+
+# Ratelimiting settings for incoming federation
+#
+# The rc_federation configuration is made up of the following settings:
+#   - window_size: window size in milliseconds
+#   - sleep_limit: number of federation requests from a single server in
+#     a window before the server will delay processing the request.
+#   - sleep_delay: duration in milliseconds to delay processing events
+#     from remote servers by if they go over the sleep limit.
+#   - reject_limit: maximum number of concurrent federation requests
+#     allowed from a single server
+#   - concurrent: number of federation requests to concurrently process
+#     from a single server
+#
+# The defaults are as shown below.
+#
+#rc_federation:
+#  window_size: 1000
+#  sleep_limit: 10
+#  sleep_delay: 500
+#  reject_limit: 50
+#  concurrent: 3
+
+# Target outgoing federation transaction frequency for sending read-receipts,
+# per-room.
+#
+# If we end up trying to send out more read-receipts, they will get buffered up
+# into fewer transactions.
+#
+#federation_rr_transactions_per_room_per_second: 50
+
+
+
+## Media Store ##
+
+# Enable the media store service in the Synapse master. Uncomment the
+# following if you are using a separate media store worker.
+#
+#enable_media_repo: false
+
+# Directory where uploaded images and attachments are stored.
+#
+media_store_path: "$DATA_DIR/media_store"
+
+# Media storage providers allow media to be stored in different
+# locations.
+#
+#media_storage_providers:
+#  - module: file_system
+#    # Whether to write new local files.
+#    store_local: false
+#    # Whether to write new remote media
+#    store_remote: false
+#    # Whether to block upload requests waiting for write to this
+#    # provider to complete
+#    store_synchronous: false
+#    config:
+#       directory: /mnt/some/other/directory
+
+# Directory where in-progress uploads are stored.
+#
+uploads_path: "$DATA_DIR/uploads"
+
+# The largest allowed upload size in bytes
+#
+max_upload_size: "$MAX_UPLOAD_SIZE"
+
+# Maximum number of pixels that will be thumbnailed
+#
+#max_image_pixels: 32M
+
+# Whether to generate new thumbnails on the fly to precisely match
+# the resolution requested by the client. If true then whenever
+# a new resolution is requested by the client the server will
+# generate a new thumbnail. If false the server will pick a thumbnail
+# from a precalculated list.
+#
+#dynamic_thumbnails: false
+
+# List of thumbnails to precalculate when an image is uploaded.
+#
+#thumbnail_sizes:
+#  - width: 32
+#    height: 32
+#    method: crop
+#  - width: 96
+#    height: 96
+#    method: crop
+#  - width: 320
+#    height: 240
+#    method: scale
+#  - width: 640
+#    height: 480
+#    method: scale
+#  - width: 800
+#    height: 600
+#    method: scale
+
+# Is the preview URL API enabled?
+#
+# 'false' by default: uncomment the following to enable it (and specify a
+# url_preview_ip_range_blacklist blacklist).
+#
+#url_preview_enabled: true
+
+# List of IP address CIDR ranges that the URL preview spider is denied
+# from accessing.  There are no defaults: you must explicitly
+# specify a list for URL previewing to work.  You should specify any
+# internal services in your network that you do not want synapse to try
+# to connect to, otherwise anyone in any Matrix room could cause your
+# synapse to issue arbitrary GET requests to your internal services,
+# causing serious security issues.
+#
+# (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly
+# listed here, since they correspond to unroutable addresses.)
+#
+# This must be specified if url_preview_enabled is set. It is recommended that
+# you uncomment the following list as a starting point.
+#
+#url_preview_ip_range_blacklist:
+#  - '127.0.0.0/8'
+#  - '10.0.0.0/8'
+#  - '172.16.0.0/12'
+#  - '192.168.0.0/16'
+#  - '100.64.0.0/10'
+#  - '169.254.0.0/16'
+#  - '::1/128'
+#  - 'fe80::/64'
+#  - 'fc00::/7'
+
+# List of IP address CIDR ranges that the URL preview spider is allowed
+# to access even if they are specified in url_preview_ip_range_blacklist.
+# This is useful for specifying exceptions to wide-ranging blacklisted
+# target IP ranges - e.g. for enabling URL previews for a specific private
+# website only visible in your network.
+#
+#url_preview_ip_range_whitelist:
+#   - '192.168.1.1'
+
+# Optional list of URL matches that the URL preview spider is
+# denied from accessing.  You should use url_preview_ip_range_blacklist
+# in preference to this, otherwise someone could define a public DNS
+# entry that points to a private IP address and circumvent the blacklist.
+# This is more useful if you know there is an entire shape of URL that
+# you know that will never want synapse to try to spider.
+#
+# Each list entry is a dictionary of url component attributes as returned
+# by urlparse.urlsplit as applied to the absolute form of the URL.  See
+# https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit
+# The values of the dictionary are treated as an filename match pattern
+# applied to that component of URLs, unless they start with a ^ in which
+# case they are treated as a regular expression match.  If all the
+# specified component matches for a given list item succeed, the URL is
+# blacklisted.
+#
+#url_preview_url_blacklist:
+#  # blacklist any URL with a username in its URI
+#  - username: '*'
+#
+#  # blacklist all *.google.com URLs
+#  - netloc: 'google.com'
+#  - netloc: '*.google.com'
+#
+#  # blacklist all plain HTTP URLs
+#  - scheme: 'http'
+#
+#  # blacklist http(s)://www.acme.com/foo
+#  - netloc: 'www.acme.com'
+#    path: '/foo'
+#
+#  # blacklist any URL with a literal IPv4 address
+#  - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'
+
+# The largest allowed URL preview spidering size in bytes
+#
+#max_spider_size: 10M
+
+
+## Captcha ##
+# See docs/CAPTCHA_SETUP for full details of configuring this.
+
+# This Home Server's ReCAPTCHA public key.
+#
+#recaptcha_public_key: "YOUR_PUBLIC_KEY"
+
+# This Home Server's ReCAPTCHA private key.
+#
+#recaptcha_private_key: "YOUR_PRIVATE_KEY"
+
+# Enables ReCaptcha checks when registering, preventing signup
+# unless a captcha is answered. Requires a valid ReCaptcha
+# public/private key.
+#
+#enable_registration_captcha: false
+
+# A secret key used to bypass the captcha test entirely.
+#
+#captcha_bypass_secret: "YOUR_SECRET_HERE"
+
+# The API endpoint to use for verifying m.login.recaptcha responses.
+#
+#recaptcha_siteverify_api: "https://www.recaptcha.net/recaptcha/api/siteverify"
+
+
+## TURN ##
+
+# The public URIs of the TURN server to give to clients
+#
+turn_uris: $TURN_URIS
+
+# The shared secret used to compute passwords for the TURN server
+#
+turn_shared_secret: "$TURN_SHARED_SECRET"
+
+# The Username and password if the TURN server needs them and
+# does not use a token
+#
+#turn_username: "TURNSERVER_USERNAME"
+#turn_password: "TURNSERVER_PASSWORD"
+
+# How long generated TURN credentials last
+#
+turn_user_lifetime: "$TURN_USER_LIFETIME"
+
+# Whether guests should be allowed to use the TURN server.
+# This defaults to True, otherwise VoIP will be unreliable for guests.
+# However, it does introduce a slight security risk as it allows users to
+# connect to arbitrary endpoints without having first signed up for a
+# valid account (e.g. by passing a CAPTCHA).
+#
+#turn_allow_guests: true
+
+## Registration ##
+#
+# Registration can be rate-limited using the parameters in the "Ratelimiting"
+# section of this file.
+
+# Enable registration for new users.
+#
+enable_registration: $ALLOW_REGISTRATION
+
+# Optional account validity configuration. This allows for accounts to be denied
+# any request after a given period.
+#
+# \`\`enabled\`\` defines whether the account validity feature is enabled. Defaults
+# to False.
+#
+# \`\`period\`\` allows setting the period after which an account is valid
+# after its registration. When renewing the account, its validity period
+# will be extended by this amount of time. This parameter is required when using
+# the account validity feature.
+#
+# \`\`renew_at\`\` is the amount of time before an account's expiry date at which
+# Synapse will send an email to the account's email address with a renewal link.
+# This needs the \`\`email\`\` and \`\`public_baseurl\`\` configuration sections to be
+# filled.
+#
+# \`\`renew_email_subject\`\` is the subject of the email sent out with the renewal
+# link. \`\`%(app)s\`\` can be used as a placeholder for the \`\`app_name\`\` parameter
+# from the \`\`email\`\` section.
+#
+# Once this feature is enabled, Synapse will look for registered users without an
+# expiration date at startup and will add one to every account it found using the
+# current settings at that time.
+# This means that, if a validity period is set, and Synapse is restarted (it will
+# then derive an expiration date from the current validity period), and some time
+# after that the validity period changes and Synapse is restarted, the users'
+# expiration dates won't be updated unless their account is manually renewed. This
+# date will be randomly selected within a range [now + period - d ; now + period],
+# where d is equal to 10% of the validity period.
+#
+#account_validity:
+#  enabled: true
+#  period: 6w
+#  renew_at: 1w
+#  renew_email_subject: "Renew your %(app)s account"
+#  # Directory in which Synapse will try to find the HTML files to serve to the
+#  # user when trying to renew an account. Optional, defaults to
+#  # synapse/res/templates.
+#  template_dir: "res/templates"
+#  # HTML to be displayed to the user after they successfully renewed their
+#  # account. Optional.
+#  account_renewed_html_path: "account_renewed.html"
+#  # HTML to be displayed when the user tries to renew an account with an invalid
+#  # renewal token. Optional.
+#  invalid_token_html_path: "invalid_token.html"
+
+# Time that a user's session remains valid for, after they log in.
+#
+# Note that this is not currently compatible with guest logins.
+#
+# Note also that this is calculated at login time: changes are not applied
+# retrospectively to users who have already logged in.
+#
+# By default, this is infinite.
+#
+#session_lifetime: 24h
+
+# The user must provide all of the below types of 3PID when registering.
+#
+#registrations_require_3pid:
+#  - email
+#  - msisdn
+
+# Explicitly disable asking for MSISDNs from the registration
+# flow (overrides registrations_require_3pid if MSISDNs are set as required)
+#
+#disable_msisdn_registration: true
+
+# Mandate that users are only allowed to associate certain formats of
+# 3PIDs with accounts on this server.
+#
+#allowed_local_3pids:
+#  - medium: email
+#    pattern: '.*@matrix\.org'
+#  - medium: email
+#    pattern: '.*@vector\.im'
+#  - medium: msisdn
+#    pattern: '\+44'
+
+# Enable 3PIDs lookup requests to identity servers from this server.
+#
+#enable_3pid_lookup: true
+
+# If set, allows registration of standard or admin accounts by anyone who
+# has the shared secret, even if registration is otherwise disabled.
+#
+# registration_shared_secret: <PRIVATE STRING>
+
+# Set the number of bcrypt rounds used to generate password hash.
+# Larger numbers increase the work factor needed to generate the hash.
+# The default number is 12 (which equates to 2^12 rounds).
+# N.B. that increasing this will exponentially increase the time required
+# to register or login - e.g. 24 => 2^24 rounds which will take >20 mins.
+#
+#bcrypt_rounds: 12
+
+# Allows users to register as guests without a password/email/etc, and
+# participate in rooms hosted on this server which have been made
+# accessible to anonymous users.
+#
+#allow_guest_access: false
+
+# The identity server which we suggest that clients should use when users log
+# in on this server.
+#
+# (By default, no suggestion is made, so it is left up to the client.
+# This setting is ignored unless public_baseurl is also set.)
+#
+#default_identity_server: https://matrix.org
+
+# The list of identity servers trusted to verify third party
+# identifiers by this server.
+#
+# Also defines the ID server which will be called when an account is
+# deactivated (one will be picked arbitrarily).
+#
+# Note: This option is deprecated. Since v0.99.4, Synapse has tracked which identity
+# server a 3PID has been bound to. For 3PIDs bound before then, Synapse runs a
+# background migration script, informing itself that the identity server all of its
+# 3PIDs have been bound to is likely one of the below.
+#
+# As of Synapse v1.4.0, all other functionality of this option has been deprecated, and
+# it is now solely used for the purposes of the background migration script, and can be
+# removed once it has run.
+#trusted_third_party_id_servers:
+#  - matrix.org
+#  - vector.im
+
+# Handle threepid (email/phone etc) registration and password resets through a set of
+# *trusted* identity servers. Note that this allows the configured identity server to
+# reset passwords for accounts!
+#
+# Be aware that if \`email\` is not set, and SMTP options have not been
+# configured in the email config block, registration and user password resets via
+# email will be globally disabled.
+#
+# Additionally, if \`msisdn\` is not set, registration and password resets via msisdn
+# will be disabled regardless. This is due to Synapse currently not supporting any
+# method of sending SMS messages on its own.
+#
+# To enable using an identity server for operations regarding a particular third-party
+# identifier type, set the value to the URL of that identity server as shown in the
+# examples below.
+#
+# Servers handling the these requests must answer the \`/requestToken\` endpoints defined
+# by the Matrix Identity Service API specification:
+# https://matrix.org/docs/spec/identity_service/latest
+#
+# If a delegate is specified, the config option public_baseurl must also be filled out.
+#
+account_threepid_delegates:
+    #email: https://example.com     # Delegate email sending to example.org
+    #msisdn: http://localhost:8090  # Delegate SMS sending to this local process
+
+# Users who register on this homeserver will automatically be joined
+# to these rooms
+#
+#auto_join_rooms:
+#  - "#example:example.com"
+
+# Where auto_join_rooms are specified, setting this flag ensures that the
+# the rooms exist by creating them when the first user on the
+# homeserver registers.
+# Setting to false means that if the rooms are not manually created,
+# users cannot be auto-joined since they do not exist.
+#
+#autocreate_auto_join_rooms: true
+
+
+## Metrics ###
+
+# Enable collection and rendering of performance metrics
+#
+enable_metrics: $EXPOSE_METRICS
+
+# Enable sentry integration
+# NOTE: While attempts are made to ensure that the logs don't contain
+# any sensitive information, this cannot be guaranteed. By enabling
+# this option the sentry server may therefore receive sensitive
+# information, and it in turn may then diseminate sensitive information
+# through insecure notification channels if so configured.
+#
+#sentry:
+#    dsn: "..."
+
+# Flags to enable Prometheus metrics which are not suitable to be
+# enabled by default, either for performance reasons or limited use.
+#
+metrics_flags:
+    # Publish synapse_federation_known_servers, a g auge of the number of
+    # servers this homeserver knows about, including itself. May cause
+    # performance problems on large homeservers.
+    #
+    #known_servers: true
+
+# Whether or not to report anonymized homeserver usage statistics.
+report_stats: $REPORT_STATS
+
+# The endpoint to report the anonymized homeserver usage statistics to.
+# Defaults to https://matrix.org/report-usage-stats/push
+#
+#report_stats_endpoint: https://example.com/report-usage-stats/push
+
+
+## API Configuration ##
+
+# A list of event types that will be included in the room_invite_state
+#
+#room_invite_state_types:
+#  - "m.room.join_rules"
+#  - "m.room.canonical_alias"
+#  - "m.room.avatar"
+#  - "m.room.encryption"
+#  - "m.room.name"
+
+
+# A list of application service config files to use
+#
+#app_service_config_files:
+#  - app_service_1.yaml
+#  - app_service_2.yaml
+
+# Uncomment to enable tracking of application service IP addresses. Implicitly
+# enables MAU tracking for application service users.
+#
+#track_appservice_user_ips: true
+
+
+# a secret which is used to sign access tokens. If none is specified,
+# the registration_shared_secret is used, if one is given; otherwise,
+# a secret key is derived from the signing key.
+#
+# macaroon_secret_key: <PRIVATE STRING>
+
+# a secret which is used to calculate HMACs for form values, to stop
+# falsification of values. Must be specified for the User Consent
+# forms to work.
+#
+# form_secret: <PRIVATE STRING>
+
+## Signing Keys ##
+
+# Path to the signing key to sign messages with
+#
+signing_key_path: "$SIGNING_KEY_PATH"
+
+# The keys that the server used to sign messages with but won't use
+# to sign new messages. E.g. it has lost its private key
+#
+#old_signing_keys:
+#  "ed25519:auto":
+#    # Base64 encoded public key
+#    key: "The public part of your old signing key."
+#    # Millisecond POSIX timestamp when the key expired.
+#    expired_ts: 123456789123
+
+# How long key response published by this server is valid for.
+# Used to set the valid_until_ts in /key/v2 APIs.
+# Determines how quickly servers will query to check which keys
+# are still valid.
+#
+#key_refresh_interval: 1d
+
+# The trusted servers to download signing keys from.
+#
+# When we need to fetch a signing key, each server is tried in parallel.
+#
+# Normally, the connection to the key server is validated via TLS certificates.
+# Additional security can be provided by configuring a \`verify key\`, which
+# will make synapse check that the response is signed by that key.
+#
+# This setting supercedes an older setting named \`perspectives\`. The old format
+# is still supported for backwards-compatibility, but it is deprecated.
+#
+# 'trusted_key_servers' defaults to matrix.org, but using it will generate a
+# warning on start-up. To suppress this warning, set
+# 'suppress_key_server_warning' to true.
+#
+# Options for each entry in the list include:
+#
+#    server_name: the name of the server. required.
+#
+#    verify_keys: an optional map from key id to base64-encoded public key.
+#       If specified, we will check that the response is signed by at least
+#       one of the given keys.
+#
+#    accept_keys_insecurely: a boolean. Normally, if \`verify_keys\` is unset,
+#       and federation_verify_certificates is not \`true\`, synapse will refuse
+#       to start, because this would allow anyone who can spoof DNS responses
+#       to masquerade as the trusted key server. If you know what you are doing
+#       and are sure that your network environment provides a secure connection
+#       to the key server, you can set this to \`true\` to override this
+#       behaviour.
+#
+# An example configuration might look like:
+#
+#trusted_key_servers:
+#  - server_name: "my_trusted_server.example.com"
+#    verify_keys:
+#      "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr"
+#  - server_name: "my_other_trusted_server.example.com"
+#
+trusted_key_servers:
+  - server_name: "matrix.org"
+
+# Uncomment the following to disable the warning that is emitted when the
+# trusted_key_servers include 'matrix.org'. See above.
+#
+#suppress_key_server_warning: true
+
+# The signing keys to use when acting as a trusted key server. If not specified
+# defaults to the server signing key.
+#
+# Can contain multiple keys, one per line.
+#
+#key_server_signing_keys_path: "key_server_signing_keys.key"
+
+
+# Enable SAML2 for registration and login. Uses pysaml2.
+#
+# At least one of \`sp_config\` or \`config_path\` must be set in this section to
+# enable SAML login.
+#
+# (You will probably also want to set the following options to \`false\` to
+# disable the regular login/registration flows:
+#   * enable_registration
+#   * password_config.enabled
+#
+# Once SAML support is enabled, a metadata file will be exposed at
+# https://<server>:<port>/_matrix/saml2/metadata.xml, which you may be able to
+# use to configure your SAML IdP with. Alternatively, you can manually configure
+# the IdP to use an ACS location of
+# https://<server>:<port>/_matrix/saml2/authn_response.
+#
+saml2_config:
+  # \`sp_config\` is the configuration for the pysaml2 Service Provider.
+  # See pysaml2 docs for format of config.
+  #
+  # Default values will be used for the 'entityid' and 'service' settings,
+  # so it is not normally necessary to specify them unless you need to
+  # override them.
+  #
+  #sp_config:
+  #  # point this to the IdP's metadata. You can use either a local file or
+  #  # (preferably) a URL.
+  #  metadata:
+  #    #local: ["saml2/idp.xml"]
+  #    remote:
+  #      - url: https://our_idp/metadata.xml
+  #
+  #    # By default, the user has to go to our login page first. If you'd like
+  #    # to allow IdP-initiated login, set 'allow_unsolicited: true' in a
+  #    # 'service.sp' section:
+  #    #
+  #    #service:
+  #    #  sp:
+  #    #    allow_unsolicited: true
+  #
+  #    # The examples below are just used to generate our metadata xml, and you
+  #    # may well not need them, depending on your setup. Alternatively you
+  #    # may need a whole lot more detail - see the pysaml2 docs!
+  #
+  #    description: ["My awesome SP", "en"]
+  #    name: ["Test SP", "en"]
+  #
+  #    organization:
+  #      name: Example com
+  #      display_name:
+  #        - ["Example co", "en"]
+  #      url: "http://example.com"
+  #
+  #    contact_person:
+  #      - given_name: Bob
+  #        sur_name: "the Sysadmin"
+  #        email_address": ["admin@example.com"]
+  #        contact_type": technical
+
+  # Instead of putting the config inline as above, you can specify a
+  # separate pysaml2 configuration file:
+  #
+  #config_path: "CONFDIR/sp_conf.py"
+
+  # the lifetime of a SAML session. This defines how long a user has to
+  # complete the authentication process, if allow_unsolicited is unset.
+  # The default is 5 minutes.
+  #
+  #saml_session_lifetime: 5m
+
+  # The SAML attribute (after mapping via the attribute maps) to use to derive
+  # the Matrix ID from. 'uid' by default.
+  #
+  #mxid_source_attribute: displayName
+
+  # The mapping system to use for mapping the saml attribute onto a matrix ID.
+  # Options include:
+  #  * 'hexencode' (which maps unpermitted characters to '=xx')
+  #  * 'dotreplace' (which replaces unpermitted characters with '.').
+  # The default is 'hexencode'.
+  #
+  #mxid_mapping: dotreplace
+
+  # In previous versions of synapse, the mapping from SAML attribute to MXID was
+  # always calculated dynamically rather than stored in a table. For backwards-
+  # compatibility, we will look for user_ids matching such a pattern before
+  # creating a new account.
+  #
+  # This setting controls the SAML attribute which will be used for this
+  # backwards-compatibility lookup. Typically it should be 'uid', but if the
+  # attribute maps are changed, it may be necessary to change it.
+  #
+  # The default is 'uid'.
+  #
+  #grandfathered_mxid_source_attribute: upn
+
+
+
+# Enable CAS for registration and login.
+#
+#cas_config:
+#   enabled: true
+#   server_url: "https://cas-server.com"
+#   service_url: "https://homeserver.domain.com:8448"
+#   #displayname_attribute: name
+#   #required_attributes:
+#   #    name: value
+
+
+# The JWT needs to contain a globally unique "sub" (subject) claim.
+#
+#jwt_config:
+#   enabled: true
+#   secret: "a secret"
+#   algorithm: "HS256"
+
+
+password_config:
+   # Uncomment to disable password login
+   #
+   #enabled: false
+
+   # Uncomment to disable authentication against the local password
+   # database. This is ignored if \`enabled\` is false, and is only useful
+   # if you have other password_providers.
+   #
+   #localdb_enabled: false
+
+   # Uncomment and change to a secret random string for extra security.
+   # DO NOT CHANGE THIS AFTER INITIAL SETUP!
+   #
+   #pepper: "EVEN_MORE_SECRET"
+
+
+
+# Enable sending emails for password resets, notification events or
+# account expiry notices
+#
+# If your SMTP server requires authentication, the optional smtp_user &
+# smtp_pass variables should be used
+#
+#email:
+#   enable_notifs: false
+#   smtp_host: "localhost"
+#   smtp_port: 25 # SSL: 465, STARTTLS: 587
+#   smtp_user: "exampleusername"
+#   smtp_pass: "examplepassword"
+#   require_transport_security: false
+#   notif_from: "Your Friendly %(app)s Home Server <noreply@example.com>"
+#   app_name: Matrix
+#
+#   # Enable email notifications by default
+#   #
+#   notif_for_new_users: true
+#
+#   # Defining a custom URL for Riot is only needed if email notifications
+#   # should contain links to a self-hosted installation of Riot; when set
+#   # the "app_name" setting is ignored
+#   #
+#   riot_base_url: "http://localhost/riot"
+#
+#   # Configure the time that a validation email or text message code
+#   # will expire after sending
+#   #
+#   # This is currently used for password resets
+#   #
+#   #validation_token_lifetime: 1h
+#
+#   # Template directory. All template files should be stored within this
+#   # directory. If not set, default templates from within the Synapse
+#   # package will be used
+#   #
+#   # For the list of default templates, please see
+#   # https://github.com/matrix-org/synapse/tree/master/synapse/res/templates
+#   #
+#   #template_dir: res/templates
+#
+#   # Templates for email notifications
+#   #
+#   notif_template_html: notif_mail.html
+#   notif_template_text: notif_mail.txt
+#
+#   # Templates for account expiry notices
+#   #
+#   expiry_template_html: notice_expiry.html
+#   expiry_template_text: notice_expiry.txt
+#
+#   # Templates for password reset emails sent by the homeserver
+#   #
+#   #password_reset_template_html: password_reset.html
+#   #password_reset_template_text: password_reset.txt
+#
+#   # Templates for registration emails sent by the homeserver
+#   #
+#   #registration_template_html: registration.html
+#   #registration_template_text: registration.txt
+#
+#   # Templates for validation emails sent by the homeserver when adding an email to
+#   # your user account
+#   #
+#   #add_threepid_template_html: add_threepid.html
+#   #add_threepid_template_text: add_threepid.txt
+#
+#   # Templates for password reset success and failure pages that a user
+#   # will see after attempting to reset their password
+#   #
+#   #password_reset_template_success_html: password_reset_success.html
+#   #password_reset_template_failure_html: password_reset_failure.html
+#
+#   # Templates for registration success and failure pages that a user
+#   # will see after attempting to register using an email or phone
+#   #
+#   #registration_template_success_html: registration_success.html
+#   #registration_template_failure_html: registration_failure.html
+#
+#   # Templates for success and failure pages that a user will see after attempting
+#   # to add an email or phone to their account
+#   #
+#   #add_threepid_success_html: add_threepid_success.html
+#   #add_threepid_failure_html: add_threepid_failure.html
+
+$(generate_password_providers)
+
+# Clients requesting push notifications can either have the body of
+# the message sent in the notification poke along with other details
+# like the sender, or just the event ID and room ID (\`event_id_only\`).
+# If clients choose the former, this option controls whether the
+# notification request includes the content of the event (other details
+# like the sender are still included). For \`event_id_only\` push, it
+# has no effect.
+#
+# For modern android devices the notification content will still appear
+# because it is loaded by the app. iPhone, however will send a
+# notification saying only that a message arrived and who it came from.
+#
+#push:
+#  include_content: true
+
+
+#spam_checker:
+#  module: "my_custom_project.SuperSpamChecker"
+#  config:
+#    example_option: 'things'
+
+
+# Uncomment to allow non-server-admin users to create groups on this server
+#
+#enable_group_creation: true
+
+# If enabled, non server admins can only create groups with local parts
+# starting with this prefix
+#
+#group_creation_prefix: "unofficial/"
+
+
+
+# User Directory configuration
+#
+# 'enabled' defines whether users can search the user directory. If
+# false then empty responses are returned to all queries. Defaults to
+# true.
+#
+# 'search_all_users' defines whether to search all users visible to your HS
+# when searching the user directory, rather than limiting to users visible
+# in public rooms.  Defaults to false.  If you set it True, you'll have to
+# rebuild the user_directory search indexes, see
+# https://github.com/matrix-org/synapse/blob/master/docs/user_directory.md
+#
+#user_directory:
+#  enabled: true
+#  search_all_users: false
+
+
+# User Consent configuration
+#
+# for detailed instructions, see
+# https://github.com/matrix-org/synapse/blob/master/docs/consent_tracking.md
+#
+# Parts of this section are required if enabling the 'consent' resource under
+# 'listeners', in particular 'template_dir' and 'version'.
+#
+# 'template_dir' gives the location of the templates for the HTML forms.
+# This directory should contain one subdirectory per language (eg, 'en', 'fr'),
+# and each language directory should contain the policy document (named as
+# '<version>.html') and a success page (success.html).
+#
+# 'version' specifies the 'current' version of the policy document. It defines
+# the version to be served by the consent resource if there is no 'v'
+# parameter.
+#
+# 'server_notice_content', if enabled, will send a user a "Server Notice"
+# asking them to consent to the privacy policy. The 'server_notices' section
+# must also be configured for this to work. Notices will *not* be sent to
+# guest users unless 'send_server_notice_to_guests' is set to true.
+#
+# 'block_events_error', if set, will block any attempts to send events
+# until the user consents to the privacy policy. The value of the setting is
+# used as the text of the error.
+#
+# 'require_at_registration', if enabled, will add a step to the registration
+# process, similar to how captcha works. Users will be required to accept the
+# policy before their account is created.
+#
+# 'policy_name' is the display name of the policy users will see when registering
+# for an account. Has no effect unless \`require_at_registration\` is enabled.
+# Defaults to "Privacy Policy".
+#
+#user_consent:
+#  template_dir: res/templates/privacy
+#  version: 1.0
+#  server_notice_content:
+#    msgtype: m.text
+#    body: >-
+#      To continue using this homeserver you must review and agree to the
+#      terms and conditions at %(consent_uri)s
+#  send_server_notice_to_guests: true
+#  block_events_error: >-
+#    To continue using this homeserver you must review and agree to the
+#    terms and conditions at %(consent_uri)s
+#  require_at_registration: false
+#  policy_name: Privacy Policy
+#
+
+
+
+# Local statistics collection. Used in populating the room directory.
+#
+# 'bucket_size' controls how large each statistics timeslice is. It can
+# be defined in a human readable short form -- e.g. "1d", "1y".
+#
+# 'retention' controls how long historical statistics will be kept for.
+# It can be defined in a human readable short form -- e.g. "1d", "1y".
+#
+#
+#stats:
+#   enabled: true
+#   bucket_size: 1d
+#   retention: 1y
+
+
+# Server Notices room configuration
+#
+# Uncomment this section to enable a room which can be used to send notices
+# from the server to users. It is a special room which cannot be left; notices
+# come from a special "notices" user id.
+#
+# If you uncomment this section, you *must* define the system_mxid_localpart
+# setting, which defines the id of the user which will be used to send the
+# notices.
+#
+# It's also possible to override the room name, the display name of the
+# "notices" user, and the avatar for the user.
+#
+#server_notices:
+#  system_mxid_localpart: notices
+#  system_mxid_display_name: "Server Notices"
+#  system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
+#  room_name: "Server Notices"
+
+
+
+# Uncomment to disable searching the public room list. When disabled
+# blocks searching local and remote room lists for local and remote
+# users by always returning an empty list for all queries.
+#
+#enable_room_list_search: false
+
+# The \`alias_creation\` option controls who's allowed to create aliases
+# on this server.
+#
+# The format of this option is a list of rules that contain globs that
+# match against user_id, room_id and the new alias (fully qualified with
+# server name). The action in the first rule that matches is taken,
+# which can currently either be "allow" or "deny".
+#
+# Missing user_id/room_id/alias fields default to "*".
+#
+# If no rules match the request is denied. An empty list means no one
+# can create aliases.
+#
+# Options for the rules include:
+#
+#   user_id: Matches against the creator of the alias
+#   alias: Matches against the alias being created
+#   room_id: Matches against the room ID the alias is being pointed at
+#   action: Whether to "allow" or "deny" the request if the rule matches
+#
+# The default is:
+#
+#alias_creation_rules:
+#  - user_id: "*"
+#    alias: "*"
+#    room_id: "*"
+#    action: allow
+
+# The \`room_list_publication_rules\` option controls who can publish and
+# which rooms can be published in the public room list.
+#
+# The format of this option is the same as that for
+# \`alias_creation_rules\`.
+#
+# If the room has one or more aliases associated with it, only one of
+# the aliases needs to match the alias rule. If there are no aliases
+# then only rules with \`alias: *\` match.
+#
+# If no rules match the request is denied. An empty list means no one
+# can publish rooms.
+#
+# Options for the rules include:
+#
+#   user_id: Matches agaisnt the creator of the alias
+#   room_id: Matches against the room ID being published
+#   alias: Matches against any current local or canonical aliases
+#            associated with the room
+#   action: Whether to "allow" or "deny" the request if the rule matches
+#
+# The default is:
+#
+#room_list_publication_rules:
+#  - user_id: "*"
+#    alias: "*"
+#    room_id: "*"
+#    action: allow
+
+
+# Server admins can define a Python module that implements extra rules for
+# allowing or denying incoming events. In order to work, this module needs to
+# override the methods defined in synapse/events/third_party_rules.py.
+#
+# This feature is designed to be used in closed federations only, where each
+# participating server enforces the same rules.
+#
+#third_party_event_rules:
+#  module: "my_custom_project.SuperRulesSet"
+#  config:
+#    example_option: 'things'
+
+
+## Opentracing ##
+
+# These settings enable opentracing, which implements distributed tracing.
+# This allows you to observe the causal chains of events across servers
+# including requests, key lookups etc., across any server running
+# synapse or any other other services which supports opentracing
+# (specifically those implemented with Jaeger).
+#
+opentracing:
+# tracing is disabled by default. Uncomment the following line to enable it.
+#
+#enabled: true
+
+    # The list of homeservers we wish to send and receive span contexts and span baggage.
+    # See docs/opentracing.rst
+    # This is a list of regexes which are matched against the server_name of the
+    # homeserver.
+    #
+    # By defult, it is empty, so no servers are matched.
+    #
+    #homeserver_whitelist:
+    #  - ".*"
+
+    # Jaeger can be configured to sample traces at different rates.
+    # All configuration options provided by Jaeger can be set here.
+    # Jaeger's configuration mostly related to trace sampling which
+    # is documented here:
+    # https://www.jaegertracing.io/docs/1.13/sampling/.
+    #
+    #jaeger_config:
+    #  sampler:
+    #    type: const
+    #    param: 1
+
+    #  Logging whether spans were started and reported
+    #
+    #  logging:
+    #    false
+    #
+EOF
diff --git a/cdist/conf/type/__matrix_synapse/files/log.config.sh b/cdist/conf/type/__matrix_synapse/files/log.config.sh
new file mode 100755
index 00000000..c156ba4c
--- /dev/null
+++ b/cdist/conf/type/__matrix_synapse/files/log.config.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+
+cat << EOF
+version: 1
+
+formatters:
+  precise:
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s'
+
+filters:
+  context:
+    (): synapse.util.logcontext.LoggingContextFilter
+    request: ""
+
+handlers:
+  file:
+    class: logging.handlers.RotatingFileHandler
+    formatter: precise
+    filename: $LOG_DIR/homeserver.log
+    maxBytes: 104857600
+    backupCount: 10
+    filters: [context]
+    level: INFO
+  console:
+    class: logging.StreamHandler
+    formatter: precise
+
+loggers:
+    synapse:
+        level: INFO
+
+    synapse.storage.SQL:
+        level: INFO
+
+root:
+    level: INFO
+    handlers: [file, console]
+EOF
diff --git a/cdist/conf/type/__matrix_synapse/man.rst b/cdist/conf/type/__matrix_synapse/man.rst
new file mode 100644
index 00000000..efda5675
--- /dev/null
+++ b/cdist/conf/type/__matrix_synapse/man.rst
@@ -0,0 +1,122 @@
+cdist-type__matrix_synapse(7)
+======================
+
+NAME
+----
+cdist-type__matrix_synapse - Install and configure Synapse, a Matrix homeserver
+
+
+DESCRIPTION
+-----------
+This type install and configure the Synapse Matrix homeserver. This is a
+signleton type.
+
+
+REQUIRED PARAMETERS
+-------------------
+server_name
+  Name of your homeserver (e.g. ungleich.ch) used as part of your MXIDs. This
+  value cannot be changed without meddling with the database once the server is
+  being used.
+
+base_url
+  Public URL of your homeserver (e.g. http://matrix.ungleich.ch).
+
+database_engine
+  'sqlite3' or 'postgresql'
+
+database_name
+  Path to the database if SQLite3 is used or database name if PostgresSQL is
+  used.
+
+OPTIONAL PARAMETERS
+-------------------
+database_host
+  Database node address, only used with PostgresSQL.
+
+database_user
+  Database user, only used with PostgresSQL.
+
+database_password
+  Database password, only used with PostgresSQL.
+
+ldap_uri
+  Address of your LDAP server.
+
+ldap_base_dn
+  Base DN of your LDAP tree.
+
+ldap_uid_attribute
+  LDAP attriute mapping to Synapse's uid field, default to uid.
+
+ldap_mail_attribute
+  LDAP attriute mapping to Synapse's mail field, default to mail.
+
+ldap_name_attribute
+  LDAP attriute mapping to Synapse's name field, default to givenName.
+
+ldap_bind_dn
+  User used to authenticate against your LDAP server in 'search' mode.
+
+ldap_bind_password
+  Password used to authenticate against your LDAP server in 'search' mode.
+
+ldap_filter
+  LDAP user filter, defaulting to `(objectClass=posixAccount)`.
+
+turn_uri
+  URI to TURN server, can be provided multiple times if there is more than one
+  server.
+
+turn_shared_secret
+  Shared secret used to access the TURN REST API.
+
+turn_user_lifetime
+  Lifetime of TURN credentials. Defaults to 1h.
+
+max_upload_size
+  Maximum size for user-uploaded files. Defaults to 10M.
+
+BOOLEAN PARAMETERS
+------------------
+allow_registration
+  Enables user registration on the homeserver.
+
+enable_ldap_auth
+  Enables ldap-backed authentication.
+
+ldap_search_mode
+  Enables 'search' mode for LDAP auth backend.
+
+report_stats
+  Whether or not to report anonymized homeserver usage statistics.
+
+expose_metrics
+  Expose metrics endpoint for Prometheus.
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __matrix_synapse --server_name ungleich.ch \
+      --base_url https://matrix.ungleich.ch \
+      --database_engine sqlite3 \
+      --database_name /var/lib/matrix-syanpse/homeserver.db
+
+SEE ALSO
+--------
+- `cdist-type__matrix_riot(7) <cdist-type__matrix_riot.html>`_
+
+
+AUTHORS
+-------
+Timothée Floure <timothee.floure@ungleich.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2019 Timothée Floure. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__matrix_synapse/manifest b/cdist/conf/type/__matrix_synapse/manifest
new file mode 100755
index 00000000..86c772c0
--- /dev/null
+++ b/cdist/conf/type/__matrix_synapse/manifest
@@ -0,0 +1,234 @@
+#!/bin/sh -e
+#
+# 2019 Timothée Floure (timothee.floure@ungleich.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# OS-specific configuration.
+os=$(cat "$__global/explorer/os")
+distribution=$(cat "$__global/explorer/lsb_codename")
+
+case "$os" in
+   debian)
+     synapse_user=matrix-synapse
+     synapse_pkg=matrix-synapse
+     synapse_service=matrix-synapse
+     ldap_auth_provider_pkg=matrix-synapse-ldap3
+     psycopg2_pkg=python3-psycopg2
+     synapse_conf_dir='/etc/matrix-synapse'
+     synapse_data_dir='/var/lib/matrix-synapse'
+
+     if [ ! -f "$__global/explorer/lsb_codename" ]; then
+       ls "$__global/explorer" >&2
+       echo "Could not determine Debian release, ensure that lsb-release is installed on the target." >&2
+       exit 1
+     fi
+     ;;
+   fedora)
+     synapse_user=synapse
+     synapse_pkg=matrix-synapse
+     synapse_service=synapse
+     ldap_auth_provider_pkg=python-matrix-synapse-ldap3
+     synapse_conf_dir='/etc/synapse'
+     synapse_data_dir='/var/lib/synapse'
+     ;;
+   freebsd)
+     synapse_user=synapse
+     synapse_pkg=py36-matrix-synapse
+     synapse_service=synapse
+     ldap_auth_provider_pkg=py36-matrix-synapse-ldap3
+     synapse_conf_dir='/usr/local/etc/matrix-synapse'
+     synapse_data_dir='/var/matrix-synapse'
+     ;;
+   alpine)
+     echo "As of 2019-12-19  matrix-synapse is not in alpine stable. Exiting."
+     exit 1
+     ;;
+   *)
+      printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2
+      printf "Please contribute an implementation for it if you can.\n" >&2
+      exit 1
+   ;;
+esac
+
+# Required parameters:
+server_name=$(cat "$__object/parameter/server_name")
+export SERVER_NAME=$server_name
+base_url=$(cat "$__object/parameter/base_url")
+export BASE_URL=$base_url
+
+export DATA_DIR=$synapse_data_dir
+export LOG_DIR='/var/log/matrix-synapse'
+export PIDFILE='/var/run/matrix/homeserver.pid'
+export LOG_CONFIG_PATH="$synapse_conf_dir/log.yaml"
+export SIGNING_KEY_PATH="$synapse_conf_dir/signin.key"
+
+database_engine=$(cat "$__object/parameter/database_engine")
+export DATABASE_ENGINE=$database_engine
+database_name=$(cat "$__object/parameter/database_name")
+export DATABASE_NAME=$database_name
+
+# Optional parameters:
+database_host=$(cat "$__object/parameter/database_host")
+export DATABASE_HOST=$database_host
+database_user=$(cat "$__object/parameter/database_user")
+export DATABASE_USER=$database_user
+database_password=$(cat "$__object/parameter/database_password")
+export DATABASE_PASSWORD=$database_password
+
+ldap_filter=$(cat "$__object/parameter/ldap_filter")
+export LDAP_FILTER=$ldap_filter
+ldap_uid_attribute=$(cat "$__object/parameter/ldap_uid_attribute")
+export LDAP_UID_ATTRIBUTE=$ldap_uid_attribute
+ldap_mail_attribute=$(cat "$__object/parameter/ldap_mail_attribute")
+export LDAP_MAIL_ATTRIBUTE=$ldap_mail_attribute
+ldap_name_attribute=$(cat "$__object/parameter/ldap_name_attribute")
+export LDAP_NAME_ATTRIBUTE=$ldap_name_attribute
+ldap_uri=$(cat "$__object/parameter/ldap_uri")
+export LDAP_URI=$ldap_uri
+ldap_base_dn=$(cat "$__object/parameter/ldap_base_dn")
+export LDAP_BASE_DN=$ldap_base_dn
+ldap_bind_dn=$(cat "$__object/parameter/ldap_bind_dn")
+export LDAP_BIND_DN=$ldap_bind_dn
+ldap_bind_password=$(cat "$__object/parameter/ldap_bind_password")
+export LDAP_BIND_PASSWORD=$ldap_bind_password
+
+turn_user_lifetime=$(cat "$__object/parameter/turn_user_lifetime")
+export TURN_USER_LIFETIME=$turn_user_lifetime
+if [ -f "$__object/parameter/turn_shared_secret" ]; then
+    turn_shared_secret=$(cat "$__object/parameter/turn_shared_secret")
+    export TURN_SHARED_SECRET=$turn_shared_secret
+fi
+if [ -f "$__object/parameter/turn_uri" ]; then
+    uris=$(tr "\n" "," < "$__object/parameter/turn_uri" | sed 's/,$//')
+    export TURN_URIS="[$uris]"
+fi
+
+max_upload_size=$(cat "$__object/parameter/max_upload_size")
+export MAX_UPLOAD_SIZE=$max_upload_size
+
+# Boolean parameters:
+if [ -f "$__object/parameter/report_stats" ]; then
+    export REPORT_STATS='true'
+else
+    export REPORT_STATS='false'
+fi
+if [ -f "$__object/parameter/allow_registration" ]; then
+    export ALLOW_REGISTRATION='true'
+else
+    export ALLOW_REGISTRATION='false'
+fi
+if [ -f "$__object/parameter/enable_ldap_auth" ]; then
+    export ENABLE_LDAP_AUTH='true'
+else
+    export ENABLE_LDAP_AUTH='false'
+fi
+if [ -f "$__object/parameter/ldap_search_mode" ]; then
+    export LDAP_SEARCH_MODE=1
+fi
+if [ -f "$__object/parameter/expose_metrics" ]; then
+    export EXPOSE_METRICS='true'
+else
+    export EXPOSE_METRICS='false'
+fi
+
+# Specific case for debian-buster, boilerplate but there's not much I can do
+# about it.
+
+installation_reqs=""
+if [ "$os" = "debian" ] && [ "$distribution" = "buster" ]; then
+  # Enable debian-backports for debian Buster, as the 'stable'
+  # matrix-synapse package is ways too old (< 1.0).
+  __apt_source debian-backports \
+    --uri http://deb.debian.org/debian/ \
+    --distribution "$distribution-backports" \
+    --component main
+   require="__apt_source/debian-backports" __apt_update_index
+
+  # Install base matrix-synapse package.
+  require="__apt_update_index" __package_apt $synapse_pkg \
+    --state present \
+    --target-release "$distribution-backports"
+
+  # Install LdapAuthProvider module if LDAP auth is enabled.
+  if [ "$ENABLE_LDAP_AUTH" = "true" ]; then
+    require="__package_apt/$synapse_pkg" __package_apt $ldap_auth_provider_pkg \
+      --state present \
+      --target-release "$distribution-backports"
+    installation_reqs="$installation_reqs __package_apt/$ldap_auth_provider_pkg"
+  fi
+
+  # For some reason, psycopg2 is not considered a dependency of
+  # matrix-synapse in matrix.org's APT repository.
+  if [ "$DATABASE_ENGINE" = "psycopg2" ]; then
+    require="__package_apt/$synapse_pkg" __package_apt $psycopg2_pkg \
+      --state present
+      installation_reqs="$installation_reqs __package_apt/$psycopg2_pkg"
+  fi
+
+  # Used for dependency order resolution.
+  installation_reqs="$installation_reqs __package_apt/$synapse_pkg"
+else
+  # Install base matrix-synapse package.
+  __package $synapse_pkg --state present
+
+  # Install LdapAuthProvider module if LDAP auth is enabled.
+  if [ "$ENABLE_LDAP_AUTH" = "true" ]; then
+    require="__package/$synapse_pkg" __package $ldap_auth_provider_pkg \
+      --state present
+  fi
+
+  # Used for dependency order resolution.
+  installation_reqs="__package/$synapse_pkg"
+fi
+
+# Generate and deploy configuration files.
+mkdir -p "$__object/files"
+"$__type/files/homeserver.yaml.sh" > "$__object/files/homeserver.yaml"
+"$__type/files/log.config.sh" > "$__object/files/log.config"
+
+require="$installation_reqs" __file "$synapse_conf_dir/homeserver.yaml" \
+  --state present \
+  --owner $synapse_user \
+  --mode 600 \
+  --source "$__object/files/homeserver.yaml"
+require="$installation_reqs" __file "$LOG_CONFIG_PATH" \
+  --state present \
+  --owner $synapse_user \
+  --mode 600 \
+  --source "$__object/files/log.config"
+require="$installation_reqs" __directory $DATA_DIR --state present --owner $synapse_user
+require="$installation_reqs" __directory $LOG_DIR --state present --owner $synapse_user
+
+# Work around dpkg-reconfigure for Debian package.
+RESTART_REQUIRES="__file/$synapse_conf_dir/homeserver.yaml"
+if [ "$os" = "debian" ]; then
+  require="$installation_reqs" __file "$synapse_conf_dir/conf.d/server_name.yaml" \
+    --state present --owner $synapse_user --source - << EOF
+server_name: "$SERVER_NAME"
+EOF
+  require="$installation_reqs" __file "$synapse_conf_dir/conf.d/report_stats.yaml" \
+    --state present --owner $synapse_user --source - << EOF
+report_stats: $REPORT_STATS
+EOF
+
+  RESTART_REQUIRES="$RESTART_REQUIRES __file/$synapse_conf_dir/conf.d/server_name.yaml \
+      __file/$synapse_conf_dir/conf.d/report_stats.yaml"
+fi
+
+# Restart synapse homeserver to reload configuration.
+require="$RESTART_REQUIRES" __service $synapse_service --action restart
diff --git a/cdist/conf/type/__matrix_synapse/parameter/boolean b/cdist/conf/type/__matrix_synapse/parameter/boolean
new file mode 100644
index 00000000..102fec29
--- /dev/null
+++ b/cdist/conf/type/__matrix_synapse/parameter/boolean
@@ -0,0 +1,5 @@
+allow_registration
+enable_ldap_auth
+ldap_search_mode
+report_stats
+expose_metrics
diff --git a/cdist/conf/type/__matrix_synapse/parameter/default/database_host b/cdist/conf/type/__matrix_synapse/parameter/default/database_host
new file mode 100644
index 00000000..e69de29b
diff --git a/cdist/conf/type/__matrix_synapse/parameter/default/database_password b/cdist/conf/type/__matrix_synapse/parameter/default/database_password
new file mode 100644
index 00000000..e69de29b
diff --git a/cdist/conf/type/__matrix_synapse/parameter/default/database_user b/cdist/conf/type/__matrix_synapse/parameter/default/database_user
new file mode 100644
index 00000000..e69de29b
diff --git a/cdist/conf/type/__matrix_synapse/parameter/default/ldap_base_dn b/cdist/conf/type/__matrix_synapse/parameter/default/ldap_base_dn
new file mode 100644
index 00000000..e69de29b
diff --git a/cdist/conf/type/__matrix_synapse/parameter/default/ldap_bind_dn b/cdist/conf/type/__matrix_synapse/parameter/default/ldap_bind_dn
new file mode 100644
index 00000000..e69de29b
diff --git a/cdist/conf/type/__matrix_synapse/parameter/default/ldap_bind_password b/cdist/conf/type/__matrix_synapse/parameter/default/ldap_bind_password
new file mode 100644
index 00000000..e69de29b
diff --git a/cdist/conf/type/__matrix_synapse/parameter/default/ldap_filter b/cdist/conf/type/__matrix_synapse/parameter/default/ldap_filter
new file mode 100644
index 00000000..f760ae9c
--- /dev/null
+++ b/cdist/conf/type/__matrix_synapse/parameter/default/ldap_filter
@@ -0,0 +1 @@
+(objectClass=posixAccount)
diff --git a/cdist/conf/type/__matrix_synapse/parameter/default/ldap_mail_attribute b/cdist/conf/type/__matrix_synapse/parameter/default/ldap_mail_attribute
new file mode 100644
index 00000000..fa7963cc
--- /dev/null
+++ b/cdist/conf/type/__matrix_synapse/parameter/default/ldap_mail_attribute
@@ -0,0 +1 @@
+mail
diff --git a/cdist/conf/type/__matrix_synapse/parameter/default/ldap_name_attribute b/cdist/conf/type/__matrix_synapse/parameter/default/ldap_name_attribute
new file mode 100644
index 00000000..4368fce0
--- /dev/null
+++ b/cdist/conf/type/__matrix_synapse/parameter/default/ldap_name_attribute
@@ -0,0 +1 @@
+givenName
diff --git a/cdist/conf/type/__matrix_synapse/parameter/default/ldap_uid_attribute b/cdist/conf/type/__matrix_synapse/parameter/default/ldap_uid_attribute
new file mode 100644
index 00000000..8cd23017
--- /dev/null
+++ b/cdist/conf/type/__matrix_synapse/parameter/default/ldap_uid_attribute
@@ -0,0 +1 @@
+uid
diff --git a/cdist/conf/type/__matrix_synapse/parameter/default/ldap_uri b/cdist/conf/type/__matrix_synapse/parameter/default/ldap_uri
new file mode 100644
index 00000000..e69de29b
diff --git a/cdist/conf/type/__matrix_synapse/parameter/default/max_upload_size b/cdist/conf/type/__matrix_synapse/parameter/default/max_upload_size
new file mode 100644
index 00000000..a1f4d0d1
--- /dev/null
+++ b/cdist/conf/type/__matrix_synapse/parameter/default/max_upload_size
@@ -0,0 +1 @@
+10M
diff --git a/cdist/conf/type/__matrix_synapse/parameter/default/turn_user_lifetime b/cdist/conf/type/__matrix_synapse/parameter/default/turn_user_lifetime
new file mode 100644
index 00000000..6f96dba0
--- /dev/null
+++ b/cdist/conf/type/__matrix_synapse/parameter/default/turn_user_lifetime
@@ -0,0 +1 @@
+1h
diff --git a/cdist/conf/type/__matrix_synapse/parameter/optional b/cdist/conf/type/__matrix_synapse/parameter/optional
new file mode 100644
index 00000000..74acc5b5
--- /dev/null
+++ b/cdist/conf/type/__matrix_synapse/parameter/optional
@@ -0,0 +1,14 @@
+database_host
+database_user
+database_password
+ldap_uri
+ldap_base_dn
+ldap_uid_attribute
+ldap_mail_attribute
+ldap_name_attribute
+ldap_bind_dn
+ldap_bind_password
+ldap_filter
+turn_shared_secret
+turn_user_lifetime
+max_upload_size
diff --git a/cdist/conf/type/__matrix_synapse/parameter/optional_multiple b/cdist/conf/type/__matrix_synapse/parameter/optional_multiple
new file mode 100644
index 00000000..b7f30d6f
--- /dev/null
+++ b/cdist/conf/type/__matrix_synapse/parameter/optional_multiple
@@ -0,0 +1 @@
+turn_uri
diff --git a/cdist/conf/type/__matrix_synapse/parameter/required b/cdist/conf/type/__matrix_synapse/parameter/required
new file mode 100644
index 00000000..de3e9a2c
--- /dev/null
+++ b/cdist/conf/type/__matrix_synapse/parameter/required
@@ -0,0 +1,4 @@
+server_name
+base_url
+database_engine
+database_name
diff --git a/cdist/conf/type/__matrix_synapse/singleton b/cdist/conf/type/__matrix_synapse/singleton
new file mode 100644
index 00000000..e69de29b

From b8266d5a28865a07f4592c21c9959afcfb1e2490 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 Feb 2020 10:31:31 +0100
Subject: [PATCH 010/527] Import __coturn type

---
 .../type/__coturn/files/turnserver.conf.sh    | 744 ++++++++++++++++++
 cdist/conf/type/__coturn/man.rst              |  61 ++
 cdist/conf/type/__coturn/manifest             |  55 ++
 cdist/conf/type/__coturn/parameter/boolean    |   2 +
 cdist/conf/type/__coturn/parameter/optional   |   2 +
 cdist/conf/type/__coturn/singleton            |   0
 6 files changed, 864 insertions(+)
 create mode 100755 cdist/conf/type/__coturn/files/turnserver.conf.sh
 create mode 100644 cdist/conf/type/__coturn/man.rst
 create mode 100755 cdist/conf/type/__coturn/manifest
 create mode 100644 cdist/conf/type/__coturn/parameter/boolean
 create mode 100644 cdist/conf/type/__coturn/parameter/optional
 create mode 100644 cdist/conf/type/__coturn/singleton

diff --git a/cdist/conf/type/__coturn/files/turnserver.conf.sh b/cdist/conf/type/__coturn/files/turnserver.conf.sh
new file mode 100755
index 00000000..0e2f2995
--- /dev/null
+++ b/cdist/conf/type/__coturn/files/turnserver.conf.sh
@@ -0,0 +1,744 @@
+#!/bin/sh
+
+generate_use_auth_secret () {
+  if [ $USE_AUTH_SECRET ]; then
+    echo 'use-auth-secret'
+  else
+    echo '#use-auth-secret'
+  fi
+}
+
+generate_static_auth_secret () {
+  if [ "$STATIC_AUTH_SECRET" $! "" ]; then
+    echo "static-auth-secret=$STATIC_AUTH_SECRET"
+  else
+    echo "#static-auth-secret=north"
+  fi
+}
+
+generate_realm () {
+  if [ "$REALM" != "" ]; then
+    echo "realm=$REALM"
+  else
+    echo "#realm=mycompany.org"
+  fi
+}
+
+generate_no_tcp_relay () {
+  if [ $NO_TCP_RELAY ]; then
+    echo 'no-tcp-releay'
+  else
+    echo '#no-tcp-relay'
+  fi
+}
+
+cat << EOF
+# Coturn TURN SERVER configuration file
+#
+# Boolean values note: where boolean value is supposed to be used,
+# you can use '0', 'off', 'no', 'false', 'f' as 'false,
+# and you can use '1', 'on', 'yes', 'true', 't' as 'true'
+# If the value is missed, then it means 'true'.
+#
+
+# Listener interface device (optional, Linux only).
+# NOT RECOMMENDED.
+#
+#listening-device=eth0
+
+# TURN listener port for UDP and TCP (Default: 3478).
+# Note: actually, TLS & DTLS sessions can connect to the
+# "plain" TCP & UDP port(s), too - if allowed by configuration.
+#
+#listening-port=3478
+
+# TURN listener port for TLS (Default: 5349).
+# Note: actually, "plain" TCP & UDP sessions can connect to the TLS & DTLS
+# port(s), too - if allowed by configuration. The TURN server
+# "automatically" recognizes the type of traffic. Actually, two listening
+# endpoints (the "plain" one and the "tls" one) are equivalent in terms of
+# functionality; but we keep both endpoints to satisfy the RFC 5766 specs.
+# For secure TCP connections, we currently support SSL version 3 and
+# TLS version 1.0, 1.1 and 1.2.
+# For secure UDP connections, we support DTLS version 1.
+#
+#tls-listening-port=5349
+
+# Alternative listening port for UDP and TCP listeners;
+# default (or zero) value means "listening port plus one".
+# This is needed for RFC 5780 support
+# (STUN extension specs, NAT behavior discovery). The TURN Server
+# supports RFC 5780 only if it is started with more than one
+# listening IP address of the same family (IPv4 or IPv6).
+# RFC 5780 is supported only by UDP protocol, other protocols
+# are listening to that endpoint only for "symmetry".
+#
+#alt-listening-port=0
+
+# Alternative listening port for TLS and DTLS protocols.
+# Default (or zero) value means "TLS listening port plus one".
+#
+#alt-tls-listening-port=0
+
+# Listener IP address of relay server. Multiple listeners can be specified.
+# If no IP(s) specified in the config file or in the command line options,
+# then all IPv4 and IPv6 system IPs will be used for listening.
+#
+#listening-ip=172.17.19.101
+#listening-ip=10.207.21.238
+#listening-ip=2607:f0d0:1002:51::4
+
+# Auxiliary STUN/TURN server listening endpoint.
+# Aux servers have almost full TURN and STUN functionality.
+# The (minor) limitations are:
+#
+# 1) Auxiliary servers do not have alternative ports and
+# they do not support STUN RFC 5780 functionality (CHANGE REQUEST).
+#
+# 2) Auxiliary servers also are never returning ALTERNATIVE-SERVER reply.
+#
+# Valid formats are 1.2.3.4:5555 for IPv4 and [1:2::3:4]:5555 for IPv6.
+#
+# There may be multiple aux-server options, each will be used for listening
+# to client requests.
+#
+#aux-server=172.17.19.110:33478
+#aux-server=[2607:f0d0:1002:51::4]:33478
+
+# (recommended for older Linuxes only)
+# Automatically balance UDP traffic over auxiliary servers (if configured).
+# The load balancing is using the ALTERNATE-SERVER mechanism.
+# The TURN client must support 300 ALTERNATE-SERVER response for this
+# functionality.
+#
+#udp-self-balance
+
+# Relay interface device for relay sockets (optional, Linux only).
+# NOT RECOMMENDED.
+#
+#relay-device=eth1
+
+# Relay address (the local IP address that will be used to relay the
+# packets to the peer).
+# Multiple relay addresses may be used.
+# The same IP(s) can be used as both listening IP(s) and relay IP(s).
+#
+# If no relay IP(s) specified, then the turnserver will apply the default
+# policy: it will decide itself which relay addresses to be used, and it
+# will always be using the client socket IP address as the relay IP address
+# of the TURN session (if the requested relay address family is the same
+# as the family of the client socket).
+#
+#relay-ip=172.17.19.105
+#relay-ip=2607:f0d0:1002:51::5
+
+# For Amazon EC2 users:
+#
+# TURN Server public/private address mapping, if the server is behind NAT.
+# In that situation, if a -X is used in form "-X <ip>" then that ip will be reported
+# as relay IP address of all allocations. This scenario works only in a simple case
+# when one single relay address is be used, and no RFC5780 functionality is required.
+# That single relay address must be mapped by NAT to the 'external' IP.
+# The "external-ip" value, if not empty, is returned in XOR-RELAYED-ADDRESS field.
+# For that 'external' IP, NAT must forward ports directly (relayed port 12345
+# must be always mapped to the same 'external' port 12345).
+#
+# In more complex case when more than one IP address is involved,
+# that option must be used several times, each entry must
+# have form "-X <public-ip/private-ip>", to map all involved addresses.
+# RFC5780 NAT discovery STUN functionality will work correctly,
+# if the addresses are mapped properly, even when the TURN server itself
+# is behind A NAT.
+#
+# By default, this value is empty, and no address mapping is used.
+#
+#external-ip=60.70.80.91
+#
+#OR:
+#
+#external-ip=60.70.80.91/172.17.19.101
+#external-ip=60.70.80.92/172.17.19.102
+
+
+# Number of the relay threads to handle the established connections
+# (in addition to authentication thread and the listener thread).
+# If explicitly set to 0 then application runs relay process in a
+# single thread, in the same thread with the listener process
+# (the authentication thread will still be a separate thread).
+#
+# If this parameter is not set, then the default OS-dependent
+# thread pattern algorithm will be employed. Usually the default
+# algorithm is the most optimal, so you have to change this option
+# only if you want to make some fine tweaks.
+#
+# In the older systems (Linux kernel before 3.9),
+# the number of UDP threads is always one thread per network listening
+# endpoint - including the auxiliary endpoints - unless 0 (zero) or
+# 1 (one) value is set.
+#
+#relay-threads=0
+
+# Lower and upper bounds of the UDP relay endpoints:
+# (default values are 49152 and 65535)
+#
+#min-port=49152
+#max-port=65535
+
+# Uncomment to run TURN server in 'normal' 'moderate' verbose mode.
+# By default the verbose mode is off.
+#verbose
+
+# Uncomment to run TURN server in 'extra' verbose mode.
+# This mode is very annoying and produces lots of output.
+# Not recommended under any normal circumstances.
+#
+#Verbose
+
+# Uncomment to use fingerprints in the TURN messages.
+# By default the fingerprints are off.
+#
+#fingerprint
+
+# Uncomment to use long-term credential mechanism.
+# By default no credentials mechanism is used (any user allowed).
+#
+#lt-cred-mech
+
+# This option is opposite to lt-cred-mech.
+# (TURN Server with no-auth option allows anonymous access).
+# If neither option is defined, and no users are defined,
+# then no-auth is default. If at least one user is defined,
+# in this file or in command line or in usersdb file, then
+# lt-cred-mech is default.
+#
+#no-auth
+
+# TURN REST API flag.
+# (Time Limited Long Term Credential)
+# Flag that sets a special authorization option that is based upon authentication secret.
+#
+# This feature's purpose is to support "TURN Server REST API", see
+# "TURN REST API" link in the project's page
+# https://github.com/coturn/coturn/
+#
+# This option is used with timestamp:
+#
+# usercombo -> "timestamp:userid"
+# turn user -> usercombo
+# turn password -> base64(hmac(secret key, usercombo))
+#
+# This allows TURN credentials to be accounted for a specific user id.
+# If you don't have a suitable id, the timestamp alone can be used.
+# This option is just turning on secret-based authentication.
+# The actual value of the secret is defined either by option static-auth-secret,
+# or can be found in the turn_secret table in the database (see below).
+#
+# Read more about it:
+#  - https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00
+#  - https://www.ietf.org/proceedings/87/slides/slides-87-behave-10.pdf
+#
+# Be aware that use-auth-secret overrides some part of lt-cred-mech.
+# Notice that this feature depends internally on lt-cred-mech, so if you set
+# use-auth-secret then it enables internally automatically lt-cred-mech option
+# like if you enable both.
+#
+# You can use only one of the to auth mechanisms in the same time because,
+# both mechanism use the username and password validation in different way.
+#
+# This way be aware that you can't use both auth mechnaism in the same time!
+# Use in config either the lt-cred-mech or the use-auth-secret
+# to avoid any confusion.
+#
+$(generate_use_auth_secret)
+
+# 'Static' authentication secret value (a string) for TURN REST API only.
+# If not set, then the turn server
+# will try to use the 'dynamic' value in turn_secret table
+# in user database (if present). The database-stored  value can be changed on-the-fly
+# by a separate program, so this is why that other mode is 'dynamic'.
+#
+$(generate_static_auth_secret)
+
+# Server name used for
+# the oAuth authentication purposes.
+# The default value is the realm name.
+#
+#server-name=blackdow.carleon.gov
+
+# Flag that allows oAuth authentication.
+#
+#oauth
+
+# 'Static' user accounts for long term credentials mechanism, only.
+# This option cannot be used with TURN REST API.
+# 'Static' user accounts are NOT dynamically checked by the turnserver process,
+# so that they can NOT be changed while the turnserver is running.
+#
+#user=username1:key1
+#user=username2:key2
+# OR:
+#user=username1:password1
+#user=username2:password2
+#
+# Keys must be generated by turnadmin utility. The key value depends
+# on user name, realm, and password:
+#
+# Example:
+# $ turnadmin -k -u ninefingers -r north.gov -p youhavetoberealistic
+# Output: 0xbc807ee29df3c9ffa736523fb2c4e8ee
+# ('0x' in the beginning of the key is what differentiates the key from
+# password. If it has 0x then it is a key, otherwise it is a password).
+#
+# The corresponding user account entry in the config file will be:
+#
+#user=ninefingers:0xbc807ee29df3c9ffa736523fb2c4e8ee
+# Or, equivalently, with open clear password (less secure):
+#user=ninefingers:youhavetoberealistic
+#
+
+# SQLite database file name.
+#
+# Default file name is /var/db/turndb or /usr/local/var/db/turndb or
+# /var/lib/turn/turndb.
+#
+#userdb=/var/db/turndb
+
+# PostgreSQL database connection string in the case that we are using PostgreSQL
+# as the user database.
+# This database can be used for long-term credential mechanism
+# and it can store the secret value for secret-based timed authentication in TURN RESP API.
+# See http://www.postgresql.org/docs/8.4/static/libpq-connect.html for 8.x PostgreSQL
+# versions connection string format, see
+# http://www.postgresql.org/docs/9.2/static/libpq-connect.html#LIBPQ-CONNSTRING
+# for 9.x and newer connection string formats.
+#
+#psql-userdb="host=<host> dbname=<database-name> user=<database-user> password=<database-user-password> connect_timeout=30"
+
+# MySQL database connection string in the case that we are using MySQL
+# as the user database.
+# This database can be used for long-term credential mechanism
+# and it can store the secret value for secret-based timed authentication in TURN RESP API.
+#
+# Optional connection string parameters for the secure communications (SSL):
+# ca, capath, cert, key, cipher
+# (see http://dev.mysql.com/doc/refman/5.1/en/ssl-options.html for the
+# command options description).
+#
+# Use string format as below (space separated parameters, all optional):
+#
+#mysql-userdb="host=<host> dbname=<database-name> user=<database-user> password=<database-user-password> port=<port> connect_timeout=<seconds> read_timeout=<seconds>"
+
+# If you want to use in the MySQL connection string the password in encrypted format,
+# then set in this option the MySQL password encryption secret key file.
+#
+# Warning: If this option is set, then mysql password must be set in "mysql-userdb" in encrypted format!
+# If you want to use cleartext password then do not set this option!
+#
+# This is the file path which contain secret key of aes encryption while using password encryption.
+#
+#secret-key-file=/path/
+
+# MongoDB database connection string in the case that we are using MongoDB
+# as the user database.
+# This database can be used for long-term credential mechanism
+# and it can store the secret value for secret-based timed authentication in TURN RESP API.
+# Use string format is described at http://hergert.me/docs/mongo-c-driver/mongoc_uri.html
+#
+#mongo-userdb="mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]"
+
+# Redis database connection string in the case that we are using Redis
+# as the user database.
+# This database can be used for long-term credential mechanism
+# and it can store the secret value for secret-based timed authentication in TURN RESP API.
+# Use string format as below (space separated parameters, all optional):
+#
+#redis-userdb="ip=<ip-address> dbname=<database-number> password=<database-user-password> port=<port> connect_timeout=<seconds>"
+
+# Redis status and statistics database connection string, if used (default - empty, no Redis stats DB used).
+# This database keeps allocations status information, and it can be also used for publishing
+# and delivering traffic and allocation event notifications.
+# The connection string has the same parameters as redis-userdb connection string.
+# Use string format as below (space separated parameters, all optional):
+#
+#redis-statsdb="ip=<ip-address> dbname=<database-number> password=<database-user-password> port=<port> connect_timeout=<seconds>"
+
+# The default realm to be used for the users when no explicit
+# origin/realm relationship was found in the database, or if the TURN
+# server is not using any database (just the commands-line settings
+# and the userdb file). Must be used with long-term credentials
+# mechanism or with TURN REST API.
+#
+# Note: If default realm is not specified at all, then realm falls back to the host domain name.
+#       If domain name is empty string, or '(None)', then it is initialized to am empty string.
+#
+$(generate_realm)
+
+# The flag that sets the origin consistency
+# check: across the session, all requests must have the same
+# main ORIGIN attribute value (if the ORIGIN was
+# initially used by the session).
+#
+#check-origin-consistency
+
+# Per-user allocation quota.
+# default value is 0 (no quota, unlimited number of sessions per user).
+# This option can also be set through the database, for a particular realm.
+#
+#user-quota=0
+
+# Total allocation quota.
+# default value is 0 (no quota).
+# This option can also be set through the database, for a particular realm.
+#
+#total-quota=0
+
+# Max bytes-per-second bandwidth a TURN session is allowed to handle
+# (input and output network streams are treated separately). Anything above
+# that limit will be dropped or temporary suppressed (within
+# the available buffer limits).
+# This option can also be set through the database, for a particular realm.
+#
+#max-bps=0
+
+#
+# Maximum server capacity.
+# Total bytes-per-second bandwidth the TURN server is allowed to allocate
+# for the sessions, combined (input and output network streams are treated separately).
+#
+# bps-capacity=0
+
+# Uncomment if no UDP client listener is desired.
+# By default UDP client listener is always started.
+#
+#no-udp
+
+# Uncomment if no TCP client listener is desired.
+# By default TCP client listener is always started.
+#
+#no-tcp
+
+# Uncomment if no TLS client listener is desired.
+# By default TLS client listener is always started.
+#
+#no-tls
+
+# Uncomment if no DTLS client listener is desired.
+# By default DTLS client listener is always started.
+#
+#no-dtls
+
+# Uncomment if no UDP relay endpoints are allowed.
+# By default UDP relay endpoints are enabled (like in RFC 5766).
+#
+#no-udp-relay
+
+# Uncomment if no TCP relay endpoints are allowed.
+# By default TCP relay endpoints are enabled (like in RFC 6062).
+#
+$(generate_no_tcp_relay)
+
+# Uncomment if extra security is desired,
+# with nonce value having limited lifetime.
+# By default, the nonce value is unique for a session,
+# and has unlimited lifetime.
+# Set this option to limit the nonce lifetime.
+# It defaults to 600 secs (10 min) if no value is provided. After that delay,
+# the client will get 438 error and will have to re-authenticate itself.
+#
+#stale-nonce=600
+
+# Uncomment if you want to set the maximum allocation
+# time before it has to be refreshed.
+# Default is 3600s.
+#
+#max-allocate-lifetime=3600
+
+
+# Uncomment to set the lifetime for the channel.
+# Default value is 600 secs (10 minutes).
+# This value MUST not be changed for production purposes.
+#
+#channel-lifetime=600
+
+# Uncomment to set the permission lifetime.
+# Default to 300 secs (5 minutes).
+# In production this value MUST not be changed,
+# however it can be useful for test purposes.
+#
+#permission-lifetime=300
+
+# Certificate file.
+# Use an absolute path or path relative to the
+# configuration file.
+#
+#cert=/usr/local/etc/turn_server_cert.pem
+
+# Private key file.
+# Use an absolute path or path relative to the
+# configuration file.
+# Use PEM file format.
+#
+#pkey=/usr/local/etc/turn_server_pkey.pem
+
+# Private key file password, if it is in encoded format.
+# This option has no default value.
+#
+#pkey-pwd=...
+
+# Allowed OpenSSL cipher list for TLS/DTLS connections.
+# Default value is "DEFAULT".
+#
+#cipher-list="DEFAULT"
+
+# CA file in OpenSSL format.
+# Forces TURN server to verify the client SSL certificates.
+# By default it is not set: there is no default value and the client
+# certificate is not checked.
+#
+# Example:
+#CA-file=/etc/ssh/id_rsa.cert
+
+# Curve name for EC ciphers, if supported by OpenSSL
+# library (TLS and DTLS). The default value is prime256v1,
+# if pre-OpenSSL 1.0.2 is used. With OpenSSL 1.0.2+,
+# an optimal curve will be automatically calculated, if not defined
+# by this option.
+#
+#ec-curve-name=prime256v1
+
+# Use 566 bits predefined DH TLS key. Default size of the key is 1066.
+#
+#dh566
+
+# Use 2066 bits predefined DH TLS key. Default size of the key is 1066.
+#
+#dh2066
+
+# Use custom DH TLS key, stored in PEM format in the file.
+# Flags --dh566 and --dh2066 are ignored when the DH key is taken from a file.
+#
+#dh-file=<DH-PEM-file-name>
+
+# Flag to prevent stdout log messages.
+# By default, all log messages are going to both stdout and to
+# the configured log file. With this option everything will be
+# going to the configured log only (unless the log file itself is stdout).
+#
+#no-stdout-log
+
+# Option to set the log file name.
+# By default, the turnserver tries to open a log file in
+# /var/log, /var/tmp, /tmp and current directories directories
+# (which open operation succeeds first that file will be used).
+# With this option you can set the definite log file name.
+# The special names are "stdout" and "-" - they will force everything
+# to the stdout. Also, the "syslog" name will force everything to
+# the system log (syslog).
+# In the runtime, the logfile can be reset with the SIGHUP signal
+# to the turnserver process.
+#
+#log-file=/var/tmp/turn.log
+
+# Option to redirect all log output into system log (syslog).
+#
+syslog
+
+# This flag means that no log file rollover will be used, and the log file
+# name will be constructed as-is, without PID and date appendage.
+# This option can be used, for example, together with the logrotate tool.
+#
+#simple-log
+
+# Option to set the "redirection" mode. The value of this option
+# will be the address of the alternate server for UDP & TCP service in form of
+# <ip>[:<port>]. The server will send this value in the attribute
+# ALTERNATE-SERVER, with error 300, on ALLOCATE request, to the client.
+# Client will receive only values with the same address family
+# as the client network endpoint address family.
+# See RFC 5389 and RFC 5766 for ALTERNATE-SERVER functionality description.
+# The client must use the obtained value for subsequent TURN communications.
+# If more than one --alternate-server options are provided, then the functionality
+# can be more accurately described as "load-balancing" than a mere "redirection".
+# If the port number is omitted, then the default port
+# number 3478 for the UDP/TCP protocols will be used.
+# Colon (:) characters in IPv6 addresses may conflict with the syntax of
+# the option. To alleviate this conflict, literal IPv6 addresses are enclosed
+# in square brackets in such resource identifiers, for example:
+# [2001:db8:85a3:8d3:1319:8a2e:370:7348]:3478 .
+# Multiple alternate servers can be set. They will be used in the
+# round-robin manner. All servers in the pool are considered of equal weight and
+# the load will be distributed equally. For example, if we have 4 alternate servers,
+# then each server will receive 25% of ALLOCATE requests. A alternate TURN server
+# address can be used more than one time with the alternate-server option, so this
+# can emulate "weighting" of the servers.
+#
+# Examples:
+#alternate-server=1.2.3.4:5678
+#alternate-server=11.22.33.44:56789
+#alternate-server=5.6.7.8
+#alternate-server=[2001:db8:85a3:8d3:1319:8a2e:370:7348]:3478
+
+# Option to set alternative server for TLS & DTLS services in form of
+# <ip>:<port>. If the port number is omitted, then the default port
+# number 5349 for the TLS/DTLS protocols will be used. See the previous
+# option for the functionality description.
+#
+# Examples:
+#tls-alternate-server=1.2.3.4:5678
+#tls-alternate-server=11.22.33.44:56789
+#tls-alternate-server=[2001:db8:85a3:8d3:1319:8a2e:370:7348]:3478
+
+# Option to suppress TURN functionality, only STUN requests will be processed.
+# Run as STUN server only, all TURN requests will be ignored.
+# By default, this option is NOT set.
+#
+#stun-only
+
+# Option to suppress STUN functionality, only TURN requests will be processed.
+# Run as TURN server only, all STUN requests will be ignored.
+# By default, this option is NOT set.
+#
+#no-stun
+
+# This is the timestamp/username separator symbol (character) in TURN REST API.
+# The default value is ':'.
+# rest-api-separator=:
+
+# Flag that can be used to allow peers on the loopback addresses (127.x.x.x and ::1).
+# This is an extra security measure.
+#
+# (To avoid any security issue that allowing loopback access may raise,
+# the no-loopback-peers option is replaced by allow-loopback-peers.)
+#
+# Allow it only for testing in a development environment!
+# In production it adds a possible security vulnerability, so for security reasons
+# it is not allowed using it together with empty cli-password.
+#
+#allow-loopback-peers
+
+# Flag that can be used to disallow peers on well-known broadcast addresses (224.0.0.0 and above, and FFXX:*).
+# This is an extra security measure.
+#
+#no-multicast-peers
+
+# Option to set the max time, in seconds, allowed for full allocation establishment.
+# Default is 60 seconds.
+#
+#max-allocate-timeout=60
+
+# Option to allow or ban specific ip addresses or ranges of ip addresses.
+# If an ip address is specified as both allowed and denied, then the ip address is
+# considered to be allowed. This is useful when you wish to ban a range of ip
+# addresses, except for a few specific ips within that range.
+#
+# This can be used when you do not want users of the turn server to be able to access
+# machines reachable by the turn server, but would otherwise be unreachable from the
+# internet (e.g. when the turn server is sitting behind a NAT)
+#
+# Examples:
+# denied-peer-ip=83.166.64.0-83.166.95.255
+# allowed-peer-ip=83.166.68.45
+
+# File name to store the pid of the process.
+# Default is /var/run/turnserver.pid (if superuser account is used) or
+# /var/tmp/turnserver.pid .
+#
+#pidfile="/var/run/turnserver.pid"
+
+# Require authentication of the STUN Binding request.
+# By default, the clients are allowed anonymous access to the STUN Binding functionality.
+#
+#secure-stun
+
+# Mobility with ICE (MICE) specs support.
+#
+#mobility
+
+# Allocate Address Family according
+# If enabled then TURN server allocates address family according  the TURN
+# Client <=> Server communication address family.
+# (By default coTURN works according RFC 6156.)
+# !!Warning: Enabling this option breaks RFC6156 section-4.2 (violates use default IPv4)!!
+#
+#keep-address-family
+
+
+# User name to run the process. After the initialization, the turnserver process
+# will make an attempt to change the current user ID to that user.
+#
+#proc-user=<user-name>
+
+# Group name to run the process. After the initialization, the turnserver process
+# will make an attempt to change the current group ID to that group.
+#
+#proc-group=<group-name>
+
+# Turn OFF the CLI support.
+# By default it is always ON.
+# See also options cli-ip and cli-port.
+#
+#no-cli
+
+#Local system IP address to be used for CLI server endpoint. Default value
+# is 127.0.0.1.
+#
+#cli-ip=127.0.0.1
+
+# CLI server port. Default is 5766.
+#
+#cli-port=5766
+
+# CLI access password. Default is empty (no password).
+# For the security reasons, it is recommended to use the encrypted
+# for of the password (see the -P command in the turnadmin utility).
+#
+# Secure form for password 'qwerty':
+#
+#cli-password=$5$79a316b350311570$81df9cfb9af7f5e5a76eada31e7097b663a0670f99a3c07ded3f1c8e59c5658a
+#
+# Or unsecure form for the same password:
+#
+#cli-password=qwerty
+
+# Enable Web-admin support on https. By default it is Disabled.
+# If it is enabled it also enables a http a simple static banner page
+# with a small reminder that the admin page is available only on https.
+#
+#web-admin
+
+# Local system IP address to be used for Web-admin server endpoint. Default value is 127.0.0.1.
+#
+#web-admin-ip=127.0.0.1
+
+# Web-admin server port. Default is 8080.
+#
+#web-admin-port=8080
+
+# Web-admin server listen on STUN/TURN worker threads
+# By default it is disabled for security resons! (Not recommended in any production environment!)
+#
+#web-admin-listen-on-workers
+
+# Server relay. NON-STANDARD AND DANGEROUS OPTION.
+# Only for those applications when we want to run
+# server applications on the relay endpoints.
+# This option eliminates the IP permissions check on
+# the packets incoming to the relay endpoints.
+#
+#server-relay
+
+# Maximum number of output sessions in ps CLI command.
+# This value can be changed on-the-fly in CLI. The default value is 256.
+#
+#cli-max-output-sessions
+
+# Set network engine type for the process (for internal purposes).
+#
+#ne=[1|2|3]
+
+# Do not allow an TLS/DTLS version of protocol
+#
+#no-tlsv1
+#no-tlsv1_1
+#no-tlsv1_2
+EOF
diff --git a/cdist/conf/type/__coturn/man.rst b/cdist/conf/type/__coturn/man.rst
new file mode 100644
index 00000000..219da4ea
--- /dev/null
+++ b/cdist/conf/type/__coturn/man.rst
@@ -0,0 +1,61 @@
+cdist-type__coturn(7)
+=====================
+
+NAME
+----
+cdist-type__coturn - Install and configure a coturn TURN server
+
+
+DESCRIPTION
+-----------
+This (singleton) type install and configure a coturn TURN
+server.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+static_auth_secret
+  Secret used to access the TURN REST API.
+
+realm
+  Defailt realm.
+
+BOOLEAN PARAMETERS
+------------------
+use_auth_secret
+  Allows TURN credentials to be accounted for a specific user id.
+
+no_tcp_relay
+  Disable TCP relay endpoints.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __coturn \
+      --realm turn.domain.tld \
+      --no_tcp_relay
+
+
+SEE ALSO
+--------
+- `coturn Github repository <https://github.com/coturn/coturn>`_
+
+AUTHORS
+-------
+Timothée Floure <timothee.floure@ungleich.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Timothée Floure. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__coturn/manifest b/cdist/conf/type/__coturn/manifest
new file mode 100755
index 00000000..915f587d
--- /dev/null
+++ b/cdist/conf/type/__coturn/manifest
@@ -0,0 +1,55 @@
+#!/bin/sh -e
+#
+# 2020 Timothée Floure (timothee.floure@ungleich.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+__package coturn
+
+# Optional parameters.
+if [ -f "$__object/parameter/use_auth_secret" ]; then
+    export USE_AUTH_SECRET=1
+fi
+
+if [ -f "$__object/parameter/static_auth_secret" ]; then
+    static_auth_secret=$(cat "$__object/parameter/static_auth_secret")
+    export STATIC_AUTH_SECRET=$static_auth_secret
+fi
+
+if [ -f "$__object/parameter/realm" ]; then
+    realm=$(cat "$__object/parameter/realm")
+    export REALM=$realm
+fi
+
+if [ -f "$__object/parameter/no_tcp_relay" ]; then
+    no_tcp_relay=$(cat "$__object/parameter/no_tcp_relay")
+    export NO_TCP_RELAY=$no_tcp_relay
+fi
+
+# Hardcoded.
+coturn_config='/etc/turnserver.conf'
+
+# Generate and deploy configuration file.
+mkdir -p "$__object/files"
+"$__type/files/turnserver.conf.sh" > "$__object/files/turnserver.conf"
+
+__file $coturn_config \
+  --source "$__object/files/turnserver.conf" \
+  --state present
+
+# Restart coturn server.
+require="__file/$coturn_config" __service coturn --action restart
diff --git a/cdist/conf/type/__coturn/parameter/boolean b/cdist/conf/type/__coturn/parameter/boolean
new file mode 100644
index 00000000..45f9e704
--- /dev/null
+++ b/cdist/conf/type/__coturn/parameter/boolean
@@ -0,0 +1,2 @@
+use_auth_secret
+no_tcp_relay
diff --git a/cdist/conf/type/__coturn/parameter/optional b/cdist/conf/type/__coturn/parameter/optional
new file mode 100644
index 00000000..33218485
--- /dev/null
+++ b/cdist/conf/type/__coturn/parameter/optional
@@ -0,0 +1,2 @@
+static_auth_secret
+realm
diff --git a/cdist/conf/type/__coturn/singleton b/cdist/conf/type/__coturn/singleton
new file mode 100644
index 00000000..e69de29b

From 07df829bd0239b6e71672597656a5f80c7925d85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 Feb 2020 10:31:52 +0100
Subject: [PATCH 011/527] Import __matterbridge type

---
 .../files/matterbridge.service.sh             |  18 ++++
 cdist/conf/type/__matterbridge/man.rst        |  56 ++++++++++
 cdist/conf/type/__matterbridge/manifest       | 100 ++++++++++++++++++
 .../type/__matterbridge/parameter/required    |   2 +
 cdist/conf/type/__matterbridge/singleton      |   0
 5 files changed, 176 insertions(+)
 create mode 100755 cdist/conf/type/__matterbridge/files/matterbridge.service.sh
 create mode 100644 cdist/conf/type/__matterbridge/man.rst
 create mode 100755 cdist/conf/type/__matterbridge/manifest
 create mode 100644 cdist/conf/type/__matterbridge/parameter/required
 create mode 100644 cdist/conf/type/__matterbridge/singleton

diff --git a/cdist/conf/type/__matterbridge/files/matterbridge.service.sh b/cdist/conf/type/__matterbridge/files/matterbridge.service.sh
new file mode 100755
index 00000000..9dbd1cb6
--- /dev/null
+++ b/cdist/conf/type/__matterbridge/files/matterbridge.service.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+cat <<EOF
+[Unit]
+Description=IM bridging daemon
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+User=$USER
+Group=$GROUP
+Type=simple
+Restart=on-failure
+ExecStart=$BINARY_PATH -conf=/etc/matterbridge/matterbridge.toml
+
+[Install]
+WantedBy=multi-user.target
+EOF
diff --git a/cdist/conf/type/__matterbridge/man.rst b/cdist/conf/type/__matterbridge/man.rst
new file mode 100644
index 00000000..02eef2c3
--- /dev/null
+++ b/cdist/conf/type/__matterbridge/man.rst
@@ -0,0 +1,56 @@
+cdist-type__matterbridge(7)
+===========================
+
+NAME
+----
+cdist-type__matterbridge - Install matterbridge from upstream binary
+
+
+DESCRIPTION
+-----------
+This singleton type install a matterbridge service from binary.
+
+REQUIRED PARAMETERS
+-------------------
+version
+  Release (git tag) to fetch from the project github's page.
+
+config
+  Matterbridge configuration (TOML).
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __matterbridge --version 1.16.3 --config - << EOF
+    [...]
+EOF
+
+
+SEE ALSO
+--------
+- `Matterbridge github repository <https://github.com/42wim/matterbridge>`_
+
+
+AUTHORS
+-------
+Timothée Floure <timothee.floure@ungleich.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Timothée Floure. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__matterbridge/manifest b/cdist/conf/type/__matterbridge/manifest
new file mode 100755
index 00000000..35574661
--- /dev/null
+++ b/cdist/conf/type/__matterbridge/manifest
@@ -0,0 +1,100 @@
+#!/bin/sh -e
+#
+# 2020 Timothée Floure (timothee.floure@ungleich.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$(cat "$__global/explorer/os")
+case "$os" in
+   debian)
+     # This type assume systemd for service installation.
+   ;;
+   *)
+      printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2
+      printf "Please contribute an implementation for it if you can.\n" >&2
+      exit 1
+   ;;
+esac
+
+# Required parameters.
+VERSION=$(cat "$__object/parameter/version")
+if [ -f "$__object/parameter/config" ]; then
+    CONFIG="$(cat "$__object/parameter/config")"
+    if [ "$CONFIG" = "-" ]; then
+        CONFIG=$(cat "$__object/stdin")
+    fi
+fi
+
+# Hardcoded values used in templates.
+export BINARY_PATH=/usr/local/bin/matterbridge
+export CONFIG_PATH=/etc/matterbridge/matterbridge.toml
+export USER=matterbridge
+export GROUP=$USER
+
+# Internal variables.
+artefact="matterbridge-$VERSION-linux-64bit"
+checksum_file="checksums.txt"
+release_download_url=https://github.com/42wim/matterbridge/releases/download
+binary_url="$release_download_url/v$VERSION/$artefact"
+checksum_file_url="$release_download_url/v$VERSION/$checksum_file"
+config_dir=$(dirname $CONFIG_PATH)
+systemd_unit_path='/etc/systemd/system/matterbridge.service'
+
+# Check if curl is available.
+if ! command -v curl > /dev/null; then
+     echo "curl is required for this type, but could not be found. Exiting." >&2
+     exit 1
+fi
+
+# Initialize working directory.
+mkdir -p "$__object/files"
+
+# Download and check matterbridge binary.
+curl -L "$binary_url" -o "$__object/files/$artefact"
+curl -L "$checksum_file_url" -o "$__object/files/$checksum_file"
+ls "$__object/files/" >&2
+cat "$__object/files/checksums.txt" >&2
+if ! (cd "$__object/files"; sha256sum --ignore-missing --check $checksum_file); then
+    echo "Matterbridge binary checksum failed." >&2
+    exit 1
+fi
+
+# Create service user.
+__user $USER --home "/var/lib/$USER"
+
+# Deploy matterbridge binary.
+require="__user/$USER" __file "$BINARY_PATH" \
+  --source "$__object/files/$artefact" \
+  --owner "$USER" --mode 755
+
+# Generate and deploy configuration file.
+"$__type/files/matterbridge.service.sh" > "$__object/files/matterbridge.service"
+
+require="__user/$USER" __directory "$config_dir" \
+  --owner "$USER" --mode 0755 --parents \
+
+require="__directory/$config_dir" echo "$CONFIG" | __file "$CONFIG_PATH" \
+  --owner "$USER" \
+  --mode 0640 \
+  --source -
+
+__file "$systemd_unit_path" \
+  --source "$__object/files/matterbridge.service"
+
+# Deal with init system.
+require="__file/$systemd_unit_path" __start_on_boot matterbridge
+require="__file/$BINARY_PATH __file/$CONFIG_PATH __file/$systemd_unit_path" __service matterbridge --action restart
diff --git a/cdist/conf/type/__matterbridge/parameter/required b/cdist/conf/type/__matterbridge/parameter/required
new file mode 100644
index 00000000..ed5d8b33
--- /dev/null
+++ b/cdist/conf/type/__matterbridge/parameter/required
@@ -0,0 +1,2 @@
+version
+config
diff --git a/cdist/conf/type/__matterbridge/singleton b/cdist/conf/type/__matterbridge/singleton
new file mode 100644
index 00000000..e69de29b

From 6b4b6534a1187e2533f627ff485873aeac7c4a32 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 26 Feb 2020 16:12:21 +0100
Subject: [PATCH 012/527] [__directory] Give more precise error message when
 --state pre-exists

---
 cdist/conf/type/__directory/gencode-remote | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote
index e1ab69d7..a1a32ea2 100755
--- a/cdist/conf/type/__directory/gencode-remote
+++ b/cdist/conf/type/__directory/gencode-remote
@@ -109,10 +109,24 @@ case "$state_should" in
       done
    ;;
    pre-exists)
-      if [ "$type" != "directory" ]; then
-         echo "Directory \"$destination\" does not exist" >&2
-         exit 1
-      fi
+      case $type in
+         directory)
+            # all good
+            exit 0
+         ;;
+         none)
+            printf 'Directory "%s" does not exist\n' "$destination" >&2
+            exit 1
+         ;;
+         file|symlink)
+            printf 'File "%s" exists and is a %s, but should be a directory\n' "$destination" "$type" >&2
+            exit 1
+         ;;
+         *)
+            printf 'File or directory "%s" is in an unknown state\n' "$destination" >&2
+            exit 1
+         ;;
+      esac
    ;;
    absent)
         if [ "$type" = "directory" ]; then

From 046f7d0663fb6e117f76747bf71c7976abc90962 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 26 Feb 2020 16:32:03 +0100
Subject: [PATCH 013/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 35590655..4fdfcd68 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* New type: __service (Timothée Floure)
 	* Types __consul_*: Add optional parameter for using distribution packages (Timothée Floure)
 	* Explorer disks: Fix NetBSD, support Linux w/o lsblk (Dennis Camera)
+	* Type __directory: Add 'exists' and 'pre-exists' states (Dennis Camera)
 
 6.5.1: 2020-02-15
 	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)

From 46d2487f08e04d3f6f3cbd434404dda006e83276 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 26 Feb 2020 16:40:53 +0100
Subject: [PATCH 014/527] [__file] Give more precise error message when --state
 pre-exists

---
 cdist/conf/type/__file/gencode-local | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local
index fb9f9a92..231b6927 100755
--- a/cdist/conf/type/__file/gencode-local
+++ b/cdist/conf/type/__file/gencode-local
@@ -31,12 +31,24 @@ if [ "$state_should" = "pre-exists" ]; then
       exit 1
    fi
 
-   if [ "$type" = "file" ]; then
-      exit 0 # nothing to do
-   else
-      echo "File \"$destination\" does not exist"
-      exit 1
-   fi
+   case $type in
+      file)
+         # nothing to do
+         exit 0
+      ;;
+      none)
+         printf 'File "%s" does not exist\n' "$destination" >&2
+         exit 1
+      ;;
+      directory|symlink)
+         printf 'File "%s" exists and is a %s, but should be a regular file\n' "$destination" "$type" >&2
+         exit 1
+      ;;
+      *)
+         printf 'File or directory "%s" is in an unknown state\n' "$destination" >&2
+         exit 1
+      ;;
+   esac
 fi
 
 upload_file=

From 9eacba06bb218bd0627ef633b363495331ac6c5b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 26 Feb 2020 16:48:14 +0100
Subject: [PATCH 015/527] [__file] Treat pre-exists as a special case in
 gencode-remote as it should never get there

---
 cdist/conf/type/__file/gencode-remote | 56 ++++++++++++++-------------
 1 file changed, 30 insertions(+), 26 deletions(-)

diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index b04c471e..815593bd 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -55,37 +55,41 @@ set_owner() {
 }
 
 set_mode() {
-   echo "chmod '$1' '$destination'"
-   echo "chmod '$1'" >> "$__messages_out"
-   fire_onchange=1
+    echo "chmod '$1' '$destination'"
+    echo "chmod '$1'" >> "$__messages_out"
+    fire_onchange=1
 }
 
 case "$state_should" in
-    present|exists|pre-exists)
-    # Note: Mode - needs to happen last as a chown/chgrp can alter mode by
-    #  clearing S_ISUID and S_ISGID bits (see chown(2))
-    for attribute in group owner mode; do
-        if [ -f "$__object/parameter/$attribute" ]; then
-            value_should="$(cat "$__object/parameter/$attribute")"
+    present|exists)
+        # Note: Mode - needs to happen last as a chown/chgrp can alter mode by
+        #  clearing S_ISUID and S_ISGID bits (see chown(2))
+        for attribute in group owner mode; do
+            if [ -f "$__object/parameter/$attribute" ]; then
+                value_should="$(cat "$__object/parameter/$attribute")"
 
-            # change 0xxx format to xxx format => same as stat returns
-            if [ "$attribute" = mode ]; then
-                value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')"
-            fi
-            
-            value_is="$(get_current_value "$attribute" "$value_should")"
-            if [ -f "$__object/files/set-attributes" ] || [ "$value_should" != "$value_is" ]; then
-                "set_$attribute" "$value_should"
+                # change 0xxx format to xxx format => same as stat returns
+                if [ "$attribute" = mode ]; then
+                    value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')"
+                fi
+
+                value_is="$(get_current_value "$attribute" "$value_should")"
+                if [ -f "$__object/files/set-attributes" ] || [ "$value_should" != "$value_is" ]; then
+                    "set_$attribute" "$value_should"
+                fi
             fi
+        done
+        if [ -f "$__object/files/set-attributes" ]; then
+            # set-attributes is created if file is created or uploaded in gencode-local
+            fire_onchange=1
         fi
-    done
-    if [ -f "$__object/files/set-attributes" ]; then
-        # set-attributes is created if file is created or uploaded in gencode-local
-        fire_onchange=1
-    fi
-
     ;;
 
+    pre-exists)
+        # pre-exists should never reach gencode-remote…
+        exit 1
+   ;;
+
     absent)
         if [ "$type" = "file" ]; then
             echo "rm -f '$destination'"
@@ -101,7 +105,7 @@ case "$state_should" in
 esac
 
 if [ -f "$__object/parameter/onchange" ]; then
-   if [ -n "$fire_onchange" ]; then
-      cat "$__object/parameter/onchange"
-   fi
+    if [ -n "$fire_onchange" ]; then
+        cat "$__object/parameter/onchange"
+    fi
 fi

From da6ccf808ef4417b90e0882abd042f0a40f28a8e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 26 Feb 2020 21:48:08 +0100
Subject: [PATCH 016/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 4fdfcd68..9075cf3f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,7 @@ next:
 	* Types __consul_*: Add optional parameter for using distribution packages (Timothée Floure)
 	* Explorer disks: Fix NetBSD, support Linux w/o lsblk (Dennis Camera)
 	* Type __directory: Add 'exists' and 'pre-exists' states (Dennis Camera)
+	* Type __file: Improve error messages for pre-exists state (Dennis Camera)
 
 6.5.1: 2020-02-15
 	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)

From 213f1b049c55205ffe233de22e68c4a4738c8b82 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 27 Feb 2020 20:23:04 +0100
Subject: [PATCH 017/527] Release 6.5.2

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 9075cf3f..64b124e8 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.5.2: 2020-02-27
 	* Type __update_alternatives: Add state explorer (Ander Punnar)
 	* Explorer os_version: Add support for Alpine Linux (Jin-Guk Kwon)
 	* Explorer init: Rewrite and support more init systems (Dennis Camera)

From fb32d6ed3f9421d515b0ec2eabe9ab4fcc6ccacb Mon Sep 17 00:00:00 2001
From: llnu <llnu@protonmail.ch>
Date: Sun, 8 Mar 2020 16:04:02 +0100
Subject: [PATCH 018/527] alpine uses a different getent lib which doesnt
 support: getent shadow

---
 cdist/conf/type/__user/explorer/shadow | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__user/explorer/shadow b/cdist/conf/type/__user/explorer/shadow
index 73ce0e29..63d38f0d 100755
--- a/cdist/conf/type/__user/explorer/shadow
+++ b/cdist/conf/type/__user/explorer/shadow
@@ -24,7 +24,7 @@
 name=$__object_id
 
 case $("$__explorer/os") in
-  'freebsd'|'netbsd'|'openbsd')
+  'freebsd'|'netbsd'|'openbsd'|'alpine')
     database='passwd'
     ;;
   # Default to using shadow passwords

From f7d5f5bc974fff858c9999752badce3c6e8ba72e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 9 Mar 2020 08:02:18 +0100
Subject: [PATCH 019/527] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 64b124e8..93df32a2 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __user: Fix missing shadow for alpine (llnu)
+
 6.5.2: 2020-02-27
 	* Type __update_alternatives: Add state explorer (Ander Punnar)
 	* Explorer os_version: Add support for Alpine Linux (Jin-Guk Kwon)

From 358e04b2afa380b63843869f1f57967e0ef8de22 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 14 Mar 2020 09:58:38 +0100
Subject: [PATCH 020/527] Handle specially if no time about the last index
 update found.

The explorer 'currage' now returns -1 if he can not find any value about
this. The gencode-remote script handle this value special to not exit if
-1 given as value.

This fixes the bug
  https://code.ungleich.ch/ungleich-public/cdist/issues/803
---
 cdist/conf/type/__package_update_index/explorer/currage | 6 +++---
 cdist/conf/type/__package_update_index/gencode-remote   | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__package_update_index/explorer/currage b/cdist/conf/type/__package_update_index/explorer/currage
index cfb778d5..bd51ee86 100644
--- a/cdist/conf/type/__package_update_index/explorer/currage
+++ b/cdist/conf/type/__package_update_index/explorer/currage
@@ -24,18 +24,18 @@ case "$type" in
         if [ -f "/var/cache/apt/pkgcache.bin" ]; then
             echo $(($(date +"%s")-$(stat --format '%Y' /var/cache/apt/pkgcache.bin)))
         else
-            echo 0
+            echo -1
         fi
         ;;
     pacman)
         if [ -d "/var/lib/pacman/sync" ]; then
             echo $(($(date +"%s")-$(stat --format '%Y' /var/lib/pacman/sync)))
         else
-            echo 0
+            echo -1
         fi
         ;;
     alpine)
-        echo 0
+        echo -1
         ;;
     *)  echo "Your specified type ($type) is currently not supported." >&2
         echo "Please contribute an implementation for it if you can." >&2
diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote
index 6c51cbed..803468b5 100755
--- a/cdist/conf/type/__package_update_index/gencode-remote
+++ b/cdist/conf/type/__package_update_index/gencode-remote
@@ -31,7 +31,8 @@ if [ -n "$maxage" ]; then
     if [ "$type" != "apt" ] && [ "$type" != "pacman" ]; then
         echo "ERROR: \"--maxage\" only supported for \"apt\" or \"pacman\" pkg-manager." >&2
         exit 1
-    elif [ "$currage" -lt "$maxage" ]; then
+    # do not exit if no value found (represented as -1)
+    elif [ "$currage" -ne -1 ] && [ "$currage" -lt "$maxage" ]; then
         exit 0 # no need to update
     fi
 fi

From f00e4af5f0f1f49ab93be001468327ea90df6bd0 Mon Sep 17 00:00:00 2001
From: Andrew Schleifer <me@andrewschleifer.name>
Date: Thu, 26 Mar 2020 21:17:32 +0800
Subject: [PATCH 021/527] fix typo

---
 cdist/conf/type/__letsencrypt_cert/man.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__letsencrypt_cert/man.rst b/cdist/conf/type/__letsencrypt_cert/man.rst
index c4ffc6bc..85eb88ea 100644
--- a/cdist/conf/type/__letsencrypt_cert/man.rst
+++ b/cdist/conf/type/__letsencrypt_cert/man.rst
@@ -59,13 +59,13 @@ MESSAGES
 --------
 
 change
-    Certificte was changed.
+    Certificate was changed.
 
 create
-    Certificte was created.
+    Certificate was created.
 
 remove
-    Certificte was removed.
+    Certificate was removed.
 
 EXAMPLES
 --------

From 66d990502987aae97695a4b8678e7d160f1066f7 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 26 Mar 2020 21:48:17 +0100
Subject: [PATCH 022/527] [__consul_agent] make conf_dir depent on the OS

---
 cdist/conf/type/__consul_agent/manifest | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest
index 0d819d45..a7fe0bc5 100755
--- a/cdist/conf/type/__consul_agent/manifest
+++ b/cdist/conf/type/__consul_agent/manifest
@@ -1,7 +1,7 @@
 #!/bin/sh -e
 #
 # 2015 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2015-2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2015-2020 Nico Schottelius (nico-cdist at schottelius.org)
 # 2019 Timothée Floure (timothee.floure at ungleich.ch)
 #
 # This file is part of cdist.
@@ -37,10 +37,22 @@ fi
 # Those are default that might be overriden by os-specific logic.
 
 data_dir="/var/lib/consul"
-conf_dir="/etc/consul/conf.d"
-conf_file="config.json"
+
+
+
 tls_dir="$conf_dir/tls"
 
+case "$os" in
+    alpine)
+        conf_dir="/etc/consul"
+        conf_file="server.json"
+        ;;
+    *)
+        conf_dir="/etc/consul/conf.d"
+        conf_file="config.json"
+        ;;
+esac
+
 ###
 # Sane deployment, based on distribution package when available.
 

From ec11f04ab83504035271483a61549a4e0c51053c Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 26 Mar 2020 21:49:43 +0100
Subject: [PATCH 023/527] ++changes

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 93df32a2..76f47679 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,8 @@ Changelog
 
 next:
 	* Type __user: Fix missing shadow for alpine (llnu)
+	* Type __consule_agent: Make conf_dir dependent on OS - fixes
+	Alpine (Nico Schottelius)
 
 6.5.2: 2020-02-27
 	* Type __update_alternatives: Add state explorer (Ander Punnar)

From df63cfe0884d460942a34b31d9296a31aa8edb2f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 27 Mar 2020 08:50:27 +0100
Subject: [PATCH 024/527] ++changelog

---
 docs/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/changelog b/docs/changelog
index 76f47679..8e51b949 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,8 +3,8 @@ Changelog
 
 next:
 	* Type __user: Fix missing shadow for alpine (llnu)
-	* Type __consule_agent: Make conf_dir dependent on OS - fixes
-	Alpine (Nico Schottelius)
+	* Type __consule_agent: Make conf_dir dependent on OS - fixes Alpine (Nico Schottelius)
+	* Type __letsencrypt_cert: Fix typo (Andrew Schleifer)
 
 6.5.2: 2020-02-27
 	* Type __update_alternatives: Add state explorer (Ander Punnar)

From b25939cdd64fdab7d71ab9285c64ad5bf14797f5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 27 Mar 2020 13:30:52 +0100
Subject: [PATCH 025/527] Fix shellcheck

---
 cdist/conf/type/__consul_agent/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest
index a7fe0bc5..7b54529c 100755
--- a/cdist/conf/type/__consul_agent/manifest
+++ b/cdist/conf/type/__consul_agent/manifest
@@ -232,7 +232,7 @@ if [ -f "$__object/parameter/ca-file-source" ] || \
    [ -f "$__object/parameter/cert-file-source" ] || \
    [ -f "$__object/parameter/key-file-source" ]; then
 
-   requires="$config_deployment_requires" __directory $tls_dir \
+   requires="$config_deployment_requires" __directory "$tls_dir" \
      --owner root --group "$group" --mode 750 --state "$state"
 
    # Append to service restart requirements.

From da30afe791e1eca11d68fedc88d7a943ecf089e5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sun, 29 Mar 2020 12:54:02 +0200
Subject: [PATCH 026/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8e51b949..62ef28be 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __user: Fix missing shadow for alpine (llnu)
 	* Type __consule_agent: Make conf_dir dependent on OS - fixes Alpine (Nico Schottelius)
 	* Type __letsencrypt_cert: Fix typo (Andrew Schleifer)
+	* Type __package_update_index: Fix maxage false positives (Matthias Stecher)
 
 6.5.2: 2020-02-27
 	* Type __update_alternatives: Add state explorer (Ander Punnar)

From 0f639a9278e32b395bbc0f20b53b89ce1bb324a8 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 2 Apr 2020 18:14:09 +0200
Subject: [PATCH 027/527] Make __cron nonparallel

It fails due to removal / edit of crontab at the same time

VERBOSE: [18331]: uncloud.ungleich.ch: Running object __cron/pg_dump_cleanup
VERBOSE: [18332]: uncloud.ungleich.ch: Running object __ungleich_packages/server
VERBOSE: [18335]: uncloud.ungleich.ch: Running object __postgres_role/app
INFO: [18335]: uncloud.ungleich.ch: Processing __postgres_role/app
INFO: [18331]: uncloud.ungleich.ch: Processing __cron/pg_dump_cleanup
ERROR: [16451]: uncloud.ungleich.ch: ssh -o User=root -o ControlPath=/tmp/tmp_8eu4oep/s -o ControlMaster=auto -o ControlPersist=2h uncloud.ungleich.ch /bin/sh -c ' export __object=/var/lib/cdist/object/__cron/pg_dumpall_under_day/.cdist-mfd00npk;  export __object_id=pg_dumpall_under_day;/bin/sh -e /var/lib/cdist/object/__cron/pg_dumpall_under_day/.cdist-mfd00npk/code-remote': ['ssh', '-o', 'User=root', '-o', 'ControlPath=/tmp/tmp_8eu4oep/s', '-o', 'ControlMaster=auto', '-o', 'ControlPersist=2h', 'uncloud.ungleich.ch', "/bin/sh -c ' export __object=/var/lib/cdist/object/__cron/pg_dumpall_under_day/.cdist-mfd00npk;  export __object_id=pg_dumpall_under_day;/bin/sh -e /var/lib/cdist/object/__cron/pg_dumpall_under_day/.cdist-mfd00npk/code-remote'"]

Error processing object '__cron/pg_dumpall_under_day'
=====================================================
name: __cron/pg_dumpall_under_day
path: /tmp/tmplaq9cwdh/6318c251013a449595327745daacf3ee/data/object/__cron/pg_dumpall_under_day/.cdist-mfd00npk
source: /tmp/tmplaq9cwdh/6318c251013a449595327745daacf3ee/data/conf/type/__ungleich_postgresql/manifest
type: /home/nico/vcs/cdist/cdist/conf/type/__cron

code-remote:stderr
------------------
crontab: can't move 'postgres.new' to 'postgres': No such file or directory

VERBOSE: [16451]: config: Total processing time for 1 host(s): 13.98031210899353
[18:09] line:~%
---
 cdist/conf/type/__cron/nonparallel | 0
 docs/changelog                     | 1 +
 2 files changed, 1 insertion(+)
 create mode 100644 cdist/conf/type/__cron/nonparallel

diff --git a/cdist/conf/type/__cron/nonparallel b/cdist/conf/type/__cron/nonparallel
new file mode 100644
index 00000000..e69de29b
diff --git a/docs/changelog b/docs/changelog
index 62ef28be..b608b35f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -2,6 +2,7 @@ Changelog
 ---------
 
 next:
+	* Type __cron: Make non parallel due to race condition (Nico Schottelius)
 	* Type __user: Fix missing shadow for alpine (llnu)
 	* Type __consule_agent: Make conf_dir dependent on OS - fixes Alpine (Nico Schottelius)
 	* Type __letsencrypt_cert: Fix typo (Andrew Schleifer)

From d034fe9369f1f7aa00a7b0136ffd940ecbceb588 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 2 Apr 2020 18:24:13 +0200
Subject: [PATCH 028/527] [__pyvenv] use python3 -m venv on alpine

Until python4 comes, this will work
---
 cdist/conf/type/__pyvenv/gencode-remote | 12 +++++++++++-
 docs/changelog                          |  1 +
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__pyvenv/gencode-remote b/cdist/conf/type/__pyvenv/gencode-remote
index 04700683..05ec3b29 100755
--- a/cdist/conf/type/__pyvenv/gencode-remote
+++ b/cdist/conf/type/__pyvenv/gencode-remote
@@ -37,11 +37,21 @@ mode="$(cat "$__object/parameter/mode")"
 destination="/$__object_id"
 venvparams="$(cat "$__object/parameter/venvparams")"
 pyvenvparam="$__object/parameter/pyvenv"
+
+os=$(cat $__global/explorer/os)
+
 if [ -f "$pyvenvparam" ]
 then
     pyvenv=$(cat "$pyvenvparam")
 else
-    pyvenv="pyvenv"
+    case "$os" in
+        alpine) # no pyvenv on alpine - I assume others will follow
+            pyvenv="python3 -m venv"
+            ;;
+        *)
+            pyvenv="pyvenv"
+            ;;
+    esac
 fi
 
 case $state_should in
diff --git a/docs/changelog b/docs/changelog
index b608b35f..f5846925 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __cron: Make non parallel due to race condition (Nico Schottelius)
+	* Type __pyvenv: Use python3 -m venv on Alpine (Nico Schottelius)
 	* Type __user: Fix missing shadow for alpine (llnu)
 	* Type __consule_agent: Make conf_dir dependent on OS - fixes Alpine (Nico Schottelius)
 	* Type __letsencrypt_cert: Fix typo (Andrew Schleifer)

From 890c73f6bdd027e60cedd868323f9d0438408b7a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 2 Apr 2020 20:18:04 +0200
Subject: [PATCH 029/527] Fix shellcheck issues

---
 cdist/conf/type/__package_update_index/explorer/currage | 6 +++---
 cdist/conf/type/__pyvenv/gencode-remote                 | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__package_update_index/explorer/currage b/cdist/conf/type/__package_update_index/explorer/currage
index bd51ee86..8eadaf53 100644
--- a/cdist/conf/type/__package_update_index/explorer/currage
+++ b/cdist/conf/type/__package_update_index/explorer/currage
@@ -24,18 +24,18 @@ case "$type" in
         if [ -f "/var/cache/apt/pkgcache.bin" ]; then
             echo $(($(date +"%s")-$(stat --format '%Y' /var/cache/apt/pkgcache.bin)))
         else
-            echo -1
+            echo -- -1
         fi
         ;;
     pacman)
         if [ -d "/var/lib/pacman/sync" ]; then
             echo $(($(date +"%s")-$(stat --format '%Y' /var/lib/pacman/sync)))
         else
-            echo -1
+            echo -- -1
         fi
         ;;
     alpine)
-        echo -1
+        echo -- -1
         ;;
     *)  echo "Your specified type ($type) is currently not supported." >&2
         echo "Please contribute an implementation for it if you can." >&2
diff --git a/cdist/conf/type/__pyvenv/gencode-remote b/cdist/conf/type/__pyvenv/gencode-remote
index 05ec3b29..9c7b7fab 100755
--- a/cdist/conf/type/__pyvenv/gencode-remote
+++ b/cdist/conf/type/__pyvenv/gencode-remote
@@ -38,7 +38,7 @@ destination="/$__object_id"
 venvparams="$(cat "$__object/parameter/venvparams")"
 pyvenvparam="$__object/parameter/pyvenv"
 
-os=$(cat $__global/explorer/os)
+os=$(cat "$__global/explorer/os")
 
 if [ -f "$pyvenvparam" ]
 then

From ceb6b597ef66dc21c901b5b52f251f4c8c61cd54 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 3 Apr 2020 19:45:44 +0200
Subject: [PATCH 030/527] Release 6.5.3

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index f5846925..aa5439d5 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.5.3: 2020-04-03
 	* Type __cron: Make non parallel due to race condition (Nico Schottelius)
 	* Type __pyvenv: Use python3 -m venv on Alpine (Nico Schottelius)
 	* Type __user: Fix missing shadow for alpine (llnu)

From d53077f4e8fbca26c6f1105efa1060df749900b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 6 Apr 2020 09:26:52 +0200
Subject: [PATCH 031/527] Add --encoding, --lc-collate, --lc-support to
 __postgres_database type

---
 .../type/__postgres_database/gencode-remote    | 18 +++++++++++++++++-
 cdist/conf/type/__postgres_database/man.rst    | 12 ++++++++++--
 .../__postgres_database/parameter/optional     |  3 +++
 3 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote
index 47e6b97c..b5f52f50 100755
--- a/cdist/conf/type/__postgres_database/gencode-remote
+++ b/cdist/conf/type/__postgres_database/gencode-remote
@@ -43,8 +43,24 @@ if [ "$state_should" != "$state_is" ]; then
          if [ -f "$__object/parameter/owner" ]; then
             owner="-O \"$(cat "$__object/parameter/owner")\""
          fi
+
+         encoding=""
+         if [ -f "$__object/parameter/encoding" ]; then
+            encoding="--encoding \"$(cat "$__object/parameter/encoding")\""
+         fi
+
+         lc_collate=""
+         if [ -f "$__object/parameter/lc-collate" ]; then
+            lc_collate="--lc-collate \"$(cat "$__object/parameter/lc-collate")\""
+         fi
+
+         lc_ctype=""
+         if [ -f "$__object/parameter/lc-ctype" ]; then
+            lc_ctype="--lc-ctype \"$(cat "$__object/parameter/lc-ctype")\""
+         fi
+
          cat << EOF
-su - '$postgres_user' -c "createdb $owner \"$name\""
+su - '$postgres_user' -c "createdb $owner \"$name\" $encoding $lc_collate $lc_ctype"
 EOF
       ;;
       absent)
diff --git a/cdist/conf/type/__postgres_database/man.rst b/cdist/conf/type/__postgres_database/man.rst
index acceec9b..97ac95eb 100644
--- a/cdist/conf/type/__postgres_database/man.rst
+++ b/cdist/conf/type/__postgres_database/man.rst
@@ -14,11 +14,19 @@ This cdist type allows you to create or drop postgres databases.
 OPTIONAL PARAMETERS
 -------------------
 state
-   either 'present' or 'absent', defaults to 'present'.
+   Either 'present' or 'absent', defaults to 'present'.
 
 owner
-   the role owning this database
+   Specifies the database user who will own the new database.
 
+encoding
+   Specifies the character encoding scheme to be used in this database.
+
+lc-collate
+   Specifies the LC_COLLATE setting to be used in this database.
+
+lc-ctype
+   Specifies the LC_CTYPE setting to be used in this database.
 
 EXAMPLES
 --------
diff --git a/cdist/conf/type/__postgres_database/parameter/optional b/cdist/conf/type/__postgres_database/parameter/optional
index d86b6469..fed2581e 100644
--- a/cdist/conf/type/__postgres_database/parameter/optional
+++ b/cdist/conf/type/__postgres_database/parameter/optional
@@ -1,2 +1,5 @@
 state
 owner
+encoding
+lc-collate
+lc-ctype

From 76d978d3d85051fe943334771c30dfff907f6e21 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Fri, 10 Apr 2020 10:51:17 +0200
Subject: [PATCH 032/527] explorer/init: do not grep on non-existent init

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/conf/explorer/init | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init
index 1b921c68..f27c77ef 100755
--- a/cdist/conf/explorer/init
+++ b/cdist/conf/explorer/init
@@ -221,6 +221,7 @@ check_systemstarter() {
 
 check_sysvinit() (
 	init_path=${1:-/sbin/init}
+	test -x "${init_path}" || return 1
 	grep -q 'INIT_VERSION=sysvinit-[0-9.]*' "${init_path}" || return 1
 
 	# It is quite common to use SysVinit to stack other init systemd

From e19c1bb1e0072a4bfd6ec64ada2d3c4fab1ac064 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Fri, 10 Apr 2020 21:50:39 +0200
Subject: [PATCH 033/527] remove duplicates from conf dirs while preserving
 order

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/exec/util.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cdist/exec/util.py b/cdist/exec/util.py
index 9787f431..e3be2235 100644
--- a/cdist/exec/util.py
+++ b/cdist/exec/util.py
@@ -199,7 +199,9 @@ def resolve_conf_dirs(configuration, add_conf_dirs):
 
     if add_conf_dirs:
         conf_dirs.extend(add_conf_dirs)
-    conf_dirs = set(conf_dirs)
+
+    # Remove duplicates.
+    conf_dirs = list(dict.fromkeys(conf_dirs))
     return conf_dirs
 
 

From 1ebcc219c26f91bcc634a9526171871f15210bd9 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 11 Apr 2020 09:54:57 +0200
Subject: [PATCH 034/527] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index aa5439d5..a927a08d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Explorer init: Do not grep on non-existent init (Steven Armstrong)
+
 6.5.3: 2020-04-03
 	* Type __cron: Make non parallel due to race condition (Nico Schottelius)
 	* Type __pyvenv: Use python3 -m venv on Alpine (Nico Schottelius)

From 704e78322ed5794fd9baf324f64363858fda2b26 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 11 Apr 2020 20:26:20 +0200
Subject: [PATCH 035/527] Use OrderedDict to guarantee order

Note:
> Changed in version 3.7: Dictionary order is guaranteed to be
> insertion order. This behavior was an implementation detail of
> CPython from 3.6.
---
 cdist/exec/util.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/exec/util.py b/cdist/exec/util.py
index e3be2235..90a26ad3 100644
--- a/cdist/exec/util.py
+++ b/cdist/exec/util.py
@@ -22,6 +22,7 @@
 import subprocess
 import os
 from tempfile import TemporaryFile
+from collections import OrderedDict
 
 import cdist
 import cdist.configuration
@@ -201,7 +202,7 @@ def resolve_conf_dirs(configuration, add_conf_dirs):
         conf_dirs.extend(add_conf_dirs)
 
     # Remove duplicates.
-    conf_dirs = list(dict.fromkeys(conf_dirs))
+    conf_dirs = list(OrderedDict.fromkeys(conf_dirs))
     return conf_dirs
 
 

From d1eecb93eeef11da057052fe14818ff442a0d0c8 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 11 Apr 2020 20:30:56 +0200
Subject: [PATCH 036/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index a927a08d..8c7d497b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Explorer init: Do not grep on non-existent init (Steven Armstrong)
+	* Core: Bugfix to preserve conf dirs order (Steven Armstrong)
 
 6.5.3: 2020-04-03
 	* Type __cron: Make non parallel due to race condition (Nico Schottelius)

From 0805fac7e91b9ea3d0dd92fb00e36bc33d8cf779 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 11 Apr 2020 20:35:29 +0200
Subject: [PATCH 037/527] Release 6.5.4

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 8c7d497b..b6b1bb39 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.5.4: 2020-04-11
 	* Explorer init: Do not grep on non-existent init (Steven Armstrong)
 	* Core: Bugfix to preserve conf dirs order (Steven Armstrong)
 

From c3f924d350772d7c3af1ce8bbbe29f1b8010d4f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 14 Apr 2020 10:23:08 +0200
Subject: [PATCH 038/527] Add --template flag to __postgres_database type

---
 cdist/conf/type/__postgres_database/gencode-remote     | 7 ++++++-
 cdist/conf/type/__postgres_database/man.rst            | 4 ++++
 cdist/conf/type/__postgres_database/parameter/optional | 1 +
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote
index b5f52f50..0f11cff4 100755
--- a/cdist/conf/type/__postgres_database/gencode-remote
+++ b/cdist/conf/type/__postgres_database/gencode-remote
@@ -44,6 +44,11 @@ if [ "$state_should" != "$state_is" ]; then
             owner="-O \"$(cat "$__object/parameter/owner")\""
          fi
 
+         template=""
+         if [ -f "$__object/parameter/template" ]; then
+            template="--template \"$(cat "$__object/parameter/template")\""
+         fi
+
          encoding=""
          if [ -f "$__object/parameter/encoding" ]; then
             encoding="--encoding \"$(cat "$__object/parameter/encoding")\""
@@ -60,7 +65,7 @@ if [ "$state_should" != "$state_is" ]; then
          fi
 
          cat << EOF
-su - '$postgres_user' -c "createdb $owner \"$name\" $encoding $lc_collate $lc_ctype"
+su - '$postgres_user' -c "createdb $owner \"$name\" $template $encoding $lc_collate $lc_ctype"
 EOF
       ;;
       absent)
diff --git a/cdist/conf/type/__postgres_database/man.rst b/cdist/conf/type/__postgres_database/man.rst
index 97ac95eb..870b4917 100644
--- a/cdist/conf/type/__postgres_database/man.rst
+++ b/cdist/conf/type/__postgres_database/man.rst
@@ -28,6 +28,10 @@ lc-collate
 lc-ctype
    Specifies the LC_CTYPE setting to be used in this database.
 
+template
+   Specifies the template database from which to build this database.
+
+
 EXAMPLES
 --------
 
diff --git a/cdist/conf/type/__postgres_database/parameter/optional b/cdist/conf/type/__postgres_database/parameter/optional
index fed2581e..877fbf32 100644
--- a/cdist/conf/type/__postgres_database/parameter/optional
+++ b/cdist/conf/type/__postgres_database/parameter/optional
@@ -3,3 +3,4 @@ owner
 encoding
 lc-collate
 lc-ctype
+template

From 742163e38c2a8124390cecee02a9c865fe18fdff Mon Sep 17 00:00:00 2001
From: Joachim Desroches <joachim.desroches@epfl.ch>
Date: Wed, 15 Apr 2020 17:10:33 +0200
Subject: [PATCH 039/527] Fix configuration file location

---
 cdist/configuration.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/configuration.py b/cdist/configuration.py
index f05a5963..1011a382 100644
--- a/cdist/configuration.py
+++ b/cdist/configuration.py
@@ -274,7 +274,8 @@ class Configuration(metaclass=Singleton):
              os.path.isfile(_local_config_file))):
         _local_config_file = os.path.join(
             os.environ.get('XDG_CONFIG_HOME',
-                           os.path.expanduser('~/.config/cdist')),
+                           os.path.expanduser('~/.config/')),
+            'cdist',
             _config_basename)
     _dist_config_file = os.path.join(
         os.path.abspath(os.path.join(os.path.dirname(cdist.__file__), "conf")),

From c9c1e7d79050afe2bc69b875096dbd1379d70eea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 21 Apr 2020 08:13:32 +0200
Subject: [PATCH 040/527] Import CI image definition

---
 other/ci/Dockerfile   | 8 ++++++++
 other/ci/README.md    | 3 +++
 other/ci/repositories | 3 +++
 3 files changed, 14 insertions(+)
 create mode 100644 other/ci/Dockerfile
 create mode 100644 other/ci/README.md
 create mode 100644 other/ci/repositories

diff --git a/other/ci/Dockerfile b/other/ci/Dockerfile
new file mode 100644
index 00000000..03d6b546
--- /dev/null
+++ b/other/ci/Dockerfile
@@ -0,0 +1,8 @@
+FROM alpine:latest
+
+COPY ./repositories /etc/apk/
+
+RUN apk update
+RUN apk upgrade
+RUN apk add python3 py3-pycodestyle rsync make shellcheck git
+RUN apk fix
diff --git a/other/ci/README.md b/other/ci/README.md
new file mode 100644
index 00000000..0bd64613
--- /dev/null
+++ b/other/ci/README.md
@@ -0,0 +1,3 @@
+This container is used for cdist's CI pipeline, and deployed in ungleich's docker registry at:
+
+	code.ungleich.ch:5050/ungleich-public/cdist-ci:latest
diff --git a/other/ci/repositories b/other/ci/repositories
new file mode 100644
index 00000000..46cabcc3
--- /dev/null
+++ b/other/ci/repositories
@@ -0,0 +1,3 @@
+https://mirror.ungleich.ch/mirror/packages/alpine/edge/main
+https://mirror.ungleich.ch/mirror/packages/alpine/edge/community
+https://mirror.ungleich.ch/mirror/packages/alpine/edge/testing

From 7d576554708115cef78eac364b20265386998882 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 21 Apr 2020 08:15:51 +0200
Subject: [PATCH 041/527] Fix typo in cdist-ci image README

---
 other/ci/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/other/ci/README.md b/other/ci/README.md
index 0bd64613..6a9b0ac5 100644
--- a/other/ci/README.md
+++ b/other/ci/README.md
@@ -1,3 +1,3 @@
 This container is used for cdist's CI pipeline, and deployed in ungleich's docker registry at:
 
-	code.ungleich.ch:5050/ungleich-public/cdist-ci:latest
+	code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest

From e2b26aa233fe702f19a39fa1378a6c531d36720f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 21 Apr 2020 08:16:46 +0200
Subject: [PATCH 042/527] Patch CI configuration to use cdist-ci image

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1cc17995..e215652c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,8 @@
 stages:
     - test
 
+image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
+
 unit_tests:
     stage: test
     script:

From 38ccdfda321e6bdb683842d227b379feea5e752c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 22 Apr 2020 23:21:34 +0200
Subject: [PATCH 043/527] Fix newly found shellcheck issues

---
 cdist/conf/type/__cdist/manifest           | 1 +
 cdist/conf/type/__openldap_server/manifest | 1 +
 2 files changed, 2 insertions(+)

diff --git a/cdist/conf/type/__cdist/manifest b/cdist/conf/type/__cdist/manifest
index a97cf288..0b0f1263 100755
--- a/cdist/conf/type/__cdist/manifest
+++ b/cdist/conf/type/__cdist/manifest
@@ -37,6 +37,7 @@ source="$(cat "$__object/parameter/source")"
 # out of it
 home=/home/$username
 
+# shellcheck disable=SC2086
 __user "$username" --home "$home" $shell
 
 require="__user/$username" __directory "$home" \
diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest
index dadc9f20..d35603c4 100644
--- a/cdist/conf/type/__openldap_server/manifest
+++ b/cdist/conf/type/__openldap_server/manifest
@@ -168,6 +168,7 @@ if [ -z "${_skip_letsencrypt_cert}" ]; then
         staging=""
     fi
 
+    # shellcheck disable=SC2086
     __letsencrypt_cert "${name}" --admin-email "${admin_email}" \
         --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R openldap:openldap ${SLAPD_DIR}/sasl2 && service slapd restart" \
         --automatic-renewal ${staging}

From 41e59a748dc45aaec88edc7a7eaed96c6086b1b1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 22 Apr 2020 23:24:34 +0200
Subject: [PATCH 044/527] Fix newly found shellcheck issues

---
 scripts/cdist-dump | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/cdist-dump b/scripts/cdist-dump
index 83b09eb8..d29e5985 100755
--- a/scripts/cdist-dump
+++ b/scripts/cdist-dump
@@ -224,6 +224,7 @@ hor_line()
 if [ "${do_global_explorer}" ]
 then
     print_verbose 2 "Dumping global explorers"
+    # shellcheck disable=SC2086
     set -- "$@" ${or} \( \
                 -path "*/explorer/*" -a \
                 ! -path "*/conf/*" -a \

From de37b0ce450109306e31affa67dd2ff3653b6d68 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 24 Apr 2020 20:26:44 +0200
Subject: [PATCH 045/527] [__motd] Improve documentation and support for
 FreeBSD

This makes it easier to use the type just by reading the man page and also
treats FreeBSD's MOTD better.
---
 cdist/conf/type/__motd/gencode-remote | 11 +++++++++++
 cdist/conf/type/__motd/man.rst        | 19 ++++++++++++++++++-
 cdist/conf/type/__motd/manifest       |  8 ++++++++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__motd/gencode-remote b/cdist/conf/type/__motd/gencode-remote
index bc842cc8..738ea834 100755
--- a/cdist/conf/type/__motd/gencode-remote
+++ b/cdist/conf/type/__motd/gencode-remote
@@ -29,7 +29,18 @@ case "$os" in
         echo "uname -snrvm > /var/run/motd"
         echo "cat /etc/motd.tail >> /var/run/motd"
     ;;
+    freebsd)
+        # FreeBSD only updates /etc/motd on boot,
+	# as seen in /etc/rc.d/motd
+	echo "uname -sri > /etc/motd"
+	echo "cat /etc/motd.template >> /etc/motd"
+	# FreeBSD 13 starts treating motd slightly different from previous
+	# versions this ensures hosts have the expected config.
+	echo "rm /etc/motd.template || true"
+	echo "service motd start"
+    ;;
     *)
+        # Other OS tend to treat /etc/motd statically
         exit 0
     ;;
 esac
diff --git a/cdist/conf/type/__motd/man.rst b/cdist/conf/type/__motd/man.rst
index 17369684..a567dc80 100644
--- a/cdist/conf/type/__motd/man.rst
+++ b/cdist/conf/type/__motd/man.rst
@@ -10,6 +10,13 @@ DESCRIPTION
 -----------
 This cdist type allows you to easily setup /etc/motd.
 
+.. note::
+      In some OS, motd is a bit special, check `motd(5)`.
+      Currently Debian, Devuan, Ubuntu and FreeBSD are taken into account.
+      If your OS of choice does something besides /etc/motd, check the source
+      and contribute support for it.
+      Otherwise it will likely just work.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -20,6 +27,7 @@ OPTIONAL PARAMETERS
 -------------------
 source
    If supplied, copy this file from the host running cdist to the target.
+   If source is '-' (dash), take what was written to stdin as the file content.
    If not supplied, a default message will be placed onto the target.
 
 
@@ -34,6 +42,15 @@ EXAMPLES
     # Supply source file from a different type
     __motd --source "$__type/files/my-motd"
 
+    # Supply source from stdin
+    __motd --source "-" <<EOF
+    Take this kiss upon the brow!
+    And, in parting from you now,
+    Thus much let me avow-
+    You are not wrong, who deem
+    That my days have been a dream
+    EOF
+
 
 AUTHORS
 -------
@@ -42,7 +59,7 @@ Nico Schottelius <nico-cdist--@--schottelius.org>
 
 COPYING
 -------
-Copyright \(C) 2011 Nico Schottelius. You can redistribute it
+Copyright \(C) 2020 Nico Schottelius. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__motd/manifest b/cdist/conf/type/__motd/manifest
index cd741cf4..ded734d7 100755
--- a/cdist/conf/type/__motd/manifest
+++ b/cdist/conf/type/__motd/manifest
@@ -34,9 +34,17 @@ os=$(cat "$__global/explorer/os")
 
 case "$os" in
    debian|ubuntu|devuan)
+      # Debian-based systems use /etc/motd.tail as a template
       destination=/etc/motd.tail
    ;;
+   freebsd)
+      # FreeBSD uses motd.template to prepend system information on boot
+      # (this actually only applies starting with version 13,
+      #  but we fix that for whatever version in gencode-remote)
+      destination=/etc/motd.template
+   ;;
    *)
+      # Most UNIX systems, including other Linux and OpenBSD just use /etc/motd
       destination=/etc/motd
    ;;
 esac

From 056c7c5400e7beeb15de800be6b62661998259ff Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 25 Apr 2020 00:12:24 +0200
Subject: [PATCH 046/527] [__openldap_server] Support extra config parameter.

This allows the user to, e.g. manually define ACLs, while this type does not
support that.
---
 cdist/conf/type/__openldap_server/man.rst            | 3 +++
 cdist/conf/type/__openldap_server/manifest           | 3 +++
 cdist/conf/type/__openldap_server/parameter/optional | 3 ++-
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
index d20101d1..fbad21d8 100644
--- a/cdist/conf/type/__openldap_server/man.rst
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -92,6 +92,9 @@ tls-ca
     Required if `tls-cert` is defined.
     Path in the remote hosts to the PEM-encoded CA certificate file.
 
+extra-config
+    Custom settings to be added in `slapd.conf(5)`.
+
 
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest
index d35603c4..84ba176f 100644
--- a/cdist/conf/type/__openldap_server/manifest
+++ b/cdist/conf/type/__openldap_server/manifest
@@ -9,6 +9,7 @@ slapd_modules=$(cat "${__object}/parameter/module" 2>/dev/null || true)
 schemas=$(cat "${__object}/parameter/schema")
 slapd_urls=$(tr '\n' ' ' < "${__object}/parameter/slapd-url")
 tls_cipher_suite=$(cat "${__object}/parameter/tls-cipher-suite" 2>/dev/null || true)
+extra_config=$(cat "${__object}/parameter/extra-config" || true)
 
 
 os="$(cat "${__global}/explorer/os")"
@@ -231,6 +232,8 @@ index uid,memberUid eq,pres,sub
 index nisMapName,nisMapEntry eq,pres,sub
 index entryCSN,entryUUID eq
 
+${extra_config}
+
 serverid ${serverid}
 EOF
 
diff --git a/cdist/conf/type/__openldap_server/parameter/optional b/cdist/conf/type/__openldap_server/parameter/optional
index a92b9c6e..71c64659 100644
--- a/cdist/conf/type/__openldap_server/parameter/optional
+++ b/cdist/conf/type/__openldap_server/parameter/optional
@@ -5,4 +5,5 @@ admin-email
 tls-cipher-suite
 tls-cert
 tls-privkey
-tls-ca
\ No newline at end of file
+tls-ca
+extra-config

From 5981d0a5f1deb2e60ea3c95ad3b535f1425060f6 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 25 Apr 2020 00:22:28 +0200
Subject: [PATCH 047/527] [__postfix] Automagically support more OSs by not
 checking too much.

It is quite likely that the package is going to be called postfix, rather
than trying to have an exhaustive "allow list" for this package, we can
just add special cases for OSs where that is not the case (not aware of any
atm).
---
 cdist/conf/type/__postfix/manifest | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/cdist/conf/type/__postfix/manifest b/cdist/conf/type/__postfix/manifest
index f3616979..121bba96 100755
--- a/cdist/conf/type/__postfix/manifest
+++ b/cdist/conf/type/__postfix/manifest
@@ -19,16 +19,4 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-
-os=$(cat "$__global/explorer/os")
-
-case "$os" in
-   alpine|ubuntu|debian|archlinux|suse|scientific|centos|devuan)
-      __package postfix --state present
-   ;;
-   *)
-      echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
-      echo "Please contribute an implementation for it if you can." >&2
-      exit 1
-   ;;
-esac
+__package postfix --state present

From fefe90e9c9bde2ad00c142c01d5a588eb99751e5 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 25 Apr 2020 12:23:27 +0200
Subject: [PATCH 048/527] [__pf*] (~) __pf_ruleset (+)__pf_apply_anchor,
 deprecate __pf_apply
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

__pf_apply the way it exists on cdist is not really useful and __pf_ruleset does
not take advantage of other types as it should, being instead overly complex and
not as reliable.

The new __pf_ruleset is compatible with the previous one, and __pf_apply_anchors
allows for a simple and powerful way of managing pf anchors.
The functionality previously provided by __pf_apply is still possible out of the
box in __pf_ruleset.

These patches were mostly contributed by Kamila Součková and made fit for
upstreaming by Evilham.
---
 cdist/conf/type/__pf_apply/deprecated         |  1 +
 .../gencode-remote}                           | 28 +++----
 cdist/conf/type/__pf_apply_anchor/man.rst     | 62 ++++++++++++++
 .../type/__pf_apply_anchor/parameter/optional |  1 +
 cdist/conf/type/__pf_ruleset/gencode-local    | 81 -------------------
 cdist/conf/type/__pf_ruleset/man.rst          | 14 ++--
 .../__pf_ruleset/{gencode-remote => manifest} | 47 +++++------
 7 files changed, 105 insertions(+), 129 deletions(-)
 create mode 100644 cdist/conf/type/__pf_apply/deprecated
 rename cdist/conf/type/{__pf_ruleset/explorer/cksum => __pf_apply_anchor/gencode-remote} (55%)
 create mode 100644 cdist/conf/type/__pf_apply_anchor/man.rst
 create mode 100644 cdist/conf/type/__pf_apply_anchor/parameter/optional
 delete mode 100755 cdist/conf/type/__pf_ruleset/gencode-local
 rename cdist/conf/type/__pf_ruleset/{gencode-remote => manifest} (51%)

diff --git a/cdist/conf/type/__pf_apply/deprecated b/cdist/conf/type/__pf_apply/deprecated
new file mode 100644
index 00000000..36cfed90
--- /dev/null
+++ b/cdist/conf/type/__pf_apply/deprecated
@@ -0,0 +1 @@
+Consider moving to __pf_apply_anchor. Get in touch if you need __pf_apply.
diff --git a/cdist/conf/type/__pf_ruleset/explorer/cksum b/cdist/conf/type/__pf_apply_anchor/gencode-remote
similarity index 55%
rename from cdist/conf/type/__pf_ruleset/explorer/cksum
rename to cdist/conf/type/__pf_apply_anchor/gencode-remote
index 9be6c901..36c26521 100755
--- a/cdist/conf/type/__pf_ruleset/explorer/cksum
+++ b/cdist/conf/type/__pf_apply_anchor/gencode-remote
@@ -1,6 +1,6 @@
-#!/bin/sh
+#!/bin/sh -e
 #
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
+# 2016 Kamila Součková (coding at kamila.is)
 #
 # This file is part of cdist.
 #
@@ -18,24 +18,16 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Get the 256 bit SHA2 checksum of the pf ruleset on the target host.
+# Apply pf(4) ruleset on *BSD
 #
 
-# Debug
-#exec >&2
-#set -x
+ANCHORS_DIR="/etc/pf.d"
 
-# Check /etc/rc.conf for pf's configuration file name. Default to /etc/pf.conf
-# See if file exists and if so, get checksum
-
-RC="/etc/rc.conf"
-TMP="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')"
-PFCONF="${TMP:-"/etc/pf.conf"}"
-
-if [ -f "${PFCONF}" ]; then	# The pf config file exists, find its cksum.
-	cksum -o 1 "${PFCONF}" | cut -d= -f2 | awk '{print $1}'
+if [ -f "${__object}/parameter/anchor_name" ]; then
+    anchor_name="$(cat "${__object}/parameter/anchor_name")"
+else
+    anchor_name="${__object_id}"
 fi
+anchor_file="${ANCHORS_DIR}/${anchor_name}"
 
-# Debug
-#set +x
-
+echo "pfctl -a \"${anchor_name}\" -f \"${anchor_file}\""
diff --git a/cdist/conf/type/__pf_apply_anchor/man.rst b/cdist/conf/type/__pf_apply_anchor/man.rst
new file mode 100644
index 00000000..aef6cdf4
--- /dev/null
+++ b/cdist/conf/type/__pf_apply_anchor/man.rst
@@ -0,0 +1,62 @@
+cdist-type__pf_apply_anchor(7)
+==============================
+
+NAME
+----
+cdist-type__pf_apply_anchor - Apply a pf(4) anchor on $__target_host
+
+
+DESCRIPTION
+-----------
+This type is used on \*BSD systems to manage anchors for the pf firewall.
+
+Notice this type does not take care of copying the ruleset, that must be
+done by the user with, e.g. `__file`.
+
+
+OPTIONAL PARAMETERS
+-------------------
+anchor_name
+   The name of the anchor to apply. If not set, `${__object_id}` is used.
+   This type requires `/etc/pf.d/${anchor_name}` to exist on
+   `$__target_host`.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Copy anchor file to ${__target_host}
+    __file "/etc/pf.d/80_dns" --source - <<EOF
+    # Managed remotely, changes will be lost
+
+    pass quick proto {tcp,udp} from any to any port domain
+    EOF
+
+    # Apply the anchor
+    require="__file/etc/pf.d/80_dns" __pf_apply_anchor 80_dns
+    # This is roughly equivalent to:
+    #   pfctl -a "${anchor_name}" -f "/etc/pf.d/${anchor_name}"
+
+
+SEE ALSO
+--------
+:strong:`pf`\ (4)
+
+
+AUTHORS
+-------
+Evilham <contact--@--evilham.com>
+Kamila Součková <coding--@--kamila.is>
+Jake Guffey <jake.guffey--@--eprotex.com>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Evilham.
+Copyright \(C) 2016 Kamila Součková.
+Copyright \(C) 2012 Jake Guffey. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__pf_apply_anchor/parameter/optional b/cdist/conf/type/__pf_apply_anchor/parameter/optional
new file mode 100644
index 00000000..b9f61e28
--- /dev/null
+++ b/cdist/conf/type/__pf_apply_anchor/parameter/optional
@@ -0,0 +1 @@
+anchor_name
diff --git a/cdist/conf/type/__pf_ruleset/gencode-local b/cdist/conf/type/__pf_ruleset/gencode-local
deleted file mode 100755
index 11bfb0b1..00000000
--- a/cdist/conf/type/__pf_ruleset/gencode-local
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/bin/sh -e
-#
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Manage pf(4) on *BSD
-#
-
-# Debug
-#exec >&2
-#set -x
-
-# Send files to $__target_host via $__remote_copy
-
-uname=$(uname)   # Need to know what the cdist host is running so we know how to compute the ruleset's checksum
-state=$(cat "$__object/parameter/state")
-
-if [ "$state" = "absent" ]; then   # There is nothing more for a *local* script to do
-   exit 0
-fi
-
-if [ -f "$__object/parameter/source" ]; then
-   source=$(cat "$__object/parameter/source")
-fi
-
-rcvar=$(cat "$__object/explorer/rcvar")
-cksum=$(cat "$__object/explorer/cksum")
-
-
-cat <<EOF
-case $uname in
-   Darwin)
-      currentSum=\$(cksum -o 1 ${source} | cut '-d ' -f1)
-      ;;
-   Linux)
-      currentSum=\$(cksum ${source} | cut '-d ' -f1)
-      ;;
-   FreeBSD)
-      currentSum=\$(cksum -o 1 ${source} | cut -d= -f2 | sed 's/ //g')
-      ;;
-   *)
-      echo "Sorry, I do not know how to find a cksum on ${uname}." >&2
-      exit 1
-      ;;
-esac
-
-# IPv6 fix
-if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$')
-then
-    my_target_host="[${__target_host}]"
-else
-    my_target_host="${__target_host}"
-fi
-
-if [ -n "${cksum}" ]; then
-   if [ ! "\${currentSum}" = "${cksum}" ]; then
-      $__remote_copy "${source}" "\${my_target_host}:${rcvar}.new"
-   fi
-else # File just doesn't exist yet
-   $__remote_copy "${source}" "\${my_target_host}:${rcvar}.new"
-fi
-EOF
-
-# Debug
-#exec +x
-
diff --git a/cdist/conf/type/__pf_ruleset/man.rst b/cdist/conf/type/__pf_ruleset/man.rst
index 5719e94e..db8873ac 100644
--- a/cdist/conf/type/__pf_ruleset/man.rst
+++ b/cdist/conf/type/__pf_ruleset/man.rst
@@ -10,6 +10,9 @@ DESCRIPTION
 -----------
 This type is used on \*BSD systems to manage the pf firewall's ruleset.
 
+It will also enable and disable the pf firewall as requested in the `state`
+parameter.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -20,9 +23,8 @@ state
 OPTIONAL PARAMETERS
 -------------------
 source
-   If supplied, use to define the ruleset to load onto the $__target_host for pf(4).
-   Note that this type is almost useless without a ruleset defined, but it's technically not
-   needed, e.g. for the case of disabling the firewall temporarily.
+   Required when state is "present".
+   Defines the ruleset to load onto the $__target_host for `pf(4)`.
 
 
 EXAMPLES
@@ -30,10 +32,10 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # Remove the current ruleset in place
+    # Remove the current ruleset in place and disable pf
     __pf_ruleset --state absent
 
-    # Enable the firewall with the ruleset defined in $__manifest/files/pf.conf
+    # Enable pf with the ruleset defined in $__manifest/files/pf.conf
     __pf_ruleset --state present --source $__manifest/files/pf.conf
 
 
@@ -44,11 +46,13 @@ SEE ALSO
 
 AUTHORS
 -------
+Kamila Součková <coding--@--kamila.is>
 Jake Guffey <jake.guffey--@--eprotex.com>
 
 
 COPYING
 -------
+Copyright \(C) 2016 Kamila Součková.
 Copyright \(C) 2012 Jake Guffey. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
diff --git a/cdist/conf/type/__pf_ruleset/gencode-remote b/cdist/conf/type/__pf_ruleset/manifest
similarity index 51%
rename from cdist/conf/type/__pf_ruleset/gencode-remote
rename to cdist/conf/type/__pf_ruleset/manifest
index 12760fdf..34fee5dd 100755
--- a/cdist/conf/type/__pf_ruleset/gencode-remote
+++ b/cdist/conf/type/__pf_ruleset/manifest
@@ -1,6 +1,6 @@
 #!/bin/sh -e
 #
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
+# 2016 Kamila Součková (coding at kamila.is)
 #
 # This file is part of cdist.
 #
@@ -21,29 +21,26 @@
 # Manage pf(4) on *BSD
 #
 
-# Debug
-#exec >&2
-#set -x
-
-# Remove ${rcvar} in the case of --state absent
-
-state=$(cat "$__object/parameter/state")
-rcvar=$(cat "$__object/explorer/rcvar")
-
-if [ "$state" = "present" ]; then   # There is nothing more for a *remote* script to do
-   exit 0
-elif [ "$state" = "absent" ]; then
-   # --state absent, so ensure that .new doesn't exist and that conf is renamed to .old
-   cat <<EOF
-   if [ -f "${rcvar}.new" ]; then
-      rm "${rcvar}.new"
-   fi
-   if [ -f "${rcvar}" ]; then
-      mv "${rcvar}" "${rcvar}.old"
-   fi
-EOF
-else
-   echo "Unknown state ${state}!" >&2
-   exit 1
+rcvar="$(cat "${__object}/explorer/rcvar")"
+state="$(cat "${__object}/parameter/state")"
+if [ -f "${__object}/parameter/source" ]; then
+    source="$(cat "${__object}/parameter/source")"
 fi
 
+if [ "${state}" = "absent" ]; then
+    action="/etc/rc.d/pf stop"
+else
+    action="/etc/rc.d/pf reload || /etc/rc.d/pf start"
+fi
+
+__key_value __pf_ruleset/rcvar \
+    --state "${state}" \
+    --file /etc/rc.conf \
+    --delimiter "=" \
+    --key "pf_enable" \
+    --value "YES"
+
+require="__key_value/__pf_ruleset/rcvar" __config_file ${rcvar} \
+    --source "${source}" \
+    --state "${state}" \
+    --onchange "${action}"

From 292879544154f11f4cd719ebf0c67c25000a25fd Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 25 Apr 2020 14:54:29 +0200
Subject: [PATCH 049/527] [__pf_ruleset] Fix shellcheck issue.

---
 cdist/conf/type/__pf_ruleset/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__pf_ruleset/manifest b/cdist/conf/type/__pf_ruleset/manifest
index 34fee5dd..27b35328 100755
--- a/cdist/conf/type/__pf_ruleset/manifest
+++ b/cdist/conf/type/__pf_ruleset/manifest
@@ -40,7 +40,7 @@ __key_value __pf_ruleset/rcvar \
     --key "pf_enable" \
     --value "YES"
 
-require="__key_value/__pf_ruleset/rcvar" __config_file ${rcvar} \
+require="__key_value/__pf_ruleset/rcvar" __config_file "${rcvar}" \
     --source "${source}" \
     --state "${state}" \
     --onchange "${action}"

From 04b7f240ebc1ae5b0f23279d75b65b50d062ccd1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sun, 26 Apr 2020 10:01:18 +0200
Subject: [PATCH 050/527] ++changelog

---
 docs/changelog | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index b6b1bb39..7cacf3cf 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,16 @@
 Changelog
 ---------
 
+next:
+	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)
+	* Type __postgres_database: Add encoding, lc-collate, lc-ctype, template parameters (Timothée Floure)
+	* Type __motd: Improve documentation and support for FreeBSD (Evil Ham)
+	* Type __openldap_server: Support extra config parameter (Evil Ham)
+	* Type __postfix: Automagically support more OSs by not checking too much (Evil Ham)
+	* New type: __pf_apply_anchor (Kamila Součková, Evil Ham)
+	* Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham)
+	* Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham)
+
 6.5.4: 2020-04-11
 	* Explorer init: Do not grep on non-existent init (Steven Armstrong)
 	* Core: Bugfix to preserve conf dirs order (Steven Armstrong)

From fefc828780bc76c4245fa96ef517a66ec3eda3fa Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sun, 26 Apr 2020 19:06:42 +0200
Subject: [PATCH 051/527] [docs] Improve cdist.cfg.skeleton

---
 configuration/cdist.cfg.skeleton | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton
index 22c1ccaf..bfac9f5c 100644
--- a/configuration/cdist.cfg.skeleton
+++ b/configuration/cdist.cfg.skeleton
@@ -19,6 +19,9 @@
 #     such as ':' for POSIX or ';' for Windows.
 #     If also specified at command line then values from command line are
 #     appended to this value.
+#     Notice that this works in a "last one wins" fashion, so if a type is redefined
+#     in multiple conf_dirs, the last one in which it is defined will be used.
+#     Consider using a unique prefix for your own roles if this can be an issue.
 # conf_dir = <dir1>:<dir2>
 #
 # init_manifest

From 678df1ec8a24b4c3884554b1931445435e098787 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 27 Apr 2020 01:23:48 +0200
Subject: [PATCH 052/527] [explorers] Improve *BSD support.

cpu_cores and memory did lacked support for other BSDs.
---
 cdist/conf/explorer/cpu_cores | 4 ++++
 cdist/conf/explorer/memory    | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/cpu_cores b/cdist/conf/explorer/cpu_cores
index a52bddac..c6744142 100755
--- a/cdist/conf/explorer/cpu_cores
+++ b/cdist/conf/explorer/cpu_cores
@@ -32,6 +32,10 @@ case "$os" in
         sysctl -n hw.ncpuonline
     ;;
 
+    "freebsd"|"netbsd")
+        sysctl -n hw.ncpu
+    ;;
+
     *)
         if [ -r /proc/cpuinfo ]; then
             cores="$(grep "core id" /proc/cpuinfo | sort | uniq | wc -l)"
diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory
index 4e3efff8..302b4cda 100755
--- a/cdist/conf/explorer/memory
+++ b/cdist/conf/explorer/memory
@@ -29,7 +29,7 @@ case "$os" in
         echo "$(sysctl -n hw.memsize)/1024" | bc
     ;;
 
-    "openbsd")
+    *"bsd")
         echo "$(sysctl -n hw.physmem) / 1048576" | bc
     ;;
 

From 0b3c417aef13eceb51195bb42e9a17205a9afc6c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 27 Apr 2020 14:00:39 +0300
Subject: [PATCH 053/527] update README

---
 README    |  7 -------
 README.md | 20 ++++++++++++++++++++
 2 files changed, 20 insertions(+), 7 deletions(-)
 delete mode 100644 README
 create mode 100644 README.md

diff --git a/README b/README
deleted file mode 100644
index caf2dac8..00000000
--- a/README
+++ /dev/null
@@ -1,7 +0,0 @@
-cdist
------
-
-cdist is a usable configuration management system.
-
-For the web documentation have a look at https://www.cdi.st/
-or at docs/src for reStructuredText manual.
diff --git a/README.md b/README.md
new file mode 100644
index 00000000..9e49b053
--- /dev/null
+++ b/README.md
@@ -0,0 +1,20 @@
+# cdist
+
+**cdist** is a usable configuration management system.
+
+It adheres to the [**KISS principle**](https://en.wikipedia.org/wiki/KISS_principle)
+and is being used in small up to enterprise grade environments.
+
+For more information have a look at [**homepage**](https://cdi.st)
+or at **``docs/src``** for manual in **reStructuredText** format.
+
+## Contributing
+
+Merge/Pull requests can be made in both
+[upstream **GitLab**](https://code.ungleich.ch/ungleich-public/cdist/merge_requests)
+(managed by [**ungleich**](https://ungleich.ch))
+and [**GitHub** project](https://github.com/ungleich/cdist/pulls).
+
+Issues can be made and other project management activites happen
+[**only in GitLab**](https://code.ungleich.ch/ungleich-public/cdist)
+(needs [**ungleich** account](https://account.ungleich.ch)).

From 56a65518ab6171d0f19152642fccb94038a3c5ad Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 27 Apr 2020 15:25:43 +0300
Subject: [PATCH 054/527] README: add participating section

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index 9e49b053..0a0d6e6d 100644
--- a/README.md
+++ b/README.md
@@ -18,3 +18,11 @@ and [**GitHub** project](https://github.com/ungleich/cdist/pulls).
 Issues can be made and other project management activites happen
 [**only in GitLab**](https://code.ungleich.ch/ungleich-public/cdist)
 (needs [**ungleich** account](https://account.ungleich.ch)).
+
+## Participating
+
+IRC: ``#cdist`` @ freenode
+
+Matrix: ``#cdist:ungleich.ch``
+
+Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist

From b31e13eacf2cb23e6a7bdadc33741026ae88553a Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 27 Apr 2020 16:30:52 +0300
Subject: [PATCH 055/527] README: add bits about cdist-contrib

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index 0a0d6e6d..de6901c7 100644
--- a/README.md
+++ b/README.md
@@ -19,6 +19,9 @@ Issues can be made and other project management activites happen
 [**only in GitLab**](https://code.ungleich.ch/ungleich-public/cdist)
 (needs [**ungleich** account](https://account.ungleich.ch)).
 
+For community-maintained types there is
+[**cdist-contrib** project](https://code.ungleich.ch/ungleich-public/cdist-contrib).
+
 ## Participating
 
 IRC: ``#cdist`` @ freenode

From 515992249de513492a725dbf4072a6c3f376668a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 27 Apr 2020 22:55:57 +0200
Subject: [PATCH 056/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 7cacf3cf..1e213187 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,6 +10,8 @@ next:
 	* New type: __pf_apply_anchor (Kamila Součková, Evil Ham)
 	* Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham)
 	* Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham)
+	* Configuration: Add notes to cdist.cfg.skeleton (Evil Ham)
+	* Explorers cpu_cores, memory: Improve *BSD support (Evil Ham)
 
 6.5.4: 2020-04-11
 	* Explorer init: Do not grep on non-existent init (Steven Armstrong)

From ea3bd14d8b377818a16578bd5032a853188baeec Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 28 Apr 2020 14:54:51 +0200
Subject: [PATCH 057/527] [logging] Mute warning on return_output=True when
 running scripts.

This fixes #806 which contains more information about the issue.

The TL;DR: this warning is not being useful and hinders debugging types because
it creates an innecessary line for each explorer.

An alternative proposal was #807 but was abandoned in favour of just dropping
the warning.
---
 cdist/exec/remote.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py
index e0ef66ec..f72bf3bf 100644
--- a/cdist/exec/remote.py
+++ b/cdist/exec/remote.py
@@ -280,9 +280,6 @@ class Remote(object):
         assert isinstance(command, (list, tuple)), (
                 "list or tuple argument expected, got: %s" % command)
 
-        if return_output and stdout is not subprocess.PIPE:
-            self.log.debug("return_output is True, ignoring stdout")
-
         close_stdout = False
         close_stderr = False
         if self.save_output_streams:

From 250161e42d1fe32bcb7b6945014aa4786d3b46ed Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 28 Apr 2020 23:08:03 +0200
Subject: [PATCH 058/527] ++

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 1e213187..c29f70de 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,7 @@ next:
 	* Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham)
 	* Configuration: Add notes to cdist.cfg.skeleton (Evil Ham)
 	* Explorers cpu_cores, memory: Improve *BSD support (Evil Ham)
+	* Core: Remove debug logging noise (Evil Ham)
 
 6.5.4: 2020-04-11
 	* Explorer init: Do not grep on non-existent init (Steven Armstrong)

From 310045d9fb5a5a63a337867a90009f89e0ded5ed Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 1 May 2020 13:02:00 +0200
Subject: [PATCH 059/527] Release 6.5.5

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index c29f70de..e6a7927e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)
 	* Type __postgres_database: Add encoding, lc-collate, lc-ctype, template parameters (Timothée Floure)
 	* Type __motd: Improve documentation and support for FreeBSD (Evil Ham)

From f58d662b32fb9c1c226b5340ccd06c6dd3e2a3f9 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 1 May 2020 15:28:01 +0200
Subject: [PATCH 060/527] [__pyvenv] Switch to python3 -m venv for ubuntu

---
 cdist/conf/type/__pyvenv/gencode-remote | 3 ++-
 cdist/conf/type/__pyvenv/man.rst        | 5 ++---
 docs/changelog                          | 4 ++++
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__pyvenv/gencode-remote b/cdist/conf/type/__pyvenv/gencode-remote
index 9c7b7fab..c5b64eff 100755
--- a/cdist/conf/type/__pyvenv/gencode-remote
+++ b/cdist/conf/type/__pyvenv/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2016 Darko Poljak (darko.poljak at gmail.com)
+# 2020 Nico Schotetlius (nico.schottelius at ungleich.ch)
 #
 # This file is part of cdist.
 #
@@ -45,7 +46,7 @@ then
     pyvenv=$(cat "$pyvenvparam")
 else
     case "$os" in
-        alpine) # no pyvenv on alpine - I assume others will follow
+        alpine|ubuntu) # no pyvenv on alpine - I assume others will follow
             pyvenv="python3 -m venv"
             ;;
         *)
diff --git a/cdist/conf/type/__pyvenv/man.rst b/cdist/conf/type/__pyvenv/man.rst
index d7de92fa..8085ff12 100644
--- a/cdist/conf/type/__pyvenv/man.rst
+++ b/cdist/conf/type/__pyvenv/man.rst
@@ -9,7 +9,7 @@ cdist-type__pyvenv - Create or remove python virtual environment
 DESCRIPTION
 -----------
 This cdist type allows you to create or remove python virtual
-environment using pyvenv.
+environment using pyvenv on python3 -m venv.
 It assumes pyvenv is already installed. Concrete package depends
 on concrete OS and/or OS version/distribution.
 Ensure this for e.g. in your init manifest as in the following example:
@@ -57,7 +57,7 @@ EXAMPLES
 
     __pyvenv /home/services/djangoenv
 
-    # Use specific pyvenv 
+    # Use specific pyvenv
     __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4
 
     # Create python virtualenv for user foo.
@@ -76,4 +76,3 @@ COPYING
 -------
 Copyright \(C) 2016 Darko Poljak. Free use of this software is
 granted under the terms of the GNU General Public License v3 or later (GPLv3+).
-
diff --git a/docs/changelog b/docs/changelog
index e6a7927e..4ee47d37 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)
+
+
 6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)
 	* Type __postgres_database: Add encoding, lc-collate, lc-ctype, template parameters (Timothée Floure)

From d4059fd29ecb93e094c411bb8ca09a234904b763 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 1 May 2020 15:31:23 +0200
Subject: [PATCH 061/527] [__letsencrypt_cert] whitelist Ubuntu

---
 cdist/conf/type/__letsencrypt_cert/manifest | 3 +++
 docs/changelog                              | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 68ecf9d4..b4464366 100755
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -91,6 +91,9 @@ if [ -z "${certbot_fullpath}" ]; then
 
 			certbot_fullpath=/usr/local/bin/certbot
 		;;
+		ubuntu)
+            __package certbot
+            ;;
 		*)
 			echo "Unsupported os: $os" >&2
 			exit 1
diff --git a/docs/changelog b/docs/changelog
index 4ee47d37..e61933ab 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,7 +3,7 @@ Changelog
 
 next:
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)
-
+	* Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius)
 
 6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)

From 6f4649efc69593fb4178fb6c2c5c95bbd93054d1 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 8 May 2020 16:08:21 +0200
Subject: [PATCH 062/527] Reference the new cdist chat on matrix

---
 docs/src/cdist-support.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/cdist-support.rst b/docs/src/cdist-support.rst
index 19afde2f..f9f61f01 100644
--- a/docs/src/cdist-support.rst
+++ b/docs/src/cdist-support.rst
@@ -3,7 +3,7 @@ Support
 
 Chat
 ~~~~
-Chat with us: `ungleich chat <https://chat.ungleich.ch/ungleich/channels/cdist>`_.
+Chat with us on `#cdist:ungleich.ch <https://ungleich.ch/u/projects/open-chat/>`_.
 
 Mailing list
 ~~~~~~~~~~~~

From 42f2dceeb12fee55616202bf085e6823d18ee714 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 9 May 2020 18:40:44 +0200
Subject: [PATCH 063/527] [__link] fix typo in the manual

---
 cdist/conf/type/__link/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__link/man.rst b/cdist/conf/type/__link/man.rst
index fe0ce425..2e81aea9 100644
--- a/cdist/conf/type/__link/man.rst
+++ b/cdist/conf/type/__link/man.rst
@@ -18,7 +18,7 @@ source
    Specifies the link source.
 
 type
-   Specifies the link type: Either hard or symoblic.
+   Specifies the link type: Either hard or symbolic.
 
 
 OPTIONAL PARAMETERS

From dcfabf9268865794602d3e59af2f312a1b0392f0 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 16 May 2020 14:00:02 +0200
Subject: [PATCH 064/527] [__cron] add hint for default values of time-related
 parameters

This should resolve some misunderstanding when leave a time-related
parameter with it's default value.
---
 cdist/conf/type/__cron/man.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__cron/man.rst b/cdist/conf/type/__cron/man.rst
index d0694738..e39bfb5c 100644
--- a/cdist/conf/type/__cron/man.rst
+++ b/cdist/conf/type/__cron/man.rst
@@ -21,6 +21,11 @@ command
 
 OPTIONAL PARAMETERS
 -------------------
+**NOTE**: All time-related parameters (``--minute``, ``--hour``, ``--day_of_month``
+``--month`` and ``--day_of_week``) defaults to ``*``, which means to execute it
+**always**. If you set ``--hour 0`` to execute the cronjob only at midnight, it
+will execute **every** minute in the first hour of the morning all days.
+
 state
    Either present or absent. Defaults to present.
 minute

From 086e683c997729817300c600307592c6fc02ffd8 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 16 May 2020 16:33:38 +0200
Subject: [PATCH 065/527] [__file] add "default values" in manual for group,
 mode and owner

Because at least the --mode default value may not be expected, the
manual lists the "default values". As they are not explicitly set in the
`__file` type, it is a bit more difficult, but should resolve to
following:
  - the mode comes from the umask set in the cdist code to protect file
    creation and uses the strongest umask possible.
  - the owner and group comes from the ssh user, which should always be
    the root user. (I think this can be swaped, too, but who will?)

At the owner and group parameter, it could be replaced with something
like "the ssh user and group", which would be more correct, but less
understandable.
---
 cdist/conf/type/__file/man.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__file/man.rst b/cdist/conf/type/__file/man.rst
index 7a0603bb..2f3b9e69 100644
--- a/cdist/conf/type/__file/man.rst
+++ b/cdist/conf/type/__file/man.rst
@@ -50,13 +50,13 @@ state
       create or modify it
 
 group
-   Group to chgrp to.
+   Group to chgrp to. Defaults to ``root``.
 
 mode
-   Unix permissions, suitable for chmod.
+   Unix permissions, suitable for chmod. Defaults to a very secure ``0600``.
 
 owner
-   User to chown to.
+   User to chown to. Defaults to ``root``.
 
 source
    If supplied, copy this file from the host running cdist to the target.

From 7ca2bfc14a8ed0cb73b3a01c85a562060264d02b Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 18 May 2020 16:00:23 +0200
Subject: [PATCH 066/527] [explorer/machine_type] Add support for FreeBSD.

More research is needed for {Net,Open}BSD support.

Indentation is left as-is for the linux code as I intend to simplify it in a
future MR, this way the diff is minimal.
---
 cdist/conf/explorer/machine_type | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index bb21f69c..fe0ae7d5 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -2,6 +2,7 @@
 #
 # 2014 Daniel Heule  (hda at sfs.biz)
 # 2014 Thomas Oettli (otho at sfs.biz)
+# 2020 Evilham (contact at evilham.com)
 #
 # This file is part of cdist.
 #
@@ -18,9 +19,27 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
 
-# FIXME: other system types (not linux ...)
+os=$("$__explorer/os")
+
+case "$os" in
+    "freebsd")
+        # FreeBSD does not have /proc/cpuinfo even when procfs is used.
+        # Instead there is a sysctl kern.vm_guest.
+        # Which is 'none' if physical, else the virtualisation.
+        vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)"
+        if [ -n "${vm_guest}" ]; then
+            if [ "${vm_guest}" = "none" ]; then
+                echo "physical"
+                exit
+            fi
+            echo "virtual_by_${vm_guest}"
+            exit
+        fi
+    ;;
+
+    *)
+        # Defaulting to linux for compatibility with previous cdist behaviour
 
 if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then
     echo openvz
@@ -72,9 +91,13 @@ if [ -r /proc/cpuinfo ]; then
             fi                        
         fi
         echo "virtual_by_unknown"
+        exit
     else
         echo "physical"
+        exit
     fi
-else
-    echo "unknown"
 fi
+    ;;
+esac
+
+echo "unknown"

From cf44c4a01bcfba95e11ba8d6aa64156c5470adc1 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 18 May 2020 16:35:50 +0200
Subject: [PATCH 067/527] [explorer/machine_type] Simplify Linux + basic
 OpenBSD support.

By abstracting away vendor-dependent pattern matching for the linux code, we can
re-use that and be reasonably sure about OpenBSD machines being virtualised when
we can identify the undelrying virtualisation technology.
It remains to be solved how to tell if an OpenBSD machine is physical; in that
case previous cdist behaviour ("unknown") remains.

For NetBSD something similar to OpenBSD could be done, with different sysctls:
hw.machine and hw.model wary of adding those without testing though, so for
NetBSD previous cdist behaviour ("unknown") remains.
https://netbsd.gw.com/cgi-bin/man-cgi?sysctl+7.i386+NetBSD-9.0
---
 cdist/conf/explorer/machine_type | 116 ++++++++++++++++---------------
 1 file changed, 61 insertions(+), 55 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index fe0ae7d5..6800234d 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -22,6 +22,18 @@
 
 os=$("$__explorer/os")
 
+vendor_string_to_machine_type() {
+    for vendor in vmware bochs kvm qemu virtualbox bhyve; do
+        if echo "${1}" | grep -q -i "${vendor}"; then
+            if [ "${vendor}" = "bochs" -o "${vendor}" = "qemu" ]; then
+                vendor="kvm"
+            fi
+            echo "virtual_by_${vendor}"
+            exit
+        fi
+    done
+}
+
 case "$os" in
     "freebsd")
         # FreeBSD does not have /proc/cpuinfo even when procfs is used.
@@ -38,65 +50,59 @@ case "$os" in
         fi
     ;;
 
+    "openbsd")
+        # OpenBSD can also use the sysctl's: hw.vendor or hw.product.
+        # Note we can be reasonably sure about a machine being virtualised
+        # as long as we can identify the virtualisation technology.
+        # But not so much about it being physical...
+        # Patches are welcome / reach out if you have better ideas.
+        for sysctl in hw.vendor hw.product; do
+            # This exits if we can make a reasonable judgement
+            vendor_string_to_machine_type "$(sysctl -n "${sysctl}")"
+        done
+    ;;
+
     *)
         # Defaulting to linux for compatibility with previous cdist behaviour
 
-if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then
-    echo openvz
-    exit
-fi
-
-if [ -e "/proc/1/environ" ] &&
-    tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then
-    echo lxc
-    exit
-fi
-
-if [ -r /proc/cpuinfo ]; then
-    # this should only exist on virtual guest machines,
-    # tested on vmware, xen, kvm
-    if grep -q "hypervisor" /proc/cpuinfo; then
-        # this file is aviable in xen guest systems
-        if [ -r /sys/hypervisor/type ]; then
-            if grep -q -i "xen" /sys/hypervisor/type; then
-                echo virtual_by_xen
-                exit 
-            fi
-        else
-            if [ -r /sys/class/dmi/id/product_name ]; then
-                if grep -q -i 'vmware' /sys/class/dmi/id/product_name; then
-                    echo "virtual_by_vmware"
-                    exit
-                elif grep -q -i 'bochs' /sys/class/dmi/id/product_name; then
-                    echo "virtual_by_kvm"
-                    exit 
-                elif grep -q -i 'virtualbox' /sys/class/dmi/id/product_name; then
-                    echo "virtual_by_virtualbox"
-                    exit 
-                fi
-            fi
-
-            if [ -r /sys/class/dmi/id/sys_vendor ]; then
-                if grep -q -i 'qemu' /sys/class/dmi/id/sys_vendor; then
-                    echo "virtual_by_kvm"
-                    exit
-                fi
-            fi
-
-            if [ -r /sys/class/dmi/id/chassis_vendor ]; then
-                if grep -q -i 'qemu' /sys/class/dmi/id/chassis_vendor; then
-                    echo "virtual_by_kvm"
-                    exit
-                fi
-            fi                        
+        if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then
+            echo openvz
+            exit
+        fi
+
+        if [ -e "/proc/1/environ" ] &&
+            tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then
+            echo lxc
+            exit
+        fi
+
+        if [ -r /proc/cpuinfo ]; then
+            # this should only exist on virtual guest machines,
+            # tested on vmware, xen, kvm, bhyve
+            if grep -q "hypervisor" /proc/cpuinfo; then
+                # this file is aviable in xen guest systems
+                if [ -r /sys/hypervisor/type ]; then
+                    if grep -q -i "xen" /sys/hypervisor/type; then
+                        echo virtual_by_xen
+                        exit
+                    fi
+                else
+                    for vendor_file in /sys/class/dmi/id/product_name \
+                                       /sys/class/dmi/id/sys_vendor \
+                                       /sys/class/dmi/id/chasis_vendor; do
+                        if [ -r ${vendor_file} ]; then
+                            # This exits if we can make a reasonable judgement
+                            vendor_string_to_machine_type "$(cat "${vendor_file}")"
+                        fi
+                    done
+                fi
+                echo "virtual_by_unknown"
+                exit
+            else
+                echo "physical"
+                exit
+            fi
         fi
-        echo "virtual_by_unknown"
-        exit
-    else
-        echo "physical"
-        exit
-    fi
-fi
     ;;
 esac
 

From 8b790b0a54aecf93298c97948c48be80b67ca7b5 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 18 May 2020 16:47:20 +0200
Subject: [PATCH 068/527] [explorer/machine_type] Make shellcheck happy!

---
 cdist/conf/explorer/machine_type | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 6800234d..1c84f4d7 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -25,7 +25,7 @@ os=$("$__explorer/os")
 vendor_string_to_machine_type() {
     for vendor in vmware bochs kvm qemu virtualbox bhyve; do
         if echo "${1}" | grep -q -i "${vendor}"; then
-            if [ "${vendor}" = "bochs" -o "${vendor}" = "qemu" ]; then
+            if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then
                 vendor="kvm"
             fi
             echo "virtual_by_${vendor}"

From 6d502f737aac8a836949d2b4b8a55c8ba20161fa Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 18 May 2020 18:37:16 +0200
Subject: [PATCH 069/527] [__ssh_authorizedkey{,s}] Improve documentation.

---
 cdist/conf/type/__ssh_authorized_key/man.rst  | 16 +++++-----
 cdist/conf/type/__ssh_authorized_keys/man.rst | 29 ++++++++++---------
 2 files changed, 25 insertions(+), 20 deletions(-)

diff --git a/cdist/conf/type/__ssh_authorized_key/man.rst b/cdist/conf/type/__ssh_authorized_key/man.rst
index 087a3dae..5bae02aa 100644
--- a/cdist/conf/type/__ssh_authorized_key/man.rst
+++ b/cdist/conf/type/__ssh_authorized_key/man.rst
@@ -15,25 +15,27 @@ This type was created to be used by the __ssh_authorized_keys type.
 REQUIRED PARAMETERS
 -------------------
 file
-   the authorized_keys file to which the given key should be added
+   The authorized_keys file where the given key should be managed.
 
 key
-   a string containing the ssh keytype, base 64 encoded key and optional
-   trailing comment which shall be added to the given authorized_keys file.
+   The ssh key which shall be managed in this authorized_keys file.
+   Must be a string containing the ssh keytype, base 64 encoded key and
+   optional trailing comment which shall be added to the given
+   authorized_keys file.
 
 
 OPTIONAL PARAMETERS
 -------------------
 comment
-   explicit comment instead of the one which may be trailing the given key
+   Use this comment instead of the one which may be trailing in the key.
 
 option
-   an option to set for this authorized_key entry.
+   An option to set for this authorized_key entry.
    Can be specified multiple times.
    See sshd(8) for available options.
 
 state
-   if the given keys should be 'present' or 'absent', defaults to 'present'.
+   If the managed key should be 'present' or 'absent', defaults to 'present'.
 
 
 MESSAGES
@@ -64,7 +66,7 @@ EXAMPLES
 
 SEE ALSO
 --------
-:strong:`cdist__ssh_authorized_keys`\ (7), :strong:`sshd`\ (8)
+:strong:`cdist-type__ssh_authorized_keys`\ (7), :strong:`sshd`\ (8)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__ssh_authorized_keys/man.rst b/cdist/conf/type/__ssh_authorized_keys/man.rst
index ba310ff9..93357b1d 100644
--- a/cdist/conf/type/__ssh_authorized_keys/man.rst
+++ b/cdist/conf/type/__ssh_authorized_keys/man.rst
@@ -20,42 +20,45 @@ then left to the user to ensure that the file exists and that ownership and
 permissions work with ssh.
 
 
-REQUIRED PARAMETERS
--------------------
+REQUIRED MULTIPLE PARAMETERS
+----------------------------
 key
-   the ssh key which shall be added to this authorized_keys file.
-   Must be a string and can be specified multiple times.
+   An ssh key which shall be managed in this authorized_keys file.
+   Must be a string containing the ssh keytype, base 64 encoded key and
+   optional trailing comment which shall be added to the given
+   authorized_keys file.
+   Can be specified multiple times.
 
 
 OPTIONAL PARAMETERS
 -------------------
 comment
-   explicit comment instead of the one which may be trailing the given key
+   Use this comment instead of the one which may be trailing in each key.
 
 file
-   an alternative destination file, defaults to ~$owner/.ssh/authorized_keys
+   An alternative destination file, defaults to ~$owner/.ssh/authorized_keys.
 
 option
-   an option to set for all created authorized_key entries.
+   An option to set for all authorized_key entries in the key parameter.
    Can be specified multiple times.
    See sshd(8) for available options.
 
 owner
-   the user owning the authorized_keys file, defaults to object_id.
+   The user owning the authorized_keys file, defaults to object_id.
 
 state
-   if the given keys should be 'present' or 'absent', defaults to 'present'.
+   If the given keys should be 'present' or 'absent', defaults to 'present'.
 
 
 BOOLEAN PARAMETERS
 ------------------
 noparent
-   don't create or change ownership and permissions of the directory containing
-   the authorized_keys file
+   Don't create or change ownership and permissions of the directory containing
+   the authorized_keys file.
 
 nofile
-   don't manage existence, ownership and permissions of the the authorized_keys
-   file
+   Don't manage existence, ownership and permissions of the the authorized_keys
+   file.
 
 
 EXAMPLES

From f9afac4dd617c5ee7934cdb03477e6a30ae70af7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 18 May 2020 19:51:08 +0200
Subject: [PATCH 070/527] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index e61933ab..013e0083 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,9 @@ Changelog
 next:
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)
 	* Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius)
+	* Types __cron, __file, __link: Improve manpages (Matthias Stecher)
+	* Explorer machine_type: Add support for FreeBSD and OpenBSD, and simplify Linux code (Evil Ham)
+	* Type __ssh_authorized_key, __ssh_authorized_keys: Improve manpages (Evil Ham)
 
 6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)

From ad58ea79c2cd3edd11a828d26a28975aa332bdd0 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 18 May 2020 20:01:01 +0200
Subject: [PATCH 071/527] [__ssh_authorized_keys] Fix bug where --option was
 not multiple

This went against both documentation and intent.
---
 cdist/conf/type/__ssh_authorized_keys/parameter/optional         | 1 -
 .../conf/type/__ssh_authorized_keys/parameter/optional_multiple  | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)
 create mode 100644 cdist/conf/type/__ssh_authorized_keys/parameter/optional_multiple

diff --git a/cdist/conf/type/__ssh_authorized_keys/parameter/optional b/cdist/conf/type/__ssh_authorized_keys/parameter/optional
index 21f9bc29..fa64fc43 100644
--- a/cdist/conf/type/__ssh_authorized_keys/parameter/optional
+++ b/cdist/conf/type/__ssh_authorized_keys/parameter/optional
@@ -1,5 +1,4 @@
 comment
 file
-option
 owner
 state
diff --git a/cdist/conf/type/__ssh_authorized_keys/parameter/optional_multiple b/cdist/conf/type/__ssh_authorized_keys/parameter/optional_multiple
new file mode 100644
index 00000000..01925a15
--- /dev/null
+++ b/cdist/conf/type/__ssh_authorized_keys/parameter/optional_multiple
@@ -0,0 +1 @@
+option

From f354d80308b1f048416b2d52a397035a93a8ee03 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 18 May 2020 21:03:29 +0200
Subject: [PATCH 072/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 013e0083..782d9bd7 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Types __cron, __file, __link: Improve manpages (Matthias Stecher)
 	* Explorer machine_type: Add support for FreeBSD and OpenBSD, and simplify Linux code (Evil Ham)
 	* Type __ssh_authorized_key, __ssh_authorized_keys: Improve manpages (Evil Ham)
+	* Type __ssh_authorized_keys: Fix bug where --option was not multiple (Evil Ham)
 
 6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)

From bf25a18a048ae3e3d1fd4a450d77fe516c18c861 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 22 May 2020 02:31:38 +0300
Subject: [PATCH 073/527] [__group] fix --gid on freebsd

---
 cdist/conf/type/__group/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__group/gencode-remote b/cdist/conf/type/__group/gencode-remote
index 6091c548..ff63e218 100755
--- a/cdist/conf/type/__group/gencode-remote
+++ b/cdist/conf/type/__group/gencode-remote
@@ -88,7 +88,7 @@ if [ "$state" = "present" ]; then
          fi
       done
       if [ "$os" = "freebsd" ]; then
-         echo pw groupadd "$@" "$name"
+         echo pw groupadd "$name" "$@"
       else
          echo groupadd "$@" "$name"
       fi

From 3bcbd952698cd5a4b8333551660ba308e4f30d7b Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 22 May 2020 02:36:49 +0300
Subject: [PATCH 074/527] [__motd] debian|ubuntu|devuan use /etc/motd

---
 cdist/conf/type/__motd/gencode-remote | 7 -------
 cdist/conf/type/__motd/manifest       | 4 ----
 2 files changed, 11 deletions(-)

diff --git a/cdist/conf/type/__motd/gencode-remote b/cdist/conf/type/__motd/gencode-remote
index 738ea834..cb7bfc84 100755
--- a/cdist/conf/type/__motd/gencode-remote
+++ b/cdist/conf/type/__motd/gencode-remote
@@ -22,13 +22,6 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-    debian|ubuntu|devuan)
-
-        # Debian and Ubuntu need to be updated,
-        # as seen in /etc/init.d/bootlogs
-        echo "uname -snrvm > /var/run/motd"
-        echo "cat /etc/motd.tail >> /var/run/motd"
-    ;;
     freebsd)
         # FreeBSD only updates /etc/motd on boot,
 	# as seen in /etc/rc.d/motd
diff --git a/cdist/conf/type/__motd/manifest b/cdist/conf/type/__motd/manifest
index ded734d7..b8f74ebf 100755
--- a/cdist/conf/type/__motd/manifest
+++ b/cdist/conf/type/__motd/manifest
@@ -33,10 +33,6 @@ os=$(cat "$__global/explorer/os")
 
 
 case "$os" in
-   debian|ubuntu|devuan)
-      # Debian-based systems use /etc/motd.tail as a template
-      destination=/etc/motd.tail
-   ;;
    freebsd)
       # FreeBSD uses motd.template to prepend system information on boot
       # (this actually only applies starting with version 13,

From 226ed02c1c2556ed30e4f8aaf1946099bd590725 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 22 May 2020 10:09:05 +0200
Subject: [PATCH 075/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 782d9bd7..f0aa3aa7 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,8 @@ next:
 	* Explorer machine_type: Add support for FreeBSD and OpenBSD, and simplify Linux code (Evil Ham)
 	* Type __ssh_authorized_key, __ssh_authorized_keys: Improve manpages (Evil Ham)
 	* Type __ssh_authorized_keys: Fix bug where --option was not multiple (Evil Ham)
+	* Type __motd: Debian/Ubuntu/Devuan use /etc/motd (Ander Punnar)
+	* Type __group: Fix --gid on FreeBSD (Ander Punnar)
 
 6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)

From 66f4421089660fc0a083abe3d2c93470eacd55ce Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Fri, 22 May 2020 17:14:29 +0300
Subject: [PATCH 076/527] [docs] Fixed two typos in cdist.cfg.skeleton

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 configuration/cdist.cfg.skeleton | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton
index bfac9f5c..91c5ab02 100644
--- a/configuration/cdist.cfg.skeleton
+++ b/configuration/cdist.cfg.skeleton
@@ -26,7 +26,7 @@
 #
 # init_manifest
 #     Specify default initial manifest.
-# init_mainfest = <path-to-init-manifst>
+# init_manifest = <path-to-init-manifest>
 #
 # inventory_dir
 #     Specify inventory directory.

From 716d3554f3dd9a7cf83cb0fd4f706b6458cb13d8 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sat, 23 May 2020 00:48:35 +0300
Subject: [PATCH 077/527] [__user] fix user delete on freebsd

---
 cdist/conf/type/__user/gencode-remote | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__user/gencode-remote b/cdist/conf/type/__user/gencode-remote
index ee18c18f..41c3a57b 100755
--- a/cdist/conf/type/__user/gencode-remote
+++ b/cdist/conf/type/__user/gencode-remote
@@ -135,11 +135,19 @@ elif [ "$state" = "absent" ]; then
     if grep -q "^${name}:" "$__object/explorer/passwd"; then
         #user exists, but state != present, so delete it
         if [ -f "$__object/parameter/remove-home" ]; then
-            printf "userdel -r '%s' >/dev/null 2>&1\\n" "${name}"
-	    echo "userdel -r" >> "$__messages_out"
+            if [ "$os" = "freebsd" ]; then
+                printf "pw userdel '%s' -r >/dev/null 2>&1\\n" "${name}"
+            else
+                printf "userdel -r '%s' >/dev/null 2>&1\\n" "${name}"
+            fi
+            echo "userdel -r" >> "$__messages_out"
         else
-            printf "userdel '%s' >/dev/null 2>&1\\n" "${name}"
-	    echo "userdel" >> "$__messages_out"
+            if [ "$os" = "freebsd" ]; then
+                printf "pw userdel '%s' >/dev/null 2>&1\\n" "${name}"
+            else
+                printf "userdel '%s' >/dev/null 2>&1\\n" "${name}"
+            fi
+            echo "userdel" >> "$__messages_out"
         fi
     fi
 else

From 6ba73c4be6a57007c035f748c3273dd7a7041992 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 23 May 2020 19:03:13 +0200
Subject: [PATCH 078/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index f0aa3aa7..6746bec6 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,6 +10,8 @@ next:
 	* Type __ssh_authorized_keys: Fix bug where --option was not multiple (Evil Ham)
 	* Type __motd: Debian/Ubuntu/Devuan use /etc/motd (Ander Punnar)
 	* Type __group: Fix --gid on FreeBSD (Ander Punnar)
+	* Configuration: Fix typos in cdist.cfg.skeleton (Jaak Ristioja)
+	* Type __user: Fix user deletion on FreeBSD (Ander Punnar)
 
 6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)

From f4e1bbc87e9127ed4ac1789c700e1cbee02ff61c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sun, 24 May 2020 01:10:56 +0200
Subject: [PATCH 079/527] Fix multiple log lines

Fixes #813.
---
 cdist/log.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/log.py b/cdist/log.py
index 5d431130..2d0bef0b 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -64,6 +64,7 @@ class DefaultLog(logging.Logger):
 
     def __init__(self, name):
         super().__init__(name)
+        self.propagate = False
 
         formatter = logging.Formatter(self.FORMAT)
 

From ba64971a567722c29cfe5a93c7ccc7e03bbf7940 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sun, 24 May 2020 17:05:01 +0200
Subject: [PATCH 080/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 6746bec6..5a2f905a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,7 @@ next:
 	* Type __group: Fix --gid on FreeBSD (Ander Punnar)
 	* Configuration: Fix typos in cdist.cfg.skeleton (Jaak Ristioja)
 	* Type __user: Fix user deletion on FreeBSD (Ander Punnar)
+	* Core: Fix double log lines (Darko Poljak)
 
 6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)

From abac79d4a50857c81e12660aaa13424d371f2137 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 25 May 2020 11:16:48 +0200
Subject: [PATCH 081/527] Release 6.5.6

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 5a2f905a..ad008cb2 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)
 	* Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius)
 	* Types __cron, __file, __link: Improve manpages (Matthias Stecher)

From 29c0180204beced2813f2d1ec2b50ee069d205c6 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 28 May 2020 23:31:13 +0300
Subject: [PATCH 082/527] [__ssh_authorized_keys] add --remove-unknown
 parameter

---
 .../type/__ssh_authorized_keys/explorer/keys  |  8 ++++++
 cdist/conf/type/__ssh_authorized_keys/man.rst |  9 +++++++
 .../conf/type/__ssh_authorized_keys/manifest  | 27 ++++++++++++++++++-
 .../__ssh_authorized_keys/parameter/boolean   |  1 +
 4 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100755 cdist/conf/type/__ssh_authorized_keys/explorer/keys

diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
new file mode 100755
index 00000000..b515c630
--- /dev/null
+++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+file="$( . "$__type_explorer/file" )"
+
+if [ -f "$file" ]
+then
+    cat "$file"
+fi
diff --git a/cdist/conf/type/__ssh_authorized_keys/man.rst b/cdist/conf/type/__ssh_authorized_keys/man.rst
index 93357b1d..dac6adeb 100644
--- a/cdist/conf/type/__ssh_authorized_keys/man.rst
+++ b/cdist/conf/type/__ssh_authorized_keys/man.rst
@@ -60,6 +60,9 @@ nofile
    Don't manage existence, ownership and permissions of the the authorized_keys
    file.
 
+remove-unknown
+   Remove undefined keys.
+
 
 EXAMPLES
 --------
@@ -70,6 +73,12 @@ EXAMPLES
     __ssh_authorized_keys root \
        --key "$(cat ~/.ssh/id_rsa.pub)"
 
+    # same as above, but make sure your key is only key in
+    # root's authorized_keys file
+    __ssh_authorized_keys root \
+       --key "$(cat ~/.ssh/id_rsa.pub)" \
+       --remove-unknown
+
     # allow key to login as user-name
     __ssh_authorized_keys user-name \
        --key "ssh-rsa AXYZAAB3NzaC1yc2..."
diff --git a/cdist/conf/type/__ssh_authorized_keys/manifest b/cdist/conf/type/__ssh_authorized_keys/manifest
index b9f0582e..b319316b 100755
--- a/cdist/conf/type/__ssh_authorized_keys/manifest
+++ b/cdist/conf/type/__ssh_authorized_keys/manifest
@@ -55,8 +55,12 @@ _cksum() {
    echo "$1" | cksum | cut -d' ' -f 1
 }
 
+_type_and_key() {
+   echo "$1" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }'
+}
+
 while read -r key; do
-   type_and_key="$(echo "$key" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }')"
+   type_and_key="$( _type_and_key "$key" )"
    object_id="$(_cksum "$file")-$(_cksum "$type_and_key")"
    set -- "$object_id"
    set -- "$@" --file "$file"
@@ -72,3 +76,24 @@ while read -r key; do
    # Ensure __ssh_authorized_key does not read stdin
    __ssh_authorized_key "$@" < /dev/null
 done < "$__object/parameter/key"
+
+if [ -f "$__object/parameter/remove-unknown" ] &&
+    [ -s "$__object/explorer/keys" ]
+then
+    while read -r key
+    do
+        type_and_key="$( _type_and_key "$key" )"
+
+        if grep -Fq "$type_and_key" "$__object/parameter/key"
+        then
+            continue
+        fi
+
+        __ssh_authorized_key "remove-$( _cksum "$file$key" )" \
+            --file "$file" \
+            --key "$key" \
+            --state absent \
+                < /dev/null
+    done \
+        < "$__object/explorer/keys"
+fi
diff --git a/cdist/conf/type/__ssh_authorized_keys/parameter/boolean b/cdist/conf/type/__ssh_authorized_keys/parameter/boolean
index 4bb126fe..7388fed5 100644
--- a/cdist/conf/type/__ssh_authorized_keys/parameter/boolean
+++ b/cdist/conf/type/__ssh_authorized_keys/parameter/boolean
@@ -1,2 +1,3 @@
 noparent
 nofile
+remove-unknown

From b354ea6e9430816bd62168c4c8e7b936ad4a4885 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 30 May 2020 10:49:13 +0200
Subject: [PATCH 083/527] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index ad008cb2..76b5edfa 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)
+
 6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)
 	* Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius)

From 988190363aeff5fba9be80355c318e1ff372eda9 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 30 May 2020 15:10:13 +0200
Subject: [PATCH 084/527] Resolve shellcheck SC1090

---
 cdist/conf/type/__ssh_authorized_keys/explorer/keys | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
index b515c630..cec25746 100755
--- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys
+++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
@@ -1,5 +1,6 @@
 #!/bin/sh -e
 
+# shellcheck disable=SC1090
 file="$( . "$__type_explorer/file" )"
 
 if [ -f "$file" ]

From ba77ea9edcf2d99c6edd76adaa83017821babae3 Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Mon, 1 Jun 2020 19:11:58 +0200
Subject: [PATCH 085/527] [UX] Add option to enable LogLevel-based coloured
 output.

This makes it easier for new and experienced users to run cdist with higher
verbosity levels, both to know that things are working as expected and to debug
issues.

Documentation has been modified accordingly and default behaviour is not
changed.
---
 cdist/argparse.py                    | 31 ++++++++++++++++--
 cdist/configuration.py               | 11 +++++++
 cdist/core/manifest.py               |  1 +
 cdist/emulator.py                    |  3 ++
 cdist/log.py                         | 26 ++++++++++++++-
 cdist/test/configuration/__init__.py | 28 +++++++++++++++++
 configuration/cdist.cfg.skeleton     |  6 ++++
 docs/src/cdist-reference.rst.sh      |  5 +++
 docs/src/man1/cdist.rst              | 47 +++++++++++++++++++---------
 9 files changed, 140 insertions(+), 18 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 611c484a..c30e2030 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -5,6 +5,7 @@ import logging
 import collections
 import functools
 import cdist.configuration
+import cdist.log
 import cdist.preos
 import cdist.info
 
@@ -88,6 +89,13 @@ def check_lower_bounded_int(value, lower_bound, name):
     return val
 
 
+def colored_output_type(val):
+    boolean_states = cdist.configuration.ColoredOutputOption.BOOLEAN_STATES
+    if val not in boolean_states.keys():
+        raise argparse.ArgumentError()
+    return boolean_states[val]
+
+
 def get_parsers():
     global parser
 
@@ -125,6 +133,15 @@ def get_parsers():
                   'value.'),
             action='count', default=None)
 
+    parser['colored_output'] = argparse.ArgumentParser(add_help=False)
+    parser['colored_output'].add_argument(
+            '--colors',
+            help='Use a colored output for different log levels.'
+                 'It can be a boolean or "auto" (default) which enables this '
+                 'feature if stdout is a tty and disables it otherwise.',
+            action='store', dest='colored_output', required=False,
+            type=colored_output_type)
+
     parser['beta'] = argparse.ArgumentParser(add_help=False)
     parser['beta'].add_argument(
            '-b', '--beta',
@@ -283,6 +300,7 @@ def get_parsers():
             'host', nargs='*', help='Host(s) to operate on.')
     parser['config'] = parser['sub'].add_parser(
             'config', parents=[parser['loglevel'], parser['beta'],
+                               parser['colored_output'],
                                parser['common'],
                                parser['config_main'],
                                parser['inventory_common'],
@@ -301,6 +319,7 @@ def get_parsers():
 
     parser['add-host'] = parser['invsub'].add_parser(
             'add-host', parents=[parser['loglevel'], parser['beta'],
+                                 parser['colored_output'],
                                  parser['common'],
                                  parser['inventory_common']])
     parser['add-host'].add_argument(
@@ -315,6 +334,7 @@ def get_parsers():
 
     parser['add-tag'] = parser['invsub'].add_parser(
             'add-tag', parents=[parser['loglevel'], parser['beta'],
+                                parser['colored_output'],
                                 parser['common'],
                                 parser['inventory_common']])
     parser['add-tag'].add_argument(
@@ -346,6 +366,7 @@ def get_parsers():
 
     parser['del-host'] = parser['invsub'].add_parser(
             'del-host', parents=[parser['loglevel'], parser['beta'],
+                                 parser['colored_output'],
                                  parser['common'],
                                  parser['inventory_common']])
     parser['del-host'].add_argument(
@@ -363,6 +384,7 @@ def get_parsers():
 
     parser['del-tag'] = parser['invsub'].add_parser(
             'del-tag', parents=[parser['loglevel'], parser['beta'],
+                                parser['colored_output'],
                                 parser['common'],
                                 parser['inventory_common']])
     parser['del-tag'].add_argument(
@@ -398,6 +420,7 @@ def get_parsers():
 
     parser['list'] = parser['invsub'].add_parser(
             'list', parents=[parser['loglevel'], parser['beta'],
+                             parser['colored_output'],
                              parser['common'],
                              parser['inventory_common']])
     parser['list'].add_argument(
@@ -430,7 +453,7 @@ def get_parsers():
 
     # Shell
     parser['shell'] = parser['sub'].add_parser(
-            'shell', parents=[parser['loglevel']])
+            'shell', parents=[parser['loglevel'], parser['colored_output']])
     parser['shell'].add_argument(
             '-s', '--shell',
             help=('Select shell to use, defaults to current shell. Used shell'
@@ -495,9 +518,13 @@ def parse_and_configure(argv, singleton=True):
 
     log = logging.getLogger("cdist")
 
+    config = cfg.get_config()
+    if config.get('GLOBAL', {}).get('colored_output', False):
+        cdist.log.ColorFormatter.USE_COLORS = True
+
     log.verbose("version %s" % cdist.VERSION)
     log.trace('command line args: {}'.format(cfg.command_line_args))
-    log.trace('configuration: {}'.format(cfg.get_config()))
+    log.trace('configuration: {}'.format(config))
     log.trace('configured args: {}'.format(args))
 
     check_beta(vars(args))
diff --git a/cdist/configuration.py b/cdist/configuration.py
index 1011a382..6f07c27f 100644
--- a/cdist/configuration.py
+++ b/cdist/configuration.py
@@ -27,6 +27,7 @@ import cdist.argparse
 import re
 import multiprocessing
 import logging
+import sys
 
 
 class Singleton(type):
@@ -47,6 +48,7 @@ _VERBOSITY_VALUES = (
 _ARCHIVING_VALUES = (
     'tar', 'tgz', 'tbz2', 'txz', 'none',
 )
+_COLORED_OUTPUT_DEFAULT = sys.stdout.isatty()
 
 
 class OptionBase:
@@ -246,9 +248,15 @@ class LogLevelOption(OptionBase):
         return VerbosityOption().translate(val)
 
 
+class ColoredOutputOption(BooleanOption):
+    BOOLEAN_STATES = dict(configparser.ConfigParser.BOOLEAN_STATES,
+                          auto=_COLORED_OUTPUT_DEFAULT)
+
+
 _ARG_OPTION_MAPPING = {
     'beta': 'beta',
     'cache_path_pattern': 'cache_path_pattern',
+    'colored_output': 'colored_output',
     'conf_dir': 'conf_dir',
     'manifest': 'init_manifest',
     'out_path': 'out_path',
@@ -294,6 +302,7 @@ class Configuration(metaclass=Singleton):
             'remote_shell': StringOption('remote_shell'),
             'cache_path_pattern': StringOption('cache_path_pattern'),
             'conf_dir': ConfDirOption(),
+            'colored_output': ColoredOutputOption('colored_output'),
             'init_manifest': StringOption('init_manifest'),
             'out_path': StringOption('out_path'),
             'remote_out_path': StringOption('remote_out_path'),
@@ -319,6 +328,7 @@ class Configuration(metaclass=Singleton):
         'CDIST_REMOTE_COPY': 'remote_copy',
         'CDIST_INVENTORY_DIR': 'inventory_dir',
         'CDIST_CACHE_PATH_PATTERN': 'cache_path_pattern',
+        'CDIST_COLORED_OUTPUT': 'colored_output',
         '__cdist_log_level': 'verbosity',
     }
     ENV_VAR_BOOLEAN_OPTIONS = ('CDIST_BETA', )
@@ -332,6 +342,7 @@ class Configuration(metaclass=Singleton):
     }
     REQUIRED_DEFAULT_CONFIG_VALUES = {
         'GLOBAL': {
+            'colored_output': _COLORED_OUTPUT_DEFAULT,
             'verbosity': 0,
         },
     }
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 8aeaf860..32520e49 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -119,6 +119,7 @@ class Manifest(object):
             '__cdist_log_level': util.log_level_env_var_val(self.log),
             '__cdist_log_level_name': util.log_level_name_env_var_val(
                 self.log),
+            '__cdist_colored_log': str(cdist.log.ColorFormatter.USE_COLORS),
         }
 
         if dry_run:
diff --git a/cdist/emulator.py b/cdist/emulator.py
index 4800e2a3..87c9fe12 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -129,6 +129,9 @@ class Emulator(object):
             # if invalid __cdist_log_level value
             logging.root.setLevel(logging.WARNING)
 
+        colored_log = self.env.get('__cdist_colored_log', 'False')
+        cdist.log.ColorFormatter.USE_COLORS = colored_log == 'True'
+
         self.log = logging.getLogger(self.target_host[0])
 
     def commandline(self):
diff --git a/cdist/log.py b/cdist/log.py
index 2d0bef0b..5f2d8f53 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -50,6 +50,30 @@ def _trace(msg, *args, **kwargs):
 logging.trace = _trace
 
 
+class ColorFormatter(logging.Formatter):
+    USE_COLORS = False
+    RESET = '\033[0m'
+    COLOR_MAP = {
+        'ERROR': '\033[0;31m',
+        'WARNING': '\033[0;33m',
+        'INFO': '\033[0;94m',
+        'VERBOSE': '\033[0;34m',
+        'DEBUG': '\033[0;90m',
+        'TRACE': '\033[0;37m',
+    }
+
+    def __init__(self, msg):
+        super().__init__(msg)
+
+    def format(self, record):
+        msg = super().format(record)
+        if self.USE_COLORS:
+            color = self.COLOR_MAP.get(record.levelname)
+            if color:
+                msg = color + msg + self.RESET
+        return msg
+
+
 class DefaultLog(logging.Logger):
 
     FORMAT = '%(levelname)s: %(message)s'
@@ -66,7 +90,7 @@ class DefaultLog(logging.Logger):
         super().__init__(name)
         self.propagate = False
 
-        formatter = logging.Formatter(self.FORMAT)
+        formatter = ColorFormatter(self.FORMAT)
 
         self.addFilter(self)
 
diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py
index 182868a6..07a73bda 100644
--- a/cdist/test/configuration/__init__.py
+++ b/cdist/test/configuration/__init__.py
@@ -28,10 +28,12 @@ import argparse
 from cdist import test
 import cdist.argparse as cap
 import logging
+import sys
 
 my_dir = op.abspath(op.dirname(__file__))
 fixtures = op.join(my_dir, 'fixtures')
 interpolation_config_file = op.join(fixtures, "interpolation-test.cfg")
+colored_output_default = sys.stdout.isatty()
 
 
 def newConfigParser():
@@ -153,6 +155,7 @@ class ConfigurationTestCase(test.CdistTestCase):
             'remote_shell': '/bin/sh',
             'inventory_dir': '',
             'cache_path_pattern': '',
+            'colored_output': colored_output_default,
             'conf_dir': '',
             'init_manifest': '',
             'out_path': '',
@@ -184,6 +187,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/bin/sh',
                 'inventory_dir': None,
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': None,
                 'init_manifest': None,
                 'out_path': None,
@@ -390,6 +394,7 @@ class ConfigurationTestCase(test.CdistTestCase):
         args = argparse.Namespace()
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'verbosity': 0,
             },
         }
@@ -440,6 +445,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/bin/sh',
                 'inventory_dir': None,
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': None,
                 'init_manifest': None,
                 'out_path': None,
@@ -515,6 +521,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': '/var/db/cdist/inventory',
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': ['/opt/cdist', ],
                 'init_manifest': None,
                 'out_path': None,
@@ -556,6 +563,7 @@ class ConfigurationTestCase(test.CdistTestCase):
             'remote_shell': '/bin/sh',
             'inventory_dir': '',
             'cache_path_pattern': '',
+            'colored_output': colored_output_default,
             'conf_dir': '',
             'init_manifest': '',
             'out_path': '',
@@ -579,6 +587,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': None,
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': [
                     '/opt/cdist/conf',
                     '/usr/local/share/cdist/conf',
@@ -623,6 +632,7 @@ class ConfigurationTestCase(test.CdistTestCase):
             'remote_shell': '/bin/sh',
             'inventory_dir': '',
             'cache_path_pattern': '',
+            'colored_output': colored_output_default,
             'conf_dir': '',
             'init_manifest': '',
             'out_path': '',
@@ -645,6 +655,7 @@ class ConfigurationTestCase(test.CdistTestCase):
             'local_shell': '/usr/bin/sh',
             'remote_shell': '/usr/bin/sh',
             'inventory_dir': '/var/db/cdist/inventory',
+            'colored_output': colored_output_default,
             'conf_dir': '/opt/cdist',
             'remote_copy': 'myscp',
             'remote_exec': 'myexec',
@@ -663,6 +674,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': '/var/db/cdist/inventory',
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': [
                     '/opt/cdist/conf',
                     '/usr/local/share/cdist/conf',
@@ -694,6 +706,7 @@ class ConfigurationTestCase(test.CdistTestCase):
         }
         expected_config = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'verbosity': 0,
             },
         }
@@ -767,6 +780,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': '/opt/sysadmin/cdist/inventory',
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': [
                     '/opt/cdist/conf',
                     '/usr/local/share/cdist/conf',
@@ -865,6 +879,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': '/var/db/cdist/inventory',
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': [
                     '/opt/conf/cdist',
                 ],
@@ -964,6 +979,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': '/var/db/cdist/inventory',
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': [
                     '/opt/conf/cdist',
                 ],
@@ -1063,6 +1079,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': '/var/db/cdist/inventory',
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': [
                     '/opt/conf/cdist',
                 ],
@@ -1095,6 +1112,7 @@ class ConfigurationTestCase(test.CdistTestCase):
             'beta': True,
             'inventory_dir': '/var/db/cdist/inventory',
             'cache_path_pattern': None,
+            'colored_output': colored_output_default,
             'conf_dir': [
                 '/opt/conf/cdist',
             ],
@@ -1125,6 +1143,7 @@ class ConfigurationTestCase(test.CdistTestCase):
         expected_config_dict = {
             'GLOBAL': {
                 'inventory_dir': None,
+                'colored_output': colored_output_default,
                 'conf_dir': None,
                 'verbosity': 0,
             },
@@ -1148,6 +1167,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'verbosity': cap.VERBOSE_DEBUG,
             },
         }
@@ -1185,6 +1205,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'save_output_streams': True,
                 'verbosity': 0,
             },
@@ -1213,6 +1234,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'save_output_streams': False,
                 'verbosity': 0,
             },
@@ -1241,6 +1263,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'save_output_streams': False,
                 'verbosity': 0,
             },
@@ -1269,6 +1292,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'save_output_streams': False,
                 'verbosity': 0,
             },
@@ -1308,6 +1332,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'timestamp': True,
                 'verbosity': 0,
             },
@@ -1336,6 +1361,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'timestamp': True,
                 'verbosity': 0,
             },
@@ -1364,6 +1390,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'timestamp': False,
                 'verbosity': 0,
             },
@@ -1392,6 +1419,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'timestamp': False,
                 'verbosity': 0,
             },
diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton
index 91c5ab02..f2a09064 100644
--- a/configuration/cdist.cfg.skeleton
+++ b/configuration/cdist.cfg.skeleton
@@ -13,6 +13,12 @@
 #     Specify cache path pattern.
 # cache_path_pattern = %h
 #
+# colored_output
+#     Use a colored output for different log levels.
+#     It can be a boolean or 'auto' (default) which enables this feature if
+#     stdout is a tty and disables it otherwise.
+# colored_output = auto
+#
 # conf_dir
 #     List of configuration directories separated with the character conventionally
 #     used by the operating system to separate search path components (as in PATH),
diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh
index e77d98f6..3b997f63 100755
--- a/docs/src/cdist-reference.rst.sh
+++ b/docs/src/cdist-reference.rst.sh
@@ -344,6 +344,11 @@ CDIST_INVENTORY_DIR
 CDIST_BETA
     Enable beta functionalities.
 
+CDIST_COLORED_OUTPUT
+    Use a colored output for different log levels.
+    It can be a boolean or 'auto' (default) which enables this feature if
+    stdout is a tty and disables it otherwise.
+
 CDIST_CACHE_PATH_PATTERN
     Custom cache path pattern.
 eof
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 38248821..4c34c4b7 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -15,8 +15,9 @@ SYNOPSIS
 
     cdist banner [-h] [-l LOGLEVEL] [-q] [-v]
 
-    cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4]
-                 [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
+    cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b]
+                 [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6]
+                 [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
                  [-j [JOBS]] [-n] [-o OUT_PATH] [-P]
                  [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
                  [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
@@ -24,8 +25,9 @@ SYNOPSIS
                  [-p [HOST_MAX]] [-s] [-t]
                  [host [host ...]]
 
-    cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4]
-                  [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
+    cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b]
+                  [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6]
+                  [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
                   [-j [JOBS]] [-n] [-o OUT_PATH] [-P]
                   [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
                   [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
@@ -36,26 +38,29 @@ SYNOPSIS
     cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ...
 
     cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                             [-g CONFIG_FILE] [-I INVENTORY_DIR]
-                             [-f HOSTFILE]
+                             [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
+                             [-I INVENTORY_DIR] [-f HOSTFILE]
                              [host [host ...]]
 
     cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                            [-g CONFIG_FILE] [-I INVENTORY_DIR]
-                            [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
+                            [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
+                            [-I INVENTORY_DIR] [-f HOSTFILE] [-T TAGFILE]
+                            [-t TAGLIST]
                             [host [host ...]]
 
     cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                             [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a]
-                             [-f HOSTFILE]
+                             [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
+                             [-I INVENTORY_DIR] [-a] [-f HOSTFILE]
                              [host [host ...]]
 
     cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                            [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a]
-                            [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
+                            [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
+                            [-I INVENTORY_DIR] [-a] [-f HOSTFILE]
+                            [-T TAGFILE] [-t TAGLIST]
                             [host [host ...]]
 
-    cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE]
+    cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b]
+                         [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
                          [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t]
                          [host [host ...]]
 
@@ -84,9 +89,11 @@ SYNOPSIS
                                        [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
                                        target_dir
 
-    cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL]
+    cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [--colors COLORED_OUTPUT]
+                [-s SHELL]
 
-    cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern]
+    cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t]
+               [pattern]
 
 
 DESCRIPTION
@@ -104,6 +111,11 @@ All commands accept the following options:
 **-h, --help**
     Show the help screen.
 
+**--colors COLORED_OUTPUT**
+    Use a colored output for different log levels.It can
+    be a boolean or "auto" (default) which enables this
+    feature if stdout is a tty and disables it otherwise.
+
 **-l LOGLEVEL, --log-level LOGLEVEL**
     Set the specified verbosity level. The levels, in
     order from the lowest to the highest, are: ERROR (-1),
@@ -893,6 +905,11 @@ CDIST_BETA
 CDIST_CACHE_PATH_PATTERN
     Custom cache path pattern.
 
+CDIST_COLORED_OUTPUT
+    Use a colored output for different log levels.
+    It can be a boolean or 'auto' (default) which enables this feature if
+    stdout is a tty and disables it otherwise.
+
 CDIST_CONFIG_FILE
     Custom configuration file.
 

From 9a4e3488c2104ad5562b5ab9aad8120969df73a8 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 1 Jun 2020 19:17:02 +0200
Subject: [PATCH 086/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 76b5edfa..5408c54f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)
+	* Core: Support colored log output (Evil Ham)
 
 6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)

From 55ebd1a4c56e788b18911f273d931f950a0c5271 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 1 Jun 2020 20:22:40 +0200
Subject: [PATCH 087/527] Fix man build failure in newer sphinx versions

---
 cdist/sphinxext/manpage.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/cdist/sphinxext/manpage.py b/cdist/sphinxext/manpage.py
index 135fe22e..1f8ac4f6 100644
--- a/cdist/sphinxext/manpage.py
+++ b/cdist/sphinxext/manpage.py
@@ -7,6 +7,7 @@ from docutils.io import FileOutput
 from os import path
 from sphinx.util.nodes import inline_all_toctrees
 from sphinx import addnodes
+from sphinx.util import logging
 
 """
     Extension based on sphinx builtin manpage.
@@ -15,6 +16,9 @@ from sphinx import addnodes
 """
 
 
+logger = logging.getLogger(__name__)
+
+
 class ManualPageTranslator(sphinx.writers.manpage.ManualPageTranslator):
 
     def header(self):
@@ -28,7 +32,7 @@ class ManualPageWriter(sphinx.writers.manpage.ManualPageWriter):
     def __init__(self, builder):
         super().__init__(builder)
         self.translator_class = (
-            self.builder.translator_class or ManualPageTranslator)
+            self.builder.get_translator_class() or ManualPageTranslator)
 
 
 class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
@@ -43,7 +47,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
             components=(docwriter,),
             read_config_files=True).get_default_values()
 
-        self.info(bold('writing... '), nonl=True)
+        logger.info(bold('writing... '), nonl=True)
 
         for info in self.config.man_pages:
             docname, name, description, authors, section = info
@@ -54,7 +58,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
                     authors = []
 
             targetname = '%s.%s' % (name, section)
-            self.info(darkgreen(targetname) + ' { ', nonl=True)
+            logger.info(darkgreen(targetname) + ' { ', nonl=True)
             destination = FileOutput(
                 destination_path=path.join(self.outdir, targetname),
                 encoding='utf-8')
@@ -63,7 +67,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
             docnames = set()
             largetree = inline_all_toctrees(self, docnames, docname, tree,
                                             darkgreen, [docname])
-            self.info('} ', nonl=True)
+            logger.info('} ', nonl=True)
             self.env.resolve_references(largetree, docname, self)
             # remove pending_xref nodes
             for pendingnode in largetree.traverse(addnodes.pending_xref):
@@ -76,7 +80,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
             largetree.settings.section = section
 
             docwriter.write(largetree, destination)
-        self.info()
+        logger.info("")
 
 
 def setup(app):

From 48d66b014351982e70aa3e065ffc1ed344660bab Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 1 Jun 2020 22:25:15 +0200
Subject: [PATCH 088/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 5408c54f..5b57e98b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)
 	* Core: Support colored log output (Evil Ham)
+	* Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak)
 
 6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)

From 6a611e556a54a60952a833989a5a0fca79011217 Mon Sep 17 00:00:00 2001
From: Daniel Heule <nerve@ipwar.ch>
Date: Tue, 2 Jun 2020 13:40:21 +0200
Subject: [PATCH 089/527] fix os explorer for sles15

---
 cdist/conf/explorer/os | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os
index 563fa4cf..2d2aede6 100755
--- a/cdist/conf/explorer/os
+++ b/cdist/conf/explorer/os
@@ -143,6 +143,11 @@ case "$uname_s" in
 esac
 
 if [ -f /etc/os-release ]; then
+   # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse
+   if grep -q ^ID_LIKE=\"suse\" /etc/os-release 2>/dev/null; then
+      echo suse
+      exit 0
+   fi
    # already lowercase, according to:
    # https://www.freedesktop.org/software/systemd/man/os-release.html
    awk -F= '/^ID=/ { if ($2 ~ /^'"'"'(.*)'"'"'$/ || $2 ~ /^"(.*)"$/) { print substr($2, 2, length($2) - 2) } else { print $2 } }' /etc/os-release

From 747c6b10762e18bbda24fbb876ac2e88f29dc70b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 21:45:04 +0200
Subject: [PATCH 090/527] Respect NO_COLOR environment variable

---
 cdist/argparse.py                    | 23 +++++++++--------------
 cdist/config.py                      |  1 +
 cdist/configuration.py               | 28 +++++++++++++++++++---------
 cdist/core/manifest.py               |  2 +-
 cdist/emulator.py                    |  4 ++--
 cdist/log.py                         | 19 ++++++++++---------
 cdist/test/configuration/__init__.py |  2 +-
 configuration/cdist.cfg.skeleton     |  6 ++++--
 8 files changed, 47 insertions(+), 38 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index c30e2030..0782654f 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -89,13 +89,6 @@ def check_lower_bounded_int(value, lower_bound, name):
     return val
 
 
-def colored_output_type(val):
-    boolean_states = cdist.configuration.ColoredOutputOption.BOOLEAN_STATES
-    if val not in boolean_states.keys():
-        raise argparse.ArgumentError()
-    return boolean_states[val]
-
-
 def get_parsers():
     global parser
 
@@ -140,7 +133,7 @@ def get_parsers():
                  'It can be a boolean or "auto" (default) which enables this '
                  'feature if stdout is a tty and disables it otherwise.',
             action='store', dest='colored_output', required=False,
-            type=colored_output_type)
+            choices=cdist.configuration.ColoredOutputOption.CHOICES)
 
     parser['beta'] = argparse.ArgumentParser(add_help=False)
     parser['beta'].add_argument(
@@ -501,7 +494,12 @@ def handle_loglevel(args):
     if hasattr(args, 'quiet') and args.quiet:
         args.verbose = _verbosity_level_off
 
-    logging.root.setLevel(_verbosity_level[args.verbose])
+    logging.getLogger().setLevel(_verbosity_level[args.verbose])
+
+
+def handle_log_colors(args):
+    if cdist.configuration.ColoredOutputOption.translate(args.colored_output):
+        cdist.log.DefaultLog.USE_COLORS = True
 
 
 def parse_and_configure(argv, singleton=True):
@@ -515,16 +513,13 @@ def parse_and_configure(argv, singleton=True):
         raise cdist.Error(str(e))
     # Loglevels are handled globally in here
     handle_loglevel(args)
+    handle_log_colors(args)
 
     log = logging.getLogger("cdist")
 
-    config = cfg.get_config()
-    if config.get('GLOBAL', {}).get('colored_output', False):
-        cdist.log.ColorFormatter.USE_COLORS = True
-
     log.verbose("version %s" % cdist.VERSION)
     log.trace('command line args: {}'.format(cfg.command_line_args))
-    log.trace('configuration: {}'.format(config))
+    log.trace('configuration: {}'.format(cfg.get_config()))
     log.trace('configured args: {}'.format(args))
 
     check_beta(vars(args))
diff --git a/cdist/config.py b/cdist/config.py
index 97cc1da6..b2d72f05 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -203,6 +203,7 @@ class Config(object):
                 cdist.log.setupParallelLogging()
         elif args.timestamp:
             cdist.log.setupTimestampingLogging()
+
         log = logging.getLogger("config")
 
         # No new child process if only one host at a time.
diff --git a/cdist/configuration.py b/cdist/configuration.py
index 6f07c27f..c0fbc063 100644
--- a/cdist/configuration.py
+++ b/cdist/configuration.py
@@ -48,7 +48,6 @@ _VERBOSITY_VALUES = (
 _ARCHIVING_VALUES = (
     'tar', 'tgz', 'tbz2', 'txz', 'none',
 )
-_COLORED_OUTPUT_DEFAULT = sys.stdout.isatty()
 
 
 class OptionBase:
@@ -249,8 +248,22 @@ class LogLevelOption(OptionBase):
 
 
 class ColoredOutputOption(BooleanOption):
-    BOOLEAN_STATES = dict(configparser.ConfigParser.BOOLEAN_STATES,
-                          auto=_COLORED_OUTPUT_DEFAULT)
+    CHOICES = tuple(configparser.ConfigParser.BOOLEAN_STATES) + ('auto',)
+    DEFAULT = 'auto'
+
+    def get_converter(self):
+        return self.translate
+
+    @staticmethod
+    def translate(val):
+        if 'NO_COLOR' in os.environ:
+            return False
+        elif isinstance(val, bool):
+            return val
+        elif val == 'auto':
+            return sys.stdout.isatty()
+        else:
+            return configparser.ConfigParser.BOOLEAN_STATES[val]
 
 
 _ARG_OPTION_MAPPING = {
@@ -337,12 +350,10 @@ class Configuration(metaclass=Singleton):
     }
 
     ARG_OPTION_MAPPING = _ARG_OPTION_MAPPING
-    ADJUST_ARG_OPTION_MAPPING = {
-       _ARG_OPTION_MAPPING[key]: key for key in _ARG_OPTION_MAPPING
-    }
+    ADJUST_ARG_OPTION_MAPPING = {v: k for k, v in _ARG_OPTION_MAPPING.items()}
     REQUIRED_DEFAULT_CONFIG_VALUES = {
         'GLOBAL': {
-            'colored_output': _COLORED_OUTPUT_DEFAULT,
+            'colored_output': 'auto',
             'verbosity': 0,
         },
     }
@@ -495,8 +506,7 @@ class Configuration(metaclass=Singleton):
             newconfig = self._read_config_file(config_file)
             self._update_config_dict(config, newconfig)
         # command line config file
-        if (self.args and 'config_file' in self.args and
-                self.args['config_file']):
+        if (self.args and self.args.get('config_file', None)):
             newconfig = self._read_config_file(self.args['config_file'])
             self._update_config_dict(config, newconfig)
         # command line
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 32520e49..8b833ff2 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -119,7 +119,7 @@ class Manifest(object):
             '__cdist_log_level': util.log_level_env_var_val(self.log),
             '__cdist_log_level_name': util.log_level_name_env_var_val(
                 self.log),
-            '__cdist_colored_log': str(cdist.log.ColorFormatter.USE_COLORS),
+            '__cdist_colored_log': str(self.log.USE_COLORS).lower(),
         }
 
         if dry_run:
diff --git a/cdist/emulator.py b/cdist/emulator.py
index 87c9fe12..4eaf2c93 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -129,8 +129,8 @@ class Emulator(object):
             # if invalid __cdist_log_level value
             logging.root.setLevel(logging.WARNING)
 
-        colored_log = self.env.get('__cdist_colored_log', 'False')
-        cdist.log.ColorFormatter.USE_COLORS = colored_log == 'True'
+        colored_log = self.env.get('__cdist_colored_log', 'false')
+        cdist.log.ColorFormatter.USE_COLORS = colored_log == 'true'
 
         self.log = logging.getLogger(self.target_host[0])
 
diff --git a/cdist/log.py b/cdist/log.py
index 5f2d8f53..19efebdb 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -51,7 +51,6 @@ logging.trace = _trace
 
 
 class ColorFormatter(logging.Formatter):
-    USE_COLORS = False
     RESET = '\033[0m'
     COLOR_MAP = {
         'ERROR': '\033[0;31m',
@@ -62,20 +61,19 @@ class ColorFormatter(logging.Formatter):
         'TRACE': '\033[0;37m',
     }
 
-    def __init__(self, msg):
-        super().__init__(msg)
+    def __init__(self, fmt):
+        super().__init__(fmt=fmt)
 
     def format(self, record):
         msg = super().format(record)
-        if self.USE_COLORS:
-            color = self.COLOR_MAP.get(record.levelname)
-            if color:
-                msg = color + msg + self.RESET
+        color = self.COLOR_MAP.get(record.levelname)
+        if color:
+            msg = color + msg + self.RESET
         return msg
 
 
 class DefaultLog(logging.Logger):
-
+    USE_COLORS = False
     FORMAT = '%(levelname)s: %(message)s'
 
     class StdoutFilter(logging.Filter):
@@ -90,7 +88,10 @@ class DefaultLog(logging.Logger):
         super().__init__(name)
         self.propagate = False
 
-        formatter = ColorFormatter(self.FORMAT)
+        if self.USE_COLORS:
+            formatter = ColorFormatter(self.FORMAT)
+        else:
+            formatter = logging.Formatter(self.FORMAT)
 
         self.addFilter(self)
 
diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py
index 07a73bda..5305b6d3 100644
--- a/cdist/test/configuration/__init__.py
+++ b/cdist/test/configuration/__init__.py
@@ -33,7 +33,7 @@ import sys
 my_dir = op.abspath(op.dirname(__file__))
 fixtures = op.join(my_dir, 'fixtures')
 interpolation_config_file = op.join(fixtures, "interpolation-test.cfg")
-colored_output_default = sys.stdout.isatty()
+colored_output_default = 'auto'
 
 
 def newConfigParser():
diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton
index f2a09064..0730201d 100644
--- a/configuration/cdist.cfg.skeleton
+++ b/configuration/cdist.cfg.skeleton
@@ -17,6 +17,8 @@
 #     Use a colored output for different log levels.
 #     It can be a boolean or 'auto' (default) which enables this feature if
 #     stdout is a tty and disables it otherwise.
+#     Colored output is always disabled if the NO_COLOR environment variable is
+#     defined (https://no-color.org/).
 # colored_output = auto
 #
 # conf_dir
@@ -51,7 +53,7 @@
 #
 # out_path
 #     Directory to save cdist output in.
-# out_path = 
+# out_path =
 #
 # parallel
 #     Process hosts in parallel. If -1 then the default, number of CPU's in
@@ -77,6 +79,6 @@
 # remote_shell = /bin/sh
 #
 # verbosity
-#     Set verbosity level. Valid values are: 
+#     Set verbosity level. Valid values are:
 #     ERROR, WARNING, INFO, VERBOSE, DEBUG, TRACE and OFF.
 # verbosity = INFO

From cdb0d2be413114a237a0f56c948cc5cd6cfc2b0b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 23:21:50 +0200
Subject: [PATCH 091/527] Patch tests

---
 cdist/configuration.py               | 4 ++++
 cdist/test/configuration/__init__.py | 9 ++++++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/cdist/configuration.py b/cdist/configuration.py
index c0fbc063..f68a5e09 100644
--- a/cdist/configuration.py
+++ b/cdist/configuration.py
@@ -266,6 +266,10 @@ class ColoredOutputOption(BooleanOption):
             return configparser.ConfigParser.BOOLEAN_STATES[val]
 
 
+ColoredOutputOption.DEFAULT = ColoredOutputOption.translate(
+    ColoredOutputOption.DEFAULT)
+
+
 _ARG_OPTION_MAPPING = {
     'beta': 'beta',
     'cache_path_pattern': 'cache_path_pattern',
diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py
index 5305b6d3..3fd24ca5 100644
--- a/cdist/test/configuration/__init__.py
+++ b/cdist/test/configuration/__init__.py
@@ -187,7 +187,8 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/bin/sh',
                 'inventory_dir': None,
                 'cache_path_pattern': None,
-                'colored_output': colored_output_default,
+                'colored_output': cc.ColoredOutputOption.translate(
+                    colored_output_default),
                 'conf_dir': None,
                 'init_manifest': None,
                 'out_path': None,
@@ -587,7 +588,8 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': None,
                 'cache_path_pattern': None,
-                'colored_output': colored_output_default,
+                'colored_output': cc.ColoredOutputOption.translate(
+                    colored_output_default),
                 'conf_dir': [
                     '/opt/cdist/conf',
                     '/usr/local/share/cdist/conf',
@@ -674,7 +676,8 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': '/var/db/cdist/inventory',
                 'cache_path_pattern': None,
-                'colored_output': colored_output_default,
+                'colored_output': cc.ColoredOutputOption.translate(
+                    colored_output_default),
                 'conf_dir': [
                     '/opt/cdist/conf',
                     '/usr/local/share/cdist/conf',

From 89e48734bf13446024e9009bce7e77cbd4255840 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 5 Jun 2020 12:22:49 +0200
Subject: [PATCH 092/527] Let config file and command line override NO_COLOR
 envvar

---
 cdist/configuration.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/cdist/configuration.py b/cdist/configuration.py
index f68a5e09..a12940b0 100644
--- a/cdist/configuration.py
+++ b/cdist/configuration.py
@@ -256,14 +256,12 @@ class ColoredOutputOption(BooleanOption):
 
     @staticmethod
     def translate(val):
-        if 'NO_COLOR' in os.environ:
-            return False
-        elif isinstance(val, bool):
+        if isinstance(val, bool):
             return val
-        elif val == 'auto':
-            return sys.stdout.isatty()
-        else:
+        elif val in configparser.ConfigParser.BOOLEAN_STATES:
             return configparser.ConfigParser.BOOLEAN_STATES[val]
+        elif val == 'auto':
+            return 'NO_COLOR' not in os.environ and sys.stdout.isatty()
 
 
 ColoredOutputOption.DEFAULT = ColoredOutputOption.translate(

From 790c6efae9e379cb140e59469f07d2c3732f536c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 5 Jun 2020 13:56:30 +0200
Subject: [PATCH 093/527] Update colored output documentation

---
 docs/src/cdist-reference.rst.sh | 38 +++++++++++++++++++++++++++------
 docs/src/man1/cdist.rst         | 35 +++++++++++++++++-------------
 2 files changed, 51 insertions(+), 22 deletions(-)

diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh
index 3b997f63..a841862e 100755
--- a/docs/src/cdist-reference.rst.sh
+++ b/docs/src/cdist-reference.rst.sh
@@ -34,7 +34,7 @@ dest="$__cdist_abs_mydir/$filename"
 cd "$__cdist_abs_mydir"
 
 exec > "$dest"
-cat << eof 
+cat << eof
 Reference
 =========
 Variable, path and type reference for cdist
@@ -51,7 +51,7 @@ eof
     done
 )
 
-cat << eof 
+cat << eof
 
 Paths
 -----
@@ -187,13 +187,13 @@ usable within a object directory:
 
 files
     This directory is reserved for user data and will not be used
-    by cdist at any time. It can be used freely by the type 
+    by cdist at any time. It can be used freely by the type
     (for instance to store template results).
 changed
     This empty file exists in an object directory, if the object has
     code to be executed (either remote or local).
 stdin
-    This file exists and contains data, if data was provided on stdin 
+    This file exists and contains data, if data was provided on stdin
     when the type was called.
 
 
@@ -222,65 +222,89 @@ __cdist_log_level, __cdist_log_level_name
     | TRACE          | 5               |
     +----------------+-----------------+
 
+    Available for: initial manifest, explorer, type manifest, type explorer,
+    type gencode.
+__cdist_colored_log
+    whether or not cdist's log has colors enabled.
+    Is set to the string ``true`` if cdist's output is using colors,
+    otherwise the variable contains the string ``false``.
+
     Available for: initial manifest, explorer, type manifest, type explorer,
     type gencode.
 __cdist_dry_run
     Is set only when doing dry run (``-n`` flag).
+
     Available for: initial manifest, explorer, type manifest, type explorer,
     type gencode.
 __explorer
     Directory that contains all global explorers.
+
     Available for: initial manifest, explorer, type explorer, shell.
 __files
     Directory that contains content from the "files" subdirectories
     from the configuration directories.
+
     Available for: initial manifest, type manifest, type gencode, shell.
 __manifest
     Directory that contains the initial manifest.
+
     Available for: initial manifest, type manifest, shell.
 __global
     Directory that contains generic output like explorer.
+
     Available for: initial manifest, type manifest, type gencode, shell.
 __messages_in
     File to read messages from.
+
     Available for: initial manifest, type manifest, type gencode.
 __messages_out
     File to write messages.
+
     Available for: initial manifest, type manifest, type gencode.
 __object
     Directory that contains the current object.
+
     Available for: type manifest, type explorer, type gencode and code scripts.
 __object_id
     The type unique object id.
+
     Available for: type manifest, type explorer, type gencode and code scripts.
-    Note: The leading and the trailing "/" will always be stripped (caused by
-    the filesystem database and ensured by the core).
-    Note: Double slashes ("//") will not be fixed and result in an error.
+
+    | Note: The leading and the trailing "/" will always be stripped (caused by
+      the filesystem database and ensured by the core).
+    | Note: Double slashes ("//") will not be fixed and result in an error.
 __object_name
     The full qualified name of the current object.
+
     Available for: type manifest, type explorer, type gencode.
 __target_host
     The host we are deploying to. This is primary variable. It's content is
     literally the one user passed in.
+
     Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
 __target_hostname
     The hostname of host we are deploying to. This variable is derived from
     **__target_host** (using **socket.getaddrinfo(__target_host)** and then
     **socket.gethostbyaddr()**).
+
     Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
 __target_fqdn
     The fully qualified domain name of the host we are deploying to.
     This variable is derived from **__target_host**
     (using **socket.getfqdn()**).
+
     Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
 __target_host_tags
     Comma separated list of target host tags.
+
     Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
 __type
     Path to the current type.
+
     Available for: type manifest, type gencode.
 __type_explorer
     Directory that contains the type explorers.
+
     Available for: type explorer.
 
 Environment variables (for writing)
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 4c34c4b7..4def97b5 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -112,9 +112,13 @@ All commands accept the following options:
     Show the help screen.
 
 **--colors COLORED_OUTPUT**
-    Use a colored output for different log levels.It can
-    be a boolean or "auto" (default) which enables this
-    feature if stdout is a tty and disables it otherwise.
+    Colorize cdist's output. If enabled, cdist will use different colors for
+    different log levels.
+    COLORED_OUTPUT recognizes the boolean values 'yes'/'no', 'on'/'off',
+    'true'/'false', '1'/'0', and 'auto' (the default).
+
+    If the value is 'auto', colored output is enabled if stdout is a TTY
+    unless the NO_COLOR (https://no-color.org/) environment variable is defined.
 
 **-l LOGLEVEL, --log-level LOGLEVEL**
     Set the specified verbosity level. The levels, in
@@ -168,7 +172,7 @@ Install command is currently in beta.
 
 **-b, --beta**
     Enable beta functionality.
-    
+
 **-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN**
     Specify custom cache path pattern. If it is not set then
     default hostdir is used. For more info on format see
@@ -191,7 +195,7 @@ Install command is currently in beta.
 
 **-I INVENTORY_DIR, --inventory INVENTORY_DIR**
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if cdist 
+    directory is set up by the following rules: if cdist
     configuration resolves this value then specified
     directory is used, if HOME env var is set then
     ~/.cdit/inventory is used, otherwise distribution
@@ -306,7 +310,7 @@ Add host(s) to inventory database.
 
 **-I INVENTORY_DIR, --inventory INVENTORY_DIR**
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if cdist 
+    directory is set up by the following rules: if cdist
     configuration resolves this value then specified
     directory is used, if HOME env var is set then
     ~/.cdit/inventory is used, otherwise distribution
@@ -336,7 +340,7 @@ Add tag(s) to inventory database.
 
 **-I INVENTORY_DIR, --inventory INVENTORY_DIR**
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if cdist 
+    directory is set up by the following rules: if cdist
     configuration resolves this value then specified
     directory is used, if HOME env var is set then
     ~/.cdit/inventory is used, otherwise distribution
@@ -379,7 +383,7 @@ Delete host(s) from inventory database.
 
 **-I INVENTORY_DIR, --inventory INVENTORY_DIR**
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if cdist 
+    directory is set up by the following rules: if cdist
     configuration resolves this value then specified
     directory is used, if HOME env var is set then
     ~/.cdit/inventory is used, otherwise distribution
@@ -413,7 +417,7 @@ Delete tag(s) from inventory database.
 
 **-I INVENTORY_DIR, --inventory INVENTORY_DIR**
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if cdist 
+    directory is set up by the following rules: if cdist
     configuration resolves this value then specified
     directory is used, if HOME env var is set then
     ~/.cdit/inventory is used, otherwise distribution
@@ -460,7 +464,7 @@ List inventory database.
 
 **-I INVENTORY_DIR, --inventory INVENTORY_DIR**
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if cdist 
+    directory is set up by the following rules: if cdist
     configuration resolves this value then specified
     directory is used, if HOME env var is set then
     ~/.cdit/inventory is used, otherwise distribution
@@ -685,6 +689,9 @@ The possible keywords and their meanings are as follows:
 :strong:`cache_path_pattern`
     Specify cache path pattern.
 
+:strong:`colored_output`
+    Colorize cdist's output. cf. the :code:`--colors` option.
+
 :strong:`conf_dir`
     List of configuration directories separated with the character conventionally
     used by the operating system to separate search path components (as in PATH),
@@ -738,7 +745,7 @@ The possible keywords and their meanings are as follows:
     in the format: YYYYMMDDHHMMSS.us.
 
 :strong:`verbosity`
-    Set verbosity level. Valid values are: 
+    Set verbosity level. Valid values are:
     'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'.
 
 
@@ -770,7 +777,7 @@ cdist/preos
 NOTES
 -----
 cdist detects if host is specified by IPv6 address. If so then remote_copy
-command is executed with host address enclosed in square brackets 
+command is executed with host address enclosed in square brackets
 (see :strong:`scp`\ (1)).
 
 EXAMPLES
@@ -906,9 +913,7 @@ CDIST_CACHE_PATH_PATTERN
     Custom cache path pattern.
 
 CDIST_COLORED_OUTPUT
-    Use a colored output for different log levels.
-    It can be a boolean or 'auto' (default) which enables this feature if
-    stdout is a tty and disables it otherwise.
+    Colorize cdist's output. cf. the :code:`--colors` option.
 
 CDIST_CONFIG_FILE
     Custom configuration file.

From 7a570f8692a727662a6eda2c13f2b578b55864e4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 5 Jun 2020 13:59:17 +0200
Subject: [PATCH 094/527] [cdist.cfg.skeleton] Update colored_output
 documentation based on cdist(1)

---
 configuration/cdist.cfg.skeleton | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton
index 0730201d..9c66b51b 100644
--- a/configuration/cdist.cfg.skeleton
+++ b/configuration/cdist.cfg.skeleton
@@ -14,11 +14,12 @@
 # cache_path_pattern = %h
 #
 # colored_output
-#     Use a colored output for different log levels.
-#     It can be a boolean or 'auto' (default) which enables this feature if
-#     stdout is a tty and disables it otherwise.
-#     Colored output is always disabled if the NO_COLOR environment variable is
-#     defined (https://no-color.org/).
+#     Colorize cdist's output. If enabled, cdist will use different colors for
+#     different log levels.
+#     Recognized values are 'yes'/'no', 'on'/'off', 'true'/'false', '1'/'0',
+#     and 'auto'
+#     If the value is 'auto', colored output is enabled if stdout is a TTY
+#     unless the NO_COLOR (https://no-color.org/) environment variable is defined.
 # colored_output = auto
 #
 # conf_dir

From 23e66e08fab14f254c426c88271f461fe8499ba4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 6 Jun 2020 13:39:29 +0200
Subject: [PATCH 095/527] Restrict colored_output value to always/never/auto.

---
 cdist/argparse.py                | 7 +++----
 cdist/configuration.py           | 8 +++++---
 configuration/cdist.cfg.skeleton | 7 +++----
 docs/src/cdist-reference.rst.sh  | 6 +++---
 docs/src/man1/cdist.rst          | 6 +++---
 5 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 0782654f..85dba246 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -128,10 +128,9 @@ def get_parsers():
 
     parser['colored_output'] = argparse.ArgumentParser(add_help=False)
     parser['colored_output'].add_argument(
-            '--colors',
-            help='Use a colored output for different log levels.'
-                 'It can be a boolean or "auto" (default) which enables this '
-                 'feature if stdout is a tty and disables it otherwise.',
+            '--colors', metavar='WHEN',
+            help="Colorize cdist's output based on log level; "
+                 "WHEN is 'always', 'never', or 'auto'.",
             action='store', dest='colored_output', required=False,
             choices=cdist.configuration.ColoredOutputOption.CHOICES)
 
diff --git a/cdist/configuration.py b/cdist/configuration.py
index a12940b0..4ba43a7a 100644
--- a/cdist/configuration.py
+++ b/cdist/configuration.py
@@ -248,7 +248,7 @@ class LogLevelOption(OptionBase):
 
 
 class ColoredOutputOption(BooleanOption):
-    CHOICES = tuple(configparser.ConfigParser.BOOLEAN_STATES) + ('auto',)
+    CHOICES = ('always', 'never', 'auto')
     DEFAULT = 'auto'
 
     def get_converter(self):
@@ -258,8 +258,10 @@ class ColoredOutputOption(BooleanOption):
     def translate(val):
         if isinstance(val, bool):
             return val
-        elif val in configparser.ConfigParser.BOOLEAN_STATES:
-            return configparser.ConfigParser.BOOLEAN_STATES[val]
+        elif val == 'always':
+            return True
+        elif val == 'never':
+            return False
         elif val == 'auto':
             return 'NO_COLOR' not in os.environ and sys.stdout.isatty()
 
diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton
index 9c66b51b..14861592 100644
--- a/configuration/cdist.cfg.skeleton
+++ b/configuration/cdist.cfg.skeleton
@@ -16,10 +16,9 @@
 # colored_output
 #     Colorize cdist's output. If enabled, cdist will use different colors for
 #     different log levels.
-#     Recognized values are 'yes'/'no', 'on'/'off', 'true'/'false', '1'/'0',
-#     and 'auto'
-#     If the value is 'auto', colored output is enabled if stdout is a TTY
-#     unless the NO_COLOR (https://no-color.org/) environment variable is defined.
+#     Recognized values are 'always', 'never', and 'auto'.
+#     If the value is 'auto', colors are enabled if stdout is a TTY unless
+#     the NO_COLOR (https://no-color.org/) environment variable is defined.
 # colored_output = auto
 #
 # conf_dir
diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh
index a841862e..c0ac2c3e 100755
--- a/docs/src/cdist-reference.rst.sh
+++ b/docs/src/cdist-reference.rst.sh
@@ -369,9 +369,9 @@ CDIST_BETA
     Enable beta functionalities.
 
 CDIST_COLORED_OUTPUT
-    Use a colored output for different log levels.
-    It can be a boolean or 'auto' (default) which enables this feature if
-    stdout is a tty and disables it otherwise.
+    Colorize cdist's output. If enabled, cdist will use different colors for
+    different log levels.
+    Recognized values are 'always', 'never', and 'auto' (the default).
 
 CDIST_CACHE_PATH_PATTERN
     Custom cache path pattern.
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 4def97b5..760ff8f6 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -111,11 +111,11 @@ All commands accept the following options:
 **-h, --help**
     Show the help screen.
 
-**--colors COLORED_OUTPUT**
+**--colors WHEN**
     Colorize cdist's output. If enabled, cdist will use different colors for
     different log levels.
-    COLORED_OUTPUT recognizes the boolean values 'yes'/'no', 'on'/'off',
-    'true'/'false', '1'/'0', and 'auto' (the default).
+    COLORED_OUTPUT recognizes the values 'always', 'never',
+    and 'auto' (the default).
 
     If the value is 'auto', colored output is enabled if stdout is a TTY
     unless the NO_COLOR (https://no-color.org/) environment variable is defined.

From 89ebd7a4f79bef894cb3f31931571cf577778b26 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 7 Jun 2020 16:45:48 +0200
Subject: [PATCH 096/527] cdist man page: update --colors metavar name

---
 docs/src/man1/cdist.rst | 62 ++++++++++++++++++-----------------------
 1 file changed, 27 insertions(+), 35 deletions(-)

diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 760ff8f6..aae98538 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -15,21 +15,19 @@ SYNOPSIS
 
     cdist banner [-h] [-l LOGLEVEL] [-q] [-v]
 
-    cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                 [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6]
-                 [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
-                 [-j [JOBS]] [-n] [-o OUT_PATH] [-P]
+    cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
+                 [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN]
+                 [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-P]
                  [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
                  [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
                  [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE]
                  [-p [HOST_MAX]] [-s] [-t]
                  [host [host ...]]
 
-    cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                  [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6]
-                  [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
-                  [-j [JOBS]] [-n] [-o OUT_PATH] [-P]
-                  [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
+    cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
+                  [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN]
+                  [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
+                  [-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
                   [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
                   [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE]
                   [-p [HOST_MAX]] [-s] [-t]
@@ -37,35 +35,31 @@ SYNOPSIS
 
     cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ...
 
-    cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                             [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
-                             [-I INVENTORY_DIR] [-f HOSTFILE]
+    cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
+                             [-g CONFIG_FILE] [-I INVENTORY_DIR] [-f HOSTFILE]
                              [host [host ...]]
 
-    cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                            [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
-                            [-I INVENTORY_DIR] [-f HOSTFILE] [-T TAGFILE]
-                            [-t TAGLIST]
-                            [host [host ...]]
-
-    cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                             [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
-                             [-I INVENTORY_DIR] [-a] [-f HOSTFILE]
-                             [host [host ...]]
-
-    cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                            [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
-                            [-I INVENTORY_DIR] [-a] [-f HOSTFILE]
+    cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
+                            [-g CONFIG_FILE] [-I INVENTORY_DIR] [-f HOSTFILE]
                             [-T TAGFILE] [-t TAGLIST]
                             [host [host ...]]
 
-    cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                         [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
-                         [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t]
+    cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
+                             [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a]
+                             [-f HOSTFILE]
+                             [host [host ...]]
+
+    cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
+                            [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a]
+                            [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
+                            [host [host ...]]
+
+    cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
+                         [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] [-f HOSTFILE]
+                         [-H] [-t]
                          [host [host ...]]
 
-    cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-g CONFIG_FILE]
-                [-L]
+    cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-g CONFIG_FILE] [-L]
                 [preos] ...
 
     cdist preos [preos-options] debian [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
@@ -89,8 +83,7 @@ SYNOPSIS
                                        [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
                                        target_dir
 
-    cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [--colors COLORED_OUTPUT]
-                [-s SHELL]
+    cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [--colors WHEN] [-s SHELL]
 
     cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t]
                [pattern]
@@ -114,8 +107,7 @@ All commands accept the following options:
 **--colors WHEN**
     Colorize cdist's output. If enabled, cdist will use different colors for
     different log levels.
-    COLORED_OUTPUT recognizes the values 'always', 'never',
-    and 'auto' (the default).
+    WHEN recognizes the values 'always', 'never', and 'auto' (the default).
 
     If the value is 'auto', colored output is enabled if stdout is a TTY
     unless the NO_COLOR (https://no-color.org/) environment variable is defined.

From be47619b1ed7a87090f7615c19a9504f7cf2b590 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 8 Jun 2020 09:11:51 +0200
Subject: [PATCH 097/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 5b57e98b..94ef55d8 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)
 	* Core: Support colored log output (Evil Ham)
+	* Core: Tune colored log output and respect NO_COLOR env var (Dennis Camera)
 	* Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak)
 
 6.5.6: 2020-05-25

From f5630297bd58fc92d95ab174ed6c8a8e14abf0b9 Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Wed, 3 Jun 2020 23:06:49 +0300
Subject: [PATCH 098/527] [docs] Fixed typo in cdist-configuration.rst

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 docs/src/cdist-configuration.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/cdist-configuration.rst b/docs/src/cdist-configuration.rst
index 706ed761..a2024584 100644
--- a/docs/src/cdist-configuration.rst
+++ b/docs/src/cdist-configuration.rst
@@ -21,7 +21,7 @@ precedence. Configuration option value read from source with higher
 precedence will overwrite option value, if exists, read from source with
 lower precedence. That means that command-line option wins them all.
 
-Users can decide on the local conifguration file location. It can be either
+Users can decide on the local configuration file location. It can be either
 ~/.cdist.cfg or $XDG_CONFIG_HOME/cdist/cdist.cfg. Note that, if both exist,
 then ~/.cdist.cfg is used.
 

From 3d725f12dace8d2e977d408b7c01fa28b8b17bde Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Wed, 3 Jun 2020 23:10:27 +0300
Subject: [PATCH 099/527] [docs] Fixed a typo and s/posix/POSIX/ in
 cdist-why.rst

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 docs/src/cdist-why.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/src/cdist-why.rst b/docs/src/cdist-why.rst
index 1123a1de..0e2cd34d 100644
--- a/docs/src/cdist-why.rst
+++ b/docs/src/cdist-why.rst
@@ -21,7 +21,7 @@ Not only is shell scripting widely known by system engineers,
 but it is also a very powerful language. Here are some features
 which make daily work easy:
 
- * Configuration can react dynamicly on explored values
+ * Configuration can react dynamically on explored values
  * High level string manipulation (using sed, awk, grep)
  * Conditional support (**if, case**)
  * Loop support (**for, while**)
@@ -44,7 +44,7 @@ Cdist requires very little on a target system. Even better,
 in almost all cases all dependencies are usually fulfilled.
 Cdist does not require an agent or high level programming
 languages on the target host: it will run on any host that
-has a **ssh server running** and a posix compatible shell
+has a **ssh server running** and a POSIX compatible shell
 (**/bin/sh**). Compared to other configuration management systems,
 it does not require to open up an additional port.
 

From 46574fc5779b939db5d90e8c11f28e808be458be Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Wed, 3 Jun 2020 23:12:10 +0300
Subject: [PATCH 100/527] [docs] Fixed three typos in cdist-upgrade.rst

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 docs/src/cdist-upgrade.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/src/cdist-upgrade.rst b/docs/src/cdist-upgrade.rst
index 67fd4934..f745b212 100644
--- a/docs/src/cdist-upgrade.rst
+++ b/docs/src/cdist-upgrade.rst
@@ -14,7 +14,7 @@ To upgrade cdist in the current branch use
     make man
     export MANPATH=$MANPATH:$(pwd -P)/doc/man
 
-If you stay on a version branche (i.e. 1.0, 1.1., ...), nothing should break.
+If you stay on a version branch (i.e. 1.0, 1.1., ...), nothing should break.
 The master branch on the other hand is the development branch and may not be
 working, break your setup or eat the tree in your garden.
 
@@ -61,7 +61,7 @@ After that, you can go back and continue the upgrade:
 Update the python package
 -------------------------
 
-To upgrade to the lastet version do
+To upgrade to the latest version do
 
 .. code-block:: sh
 
@@ -158,7 +158,7 @@ Updating from 1.5 to 1.6
 * If you used **\_\_package_apt --preseed**, you need to use the new
   type **\_\_debconf_set_selections** instead.
 * The **\_\_package** types accepted either --state deinstalled or
-  --state uninstaaled. Starting with 1.6, it was made consistently
+  --state uninstalled. Starting with 1.6, it was made consistently
   to --state removed.
 
 Updating from 1.3 to 1.5

From e1ff1bfdffd9a188a045669c4168dcd1fb561b0b Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Wed, 3 Jun 2020 23:15:38 +0300
Subject: [PATCH 101/527] [docs] Fixed two typos in cdist-real-world.rst

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 docs/src/cdist-real-world.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/src/cdist-real-world.rst b/docs/src/cdist-real-world.rst
index ba118d63..973dcd6d 100644
--- a/docs/src/cdist-real-world.rst
+++ b/docs/src/cdist-real-world.rst
@@ -198,7 +198,7 @@ We require package uWSGI present in order to create **/etc/uwsgi/apps-enabled/$u
 Installation of uWSGI also creates configuration layout: **/etc/uwsgi/apps-enabled**.
 If this directory does not exist then **__file** type would error.
 We also use stdin as file content source. For details see `Input from stdin <cdist-type.html#input-from-stdin>`_.
-For feading stdin we use here-document (**<<** operator). It allows redirection of subsequent
+For feeding stdin we use here-document (**<<** operator). It allows redirection of subsequent
 lines read by the shell to the input of a command until a line containing only the delimiter
 and a newline, with no blank characters in between (EOF in our case).
 
@@ -546,7 +546,7 @@ we have changed our **wsgi.py** file uWSGI reloads the application.
 
 Our application selects and lists items from **items** table.
 
-Openning application
+Opening application
 ~~~~~~~~~~~~~~~~~~~~
 
 Finally try the application::

From 978e24904336e6af074a7de757282456ff0314a4 Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Wed, 3 Jun 2020 23:23:49 +0300
Subject: [PATCH 102/527] [docs] Fixed capitalization of POSIX in
 cdist-install.rst

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 docs/src/cdist-install.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst
index 880b9f8e..0b65cad6 100644
--- a/docs/src/cdist-install.rst
+++ b/docs/src/cdist-install.rst
@@ -9,7 +9,7 @@ Source Host
 
 This is the machine from which you will configure target hosts.
 
- * /bin/sh: A posix like shell (for instance bash, dash, zsh)
+ * /bin/sh: A POSIX like shell (for instance bash, dash, zsh)
  * Python >= 3.2
  * SSH client
  * sphinx (for building html docs and/or the man pages)
@@ -17,7 +17,7 @@ This is the machine from which you will configure target hosts.
 Target Hosts
 ~~~~~~~~~~~~
 
- * /bin/sh: A posix like shell (for instance bash, dash, zsh)
+ * /bin/sh: A POSIX like shell (for instance bash, dash, zsh)
  * SSH server
 
 Install cdist

From dc018fdb16ba8b6044bbb55efd44e975fc5247ae Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Wed, 3 Jun 2020 23:25:46 +0300
Subject: [PATCH 103/527] [docs] Fixed typo in cdist-cache.rst

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 docs/src/cdist-cache.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/cdist-cache.rst b/docs/src/cdist-cache.rst
index 0e5361ee..d2d2d56c 100644
--- a/docs/src/cdist-cache.rst
+++ b/docs/src/cdist-cache.rst
@@ -59,7 +59,7 @@ typeorder
 
 Object cache overview
 ~~~~~~~~~~~~~~~~~~~~~
-Each object under :strong:`object` directory has its own structurue.
+Each object under :strong:`object` directory has its own structure.
 
 code-local
     code generated from gencode-local, present only if something is

From 7b262c0cec2c3c42822cef103220965424500bcb Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Wed, 3 Jun 2020 23:27:16 +0300
Subject: [PATCH 104/527] [docs] Fixed capitalization of URLs in
 cdist-bootstrap.rst

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 docs/src/cdist-bootstrap.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/cdist-bootstrap.rst b/docs/src/cdist-bootstrap.rst
index c9972a99..10f86c5a 100644
--- a/docs/src/cdist-bootstrap.rst
+++ b/docs/src/cdist-bootstrap.rst
@@ -25,7 +25,7 @@ people, have a look at `cdist best practice <cdist-best-practice.html>`_.
 Setup working directory and branch
 ----------------------------------
 I assume you have a fresh copy of the cdist tree in ~/cdist, cloned from
-one of the official urls (see `cdist quickstart <cdist-quickstart.html>`_ if you don't).
+one of the official URLs (see `cdist quickstart <cdist-quickstart.html>`_ if you don't).
 Entering the command "git branch" should show you "* master", which indicates
 you are on the **master** branch.
 

From 191f45eb7f8ab8ce97f996f3bb8375da5142d39a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 8 Jun 2020 13:48:37 +0200
Subject: [PATCH 105/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 94ef55d8..e73cfa6f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,8 @@ next:
 	* Core: Support colored log output (Evil Ham)
 	* Core: Tune colored log output and respect NO_COLOR env var (Dennis Camera)
 	* Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak)
+	* Documentation: Fix trivial grammatical mistakes (Jaak Ristioja)
+	* Explorer os: Fix for sles15 (Daniel Heule)
 
 6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)

From 955243a93b28d3524f93eb6d69af1f0eec5ff36d Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 9 Jun 2020 12:51:19 +0200
Subject: [PATCH 106/527] Update cdist man page copyright years

---
 docs/src/man1/cdist.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index aae98538..68a9a4ee 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -969,5 +969,5 @@ such case and display a warning message. An example of such a case:
 
 COPYING
 -------
-Copyright \(C) 2011-2019 Nico Schottelius. Free use of this software is
+Copyright \(C) 2011-2020 Nico Schottelius. Free use of this software is
 granted under the terms of the GNU General Public License v3 or later (GPLv3+).

From 7c490a703d0203747e481a9fc3405bf7eb5ba01a Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 10 Jun 2020 11:38:14 +0300
Subject: [PATCH 107/527] [__clean_path] add --path parameter

---
 cdist/conf/type/__clean_path/explorer/list      |  7 ++++++-
 cdist/conf/type/__clean_path/gencode-remote     |  7 ++++++-
 cdist/conf/type/__clean_path/man.rst            | 10 +++++++++-
 cdist/conf/type/__clean_path/parameter/optional |  1 +
 4 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__clean_path/explorer/list b/cdist/conf/type/__clean_path/explorer/list
index 07d38127..2bdc63a5 100755
--- a/cdist/conf/type/__clean_path/explorer/list
+++ b/cdist/conf/type/__clean_path/explorer/list
@@ -18,7 +18,12 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-path="/$__object_id"
+if [ -f "$__object/parameter/path" ]
+then
+    path="$( cat "$__object/parameter/path" )"
+else
+    path="/$__object_id"
+fi
 
 [ ! -d "$path" ] && exit 0
 
diff --git a/cdist/conf/type/__clean_path/gencode-remote b/cdist/conf/type/__clean_path/gencode-remote
index 998a70d8..2899c4a5 100755
--- a/cdist/conf/type/__clean_path/gencode-remote
+++ b/cdist/conf/type/__clean_path/gencode-remote
@@ -20,7 +20,12 @@
 
 [ ! -s "$__object/explorer/list" ] && exit 0
 
-path="/$__object_id"
+if [ -f "$__object/parameter/path" ]
+then
+    path="$( cat "$__object/parameter/path" )"
+else
+    path="/$__object_id"
+fi
 
 pattern="$( cat "$__object/parameter/pattern" )"
 
diff --git a/cdist/conf/type/__clean_path/man.rst b/cdist/conf/type/__clean_path/man.rst
index 826f4589..31d90701 100644
--- a/cdist/conf/type/__clean_path/man.rst
+++ b/cdist/conf/type/__clean_path/man.rst
@@ -10,7 +10,7 @@ DESCRIPTION
 -----------
 Remove files and directories which match the pattern.
 
-Provided path (as __object_id) must be a directory.
+Provided path must be a directory.
 
 Patterns are passed to ``find``'s ``-regex`` - see ``find(1)`` for more details.
 
@@ -29,6 +29,9 @@ pattern
 
 OPTIONAL PARAMETERS
 -------------------
+path
+   Path which will be cleaned. Defaults to ``$__object_id``.
+
 exclude
    Pattern of files which are excluded from removal.
 
@@ -46,6 +49,11 @@ EXAMPLES
         --exclude '.+\(charset\.conf\|security\.conf\)' \
         --onchange 'service apache2 restart'
 
+    __clean_path apache2-conf-enabled \
+        --path /etc/apache2/conf-enabled \
+        --pattern '.+' \
+        --exclude '.+\(charset\.conf\|security\.conf\)' \
+        --onchange 'service apache2 restart'
 
 AUTHORS
 -------
diff --git a/cdist/conf/type/__clean_path/parameter/optional b/cdist/conf/type/__clean_path/parameter/optional
index 6f313474..3b97f71c 100644
--- a/cdist/conf/type/__clean_path/parameter/optional
+++ b/cdist/conf/type/__clean_path/parameter/optional
@@ -1,2 +1,3 @@
 exclude
 onchange
+path

From 74e5d7182a466cd02b79137f1d73500c0cc50808 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 10 Jun 2020 10:45:20 +0200
Subject: [PATCH 108/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index e73cfa6f..bcde9268 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak)
 	* Documentation: Fix trivial grammatical mistakes (Jaak Ristioja)
 	* Explorer os: Fix for sles15 (Daniel Heule)
+	* Type __clean_path: Add --path parameter (Ander Punnar)
 
 6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)

From 840e417eb79c7135762d3353f34a3f8330cb0e84 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 11 Jun 2020 14:16:37 +0200
Subject: [PATCH 109/527] Fix emulator colored logging

---
 cdist/argparse.py      |  2 +-
 cdist/core/manifest.py |  3 ++-
 cdist/emulator.py      |  2 +-
 cdist/flock.py         |  1 +
 cdist/log.py           | 16 +++++++---------
 5 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 85dba246..1c83237c 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -498,7 +498,7 @@ def handle_loglevel(args):
 
 def handle_log_colors(args):
     if cdist.configuration.ColoredOutputOption.translate(args.colored_output):
-        cdist.log.DefaultLog.USE_COLORS = True
+        cdist.log.CdistFormatter.USE_COLORS = True
 
 
 def parse_and_configure(argv, singleton=True):
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 8b833ff2..2e7b873e 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -119,7 +119,8 @@ class Manifest(object):
             '__cdist_log_level': util.log_level_env_var_val(self.log),
             '__cdist_log_level_name': util.log_level_name_env_var_val(
                 self.log),
-            '__cdist_colored_log': str(self.log.USE_COLORS).lower(),
+            '__cdist_colored_log': str(
+                cdist.log.CdistFormatter.USE_COLORS).lower(),
         }
 
         if dry_run:
diff --git a/cdist/emulator.py b/cdist/emulator.py
index 4eaf2c93..08e15a78 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -130,7 +130,7 @@ class Emulator(object):
             logging.root.setLevel(logging.WARNING)
 
         colored_log = self.env.get('__cdist_colored_log', 'false')
-        cdist.log.ColorFormatter.USE_COLORS = colored_log == 'true'
+        cdist.log.CdistFormatter.USE_COLORS = colored_log == 'true'
 
         self.log = logging.getLogger(self.target_host[0])
 
diff --git a/cdist/flock.py b/cdist/flock.py
index d8bac916..3afacfd9 100644
--- a/cdist/flock.py
+++ b/cdist/flock.py
@@ -22,6 +22,7 @@
 import fcntl
 import logging
 import os
+import cdist.log
 
 
 log = logging.getLogger('cdist-flock')
diff --git a/cdist/log.py b/cdist/log.py
index 19efebdb..efae2cd3 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -50,7 +50,8 @@ def _trace(msg, *args, **kwargs):
 logging.trace = _trace
 
 
-class ColorFormatter(logging.Formatter):
+class CdistFormatter(logging.Formatter):
+    USE_COLORS = False
     RESET = '\033[0m'
     COLOR_MAP = {
         'ERROR': '\033[0;31m',
@@ -66,14 +67,14 @@ class ColorFormatter(logging.Formatter):
 
     def format(self, record):
         msg = super().format(record)
-        color = self.COLOR_MAP.get(record.levelname)
-        if color:
-            msg = color + msg + self.RESET
+        if self.USE_COLORS:
+            color = self.COLOR_MAP.get(record.levelname)
+            if color:
+                msg = color + msg + self.RESET
         return msg
 
 
 class DefaultLog(logging.Logger):
-    USE_COLORS = False
     FORMAT = '%(levelname)s: %(message)s'
 
     class StdoutFilter(logging.Filter):
@@ -88,10 +89,7 @@ class DefaultLog(logging.Logger):
         super().__init__(name)
         self.propagate = False
 
-        if self.USE_COLORS:
-            formatter = ColorFormatter(self.FORMAT)
-        else:
-            formatter = logging.Formatter(self.FORMAT)
+        formatter = CdistFormatter(self.FORMAT)
 
         self.addFilter(self)
 

From 4167f9f60c30a4587b492c2fa3dce2f02ded463a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 11 Jun 2020 14:22:54 +0200
Subject: [PATCH 110/527] Use proper format string with name

---
 cdist/log.py | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/cdist/log.py b/cdist/log.py
index efae2cd3..790059df 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -75,7 +75,7 @@ class CdistFormatter(logging.Formatter):
 
 
 class DefaultLog(logging.Logger):
-    FORMAT = '%(levelname)s: %(message)s'
+    FORMAT = '%(levelname)s: %(name)s: %(message)s'
 
     class StdoutFilter(logging.Filter):
         def filter(self, rec):
@@ -91,8 +91,6 @@ class DefaultLog(logging.Logger):
 
         formatter = CdistFormatter(self.FORMAT)
 
-        self.addFilter(self)
-
         stdout_handler = logging.StreamHandler(sys.stdout)
         stdout_handler.addFilter(self.StdoutFilter())
         stdout_handler.setLevel(logging.TRACE)
@@ -106,13 +104,6 @@ class DefaultLog(logging.Logger):
         self.addHandler(stdout_handler)
         self.addHandler(stderr_handler)
 
-    def filter(self, record):
-        """Prefix messages with logger name"""
-
-        record.msg = self.name + ": " + str(record.msg)
-
-        return True
-
     def verbose(self, msg, *args, **kwargs):
         self.log(logging.VERBOSE, msg, *args, **kwargs)
 
@@ -134,7 +125,7 @@ class TimestampingLog(DefaultLog):
 
 
 class ParallelLog(DefaultLog):
-    FORMAT = '%(levelname)s: [%(process)d]: %(message)s'
+    FORMAT = '%(levelname)s: [%(process)d]: %(name)s: %(message)s'
 
 
 class TimestampingParallelLog(TimestampingLog, ParallelLog):

From eec7ab8e4577008c6a48a473e295a9d1e31f2358 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 10 Jun 2020 16:59:18 +0200
Subject: [PATCH 111/527] Increase minimum supported Python version to 3.5

---
 docs/src/cdist-install.rst | 2 +-
 docs/src/man1/cdist.rst    | 2 +-
 scripts/cdist              | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst
index 0b65cad6..6f4f14d7 100644
--- a/docs/src/cdist-install.rst
+++ b/docs/src/cdist-install.rst
@@ -10,7 +10,7 @@ Source Host
 This is the machine from which you will configure target hosts.
 
  * /bin/sh: A POSIX like shell (for instance bash, dash, zsh)
- * Python >= 3.2
+ * Python >= 3.5
  * SSH client
  * sphinx (for building html docs and/or the man pages)
 
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 68a9a4ee..9bf8fc9b 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -95,7 +95,7 @@ cdist is the frontend executable to the cdist configuration management.
 It supports different subcommands as explained below.
 
 It is written in Python so it requires :strong:`python`\ (1) to be installed.
-It requires a minimal Python version 3.2.
+It requires a minimal Python version 3.5.
 
 GENERAL
 -------
diff --git a/scripts/cdist b/scripts/cdist
index 7bf12c01..664504a0 100755
--- a/scripts/cdist
+++ b/scripts/cdist
@@ -60,7 +60,7 @@ def commandline():
 
 
 if __name__ == "__main__":
-    cdistpythonversion = '3.2'
+    cdistpythonversion = '3.5'
     if sys.version < cdistpythonversion:
         print('Python >= {} is required on the source host.'.format(
                 cdistpythonversion), file=sys.stderr)

From eba3d0505bba4ee0472df3452142cc63c8fe2f5c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 12 Jun 2020 06:29:39 +0200
Subject: [PATCH 112/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index bcde9268..0cc2a714 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,7 @@ next:
 	* Documentation: Fix trivial grammatical mistakes (Jaak Ristioja)
 	* Explorer os: Fix for sles15 (Daniel Heule)
 	* Type __clean_path: Add --path parameter (Ander Punnar)
+	* Core: Increase minimal supported Python version to 3.5 (Darko Poljak)
 
 6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)

From 6e9e9ad557479354938257f04c8fab283b8251e8 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Tue, 12 Nov 2019 00:40:58 +0100
Subject: [PATCH 113/527] implement log server to capture nested logging output

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/core/code.py |  4 +++
 cdist/install.py   | 23 +++++++++++++-
 cdist/log.py       | 77 ++++++++++++++++++++++++++++++++++++++--------
 3 files changed, 90 insertions(+), 14 deletions(-)

diff --git a/cdist/core/code.py b/cdist/core/code.py
index 1550880a..a7d9b7ca 100644
--- a/cdist/core/code.py
+++ b/cdist/core/code.py
@@ -116,6 +116,10 @@ class Code(object):
         if dry_run:
             self.env['__cdist_dry_run'] = '1'
 
+        if '__cdist_log_server_socket_to_export' in os.environ:
+            self.env['__cdist_log_server_socket'] = os.environ['__cdist_log_server_socket_to_export']
+
+
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.cdist_type
         script = os.path.join(self.local.type_path,
diff --git a/cdist/install.py b/cdist/install.py
index b88ad016..3f94ca68 100644
--- a/cdist/install.py
+++ b/cdist/install.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
-# 2013 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2013-2019 Steven Armstrong (steven-cdist at armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -20,11 +20,32 @@
 #
 #
 
+import os
+import logging
+import tempfile
+
 import cdist.config
 import cdist.core
 
 
 class Install(cdist.config.Config):
+
+    @classmethod
+    def onehost(cls, host, host_tags, host_base_path, host_dir_name, args,
+                parallel, configuration, remove_remote_files_dirs=False):
+        # Start a log server so nested `cdist config` runs have a place to
+        # send their logs to.
+        log_server_socket_dir = tempfile.mkdtemp()
+        log_server_socket = os.path.join(log_server_socket_dir, 'log-server')
+        cls._register_path_for_removal(log_server_socket_dir)
+        log = logging.getLogger(host)
+        log.debug('Starting logging server on: %s', log_server_socket)
+        os.environ['__cdist_log_server_socket_to_export'] = log_server_socket
+        cdist.log.setupLogServer(log_server_socket)
+
+        super().onehost(host, host_tags, host_base_path, host_dir_name, args,
+                parallel, configuration, remove_remote_files_dirs=False)
+
     def object_list(self):
         """Short name for object list retrieval.
         In install mode, we only care about install objects.
diff --git a/cdist/log.py b/cdist/log.py
index 790059df..94dd11e8 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 #
 # 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2019-2020 Steven Armstrong
 #
 # This file is part of cdist.
 #
@@ -20,9 +21,17 @@
 #
 #
 
-import logging
-import sys
+import asyncio
+import contextlib
 import datetime
+import logging
+import logging.handlers
+import os
+import pickle
+import struct
+import sys
+import threading
+import time
 
 
 # Define additional cdist logging levels.
@@ -89,20 +98,25 @@ class DefaultLog(logging.Logger):
         super().__init__(name)
         self.propagate = False
 
-        formatter = CdistFormatter(self.FORMAT)
+        if '__cdist_log_server_socket' in os.environ:
+            log_server_socket = os.environ['__cdist_log_server_socket']
+            socket_handler = logging.handlers.SocketHandler(log_server_socket, None)
+            self.addHandler(socket_handler)
+        else:
+            formatter = CdistFormatter(self.FORMAT)
 
-        stdout_handler = logging.StreamHandler(sys.stdout)
-        stdout_handler.addFilter(self.StdoutFilter())
-        stdout_handler.setLevel(logging.TRACE)
-        stdout_handler.setFormatter(formatter)
+            stdout_handler = logging.StreamHandler(sys.stdout)
+            stdout_handler.addFilter(self.StdoutFilter())
+            stdout_handler.setLevel(logging.TRACE)
+            stdout_handler.setFormatter(formatter)
 
-        stderr_handler = logging.StreamHandler(sys.stderr)
-        stderr_handler.addFilter(self.StderrFilter())
-        stderr_handler.setLevel(logging.ERROR)
-        stderr_handler.setFormatter(formatter)
+            stderr_handler = logging.StreamHandler(sys.stderr)
+            stderr_handler.addFilter(self.StderrFilter())
+            stderr_handler.setLevel(logging.ERROR)
+            stderr_handler.setFormatter(formatter)
 
-        self.addHandler(stdout_handler)
-        self.addHandler(stderr_handler)
+            self.addHandler(stdout_handler)
+            self.addHandler(stderr_handler)
 
     def verbose(self, msg, *args, **kwargs):
         self.log(logging.VERBOSE, msg, *args, **kwargs)
@@ -152,4 +166,41 @@ def setupParallelLogging():
     logging.setLoggerClass(ParallelLog)
 
 
+async def handle_log_client(reader, writer):
+    while True:
+        chunk = await reader.read(4)
+        if len(chunk) < 4:
+            return
+
+        data_size = struct.unpack('>L', chunk)[0]
+        data = bytearray(data_size)
+        view = memoryview(data)
+        data_pending = data_size
+        data = await reader.read(data_size)
+
+        obj = pickle.loads(data)
+        record = logging.makeLogRecord(obj)
+        logger = logging.getLogger(record.name)
+        logger.handle(record)
+
+
+def run_log_server(server_address):
+    # Get a new loop inside the current thread to run the log server.
+    loop = asyncio.new_event_loop()
+    loop.create_task(asyncio.start_unix_server(handle_log_client, server_address))
+    loop.run_forever()
+
+
+def setupLogServer(log_server_socket):
+    """Run a asyncio based unix socket log server in a background thread.
+    """
+    with contextlib.suppress(FileNotFoundError):
+        os.remove(log_server_socket)
+    t = threading.Thread(target=run_log_server, args=(log_server_socket,))
+    # Deamonizing the thread means we don't have to care about stoping it.
+    # It will die together with the main process.
+    t.daemon = True
+    t.start()
+
+
 setupDefaultLogging()

From 831bfc822b61a5e5cf8186a32ea592622693efdb Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Tue, 12 Nov 2019 09:22:28 +0100
Subject: [PATCH 114/527] remove unused code

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/log.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/cdist/log.py b/cdist/log.py
index 94dd11e8..c1376a58 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -173,9 +173,6 @@ async def handle_log_client(reader, writer):
             return
 
         data_size = struct.unpack('>L', chunk)[0]
-        data = bytearray(data_size)
-        view = memoryview(data)
-        data_pending = data_size
         data = await reader.read(data_size)
 
         obj = pickle.loads(data)

From 57e352cd1e7b400ca1daf7e1e680874bd9a6cc01 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Sat, 11 Jan 2020 01:59:18 +0100
Subject: [PATCH 115/527] log server is also usefull for cdist config

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/argparse.py |  6 ++++++
 cdist/config.py   | 10 ++++++++++
 cdist/install.py  | 13 +++----------
 3 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 1c83237c..15b9ca7a 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -250,6 +250,12 @@ def get_parsers():
            '-S', '--disable-saving-output-streams',
            help='Disable saving output streams.',
            action='store_false', dest='save_output_streams', default=True)
+    parser['config_main'].add_argument(
+           '--log-server',
+           action='store_true',
+           help=('Start a log server for sub processes to use.'
+            'This is mainly usefull when running cdist nested'
+            'from a code-local script.'))
 
     # Config
     parser['config_args'] = argparse.ArgumentParser(add_help=False)
diff --git a/cdist/config.py b/cdist/config.py
index b2d72f05..82e8559b 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -385,6 +385,16 @@ class Config(object):
 
         log = logging.getLogger(host)
 
+        if args.log_server:
+            # Start a log server so that nested `cdist config` runs have a place
+            # to send their logs to.
+            log_server_socket_dir = tempfile.mkdtemp()
+            log_server_socket = os.path.join(log_server_socket_dir, 'log-server')
+            cls._register_path_for_removal(log_server_socket_dir)
+            log.debug('Starting logging server on: %s', log_server_socket)
+            os.environ['__cdist_log_server_socket_to_export'] = log_server_socket
+            cdist.log.setupLogServer(log_server_socket)
+
         try:
             remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds(
                 args)
diff --git a/cdist/install.py b/cdist/install.py
index 3f94ca68..ec0a7270 100644
--- a/cdist/install.py
+++ b/cdist/install.py
@@ -33,16 +33,9 @@ class Install(cdist.config.Config):
     @classmethod
     def onehost(cls, host, host_tags, host_base_path, host_dir_name, args,
                 parallel, configuration, remove_remote_files_dirs=False):
-        # Start a log server so nested `cdist config` runs have a place to
-        # send their logs to.
-        log_server_socket_dir = tempfile.mkdtemp()
-        log_server_socket = os.path.join(log_server_socket_dir, 'log-server')
-        cls._register_path_for_removal(log_server_socket_dir)
-        log = logging.getLogger(host)
-        log.debug('Starting logging server on: %s', log_server_socket)
-        os.environ['__cdist_log_server_socket_to_export'] = log_server_socket
-        cdist.log.setupLogServer(log_server_socket)
-
+        # Always start log server during cdist install so that nested
+        # `cdist config` runs have a place to send their logs to.
+        args.log_server = True
         super().onehost(host, host_tags, host_base_path, host_dir_name, args,
                 parallel, configuration, remove_remote_files_dirs=False)
 

From 59b98091d78c5195d2aa4a2ea3ef79f93e5ba67c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 9 Jun 2020 12:47:50 +0200
Subject: [PATCH 116/527] Adapt; update docs and code style

---
 cdist/__init__.py       |  4 ++++
 cdist/argparse.py       | 13 +++++++------
 cdist/config.py         | 27 ++++++++++++---------------
 cdist/core/code.py      |  6 +++---
 cdist/install.py        |  8 +++-----
 cdist/log.py            | 18 +++++++++++-------
 docs/src/man1/cdist.rst | 26 ++++++++++++++++----------
 scripts/cdist           |  5 ++---
 8 files changed, 58 insertions(+), 49 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index c673b3ba..be573170 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -26,6 +26,7 @@ import hashlib
 import cdist.log
 import cdist.version
 
+
 VERSION = cdist.version.VERSION
 
 BANNER = """
@@ -48,6 +49,9 @@ REMOTE_EXEC = "ssh -o User=root"
 REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
 
 
+MIN_SUPPORTED_PYTHON_VERSION = '3.5'
+
+
 class Error(Exception):
     """Base exception class for this project"""
     pass
diff --git a/cdist/argparse.py b/cdist/argparse.py
index 15b9ca7a..77303591 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -206,6 +206,13 @@ def get_parsers():
                  'supported. Without argument CPU count is used by default. '),
            action='store', dest='jobs',
            const=multiprocessing.cpu_count())
+    parser['config_main'].add_argument(
+           '--log-server',
+           action='store_true',
+           help=('Start a log server for sub processes to use. '
+                 'This is mainly useful when running cdist nested '
+                 'from a code-local script. Log server is alwasy '
+                 'implicitly started for \'install\' command.'))
     parser['config_main'].add_argument(
            '-n', '--dry-run',
            help='Do not execute code.', action='store_true')
@@ -250,12 +257,6 @@ def get_parsers():
            '-S', '--disable-saving-output-streams',
            help='Disable saving output streams.',
            action='store_false', dest='save_output_streams', default=True)
-    parser['config_main'].add_argument(
-           '--log-server',
-           action='store_true',
-           help=('Start a log server for sub processes to use.'
-            'This is mainly usefull when running cdist nested'
-            'from a code-local script.'))
 
     # Config
     parser['config_args'] = argparse.ArgumentParser(add_help=False)
diff --git a/cdist/config.py b/cdist/config.py
index 82e8559b..9fe9b676 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -29,18 +29,20 @@ import time
 import itertools
 import tempfile
 import multiprocessing
-from cdist.mputil import mp_pool_run, mp_sig_handler
 import atexit
 import shutil
 import socket
+
+from cdist.mputil import mp_pool_run, mp_sig_handler
+from cdist import core, inventory
+from cdist.util.remoteutil import inspect_ssh_mux_opts
+
 import cdist
 import cdist.hostsource
 import cdist.exec.local
 import cdist.exec.remote
 import cdist.util.ipaddr as ipaddr
 import cdist.configuration
-from cdist import core, inventory
-from cdist.util.remoteutil import inspect_ssh_mux_opts
 
 
 def graph_check_cycle(graph):
@@ -195,7 +197,6 @@ class Config(object):
     @classmethod
     def commandline(cls, args):
         """Configure remote system"""
-
         if (args.parallel and args.parallel != 1) or args.jobs:
             if args.timestamp:
                 cdist.log.setupTimestampingParallelLogging()
@@ -382,20 +383,16 @@ class Config(object):
            If operating in parallel then return tuple (host, True|False, )
            so that main process knows for which host function was successful.
         """
-
         log = logging.getLogger(host)
 
-        if args.log_server:
-            # Start a log server so that nested `cdist config` runs have a place
-            # to send their logs to.
-            log_server_socket_dir = tempfile.mkdtemp()
-            log_server_socket = os.path.join(log_server_socket_dir, 'log-server')
-            cls._register_path_for_removal(log_server_socket_dir)
-            log.debug('Starting logging server on: %s', log_server_socket)
-            os.environ['__cdist_log_server_socket_to_export'] = log_server_socket
-            cdist.log.setupLogServer(log_server_socket)
-
         try:
+            if args.log_server:
+                # Start a log server so that nested `cdist config` runs
+                # have a place to send their logs to.
+                log_server_socket_dir = tempfile.mkdtemp()
+                cls._register_path_for_removal(log_server_socket_dir)
+                cdist.log.setupLogServer(log_server_socket_dir, log)
+
             remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds(
                 args)
             log.debug("remote_exec for host \"{}\": {}".format(
diff --git a/cdist/core/code.py b/cdist/core/code.py
index a7d9b7ca..2a30908e 100644
--- a/cdist/core/code.py
+++ b/cdist/core/code.py
@@ -116,9 +116,9 @@ class Code(object):
         if dry_run:
             self.env['__cdist_dry_run'] = '1'
 
-        if '__cdist_log_server_socket_to_export' in os.environ:
-            self.env['__cdist_log_server_socket'] = os.environ['__cdist_log_server_socket_to_export']
-
+        if '__cdist_log_server_socket_export' in os.environ:
+            self.env['__cdist_log_server_socket'] = os.environ[
+                '__cdist_log_server_socket_export']
 
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.cdist_type
diff --git a/cdist/install.py b/cdist/install.py
index ec0a7270..561b2fa6 100644
--- a/cdist/install.py
+++ b/cdist/install.py
@@ -20,12 +20,9 @@
 #
 #
 
-import os
-import logging
-import tempfile
-
 import cdist.config
 import cdist.core
+import cdist.log
 
 
 class Install(cdist.config.Config):
@@ -36,8 +33,9 @@ class Install(cdist.config.Config):
         # Always start log server during cdist install so that nested
         # `cdist config` runs have a place to send their logs to.
         args.log_server = True
+
         super().onehost(host, host_tags, host_base_path, host_dir_name, args,
-                parallel, configuration, remove_remote_files_dirs=False)
+                        parallel, configuration, remove_remote_files_dirs)
 
     def object_list(self):
         """Short name for object list retrieval.
diff --git a/cdist/log.py b/cdist/log.py
index c1376a58..bee99fac 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -21,17 +21,16 @@
 #
 #
 
-import asyncio
-import contextlib
 import datetime
 import logging
 import logging.handlers
+import sys
 import os
+import asyncio
+import contextlib
 import pickle
 import struct
-import sys
 import threading
-import time
 
 
 # Define additional cdist logging levels.
@@ -100,7 +99,8 @@ class DefaultLog(logging.Logger):
 
         if '__cdist_log_server_socket' in os.environ:
             log_server_socket = os.environ['__cdist_log_server_socket']
-            socket_handler = logging.handlers.SocketHandler(log_server_socket, None)
+            socket_handler = logging.handlers.SocketHandler(log_server_socket,
+                                                            None)
             self.addHandler(socket_handler)
         else:
             formatter = CdistFormatter(self.FORMAT)
@@ -184,13 +184,17 @@ async def handle_log_client(reader, writer):
 def run_log_server(server_address):
     # Get a new loop inside the current thread to run the log server.
     loop = asyncio.new_event_loop()
-    loop.create_task(asyncio.start_unix_server(handle_log_client, server_address))
+    loop.create_task(asyncio.start_unix_server(handle_log_client,
+                                               server_address))
     loop.run_forever()
 
 
-def setupLogServer(log_server_socket):
+def setupLogServer(socket_dir, log=logging.getLogger(__name__)):
     """Run a asyncio based unix socket log server in a background thread.
     """
+    log_server_socket = os.path.join(socket_dir, 'log-server')
+    log.debug('Starting logging server on: %s', log_server_socket)
+    os.environ['__cdist_log_server_socket_export'] = log_server_socket
     with contextlib.suppress(FileNotFoundError):
         os.remove(log_server_socket)
     t = threading.Thread(target=run_log_server, args=(log_server_socket,))
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 9bf8fc9b..aa2607f8 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -17,20 +17,20 @@ SYNOPSIS
 
     cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
                  [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN]
-                 [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-P]
-                 [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
-                 [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
-                 [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE]
-                 [-p [HOST_MAX]] [-s] [-t]
+                 [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [--log-server]
+                 [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]]
+                 [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY]
+                 [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A]
+                 [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
                  [host [host ...]]
 
     cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
                   [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN]
-                  [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
-                  [-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
-                  [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
-                  [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE]
-                  [-p [HOST_MAX]] [-s] [-t]
+                  [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [--log-server]
+                  [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]]
+                  [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY]
+                  [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A]
+                  [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
                   [host [host ...]]
 
     cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ...
@@ -202,6 +202,12 @@ Install command is currently in beta.
     are supported. Without argument CPU count is used by
     default.
 
+**--log-server**
+    Start a log server for sub processes to use. This is
+    mainly useful when running cdist nested from a code-
+    local script. Log server is always implicitly started
+    for 'install' command.
+
 **-n, --dry-run**
     Do not execute code.
 
diff --git a/scripts/cdist b/scripts/cdist
index 664504a0..b1d782ab 100755
--- a/scripts/cdist
+++ b/scripts/cdist
@@ -60,10 +60,9 @@ def commandline():
 
 
 if __name__ == "__main__":
-    cdistpythonversion = '3.5'
-    if sys.version < cdistpythonversion:
+    if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION:
         print('Python >= {} is required on the source host.'.format(
-                cdistpythonversion), file=sys.stderr)
+                cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr)
         sys.exit(1)
 
     exit_code = 0

From 5be8437a602907009d7603fa41ae0a4e6528ffdf Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 13 Jun 2020 13:48:34 +0200
Subject: [PATCH 117/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 0cc2a714..fa5f7405 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,6 +10,7 @@ next:
 	* Explorer os: Fix for sles15 (Daniel Heule)
 	* Type __clean_path: Add --path parameter (Ander Punnar)
 	* Core: Increase minimal supported Python version to 3.5 (Darko Poljak)
+	* Core: Add log server for nested logging (Steven Armstrong)
 
 6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)

From 201050a9e5144d0f7da2a8019e5822fedaa1d1b1 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 14 Jun 2020 11:07:18 +0300
Subject: [PATCH 118/527] new type: __download

---
 cdist/conf/type/__download/explorer/state     | 20 ++++++
 cdist/conf/type/__download/gencode-local      | 35 ++++++++++
 cdist/conf/type/__download/man.rst            | 66 +++++++++++++++++++
 .../type/__download/parameter/default/cmd-get |  1 +
 .../type/__download/parameter/default/cmd-sum |  1 +
 cdist/conf/type/__download/parameter/optional |  2 +
 cdist/conf/type/__download/parameter/required |  2 +
 7 files changed, 127 insertions(+)
 create mode 100755 cdist/conf/type/__download/explorer/state
 create mode 100755 cdist/conf/type/__download/gencode-local
 create mode 100644 cdist/conf/type/__download/man.rst
 create mode 100644 cdist/conf/type/__download/parameter/default/cmd-get
 create mode 100644 cdist/conf/type/__download/parameter/default/cmd-sum
 create mode 100644 cdist/conf/type/__download/parameter/optional
 create mode 100644 cdist/conf/type/__download/parameter/required

diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
new file mode 100755
index 00000000..6a50f5a5
--- /dev/null
+++ b/cdist/conf/type/__download/explorer/state
@@ -0,0 +1,20 @@
+#!/bin/sh -e
+
+dst="/$__object_id"
+
+# shellcheck disable=SC2059
+cmd="$( printf "$( cat "$__object/parameter/cmd-sum" )" "$dst" )"
+
+sum="$( cat "$__object/parameter/sum" )"
+
+if [ -f "$dst" ]
+then
+    if [ "$( eval "$cmd" )" = "$sum" ]
+    then
+        echo 'present'
+    else
+        echo 'mismatch'
+    fi
+else
+    echo 'absent'
+fi
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
new file mode 100755
index 00000000..49e9c699
--- /dev/null
+++ b/cdist/conf/type/__download/gencode-local
@@ -0,0 +1,35 @@
+#!/bin/sh -e
+
+state_is="$( cat "$__object/explorer/state" )"
+
+if [ "$state_is" = 'present' ]
+then
+    exit 0
+fi
+
+url="$( cat "$__object/parameter/url" )"
+
+cmd="$( cat "$__object/parameter/cmd-get" )"
+
+tmp="$( mktemp )"
+
+dst="/$__object_id"
+
+printf "$cmd > %s\n" \
+    "$url" \
+    "$tmp"
+
+if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$'
+then
+    target_host="[$__target_host]"
+else
+    target_host="$__target_host"
+fi
+
+printf '%s %s %s:%s\n' \
+    "$__remote_copy" \
+    "$tmp" \
+    "$target_host" \
+    "$dst"
+
+echo "rm -f '$tmp'"
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
new file mode 100644
index 00000000..c973448f
--- /dev/null
+++ b/cdist/conf/type/__download/man.rst
@@ -0,0 +1,66 @@
+cdist-type__download(7)
+=======================
+
+NAME
+----
+cdist-type__download - Download file to local storage and copy it to target host
+
+
+DESCRIPTION
+-----------
+You must use persistent storage in target host for destination file
+(``$__object_id``) because it will be used for checksum calculation
+in order to decide if file must be downloaded.
+
+
+REQUIRED PARAMETERS
+-------------------
+url
+   URL from which to download the file.
+
+sum
+   Checksum of downloaded file.
+
+
+OPTIONAL PARAMETERS
+-------------------
+cmd-get
+   Command used for downloading.
+   Default is ``wget -O- '%s'``.
+   Command must output to ``stdout``.
+
+cmd-sum
+   Command used for checksum calculation.
+   Default is ``md5sum '%s' | awk '{print $1}'``.
+   Command output and ``--sum`` parameter must match.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __directory /opt/cpma
+
+    require='__directory/opt/cpma' \
+        __download /opt/cpma/cnq3.zip \
+            --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
+            --sum 46da3021ca9eace277115ec9106c5b46
+
+    require='__download/opt/cpma/cnq3.zip' \
+        __unpack /opt/cpma/cnq3.zip \
+            --move-existing-destination \
+            --destination /opt/cpma/server
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__download/parameter/default/cmd-get b/cdist/conf/type/__download/parameter/default/cmd-get
new file mode 100644
index 00000000..2daa38a1
--- /dev/null
+++ b/cdist/conf/type/__download/parameter/default/cmd-get
@@ -0,0 +1 @@
+wget -O- '%s'
diff --git a/cdist/conf/type/__download/parameter/default/cmd-sum b/cdist/conf/type/__download/parameter/default/cmd-sum
new file mode 100644
index 00000000..3e8a9295
--- /dev/null
+++ b/cdist/conf/type/__download/parameter/default/cmd-sum
@@ -0,0 +1 @@
+md5sum '%s' | awk '{print $1}'
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
new file mode 100644
index 00000000..22783e02
--- /dev/null
+++ b/cdist/conf/type/__download/parameter/optional
@@ -0,0 +1,2 @@
+cmd-get
+cmd-sum
diff --git a/cdist/conf/type/__download/parameter/required b/cdist/conf/type/__download/parameter/required
new file mode 100644
index 00000000..6ea4c38f
--- /dev/null
+++ b/cdist/conf/type/__download/parameter/required
@@ -0,0 +1,2 @@
+url
+sum

From cdb998398d25f1ba99f01364bbc42b900be944d5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 17 Jun 2020 12:10:58 +0200
Subject: [PATCH 119/527] Release 6.6.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index fa5f7405..c8388b93 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)
 	* Core: Support colored log output (Evil Ham)
 	* Core: Tune colored log output and respect NO_COLOR env var (Dennis Camera)

From a6543a72ade5a3b64cab1b0164d5df0b4388e88f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 17 Jun 2020 13:40:31 +0200
Subject: [PATCH 120/527] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index c8388b93..a0de09fa 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* New type: __download (Ander Punnar)
+
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)
 	* Core: Support colored log output (Evil Ham)

From 97e48be39e80ffe50946606b912ca28d3612cc61 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 21:11:28 +0200
Subject: [PATCH 121/527] [type/__package_opkg] Fix explorer running in
 parallel

---
 .../type/__package_opkg/explorer/pkg_status   | 57 ++++++++++++++-----
 1 file changed, 44 insertions(+), 13 deletions(-)

diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status
index 5da4f742..1ba88e81 100755
--- a/cdist/conf/type/__package_opkg/explorer/pkg_status
+++ b/cdist/conf/type/__package_opkg/explorer/pkg_status
@@ -1,7 +1,8 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011 Nico Schottelius (nico-cdist at schottelius.org)
 # 2012 Giel van Schijndel (giel plus cdist at mortis dot eu)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,21 +20,51 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Retrieve the status of a package - parsed opkg output
+# Retrieve the status of a package - parses opkg output
 #
 
-if [ -f "$__object/parameter/name" ]; then
-   name="$(cat "$__object/parameter/name")"
+LOCKFILE="${__type_explorer}/cdist_opkg.lock"
+_lock() (
+	set -o noclobber
+	until echo $$>"${LOCKFILE}"
+	do
+		while test -f "${LOCKFILE}"; do sleep 1; done
+	done
+
+) 2>/dev/null
+_unlock() {
+	if test -s "${LOCKFILE}" && test "$(cat "${LOCKFILE}")" = $$
+	then
+		rm "${LOCKFILE}"
+	fi
+}
+
+
+if test -f "${__object}/parameter/name"
+then
+	pkg_name=$(cat "${__object}/parameter/name")
 else
-   name="$__object_id"
+	pkg_name=$__object_id
 fi
 
-# Except dpkg failing, if package is not known / installed
-if opkg status "$name" 2>/dev/null | grep -q "^Status: install user installed$"; then
-   echo "present"
-   exit 0
-elif [ "$(opkg info "$name" 2> /dev/null | wc -l)" -eq 0 ]; then
-   echo "absent notpresent"
-   exit 0
+
+# NOTE: We need to lock parallel execution of this explorer because opkg will
+# try to acquire the OPKG lock (usually /var/lock/opkg.lock) using lockf(2) for
+# every operation. It will not wait for the lock but terminate with an error.
+# This leads to incorrect 'absent notpresent' statuses when parallel execution
+# is enabled.
+trap _unlock EXIT
+_lock
+
+
+# Except opkg failing, if package is not known / installed
+if opkg status "${pkg_name}" 2>/dev/null \
+	| grep -q -e '^Status: [^ ][^ ]* [^ ][^ ]* installed$'
+then
+	echo 'present'
+elif opkg info "${pkg_name}" 2>/dev/null | grep -q .
+then
+	echo 'absent notpresent'
+else
+	echo 'absent'
 fi
-echo "absent"

From e79b26a61f790108b90c3625d554f6df4086d616 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 21 Jun 2020 13:15:38 +0200
Subject: [PATCH 122/527] [type/__package_opkg] Also lock execution of
 code-remote

---
 .../type/__package_opkg/explorer/pkg_status   | 11 ++-
 cdist/conf/type/__package_opkg/gencode-remote | 94 +++++++++++++------
 2 files changed, 73 insertions(+), 32 deletions(-)

diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status
index 1ba88e81..f5a6f098 100755
--- a/cdist/conf/type/__package_opkg/explorer/pkg_status
+++ b/cdist/conf/type/__package_opkg/explorer/pkg_status
@@ -23,7 +23,9 @@
 # Retrieve the status of a package - parses opkg output
 #
 
-LOCKFILE="${__type_explorer}/cdist_opkg.lock"
+__type_path=${__object%%${__object_id}*}
+
+LOCKFILE="${__type_path}/cdist_opkg.lock"
 _lock() (
 	set -o noclobber
 	until echo $$>"${LOCKFILE}"
@@ -48,9 +50,10 @@ else
 fi
 
 
-# NOTE: We need to lock parallel execution of this explorer because opkg will
-# try to acquire the OPKG lock (usually /var/lock/opkg.lock) using lockf(2) for
-# every operation. It will not wait for the lock but terminate with an error.
+# NOTE: We need to lock parallel execution of type explorers and code-remote
+# because opkg will try to acquire the OPKG lock (usually /var/lock/opkg.lock)
+# using lockf(2) for every operation.
+# It will not wait for the lock but terminate with an error.
 # This leads to incorrect 'absent notpresent' statuses when parallel execution
 # is enabled.
 trap _unlock EXIT
diff --git a/cdist/conf/type/__package_opkg/gencode-remote b/cdist/conf/type/__package_opkg/gencode-remote
index 269d5f49..ad90dc24 100755
--- a/cdist/conf/type/__package_opkg/gencode-remote
+++ b/cdist/conf/type/__package_opkg/gencode-remote
@@ -2,6 +2,7 @@
 #
 # 2011,2013 Nico Schottelius (nico-cdist at schottelius.org)
 # 2012 Giel van Schijndel (giel plus cdist at mortis dot eu)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,41 +20,78 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Manage packages on OpenWRT and co.
+# Manage packages on OpenWrt, optware, and co.
 #
 
-if [ -f "$__object/parameter/name" ]; then
-   name="$(cat "$__object/parameter/name")"
+if test -f "${__object}/parameter/name"
+then
+   name=$(cat "${__object}/parameter/name")
 else
-   name="$__object_id"
+   name=$__object_id
 fi
 
-state_should="$(cat "$__object/parameter/state")"
+state_should=$(cat "${__object}/parameter/state")
+state_is=$(cat "${__object}/explorer/pkg_status")
 
-state_is="$(cat "$__object/explorer/pkg_status")"
-case "$state_is" in
-    absent*)
-       present="$(echo "$state_is" | cut -d ' ' -f 2)"
-       state_is="absent"
-    ;;
+case $state_is
+in
+	(absent*)
+		presence=$(echo "${state_is}" | cut -d ' ' -f 2)
+		state_is='absent'
+		;;
 esac
 
-[ "$state_is" = "$state_should" ] && exit 0
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
 
-case "$state_should" in
-    present)
-        if [ "$present" = "notpresent" ]; then
-            echo "opkg --verbosity=0 update"
-        fi
-        echo "opkg --verbosity=0 install '$name'"
-        echo "installed" >> "$__messages_out"
-    ;;
-    absent)
-        echo "opkg --verbosity=0 remove '$name'"
-        echo "removed" >> "$__messages_out"
-    ;;
-    *)
-        echo "Unknown state: ${state_should}" >&2
-        exit 1
-    ;;
+cat <<'EOF'
+__type_path=${__object%%${__object_id}*}
+
+LOCKFILE="${__type_path}/cdist_opkg.lock"
+_lock() (
+	set -o noclobber
+	until echo $$>"${LOCKFILE}"
+	do
+		while test -f "${LOCKFILE}"; do sleep 1; done
+	done
+
+) 2>/dev/null
+_unlock() {
+	if test -s "${LOCKFILE}" && test "$(cat "${LOCKFILE}")" = $$
+	then
+		rm "${LOCKFILE}"
+	fi
+}
+EOF
+
+# NOTE: We need to lock parallel execution of code-remote to ensure that it is
+# not executed concurrently with a type explorer.
+# opkg will try to acquire the OPKG lock (usually /var/lock/opkg.lock) using
+# lockf(2) for every operation.
+# It will not wait for the lock but terminate with an error leading to an
+# incorrect outcome.
+echo 'trap _unlock EXIT'
+echo '_lock'
+
+case $state_should
+in
+	(present)
+		if test "${presence}" = 'notpresent'
+		then
+			echo 'opkg --verbosity=0 update'
+		fi
+
+		printf "opkg --verbosity=0 install '%s'\n" "${name}"
+		echo 'installed' >>"${__messages_out}"
+		;;
+	(absent)
+		printf "opkg --verbosity=0 remove '%s'" "${name}"
+		echo 'removed' >>"${__messages_out}"
+		;;
+	(*)
+		printf 'Unknown state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
 esac

From a6a3fb40bfcbaa5d6506a1ac5ebe2e40376dcb6f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 22:22:29 +0200
Subject: [PATCH 123/527] Remove unnecessary Python shebangs

---
 cdist/config.py                        | 1 -
 cdist/info.py                          | 1 -
 cdist/install.py                       | 1 -
 cdist/integration.py                   | 1 -
 cdist/inventory.py                     | 1 -
 cdist/log.py                           | 1 -
 cdist/preos/debootstrap/debootstrap.py | 1 -
 7 files changed, 7 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index b2d72f05..b71536a8 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
diff --git a/cdist/info.py b/cdist/info.py
index b896a3d1..9e29f5d9 100644
--- a/cdist/info.py
+++ b/cdist/info.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2019-2020 Darko Poljak (darko.poljak at gmail.com)
diff --git a/cdist/install.py b/cdist/install.py
index b88ad016..a9c8119a 100644
--- a/cdist/install.py
+++ b/cdist/install.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2013 Steven Armstrong (steven-cdist at armstrong.cc)
diff --git a/cdist/integration.py b/cdist/integration.py
index ee742cc5..03e4167d 100644
--- a/cdist/integration.py
+++ b/cdist/integration.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2017 Darko Poljak (darko.poljak at gmail.com)
diff --git a/cdist/inventory.py b/cdist/inventory.py
index 138a2034..fb5ab960 100644
--- a/cdist/inventory.py
+++ b/cdist/inventory.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2016 Darko Poljak (darko.poljak at gmail.com)
diff --git a/cdist/log.py b/cdist/log.py
index 19efebdb..c77ba8ec 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py
index f53dd4a7..d3e590f2 100644
--- a/cdist/preos/debootstrap/debootstrap.py
+++ b/cdist/preos/debootstrap/debootstrap.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2016 Darko Poljak (darko.poljak at ungleich.ch)

From ce07021580219d130c3968c547d03f451f2dea12 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 21:16:23 +0200
Subject: [PATCH 124/527] Do not subclass object

---
 cdist/config.py                        | 2 +-
 cdist/core/cdist_object.py             | 2 +-
 cdist/core/cdist_type.py               | 2 +-
 cdist/core/code.py                     | 2 +-
 cdist/core/explorer.py                 | 2 +-
 cdist/core/manifest.py                 | 2 +-
 cdist/emulator.py                      | 2 +-
 cdist/exec/local.py                    | 2 +-
 cdist/exec/remote.py                   | 2 +-
 cdist/hostsource.py                    | 2 +-
 cdist/info.py                          | 3 +--
 cdist/inventory.py                     | 2 +-
 cdist/message.py                       | 2 +-
 cdist/preos.py                         | 2 +-
 cdist/preos/debootstrap/debootstrap.py | 2 +-
 cdist/shell.py                         | 4 +---
 cdist/test/config/__init__.py          | 2 +-
 cdist/util/fsproperty.py               | 4 ++--
 18 files changed, 19 insertions(+), 22 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index b2d72f05..982d8b75 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -70,7 +70,7 @@ def _graph_dfs_cycle(graph, node, path):
     return False
 
 
-class Config(object):
+class Config:
     """Cdist main class to hold arbitrary data"""
 
     # list of paths (files and/or directories) that will be removed on finish
diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index 114a47e0..51d61e04 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -47,7 +47,7 @@ class MissingObjectIdError(cdist.Error):
         return '%s' % (self.message)
 
 
-class CdistObject(object):
+class CdistObject:
     """Represents a cdist object.
 
     All interaction with objects in cdist should be done through this class.
diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py
index 4500f50d..c0329c8a 100644
--- a/cdist/core/cdist_type.py
+++ b/cdist/core/cdist_type.py
@@ -38,7 +38,7 @@ class InvalidTypeError(cdist.Error):
                 self.type_path, self.type_absolute_path, self.source_path)
 
 
-class CdistType(object):
+class CdistType:
     """Represents a cdist type.
 
     All interaction with types in cdist should be done through this class.
diff --git a/cdist/core/code.py b/cdist/core/code.py
index 1550880a..226bc63d 100644
--- a/cdist/core/code.py
+++ b/cdist/core/code.py
@@ -92,7 +92,7 @@ code-remote
 '''
 
 
-class Code(object):
+class Code:
     """Generates and executes cdist code scripts.
 
     """
diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py
index 353d7681..a3baa959 100644
--- a/cdist/core/explorer.py
+++ b/cdist/core/explorer.py
@@ -63,7 +63,7 @@ type explorer is:
 '''
 
 
-class Explorer(object):
+class Explorer:
     """Executes cdist explorers.
 
     """
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 8b833ff2..2a0b2189 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -92,7 +92,7 @@ class NoInitialManifestError(cdist.Error):
         return repr(self.message)
 
 
-class Manifest(object):
+class Manifest:
     """Executes cdist manifests.
 
     """
diff --git a/cdist/emulator.py b/cdist/emulator.py
index 4eaf2c93..24d239fa 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -56,7 +56,7 @@ class DefaultList(list):
             return cls(initial.split('\n'))
 
 
-class Emulator(object):
+class Emulator:
     def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ):
         self.argv = argv
         self.stdin = stdin
diff --git a/cdist/exec/local.py b/cdist/exec/local.py
index ad6c6e36..e0aab190 100644
--- a/cdist/exec/local.py
+++ b/cdist/exec/local.py
@@ -39,7 +39,7 @@ import cdist.exec.util as util
 CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ]
 
 
-class Local(object):
+class Local:
     """Execute commands locally.
 
     All interaction with the local side should be done through this class.
diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py
index f72bf3bf..e5af2f34 100644
--- a/cdist/exec/remote.py
+++ b/cdist/exec/remote.py
@@ -49,7 +49,7 @@ class DecodeError(cdist.Error):
         return "Cannot decode output of " + " ".join(self.command)
 
 
-class Remote(object):
+class Remote:
     """Execute commands remotely.
 
     All interaction with the remote side should be done through this class.
diff --git a/cdist/hostsource.py b/cdist/hostsource.py
index a7b8f0b4..5f927b36 100644
--- a/cdist/hostsource.py
+++ b/cdist/hostsource.py
@@ -41,7 +41,7 @@ def hostfile_process_line(line, strip_func=str.strip):
         return None
 
 
-class HostSource(object):
+class HostSource:
     """
     Host source object.
     Source can be a sequence or filename (stdin if \'-\').
diff --git a/cdist/info.py b/cdist/info.py
index b896a3d1..a1fad237 100644
--- a/cdist/info.py
+++ b/cdist/info.py
@@ -29,8 +29,7 @@ import glob
 import fnmatch
 
 
-class Info(object):
-
+class Info:
     def __init__(self, conf_dirs, args):
         self.conf_dirs = conf_dirs
         self.all = args.all
diff --git a/cdist/inventory.py b/cdist/inventory.py
index 138a2034..c06efff0 100644
--- a/cdist/inventory.py
+++ b/cdist/inventory.py
@@ -80,7 +80,7 @@ def rstrip_nl(s):
     return str.rstrip(s, "\n")
 
 
-class Inventory(object):
+class Inventory:
     """Inventory main class"""
 
     def __init__(self, db_basedir=dist_inventory_db, configuration=None):
diff --git a/cdist/message.py b/cdist/message.py
index 450fc3c3..ffa8c2bb 100644
--- a/cdist/message.py
+++ b/cdist/message.py
@@ -27,7 +27,7 @@ import tempfile
 log = logging.getLogger(__name__)
 
 
-class Message(object):
+class Message:
     """Support messaging between types
 
     """
diff --git a/cdist/preos.py b/cdist/preos.py
index bf2a8e60..f8a5dd67 100644
--- a/cdist/preos.py
+++ b/cdist/preos.py
@@ -78,7 +78,7 @@ def get_available_preoses_string(cls):
     return "Available PreOS-es:\n{}".format("\n".join(preoses))
 
 
-class PreOS(object):
+class PreOS:
     preoses = None
 
     @classmethod
diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py
index f53dd4a7..ac01e08b 100644
--- a/cdist/preos/debootstrap/debootstrap.py
+++ b/cdist/preos/debootstrap/debootstrap.py
@@ -31,7 +31,7 @@ import os
 import subprocess
 
 
-class Debian(object):
+class Debian:
     _preos_name = 'debian'
     _cdist_preos = True
 
diff --git a/cdist/shell.py b/cdist/shell.py
index 60b6a9f0..04a68937 100644
--- a/cdist/shell.py
+++ b/cdist/shell.py
@@ -32,10 +32,8 @@ import cdist.config
 log = logging.getLogger(__name__)
 
 
-class Shell(object):
-
+class Shell:
     def __init__(self, shell=None):
-
         self.shell = shell
 
         self.target_host = (
diff --git a/cdist/test/config/__init__.py b/cdist/test/config/__init__.py
index 499593e3..0ed614b1 100644
--- a/cdist/test/config/__init__.py
+++ b/cdist/test/config/__init__.py
@@ -44,7 +44,7 @@ expected_object_names = sorted([
     '__third/moon'])
 
 
-class CdistObjectErrorContext(object):
+class CdistObjectErrorContext:
     def __init__(self, original_error):
         self.original_error = original_error
 
diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py
index 5a27c9d7..1d76fd76 100644
--- a/cdist/util/fsproperty.py
+++ b/cdist/util/fsproperty.py
@@ -177,7 +177,7 @@ class DirectoryDict(collections.MutableMapping):
             raise cdist.Error(str(e))
 
 
-class FileBasedProperty(object):
+class FileBasedProperty:
     attribute_class = None
 
     def __init__(self, path):
@@ -189,7 +189,7 @@ class FileBasedProperty(object):
 
         Usage with a sublcass:
 
-        class Foo(object):
+        class Foo:
             # note that the actual DirectoryDict is stored as __parameters
             # on the instance
             parameters = DirectoryDictProperty(

From 6aae58dea77002dea0b57d20cbb01b5e2a1066b5 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 21 Jun 2020 17:35:28 +0200
Subject: [PATCH 125/527] [type/__package_opkg] Mark lock variables readonly

---
 cdist/conf/type/__package_opkg/explorer/pkg_status | 4 ++--
 cdist/conf/type/__package_opkg/gencode-remote      | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status
index f5a6f098..9d993055 100755
--- a/cdist/conf/type/__package_opkg/explorer/pkg_status
+++ b/cdist/conf/type/__package_opkg/explorer/pkg_status
@@ -23,9 +23,9 @@
 # Retrieve the status of a package - parses opkg output
 #
 
-__type_path=${__object%%${__object_id}*}
+readonly __type_path=${__object%%${__object_id}*}
+readonly LOCKFILE="${__type_path}/cdist_opkg.lock"
 
-LOCKFILE="${__type_path}/cdist_opkg.lock"
 _lock() (
 	set -o noclobber
 	until echo $$>"${LOCKFILE}"
diff --git a/cdist/conf/type/__package_opkg/gencode-remote b/cdist/conf/type/__package_opkg/gencode-remote
index ad90dc24..c7f21751 100755
--- a/cdist/conf/type/__package_opkg/gencode-remote
+++ b/cdist/conf/type/__package_opkg/gencode-remote
@@ -47,9 +47,9 @@ then
 fi
 
 cat <<'EOF'
-__type_path=${__object%%${__object_id}*}
+readonly __type_path=${__object%%${__object_id}*}
+readonly LOCKFILE="${__type_path}/cdist_opkg.lock"
 
-LOCKFILE="${__type_path}/cdist_opkg.lock"
 _lock() (
 	set -o noclobber
 	until echo $$>"${LOCKFILE}"

From 26dfdf37c21ce3c469ae0d6a717d390f422f9fbf Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 21 Jun 2020 01:13:30 +0300
Subject: [PATCH 126/527] [__download] support multiple checksum formats and
 download utilities, add --onchange and other minor changes

---
 cdist/conf/type/__download/explorer/state     | 78 +++++++++++++++----
 cdist/conf/type/__download/gencode-local      | 25 +++++-
 cdist/conf/type/__download/gencode-remote     |  7 ++
 cdist/conf/type/__download/man.rst            | 25 ++++--
 .../type/__download/parameter/default/cmd-get |  1 -
 .../type/__download/parameter/default/cmd-sum |  1 -
 cdist/conf/type/__download/parameter/optional |  1 +
 7 files changed, 116 insertions(+), 22 deletions(-)
 create mode 100755 cdist/conf/type/__download/gencode-remote
 delete mode 100644 cdist/conf/type/__download/parameter/default/cmd-get
 delete mode 100644 cdist/conf/type/__download/parameter/default/cmd-sum

diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 6a50f5a5..00362545 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -2,19 +2,71 @@
 
 dst="/$__object_id"
 
-# shellcheck disable=SC2059
-cmd="$( printf "$( cat "$__object/parameter/cmd-sum" )" "$dst" )"
-
-sum="$( cat "$__object/parameter/sum" )"
-
-if [ -f "$dst" ]
+if [ ! -f "$dst" ]
 then
-    if [ "$( eval "$cmd" )" = "$sum" ]
-    then
-        echo 'present'
-    else
-        echo 'mismatch'
-    fi
-else
     echo 'absent'
+    exit 0
+fi
+
+sum_should="$( cat "$__object/parameter/sum" )"
+
+if [ -f "$__object/parameter/cmd-sum" ]
+then
+    # shellcheck disable=SC2059
+    sum_is="$( eval "$( printf \
+        "$( cat "$__object/parameter/cmd-sum" )" \
+        "$dst" )" )"
+else
+    os="$( "$__explorer/os" )"
+
+    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+    then
+        sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
+
+    elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="md5:$( md5 -q "$dst" )"
+            ;;
+            *)
+                sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+
+    elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="sha1:$( sha1 -q "$dst" )"
+            ;;
+            *)
+                sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+
+    elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="sha256:$( sha256 -q "$dst" )"
+            ;;
+            *)
+                sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+    fi
+fi
+
+if [ -z "$sum_is" ]
+then
+    echo 'no checksum from target' >&2
+    exit 1
+fi
+
+if [ "$sum_is" = "$sum_should" ]
+then
+    echo 'present'
+else
+    echo 'mismatch'
 fi
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 49e9c699..85ef3a60 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -9,12 +9,31 @@ fi
 
 url="$( cat "$__object/parameter/url" )"
 
-cmd="$( cat "$__object/parameter/cmd-get" )"
-
 tmp="$( mktemp )"
 
 dst="/$__object_id"
 
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cmd="$( cat "$__object/parameter/cmd-get" )"
+
+elif command -v wget > /dev/null
+then
+    cmd="wget -O - '%s'"
+
+elif command -v curl > /dev/null
+then
+    cmd="curl -o - '%s'"
+
+elif command -v fetch > /dev/null
+then
+    cmd="fetch -o - '%s'"
+
+else
+    echo 'no usable locally installed utility for downloading' >&2
+    exit 1
+fi
+
 printf "$cmd > %s\n" \
     "$url" \
     "$tmp"
@@ -33,3 +52,5 @@ printf '%s %s %s:%s\n' \
     "$dst"
 
 echo "rm -f '$tmp'"
+
+echo 'downloaded' > "$__messages_out"
diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
new file mode 100755
index 00000000..b08d0050
--- /dev/null
+++ b/cdist/conf/type/__download/gencode-remote
@@ -0,0 +1,7 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/onchange" ] \
+    && grep -Fq "$__object_id:downloaded" "$__messages_in"
+then
+    cat "$__object/parameter/onchange"
+fi
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index c973448f..63a41bc4 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -10,7 +10,13 @@ DESCRIPTION
 -----------
 You must use persistent storage in target host for destination file
 (``$__object_id``) because it will be used for checksum calculation
-in order to decide if file must be downloaded.
+in order to decide if file must be (re-)downloaded.
+
+By default type will try to use following locally installed utilities
+for downloading (in order): ``wget``, ``curl`` or ``fetch``.
+
+Environment variables like ``{http,https,ftp}_proxy`` etc can be used on
+cdist execution (``http_proxy=foo cdist config ...``).
 
 
 REQUIRED PARAMETERS
@@ -19,20 +25,29 @@ url
    URL from which to download the file.
 
 sum
-   Checksum of downloaded file.
+   Checksum of file going to be downloaded.
+   By default output of ``cksum`` without filename is expected.
+   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
+
+onchange
+   Execute this command after download.
 
 
 OPTIONAL PARAMETERS
 -------------------
 cmd-get
    Command used for downloading.
-   Default is ``wget -O- '%s'``.
    Command must output to ``stdout``.
+   Parameter will be used for ``printf`` and must include only one
+   variable ``%s`` which will become URL.
+   For example: ``wget -O - '%s'``.
 
 cmd-sum
    Command used for checksum calculation.
-   Default is ``md5sum '%s' | awk '{print $1}'``.
    Command output and ``--sum`` parameter must match.
+   Parameter will be used for ``printf`` and must include only one
+   variable ``%s`` which will become destination.
+   For example: ``md5sum '%s' | awk '{print $1}'``.
 
 
 EXAMPLES
@@ -45,7 +60,7 @@ EXAMPLES
     require='__directory/opt/cpma' \
         __download /opt/cpma/cnq3.zip \
             --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
-            --sum 46da3021ca9eace277115ec9106c5b46
+            --sum md5:46da3021ca9eace277115ec9106c5b46
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
diff --git a/cdist/conf/type/__download/parameter/default/cmd-get b/cdist/conf/type/__download/parameter/default/cmd-get
deleted file mode 100644
index 2daa38a1..00000000
--- a/cdist/conf/type/__download/parameter/default/cmd-get
+++ /dev/null
@@ -1 +0,0 @@
-wget -O- '%s'
diff --git a/cdist/conf/type/__download/parameter/default/cmd-sum b/cdist/conf/type/__download/parameter/default/cmd-sum
deleted file mode 100644
index 3e8a9295..00000000
--- a/cdist/conf/type/__download/parameter/default/cmd-sum
+++ /dev/null
@@ -1 +0,0 @@
-md5sum '%s' | awk '{print $1}'
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index 22783e02..38c0ce4d 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,2 +1,3 @@
 cmd-get
 cmd-sum
+onchange

From 3649555f3522ea80f1999ca2cb7f8ddd112cbb33 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 22 Jun 2020 09:31:59 +0200
Subject: [PATCH 127/527] [type/__package_opkg] Do not lock execution of
 code-remote (revert)

Instead, rely on `nonparallel`.
In any case cdist should never run explorer and code concurrently even if the
dependency graph would allow to do so as it would result in many more
synchronization issues than this one.
---
 cdist/conf/type/__package_opkg/gencode-remote | 32 ++-----------------
 1 file changed, 2 insertions(+), 30 deletions(-)

diff --git a/cdist/conf/type/__package_opkg/gencode-remote b/cdist/conf/type/__package_opkg/gencode-remote
index c7f21751..28caff71 100755
--- a/cdist/conf/type/__package_opkg/gencode-remote
+++ b/cdist/conf/type/__package_opkg/gencode-remote
@@ -25,9 +25,9 @@
 
 if test -f "${__object}/parameter/name"
 then
-   name=$(cat "${__object}/parameter/name")
+	name=$(cat "${__object}/parameter/name")
 else
-   name=$__object_id
+	name=$__object_id
 fi
 
 state_should=$(cat "${__object}/parameter/state")
@@ -46,34 +46,6 @@ then
 	exit 0
 fi
 
-cat <<'EOF'
-readonly __type_path=${__object%%${__object_id}*}
-readonly LOCKFILE="${__type_path}/cdist_opkg.lock"
-
-_lock() (
-	set -o noclobber
-	until echo $$>"${LOCKFILE}"
-	do
-		while test -f "${LOCKFILE}"; do sleep 1; done
-	done
-
-) 2>/dev/null
-_unlock() {
-	if test -s "${LOCKFILE}" && test "$(cat "${LOCKFILE}")" = $$
-	then
-		rm "${LOCKFILE}"
-	fi
-}
-EOF
-
-# NOTE: We need to lock parallel execution of code-remote to ensure that it is
-# not executed concurrently with a type explorer.
-# opkg will try to acquire the OPKG lock (usually /var/lock/opkg.lock) using
-# lockf(2) for every operation.
-# It will not wait for the lock but terminate with an error leading to an
-# incorrect outcome.
-echo 'trap _unlock EXIT'
-echo '_lock'
 
 case $state_should
 in

From 49dde11def71cec121e5f698d0fb6ed3a539f97a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 24 Jun 2020 07:04:06 +0200
Subject: [PATCH 128/527] Remove deprecated __pf_apply

---
 cdist/conf/type/__pf_apply/deprecated     |  1 -
 cdist/conf/type/__pf_apply/explorer/rcvar | 36 ---------------
 cdist/conf/type/__pf_apply/gencode-remote | 51 ---------------------
 cdist/conf/type/__pf_apply/man.rst        | 55 -----------------------
 cdist/conf/type/__pf_apply/singleton      |  0
 5 files changed, 143 deletions(-)
 delete mode 100644 cdist/conf/type/__pf_apply/deprecated
 delete mode 100755 cdist/conf/type/__pf_apply/explorer/rcvar
 delete mode 100755 cdist/conf/type/__pf_apply/gencode-remote
 delete mode 100644 cdist/conf/type/__pf_apply/man.rst
 delete mode 100644 cdist/conf/type/__pf_apply/singleton

diff --git a/cdist/conf/type/__pf_apply/deprecated b/cdist/conf/type/__pf_apply/deprecated
deleted file mode 100644
index 36cfed90..00000000
--- a/cdist/conf/type/__pf_apply/deprecated
+++ /dev/null
@@ -1 +0,0 @@
-Consider moving to __pf_apply_anchor. Get in touch if you need __pf_apply.
diff --git a/cdist/conf/type/__pf_apply/explorer/rcvar b/cdist/conf/type/__pf_apply/explorer/rcvar
deleted file mode 100755
index 7c8d535f..00000000
--- a/cdist/conf/type/__pf_apply/explorer/rcvar
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/sh
-#
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Get the location of the pf ruleset on the target host.
-#
-
-# Debug
-#exec >&2
-#set -x
-
-# Check /etc/rc.conf for pf's configuration file name. Default to /etc/pf.conf
-
-RC="/etc/rc.conf"
-PFCONF="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')"
-echo "${PFCONF:-"/etc/pf.conf"}"
-
-# Debug
-#set +x
-
diff --git a/cdist/conf/type/__pf_apply/gencode-remote b/cdist/conf/type/__pf_apply/gencode-remote
deleted file mode 100755
index c8f7a25a..00000000
--- a/cdist/conf/type/__pf_apply/gencode-remote
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/sh -e
-#
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Apply pf(4) ruleset on *BSD
-#
-
-# Debug
-#exec >&2
-#set -x
-
-rcvar=$(cat "$__object/explorer/rcvar")
-
-cat <<EOF
-if [ -f "${rcvar}.old" ]; then # rcvar.old exists, we must need to disable pf
-   # Disable pf
-   # If it already is disabled, pfctl -d returns 1, go on with life
-   pfctl -d || true
-   # Cleanup
-   rm -f "${rcvar}.old"
-elif [ -f "${rcvar}.new" ]; then # rcvar.new exists, we must need to apply it
-   # Ensure that pf is enabled in the first place
-   # If it already is enabled, pfctl -e returns 1, go on with life
-   mv "${rcvar}.new" "${rcvar}"
-   pfctl -e || true
-   pfctl -f "${rcvar}"
-   if [ "\$?" -ne "0" ]; then # failed to configure new ruleset
-      echo "Failed to configure the new ruleset on ${__target_host}!" >&2
-   fi
-fi
-EOF
-
-# Debug
-#set +x
-
diff --git a/cdist/conf/type/__pf_apply/man.rst b/cdist/conf/type/__pf_apply/man.rst
deleted file mode 100644
index eee345e7..00000000
--- a/cdist/conf/type/__pf_apply/man.rst
+++ /dev/null
@@ -1,55 +0,0 @@
-cdist-type__pf_apply(7)
-=======================
-
-NAME
-----
-cdist-type__pf_apply - Apply pf(4) ruleset on \*BSD
-
-
-DESCRIPTION
------------
-This type is used on \*BSD systems to manage the pf firewall's active ruleset.
-
-
-REQUIRED PARAMETERS
--------------------
-NONE
-
-
-OPTIONAL PARAMETERS
--------------------
-NONE
-
-
-EXAMPLES
---------
-
-.. code-block:: sh
-
-    # Modify the ruleset on $__target_host:
-    __pf_ruleset --state present --source /my/pf/ruleset.conf
-    require="__pf_ruleset" \
-       __pf_apply
-
-    # Remove the ruleset on $__target_host (implies disabling pf(4):
-    __pf_ruleset --state absent
-    require="__pf_ruleset" \
-       __pf_apply
-
-
-SEE ALSO
---------
-:strong:`pf`\ (4), :strong:`cdist-type__pf_ruleset`\ (7)
-
-
-AUTHORS
--------
-Jake Guffey <jake.guffey--@--eprotex.com>
-
-
-COPYING
--------
-Copyright \(C) 2012 Jake Guffey. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
diff --git a/cdist/conf/type/__pf_apply/singleton b/cdist/conf/type/__pf_apply/singleton
deleted file mode 100644
index e69de29b..00000000

From a9778965be7154c0f42cf0170142ee2ec571d7ec Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 24 Jun 2020 08:47:22 +0200
Subject: [PATCH 129/527] [type/__package_opkg] Use mkdir(1) to lock instead of
 noclobber

noclobber is potentially unsafe, because it relies on the underlying shell to
implement noclobber in a safe way that avoids race conditions between multiple
processes.
mkdir is safer because it is mandated by POSIX to "fail" if the target already
exists.
---
 .../type/__package_opkg/explorer/pkg_status   | 25 ++++++++++++-------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status
index 9d993055..09c550e5 100755
--- a/cdist/conf/type/__package_opkg/explorer/pkg_status
+++ b/cdist/conf/type/__package_opkg/explorer/pkg_status
@@ -24,21 +24,28 @@
 #
 
 readonly __type_path=${__object%%${__object_id}*}
-readonly LOCKFILE="${__type_path}/cdist_opkg.lock"
+test -d "${__type_path}" || { echo 'Cannot determine __type_path' >&2; exit 1; }
+readonly LOCKDIR="${__type_path:?}/.cdist_opkg.lock.dir"
 
-_lock() (
-	set -o noclobber
-	until echo $$>"${LOCKFILE}"
+_lock() {
+	until mkdir "${LOCKDIR:?}" 2>/dev/null
 	do
-		while test -f "${LOCKFILE}"; do sleep 1; done
+		while test -d "${LOCKDIR}"
+		do
+			# DEBUG: printf 'Locked by PID: %u\n' "$(cat "${LOCKDIR}/pid")"
+			sleep 1
+		done
 	done
-
-) 2>/dev/null
+	echo $$ >"${LOCKDIR:?}/pid"
+}
 _unlock() {
-	if test -s "${LOCKFILE}" && test "$(cat "${LOCKFILE}")" = $$
+	test -d "${LOCKDIR}" || return 0
+	if test -s "${LOCKDIR}/pid"
 	then
-		rm "${LOCKFILE}"
+		test "$(cat "${LOCKDIR}/pid")" = $$ || return 1
+		rm "${LOCKDIR:?}/pid"
 	fi
+	rmdir "${LOCKDIR:?}"
 }
 
 

From 5364d3bc9037969c415140523c5cac2f8891b967 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 24 Jun 2020 20:49:48 +0200
Subject: [PATCH 130/527] [type/__package_opkg] Implement flock locking if
 available

---
 .../type/__package_opkg/explorer/pkg_status   | 55 ++++++++++++-------
 1 file changed, 36 insertions(+), 19 deletions(-)

diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status
index 09c550e5..de7b896b 100755
--- a/cdist/conf/type/__package_opkg/explorer/pkg_status
+++ b/cdist/conf/type/__package_opkg/explorer/pkg_status
@@ -25,28 +25,45 @@
 
 readonly __type_path=${__object%%${__object_id}*}
 test -d "${__type_path}" || { echo 'Cannot determine __type_path' >&2; exit 1; }
-readonly LOCKDIR="${__type_path:?}/.cdist_opkg.lock.dir"
+readonly LOCKFILE="${__type_path:?}/.cdist_opkg.lock"
 
-_lock() {
-	until mkdir "${LOCKDIR:?}" 2>/dev/null
-	do
-		while test -d "${LOCKDIR}"
+if command -v flock >/dev/null 2>&1
+then
+	# use flock (if available) on FD 9
+	_lock() {
+		exec 9<>"${LOCKFILE:?}"
+		flock -x 9
+		echo $$>&9
+	}
+	_unlock() {
+		:>"${LOCKFILE:?}"
+		flock -u 9
+		exec 9<&-
+	}
+else
+	# fallback to mkdir if flock is missing
+	_lock() {
+		until mkdir "${LOCKFILE:?}.dir" 2>/dev/null
 		do
-			# DEBUG: printf 'Locked by PID: %u\n' "$(cat "${LOCKDIR}/pid")"
-			sleep 1
+			while test -d "${LOCKFILE}.dir"
+			do
+				# DEBUG:
+				# printf 'Locked by PID: %u\n' "$(cat "${LOCKFILE}.dir/pid")"
+				sleep 1
+			done
 		done
-	done
-	echo $$ >"${LOCKDIR:?}/pid"
-}
-_unlock() {
-	test -d "${LOCKDIR}" || return 0
-	if test -s "${LOCKDIR}/pid"
-	then
-		test "$(cat "${LOCKDIR}/pid")" = $$ || return 1
-		rm "${LOCKDIR:?}/pid"
-	fi
-	rmdir "${LOCKDIR:?}"
-}
+		echo $$ >"${LOCKFILE:?}.dir/pid"
+	}
+	_unlock() {
+		test -d "${LOCKFILE}.dir" || return 0
+		if test -s "${LOCKFILE}.dir/pid"
+		then
+			test "$(cat "${LOCKFILE}.dir/pid")" = $$ || return 1
+			rm "${LOCKFILE:?}.dir/pid"
+		fi
+		rmdir "${LOCKFILE:?}.dir"
+	}
+fi
 
 
 if test -f "${__object}/parameter/name"

From 7074f9c395d3ab6f763682f860744ac8d12a8c26 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 25 Jun 2020 06:32:10 +0200
Subject: [PATCH 131/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index a0de09fa..f1713d3b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,8 @@ Changelog
 
 next:
 	* New type: __download (Ander Punnar)
+	* Type __locale_system: Add devuan support (Dennis Camera)
+	* Type __package_opkg: Add locking (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 077989e8fd749339da9a6ca8458b2c73ac71672c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 27 Jun 2020 15:55:04 +0200
Subject: [PATCH 132/527] Remove annoying warnings

Those warnings don't have any specail meaning and usage.
Resolve #825.
---
 cdist/util/ipaddr.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py
index 9b730225..95ca74ee 100644
--- a/cdist/util/ipaddr.py
+++ b/cdist/util/ipaddr.py
@@ -45,8 +45,6 @@ def resolve_target_host_name(host, family=0):
         log.debug("derived host_name for host \"{}\": {}".format(
             host, host_name))
     except (socket.gaierror, socket.herror) as e:
-        log.warning("Could not derive host_name for {}"
-                    ", $host_name will be empty. Error is: {}".format(host, e))
         # in case of error provide empty value
         host_name = ''
     return host_name
@@ -59,8 +57,6 @@ def resolve_target_fqdn(host):
         log.debug("derived host_fqdn for host \"{}\": {}".format(
             host, host_fqdn))
     except socket.herror as e:
-        log.warning("Could not derive host_fqdn for {}"
-                    ", $host_fqdn will be empty. Error is: {}".format(host, e))
         # in case of error provide empty value
         host_fqdn = ''
     return host_fqdn

From 85614aabd6f3abef5af6d362203db956c9d72aa9 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 28 Jun 2020 16:38:15 +0300
Subject: [PATCH 133/527] [__download] add --download (local|remote), update
 manual

---
 .../conf/type/__download/explorer/remote_cmd  | 19 +++++++++++++++
 cdist/conf/type/__download/gencode-local      |  4 +++-
 cdist/conf/type/__download/gencode-remote     | 22 +++++++++++++++--
 cdist/conf/type/__download/man.rst            | 24 ++++++++++++-------
 cdist/conf/type/__download/manifest           |  6 +++++
 .../__download/parameter/default/download     |  1 +
 cdist/conf/type/__download/parameter/optional |  1 +
 7 files changed, 65 insertions(+), 12 deletions(-)
 create mode 100755 cdist/conf/type/__download/explorer/remote_cmd
 create mode 100755 cdist/conf/type/__download/manifest
 create mode 100644 cdist/conf/type/__download/parameter/default/download

diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd
new file mode 100755
index 00000000..fbd4d84c
--- /dev/null
+++ b/cdist/conf/type/__download/explorer/remote_cmd
@@ -0,0 +1,19 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cmd="$( cat "$__object/parameter/cmd-get" )"
+
+elif command -v curl > /dev/null
+then
+    cmd="curl -o - '%s'"
+
+elif command -v fetch > /dev/null
+then
+    cmd="fetch -o - '%s'"
+
+else
+    cmd="wget -O - '%s'"
+fi
+
+echo "$cmd"
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 85ef3a60..339827c2 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -1,8 +1,10 @@
 #!/bin/sh -e
 
+download="$( cat "$__object/parameter/download" )"
+
 state_is="$( cat "$__object/explorer/state" )"
 
-if [ "$state_is" = 'present' ]
+if [ "$download" != 'local' ] || [ "$state_is" = 'present' ]
 then
     exit 0
 fi
diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
index b08d0050..89ba72af 100755
--- a/cdist/conf/type/__download/gencode-remote
+++ b/cdist/conf/type/__download/gencode-remote
@@ -1,7 +1,25 @@
 #!/bin/sh -e
 
-if [ -f "$__object/parameter/onchange" ] \
-    && grep -Fq "$__object_id:downloaded" "$__messages_in"
+download="$( cat "$__object/parameter/download" )"
+
+state_is="$( cat "$__object/explorer/state" )"
+
+if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
+then
+    cmd="$( cat "$__object/explorer/remote_cmd" )"
+
+    url="$( cat "$__object/parameter/url" )"
+
+    dst="/$__object_id"
+
+    printf "$cmd > %s\n" \
+        "$url" \
+        "$dst"
+
+    echo 'downloaded' > "$__messages_out"
+fi
+
+if [ -f "$__object/parameter/onchange" ] && [ "$state" != "present" ]
 then
     cat "$__object/parameter/onchange"
 fi
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index 63a41bc4..c161f4e4 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -3,26 +3,28 @@ cdist-type__download(7)
 
 NAME
 ----
-cdist-type__download - Download file to local storage and copy it to target host
+cdist-type__download - Download a file
 
 
 DESCRIPTION
 -----------
-You must use persistent storage in target host for destination file
-(``$__object_id``) because it will be used for checksum calculation
-in order to decide if file must be (re-)downloaded.
+Persistent storage for destination file in target host must be used
+(``$__object_id``) because it will be used for checksum calculation in
+order to decide if file must be (re-)downloaded.
 
-By default type will try to use following locally installed utilities
-for downloading (in order): ``wget``, ``curl`` or ``fetch``.
+By default type will try to use ``wget``, ``curl`` or ``fetch`` for
+downloading.  If ``--download remote`` type will fallback to (and
+install) ``wget``.
 
-Environment variables like ``{http,https,ftp}_proxy`` etc can be used on
-cdist execution (``http_proxy=foo cdist config ...``).
+If ``--download local`` (default), then environment variables like
+``{http,https,ftp}_proxy`` etc can be used on cdist execution
+(``http_proxy=foo cdist config ...``).
 
 
 REQUIRED PARAMETERS
 -------------------
 url
-   URL from which to download the file.
+   File's URL.
 
 sum
    Checksum of file going to be downloaded.
@@ -35,6 +37,10 @@ onchange
 
 OPTIONAL PARAMETERS
 -------------------
+download
+   If ``local`` (default), then download file to local storage and copy
+   it to target host. If ``remote``, then download happens in target.
+
 cmd-get
    Command used for downloading.
    Command must output to ``stdout``.
diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest
new file mode 100755
index 00000000..7ec8d86d
--- /dev/null
+++ b/cdist/conf/type/__download/manifest
@@ -0,0 +1,6 @@
+#!/bin/sh -e
+
+if grep -Eq '^wget' "$__object/explorer/remote_cmd"
+then
+    __package wget
+fi
diff --git a/cdist/conf/type/__download/parameter/default/download b/cdist/conf/type/__download/parameter/default/download
new file mode 100644
index 00000000..40830374
--- /dev/null
+++ b/cdist/conf/type/__download/parameter/default/download
@@ -0,0 +1 @@
+local
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index 38c0ce4d..838e2fbf 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,3 +1,4 @@
 cmd-get
 cmd-sum
+download
 onchange

From b6bf90e3f1a775ec1bd27a99015107d24a397787 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 28 Jun 2020 16:43:45 +0300
Subject: [PATCH 134/527] [__download] update manual

---
 cdist/conf/type/__download/man.rst | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index c161f4e4..eafa9dfc 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -8,15 +8,14 @@ cdist-type__download - Download a file
 
 DESCRIPTION
 -----------
-Persistent storage for destination file in target host must be used
-(``$__object_id``) because it will be used for checksum calculation in
-order to decide if file must be (re-)downloaded.
+Destination (``$__object_id``) in target host must be persistent storage
+in order to calculate checksum and decide if file must be (re-)downloaded.
 
-By default type will try to use ``wget``, ``curl`` or ``fetch`` for
-downloading.  If ``--download remote`` type will fallback to (and
-install) ``wget``.
+By default type will try to use ``wget``, ``curl`` or ``fetch``.
+If download happens in target (see ``--download``) then type will
+fallback to (and install) ``wget``.
 
-If ``--download local`` (default), then environment variables like
+If download happens in local machine, then environment variables like
 ``{http,https,ftp}_proxy`` etc can be used on cdist execution
 (``http_proxy=foo cdist config ...``).
 

From a263fdfe584bbba467086a3b2da3dfdd58ccc7fe Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 2 Oct 2019 17:52:20 +0200
Subject: [PATCH 135/527] [__hosts] Add --alias parameter

The --alias parameter allows to specify a hostname and multiple aliases on a
single /etc/hosts line.
---
 cdist/conf/type/__hosts/man.rst               | 17 +++++++---
 cdist/conf/type/__hosts/manifest              | 34 ++++++++++++-------
 .../type/__hosts/parameter/optional_multiple  |  1 +
 3 files changed, 35 insertions(+), 17 deletions(-)
 create mode 100644 cdist/conf/type/__hosts/parameter/optional_multiple

diff --git a/cdist/conf/type/__hosts/man.rst b/cdist/conf/type/__hosts/man.rst
index bece7967..1ac706cb 100644
--- a/cdist/conf/type/__hosts/man.rst
+++ b/cdist/conf/type/__hosts/man.rst
@@ -25,6 +25,10 @@ ip
     state is ``present``, this parameter is mandatory, if state is
     ``absent``, this parameter is silently ignored.
 
+alias
+    An alias for the hostname.
+    This parameter can be specified multiple times (once per alias).
+
 EXAMPLES
 --------
 
@@ -36,6 +40,8 @@ EXAMPLES
     # previously configured via __hosts.
     __hosts happy --state absent
 
+    __hosts srv1.example.com --ip 192.168.0.42 --alias srv1
+
 SEE ALSO
 --------
 
@@ -43,13 +49,14 @@ SEE ALSO
 
 AUTHORS
 -------
-
-Dmitry Bogatov <KAction@gnu.org>
+| Dmitry Bogatov <KAction@gnu.org>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
 
-Copyright (C) 2015,2016 Dmitry Bogatov. Free use of this software is granted
-under the terms of the GNU General Public License version 3 or later
-(GPLv3+).
+Copyright \(C) 2015-2016 Dmitry Bogatov, 2019 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__hosts/manifest b/cdist/conf/type/__hosts/manifest
index c536b83b..0d9e61f8 100755
--- a/cdist/conf/type/__hosts/manifest
+++ b/cdist/conf/type/__hosts/manifest
@@ -1,29 +1,39 @@
 #!/bin/sh -e
-# Copyright (C) 2015  Bogatov Dmitry <KAction@gnu.org>
 #
-# This program is free software: you can redistribute it and/or modify
+# Copyright (C) 2015  Bogatov Dmitry <KAction@gnu.org>
+# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-set -ue
+#
 
-hostname="$__object_id"
-state="$(cat "$__object/parameter/state")"
+set -e -u
+
+hostname=$__object_id
+state=$(cat "$__object/parameter/state")
 marker="# __hosts/$hostname"
 
-set -- "__hosts/$hostname" --file /etc/hosts --state "$state"
-
-if [ "$state" = absent ] ; then
-	__line "$@" --regex "$marker"
+if [ "$state" = 'absent' ]
+then
+	set -- --regex "$marker"
 else
-	ip="$(cat "$__object/parameter/ip")"
-	__line "$@" --line "$ip $hostname $marker"
+	ip=$(cat "$__object/parameter/ip")
+	aliases=$(while read -r a; do printf '\t%s' "$a"; done <"$__object/parameter/alias")
+
+	set -- --line "$(printf '%s\t%s%s  %s' \
+		"$ip" "$hostname" "$aliases" "$marker")"
 fi
+
+__line "__hosts/$hostname" --file /etc/hosts --state "$state" "$@"
diff --git a/cdist/conf/type/__hosts/parameter/optional_multiple b/cdist/conf/type/__hosts/parameter/optional_multiple
new file mode 100644
index 00000000..d077ed80
--- /dev/null
+++ b/cdist/conf/type/__hosts/parameter/optional_multiple
@@ -0,0 +1 @@
+alias

From 999e7b013449c51b848bb9365205810fc1a80fbd Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 28 Jan 2020 09:15:56 +0100
Subject: [PATCH 136/527] [type/__user] Fix shadow explorer for OpenBSD

---
 cdist/conf/type/__user/explorer/shadow | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__user/explorer/shadow b/cdist/conf/type/__user/explorer/shadow
index 73ce0e29..d516ed10 100755
--- a/cdist/conf/type/__user/explorer/shadow
+++ b/cdist/conf/type/__user/explorer/shadow
@@ -23,18 +23,24 @@
 
 name=$__object_id
 
-case $("$__explorer/os") in
-  'freebsd'|'netbsd'|'openbsd')
+case $("${__explorer}/os") in
+  freebsd|netbsd)
     database='passwd'
     ;;
-  # Default to using shadow passwords
+  openbsd)
+    database='master.passwd'
+    ;;
   *)
+    # Default to using shadow passwords
     database='shadow'
     ;;
 esac
 
-if command -v getent >/dev/null; then
-  getent "$database" "$name" || true
-elif [ -f /etc/shadow ]; then
-  grep "^${name}:" /etc/shadow || true
+if command -v getent >/dev/null 2>&1
+then
+  getent "${database}" "${name}" 2>/dev/null && exit || true  # fallback to file
+fi
+if test -n "${database}" -a -f "/etc/${database}"
+then
+  grep -e "^${name}:" "/etc/${database}" && exit || true  # ignore failure
 fi

From 6467ccbdccd380c11e37e747216bfc3aabc1b776 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 30 Jun 2020 14:31:11 +0200
Subject: [PATCH 137/527] [type/__user] Make shellcheck happy

---
 cdist/conf/type/__user/explorer/shadow | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__user/explorer/shadow b/cdist/conf/type/__user/explorer/shadow
index d516ed10..32496ee7 100755
--- a/cdist/conf/type/__user/explorer/shadow
+++ b/cdist/conf/type/__user/explorer/shadow
@@ -38,9 +38,10 @@ esac
 
 if command -v getent >/dev/null 2>&1
 then
+  # shellcheck disable=SC2015
   getent "${database}" "${name}" 2>/dev/null && exit || true  # fallback to file
 fi
 if test -n "${database}" -a -f "/etc/${database}"
 then
-  grep -e "^${name}:" "/etc/${database}" && exit || true  # ignore failure
+  grep -e "^${name}:" "/etc/${database}" || true  # ignore failure
 fi

From 3860f1feeaa2844191eeea84444963818bb42c4b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 26 Jun 2020 14:33:16 +0200
Subject: [PATCH 138/527] [type/{__file/__directory}] Support
 setuid,setguid,sticky bits

---
 cdist/conf/type/__directory/explorer/stat  | 17 +++++++++--------
 cdist/conf/type/__directory/gencode-remote |  4 ++--
 cdist/conf/type/__file/explorer/stat       | 14 +++++++-------
 cdist/conf/type/__file/gencode-remote      |  4 ++--
 4 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat
index 105d894f..a7dc8431 100755
--- a/cdist/conf/type/__directory/explorer/stat
+++ b/cdist/conf/type/__directory/explorer/stat
@@ -33,7 +33,7 @@ fallback() {
    group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
-   mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }')
+   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
 
    printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\n' \
       "$("$__type_explorer/type")" \
@@ -51,13 +51,14 @@ then
    exit
 fi
 
-case $("$__explorer/os") in
-   "freebsd"|"netbsd"|"openbsd"|"macosx")
-      stat -f "type: %HT
+case $("$__explorer/os")
+in
+   freebsd|netbsd|openbsd|macosx)
+      stat -f 'type: %HT
 owner: %Du %Su
 group: %Dg %Sg
-mode: %Lp %Sp
-" "$destination" | awk '/^type/ { print tolower($0); next } { print }'
+mode: %Mp%03Lp %Sp
+' "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
     solaris)
         ls1="$( ls -ld "$destination" )"
@@ -92,9 +93,9 @@ mode: %Lp %Sp
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
       #       we fall through to the shell fallback.
-       stat -c "type: %F
+       stat -c 'type: %F
 owner: %u %U
 group: %g %G
-mode: %a %A" "$destination" 2>/dev/null || fallback
+mode: %04a %A' "$destination" 2>/dev/null || fallback
    ;;
 esac
diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote
index a1a32ea2..2c2c56fd 100755
--- a/cdist/conf/type/__directory/gencode-remote
+++ b/cdist/conf/type/__directory/gencode-remote
@@ -97,9 +97,9 @@ case "$state_should" in
             value_should="$(cat "$__object/parameter/$attribute")"
             value_is="$(get_current_value "$attribute" "$value_should")"
 
-            # change 0xxx format to xxx format => same as stat returns
+            # format mode in four digits => same as stat returns
             if [ "$attribute" = mode ]; then
-                value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')"
+                value_should=$(printf '%04u' "${value_should}")
             fi
 
             if [ "$set_attributes" = 1 ] || [ "$value_should" != "$value_is" ]; then
diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat
index 91c8cc84..231768f6 100755
--- a/cdist/conf/type/__file/explorer/stat
+++ b/cdist/conf/type/__file/explorer/stat
@@ -34,7 +34,7 @@ fallback() {
    group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
-   mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }')
+   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
 
    size=$(echo "$ls_line" | awk '{ print $5 }')
    links=$(echo "$ls_line" | awk '{ print $2 }')
@@ -63,13 +63,13 @@ fi
 case $("$__explorer/os")
 in
    freebsd|netbsd|openbsd|macosx)
-      stat -f "type: %HT
+      stat -f 'type: %HT
 owner: %Du %Su
 group: %Dg %Sg
-mode: %Lp %Sp
+mode: %Mp%03Lp %Sp
 size: %Dz
 links: %Dl
-" "$destination" | awk '/^type/ { print tolower($0); next } { print }'
+' "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
     solaris)
         ls1="$( ls -ld "$destination" )"
@@ -106,11 +106,11 @@ links: %Dl
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
       #       we fall through to the shell fallback.
-      stat -c "type: %F
+      stat -c 'type: %F
 owner: %u %U
 group: %g %G
-mode: %a %A
+mode: %04a %A
 size: %s
-links: %h" "$destination" 2>/dev/null || fallback
+links: %h' "$destination" 2>/dev/null || fallback
          ;;
 esac
diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index 815593bd..a69154df 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -68,9 +68,9 @@ case "$state_should" in
             if [ -f "$__object/parameter/$attribute" ]; then
                 value_should="$(cat "$__object/parameter/$attribute")"
 
-                # change 0xxx format to xxx format => same as stat returns
+                # format mode in four digits => same as stat returns
                 if [ "$attribute" = mode ]; then
-                    value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')"
+                    value_should=$(printf '%04u' "${value_should}")
                 fi
 
                 value_is="$(get_current_value "$attribute" "$value_should")"

From 88400551f9490a4f8dc9b918e962e5ab3bba1286 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 30 Jun 2020 23:59:45 +0200
Subject: [PATCH 139/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index f1713d3b..b823421b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,8 @@ next:
 	* New type: __download (Ander Punnar)
 	* Type __locale_system: Add devuan support (Dennis Camera)
 	* Type __package_opkg: Add locking (Dennis Camera)
+	* Type __hosts: Add --alias parameter (Dennis Camera)
+	* Type __user: Fix shadow explorer for OpenBSD (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From fe193ecab8addf9b1c5b6d3c06c11ae9f9803378 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 1 Jul 2020 14:05:45 +0200
Subject: [PATCH 140/527] Make code consistent

* Remove supreflous checking and warning message.
* Fix cache recording.
---
 cdist/emulator.py | 65 +++++++++++++++--------------------------------
 1 file changed, 21 insertions(+), 44 deletions(-)

diff --git a/cdist/emulator.py b/cdist/emulator.py
index adbcbd9d..9fe84056 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -91,10 +91,6 @@ class Emulator:
         self.type_name = os.path.basename(argv[0])
         self.cdist_type = core.CdistType(self.type_base_path, self.type_name)
 
-        # If set then object alreay exists and this var holds existing
-        # requirements.
-        self._existing_reqs = None
-
         self.__init_log()
 
     def run(self):
@@ -230,9 +226,6 @@ class Emulator:
                 self.parameters[key] = value
 
         if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env:
-            # Make existing requirements a set so that we can compare it
-            # later with new requirements.
-            self._existing_reqs = set(self.cdist_object.requirements)
             obj_params = self._object_params_in_context()
             if obj_params != self.parameters:
                 errmsg = ("Object %s already exists with conflicting "
@@ -251,23 +244,26 @@ class Emulator:
             else:
                 self.cdist_object.create()
             self.cdist_object.parameters = self.parameters
-            # record the created object in typeorder file
-            with open(self.typeorder_path, 'a') as typeorderfile:
-                print(self.cdist_object.name, file=typeorderfile)
-            # record the created object in parent object typeorder file
-            __object_name = self.env.get('__object_name', None)
-            depname = self.cdist_object.name
-            if __object_name:
-                parent = self.cdist_object.object_from_name(__object_name)
-                parent.typeorder.append(self.cdist_object.name)
-                if self._order_dep_on():
-                    self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
-                                    ' for %s'), depname, parent.name)
-                    parent.typeorder_dep.append(depname)
-            elif self._order_dep_on():
-                self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
-                               depname)
-                self._add_typeorder_dep(depname)
+        # Do the following recording even if object exists, but with
+        # different requirements.
+
+        # record the created object in typeorder file
+        with open(self.typeorder_path, 'a') as typeorderfile:
+            print(self.cdist_object.name, file=typeorderfile)
+        # record the created object in parent object typeorder file
+        __object_name = self.env.get('__object_name', None)
+        depname = self.cdist_object.name
+        if __object_name:
+            parent = self.cdist_object.object_from_name(__object_name)
+            parent.typeorder.append(self.cdist_object.name)
+            if self._order_dep_on():
+                self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
+                                ' for %s'), depname, parent.name)
+                parent.typeorder_dep.append(depname)
+        elif self._order_dep_on():
+            self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
+                           depname)
+            self._add_typeorder_dep(depname)
 
         # Record / Append source
         self.cdist_object.source.append(self.object_source)
@@ -322,8 +318,6 @@ class Emulator:
         # This ensures pattern matching is done against sanitised list
         self.cdist_object.requirements.append(cdist_object.name)
 
-        return cdist_object.name
-
     def _order_dep_on(self):
         return os.path.exists(self.order_dep_state_path)
 
@@ -392,7 +386,6 @@ class Emulator:
                 # so do not set a requirement
                 pass
 
-        reqs = set()
         if "require" in self.env:
             requirements = self.env['require']
             self.log.debug("reqs = " + requirements)
@@ -400,23 +393,7 @@ class Emulator:
                 # Ignore empty fields - probably the only field anyway
                 if len(requirement) == 0:
                     continue
-                object_name = self.record_requirement(requirement)
-                reqs.add(object_name)
-        if self._existing_reqs is not None:
-            # If object exists then compare existing and new requirements.
-            if self._existing_reqs != reqs:
-                warnmsg = ("Object {} already exists with requirements:\n"
-                           "{}: {}\n"
-                           "{}: {}\n"
-                           "Dependency resolver could not handle dependencies "
-                           "as expected.".format(
-                               self.cdist_object.name,
-                               " ".join(self.cdist_object.source),
-                               self._existing_reqs,
-                               self.object_source,
-                               reqs
-                           ))
-                self.log.warning(warnmsg)
+                self.record_requirement(requirement)
 
     def record_auto_requirements(self):
         """An object shall automatically depend on all objects that it

From 93506d2113e1d05202527639efe883a5b44b220d Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 8 Jul 2020 00:17:12 +0300
Subject: [PATCH 141/527] [__download] curl follow redirects

---
 cdist/conf/type/__download/explorer/remote_cmd | 2 +-
 cdist/conf/type/__download/gencode-local       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd
index fbd4d84c..e3e35b45 100755
--- a/cdist/conf/type/__download/explorer/remote_cmd
+++ b/cdist/conf/type/__download/explorer/remote_cmd
@@ -6,7 +6,7 @@ then
 
 elif command -v curl > /dev/null
 then
-    cmd="curl -o - '%s'"
+    cmd="curl -L -o - '%s'"
 
 elif command -v fetch > /dev/null
 then
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 339827c2..571d2c3c 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -25,7 +25,7 @@ then
 
 elif command -v curl > /dev/null
 then
-    cmd="curl -o - '%s'"
+    cmd="curl -L -o - '%s'"
 
 elif command -v fetch > /dev/null
 then

From e9062662868d7677a235889ba39a6091085388f0 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 8 Jul 2020 00:20:55 +0300
Subject: [PATCH 142/527] [__download] s/variable/format specification/

---
 cdist/conf/type/__download/man.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index eafa9dfc..6ec0b19a 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -44,14 +44,14 @@ cmd-get
    Command used for downloading.
    Command must output to ``stdout``.
    Parameter will be used for ``printf`` and must include only one
-   variable ``%s`` which will become URL.
+   format specification ``%s`` which will become URL.
    For example: ``wget -O - '%s'``.
 
 cmd-sum
    Command used for checksum calculation.
    Command output and ``--sum`` parameter must match.
    Parameter will be used for ``printf`` and must include only one
-   variable ``%s`` which will become destination.
+   format specification ``%s`` which will become destination.
    For example: ``md5sum '%s' | awk '{print $1}'``.
 
 

From cb9933b4a0d343a059899101810b6ee86fc87dbc Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 8 Jul 2020 12:43:55 +0200
Subject: [PATCH 143/527] Fix state -> state_is

---
 cdist/conf/type/__download/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
index 89ba72af..029a0801 100755
--- a/cdist/conf/type/__download/gencode-remote
+++ b/cdist/conf/type/__download/gencode-remote
@@ -19,7 +19,7 @@ then
     echo 'downloaded' > "$__messages_out"
 fi
 
-if [ -f "$__object/parameter/onchange" ] && [ "$state" != "present" ]
+if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ]
 then
     cat "$__object/parameter/onchange"
 fi

From b8752e9ee390e983c1ce6467761aca8101b6548d Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 10 Jul 2020 21:03:35 +0200
Subject: [PATCH 144/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index b823421b..54c42bde 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Type __package_opkg: Add locking (Dennis Camera)
 	* Type __hosts: Add --alias parameter (Dennis Camera)
 	* Type __user: Fix shadow explorer for OpenBSD (Dennis Camera)
+	* Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From a5ae26116bd611fcab76c0383998fb4b05f1ae1d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 11 Jul 2020 18:57:47 +0200
Subject: [PATCH 145/527] [type/__hosts] Fix when used without --alias

---
 cdist/conf/type/__hosts/manifest | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/type/__hosts/manifest b/cdist/conf/type/__hosts/manifest
index 0d9e61f8..8103ebd5 100755
--- a/cdist/conf/type/__hosts/manifest
+++ b/cdist/conf/type/__hosts/manifest
@@ -19,21 +19,24 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-set -e -u
+set -e
 
 hostname=$__object_id
-state=$(cat "$__object/parameter/state")
-marker="# __hosts/$hostname"
+state=$(cat "${__object}/parameter/state")
+marker="# __hosts/${hostname}"
 
-if [ "$state" = 'absent' ]
+if test "${state}" != 'absent'
 then
-	set -- --regex "$marker"
-else
-	ip=$(cat "$__object/parameter/ip")
-	aliases=$(while read -r a; do printf '\t%s' "$a"; done <"$__object/parameter/alias")
+	ip=$(cat "${__object}/parameter/ip")
+	if test -s "${__object}/parameter/alias"
+	then
+		aliases=$(while read -r a; do printf '\t%s' "$a"; done <"$__object/parameter/alias")
+	fi
 
 	set -- --line "$(printf '%s\t%s%s  %s' \
-		"$ip" "$hostname" "$aliases" "$marker")"
+		"${ip}" "${hostname}" "${aliases}" "${marker}")"
+else
+	set -- --regex "$(echo "${marker}" | sed -e 's/\./\\./')$"
 fi
 
-__line "__hosts/$hostname" --file /etc/hosts --state "$state" "$@"
+__line "/etc/hosts:${hostname}" --file /etc/hosts --state "${state}" "$@"

From 19514662b022c79c4e6a56573c32dad45055310c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 12 Jul 2020 12:24:00 +0200
Subject: [PATCH 146/527] [type/{__file/__directory}] Fix typo

---
 cdist/conf/type/__directory/explorer/stat | 2 +-
 cdist/conf/type/__file/explorer/stat      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat
index a7dc8431..422c5819 100755
--- a/cdist/conf/type/__directory/explorer/stat
+++ b/cdist/conf/type/__directory/explorer/stat
@@ -30,7 +30,7 @@ fallback() {
    gid=$(echo "$ls_line" | awk '{ print $4 }')
 
    owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd)
-   group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
+   group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
    mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat
index 231768f6..3f971488 100755
--- a/cdist/conf/type/__file/explorer/stat
+++ b/cdist/conf/type/__file/explorer/stat
@@ -31,7 +31,7 @@ fallback() {
    gid=$(echo "$ls_line" | awk '{ print $4 }')
 
    owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd)
-   group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
+   group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
    mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')

From 9fb7e151b889214576cb891595a68ca8c6196d45 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 12 Jul 2020 12:31:55 +0200
Subject: [PATCH 147/527] [type/{__file/__directory}] Remove special Solaris
 blocks

Solaris 11 has GNU stat (handled by *)
Solaris 10 (and older?) does not have stat (handled by failing command -v stat)

On Solaris 10 (at least on UFS), setgid cannot be set on directories.
Unlike on other systems `chmod 2400` is not `-r----S---`, but `-r----l---`.
---
 cdist/conf/type/__directory/explorer/stat | 40 +++--------------------
 cdist/conf/type/__file/explorer/stat      | 40 +++--------------------
 2 files changed, 9 insertions(+), 71 deletions(-)

diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat
index 422c5819..f817cb02 100755
--- a/cdist/conf/type/__directory/explorer/stat
+++ b/cdist/conf/type/__directory/explorer/stat
@@ -33,7 +33,7 @@ fallback() {
    group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
-   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
+   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[lst]/)*2^(9+i/3)}printf("%04o",k)}')
 
    printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\n' \
       "$("$__type_explorer/type")" \
@@ -45,11 +45,10 @@ fallback() {
 # nothing to work with, nothing we could do
 [ -e "$destination" ] || exit 0
 
-if ! command -v stat >/dev/null
-then
+command -v stat >/dev/null 2>&1 || {
    fallback
    exit
-fi
+}
 
 case $("$__explorer/os")
 in
@@ -60,42 +59,13 @@ group: %Dg %Sg
 mode: %Mp%03Lp %Sp
 ' "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
-    solaris)
-        ls1="$( ls -ld "$destination" )"
-        ls2="$( ls -ldn "$destination" )"
-
-        if [ -f "$__object/parameter/mode" ]
-        then mode_should="$( cat "$__object/parameter/mode" )"
-        fi
-
-        # yes, it is ugly hack, but if you know better way...
-        if [ -z "$( find "$destination" -perm "$mode_should" )" ]
-        then octets=888
-        else octets="$( echo "$mode_should" | sed 's/^0//' )"
-        fi
-
-        case "$( echo "$ls1" | cut -c1-1 )" in
-            -) echo 'type: regular file' ;;
-            d) echo 'type: directory' ;;
-        esac
-
-        echo "owner: $( echo "$ls2" \
-            | awk '{print $3}' ) $( echo "$ls1" \
-                | awk '{print $3}' )"
-
-        echo "group: $( echo "$ls2" \
-            | awk '{print $4}' ) $( echo "$ls1" \
-                | awk '{print $4}' )"
-
-        echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )"
-    ;;
    *)
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
       #       we fall through to the shell fallback.
-       stat -c 'type: %F
+      stat -c 'type: %F
 owner: %u %U
 group: %g %G
 mode: %04a %A' "$destination" 2>/dev/null || fallback
-   ;;
+      ;;
 esac
diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat
index 3f971488..29b3c8a3 100755
--- a/cdist/conf/type/__file/explorer/stat
+++ b/cdist/conf/type/__file/explorer/stat
@@ -34,7 +34,7 @@ fallback() {
    group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
-   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
+   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[lst]/)*2^(9+i/3)}printf("%04o",k)}')
 
    size=$(echo "$ls_line" | awk '{ print $5 }')
    links=$(echo "$ls_line" | awk '{ print $2 }')
@@ -53,11 +53,10 @@ fallback() {
 [ -e "$destination" ] || exit 0
 
 
-if ! command -v stat >/dev/null
-then
+command -v stat >/dev/null 2>&1 || {
    fallback
    exit
-fi
+}
 
 
 case $("$__explorer/os")
@@ -71,37 +70,6 @@ size: %Dz
 links: %Dl
 ' "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
-    solaris)
-        ls1="$( ls -ld "$destination" )"
-        ls2="$( ls -ldn "$destination" )"
-
-        if [ -f "$__object/parameter/mode" ]
-        then mode_should="$( cat "$__object/parameter/mode" )"
-        fi
-
-        # yes, it is ugly hack, but if you know better way...
-        if [ -z "$( find "$destination" -perm "$mode_should" )" ]
-        then octets=888
-        else octets="$( echo "$mode_should" | sed 's/^0//' )"
-        fi
-
-        case "$( echo "$ls1" | cut -c1-1 )" in
-            -) echo 'type: regular file' ;;
-            d) echo 'type: directory' ;;
-        esac
-
-        echo "owner: $( echo "$ls2" \
-            | awk '{print $3}' ) $( echo "$ls1" \
-                | awk '{print $3}' )"
-
-        echo "group: $( echo "$ls2" \
-            | awk '{print $4}' ) $( echo "$ls1" \
-                | awk '{print $4}' )"
-
-        echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )"
-        echo "size: $( echo "$ls1" | awk '{print $5}' )"
-        echo "links: $( echo "$ls1" | awk '{print $2}' )"
-    ;;
    *)
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
@@ -112,5 +80,5 @@ group: %g %G
 mode: %04a %A
 size: %s
 links: %h' "$destination" 2>/dev/null || fallback
-         ;;
+      ;;
 esac

From bc970731316a59c21f8aa784fb27f1fa1b13ab89 Mon Sep 17 00:00:00 2001
From: Darko Poljak <darko.poljak@ungleich.ch>
Date: Mon, 13 Jul 2020 05:44:26 +0000
Subject: [PATCH 148/527] Merge branch 'bugfix/postfix-master-option' into
 '6.6'

Fix broken --option parameter in __postfix_master type

See merge request ungleich-public/cdist!905

(cherry picked from commit 2f433a1458f3a1f7f8859e9ae165178a0ec5b7a0)

9496b234 The option parameter is actually multi-valued
4009bbd7 Protect postfix variables in options
---
 cdist/conf/type/__postfix_master/gencode-remote              | 2 +-
 cdist/conf/type/__postfix_master/parameter/optional          | 1 -
 cdist/conf/type/__postfix_master/parameter/optional_multiple | 1 +
 3 files changed, 2 insertions(+), 2 deletions(-)
 create mode 100644 cdist/conf/type/__postfix_master/parameter/optional_multiple

diff --git a/cdist/conf/type/__postfix_master/gencode-remote b/cdist/conf/type/__postfix_master/gencode-remote
index 7c109a69..73de1088 100755
--- a/cdist/conf/type/__postfix_master/gencode-remote
+++ b/cdist/conf/type/__postfix_master/gencode-remote
@@ -67,7 +67,7 @@ case "$state_should" in
          remove_entry
       fi
       cat << DONE
-cat >> "$config" << ${__type##*/}_DONE
+cat >> "$config" << "${__type##*/}_DONE"
 $(cat "$entry")
 ${__type##*/}_DONE
 DONE
diff --git a/cdist/conf/type/__postfix_master/parameter/optional b/cdist/conf/type/__postfix_master/parameter/optional
index 792b42c5..410482b8 100644
--- a/cdist/conf/type/__postfix_master/parameter/optional
+++ b/cdist/conf/type/__postfix_master/parameter/optional
@@ -4,6 +4,5 @@ unpriv
 chroot
 wakeup
 maxproc
-option
 comment
 state
diff --git a/cdist/conf/type/__postfix_master/parameter/optional_multiple b/cdist/conf/type/__postfix_master/parameter/optional_multiple
new file mode 100644
index 00000000..01925a15
--- /dev/null
+++ b/cdist/conf/type/__postfix_master/parameter/optional_multiple
@@ -0,0 +1 @@
+option

From 8903540e9106db93e490515c07929a207326be77 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 13 Jul 2020 07:54:12 +0200
Subject: [PATCH 149/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 54c42bde..dac9cbce 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,8 @@ next:
 	* Type __hosts: Add --alias parameter (Dennis Camera)
 	* Type __user: Fix shadow explorer for OpenBSD (Dennis Camera)
 	* Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak)
+	* Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera)
+	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 3965c7f73844873aabb1d839b2298eed70a6b35c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 21 Jul 2020 19:42:40 +0200
Subject: [PATCH 150/527] [type/__user] Install user{add,mod,del} packages on
 OpenWrt

---
 cdist/conf/type/__user/manifest | 42 +++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/type/__user/manifest b/cdist/conf/type/__user/manifest
index 8f10b38c..b9fad65b 100644
--- a/cdist/conf/type/__user/manifest
+++ b/cdist/conf/type/__user/manifest
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,16 +18,37 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
 # Manage users.
+#
 
-os=$(cat "$__global/explorer/os")
-
-case "$os" in
-    alpine)
-        __package shadow
-        ;;
-    *)
-        :
-        ;;
+case $(cat "${__global}/explorer/os")
+in
+	(alpine)
+		__package shadow
+		;;
+	(openwrt)
+		case $(cat "${__object}/parameter/state")
+		in
+			(present)
+				if test -s "${__object}/explorer/passwd"
+				then
+					# NOTE: The package might not be required if no changes
+					# are required, but determining if changes are required is
+					# out of scope here, and 40k should be okay, I hope.
+					__package shadow-usermod
+				else
+					__package shadow-useradd
+				fi
+				;;
+			(absent)
+				if test -s "${__object}/explorer/passwd"
+				then
+					__package shadow-userdel
+				fi
+				;;
+		esac
+		;;
+	(*)
+		:
+		;;
 esac

From d8b5c733f6cec871ceb1933807f4877a4ee77ffa Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 22 Jul 2020 06:36:27 +0200
Subject: [PATCH 151/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index dac9cbce..040de9d8 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,6 +10,7 @@ next:
 	* Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak)
 	* Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera)
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
+	* Type __user: Install user packages on OpenWRT (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From fdef468f1ab6f2163da22a477df154a2082b57b4 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 22 Jul 2020 18:28:41 +0200
Subject: [PATCH 152/527] Fix OpenWrt spelling

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 040de9d8..4c17ee58 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,7 +10,7 @@ next:
 	* Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak)
 	* Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera)
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
-	* Type __user: Install user packages on OpenWRT (Dennis Camera)
+	* Type __user: Install user packages on OpenWrt (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 595e43b8d5f06641179c13de8ac04be46c1b2820 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 23 Jul 2020 09:41:53 +0200
Subject: [PATCH 153/527] [type/{__file,__directory}] Fix incorrect
 interpretation of strings with leading 0s as octal

---
 cdist/conf/type/__directory/gencode-remote | 4 +++-
 cdist/conf/type/__file/gencode-remote      | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote
index 2c2c56fd..d9c00b56 100755
--- a/cdist/conf/type/__directory/gencode-remote
+++ b/cdist/conf/type/__directory/gencode-remote
@@ -99,7 +99,9 @@ case "$state_should" in
 
             # format mode in four digits => same as stat returns
             if [ "$attribute" = mode ]; then
-                value_should=$(printf '%04u' "${value_should}")
+                # Convert to four-digit octal number (printf interprets
+                # strings with leading 0s as octal!)
+                value_should=$(printf '%04o' "0${value_should}")
             fi
 
             if [ "$set_attributes" = 1 ] || [ "$value_should" != "$value_is" ]; then
diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index a69154df..35356b13 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -70,7 +70,9 @@ case "$state_should" in
 
                 # format mode in four digits => same as stat returns
                 if [ "$attribute" = mode ]; then
-                    value_should=$(printf '%04u' "${value_should}")
+                    # Convert to four-digit octal number (printf interprets
+                    # strings with leading 0s as octal!)
+                    value_should=$(printf '%04o' "0${value_should}")
                 fi
 
                 value_is="$(get_current_value "$attribute" "$value_should")"

From ae5f0bba0b103aabcfbad0f105c69c0b403697ac Mon Sep 17 00:00:00 2001
From: fnux <timothee.floure@ungleich.ch>
Date: Fri, 24 Jul 2020 12:26:35 +0200
Subject: [PATCH 154/527] Add Alpine support to __openldap_server

---
 cdist/conf/type/__openldap_server/man.rst  |  4 +--
 cdist/conf/type/__openldap_server/manifest | 42 ++++++++++++++++++----
 2 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
index fbad21d8..a96c7dad 100644
--- a/cdist/conf/type/__openldap_server/man.rst
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -103,8 +103,8 @@ syncrepl-host
     Set once per host that will replicate the directory.
 
 module
-    LDAP module to load. See `slapd.conf(5)`.
-    Default value is OS-dependent, see manifest.
+    LDAP module to load. See `slapd.conf(5)`. Some dependencies might have to
+    be installed beforehand. Default value is OS-dependent, see manifest.
 
 schema
     Name of LDAP schema to load. Must be the name without extension of a
diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest
index 84ba176f..2aeece26 100644
--- a/cdist/conf/type/__openldap_server/manifest
+++ b/cdist/conf/type/__openldap_server/manifest
@@ -25,6 +25,7 @@ case "${os}" in
         SLAPD_DATA_DIR="/var/db/openldap-data"
         SLAPD_RUN_DIR="/var/run/openldap"
         SLAPD_MODULE_PATH="/usr/local/libexec/openldap"
+        SLAPD_MODULE_TYPE="la"
         if [ -z "${slapd_modules}" ]; then
             # It looks like ppolicy and syncprov must be compiled
             slapd_modules="back_mdb back_monitor"
@@ -43,13 +44,34 @@ case "${os}" in
         SLAPD_DATA_DIR="/var/lib/ldap"
         SLAPD_RUN_DIR="/var/run/slapd"
         SLAPD_MODULE_PATH="/usr/lib/ldap"
+        SLAPD_MODULE_TYPE="la"
         if [ -z "${slapd_modules}" ]; then
             slapd_modules="back_mdb ppolicy syncprov back_monitor"
         fi
+        CONF_OWNER="openldap"
+        CONF_GROUP="openldap"
         if [ -z "${tls_cipher_suite}" ]; then
             tls_cipher_suite="NORMAL"
         fi
         ;;
+    alpine)
+        PKGS="openldap openldap-clients"
+        ETC="/etc"
+        SLAPD_DIR="/etc/openldap"
+        SLAPD_DATA_DIR="/var/lib/openldap"
+        SLAPD_RUN_DIR="/var/run/openldap"
+        SLAPD_MODULE_PATH="/usr/lib/openldap"
+        SLAPD_MODULE_TYPE="so"
+        if [ -z "${slapd_modules}" ]; then
+            slapd_modules="back_mdb ppolicy syncprov back_monitor"
+            PKGS="$PKGS openldap-back-mdb openldap-back-monitor openldap-overlay-all"
+        fi
+        CONF_OWNER="ldap"
+        CONF_GROUP="$SLAPD_USER"
+        if [ -z "${tls_cipher_suite}" ]; then
+            tls_cipher_suite="DEFAULT"
+        fi
+        ;;
     *)
         echo "Don't know the openldap defaults for: $os" >&2
         exit 1
@@ -156,6 +178,12 @@ case "${os}" in
                --line "SLAPD_SERVICES=\"${slapd_urls}\"" \
                --state present
         ;;
+    alpine)
+        require="__package/${PKG_MAIN}" __line add_slapd_services \
+               --file ${ETC}/conf.d/slapd \
+               --line "command_args=\"-h '${slapd_urls}'\"" \
+               --state present
+        ;;
     *)
         # Nothing to do here, move on.
         ;;
@@ -170,20 +198,22 @@ if [ -z "${_skip_letsencrypt_cert}" ]; then
     fi
 
     # shellcheck disable=SC2086
-    __letsencrypt_cert "${name}" --admin-email "${admin_email}" \
-        --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R openldap:openldap ${SLAPD_DIR}/sasl2 && service slapd restart" \
-        --automatic-renewal ${staging}
+    __directory ${SLAPD_DIR}/sasl2
+    require="__directory/${SLAPD_DIR}/sasl2" __letsencrypt_cert "${name}" \
+        --admin-email "${admin_email}" \
+        --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R ${CONF_OWNER}:${CONF_GROUP} ${SLAPD_DIR}/sasl2 && service slapd restart" \
+        --automatic-renewal "${staging}"
 fi
 
 require="__package/${PKG_MAIN}" __directory ${SLAPD_DIR}/slapd.d --state absent
 
 if [ -z "${_skip_letsencrypt_cert}" ]; then
     require="__package/${PKG_MAIN} __letsencrypt_cert/${name}" \
-           __file ${SLAPD_DIR}/slapd.conf --owner ${CONF_OWNER} --group ${CONF_GROUP} --mode 644 \
+           __file "${SLAPD_DIR}/slapd.conf" --owner "${CONF_OWNER}" --group "${CONF_GROUP}" --mode 644 \
            --source "${ldapconf}"
 else
     require="__package/${PKG_MAIN}" \
-           __file ${SLAPD_DIR}/slapd.conf --owner ${CONF_OWNER} --group ${CONF_GROUP} --mode 644 \
+           __file "${SLAPD_DIR}/slapd.conf" --owner "${CONF_OWNER}" --group "${CONF_GROUP}" --mode 644 \
            --source "${ldapconf}"
 fi
 
@@ -210,7 +240,7 @@ done
 # Add specified modules
 echo "modulepath ${SLAPD_MODULE_PATH}" >> "${ldapconf}"
 for module in ${slapd_modules}; do
-    echo "moduleload ${module}.la" >> "${ldapconf}"
+    echo "moduleload ${module}.${SLAPD_MODULE_TYPE}" >> "${ldapconf}"
 done
 
 # Rest of the config

From 8654cbe4661762f25cd75c484f28344ce91ec55b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 24 Jul 2020 12:29:02 +0200
Subject: [PATCH 155/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 4c17ee58..0b9b8308 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,7 @@ next:
 	* Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera)
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
 	* Type __user: Install user packages on OpenWrt (Dennis Camera)
+	* Type __openldap_server: Add Alpine support (Timothée Floure)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 8b53f35ffab4e7b74d3fd019e7602c8a05338173 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 24 Jul 2020 12:33:16 +0200
Subject: [PATCH 156/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 0b9b8308..2457e90b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,7 @@ next:
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
 	* Type __user: Install user packages on OpenWrt (Dennis Camera)
 	* Type __openldap_server: Add Alpine support (Timothée Floure)
+	* Type __pf_apply: Remove deprecated type (Darko Poljak)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From ee71cad047ba0c86343fe8ee9ea2bc79bf950256 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 25 Jul 2020 19:19:38 +0200
Subject: [PATCH 157/527] [type/__package_apt] Fix type for legacy APT versions

--no-install-recommends was introduced with Debian 5.
The APT::Install-Recommends option gets ignored by old versions and
produces no error.
---
 cdist/conf/type/__package_apt/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote
index e02564a2..f3d91566 100755
--- a/cdist/conf/type/__package_apt/gencode-remote
+++ b/cdist/conf/type/__package_apt/gencode-remote
@@ -64,7 +64,7 @@ esac
 
 # Hint if we need to avoid questions at some point:
 # DEBIAN_PRIORITY=critical can reduce the number of questions
-aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes --no-install-recommends -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
+aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o APT::Install-Recommends=0 -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
 
 if [ "$state_is" = "$state_should" ]; then
     if [ -z "$version" ] || [ "$version" = "$version_is" ]; then

From 46d09392f08b0659557048c02e109f189ffd85f3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 19:36:34 +0200
Subject: [PATCH 158/527] [type/__key_value] Get AWK from POSIX PATH

This is required here, because Solaris /usr/bin/awk does not support the
sub() function.
So xpg4 AWK needs to be used.
---
 cdist/conf/type/__key_value/explorer/state         | 4 +++-
 cdist/conf/type/__key_value/files/remote_script.sh | 5 ++++-
 cdist/conf/type/__key_value/gencode-remote         | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__key_value/explorer/state b/cdist/conf/type/__key_value/explorer/state
index 7b2de1df..d24600af 100755
--- a/cdist/conf/type/__key_value/explorer/state
+++ b/cdist/conf/type/__key_value/explorer/state
@@ -40,7 +40,9 @@ else
 fi
 export key state delimiter value exact_delimiter
 
-awk -f - "$file" <<"AWK_EOF"
+awk_bin=$(PATH=$(getconf PATH 2>/dev/null) && command -v awk || echo awk)
+
+"${awk_bin}" -f - "$file" <<"AWK_EOF"
 BEGIN {
     state=ENVIRON["state"]
     key=ENVIRON["key"]
diff --git a/cdist/conf/type/__key_value/files/remote_script.sh b/cdist/conf/type/__key_value/files/remote_script.sh
index f7a1add5..faf080cb 100644
--- a/cdist/conf/type/__key_value/files/remote_script.sh
+++ b/cdist/conf/type/__key_value/files/remote_script.sh
@@ -24,7 +24,10 @@ if [ -f "$file" ]; then
 else
     touch "$file"
 fi
-awk -f - "$file" >"$tmpfile" <<"AWK_EOF"
+
+awk_bin=$(PATH=$(getconf PATH 2>/dev/null) && command -v awk || echo awk)
+
+"${awk_bin}" -f - "$file" >"$tmpfile" <<"AWK_EOF"
 BEGIN {
     # import variables in a secure way ..
     state=ENVIRON["state"]
diff --git a/cdist/conf/type/__key_value/gencode-remote b/cdist/conf/type/__key_value/gencode-remote
index 13cc27c7..1174400e 100755
--- a/cdist/conf/type/__key_value/gencode-remote
+++ b/cdist/conf/type/__key_value/gencode-remote
@@ -25,7 +25,7 @@ state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
 fire_onchange=''
 
-if [  "$state_is" = "$state_should" ]; then
+if [ "$state_is" = "$state_should" ]; then
     exit 0
 fi
 

From a5905044365e0f28e793f01e0be0cac8e6b2b379 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 24 Jul 2020 10:19:10 +0200
Subject: [PATCH 159/527] [type/__locale_system] RedHat systems on systemd use
 /etc/locale.conf

---
 cdist/conf/type/__locale_system/manifest | 26 ++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 4a1fdeed..22531a40 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -3,6 +3,7 @@
 # 2012-2016 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2016 Carlos Ortigoza (carlos.ortigoza at ungleich.ch)
 # 2016 Nico Schottelius (nico.schottelius at ungleich.ch)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -32,8 +33,25 @@ case "$os" in
     archlinux)
         locale_conf="/etc/locale.conf"
         ;;
-    redhat|centos)
-        locale_conf="/etc/sysconfig/i18n"
+    centos|redhat|scientific)
+        # shellcheck source=/dev/null
+        version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
+        if expr "${version_id}" '>=' 7 >/dev/null
+        then
+            locale_conf="/etc/locale.conf"
+        else
+            locale_conf="/etc/sysconfig/i18n"
+        fi
+        ;;
+    fedora)
+        # shellcheck source=/dev/null
+        version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
+        if expr "${version_id}" '>=' 18 >/dev/null
+        then
+            locale_conf="/etc/locale.conf"
+        else
+            locale_conf="/etc/sysconfig/i18n"
+        fi
         ;;
     *)
         echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
@@ -47,9 +65,9 @@ __file "$locale_conf" \
        --state exists
 
 require="__file/$locale_conf" \
-       __key_value "$locale_conf:$__object_id" \
+__key_value "$locale_conf:$__object_id" \
        --file "$locale_conf" \
        --key "$__object_id" \
-       --delimiter = \
+       --delimiter '=' --exact_delimiter \
        --state "$(cat "$__object/parameter/state")" \
        --value "$(cat "$__object/parameter/value")"

From 47e28fc441187b5deb869f53396eac1ee0f8cbe7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 12:07:38 +0200
Subject: [PATCH 160/527] [type/__locale_system] Support old Debian derivatives

---
 cdist/conf/type/__locale_system/manifest | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 22531a40..b9991fa3 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -27,9 +27,29 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-    debian|devuan|ubuntu)
+    debian)
+        os_version=$(cat "${__global}/explorer/os_version")
+        if expr "${os_version}" '>=' 4 >/dev/null
+        then
+            # Debian 4 (etch) and later
+            locale_conf="/etc/default/locale"
+        else
+            locale_conf="/etc/environment"
+        fi
+        ;;
+    devuan)
         locale_conf="/etc/default/locale"
         ;;
+    ubuntu)
+        os_version=$(cat "${__global}/explorer/os_version")
+        if expr "${os_version}" '>=' 6.10 >/dev/null
+        then
+            # Ubuntu 6.10 (edgy) and later
+            locale_conf="/etc/default/locale"
+        else
+            locale_conf="/etc/environment"
+        fi
+        ;;
     archlinux)
         locale_conf="/etc/locale.conf"
         ;;
@@ -61,7 +81,7 @@ case "$os" in
 esac
 
 __file "$locale_conf" \
-       --owner root --group root --mode 644 \
+       --owner root --group root --mode 0644 \
        --state exists
 
 require="__file/$locale_conf" \

From 0ef54a721d6f540f70959df2b7f1321a5688a594 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 12:17:30 +0200
Subject: [PATCH 161/527] [type/__locale_system] Add support for Gentoo Linux

---
 cdist/conf/type/__locale_system/manifest | 45 ++++++++++++++++++------
 1 file changed, 34 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index b9991fa3..0049ed5f 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -24,9 +24,19 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
+onchange_cmd=  # none, by default
+quote_value=false
+
+catval() {
+    # shellcheck disable=SC2059
+    printf "$($quote_value && echo '"%s"' || echo '%s')" "$(cat "$1")"
+}
+
+
 os=$(cat "$__global/explorer/os")
 
-case "$os" in
+case $os
+in
     debian)
         os_version=$(cat "${__global}/explorer/os_version")
         if expr "${os_version}" '>=' 4 >/dev/null
@@ -69,10 +79,24 @@ case "$os" in
         if expr "${version_id}" '>=' 18 >/dev/null
         then
             locale_conf="/etc/locale.conf"
+            quote_value=false
         else
             locale_conf="/etc/sysconfig/i18n"
         fi
         ;;
+    gentoo)
+        case $(cat "${__global}/explorer/init")
+        in
+            (*openrc*)
+                locale_conf="/etc/env.d/02locale"
+                onchange_cmd="env-update --no-ldconfig"
+                quote_value=true
+                ;;
+            (systemd)
+                locale_conf="/etc/locale.conf"
+                ;;
+        esac
+        ;;
     *)
         echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
         echo "Please contribute an implementation for it if you can." >&2
@@ -80,14 +104,13 @@ case "$os" in
         ;;
 esac
 
-__file "$locale_conf" \
-       --owner root --group root --mode 0644 \
-       --state exists
+__file "${locale_conf}" --state exists --owner root --group root --mode 0644
 
-require="__file/$locale_conf" \
-__key_value "$locale_conf:$__object_id" \
-       --file "$locale_conf" \
-       --key "$__object_id" \
-       --delimiter '=' --exact_delimiter \
-       --state "$(cat "$__object/parameter/state")" \
-       --value "$(cat "$__object/parameter/value")"
+require="__file/${locale_conf}" \
+__key_value "${locale_conf}:${__object_id}" \
+    --file "${locale_conf}" \
+    --key "${__object_id}" \
+    --delimiter '=' --exact_delimiter \
+    --state "$(cat "${__object}/parameter/state")" \
+    --value "$(catval "${__object}/parameter/value")" \
+    --onchange "${onchange_cmd}"

From 630d987d5f5a40ae783fd9d4ff0a10653fcab04c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 12:28:20 +0200
Subject: [PATCH 162/527] [type/__locale_system] Add support for Void Linux

---
 cdist/conf/type/__locale_system/manifest | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 0049ed5f..ff0a8c23 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -97,6 +97,9 @@ in
                 ;;
         esac
         ;;
+    voidlinux)
+        locale_conf="/etc/locale.conf"
+        ;;
     *)
         echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
         echo "Please contribute an implementation for it if you can." >&2

From 0ae0935afafd6b10516e9689c19000ef729d1b9b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 14:05:36 +0200
Subject: [PATCH 163/527] [type/__locale_system] Add support for SuSE

---
 cdist/conf/type/__locale_system/manifest | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index ff0a8c23..26d4f7b5 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -24,6 +24,7 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
+key=$__object_id
 onchange_cmd=  # none, by default
 quote_value=false
 
@@ -97,6 +98,25 @@ in
                 ;;
         esac
         ;;
+    suse)
+        os_version=$(cat "${__global}/explorer/os_version")
+        os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
+
+        # https://documentation.suse.com/sles/15-SP2/html/SLES-all/cha-suse.html#sec-suse-l10n
+        if expr "${os_major}" '>=' 15 \& "${os_major}" != 42
+        then
+            # It seems that starting with SuSE 15 the systemd /etc/locale.conf
+            # is the preferred way to set locales, although
+            # /etc/sysconfig/language is still available.
+            # Older documentation doesn't mention /etc/locale.conf, even though
+            # is it created when localectl is used.
+            locale_conf="/etc/locale.conf"
+        else
+            locale_conf="/etc/sysconfig/language"
+            quote_value=true
+            key="RC_${__object_id}"
+        fi
+        ;;
     voidlinux)
         locale_conf="/etc/locale.conf"
         ;;
@@ -110,9 +130,9 @@ esac
 __file "${locale_conf}" --state exists --owner root --group root --mode 0644
 
 require="__file/${locale_conf}" \
-__key_value "${locale_conf}:${__object_id}" \
+__key_value "${locale_conf}:${key}" \
     --file "${locale_conf}" \
-    --key "${__object_id}" \
+    --key "${key}" \
     --delimiter '=' --exact_delimiter \
     --state "$(cat "${__object}/parameter/state")" \
     --value "$(catval "${__object}/parameter/value")" \

From cbf22f3b2c8b9d0fef8f6b7b0bf16508f96675b3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 15:23:55 +0200
Subject: [PATCH 164/527] [type/__locale_system] Add support for Solaris

---
 cdist/conf/type/__locale_system/manifest | 41 ++++++++++++++++++++++--
 1 file changed, 39 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 26d4f7b5..71491fe5 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -33,6 +33,7 @@ catval() {
     printf "$($quote_value && echo '"%s"' || echo '%s')" "$(cat "$1")"
 }
 
+state_should=$(cat "${__object}/parameter/state")
 
 os=$(cat "$__global/explorer/os")
 
@@ -98,6 +99,39 @@ in
                 ;;
         esac
         ;;
+    solaris)
+        locale_conf="/etc/default/init"
+        locale_conf_group="sys"
+
+        if expr "$(cat "${__global}/explorer/os_version")" '>=' 5.11 >/dev/null
+        then
+            # mode on Oracle Solaris 11 is actually 0444,
+            # but the write bit makes sense, IMO
+            locale_conf_mode=0644
+
+            # Oracle Solaris 11.2 and later uses SMF to store environment info.
+            # This is a hack, but I didn't feel like modifying the whole type
+            # just for some Oracle nonsense.
+            # 11.3 apparently added nlsadm(1m), but it is missing from 11.2.
+            # Illumos continues to use /etc/default/init
+            # NOTE: Remember not to use "cool" POSIX features like -q or -e with
+            # Solaris grep.
+            release_regex='Oracle Solaris 11.[2-9][0-9]*'
+            case $state_should
+            in
+                (present)
+                    svccfg_cmd="svccfg -s svc:/system/environment:init setprop environment/${key} = astring: '$(cat "${__object}/parameter/value")'"
+                    ;;
+                (absent)
+                    svccfg_cmd="svccfg -s svc:/system/environment:init delprop environment/${key}"
+                    ;;
+            esac
+            refresh_cmd='svcadm refresh svc:/system/environment'
+            onchange_cmd="grep '${release_regex}' /etc/release >&- || exit 0; ${svccfg_cmd:-:} && ${refresh_cmd}"
+        else
+            locale_conf_mode=0555
+        fi
+        ;;
     suse)
         os_version=$(cat "${__global}/explorer/os_version")
         os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
@@ -127,13 +161,16 @@ in
         ;;
 esac
 
-__file "${locale_conf}" --state exists --owner root --group root --mode 0644
+__file "${locale_conf}" --state exists \
+    --owner "${locale_conf_owner:-0}" \
+    --group "${locale_conf_group:-0}" \
+    --mode "${locale_conf_mode:-0644}"
 
 require="__file/${locale_conf}" \
 __key_value "${locale_conf}:${key}" \
     --file "${locale_conf}" \
     --key "${key}" \
     --delimiter '=' --exact_delimiter \
-    --state "$(cat "${__object}/parameter/state")" \
+    --state "${state_should}" \
     --value "$(catval "${__object}/parameter/value")" \
     --onchange "${onchange_cmd}"

From a923e75d9b0054212031507986d3b935d74d52bf Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 18:29:14 +0200
Subject: [PATCH 165/527] [type/__locale_system] Add support for NetBSD

---
 cdist/conf/type/__locale_system/manifest | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 71491fe5..92af852f 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -99,6 +99,15 @@ in
                 ;;
         esac
         ;;
+    netbsd)
+        # NetBSD doesn't  have a separate configuration file to set locales.
+        # So the shell login file will have to do.
+        # "Non-POSIX" shells like csh will not be updated here.
+
+        locale_conf="/etc/profile"
+        quote_value=true
+        value="$(catval "${__object}/parameter/value"); export ${key}"
+        ;;
     solaris)
         locale_conf="/etc/default/init"
         locale_conf_group="sys"
@@ -172,5 +181,5 @@ __key_value "${locale_conf}:${key}" \
     --key "${key}" \
     --delimiter '=' --exact_delimiter \
     --state "${state_should}" \
-    --value "$(catval "${__object}/parameter/value")" \
+    --value "${value:-$(catval "${__object}/parameter/value")}" \
     --onchange "${onchange_cmd}"

From 511d8c96aa9201049eaafc459a2de9694fe2a4e4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 19:23:17 +0200
Subject: [PATCH 166/527] [type/__locale_system] Add support for Slackware

---
 cdist/conf/type/__locale_system/manifest | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 92af852f..180788e6 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -141,6 +141,12 @@ in
             locale_conf_mode=0555
         fi
         ;;
+    slackware)
+        # NOTE: lang.csh (csh config) is ignored here.
+        locale_conf="/etc/profile.d/lang.sh"
+        locale_conf_mode=0755
+        key="export ${__object_id}"
+        ;;
     suse)
         os_version=$(cat "${__global}/explorer/os_version")
         os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
@@ -176,7 +182,7 @@ __file "${locale_conf}" --state exists \
     --mode "${locale_conf_mode:-0644}"
 
 require="__file/${locale_conf}" \
-__key_value "${locale_conf}:${key}" \
+__key_value "${locale_conf}:${key#export }" \
     --file "${locale_conf}" \
     --key "${key}" \
     --delimiter '=' --exact_delimiter \

From 70d1228dc0c6a3af105f0864b46d7dea806aedc2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 19:26:45 +0200
Subject: [PATCH 167/527] [type/__locale_system] Add support for FreeBSD

---
 cdist/conf/type/__locale_system/manifest | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 180788e6..e4286ef6 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -99,9 +99,10 @@ in
                 ;;
         esac
         ;;
-    netbsd)
-        # NetBSD doesn't  have a separate configuration file to set locales.
-        # So the shell login file will have to do.
+    freebsd|netbsd)
+        # NetBSD doesn't have a separate configuration file to set locales.
+        # In FreeBSD locales could be configured via /etc/login.conf but parsing
+        # that would be annoying, so the shell login file will have to do.
         # "Non-POSIX" shells like csh will not be updated here.
 
         locale_conf="/etc/profile"

From 73f1937636bfa59eea0432e3625c0a27cc8171b5 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 27 Jul 2020 06:20:21 +0200
Subject: [PATCH 168/527] [__unpack] no mkdir by default, because destination
 can be file, but tar needs mkdir andrar needs slash at the end

---
 cdist/conf/type/__unpack/explorer/state       | 37 +++++++++
 cdist/conf/type/__unpack/gencode-remote       | 75 ++++++++++++++++++
 cdist/conf/type/__unpack/man.rst              | 79 +++++++++++++++++++
 cdist/conf/type/__unpack/manifest             | 41 ++++++++++
 cdist/conf/type/__unpack/parameter/boolean    |  2 +
 cdist/conf/type/__unpack/parameter/optional   |  2 +
 cdist/conf/type/__unpack/parameter/required   |  1 +
 cdist/conf/type/__unpack/test/README          |  3 +
 .../type/__unpack/test/make-init-manifest.sh  | 22 ++++++
 .../type/__unpack/test/make-test-files.sh     | 44 +++++++++++
 10 files changed, 306 insertions(+)
 create mode 100755 cdist/conf/type/__unpack/explorer/state
 create mode 100755 cdist/conf/type/__unpack/gencode-remote
 create mode 100644 cdist/conf/type/__unpack/man.rst
 create mode 100755 cdist/conf/type/__unpack/manifest
 create mode 100644 cdist/conf/type/__unpack/parameter/boolean
 create mode 100644 cdist/conf/type/__unpack/parameter/optional
 create mode 100644 cdist/conf/type/__unpack/parameter/required
 create mode 100644 cdist/conf/type/__unpack/test/README
 create mode 100755 cdist/conf/type/__unpack/test/make-init-manifest.sh
 create mode 100755 cdist/conf/type/__unpack/test/make-test-files.sh

diff --git a/cdist/conf/type/__unpack/explorer/state b/cdist/conf/type/__unpack/explorer/state
new file mode 100755
index 00000000..38bc0978
--- /dev/null
+++ b/cdist/conf/type/__unpack/explorer/state
@@ -0,0 +1,37 @@
+#!/bin/sh -e
+
+src="/$__object_id"
+
+if [ -f "$__object/parameter/sum-file" ]
+then
+    src_sum_was_file="$( cat "$__object/parameter/sum-file" )"
+else
+    src_sum_was_file="$src.cdist__unpack_sum"
+fi
+
+if [ ! -f "$src" ]
+then
+    if [ -n "$__cdist_dry_run" ]
+    then
+        echo 'mismatch'
+    else
+        echo 'missing'
+    fi
+else
+    if [ ! -f "$src_sum_was_file" ]
+    then
+        echo 'mismatch'
+        exit 0
+    fi
+
+    src_sum_was="$( cat "$src_sum_was_file" )"
+
+    src_sum_is="$( cksum "$src" | awk '{ print $1$2 }' )"
+
+    if [ "$src_sum_was" = "$src_sum_is" ]
+    then
+        echo 'match'
+    else
+        echo 'mismatch'
+    fi
+fi
diff --git a/cdist/conf/type/__unpack/gencode-remote b/cdist/conf/type/__unpack/gencode-remote
new file mode 100755
index 00000000..45c7173a
--- /dev/null
+++ b/cdist/conf/type/__unpack/gencode-remote
@@ -0,0 +1,75 @@
+#!/bin/sh -e
+
+if grep -Eq '^(missing|match)$' "$__object/explorer/state"
+then
+    exit 0
+fi
+
+os="$( cat "$__global/explorer/os" )"
+
+src="/$__object_id"
+
+dst="$( sed 's/\/$//' "$__object/parameter/destination" )"
+
+cmd=''
+
+case "$src" in
+    *.tar|*.tgz|*.tar.*)
+        cmd="mkdir -p '$dst' && tar --directory='$dst' --extract --file='$src'"
+
+        if [ -f "$__object/parameter/tar-strip" ]
+        then
+            tar_strip="$( cat "$__object/parameter/tar-strip" )"
+
+            cmd="$cmd --strip-components=$tar_strip"
+        fi
+    ;;
+    *.7z)
+        case "$os" in
+            centos|fedora|redhat)
+                cmd='7za'
+            ;;
+            *)
+                cmd='7zr'
+            ;;
+        esac
+
+        cmd="$cmd e -aoa -o'$dst' '$src'"
+    ;;
+    *.bz2)
+        cmd="bunzip2 --stdout '$src' > '$dst'"
+    ;;
+    *.gz)
+        cmd="gunzip --stdout '$src' > '$dst'"
+    ;;
+    *.lzma|*.xz)
+        cmd="xz --uncompress --stdout '$src' > '$dst'"
+    ;;
+    *.rar)
+        cmd="unrar x -o+ '$src' '$dst/'"
+    ;;
+    *.zip)
+        cmd="unzip -o '$src' -d '$dst'"
+    ;;
+esac
+
+if [ -f "$__object/parameter/backup-destination" ]
+then
+    echo "if [ -e '$dst' ]; then mv '$dst' '$dst.cdist__unpack_backup_$( date +%s )'; fi"
+fi
+
+echo "$cmd"
+
+if [ -f "$__object/parameter/sum-file" ]
+then
+    sum_file="$( cat "$__object/parameter/sum-file" )"
+else
+    sum_file="$src.cdist__unpack_sum"
+fi
+
+echo "cksum '$src' | awk '{ print \$1\$2 }' > '$sum_file'"
+
+if [ ! -f "$__object/parameter/preserve-archive" ]
+then
+    echo "rm -f '$src'"
+fi
diff --git a/cdist/conf/type/__unpack/man.rst b/cdist/conf/type/__unpack/man.rst
new file mode 100644
index 00000000..8fe96e43
--- /dev/null
+++ b/cdist/conf/type/__unpack/man.rst
@@ -0,0 +1,79 @@
+cdist-type__unpack(7)
+=====================
+
+NAME
+----
+cdist-type__unpack - Unpack archives
+
+
+DESCRIPTION
+-----------
+Unpack ``.tar``, ``.tgz``, ``.tar.*``, ``.7z``, ``.bz2``, ``.gz``,
+``.lzma``, ``.xz``, ``.rar`` and ``.zip`` archives. Archive type is
+detected by extension.
+
+To achieve idempotency, checksum file will be created in target. See
+``--sum-file`` parameter for details.
+
+
+REQUIRED PARAMETERS
+-------------------
+destination
+   Depending on archive format file or directory to where archive
+   contents will be written.
+
+
+OPTIONAL PARAMETERS
+-------------------
+sum-file
+    Override archive's checksum file in target. By default
+    ``XXX.cdist__unpack_sum`` will be used, where ``XXX`` is source
+    archive path. This file must be kept in target's persistent storage.
+
+tar-strip
+    Tarball specific. See ``man tar`` for ``--strip-components``.
+
+
+OPTIONAL BOOLEAN PARAMETERS
+---------------------------
+backup-destination
+    By default destination file will be overwritten. In case destination
+    is directory, files from archive will be added to or overwritten in
+    directory. This parameter moves existing destination to
+    ``XXX.cdist__unpack_backup_YYY``, where ``XXX`` is destination and
+    ``YYY`` current UNIX timestamp.
+
+preserve-archive
+    Don't delete archive after unpacking.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __directory /opt/cpma
+
+    require='__directory/opt/cpma' \
+        __download /opt/cpma/cnq3.zip \
+            --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
+            --sum md5:46da3021ca9eace277115ec9106c5b46
+
+    require='__download/opt/cpma/cnq3.zip' \
+        __unpack /opt/cpma/cnq3.zip \
+            --backup-destination \
+            --preserve-archive \
+            --destination /opt/cpma/server
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__unpack/manifest b/cdist/conf/type/__unpack/manifest
new file mode 100755
index 00000000..6bdf5a10
--- /dev/null
+++ b/cdist/conf/type/__unpack/manifest
@@ -0,0 +1,41 @@
+#!/bin/sh -e
+
+os="$( cat "$__global/explorer/os" )"
+
+src="/$__object_id"
+
+case "$src" in
+    *.7z)
+        __package p7zip
+    ;;
+    *.bz2)
+        case "$os" in
+            freebsd)
+                # bzip2 is part of freebsd base system
+            ;;
+            *)
+                __package bzip2
+            ;;
+        esac
+    ;;
+    *.lzma|*.xz|*.txz)
+        case "$os" in
+            debian|ubuntu|devuan)
+                __package xz-utils
+            ;;
+            alpine|centos)
+                __package xz
+            ;;
+        esac
+    ;;
+    *.rar)
+        case "$os" in
+            debian|ubuntu|devuan|alpine|freebsd)
+                __package unrar
+            ;;
+        esac
+    ;;
+    *.zip)
+        __package unzip
+    ;;
+esac
diff --git a/cdist/conf/type/__unpack/parameter/boolean b/cdist/conf/type/__unpack/parameter/boolean
new file mode 100644
index 00000000..99dca934
--- /dev/null
+++ b/cdist/conf/type/__unpack/parameter/boolean
@@ -0,0 +1,2 @@
+backup-destination
+preserve-archive
diff --git a/cdist/conf/type/__unpack/parameter/optional b/cdist/conf/type/__unpack/parameter/optional
new file mode 100644
index 00000000..d136dd0c
--- /dev/null
+++ b/cdist/conf/type/__unpack/parameter/optional
@@ -0,0 +1,2 @@
+sum-file
+tar-strip
diff --git a/cdist/conf/type/__unpack/parameter/required b/cdist/conf/type/__unpack/parameter/required
new file mode 100644
index 00000000..ac459b09
--- /dev/null
+++ b/cdist/conf/type/__unpack/parameter/required
@@ -0,0 +1 @@
+destination
diff --git a/cdist/conf/type/__unpack/test/README b/cdist/conf/type/__unpack/test/README
new file mode 100644
index 00000000..54f3972a
--- /dev/null
+++ b/cdist/conf/type/__unpack/test/README
@@ -0,0 +1,3 @@
+./make-test-files.sh
+./make-init-manifest.sh | cdist config -i - localhost
+sudo find /tmp/cdist__unpack_test/ -type f -exec cat {} \; | sort
diff --git a/cdist/conf/type/__unpack/test/make-init-manifest.sh b/cdist/conf/type/__unpack/test/make-init-manifest.sh
new file mode 100755
index 00000000..404bc106
--- /dev/null
+++ b/cdist/conf/type/__unpack/test/make-init-manifest.sh
@@ -0,0 +1,22 @@
+#!/bin/sh -e
+
+p="$( pwd )"
+d=/tmp/cdist__unpack_test
+
+echo 'export CDIST_ORDER_DEPENDENCY=1'
+
+echo "__directory $d"
+
+find "$p" -name 'test.*' -and -not -name '*.cdist__unpack_sum' \
+    | sort \
+    | while read -r l
+do
+    n="$( basename "$l" )"
+
+    printf '__unpack %s --destination %s/%s\n' \
+        "$l" \
+        "$d" \
+        "$n"
+done
+
+echo "__clean_path $p --pattern '.+/test\..+'"
diff --git a/cdist/conf/type/__unpack/test/make-test-files.sh b/cdist/conf/type/__unpack/test/make-test-files.sh
new file mode 100755
index 00000000..d18e9e9f
--- /dev/null
+++ b/cdist/conf/type/__unpack/test/make-test-files.sh
@@ -0,0 +1,44 @@
+#!/bin/sh -ex
+
+echo test.7z > test
+7z a test.7z test > /dev/null
+
+echo test.bz2 > test
+bzip2 test
+
+echo test.gz > test
+gzip test
+
+echo test.lzma > test
+lzma test
+
+echo test.rar > test
+rar a test.rar test > /dev/null
+
+echo test.tar.bz2 > test
+tar cf test.tar test
+bzip2 test.tar
+
+echo test.tar.xz > test
+tar cf test.tar test
+xz test.tar
+
+echo test.tgz > test
+tar cf test.tar test
+gzip test.tar
+mv test.tar.gz test.tgz
+
+echo test.tar.gz > test
+tar cf test.tar test
+gzip test.tar
+
+echo test.tar > test
+tar cf test.tar test
+
+echo test.xz > test
+xz test
+
+echo test.zip > test
+zip test.zip test > /dev/null
+
+rm test

From 463b6cd6b52da8918d2803721164748c2a632696 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 27 Jul 2020 06:22:25 +0200
Subject: [PATCH 169/527] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 2457e90b..8d26e767 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -13,6 +13,9 @@ next:
 	* Type __user: Install user packages on OpenWrt (Dennis Camera)
 	* Type __openldap_server: Add Alpine support (Timothée Floure)
 	* Type __pf_apply: Remove deprecated type (Darko Poljak)
+	* Type __package_apt: Fix for legacy APT versions that do not support --no-install-recommends (Dennis Camera)
+	* Type __key_value: Get awk from POSIX PATH (Dennis Camera)
+	* New type: __unpack (Ander Punnar)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From d26c36914a4fa9889af57f549cb2854968663a0b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 27 Jul 2020 10:55:28 +0200
Subject: [PATCH 170/527] [__timezone] Make type singleton

---
 cdist/conf/type/__timezone/gencode-remote     |  2 +-
 cdist/conf/type/__timezone/man.rst            | 22 ++++++++++++-------
 cdist/conf/type/__timezone/manifest           |  2 +-
 cdist/conf/type/__timezone/parameter/required |  1 +
 cdist/conf/type/__timezone/singleton          |  0
 5 files changed, 17 insertions(+), 10 deletions(-)
 create mode 100644 cdist/conf/type/__timezone/parameter/required
 create mode 100644 cdist/conf/type/__timezone/singleton

diff --git a/cdist/conf/type/__timezone/gencode-remote b/cdist/conf/type/__timezone/gencode-remote
index 5299f548..b685c990 100755
--- a/cdist/conf/type/__timezone/gencode-remote
+++ b/cdist/conf/type/__timezone/gencode-remote
@@ -22,7 +22,7 @@
 # This type allows to configure the desired localtime timezone.
 
 timezone_is=$(cat "$__object/explorer/timezone_is")
-timezone_should="$__object_id"
+timezone_should=$(cat "$__object/parameter/tz")
 os=$(cat "$__global/explorer/os")
 
 if [ "$timezone_is" = "$timezone_should" ]; then
diff --git a/cdist/conf/type/__timezone/man.rst b/cdist/conf/type/__timezone/man.rst
index 8a945c16..6012c552 100644
--- a/cdist/conf/type/__timezone/man.rst
+++ b/cdist/conf/type/__timezone/man.rst
@@ -14,7 +14,8 @@ This type creates a symlink (/etc/localtime) to the selected timezone
 
 REQUIRED PARAMETERS
 -------------------
-None.
+tz
+    The name of timezone to set.
 
 
 OPTIONAL PARAMETERS
@@ -27,19 +28,24 @@ EXAMPLES
 
 .. code-block:: sh
 
-    #Set up Europe/Andorra as our timezone.
-    __timezone Europe/Andorra
+    # Set up Europe/Andorra as our timezone.
+    __timezone --tz Europe/Andorra
 
-    #Set up US/Central as our timezone.
-    __timezone US/Central
+    # Set up US/Central as our timezone.
+    __timezone --tz US/Central
 
 
 AUTHORS
 -------
-Ramon Salvadó <rsalvado--@--gnuine--dot--com>
+| Steven Armstrong <steven-cdist--@--armstrong.cc>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Ramon Salvadó <rsalvado--@--gnuine--dot--com>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Free use of this software is
-granted under the terms of the GNU General Public License version 3 (GPLv3).
+Copyright \(C) 2012-2020 the `AUTHORS`_. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__timezone/manifest b/cdist/conf/type/__timezone/manifest
index 3d28ccba..0eb7fb9c 100755
--- a/cdist/conf/type/__timezone/manifest
+++ b/cdist/conf/type/__timezone/manifest
@@ -22,7 +22,7 @@
 #
 # This type allows to configure the desired localtime timezone.
 
-timezone="$__object_id"
+timezone=$(cat "$__object/parameter/tz")
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
diff --git a/cdist/conf/type/__timezone/parameter/required b/cdist/conf/type/__timezone/parameter/required
new file mode 100644
index 00000000..975445e4
--- /dev/null
+++ b/cdist/conf/type/__timezone/parameter/required
@@ -0,0 +1 @@
+tz
diff --git a/cdist/conf/type/__timezone/singleton b/cdist/conf/type/__timezone/singleton
new file mode 100644
index 00000000..e69de29b

From 627d215b637f1893e48a4a04c7e60686b960a698 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 27 Jul 2020 13:09:53 +0200
Subject: [PATCH 171/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8d26e767..e3711be2 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -16,6 +16,7 @@ next:
 	* Type __package_apt: Fix for legacy APT versions that do not support --no-install-recommends (Dennis Camera)
 	* Type __key_value: Get awk from POSIX PATH (Dennis Camera)
 	* New type: __unpack (Ander Punnar)
+	* Type __locale_system: Support more OSes (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 5dfc996febd085e4592fe590c4837bd4802ef4c3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 22:53:34 +0200
Subject: [PATCH 172/527] Fix global explorers for NetBSD

On NetBSD sysctl is at /sbin/sysctl, but the default PATH does not
contain /sbin.
---
 cdist/conf/explorer/cpu_cores | 1 +
 cdist/conf/explorer/disks     | 5 ++---
 cdist/conf/explorer/memory    | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/explorer/cpu_cores b/cdist/conf/explorer/cpu_cores
index c6744142..81e5294e 100755
--- a/cdist/conf/explorer/cpu_cores
+++ b/cdist/conf/explorer/cpu_cores
@@ -33,6 +33,7 @@ case "$os" in
     ;;
 
     "freebsd"|"netbsd")
+        PATH=$(getconf PATH)
         sysctl -n hw.ncpu
     ;;
 
diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks
index 24540601..56d62d10 100755
--- a/cdist/conf/explorer/disks
+++ b/cdist/conf/explorer/disks
@@ -30,9 +30,8 @@ case $uname_s in
         sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+'
     ;;
     NetBSD)
-        PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin"
-        sysctl -n hw.disknames \
-        | awk 'BEGIN { RS = " " } /^[lsw]d[0-9]+/'
+        PATH=$(getconf PATH)
+        sysctl -n hw.disknames | awk -v RS=' ' '/^[lsw]d[0-9]+/'
     ;;
     Linux)
         # list of major device numbers toexclude:
diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory
index 302b4cda..5ea15ada 100755
--- a/cdist/conf/explorer/memory
+++ b/cdist/conf/explorer/memory
@@ -30,6 +30,7 @@ case "$os" in
     ;;
 
     *"bsd")
+        PATH=$(getconf PATH)
         echo "$(sysctl -n hw.physmem) / 1048576" | bc
     ;;
 

From 3a87a447d00ef12eb6ace7c240926568aec2a042 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 23:37:48 +0200
Subject: [PATCH 173/527] [type/__sysctl] Fix on NetBSD

---
 cdist/conf/type/__sysctl/explorer/value | 9 +++++++--
 cdist/conf/type/__sysctl/gencode-remote | 2 ++
 cdist/conf/type/__sysctl/man.rst        | 7 +++++++
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__sysctl/explorer/value b/cdist/conf/type/__sysctl/explorer/value
index fc85b3d8..3e93c151 100755
--- a/cdist/conf/type/__sysctl/explorer/value
+++ b/cdist/conf/type/__sysctl/explorer/value
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -18,5 +18,10 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+if test "$(uname -s)" = NetBSD
+then
+	PATH=$(getconf PATH)
+fi
+
 # get the current runtime value
-sysctl -n "$__object_id" || true
+sysctl -n "${__object_id}" || true
diff --git a/cdist/conf/type/__sysctl/gencode-remote b/cdist/conf/type/__sysctl/gencode-remote
index 711d54e5..f0f6deef 100755
--- a/cdist/conf/type/__sysctl/gencode-remote
+++ b/cdist/conf/type/__sysctl/gencode-remote
@@ -44,6 +44,8 @@ case "$os" in
       flag='-w'
       ;;
    netbsd)
+      # shellcheck disable=SC2016
+      echo 'PATH=$(getconf PATH)'
       flag='-w'
       ;;
    freebsd|openbsd)
diff --git a/cdist/conf/type/__sysctl/man.rst b/cdist/conf/type/__sysctl/man.rst
index 6873003e..dbb9a1ac 100644
--- a/cdist/conf/type/__sysctl/man.rst
+++ b/cdist/conf/type/__sysctl/man.rst
@@ -26,6 +26,13 @@ EXAMPLES
 
     __sysctl net.ipv4.ip_forward --value 1
 
+    # On some operating systems, e.g. NetBSD, to prevent an error if the
+    # MIB style name does not exist (e.g. optional kernel components),
+    # name and value can be separated by `?=`. The same effect can be achieved
+    # in cdist by appending a `?` to the key:
+
+    __sysctl ddb.onpanic? --value -1
+
 
 AUTHORS
 -------

From 76bb214b5300707426030649cf03f35808e43256 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 27 Jul 2020 15:31:38 +0200
Subject: [PATCH 174/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index e3711be2..8fef4653 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -17,6 +17,8 @@ next:
 	* Type __key_value: Get awk from POSIX PATH (Dennis Camera)
 	* New type: __unpack (Ander Punnar)
 	* Type __locale_system: Support more OSes (Dennis Camera)
+	* Explorers cpu_cores, disks, memory: Fix for NetBSD (Dennis Camera)
+	* Type __sysctl: Fix for NetBSD (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From f5b367dfdbf60e75cfb2be038a05b6f226a1c010 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 28 Jul 2020 07:14:26 +0200
Subject: [PATCH 175/527] Release 6.7.0

---
 docs/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/changelog b/docs/changelog
index 8fef4653..269a2049 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,8 @@
 Changelog
 ---------
 
-next:
+6.7.0: 2020-07-28
+	* Delete deprecated type: __pf_apply (Darko Poljak)
 	* New type: __download (Ander Punnar)
 	* Type __locale_system: Add devuan support (Dennis Camera)
 	* Type __package_opkg: Add locking (Dennis Camera)
@@ -12,7 +13,6 @@ next:
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
 	* Type __user: Install user packages on OpenWrt (Dennis Camera)
 	* Type __openldap_server: Add Alpine support (Timothée Floure)
-	* Type __pf_apply: Remove deprecated type (Darko Poljak)
 	* Type __package_apt: Fix for legacy APT versions that do not support --no-install-recommends (Dennis Camera)
 	* Type __key_value: Get awk from POSIX PATH (Dennis Camera)
 	* New type: __unpack (Ander Punnar)

From c053a2c4a000b0c15df186ef2e067ea68ec9499c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 29 Jul 2020 11:31:12 +0200
Subject: [PATCH 176/527] Fix building man pages

Resolves #830.
---
 Makefile                                  | 4 ++--
 cdist/conf/type/__openldap_server/man.rst | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index f89ac1e7..3712511c 100644
--- a/Makefile
+++ b/Makefile
@@ -81,7 +81,7 @@ version:
 	}
 
 # Manpages #3: generic part
-man: version $(MANTYPES) $(DOCSREF)
+man: version configskel $(MANTYPES) $(DOCSREF) $(DOCSTYPESREF)
 	$(SPHINXM)
 
 html: version configskel $(MANTYPES) $(DOCSREF) $(DOCSTYPESREF)
@@ -104,7 +104,7 @@ DOTMANTYPES=$(subst /man.rst,.rst,$(DOTMANTYPEPREFIX))
 $(DOTMAN7DSTDIR)/cdist-type%.rst: $(DOTTYPEDIR)/%/man.rst
 	ln -sf "$^" $@
 
-dotman: version $(DOTMANTYPES)
+dotman: version configskel $(DOTMANTYPES) $(DOCSREF) $(DOCSTYPESREF)
 	$(SPHINXM)
 
 ################################################################################
diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
index a96c7dad..fa714ec0 100644
--- a/cdist/conf/type/__openldap_server/man.rst
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -31,8 +31,8 @@ manager-password-hash
     Generate e.g. with: `slappasswd -s weneedgoodsecurity`.
     See `slappasswd(8C)`, `slapd.conf(5)`.
     TODO: implement this: http://blog.adamsbros.org/2015/06/09/openldap-ssha-salted-hashes-by-hand/
-      to derive from the manager-password parameter and ensure idempotency (care with salts).
-      At that point, manager-password-hash should be deprecated and ignored.
+    to derive from the manager-password parameter and ensure idempotency (care with salts).
+    At that point, manager-password-hash should be deprecated and ignored.
 
 serverid
     The server for the directory.

From d37d2dc307d066f4d5f1bbbb93e0480965dfe3b8 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 2 Aug 2020 13:53:38 +0300
Subject: [PATCH 177/527] [__unpack] add parameter --tar-extra-args

---
 cdist/conf/type/__unpack/gencode-remote     |  7 +++++++
 cdist/conf/type/__unpack/man.rst            | 11 +++++++++++
 cdist/conf/type/__unpack/parameter/optional |  1 +
 3 files changed, 19 insertions(+)

diff --git a/cdist/conf/type/__unpack/gencode-remote b/cdist/conf/type/__unpack/gencode-remote
index 45c7173a..3b7a19a7 100755
--- a/cdist/conf/type/__unpack/gencode-remote
+++ b/cdist/conf/type/__unpack/gencode-remote
@@ -23,6 +23,13 @@ case "$src" in
 
             cmd="$cmd --strip-components=$tar_strip"
         fi
+
+        if [ -f "$__object/parameter/tar-extra-args" ]
+        then
+            tar_extra_args="$( cat "$__object/parameter/tar-extra-args" )"
+
+            cmd="$cmd $tar_extra_args"
+        fi
     ;;
     *.7z)
         case "$os" in
diff --git a/cdist/conf/type/__unpack/man.rst b/cdist/conf/type/__unpack/man.rst
index 8fe96e43..bd0603cf 100644
--- a/cdist/conf/type/__unpack/man.rst
+++ b/cdist/conf/type/__unpack/man.rst
@@ -33,6 +33,10 @@ sum-file
 tar-strip
     Tarball specific. See ``man tar`` for ``--strip-components``.
 
+tar-extra-args
+    Tarball sepcific. Append additional arguments to ``tar`` command.
+    See ``man tar`` for possible arguments.
+
 
 OPTIONAL BOOLEAN PARAMETERS
 ---------------------------
@@ -65,6 +69,13 @@ EXAMPLES
             --preserve-archive \
             --destination /opt/cpma/server
 
+    # example usecase for --tar-* args
+    __unpack /root/strelaysrv.tar.gz \
+        --preserve-archive \
+        --destination /usr/local/bin \
+        --tar-strip 1 \
+        --tar-extra-args '--wildcards "*/strelaysrv"'
+
 
 AUTHORS
 -------
diff --git a/cdist/conf/type/__unpack/parameter/optional b/cdist/conf/type/__unpack/parameter/optional
index d136dd0c..da26ca63 100644
--- a/cdist/conf/type/__unpack/parameter/optional
+++ b/cdist/conf/type/__unpack/parameter/optional
@@ -1,2 +1,3 @@
 sum-file
 tar-strip
+tar-extra-args

From 935f2395bc8f02d373f06d1e22acad813403ead2 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 2 Aug 2020 13:54:30 +0300
Subject: [PATCH 178/527] [__locale_system] fix for debian and ubuntu

ubuntu 6.10 and debian etch are 10+ years old and EOL. rather than
preserving compatibility I'll just remove it. while /etc/environment
works too, correct place is /etc/default/locale (as it was before
breaking change). also /etc/debian_version (os_version explorer) may
contain minor version with dot (10.5) or string (bullseye/sid).
---
 cdist/conf/type/__locale_system/manifest | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index e4286ef6..f7d75387 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -40,27 +40,13 @@ os=$(cat "$__global/explorer/os")
 case $os
 in
     debian)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 4 >/dev/null
-        then
-            # Debian 4 (etch) and later
-            locale_conf="/etc/default/locale"
-        else
-            locale_conf="/etc/environment"
-        fi
+        locale_conf="/etc/default/locale"
         ;;
     devuan)
         locale_conf="/etc/default/locale"
         ;;
     ubuntu)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 6.10 >/dev/null
-        then
-            # Ubuntu 6.10 (edgy) and later
-            locale_conf="/etc/default/locale"
-        else
-            locale_conf="/etc/environment"
-        fi
+        locale_conf="/etc/default/locale"
         ;;
     archlinux)
         locale_conf="/etc/locale.conf"

From 885d5a58f40e5adfd6aa616712e4c7246eca7880 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 16:49:01 +0200
Subject: [PATCH 179/527] [type/__locale_system] Fix floating point version
 comparison

---
 cdist/conf/type/__locale_system/manifest | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index e4286ef6..521f1007 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -24,6 +24,9 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
+bccmp() { test "$(bc)" -gt 0; }
+
+
 key=$__object_id
 onchange_cmd=  # none, by default
 quote_value=false
@@ -41,7 +44,7 @@ case $os
 in
     debian)
         os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 4 >/dev/null
+        if printf '%f >= 4\n' "${os_version}" | bccmp
         then
             # Debian 4 (etch) and later
             locale_conf="/etc/default/locale"
@@ -54,7 +57,7 @@ in
         ;;
     ubuntu)
         os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 6.10 >/dev/null
+        if printf '%f >= 6.10\n' "${os_version}" | bccmp
         then
             # Ubuntu 6.10 (edgy) and later
             locale_conf="/etc/default/locale"
@@ -68,7 +71,7 @@ in
     centos|redhat|scientific)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if expr "${version_id}" '>=' 7 >/dev/null
+        if printf '%f >= 7\n' "${version_id}" | bccmp
         then
             locale_conf="/etc/locale.conf"
         else
@@ -78,7 +81,7 @@ in
     fedora)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if expr "${version_id}" '>=' 18 >/dev/null
+        if printf '%f >= 18\n' "${version_id}" | bccmp
         then
             locale_conf="/etc/locale.conf"
             quote_value=false
@@ -113,7 +116,8 @@ in
         locale_conf="/etc/default/init"
         locale_conf_group="sys"
 
-        if expr "$(cat "${__global}/explorer/os_version")" '>=' 5.11 >/dev/null
+        os_version=$(cat "${__global}/explorer/os_version")
+        if printf '%f >= 5.11\n' "${os_version}" | bccmp
         then
             # mode on Oracle Solaris 11 is actually 0444,
             # but the write bit makes sense, IMO

From 71710fa00a86e3dcda6be82b959956cb88ba9902 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 20:17:00 +0200
Subject: [PATCH 180/527] [type/__locale_system] Implement "proper" version
 comparison

Proper in the sense that it can handle all numeric version numbers even if they
are not floating point (e.g. 16.04.6).
---
 cdist/conf/type/__locale_system/manifest | 28 ++++++++++++++++--------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 521f1007..88eadb31 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -24,7 +24,20 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
-bccmp() { test "$(bc)" -gt 0; }
+version_ge() {
+    awk -F '[^0-9.]' -v target="${1:?}" '
+    function max(x, y) { return x > y ? x : y }
+    BEGIN {
+        getline
+        nx = split($1, x, ".")
+        ny = split(target, y, ".")
+        for (i = 1; i <= max(nx, ny); ++i) {
+            diff = int(x[i]) - int(y[i])
+            if (diff == 0) continue
+            exit (diff < 0)
+        }
+    }'
+}
 
 
 key=$__object_id
@@ -43,8 +56,7 @@ os=$(cat "$__global/explorer/os")
 case $os
 in
     debian)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if printf '%f >= 4\n' "${os_version}" | bccmp
+        if version_ge 4 <"${__global}/explorer/os_version"
         then
             # Debian 4 (etch) and later
             locale_conf="/etc/default/locale"
@@ -56,8 +68,7 @@ in
         locale_conf="/etc/default/locale"
         ;;
     ubuntu)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if printf '%f >= 6.10\n' "${os_version}" | bccmp
+        if version_ge 6.10 <"${__global}/explorer/os_version"
         then
             # Ubuntu 6.10 (edgy) and later
             locale_conf="/etc/default/locale"
@@ -71,7 +82,7 @@ in
     centos|redhat|scientific)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if printf '%f >= 7\n' "${version_id}" | bccmp
+        if echo "${version_id}" | version_ge 7
         then
             locale_conf="/etc/locale.conf"
         else
@@ -81,7 +92,7 @@ in
     fedora)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if printf '%f >= 18\n' "${version_id}" | bccmp
+        if echo "${version_id}" | version_ge 18
         then
             locale_conf="/etc/locale.conf"
             quote_value=false
@@ -116,8 +127,7 @@ in
         locale_conf="/etc/default/init"
         locale_conf_group="sys"
 
-        os_version=$(cat "${__global}/explorer/os_version")
-        if printf '%f >= 5.11\n' "${os_version}" | bccmp
+        if version_ge 5.11 <"${__global}/explorer/os_version"
         then
             # mode on Oracle Solaris 11 is actually 0444,
             # but the write bit makes sense, IMO

From 7b480f4293b84a97850430a9436e28c66ccf5736 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 22:08:14 +0200
Subject: [PATCH 181/527] [type/__locale_system] Fix version extraction for
 SuSE

---
 cdist/conf/type/__locale_system/manifest | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 88eadb31..4b996ebc 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -163,7 +163,13 @@ in
         key="export ${__object_id}"
         ;;
     suse)
-        os_version=$(cat "${__global}/explorer/os_version")
+        if test -s "${__global}/explorer/os_release"
+        then
+            # shellcheck source=/dev/null
+            os_version=$(. "${__global}/explorer/os_release" && echo "${VERSION}")
+        else
+            os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global}/explorer/os_version")
+        fi
         os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
 
         # https://documentation.suse.com/sles/15-SP2/html/SLES-all/cha-suse.html#sec-suse-l10n

From b370b70ff4ccc1c6efca97f61f220a5ce0b191da Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 22:13:41 +0200
Subject: [PATCH 182/527] [explorer/os] Fix OS detection for openSUSE

All distros with ID_LIKE suse should be treated as "suse".
My openSUSE Leap 15.1 installation has:
ID_LIKE="suse opensuse"

This patch doesn't require a strict "suse" value but only the word suse to be in
the list.
---
 cdist/conf/explorer/os | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os
index 2d2aede6..8f54d9c8 100755
--- a/cdist/conf/explorer/os
+++ b/cdist/conf/explorer/os
@@ -144,7 +144,8 @@ esac
 
 if [ -f /etc/os-release ]; then
    # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse
-   if grep -q ^ID_LIKE=\"suse\" /etc/os-release 2>/dev/null; then
+   if (. /etc/os-release && echo "${ID_LIKE}" | grep -q '\(^\|\ \)suse\($\|\ \)')
+   then
       echo suse
       exit 0
    fi

From 17ab4bd80c9c9dda2fa4f8ccde5ee454f7b5ba2d Mon Sep 17 00:00:00 2001
From: Joachim Desroches <joachim.desroches@epfl.ch>
Date: Thu, 6 Aug 2020 11:45:05 +0200
Subject: [PATCH 183/527] Add Alpine Linux as supported for __filesystem.

---
 cdist/conf/type/__filesystem/explorer/lsblk | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk
index 9ae544ac..9be3c575 100644
--- a/cdist/conf/type/__filesystem/explorer/lsblk
+++ b/cdist/conf/type/__filesystem/explorer/lsblk
@@ -18,16 +18,16 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$("$__explorer/os")
+os=$("${__explorer:?}/os")
 
-if [ -f "$__object/parameter/device" ]; then
+if [ -f "${__object:?}/parameter/device" ]; then
     blkdev="$(cat "$__object/parameter/device")"
 else
-    blkdev="$__object_id"
+    blkdev="${__object_id:?}"
 fi
 
 case "$os" in
-    centos|fedora|redhat|suse|gentoo)
+    alpine|centos|fedora|redhat|suse|gentoo)
         if [ ! -x "$(command -v lsblk)" ]; then
             echo "lsblk is required for __filesystem type" >&2
             exit 1

From 74dd47c8c352cc3de3b5684bbe90520f61d95f37 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 15 Aug 2020 21:11:43 +0200
Subject: [PATCH 184/527] ++changelog

---
 docs/changelog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 269a2049..2c09bfea 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,12 @@
 Changelog
 ---------
 
+next:
+	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)
+	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)
+	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)
+	* Type __filesystem: Support Alpine Linux (Joachim Desroches)
+
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)
 	* New type: __download (Ander Punnar)

From 8f94a226c7f0b875987e25e57682414204380fa2 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 15 Aug 2020 21:54:07 +0200
Subject: [PATCH 185/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 2c09bfea..9e44cc4d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)
 	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)
 	* Type __filesystem: Support Alpine Linux (Joachim Desroches)
+	* Type __locale_system: Fix version comparison (Dennis Camera)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From 6fed1785298eb506e7ea6cd177efe7d96658769f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 17 Aug 2020 09:27:48 +0200
Subject: [PATCH 186/527] [explorer/os_version] Convert Debian sid to version
 number.

Conversion of Debian sid to versions is done based on Debian codenames.
The version number is the version number of the final release - 0.01.

It is unknown if Debian < 4.0 has any sort of version information
available (apart from maybe checking base-files package version).
But I don't think any of these systems are still alive,
so I think going with 3.99 is fine for those.
---
 cdist/conf/explorer/os_version | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 1d54ea60..a7b1d3bc 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -31,7 +31,32 @@ case "$("$__explorer/os")" in
       cat /etc/arch-release
    ;;
    debian)
-      cat /etc/debian_version
+      debian_version=$(cat /etc/debian_version)
+      case $debian_version
+      in
+          testing/unstable)
+              # previous to Debian 4.0 testing/unstable was used
+              # cf. https://metadata.ftp-master.debian.org/changelogs/main/b/base-files/base-files_11_changelog
+              echo 3.99
+              ;;
+          */sid)
+              # sid versions don't have a number, so we decode by codename:
+              case $(expr "$debian_version" : '\([a-z]\{1,\}\)/')
+              in
+                  bullseye) echo 10.99 ;;
+                  buster) echo 9.99 ;;
+                  stretch) echo 8.99 ;;
+                  jessie) echo 7.99 ;;
+                  wheezy) echo 6.99 ;;
+                  squeeze) echo 5.99 ;;
+                  lenny) echo 4.99 ;;
+                  *) exit 1
+              esac
+              ;;
+          *)
+              echo "$debian_version"
+              ;;
+      esac
    ;;
    devuan)
       cat /etc/devuan_version
@@ -73,4 +98,4 @@ case "$("$__explorer/os")" in
    alpine)
        cat /etc/alpine-release
    ;;
-esac
\ No newline at end of file
+esac

From 502d75304707704d3ba73374701394b98d259506 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 18 Aug 2020 00:46:07 +0300
Subject: [PATCH 187/527] [__unpack] add --onchange

---
 cdist/conf/type/__unpack/gencode-remote     | 5 +++++
 cdist/conf/type/__unpack/man.rst            | 3 +++
 cdist/conf/type/__unpack/parameter/optional | 1 +
 3 files changed, 9 insertions(+)

diff --git a/cdist/conf/type/__unpack/gencode-remote b/cdist/conf/type/__unpack/gencode-remote
index 3b7a19a7..c4451f73 100755
--- a/cdist/conf/type/__unpack/gencode-remote
+++ b/cdist/conf/type/__unpack/gencode-remote
@@ -80,3 +80,8 @@ if [ ! -f "$__object/parameter/preserve-archive" ]
 then
     echo "rm -f '$src'"
 fi
+
+if [ -f "$__object/parameter/onchange" ]
+then
+    cat "$__object/parameter/onchange"
+fi
diff --git a/cdist/conf/type/__unpack/man.rst b/cdist/conf/type/__unpack/man.rst
index bd0603cf..daa03814 100644
--- a/cdist/conf/type/__unpack/man.rst
+++ b/cdist/conf/type/__unpack/man.rst
@@ -50,6 +50,9 @@ backup-destination
 preserve-archive
     Don't delete archive after unpacking.
 
+onchange
+    Execute this command after unpack.
+
 
 EXAMPLES
 --------
diff --git a/cdist/conf/type/__unpack/parameter/optional b/cdist/conf/type/__unpack/parameter/optional
index da26ca63..d846ac75 100644
--- a/cdist/conf/type/__unpack/parameter/optional
+++ b/cdist/conf/type/__unpack/parameter/optional
@@ -1,3 +1,4 @@
 sum-file
 tar-strip
 tar-extra-args
+onchange

From d239169c4f91a4ce33be170ed29b08299c51234f Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 18 Aug 2020 00:48:58 +0300
Subject: [PATCH 188/527] [__download] fix manual: onchange parameter in wrong
 section

---
 cdist/conf/type/__download/man.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index 6ec0b19a..eb3ac971 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -30,9 +30,6 @@ sum
    By default output of ``cksum`` without filename is expected.
    Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
 
-onchange
-   Execute this command after download.
-
 
 OPTIONAL PARAMETERS
 -------------------
@@ -54,6 +51,9 @@ cmd-sum
    format specification ``%s`` which will become destination.
    For example: ``md5sum '%s' | awk '{print $1}'``.
 
+onchange
+   Execute this command after download.
+
 
 EXAMPLES
 --------

From ba26a437be4d8034037e7a6eb42382aee01f829e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 18 Aug 2020 11:06:19 +0200
Subject: [PATCH 189/527] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 9e44cc4d..a3e4167a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,9 @@ next:
 	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)
 	* Type __filesystem: Support Alpine Linux (Joachim Desroches)
 	* Type __locale_system: Fix version comparison (Dennis Camera)
+	* Type __unpack: Add --onchange parameter (Ander Punnar)
+	* Type __download: Fix manual (Ander Punnar)
+	* Explorer os_version: Convert Debian sid to version number (Dennis Camera)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From c17541f24cf118b0989a5899cab3a439ed534a46 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 22 Aug 2020 19:53:42 +0200
Subject: [PATCH 190/527] Expand and split by consecutive require delimiters

Resolves #832.
---
 cdist/emulator.py               |  6 +++++-
 cdist/test/emulator/__init__.py | 10 ++++++++++
 docs/src/cdist-manifest.rst     |  3 ++-
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/cdist/emulator.py b/cdist/emulator.py
index 9fe84056..60b94880 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -25,6 +25,7 @@ import argparse
 import logging
 import os
 import sys
+import re
 
 import cdist
 from cdist import core
@@ -389,12 +390,15 @@ class Emulator:
         if "require" in self.env:
             requirements = self.env['require']
             self.log.debug("reqs = " + requirements)
-            for requirement in requirements.split(" "):
+            for requirement in self._parse_require(requirements):
                 # Ignore empty fields - probably the only field anyway
                 if len(requirement) == 0:
                     continue
                 self.record_requirement(requirement)
 
+    def _parse_require(self, require):
+        return re.split(r'[ \t\n]+', require)
+
     def record_auto_requirements(self):
         """An object shall automatically depend on all objects that it
            defined in it's type manifest.
diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py
index e375676c..befd7b57 100644
--- a/cdist/test/emulator/__init__.py
+++ b/cdist/test/emulator/__init__.py
@@ -685,6 +685,16 @@ class EmulatorAlreadyExistingRequirementsWarnTestCase(test.CdistTestCase):
         self.env['require'] = '__directory/spam'
         emu = emulator.Emulator(argv, env=self.env)
 
+    def test_parse_require(self):
+        require = " \t \n  \t\t\n\t\na\tb\nc d \te\t\nf\ng\t "
+        expected = ['', 'a', 'b', 'c', 'd', 'e', 'f', 'g', '', ]
+
+        argv = ['__directory', 'spam']
+        emu = emulator.Emulator(argv, env=self.env)
+        requirements = emu._parse_require(require)
+
+        self.assertEqual(expected, requirements)
+
 
 if __name__ == '__main__':
     import unittest
diff --git a/docs/src/cdist-manifest.rst b/docs/src/cdist-manifest.rst
index 5dbca479..2e49a721 100644
--- a/docs/src/cdist-manifest.rst
+++ b/docs/src/cdist-manifest.rst
@@ -95,7 +95,8 @@ Dependencies
 ------------
 If you want to describe that something requires something else, just
 setup the variable "require" to contain the requirements. Multiple
-requirements can be added white space separated.
+requirements can be added separated with (optionally consecutive)
+delimiters including space, tab and newline.
 
 ::
 

From b5a40eb0d1c8990f0feb8e918a95e54e4fd35d98 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 27 Aug 2020 12:25:11 +0200
Subject: [PATCH 191/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index a3e4167a..eda77a12 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,6 +10,7 @@ next:
 	* Type __unpack: Add --onchange parameter (Ander Punnar)
 	* Type __download: Fix manual (Ander Punnar)
 	* Explorer os_version: Convert Debian sid to version number (Dennis Camera)
+	* Core: Expand require delimiter characters, split by consecutive delimiters (Darko Poljak)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From b1375464cc7507b38dd02f2df52c75489837d129 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 6 Sep 2020 13:24:58 +0200
Subject: [PATCH 192/527] __systemd_service: fix manpage typos

---
 cdist/conf/type/__systemd_service/man.rst | 25 +++++++++++++++--------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__systemd_service/man.rst b/cdist/conf/type/__systemd_service/man.rst
index 7eca398b..cd14c985 100644
--- a/cdist/conf/type/__systemd_service/man.rst
+++ b/cdist/conf/type/__systemd_service/man.rst
@@ -1,9 +1,10 @@
-cdist-type__systemd-service(7)
+cdist-type__systemd_service(7)
 ==============================
 
 NAME
 ----
-cdist-type__systemd-service - Controls a systemd service state
+cdist-type__systemd_service - Controls a systemd service state
+
 
 DESCRIPTION
 -----------
@@ -14,11 +15,12 @@ service after configuration applied or shutdown one service.
 The activation or deactivation is out of scope. Look for the
 :strong:`cdist-type__systemd_util`\ (7) type instead.
 
+
 REQUIRED PARAMETERS
 -------------------
-
 None.
 
+
 OPTIONAL PARAMETERS
 -------------------
 
@@ -31,12 +33,12 @@ state
     running
         Service should run (default)
 
-    stoppend
-        Service should stopped
+    stopped
+        Service should be stopped
 
 action
     Executes an action on on the service. It will only execute it if the
-    service keeps the state **running**. There are following actions, where:
+    service keeps the state ``running``. There are following actions, where:
 
     reload
         Reloads the service
@@ -48,11 +50,12 @@ BOOLEAN PARAMETERS
 ------------------
 
 if-required
-    Only execute the action if minimum one required type outputs a message to
-    **$__messages_out**. Through this, the action should only executed if a
+    Only execute the action if at minimum one required type outputs a message
+    to ``$__messages_out``. Through this, the action should only executed if a
     dependency did something. The action will not executed if no dependencies
     given.
 
+
 MESSAGES
 --------
 
@@ -68,12 +71,14 @@ restart
 reload
     Reloaded the service
 
+
 ABORTS
 ------
 Aborts in following cases:
 
 systemd or the service does not exist
 
+
 EXAMPLES
 --------
 .. code-block:: sh
@@ -95,13 +100,15 @@ EXAMPLES
 
     # reload the service for a modified configuration file
     # only reloads the service if the file really changed
-    require="__config_file/etc/foo.conf" __systemd_service foo \
+    require="__file/etc/foo.conf" __systemd_service foo \
         --action reload --if-required
 
+
 AUTHORS
 -------
 Matthias Stecher <matthiasstecher at gmx.de>
 
+
 COPYRIGHT
 ---------
 Copyright \(C) 2020 Matthias Stecher. You can redistribute it

From 6b262a61c19234f6450cef11f715020dcd5f40d5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 10 Sep 2020 13:24:58 +0200
Subject: [PATCH 193/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index eda77a12..e4ec3fdd 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,8 @@ next:
 	* Type __download: Fix manual (Ander Punnar)
 	* Explorer os_version: Convert Debian sid to version number (Dennis Camera)
 	* Core: Expand require delimiter characters, split by consecutive delimiters (Darko Poljak)
+	* Type __timezone: Make singleton (Dennis Camera)
+	* Type __systemd_service: Fix manpage typos (Matthias Stecher)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From 53b91adbd8be4d858f1f4da3389b63babc85db0c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 11 Sep 2020 14:19:45 +0200
Subject: [PATCH 194/527] Fix shellcheck

---
 cdist/conf/explorer/os | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os
index 8f54d9c8..46d87f3e 100755
--- a/cdist/conf/explorer/os
+++ b/cdist/conf/explorer/os
@@ -144,6 +144,7 @@ esac
 
 if [ -f /etc/os-release ]; then
    # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse
+   # shellcheck disable=SC1091
    if (. /etc/os-release && echo "${ID_LIKE}" | grep -q '\(^\|\ \)suse\($\|\ \)')
    then
       echo suse

From 2885c6a24838956167ae6c33ebf87067e8790083 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 11 Sep 2020 14:15:26 +0200
Subject: [PATCH 195/527] Release 6.8.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index e4ec3fdd..7a0d050d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)
 	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)
 	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)

From decc0ad54dffbafcd7d3644c28ae4d149926e612 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 18 Sep 2020 19:33:36 +0300
Subject: [PATCH 196/527] [__package_pip] detect pip binary

---
 cdist/conf/type/__package_pip/explorer/pip   | 10 ++++++++++
 cdist/conf/type/__package_pip/explorer/state |  2 +-
 cdist/conf/type/__package_pip/gencode-remote |  7 ++++++-
 3 files changed, 17 insertions(+), 2 deletions(-)
 create mode 100755 cdist/conf/type/__package_pip/explorer/pip

diff --git a/cdist/conf/type/__package_pip/explorer/pip b/cdist/conf/type/__package_pip/explorer/pip
new file mode 100755
index 00000000..cf9fae89
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/pip
@@ -0,0 +1,10 @@
+#!/bin/sh -e
+
+for bin in pip3 pip
+do
+    if check="$( command -v "$bin" )"
+    then
+        echo "$check"
+        break
+    fi
+done
diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state
index 5be07280..3cc98ab9 100644
--- a/cdist/conf/type/__package_pip/explorer/state
+++ b/cdist/conf/type/__package_pip/explorer/state
@@ -32,7 +32,7 @@ pipparam="$__object/parameter/pip"
 if [ -f "$pipparam" ]; then
     pip=$(cat "$pipparam")
 else
-    pip="pip"
+    pip="$( "$__type_explorer/pip" )"
 fi
 
 # If there is no pip, it may get created from somebody else.
diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index dcc4fdf9..9c5eb0d1 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -38,7 +38,12 @@ pipparam="$__object/parameter/pip"
 if [ -f "$pipparam" ]; then
     pip=$(cat "$pipparam")
 else
-    pip="pip"
+    pip="$( cat "$__object/explorer/pip" )"
+    if [ -z "$pip" ]
+    then
+        echo 'pip not found in path' >&2
+        exit 1
+    fi
 fi
 
 runasparam="$__object/parameter/runas"

From 89b621511561105cfcfcf12bbf820b74ab03396d Mon Sep 17 00:00:00 2001
From: Darko Poljak <darko.poljak@ungleich.ch>
Date: Mon, 21 Sep 2020 09:04:05 +0200
Subject: [PATCH 197/527] Clarify stdin input

Resolve #836.
---
 cdist/argparse.py                | 36 ++++++--------------------
 cdist/config.py                  |  6 +++--
 cdist/inventory.py               |  4 +--
 cdist/test/inventory/__init__.py |  3 +--
 docs/src/man1/cdist.rst          | 44 ++++++++++----------------------
 5 files changed, 29 insertions(+), 64 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 77303591..1d16bb25 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -273,8 +273,7 @@ def get_parsers():
             '-f', '--file',
             help=('Read specified file for a list of additional hosts to '
                   'operate on or if \'-\' is given, read stdin (one host per '
-                  'line). If no host or host file is specified then, by '
-                  'default, read hosts from stdin.'),
+                  'line).'),
             dest='hostfile', required=False)
     parser['config_args'].add_argument(
            '-p', '--parallel', nargs='?', metavar='HOST_MAX',
@@ -326,9 +325,7 @@ def get_parsers():
     parser['add-host'].add_argument(
            '-f', '--file',
            help=('Read additional hosts to add from specified file '
-                 'or from stdin if \'-\' (each host on separate line). '
-                 'If no host or host file is specified then, by default, '
-                 'read from stdin.'),
+                 'or from stdin if \'-\' (each host on separate line). '),
            dest='hostfile', required=False)
 
     parser['add-tag'] = parser['invsub'].add_parser(
@@ -342,20 +339,12 @@ def get_parsers():
     parser['add-tag'].add_argument(
            '-f', '--file',
            help=('Read additional hosts to add tags from specified file '
-                 'or from stdin if \'-\' (each host on separate line). '
-                 'If no host or host file is specified then, by default, '
-                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                 ' are specified then tags are read from stdin and are'
-                 ' added to all hosts.'),
+                 'or from stdin if \'-\' (each host on separate line). '),
            dest='hostfile', required=False)
     parser['add-tag'].add_argument(
            '-T', '--tag-file',
            help=('Read additional tags to add from specified file '
-                 'or from stdin if \'-\' (each tag on separate line). '
-                 'If no tag or tag file is specified then, by default, '
-                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                 ' are specified then tags are read from stdin and are'
-                 ' added to all hosts.'),
+                 'or from stdin if \'-\' (each tag on separate line). '),
            dest='tagfile', required=False)
     parser['add-tag'].add_argument(
            '-t', '--taglist',
@@ -376,9 +365,7 @@ def get_parsers():
     parser['del-host'].add_argument(
             '-f', '--file',
             help=('Read additional hosts to delete from specified file '
-                  'or from stdin if \'-\' (each host on separate line). '
-                  'If no host or host file is specified then, by default, '
-                  'read from stdin.'),
+                  'or from stdin if \'-\' (each host on separate line). '),
             dest='hostfile', required=False)
 
     parser['del-tag'] = parser['invsub'].add_parser(
@@ -396,20 +383,13 @@ def get_parsers():
     parser['del-tag'].add_argument(
             '-f', '--file',
             help=('Read additional hosts to delete tags for from specified '
-                  'file or from stdin if \'-\' (each host on separate line). '
-                  'If no host or host file is specified then, by default, '
-                  'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                  ' are specified then tags are read from stdin and are'
-                  ' deleted from all hosts.'),
+                  'file or from stdin if \'-\' (each host on separate '
+                  'line). '),
             dest='hostfile', required=False)
     parser['del-tag'].add_argument(
             '-T', '--tag-file',
             help=('Read additional tags from specified file '
-                  'or from stdin if \'-\' (each tag on separate line). '
-                  'If no tag or tag file is specified then, by default, '
-                  'read from stdin. If no tags/tagfile nor'
-                  ' hosts/hostfile are specified then tags are read from'
-                  ' stdin and are added to all hosts.'),
+                  'or from stdin if \'-\' (each tag on separate line). '),
             dest='tagfile', required=False)
     parser['del-tag'].add_argument(
             '-t', '--taglist',
diff --git a/cdist/config.py b/cdist/config.py
index 30416008..e84f6f84 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -175,9 +175,11 @@ class Config:
             raise cdist.Error(("Cannot read both, manifest and host file, "
                                "from stdin"))
 
-        # if no host source is specified then read hosts from stdin
         if not (args.hostfile or args.host):
-            args.hostfile = '-'
+            if args.tag or args.all_tagged_hosts:
+                raise cdist.Error(("Target host tag(s) missing"))
+            else:
+                raise cdist.Error(("Target host(s) missing"))
 
         if args.manifest == '-':
             # read initial manifest from stdin
diff --git a/cdist/inventory.py b/cdist/inventory.py
index 6ab20fa7..0387f326 100644
--- a/cdist/inventory.py
+++ b/cdist/inventory.py
@@ -299,7 +299,7 @@ class InventoryHost(Inventory):
         self.all = all
 
         if not self.hosts and not self.hostfile:
-            self.hostfile = "-"
+            raise cdist.Error("Host(s) missing")
 
     def _new_hostpath(self, hostpath):
         # create empty file
@@ -355,7 +355,7 @@ class InventoryTag(Inventory):
         else:
             self.allhosts = False
         if not self.tags and not self.tagfile:
-            self.tagfile = "-"
+            raise cdist.Error("Tag(s) missing")
 
         if self.hostfile == "-" and self.tagfile == "-":
             raise cdist.Error("Cannot read both, hosts and tags, from stdin")
diff --git a/cdist/test/inventory/__init__.py b/cdist/test/inventory/__init__.py
index 287c855c..a8cd8bf8 100644
--- a/cdist/test/inventory/__init__.py
+++ b/cdist/test/inventory/__init__.py
@@ -307,11 +307,10 @@ class InventoryTestCase(test.CdistTestCase):
             raise e
 
     # InventoryTag
+    @unittest.expectedFailure
     def test_inventory_tag_init(self):
         invTag = inventory.InventoryTag(db_basedir=inventory_dir,
                                         action="add")
-        self.assertTrue(invTag.allhosts)
-        self.assertEqual(invTag.tagfile, "-")
 
     def test_inventory_tag_stdin_multiple_hosts(self):
         try:
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index aa2607f8..0ecb4a61 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -177,10 +177,8 @@ Install command is currently in beta.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read specified file for a list of additional hosts to operate on
-    or if '-' is given, read stdin (one host per line).
-    If no host or host file is specified then, by default,
-    read hosts from stdin. For the file format see
-    :strong:`HOSTFILE FORMAT` below.
+    or if '-' is given, read stdin (one host per line). For the file
+    format see :strong:`HOSTFILE FORMAT` below.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -299,9 +297,8 @@ Add host(s) to inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to add from specified file or
-    from stdin if '-' (each host on separate line). If no
-    host or host file is specified then, by default, read
-    from stdin. Hostfile format is the same as config hostfile format.
+    from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -327,11 +324,8 @@ Add tag(s) to inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to add tags from specified file
-    or from stdin if '-' (each host on separate line). If
-    no host or host file is specified then, by default,
-    read from stdin. If no tags/tagfile nor hosts/hostfile
-    are specified then tags are read from stdin and are
-    added to all hosts. Hostfile format is the same as config hostfile format.
+    or from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -346,11 +340,8 @@ Add tag(s) to inventory database.
 
 **-T TAGFILE, --tag-file TAGFILE**
     Read additional tags to add from specified file or
-    from stdin if '-' (each tag on separate line). If no
-    tag or tag file is specified then, by default, read
-    from stdin. If no tags/tagfile nor hosts/hostfile are
-    specified then tags are read from stdin and are added
-    to all hosts. Tagfile format is the same as config hostfile format.
+    from stdin if '-' (each tag on separate line).
+    Tagfile format is the same as config hostfile format.
 
 **-t TAGLIST, --taglist TAGLIST**
     Tag list to be added for specified host(s), comma
@@ -372,9 +363,8 @@ Delete host(s) from inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to delete from specified file or
-    from stdin if '-' (each host on separate line). If no
-    host or host file is specified then, by default, read
-    from stdin. Hostfile format is the same as config hostfile format.
+    from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -404,11 +394,8 @@ Delete tag(s) from inventory database.
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to delete tags for from
     specified file or from stdin if '-' (each host on
-    separate line). If no host or host file is specified
-    then, by default, read from stdin. If no tags/tagfile
-    nor hosts/hostfile are specified then tags are read
-    from stdin and are deleted from all hosts. Hostfile
-    format is the same as config hostfile format.
+    separate line). Hostfile format is the same as
+    config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -423,11 +410,8 @@ Delete tag(s) from inventory database.
 
 **-T TAGFILE, --tag-file TAGFILE**
     Read additional tags from specified file or from stdin
-    if '-' (each tag on separate line). If no tag or tag
-    file is specified then, by default, read from stdin.
-    If no tags/tagfile nor hosts/hostfile are specified
-    then tags are read from stdin and are added to all
-    hosts. Tagfile format is the same as config hostfile format.
+    if '-' (each tag on separate line).
+    Tagfile format is the same as config hostfile format.
 
 **-t TAGLIST, --taglist TAGLIST**
     Tag list to be deleted for specified host(s), comma

From 89a0080e133052b82ce19e7fdbaf045dcc833da7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:09:26 +0200
Subject: [PATCH 198/527] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 7a0d050d..260b4fe4 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* Core: Clarify stdin input (Darko Poljak)
+	* Type __package_pip: Detect pip binary (Ander Punnar)
+
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)
 	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)

From 0fc10749edc698d1f70f826603d009fba943d252 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:11:35 +0200
Subject: [PATCH 199/527] Fix shellcheck

---
 cdist/conf/type/__package_pip/gencode-remote | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 9c5eb0d1..a1375c2d 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -60,7 +60,7 @@ case "$state_should" in
         then
             echo "su -c '$pip install -q $name' $runas"
         else
-            echo $pip install -q "$name"
+            echo "$pip" install -q "$name"
         fi
         echo "installed" >> "$__messages_out"
     ;;
@@ -69,7 +69,7 @@ case "$state_should" in
         then
             echo "su -c '$pip uninstall -q -y $name' $runas"
         else
-            echo $pip uninstall -q -y "$name"
+            echo "$pip" uninstall -q -y "$name"
         fi
         echo "removed" >> "$__messages_out"
     ;;

From b6922508b9e07834009a6bfae2d3933b5ea62fbf Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:17:34 +0200
Subject: [PATCH 200/527] Update helper script

---
 bin/build-helper | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/build-helper b/bin/build-helper
index ed41e438..d4d603ed 100755
--- a/bin/build-helper
+++ b/bin/build-helper
@@ -371,7 +371,6 @@ eof
 Manual steps post release:
     - cdist-web
     - send generated mailinglist.tmp mail
-    - twitter
 eof
     ;;
 

From 84a7818121047cf6bb23d8bd3c055d09bb033d32 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 23 Sep 2020 20:29:47 +0200
Subject: [PATCH 201/527] docs: make varaibles environment-aware

There are all overwriting the environment, even the comment states
otherwise. Fixes it.
---
 docs/src/Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/src/Makefile b/docs/src/Makefile
index 2ecf7a32..ba87d170 100644
--- a/docs/src/Makefile
+++ b/docs/src/Makefile
@@ -2,10 +2,10 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-PAPER         =
-BUILDDIR      = ../dist
+SPHINXOPTS   ?=
+SPHINXBUILD  ?= sphinx-build
+PAPER        ?=
+BUILDDIR     ?= ../dist
 # for cache, etc.
 _BUILDDIR     = _build
 

From 73d6c9d469d1429999ea2e980b3ac4c63a3e0dd4 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 27 Sep 2020 10:17:35 +0200
Subject: [PATCH 202/527] Add custom remote copy/exec examples

---
 docs/src/cdist-remote-exec-copy.rst | 389 +++++++++++++++++++++++++++-
 1 file changed, 388 insertions(+), 1 deletion(-)

diff --git a/docs/src/cdist-remote-exec-copy.rst b/docs/src/cdist-remote-exec-copy.rst
index bb818310..e7b7b226 100644
--- a/docs/src/cdist-remote-exec-copy.rst
+++ b/docs/src/cdist-remote-exec-copy.rst
@@ -10,7 +10,7 @@ By default this is accomplished with ssh and scp respectively.
 The default implementations used by cdist are::
 
     __remote_exec: ssh -o User=root
-    __remote_copy: scp -o User=root
+    __remote_copy: scp -o User=root -q
 
 The user can override these defaults by providing custom implementations and
 passing them to cdist with the --remote-exec and/or --remote-copy arguments.
@@ -26,3 +26,390 @@ specified by enclosed in square brackets (see :strong:`ssh`\ (1) and
 With this simple interface the user can take total control of how cdist
 interacts with the target when required, while the default implementation 
 remains as simple as possible.
+
+
+Examples
+--------
+
+Here are examples of using alternative __remote_copy and __remote_exec scripts.
+
+All scripts from below are present in cdist sources in `other/examples/remote`
+directory.
+
+ssh
+~~~
+
+Same as cdist default.
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    #echo "$@" | logger -t "cdist-ssh-copy"
+    scp -o User=root -q $@
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    #echo "$@" | logger -t "cdist-ssh-exec"
+    ssh -o User=root $@
+
+local
+~~~~~
+
+This effectively turns remote calling into local calling. Probably most useful
+for the unit testing.
+
+**copy**
+
+.. code-block:: sh
+
+    code="$(echo "$@" | sed "s|\([[:space:]]\)$__target_host:|\1|g")"
+    cp -L $code
+
+**exec**
+
+.. code-block:: sh
+
+    target_host=$1; shift
+    echo "$@" | /bin/sh 
+
+chroot
+~~~~~~
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script /path/to/your/chroot" target-id
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-chroot-copy"
+       :
+    }
+
+    chroot="$1"; shift
+    target_host="$__target_host"
+
+    # replace target_host with chroot location
+    code="$(echo "$@" | sed "s|$target_host:|$chroot|g")"
+
+    log "target_host: $target_host"
+    log "chroot: $chroot"
+    log "$@"
+    log "$code"
+
+    # copy files into chroot
+    cp $code
+
+    log "-----"
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script /path/to/your/chroot" target-id
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-chroot-exec"
+       :
+    }
+
+    chroot="$1"; shift
+    target_host="$1"; shift
+
+    script=$(mktemp "${chroot}/tmp/chroot-${0##*/}.XXXXXXXXXX")
+    trap cleanup INT TERM EXIT
+    cleanup() {
+       [ $__cdist_debug ] || rm "$script"
+    }
+
+    log "target_host: $target_host"
+    log "script: $script"
+    log "@: $@"
+    echo "#!/bin/sh -l" > "$script"
+    echo "$@" >> "$script"
+    chmod +x "$script"
+
+    relative_script="${script#$chroot}"
+    log "relative_script: $relative_script"
+
+    # run in chroot
+    chroot "$chroot" "$relative_script"
+
+    log "-----"
+
+rsync
+~~~~~
+
+**copy**
+
+Usage: cdist config --remote-copy /path/to/this/script target_host
+
+.. code-block:: sh
+
+    # For rsync to do the right thing, the source has to end with "/" if it is
+    # a directory. The below preprocessor loop takes care of that.
+
+    # second last argument is the source
+    source_index=$(($#-1))
+    index=0
+    for arg in $@; do
+       if [ $index -eq 0 ]; then
+          # reset $@
+          set --
+       fi
+       index=$((index+=1))
+       if [ $index -eq $source_index -a -d "$arg" ]; then
+          arg="${arg%/}/"
+       fi
+       set -- "$@" "$arg"
+    done
+
+    rsync --backup --suffix=~cdist -e 'ssh -o User=root' $@
+
+schroot
+~~~~~~~
+
+__remote_copy and __remote_exec scripts to run cdist against a chroot on the
+target host over ssh.
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script schroot-chroot-name" target_host
+
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-schroot-copy"
+       :
+    }
+
+    chroot_name="$1"; shift
+    target_host="$__target_host"
+
+    # get directory for given chroot_name
+    chroot="$(ssh -o User=root -q $target_host schroot -c $chroot_name --config | awk -F = '/directory=/ {print $2}')"
+
+    # prefix destination with chroot
+    code="$(echo "$@" | sed "s|$target_host:|$target_host:$chroot|g")"
+
+    log "target_host: $target_host"
+    log "chroot_name: $chroot_name"
+    log "chroot: $chroot"
+    log "@: $@"
+    log "code: $code"
+
+    # copy files into remote chroot
+    scp -o User=root -q $code
+
+    log "-----"
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script schroot-chroot-name" target_host
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-schroot-exec"
+       :
+    }
+
+    chroot_name="$1"; shift
+    target_host="$1"; shift
+
+    code="ssh -o User=root -q $target_host schroot -c $chroot_name -- $@"
+
+    log "target_host: $target_host"
+    log "chroot_name: $chroot_name"
+    log "@: $@"
+    log "code: $code"
+
+    # run in remote chroot
+    $code
+
+    log "-----"
+
+schroot-uri
+~~~~~~~~~~~
+
+__remote_exec/__remote_copy script to run cdist against a schroot target URI.
+
+Usage::
+
+    cdist config \
+        --remote-exec "/path/to/this/script exec" \
+        --remote-copy "/path/to/this/script copy" \
+        target_uri
+
+    # target_uri examples:
+    schroot:///chroot-name
+    schroot://foo.ethz.ch/chroot-name
+    schroot://user-name@foo.ethz.ch/chroot-name
+
+    # and how to match them in .../manifest/init
+    case "$target_host" in
+    schroot://*)
+        # any schroot
+    ;;
+    schroot://foo.ethz.ch/*)
+        # any schroot on specific host
+    ;;
+    schroot://foo.ethz.ch/chroot-name)
+        # specific schroot on specific host
+    ;;
+    schroot:///chroot-name)
+        # specific schroot on localhost
+    ;;
+    esac
+
+**copy/exec**
+
+.. code-block:: sh
+
+    my_name="${0##*/}"
+    mode="$1"; shift
+
+    log() {
+       # uncomment me for debugging
+       #echo "$@" | logger -t "cdist-$my_name-$mode"
+       :
+    }
+
+    die() {
+       echo "$@" >&2
+       exit 1
+    }
+
+
+    uri="$__target_host"
+
+    scheme="${uri%%:*}"; rest="${uri#$scheme:}"; rest="${rest#//}"
+    authority="${rest%%/*}"; rest="${rest#$authority}"
+    path="${rest%\?*}"; rest="${rest#$path}"
+    schroot_name="${path#/}"
+
+    [ "$scheme" = "schroot" ] || die "Failed to parse scheme from __target_host ($__target_host). Expected 'schroot', got '$scheme'"
+    [ -n "$schroot_name" ] || die "Failed to parse schroot name from __target_host: $__target_host"
+
+    case "$authority" in
+       '')
+          # authority is empty, neither user nor host given
+          user=""
+          host=""
+       ;; 
+       *@*)
+          # authority contains @, take user from authority
+          user="${authority%@*}"
+          host="${authority#*@}"
+       ;; 
+       *) 
+          # no user in authority, default to root
+          user="root"
+          host="$authority"
+       ;;
+    esac
+
+    log "mode: $mode"
+    log "@: $@"
+    log "uri: $uri"
+    log "scheme: $scheme"
+    log "authority: $authority"
+    log "user: $user"
+    log "host: $host"
+    log "path: $path"
+    log "schroot_name: $schroot_name"
+
+    exec_prefix=""
+    copy_prefix=""
+    if [ -n "$host" ]; then
+       # we are working on a remote host
+       exec_prefix="ssh -o User=$user -q $host"
+       copy_prefix="scp -o User=$user -q"
+       copy_destination_prefix="$host:"
+    else
+       # working on local machine
+       copy_prefix="cp"
+       copy_destination_prefix=""
+    fi
+    log "exec_prefix: $exec_prefix"
+    log "copy_prefix: $copy_prefix"
+    log "copy_destination_prefix: $copy_destination_prefix"
+
+    case "$mode" in
+       exec)
+          # In exec mode the first argument is the __target_host which we already got from env. Get rid of it.
+          shift
+          code="$exec_prefix schroot -c $schroot_name -- sh -c '$@'"
+       ;;
+       copy)
+          # get directory for given chroot_name
+          schroot_directory="$($exec_prefix schroot -c $schroot_name --config | awk -F = '/directory=/ {print $2}')"
+          [ -n "$schroot_directory" ] || die "Failed to retreive schroot directory for schroot: $schroot_name"
+          log "schroot_directory: $schroot_directory"
+          # prefix destination with chroot
+          code="$copy_prefix $(echo "$@" | sed "s|$uri:|${copy_destination_prefix}${schroot_directory}|g")"
+       ;;
+       *) die "Unknown mode: $mode";;
+    esac
+
+    log "code: $code"
+
+    # Run the code
+    $code
+
+    log "-----"
+
+sudo
+~~~~
+
+**copy**
+
+Use rsync over ssh to copy files. Uses the "--rsync-path" option
+to run the remote rsync instance with sudo.
+
+This command assumes your ssh configuration is already set up in ~/.ssh/config.
+
+Usage: cdist config --remote-copy /path/to/this/script target_host
+
+.. code-block:: sh
+
+    # For rsync to do the right thing, the source has to end with "/" if it is
+    # a directory. The below preprocessor loop takes care of that.
+
+    # second last argument is the source
+    source_index=$(($#-1))
+    index=0
+    for arg in $@; do
+       if [ $index -eq 0 ]; then
+          # reset $@
+          set --
+       fi
+       index=$((index+=1))
+       if [ $index -eq $source_index -a -d "$arg" ]; then
+          arg="${arg%/}/"
+       fi
+       set -- "$@" "$arg"
+    done
+
+    rsync --copy-links --rsync-path="sudo rsync" -e 'ssh' "$@"
+
+**exec**
+
+Prefixes all remote commands with sudo.
+
+This command assumes your ssh configuration is already set up in ~/.ssh/config.
+
+Usage: cdist config --remote-exec "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    host="$1"; shift
+    ssh -q "$host" sudo sh -c \""$@"\"

From 652c89185816526e5165ff9d5686a0d245ec3e5f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 29 Sep 2020 05:57:54 +0200
Subject: [PATCH 203/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 260b4fe4..8d380524 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
+	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From f994226d0e23b667309fd75116c0c8a14d15f34d Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 29 Sep 2020 19:44:47 +0200
Subject: [PATCH 204/527] [__package_pkgng_freebsd] Bootstrap pkg if necessary

In a pristine FreeBSD base installation, pkg is really a bootstrapper utility,
in such cases the type used to fail instead of automatically bootstrapping pkg.
---
 .../type/__package_pkgng_freebsd/explorer/pkg_bootstrapped   | 4 ++++
 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version | 5 +++++
 cdist/conf/type/__package_pkgng_freebsd/gencode-remote       | 5 +++++
 3 files changed, 14 insertions(+)
 create mode 100755 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped

diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped
new file mode 100755
index 00000000..429f15d3
--- /dev/null
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+if pkg -N >/dev/null 2>&1; then
+	echo "YES"
+fi
diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
index 92ce0623..f0fb9127 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
@@ -21,6 +21,11 @@
 # Retrieve the status of a package - parsed dpkg output
 #
 
+if ! pkg -N >/dev/null 2>&1; then
+   # Nothing to do if pkg is not bootstrapped
+   exit
+fi
+
 if [ -f "$__object/parameter/name" ]; then
    name="$(cat "$__object/parameter/name")"
 else
diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
index dd36efda..b5944177 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
+++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
@@ -43,6 +43,7 @@ fi
 repo="$(cat "$__object/parameter/repo")"
 state="$(cat "$__object/parameter/state")"
 curr_version="$(cat "$__object/explorer/pkg_version")"
+pkg_bootstrapped="$(cat "$__object/explorer/pkg_bootstrapped")"
 add_cmd="pkg install -y"
 rm_cmd="pkg delete -y"
 upg_cmd="pkg upgrade -y"
@@ -73,6 +74,10 @@ execcmd(){
          ;;
    esac
 
+   if [ -z "${pkg_bootstrapped}" ]; then
+       echo "pkg bootstrap -y >/dev/null 2>&1"
+   fi
+
    echo "$_cmd >/dev/null 2>&1"   # Silence the output of the command
    echo "status=\$?"
    echo "if [ \"\$status\" -ne \"0\" ]; then"

From 52b5f05163416196403634cb9a52f746cdb96391 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 30 Sep 2020 08:56:31 +0200
Subject: [PATCH 205/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8d380524..34c97d67 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
+	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 3fa74b454a9512319e3559a53dd477f38724fd9b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 30 Sep 2020 15:43:32 +0200
Subject: [PATCH 206/527] Fix typo

---
 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
index f0fb9127..1c6ba5e5 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Retrieve the status of a package - parsed dpkg output
+# Retrieve the status of a package - parsed pkgng output
 #
 
 if ! pkg -N >/dev/null 2>&1; then

From 5aeed14b1b8b022f952680cceb12a5099a099558 Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Thu, 8 Oct 2020 16:15:20 +0200
Subject: [PATCH 207/527] Fixed calling of __systemd_service type with correct
 arguments.

---
 cdist/conf/type/__service/manifest | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__service/manifest b/cdist/conf/type/__service/manifest
index cb5af234..beb0713c 100644
--- a/cdist/conf/type/__service/manifest
+++ b/cdist/conf/type/__service/manifest
@@ -7,7 +7,9 @@ action="$(cat "$__object/parameter/action")"
 
 case "$manager" in
 	systemd)
-		__systemd_service "$name" --action "$action"
+		test "$action" = "start" && action="running"
+		test "$action" = "stop"  && action="stopped"
+		__systemd_service "$name" --state "$action"
 	;;
 	*)
 		# Unknown: handled by `service $NAME $action` in gencode-remote.

From c030deea3dbccfe73637b730f0d545db7f8d5f35 Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Fri, 9 Oct 2020 06:51:44 +0200
Subject: [PATCH 208/527] [__line] Add support for '--state replace'

It is currently counter-intuitive that something like:

    # File '/thing' contents
    #SomeSetting WrongValue

    # Manifest
    __line '/thing' \
           --line 'SomeSeting GoodValue' \
           --regex '^(#[[:space:]]*)?SomeSetting[[:space:]]'

Produces:

    # Resulting '/thing' contents
    #SomeSetting WrongValue

This makes sense given the implementation, but it masks a very common use-case.

Changing the default behaviour for such a base type is not really an option, so
instead we add a `replace` as a valid value for `--state`, which would result
in:

    # Resulting '/thing' contents with: --state replace
    SomeSetting GoodValue

For compatibility, if the regex is missing, `--state replace` behaves just as
`--state present`.
---
 cdist/conf/type/__line/explorer/state | 12 +++++++++++-
 cdist/conf/type/__line/gencode-remote | 10 +++++++---
 cdist/conf/type/__line/man.rst        | 13 +++++++++++--
 3 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state
index e8fc3630..9d480b19 100755
--- a/cdist/conf/type/__line/explorer/state
+++ b/cdist/conf/type/__line/explorer/state
@@ -53,8 +53,10 @@ function _find(_text, _pattern) {
 BEGIN {
    getline anchor < (ENVIRON["__object"] "/parameter/" position)
    getline pattern < (ENVIRON["__object"] "/parameter/" needle)
+   getline line < (ENVIRON["__object"] "/parameter/line")
 
    found_line = 0
+   correct_line = 0
    correct_pos = (position != "after" && position != "before")
 }
 {
@@ -63,15 +65,18 @@ BEGIN {
          getline
          if (_find($0, pattern)) {
             found_line++
+	    if (index($0, line) == 1) { correct_line++ }
             correct_pos = 1
             exit 0
          }
       } else if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
       }
    } else if (position == "before") {
       if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
          getline
          if (match($0, anchor)) {
             correct_pos = 1
@@ -81,13 +86,18 @@ BEGIN {
    } else {
       if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
          exit 0
       }
    }
 }
 END {
    if (found_line && correct_pos) {
-      print "present"
+      if (correct_line) {
+        print "present"
+      } else {
+        print "matching"
+      }
    } else if (found_line) {
       print "wrongposition"
    } else {
diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote
index 88cae68b..a89886da 100755
--- a/cdist/conf/type/__line/gencode-remote
+++ b/cdist/conf/type/__line/gencode-remote
@@ -38,7 +38,11 @@ if [ -z "$state_is" ]; then
    exit 1
 fi
 
-if [ "$state_should" = "$state_is" ]; then
+if [ "$state_should" = "$state_is" ] || \
+	{ [ "$state_should" = "present" ] && [ "$state_is" = "matching" ] ;} || \
+	{ [ "$state_should" = "replace" ] && [ "$state_is" = "present" ] ;} ; then
+   # If state matches already, or 'present' is used and regex matches
+   # or 'replace' is used and the exact line is present, then there is
    # nothing to do
    exit 0
 fi
@@ -61,8 +65,8 @@ fi
 add=0
 remove=0
 case "$state_should" in
-   present)
-      if [ "$state_is" = "wrongposition" ]; then
+   present|replace)
+      if [ "$state_is" = "wrongposition" ] || [ "$state_is" = "matching" ]; then
          echo updated >> "$__messages_out"
          remove=1
       else
diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst
index f76cab64..70490f68 100644
--- a/cdist/conf/type/__line/man.rst
+++ b/cdist/conf/type/__line/man.rst
@@ -31,7 +31,7 @@ file
 line
     Specifies the line which should be absent or present.
 
-    Must be present, if state is 'present'.
+    Must be present, if state is 'present' or 'replace'.
     Ignored if regex is given and state is 'absent'.
 
 regex
@@ -41,10 +41,13 @@ regex
     If state is 'absent', ensure all lines matching the regular expression
     are absent.
 
+    If state is 'replace', ensure all lines matching the regular expression
+    are exactly 'line'.
+
     The regular expression is interpreted by awk's match function.
 
 state
-    'present' or 'absent', defaults to 'present'
+    'present', 'absent' or 'replace', defaults to 'present'.
 
 onchange
     The code to run if line is added, removed or updated.
@@ -99,6 +102,12 @@ EXAMPLES
         --line '-session required pam_exec.so debug log=/tmp/classify.log /usr/local/libexec/classify' \
         --after '^session[[:space:]]+include[[:space:]]+password-auth-ac$'
 
+    # Uncomment as needed and set a value in a configuration file.
+    __line /etc/example.conf \
+        --line 'SomeSetting SomeValue' \
+        --regex '^(#[[:space:]]*)?SomeSetting[[:space:]]' \
+        --state replace
+
 
 SEE ALSO
 --------

From 4df5c91912edf9f31a6716d7cbe0adaedbc2b0e3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 9 Oct 2020 06:52:52 +0200
Subject: [PATCH 209/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 34c97d67..6d237449 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,8 @@ next:
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
 	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
+	* Type __service: Fix calling __systemd_service (Mark Verboom)
+	* Type __line: Add 'replace' state (Evil Ham)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 8ecae42199f27693eec771513174ce8143c8ca83 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:00:11 +0300
Subject: [PATCH 210/527] remove bin/cdist script

---
 bin/cdist | 33 ---------------------------------
 1 file changed, 33 deletions(-)
 delete mode 100755 bin/cdist

diff --git a/bin/cdist b/bin/cdist
deleted file mode 100755
index 645020a1..00000000
--- a/bin/cdist
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/sh
-# -*- coding: utf-8 -*-
-#
-# 2012 Nico Schottelius (nico-cdist at schottelius.org)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-# Wrapper for real script to allow execution from checkout
-dir=${0%/*}
-
-# Ensure version is present - the bundled/shipped version contains a static version,
-# the git version contains a dynamic version
-"$dir/build-helper" version
-
-libdir=$(cd "${dir}/../" && pwd -P)
-export PYTHONPATH="${libdir}"
-
-"$dir/../scripts/cdist" "$@"

From 45d51c0e1553ce12fcea09b1b9e0d2fb9c0304c5 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:00:44 +0300
Subject: [PATCH 211/527] rename build-helper -> cdist-build-helper

---
 bin/{build-helper => cdist-build-helper} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename bin/{build-helper => cdist-build-helper} (100%)

diff --git a/bin/build-helper b/bin/cdist-build-helper
similarity index 100%
rename from bin/build-helper
rename to bin/cdist-build-helper

From 3f1939716f4c4c2cdf538a22170a67fe181c675c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:02:45 +0300
Subject: [PATCH 212/527] enable running scripts/cdist directly and symlinked

---
 scripts/cdist | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scripts/cdist b/scripts/cdist
index b1d782ab..2ce40ae0 100755
--- a/scripts/cdist
+++ b/scripts/cdist
@@ -22,7 +22,15 @@
 #
 
 import logging
+import os
 import sys
+
+cdist_bin = os.path.abspath(__file__)
+if os.path.islink(cdist_bin):
+    cdist_bin = os.readlink(cdist_bin)
+cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
+sys.path.insert(0, cdist_dir)
+
 import cdist
 import cdist.argparse
 import cdist.banner

From fdc1ab93e968575b1d1f6f3b2a6cefa6ed6bb850 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:03:11 +0300
Subject: [PATCH 213/527] move scripts/* to bin/

---
 {scripts => bin}/cdist          | 0
 {scripts => bin}/cdist-dump     | 0
 {scripts => bin}/cdist-new-type | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 rename {scripts => bin}/cdist (100%)
 rename {scripts => bin}/cdist-dump (100%)
 rename {scripts => bin}/cdist-new-type (100%)

diff --git a/scripts/cdist b/bin/cdist
similarity index 100%
rename from scripts/cdist
rename to bin/cdist
diff --git a/scripts/cdist-dump b/bin/cdist-dump
similarity index 100%
rename from scripts/cdist-dump
rename to bin/cdist-dump
diff --git a/scripts/cdist-new-type b/bin/cdist-new-type
similarity index 100%
rename from scripts/cdist-new-type
rename to bin/cdist-new-type

From 86057cef19b10859cd66ceb4e3ebade5986aff89 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:05:17 +0300
Subject: [PATCH 214/527] don't die if there is no version.py

---
 cdist/__init__.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index be573170..26e7d071 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -24,10 +24,13 @@ import os
 import hashlib
 
 import cdist.log
-import cdist.version
 
 
-VERSION = cdist.version.VERSION
+try:
+    import cdist.version
+    VERSION = cdist.version.VERSION
+except ModuleNotFoundError:
+    VERSION = 'from git'
 
 BANNER = """
              ..          .       .x+=:.        s

From fd04c036139f150be79801e5fd3b49086e5d7263 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 13:42:16 +0300
Subject: [PATCH 215/527] add parent dir to module search path only when
 importing fails

---
 bin/cdist | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index 2ce40ae0..66ccbb5a 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -25,13 +25,19 @@ import logging
 import os
 import sys
 
-cdist_bin = os.path.abspath(__file__)
-if os.path.islink(cdist_bin):
-    cdist_bin = os.readlink(cdist_bin)
-cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
-sys.path.insert(0, cdist_dir)
+# try to import cdist and if that fails, then add this file's parent dir to
+# module search path and try again. additionally check if this file is
+# symlinked, so user can symlink this file to, for example, ~/.local/bin.
+try:
+    import cdist
+except ModuleNotFoundError:
+    cdist_bin = os.path.abspath(__file__)
+    if os.path.islink(cdist_bin):
+        cdist_bin = os.readlink(cdist_bin)
+    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
+    sys.path.insert(0, cdist_dir)
+    import cdist
 
-import cdist
 import cdist.argparse
 import cdist.banner
 import cdist.config

From 1614b62f702a82731b3b9c636894268b4310821b Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 13:48:28 +0300
Subject: [PATCH 216/527] fallback VERSION to "unknown version"

---
 cdist/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index 26e7d071..350a2765 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -30,7 +30,7 @@ try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
-    VERSION = 'from git'
+    VERSION = 'unknown version'
 
 BANNER = """
              ..          .       .x+=:.        s

From 174aa7728065da611dc8887d9f33ead7e27c473a Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:11:00 +0300
Subject: [PATCH 217/527] __file__ already is absolute

---
 bin/cdist | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/cdist b/bin/cdist
index 66ccbb5a..d7eae312 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -31,7 +31,7 @@ import sys
 try:
     import cdist
 except ModuleNotFoundError:
-    cdist_bin = os.path.abspath(__file__)
+    cdist_bin = __file__
     if os.path.islink(cdist_bin):
         cdist_bin = os.readlink(cdist_bin)
     cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))

From 65c8af4ba3751dddbfe07fd5077e5a99b5d240b8 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:11:12 +0300
Subject: [PATCH 218/527] overengineered version discovery

---
 cdist/__init__.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index 350a2765..f659506f 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -22,6 +22,7 @@
 
 import os
 import hashlib
+import subprocess
 
 import cdist.log
 
@@ -30,7 +31,19 @@ try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
-    VERSION = 'unknown version'
+    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+    if os.path.isdir(os.path.join(cdist_dir, '.git')):
+        run_git = subprocess.run(
+            ['git', 'describe', '--always'],
+            cwd=cdist_dir,
+            capture_output=True,
+            text=True)
+        if run_git.returncode == 0:
+            VERSION = str(run_git.stdout)
+        else:
+            VERSION = 'from git'
+    else:
+        VERSION = 'unknown version'
 
 BANNER = """
              ..          .       .x+=:.        s

From 42d5d6c3e29f6500c3f6321b01c3b268e6886ec6 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:12:39 +0300
Subject: [PATCH 219/527] redundant str()

---
 cdist/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index f659506f..c16562e7 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -39,7 +39,7 @@ except ModuleNotFoundError:
             capture_output=True,
             text=True)
         if run_git.returncode == 0:
-            VERSION = str(run_git.stdout)
+            VERSION = run_git.stdout
         else:
             VERSION = 'from git'
     else:

From b41d80075a616a1c7ac3a361079c748424409995 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:16:04 +0300
Subject: [PATCH 220/527] update paths in setup.py

---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 7b000041..002be19c 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ import subprocess
 
 
 # We have it only if it is a git cloned repo.
-build_helper = os.path.join('bin', 'build-helper')
+build_helper = os.path.join('bin', 'cdist-build-helper')
 # Version file path.
 version_file = os.path.join('cdist', 'version.py')
 # If we have build-helper we could be a git repo.
@@ -56,7 +56,7 @@ setup(
     name="cdist",
     packages=["cdist", "cdist.core", "cdist.exec", "cdist.util", ],
     package_data={'cdist': package_data},
-    scripts=["scripts/cdist", "scripts/cdist-dump", "scripts/cdist-new-type"],
+    scripts=["bin/cdist", "bin/cdist-dump", "bin/cdist-new-type"],
     version=cdist.version.VERSION,
     description="A Usable Configuration Management System",
     author="Nico Schottelius",

From e55db1b427fcc7f750adfcb3646a3eeb446113de Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 15:41:38 +0300
Subject: [PATCH 221/527] use check_output for git describe execution and
 define fallback VERSION earlier

---
 cdist/__init__.py | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index c16562e7..1c60ae0f 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -27,23 +27,21 @@ import subprocess
 import cdist.log
 
 
+VERSION = 'unknown version'
+
 try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
     cdist_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
     if os.path.isdir(os.path.join(cdist_dir, '.git')):
-        run_git = subprocess.run(
-            ['git', 'describe', '--always'],
-            cwd=cdist_dir,
-            capture_output=True,
-            text=True)
-        if run_git.returncode == 0:
-            VERSION = run_git.stdout
-        else:
-            VERSION = 'from git'
-    else:
-        VERSION = 'unknown version'
+        try:
+            VERSION = subprocess.check_output(
+                ['git', 'describe', '--always'],
+                cwd=cdist_dir,
+                universal_newlines=True)
+        except:
+            pass
 
 BANNER = """
              ..          .       .x+=:.        s

From 54d83a62118ed3f0c6328c4b25a3b9ee56adf27a Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 15:50:50 +0300
Subject: [PATCH 222/527] there is no single author anymore, also remove www.

---
 setup.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 002be19c..858c2c17 100644
--- a/setup.py
+++ b/setup.py
@@ -59,9 +59,8 @@ setup(
     scripts=["bin/cdist", "bin/cdist-dump", "bin/cdist-new-type"],
     version=cdist.version.VERSION,
     description="A Usable Configuration Management System",
-    author="Nico Schottelius",
-    author_email="nico-cdist-pypi@schottelius.org",
-    url="https://www.cdi.st/",
+    author="cdist contributors",
+    url="https://cdi.st",
     classifiers=[
         "Development Status :: 6 - Mature",
         "Environment :: Console",

From 507fa6fa93210959b52bb639a9394c8ac52b40e5 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 17 Oct 2020 17:05:09 +0200
Subject: [PATCH 223/527] __download: fix non-existent parameter of __unpack

Probably happened due to renaming .. guess it's correct now.
---
 cdist/conf/type/__download/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index eb3ac971..d8814683 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -69,7 +69,7 @@ EXAMPLES
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
-            --move-existing-destination \
+            --backup-destination \
             --destination /opt/cpma/server
 
 

From d20fb743243f0341820a5b41a5725fb705eb5aae Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sat, 17 Oct 2020 23:16:42 +0300
Subject: [PATCH 224/527] use os.path.realpath instead, because it eliminates
 any symbolic links encountered in the path

---
 bin/cdist | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index d7eae312..1f92f157 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -25,16 +25,16 @@ import logging
 import os
 import sys
 
-# try to import cdist and if that fails, then add this file's parent dir to
-# module search path and try again. additionally check if this file is
-# symlinked, so user can symlink this file to, for example, ~/.local/bin.
+# try to import cdist and if that fails,
+# then add this file's parent dir to
+# module search path and try again.
 try:
     import cdist
 except ModuleNotFoundError:
-    cdist_bin = __file__
-    if os.path.islink(cdist_bin):
-        cdist_bin = os.readlink(cdist_bin)
-    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
+    cdist_dir = os.path.realpath(
+        os.path.join(
+            os.path.dirname(os.path.realpath(__file__)),
+            os.pardir))
     sys.path.insert(0, cdist_dir)
     import cdist
 

From b2e6afb57e6799b934a2eaa387cea337d65d8aac Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 17 Oct 2020 23:01:36 +0200
Subject: [PATCH 225/527] __download: adapt download+unpack example in manpage

---
 cdist/conf/type/__download/man.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index d8814683..54503470 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -70,6 +70,7 @@ EXAMPLES
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
             --backup-destination \
+            --preserve-archive \
             --destination /opt/cpma/server
 
 

From 955b84727611caa5aa05e4521af326c94c649e89 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 18 Oct 2020 15:55:14 +0200
Subject: [PATCH 226/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 6d237449..8ad2c624 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
 	* Type __service: Fix calling __systemd_service (Mark Verboom)
 	* Type __line: Add 'replace' state (Evil Ham)
+	* Type __download: Fix man page (Matthias Stecher)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 6964070282a6c5d09a458017623012fc17e3008d Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 18 Oct 2020 17:13:22 +0300
Subject: [PATCH 227/527] s/build-helper/cdist-build-helper/

---
 .gitlab-ci.yml             | 8 ++++----
 README-maintainers         | 2 +-
 bin/cdist-build-helper     | 2 +-
 docs/src/cdist-install.rst | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e215652c..e48355ea 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,15 +6,15 @@ image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
 unit_tests:
     stage: test
     script:
-        - ./bin/build-helper version
-        - ./bin/build-helper test
+        - ./bin/cdist-build-helper version
+        - ./bin/cdist-build-helper test
 
 pycodestyle:
     stage: test
     script:
-        - ./bin/build-helper pycodestyle
+        - ./bin/cdist-build-helper pycodestyle
 
 shellcheck:
     stage: test
     script:
-        - ./bin/build-helper shellcheck
+        - ./bin/cdist-build-helper shellcheck
diff --git a/README-maintainers b/README-maintainers
index af57f475..5766dd7d 100644
--- a/README-maintainers
+++ b/README-maintainers
@@ -1,4 +1,4 @@
-Maintainers should use ./bin/build-helper script.
+Maintainers should use ./bin/cdist-build-helper script.
 
 Makefile is intended for end users. It can be used for non-maintaining
 targets that can be run from pure source (without git repository).
diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper
index d4d603ed..bdef0dbb 100755
--- a/bin/cdist-build-helper
+++ b/bin/cdist-build-helper
@@ -495,7 +495,7 @@ eof
     ;;
 
     shellcheck-build-helper)
-        ${SHELLCHECKCMD} ./bin/build-helper
+        ${SHELLCHECKCMD} ./bin/cdist-build-helper
     ;;
 
     check-shellcheck)
diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst
index 6f4f14d7..18863145 100644
--- a/docs/src/cdist-install.rst
+++ b/docs/src/cdist-install.rst
@@ -49,7 +49,7 @@ create version.py:
 
 .. code-block:: sh
 
-    ./bin/build-helper version
+    ./bin/cdist-build-helper version
 
 Then you install it with:
 
@@ -70,7 +70,7 @@ Or directly with distutils:
 
     python setup.py install
 
-Note that `bin/build-helper` script is intended for cdist maintainers.
+Note that `bin/cdist-build-helper` script is intended for cdist maintainers.
 
 
 Available versions in git

From e3d906a85fe5e0c9f254d3620e8568437c93d97c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 15 Sep 2020 00:55:26 +0300
Subject: [PATCH 228/527] [__acl] remove deprecated parameters, fix some bugs
 and improve manual

---
 cdist/conf/type/__acl/explorer/checks         | 39 -------------------
 cdist/conf/type/__acl/explorer/getent         |  4 ++
 cdist/conf/type/__acl/gencode-remote          | 36 ++++++++---------
 cdist/conf/type/__acl/man.rst                 | 12 +++---
 .../conf/type/__acl/parameter/deprecated/acl  |  1 -
 .../type/__acl/parameter/deprecated/group     |  1 -
 .../conf/type/__acl/parameter/deprecated/mask |  1 -
 .../type/__acl/parameter/deprecated/other     |  1 -
 .../conf/type/__acl/parameter/deprecated/user |  1 -
 cdist/conf/type/__acl/parameter/optional      |  2 -
 .../type/__acl/parameter/optional_multiple    |  3 --
 11 files changed, 26 insertions(+), 75 deletions(-)
 delete mode 100755 cdist/conf/type/__acl/explorer/checks
 create mode 100755 cdist/conf/type/__acl/explorer/getent
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/acl
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/group
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/mask
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/other
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/user

diff --git a/cdist/conf/type/__acl/explorer/checks b/cdist/conf/type/__acl/explorer/checks
deleted file mode 100755
index 70bb0412..00000000
--- a/cdist/conf/type/__acl/explorer/checks
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/sh -e
-#
-# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-# TODO check if filesystem has ACL turned on etc
-
-if [ -f "$__object/parameter/acl" ]
-then
-    grep -E '^(default:)?(user|group):' "$__object/parameter/acl" \
-    | while read -r acl
-    do
-        param="$( echo "$acl" | awk -F: '{print $(NF-2)}' )"
-        check="$( echo "$acl" | awk -F: '{print $(NF-1)}' )"
-
-        [ "$param" = 'user' ] && db=passwd || db="$param"
-
-        if ! getent "$db" "$check" > /dev/null
-        then
-            echo "missing $param '$check'" >&2
-            exit 1
-        fi
-    done
-fi
diff --git a/cdist/conf/type/__acl/explorer/getent b/cdist/conf/type/__acl/explorer/getent
new file mode 100755
index 00000000..7e6c2c30
--- /dev/null
+++ b/cdist/conf/type/__acl/explorer/getent
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+getent passwd | awk -F: '{print "user:"$1}'
+getent group | awk -F: '{print "group:"$1}'
diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote
index e5404a9d..32318e91 100755
--- a/cdist/conf/type/__acl/gencode-remote
+++ b/cdist/conf/type/__acl/gencode-remote
@@ -22,8 +22,8 @@ file_is="$( cat "$__object/explorer/file_is" )"
 
 if [ "$file_is" = 'missing' ] \
     && [ -z "$__cdist_dry_run" ] \
-    && \( [ ! -f "$__object/parameter/file" ] \
-        || [ ! -f "$__object/parameter/directory" ] \)
+    && [ ! -f "$__object/parameter/file" ] \
+    && [ ! -f "$__object/parameter/directory" ]
 then
     exit 0
 fi
@@ -47,28 +47,26 @@ then
 elif [ -f "$__object/parameter/entry" ]
 then
     acl_should="$( cat "$__object/parameter/entry" )"
-elif [ -f "$__object/parameter/acl" ]
-then
-    acl_should="$( cat "$__object/parameter/acl" )"
-elif
-    [ -f "$__object/parameter/user" ] \
-        || [ -f "$__object/parameter/group" ] \
-        || [ -f "$__object/parameter/mask" ] \
-        || [ -f "$__object/parameter/other" ]
-then
-    acl_should="$( for param in user group mask other
-    do
-        [ ! -f "$__object/parameter/$param" ] && continue
-
-        echo "$param" | grep -Eq 'mask|other' && sep=:: || sep=:
-
-        echo "$param$sep$( cat "$__object/parameter/$param" )"
-    done )"
 else
     echo 'no parameters set' >&2
     exit 1
 fi
 
+# instead of setfacl's non-helpful message "Option -m: Invalid argument near character X"
+# let's check if target has necessary users and groups, since mistyped or missing
+# users/groups in target is most common reason.
+echo "$acl_should" \
+    | grep -Po '(user|group):[^:]+' \
+    | sort -u \
+    | while read -r l
+    do
+        if ! grep "$l" -Fxq "$__object/explorer/getent"
+        then
+            echo "no $l' in target" | sed "s/:/ '/" >&2
+            exit 1
+        fi
+    done
+
 if [ -f "$__object/parameter/default" ]
 then
     acl_should="$( echo "$acl_should" \
diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst
index 28412871..307be72b 100644
--- a/cdist/conf/type/__acl/man.rst
+++ b/cdist/conf/type/__acl/man.rst
@@ -12,11 +12,14 @@ Fully supported and tested on Linux (ext4 filesystem), partial support for FreeB
 
 See ``setfacl`` and ``acl`` manpages for more details.
 
+One of ``--entry`` or ``--source`` must be used.
 
-REQUIRED MULTIPLE PARAMETERS
+
+OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
 entry
    Set ACL entry following ``getfacl`` output syntax.
+   Must be used if ``--source`` is not used.
 
 
 OPTIONAL PARAMETERS
@@ -25,6 +28,7 @@ source
    Read ACL entries from stdin or file.
    Ordering of entries is not important.
    When reading from file, comments and empty lines are ignored.
+   Must be used if ``--entry`` is not used.
 
 file
    Create/change file with ``__file`` using ``user:group:mode`` pattern.
@@ -48,12 +52,6 @@ remove
    ``mask`` and ``other`` entries can't be removed, but only changed.
 
 
-DEPRECATED PARAMETERS
----------------------
-Parameters ``acl``, ``user``, ``group``, ``mask`` and ``other`` are deprecated and they
-will be removed in future versions. Please use ``entry`` parameter instead.
-
-
 EXAMPLES
 --------
 
diff --git a/cdist/conf/type/__acl/parameter/deprecated/acl b/cdist/conf/type/__acl/parameter/deprecated/acl
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/acl
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/group b/cdist/conf/type/__acl/parameter/deprecated/group
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/group
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/mask b/cdist/conf/type/__acl/parameter/deprecated/mask
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/mask
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/other b/cdist/conf/type/__acl/parameter/deprecated/other
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/other
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/user b/cdist/conf/type/__acl/parameter/deprecated/user
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/user
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/optional b/cdist/conf/type/__acl/parameter/optional
index cdcbc0b8..5a0c29a3 100644
--- a/cdist/conf/type/__acl/parameter/optional
+++ b/cdist/conf/type/__acl/parameter/optional
@@ -1,5 +1,3 @@
-mask
-other
 source
 file
 directory
diff --git a/cdist/conf/type/__acl/parameter/optional_multiple b/cdist/conf/type/__acl/parameter/optional_multiple
index c615d507..4c884f03 100644
--- a/cdist/conf/type/__acl/parameter/optional_multiple
+++ b/cdist/conf/type/__acl/parameter/optional_multiple
@@ -1,4 +1 @@
 entry
-acl
-user
-group

From 716cd37281903aafd7d65df539265a6722e809b1 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 21 Sep 2020 11:18:39 +0300
Subject: [PATCH 229/527] [__update_alternatives] rewrite and support --install

---
 .../explorer/alternatives                     |  4 ++
 .../type/__update_alternatives/explorer/link  | 40 +++++++++++++++++++
 .../__update_alternatives/explorer/path_is    | 12 ++++++
 .../explorer/path_should_state                |  8 ++++
 .../type/__update_alternatives/explorer/state |  8 ----
 .../type/__update_alternatives/gencode-remote | 37 ++++++++++++++---
 cdist/conf/type/__update_alternatives/man.rst | 13 ++++--
 .../__update_alternatives/parameter/boolean   |  1 +
 8 files changed, 107 insertions(+), 16 deletions(-)
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/alternatives
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/link
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/path_is
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/path_should_state
 delete mode 100755 cdist/conf/type/__update_alternatives/explorer/state
 create mode 100644 cdist/conf/type/__update_alternatives/parameter/boolean

diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives
new file mode 100755
index 00000000..34aaca56
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/alternatives
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+update-alternatives --display "$__object_id" 2>/dev/null \
+    | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link
new file mode 100755
index 00000000..6519e7c2
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/link
@@ -0,0 +1,40 @@
+#!/bin/sh -e
+
+# fedora's (update-)alternatives --display output doesn't have
+# "link <name> is <path>" line, but debian does. so, let's find
+# out how they store this information.
+#
+# debian and friends:
+#   https://salsa.debian.org/dpkg-team/dpkg/-/blob/master/utils/update-alternatives.c
+#   see calls to altdb_print_line function
+#
+# fedora and friends:
+#   https://github.com/fedora-sysv/chkconfig/blob/master/alternatives.c
+#   see calls to parseLine function
+#
+# conclusion: it is safe to assume that (master) link is on second line
+
+for altdir in \
+    /var/lib/dpkg/alternatives \
+    /var/lib/alternatives
+do
+    if [ ! -f "$altdir/$__object_id" ]
+    then
+        continue
+    fi
+
+    link="$( awk 'NR==2' "$altdir/$__object_id" )"
+
+    if [ -n "$link" ]
+    then
+        break
+    fi
+done
+
+if [ -z "$link" ]
+then
+    echo "unable to get link for $__object_id" >&2
+    exit 1
+fi
+
+echo "$link"
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
new file mode 100755
index 00000000..fc304d5d
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -0,0 +1,12 @@
+#!/bin/sh -e
+
+path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \
+    | awk '/link currently points to/ {print $5}' )"
+
+if [ -z "$path_is" ]
+then
+    echo "unable to get current path for $__object_id" >&2
+    exit 1
+fi
+
+echo "$path_is"
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state
new file mode 100755
index 00000000..59e015c5
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+if [ -f "$( cat "$__object/parameter/path" )" ]
+then
+    echo 'present'
+else
+    echo 'absent'
+fi
diff --git a/cdist/conf/type/__update_alternatives/explorer/state b/cdist/conf/type/__update_alternatives/explorer/state
deleted file mode 100755
index 04a78aaa..00000000
--- a/cdist/conf/type/__update_alternatives/explorer/state
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh -e
-path="$(cat "$__object/parameter/path")"
-name="$__object_id"
-link="$(readlink "/etc/alternatives/$name")"
-if [ "$path" = "$link" ]
-then echo present
-else echo absent
-fi
diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index c0b49814..e393cdef 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Ander Punnar (ander@kvlt.ee)
 #
 # This file is part of cdist.
 #
@@ -16,12 +17,38 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
-if [ "$(cat "$__object/explorer/state")" = 'present' ]
-then exit 0
+path_is="$( cat "$__object/explorer/path_is" )"
+
+path_should="$( cat "$__object/parameter/path" )"
+
+if [ "$path_is" = "$path_should" ]
+then
+    exit 0
+fi
+
+if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ]
+then
+    echo "$path_should does not exist in target" >&2
+    exit 1
 fi
 
-path="$(cat "$__object/parameter/path")"
 name="$__object_id"
-echo "update-alternatives --quiet --set '$name' '$path'"
+
+alternatives="$( cat "$__object/explorer/alternatives" )"
+
+if ! echo "$alternatives" | grep -Fxq "$path_should"
+then
+    if [ ! -f "$__object/parameter/install" ]
+    then
+        echo "$path_should is not in $name alternatives." >&2
+        echo 'Please install missing packages or use --install to add path to alternatives.' >&2
+        exit 1
+    fi
+
+    link="$( cat "$__object/explorer/link" )"
+
+    echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
+fi
+
+echo "update-alternatives --set '$name' '$path_should'"
diff --git a/cdist/conf/type/__update_alternatives/man.rst b/cdist/conf/type/__update_alternatives/man.rst
index 73d82d11..0dc973f2 100644
--- a/cdist/conf/type/__update_alternatives/man.rst
+++ b/cdist/conf/type/__update_alternatives/man.rst
@@ -19,6 +19,12 @@ path
    Use this path for the given alternative
 
 
+BOOLEAN PARAMETERS
+------------------
+install
+   Add (``update-alternatives --install``) missing path to alternatives.
+
+
 EXAMPLES
 --------
 
@@ -36,11 +42,12 @@ SEE ALSO
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Ander Punnar <ander@kvlt.ee>
 
 
 COPYING
 -------
-Copyright \(C) 2013 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
+Copyright \(C) 2013 Nico Schottelius and 2020 Ander Punnar. You can
+redistribute it and/or modify it under the terms of the GNU General Public
+License as published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__update_alternatives/parameter/boolean b/cdist/conf/type/__update_alternatives/parameter/boolean
new file mode 100644
index 00000000..7c32f559
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/parameter/boolean
@@ -0,0 +1 @@
+install

From 687c1d2dd9bed4130a65885d19ce31c87c9d79de Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 19 Oct 2020 06:57:00 +0200
Subject: [PATCH 230/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 8ad2c624..a0f1ead2 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,8 @@ next:
 	* Type __service: Fix calling __systemd_service (Mark Verboom)
 	* Type __line: Add 'replace' state (Evil Ham)
 	* Type __download: Fix man page (Matthias Stecher)
+	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
+	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 367da4b77e79b82fe84c2ce1a7f75cfad92db81e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 28 Oct 2020 18:18:24 +0100
Subject: [PATCH 231/527] [type/__file] Fix --state pre-exists

---
 cdist/conf/type/__file/gencode-remote | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index 35356b13..2675b03a 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -87,11 +87,6 @@ case "$state_should" in
         fi
     ;;
 
-    pre-exists)
-        # pre-exists should never reach gencode-remote…
-        exit 1
-   ;;
-
     absent)
         if [ "$type" = "file" ]; then
             echo "rm -f '$destination'"

From 9277e0ba19c346189658712acf17a772967dfa32 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 29 Oct 2020 09:30:58 +0100
Subject: [PATCH 232/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index a0f1ead2..5977d80e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,7 @@ next:
 	* Type __download: Fix man page (Matthias Stecher)
 	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
+	* Type __file: Fix state pre-exists (Dennis Camera)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 82a9aa79020ff2369b1f4d530a3a6d5ca41915f8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 29 Oct 2020 10:39:42 +0100
Subject: [PATCH 233/527] [type/__apt_norecommends] Use 00InstallRecommends
 file as debian-installer does

debian-installer can be preseeded with `base-installer/install-recommends` to
disable installation of recommended packages already during OS installation.
d-i will then create the file `/etc/apt/apt.conf.d/00InstallRecommends`
(cf. https://salsa.debian.org/installer-team/base-installer/-/blob/master/library.sh).

__apt_norecommends should use the same file to avoid having two config files
effectively doing the same thing.
---
 cdist/conf/type/__apt_norecommends/man.rst  |  9 +++--
 cdist/conf/type/__apt_norecommends/manifest | 41 +++++++++++----------
 2 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/cdist/conf/type/__apt_norecommends/man.rst b/cdist/conf/type/__apt_norecommends/man.rst
index 001fffe4..9297b518 100644
--- a/cdist/conf/type/__apt_norecommends/man.rst
+++ b/cdist/conf/type/__apt_norecommends/man.rst
@@ -32,11 +32,12 @@ EXAMPLES
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
+Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Copyright \(C) 2014 Steven Armstrong. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2014 Steven Armstrong, 2020 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__apt_norecommends/manifest b/cdist/conf/type/__apt_norecommends/manifest
index e737df89..fc187784 100755
--- a/cdist/conf/type/__apt_norecommends/manifest
+++ b/cdist/conf/type/__apt_norecommends/manifest
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,26 +20,28 @@
 #
 
 
-os=$(cat "$__global/explorer/os")
+os=$(cat "${__global:?}/explorer/os")
 
-case "$os" in
-   ubuntu|debian|devuan)
-      # No stinking recommends thank you very much.
-      # If I want something installed I will do so myself.
-      __file /etc/apt/apt.conf.d/99-no-recommends \
-         --owner root --group root --mode 644 \
-         --source - << DONE
-APT::Install-Recommends "0";
-APT::Install-Suggests "0";
-APT::AutoRemove::RecommendsImportant "0";
-APT::AutoRemove::SuggestsImportant "0";
-DONE
-   ;;
-   *)
-      cat >&2 << DONE
+case ${os}
+in
+	(ubuntu|debian|devuan)
+		__file /etc/apt/apt.conf.d/00InstallRecommends --state present \
+			--owner root --group root --mode 0644 --source - <<-'EOF'
+		APT::Install-Recommends "false";
+		APT::Install-Suggests "false";
+		APT::AutoRemove::RecommendsImportant "false";
+		APT::AutoRemove::SuggestsImportant "false";
+		EOF
+
+		# TODO: Remove the following object after some time
+		require=__file/etc/apt/apt.conf.d/00InstallRecommends \
+		__file /etc/apt/apt.conf.d/99-no-recommends --state absent
+		;;
+	(*)
+		cat >&2 <<EOF
 The developer of this type (${__type##*/}) did not think your operating system
 ($os) would have any use for it. If you think otherwise please submit a patch.
-DONE
-      exit 1
-   ;;
+EOF
+		exit 1
+		;;
 esac

From b9ad22595fdb75c17e88e33dc028c33dbd5a7de9 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 18:03:27 +0100
Subject: [PATCH 234/527] [scanner] begin scanner implementation - non invasive

---
 cdist/scan.py | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 cdist/scan.py

diff --git a/cdist/scan.py b/cdist/scan.py
new file mode 100644
index 00000000..fa1bf5de
--- /dev/null
+++ b/cdist/scan.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+from scapy.all import *
+from scapy.data import ETHER_TYPES
+
+
+class Scanner(object):
+    def recv_msg_cpu(self, pkg):
+#        print(pkg.__repr__())
+        if ICMPv6EchoReply in pkg:
+            host = pkg['IPv6'].src
+            print(f"Host {host} is alive")
+
+
+    def scan(self):
+        sniff(iface="wlan0",
+              filter="icmp6",
+              prn=self.recv_msg_cpu)
+
+
+if __name__ == '__main__':
+    s = Scanner()
+    s.scan()

From 87b46a622411362f441b62199972e71d85d7d2d6 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 18:49:20 +0100
Subject: [PATCH 235/527] [scanner] finish prototype

ping @poljakowski - it's your turn now
---
 cdist/scan.py | 138 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 131 insertions(+), 7 deletions(-)

diff --git a/cdist/scan.py b/cdist/scan.py
index fa1bf5de..e2100499 100644
--- a/cdist/scan.py
+++ b/cdist/scan.py
@@ -19,24 +19,148 @@
 #
 #
 
-from scapy.all import *
-from scapy.data import ETHER_TYPES
+#
+# Interface to be implemented:
+# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated
+#   scan: scan / listen for icmp6 replies
+#   trigger: send trigger to multicast
+#   config: configure newly detected hosts
+#   install: install newly detected hosts
+#
+# Scanner logic
+#  - save results to configdir:
+#     basedir = ~/.cdist/scan/<ipv6-address>
+#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time or similar
+#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#
+#
+#
+#
+# cdist scan --list
+#       Show all known hosts including last seen flag
+#
+# Logic for reconfiguration:
+#
+#  - record when configured last time
+#  - introduce a parameter --reconfigure-after that takes time argument
+#  - reconfigure if a) host alive and b) reconfigure-after time passed
+#
 
 
+from multiprocessing import Process
+import os
+
+# FIXME: fail gracefully if non existent - i.e. "scapy required for scanner - please install python3-scapy"
+from scapy.all import *
+
+# Datetime overwrites scapy.all.datetime - needs to be imported AFTER
+import datetime
+
+class Trigger(object):
+    """
+    Trigger an ICMPv6EchoReply from all hosts that are alive
+    """
+
+    def __init__(self, interfaces=None, verbose=False):
+        self.interfaces = interfaces
+        self.verbose = verbose
+
+        # Wait 5 seconds before triggering again - FIXME: add parameter
+        self.sleeptime = 5
+
+    def start(self):
+        self.processes = []
+        for interface in self.interfaces:
+            p = Process(target=self.run_interface, args=(interface,))
+            self.processes.append(p)
+            p.start()
+
+    def join(self):
+        for process in self.processes:
+            process.join()
+
+    def run_interface(self, interface):
+        while True:
+            self.trigger(interface)
+            time.sleep(self.sleeptime)
+
+    def trigger(self, interface):
+        packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
+        send(packet, verbose=self.verbose)
+
 class Scanner(object):
-    def recv_msg_cpu(self, pkg):
-#        print(pkg.__repr__())
+    """
+    Scan for replies of hosts, maintain the up-to-date database
+    """
+
+    def __init__(self, interfaces=None, outdir=None):
+        self.interfaces = interfaces
+
+        if outdir:
+            self.outdir = outdir
+        else:
+            self.outdir = "."
+
+    def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
             host = pkg['IPv6'].src
             print(f"Host {host} is alive")
 
+            dir = os.path.join(self.outdir, host)
+            fname = os.path.join(dir, "last_seen")
+
+            now = datetime.datetime.now()
+
+            os.makedirs(dir, exist_ok=True)
+
+            # FIXME: maybe adjust the format so we can easily parse again
+            with open(fname, "w") as fd:
+                fd.write(f"{now}\n")
+
 
     def scan(self):
-        sniff(iface="wlan0",
+        sniff(iface=self.interfaces,
               filter="icmp6",
-              prn=self.recv_msg_cpu)
+              prn=self.handle_pkg)
 
 
 if __name__ == '__main__':
-    s = Scanner()
+    t = Trigger(interfaces=["wlan0"])
+    t.start()
+
+    # Scanner can listen on many interfaces at the same time
+    s = Scanner(interfaces=["wlan0"])
     s.scan()
+
+    # Join back the trigger processes
+    t.join()
+
+    # Test in my lan shows:
+    # [18:48] bridge:cdist% ls -1d fe80::*
+    # fe80::142d:f0a5:725b:1103
+    # fe80::20d:b9ff:fe49:ac11
+    # fe80::20d:b9ff:fe4c:547d
+    # fe80::219:d2ff:feb2:2e12
+    # fe80::21b:fcff:feee:f446
+    # fe80::21b:fcff:feee:f45c
+    # fe80::21b:fcff:feee:f4b1
+    # fe80::21b:fcff:feee:f4ba
+    # fe80::21b:fcff:feee:f4bc
+    # fe80::21b:fcff:feee:f4c1
+    # fe80::21d:72ff:fe86:46b
+    # fe80::42b0:34ff:fe6f:f6f0
+    # fe80::42b0:34ff:fe6f:f863
+    # fe80::42b0:34ff:fe6f:f9b2
+    # fe80::4a5d:60ff:fea1:e55f
+    # fe80::77a3:5e3f:82cc:f2e5
+    # fe80::9e93:4eff:fe6c:c1f4
+    # fe80::ba69:f4ff:fec5:6041
+    # fe80::ba69:f4ff:fec5:8db7
+    # fe80::bad8:12ff:fe65:313d
+    # fe80::bad8:12ff:fe65:d9b1
+    # fe80::ce2d:e0ff:fed4:2611
+    # fe80::ce32:e5ff:fe79:7ea7
+    # fe80::d66d:6dff:fe33:e00
+    # fe80::e2ff:f7ff:fe00:20e6
+    # fe80::f29f:c2ff:fe7c:275e

From 91d99bf08accb3bde0da420294fc9fb68d3d4068 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 21:22:36 +0100
Subject: [PATCH 236/527] [RFC] scanner documentation

---
 docs/dev/logs/2020-10-29.org | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 docs/dev/logs/2020-10-29.org

diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org
new file mode 100644
index 00000000..718fd68c
--- /dev/null
+++ b/docs/dev/logs/2020-10-29.org
@@ -0,0 +1,34 @@
+* The scanner, 2020-10-29, Hacking Villa Diesbach
+** Motivation
+   - The purpose of cdist is to ensure systems are in a configured state
+   - If systems reboot into a clean (think: netboot) state they are
+     stuck in an unconfigured mode
+   - We can either trigger *from* those machines
+     - this is what cdist trigger is for
+   - Or we can regulary *scan* for machines
+     - This method does not need any modification to standard OS
+** How it works
+   - cdist scan uses the all nodes multicast group ff02::1
+   - It sends a ping packet there in regular intervals
+   - This even works in non-IPv6 networks, as all operating systems
+     are IPv6 capable and usually IPv6 enabled by default
+     - Link local is always accessible!
+   - cdist scan receives an answer from all alive hosts
+     - These results are stored in ~/.cdist/scan/${hostip}
+     - We record the last_seen date  ~/.cdist/scan/${hostip}/last_seen
+   - After a host is detected, cdist *can* try to configure it
+     - It saves the result (+/- logging needs to be defined) in
+       ~/.cdist/scan/${hostip}/{config, install}_result
+     - If logging is saved: maybe in ~/.cdist/scan/${hostip}/{config, install}_log
+     - Final naming TBD
+** Benefits from the scanning approach
+   - We know when a host is alive/dead
+   - We can use standard OS w/o trigger customisation
+     - Only requirement: we can ssh into it
+     - Can make use f.i. of Alpine Linux w/ ssh keys feeding in
+   - We can trigger regular reconfiguration
+     - If alive && last_config_time > 1d -> reconfigure
+   - Data can be exported to f.i. prometheus
+     - Record when configured (successfully)
+     - Record when seen
+   - Enables configurations in stateless environments

From 09dfcfe81e0d9e6520d9bf98598037f86c84641e Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 23:16:08 +0100
Subject: [PATCH 237/527] [scanner] add to beta commands

---
 cdist/argparse.py            | 32 ++++++++++++++++++++-
 cdist/scan/commandline.py    | 55 ++++++++++++++++++++++++++++++++++++
 cdist/{ => scan}/scan.py     | 20 +++++++++----
 docs/dev/logs/2020-10-29.org | 23 +++++++++++++++
 4 files changed, 124 insertions(+), 6 deletions(-)
 create mode 100644 cdist/scan/commandline.py
 rename cdist/{ => scan}/scan.py (90%)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 1d16bb25..ff195e8c 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -8,10 +8,11 @@ import cdist.configuration
 import cdist.log
 import cdist.preos
 import cdist.info
+import cdist.scan.commandline
 
 
 # set of beta sub-commands
-BETA_COMMANDS = set(('install', 'inventory', ))
+BETA_COMMANDS = set(('install', 'inventory', 'scan', ))
 # set of beta arguments for sub-commands
 BETA_ARGS = {
     'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
@@ -470,6 +471,35 @@ def get_parsers():
             'pattern', nargs='?', help='Glob pattern.')
     parser['info'].set_defaults(func=cdist.info.Info.commandline)
 
+    # Scan = config + further
+    parser['scan'] = parser['sub'].add_parser('scan', add_help=False,
+                                                 parents=[parser['config']])
+
+    parser['scan'] = parser['sub'].add_parser(
+            'scan', parents=[parser['loglevel'],
+                             parser['beta'],
+                             parser['colored_output'],
+                             parser['common'],
+                             parser['config_main']])
+
+    parser['scan'].add_argument(
+        '-m', '--mode', help='Which modes should run',
+        action='append', default=[],
+        choices=['scan', 'trigger'])
+    parser['scan'].add_argument(
+        '--config',
+        action='store_true',
+        help='Try to configure detected hosts')
+    parser['scan'].add_argument(
+        '-I', '--interfaces',
+        action='append',  default=[],
+        help='On which interfaces to scan/trigger')
+    parser['scan'].add_argument(
+        '-d', '--delay',
+        action='store',  default=3600,
+        help='How long to wait before reconfiguring after last try')
+    parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
+
     for p in parser:
         parser[p].epilog = EPILOG
 
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
new file mode 100644
index 00000000..0fb718e5
--- /dev/null
+++ b/cdist/scan/commandline.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import logging
+
+log = logging.getLogger("scan")
+
+
+# define this outside of the class to not handle scapy import errors by default
+def commandline(args):
+    log.debug(args)
+
+    try:
+        import cdist.scan.scan as scan
+    except ModuleNotFoundError:
+        print('cdist scan requires scapy to be installed')
+
+    processes = []
+
+    if not args.mode:
+        # By default scan and trigger, but do not call any action
+        args.mode = ['scan', 'trigger' ]
+
+    if 'trigger' in args.mode:
+        t = scan.Trigger(interfaces=args.interfaces)
+        t.start()
+        processes.append(t)
+        log.debug("Trigger started")
+
+    if 'scan' in args.mode:
+        s = scan.Scanner(interfaces=args.interfaces, args=args)
+        s.start()
+        processes.append(s)
+        log.debug("Scanner started")
+
+    for process in processes:
+        process.join()
diff --git a/cdist/scan.py b/cdist/scan/scan.py
similarity index 90%
rename from cdist/scan.py
rename to cdist/scan/scan.py
index e2100499..fcbf1899 100644
--- a/cdist/scan.py
+++ b/cdist/scan/scan.py
@@ -50,13 +50,14 @@
 
 from multiprocessing import Process
 import os
-
-# FIXME: fail gracefully if non existent - i.e. "scapy required for scanner - please install python3-scapy"
+import logging
 from scapy.all import *
 
 # Datetime overwrites scapy.all.datetime - needs to be imported AFTER
 import datetime
 
+log = logging.getLogger("scan")
+
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
@@ -87,6 +88,7 @@ class Trigger(object):
 
     def trigger(self, interface):
         packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
+        log.debug(f"Sending request on {interface}")
         send(packet, verbose=self.verbose)
 
 class Scanner(object):
@@ -94,18 +96,18 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces=None, outdir=None):
+    def __init__(self, interfaces=None, args=None, outdir=None):
         self.interfaces = interfaces
 
         if outdir:
             self.outdir = outdir
         else:
-            self.outdir = "."
+            self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan')
 
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
             host = pkg['IPv6'].src
-            print(f"Host {host} is alive")
+            log.verbose(f"Host {host} is alive")
 
             dir = os.path.join(self.outdir, host)
             fname = os.path.join(dir, "last_seen")
@@ -118,13 +120,21 @@ class Scanner(object):
             with open(fname, "w") as fd:
                 fd.write(f"{now}\n")
 
+    def start(self):
+        self.process = Process(target=self.scan)
+        self.process.start()
+
+    def join(self):
+        self.process.join()
 
     def scan(self):
+        log.debug("Scanning - zzzzz")
         sniff(iface=self.interfaces,
               filter="icmp6",
               prn=self.handle_pkg)
 
 
+
 if __name__ == '__main__':
     t = Trigger(interfaces=["wlan0"])
     t.start()
diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org
index 718fd68c..4461be8c 100644
--- a/docs/dev/logs/2020-10-29.org
+++ b/docs/dev/logs/2020-10-29.org
@@ -32,3 +32,26 @@
      - Record when configured (successfully)
      - Record when seen
    - Enables configurations in stateless environments
+** Sample output v2020-10-29
+23:14] bridge:~% sudo  cdist scan -b -I wlan0 -vv
+VERBOSE: cdist: version 6.8.0-36-g91d99bf0
+VERBOSE: scan: Host fe80::21d:72ff:fe86:46b is alive
+VERBOSE: scan: Host fe80::ce2d:e0ff:fed4:2611 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4c1 is alive
+VERBOSE: scan: Host fe80::e2ff:f7ff:fe00:20e6 is alive
+VERBOSE: scan: Host fe80::20d:b9ff:fe49:ac11 is alive
+VERBOSE: scan: Host fe80::9e93:4eff:fe6c:c1f4 is alive
+VERBOSE: scan: Host fe80::ce32:e5ff:fe79:7ea7 is alive
+VERBOSE: scan: Host fe80::219:d2ff:feb2:2e12 is alive
+VERBOSE: scan: Host fe80::d66d:6dff:fe33:e00 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f446 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4b1 is alive
+VERBOSE: scan: Host fe80::20d:b9ff:fe4c:547d is alive
+VERBOSE: scan: Host fe80::bad8:12ff:fe65:313d is alive
+VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f6f0 is alive
+VERBOSE: scan: Host fe80::ba69:f4ff:fec5:6041 is alive
+VERBOSE: scan: Host fe80::f29f:c2ff:fe7c:275e is alive
+VERBOSE: scan: Host fe80::ba69:f4ff:fec5:8db7 is alive
+VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f863 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4bc is alive
+...

From e30ecdda535200ac75a6e774590d2e1b5e1b5b28 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 13:05:40 +0200
Subject: [PATCH 238/527] Add __uci and __uci_commit types

---
 cdist/conf/type/__uci/explorer/state          | 89 +++++++++++++++++++
 cdist/conf/type/__uci/gencode-remote          | 67 ++++++++++++++
 cdist/conf/type/__uci/man.rst                 | 77 ++++++++++++++++
 cdist/conf/type/__uci/manifest                | 40 +++++++++
 cdist/conf/type/__uci/parameter/default/state |  1 +
 .../type/__uci/parameter/default/transaction  |  1 +
 cdist/conf/type/__uci/parameter/optional      |  2 +
 .../type/__uci/parameter/required_multiple    |  1 +
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++
 cdist/conf/type/__uci_commit/man.rst          | 58 ++++++++++++
 cdist/conf/type/__uci_commit/nonparallel      |  0
 11 files changed, 357 insertions(+)
 create mode 100644 cdist/conf/type/__uci/explorer/state
 create mode 100755 cdist/conf/type/__uci/gencode-remote
 create mode 100644 cdist/conf/type/__uci/man.rst
 create mode 100755 cdist/conf/type/__uci/manifest
 create mode 100644 cdist/conf/type/__uci/parameter/default/state
 create mode 100644 cdist/conf/type/__uci/parameter/default/transaction
 create mode 100644 cdist/conf/type/__uci/parameter/optional
 create mode 100644 cdist/conf/type/__uci/parameter/required_multiple
 create mode 100755 cdist/conf/type/__uci_commit/gencode-remote
 create mode 100644 cdist/conf/type/__uci_commit/man.rst
 create mode 100644 cdist/conf/type/__uci_commit/nonparallel

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
new file mode 100644
index 00000000..2e98b606
--- /dev/null
+++ b/cdist/conf/type/__uci/explorer/state
@@ -0,0 +1,89 @@
+#!/bin/sh
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current state of the configuration option
+# The output of this explorer is one of these values:
+#   present
+#     The configuration option is present and has the value of the
+#     parameter --value.
+#   absent
+#     The configuration option is not defined.
+#   different
+#     The configuration option is present but has a different value than the
+#     parameter --value.
+#   rearranged
+#     The configuration option is present (a list) and has the same values as
+#     the parameter --value, but in a different order.
+
+RS=$(printf '\036')
+
+option=${__object_id:?}
+
+values_is=$(uci -s -N -d "${RS}" get "${option}" 2>/dev/null) || {
+	echo absent
+	exit 0
+}
+
+# strip off trailing newline
+printf '%s' "${values_is}" \
+| awk '
+BEGIN {
+	state = "present"  # assume all is fine
+}
+NR == FNR {
+	# memoize "should" state
+	should[FNR] = $0
+
+	# go to next line (important!)
+	next
+}
+
+# compare "is" state
+$0 == should[FNR] { next }
+
+FNR > length(should) {
+	# there are more "is" records than "should" -> definitely different
+	state = "different"
+	exit
+}
+
+{
+	# see if we can find the value somewhere in should
+	for (i in should) {
+		if ($0 == should[i]) {
+			# ... value found -> rearranged
+			# FIXME: Duplicate values are not properly handled here. Do they matter?
+			state = "rearranged"
+			next
+		}
+	}
+
+	state = "different"
+	exit
+}
+
+END {
+	if (FNR < length(should)) {
+		# "is" was shorter than "should" -> different
+		state = "different"
+	}
+
+	print state
+}
+' "${__object:?}/parameter/value" RS="${RS}" -
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
new file mode 100755
index 00000000..48f114fe
--- /dev/null
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -0,0 +1,67 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
+
+config=${__object_id:?}
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+case ${state_should}
+in
+	(present)
+		if in_list "${state_is}" 'present' 'rearranged'
+		then
+			# NOTE: order is ignored so rearranged is also fine.
+			exit 0
+		fi
+
+		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
+		then
+			# "should" is a list
+			if test "${state_is}" != 'absent'
+			then
+				printf "uci delete '%s'\n" "${config}"
+			fi
+
+			while read -r value
+			do
+				printf "uci add_list '%s'='%s'\n" "${config}" "${value}"
+			done <"${__object:?}/parameter/value"
+		else
+			# "should" is a scalar
+			value=$(cat "${__object:?}/parameter/value")
+			printf "uci set '%s'='%s'\n" "${config}" "${value}"
+		fi
+		;;
+	(absent)
+		if in_list "${state_is}" 'absent'
+		then
+			exit 0
+		fi
+
+		printf "uci delete '%s'\n" "${config}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
new file mode 100644
index 00000000..d23d8b2b
--- /dev/null
+++ b/cdist/conf/type/__uci/man.rst
@@ -0,0 +1,77 @@
+cdist-type__uci(7)
+==================
+
+NAME
+----
+cdist-type__uci - Manage configuration values in OpenWrt's
+Unified Configuration Interface (UCI)
+
+
+DESCRIPTION
+-----------
+This cdist type can be used to alter configuration options in OpenWrt's UCI
+system.
+
+Options can be applied in batches if the `--transaction` parameter is used.
+It is important to ensure that the `__uci_commit` object is executed before a
+new transaction is started.
+
+REQUIRED PARAMETERS
+-------------------
+value
+    The value to be set. Can be used multiple times.
+
+    Due to the way cdist handles arguments, values **must not** contain newline
+    characters.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    `present` or `absent`, defaults to `present`.
+transaction
+    The name of the transaction this option belongs to.
+    If none is given: "default" is used.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Set the system hostname
+    __uci system.@system[0].hostname --value 'OpenWrt'
+
+    # Enable NTP and NTPd (in one transaction)
+    __uci system.ntp.enabled --value 1 --transaction ntp
+    __uci system.ntp.enable_server --value 1 --transaction ntp
+    __uci system.ntp.server --transaction ntp \
+        --value '0.openwrt.pool.ntp.org' \
+        --value '1.openwrt.pool.ntp.org' \
+        --value '2.openwrt.pool.ntp.org' \
+        --value '3.openwrt.pool.ntp.org'
+    export require=__uci_commit/ntp
+
+
+SEE ALSO
+--------
+- https://openwrt.org/docs/guide-user/base-system/uci
+- :strong:`cdist-type__uci_commit`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
new file mode 100755
index 00000000..e5b0fb30
--- /dev/null
+++ b/cdist/conf/type/__uci/manifest
@@ -0,0 +1,40 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+os=$(cat "${__global:?}/explorer/os")
+
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+case ${os}
+in
+	(openwrt)
+		# okay
+		;;
+	(*)
+		printf "Your operating system (%s) is currently not supported by this type (%s)\n" "${os}" "${__type##*/}" >&2
+		printf "Please contribute an implementation for it if you can.\n" >&2
+		exit 1
+		;;
+esac
+
+
+# Make sure the changes are being commited
+require=${__object_name:?} __uci_commit "${transaction_name}"
diff --git a/cdist/conf/type/__uci/parameter/default/state b/cdist/conf/type/__uci/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__uci/parameter/default/transaction b/cdist/conf/type/__uci/parameter/default/transaction
new file mode 100644
index 00000000..4ad96d51
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/default/transaction
@@ -0,0 +1 @@
+default
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
new file mode 100644
index 00000000..ddbbba16
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -0,0 +1,2 @@
+state
+transaction
diff --git a/cdist/conf/type/__uci/parameter/required_multiple b/cdist/conf/type/__uci/parameter/required_multiple
new file mode 100644
index 00000000..6d4e1507
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/required_multiple
@@ -0,0 +1 @@
+value
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
new file mode 100755
index 00000000..bed0eefb
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -0,0 +1,21 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+echo 'uci commit'
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
new file mode 100644
index 00000000..c55d06e5
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/man.rst
@@ -0,0 +1,58 @@
+cdist-type__uci_commit(7)
+=========================
+
+NAME
+----
+cdist-type__uci_commit - Commit a UCI transaction.
+
+
+DESCRIPTION
+-----------
+This type executes the "uci commit" command on the target.
+It is usually not required to use this type. Use the `--transaction` parameter
+of `cdist-type__uci`\ (7) instead.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Commit the default transaction
+    __uci_commit default
+
+    # Commit another transaction
+    __uci_commit my_transaction
+
+
+SEE ALSO
+--------
+:strong:`cdist-type__uci`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_commit/nonparallel b/cdist/conf/type/__uci_commit/nonparallel
new file mode 100644
index 00000000..e69de29b

From 55e7b32449c9a86e34054c321b20a49795c6652f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 14:00:29 +0200
Subject: [PATCH 239/527] [type/__uci] Only generate __uci_commit if changes
 are required

---
 cdist/conf/type/__uci/manifest | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index e5b0fb30..74524513 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,9 +18,12 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; } }
 
 os=$(cat "${__global:?}/explorer/os")
 
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case ${os}
@@ -35,6 +38,25 @@ in
 		;;
 esac
 
+changes_required=false
 
-# Make sure the changes are being commited
-require=${__object_name:?} __uci_commit "${transaction_name}"
+case ${state_should}
+in
+	(present)
+		# NOTE: order is ignored so rearranged is also fine.
+		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
+		;;
+	(absent)
+		in_list "${state_is}" 'absent' || changes_required=true
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
+
+if ${changes_required}
+then
+	# Make sure the changes are being committed
+	require=${__object_name:?} __uci_commit "${transaction_name}"
+fi

From a09120977f231fe3dda1c7c07073fc7e03fc5ea0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 14:07:10 +0200
Subject: [PATCH 240/527] [type/__uci] Allow omission of --value parameter if
 --state absent

---
 cdist/conf/type/__uci/explorer/state                   | 10 +++++++++-
 cdist/conf/type/__uci/man.rst                          |  1 +
 cdist/conf/type/__uci/manifest                         |  5 +++++
 .../parameter/{required_multiple => optional_multiple} |  0
 4 files changed, 15 insertions(+), 1 deletion(-)
 rename cdist/conf/type/__uci/parameter/{required_multiple => optional_multiple} (100%)

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
index 2e98b606..6fb4b173 100644
--- a/cdist/conf/type/__uci/explorer/state
+++ b/cdist/conf/type/__uci/explorer/state
@@ -40,6 +40,14 @@ values_is=$(uci -s -N -d "${RS}" get "${option}" 2>/dev/null) || {
 	exit 0
 }
 
+if test -f "${__object:?}/parameter/value"
+then
+	should_file="${__object:?}/parameter/value"
+else
+	should_file='/dev/null'
+fi
+
+
 # strip off trailing newline
 printf '%s' "${values_is}" \
 | awk '
@@ -86,4 +94,4 @@ END {
 
 	print state
 }
-' "${__object:?}/parameter/value" RS="${RS}" -
+' "${should_file}" RS="${RS}" -
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index d23d8b2b..c6bb81ab 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -20,6 +20,7 @@ REQUIRED PARAMETERS
 -------------------
 value
     The value to be set. Can be used multiple times.
+    This parameter is allowed to be omitted if `--state` is `absent`.
 
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index 74524513..e56462e5 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -43,6 +43,11 @@ changes_required=false
 case ${state_should}
 in
 	(present)
+		test -s "${__object:?}/parameter/value" || {
+			echo 'The parameter --value is required.' >&2
+			exit 1
+		}
+
 		# NOTE: order is ignored so rearranged is also fine.
 		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
 		;;
diff --git a/cdist/conf/type/__uci/parameter/required_multiple b/cdist/conf/type/__uci/parameter/optional_multiple
similarity index 100%
rename from cdist/conf/type/__uci/parameter/required_multiple
rename to cdist/conf/type/__uci/parameter/optional_multiple

From d8f20a6a204142360e7fb7afd6603c69ae613e2d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 12 Jun 2020 19:39:19 +0200
Subject: [PATCH 241/527] [type/__uci] Implement "real" transactions using
 batch files

---
 .../__uci/{gencode-remote => gencode-local}   | 22 ++++++++++++++++---
 cdist/conf/type/__uci/man.rst                 |  3 +--
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++++++++++++++-
 3 files changed, 40 insertions(+), 6 deletions(-)
 rename cdist/conf/type/__uci/{gencode-remote => gencode-local} (79%)

diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-local
similarity index 79%
rename from cdist/conf/type/__uci/gencode-remote
rename to cdist/conf/type/__uci/gencode-local
index 48f114fe..bba5944d 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-local
@@ -20,11 +20,27 @@
 
 in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
 
+uci_cmd() {
+	printf 'printf "%s\n"' "$1"
+	shift
+	printf " '%s'" "$@"
+	printf " >>'%s'\n" "${tmpfile}"
+}
+
 config=${__object_id:?}
 
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+tmpdir="${__global:?}/tmp/__uci"
+# HACK
+mkdir -p "${tmpdir}"
+
+tmpfile="${tmpdir}/${transaction_name}.txt"
+
+
 case ${state_should}
 in
 	(present)
@@ -44,12 +60,12 @@ in
 
 			while read -r value
 			do
-				printf "uci add_list '%s'='%s'\n" "${config}" "${value}"
+				uci_cmd "add_list '%s'='%s'" "${config}" "${value}"
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
 			value=$(cat "${__object:?}/parameter/value")
-			printf "uci set '%s'='%s'\n" "${config}" "${value}"
+			uci_cmd "set '%s'='%s'" "${config}" "${value}"
 		fi
 		;;
 	(absent)
@@ -58,7 +74,7 @@ in
 			exit 0
 		fi
 
-		printf "uci delete '%s'\n" "${config}"
+		uci_cmd "delete '%s'" "${config}"
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index c6bb81ab..2f64355b 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -13,8 +13,7 @@ This cdist type can be used to alter configuration options in OpenWrt's UCI
 system.
 
 Options can be applied in batches if the `--transaction` parameter is used.
-It is important to ensure that the `__uci_commit` object is executed before a
-new transaction is started.
+
 
 REQUIRED PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index bed0eefb..ac74e5e4 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -18,4 +18,23 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-echo 'uci commit'
+transaction_name=${__object_id:?}
+batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
+
+test -s "${batchfile}" || exit 0
+
+cat <<'EOF'
+rollback() {
+	uci changes \
+	| sed -e 's/\..*$//' -e 's/^-//' \
+	| while read -r package
+	  do
+		  uci revert "${package}"
+	  done
+}
+
+EOF
+
+echo "uci batch <<'EOF' && uci commit || rollback"
+cat "${batchfile}"
+echo 'EOF'

From d3574b2d3e807ae436afd90aa1ed9d01b78ca6a3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 16:43:16 +0200
Subject: [PATCH 242/527] [type/__uci] Send messages when options are set to be
 altered

---
 cdist/conf/type/__uci/gencode-local | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index bba5944d..ba27dc5e 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -53,6 +53,8 @@ in
 		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
 		then
 			# "should" is a list
+			printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+
 			if test "${state_is}" != 'absent'
 			then
 				printf "uci delete '%s'\n" "${config}"
@@ -64,6 +66,8 @@ in
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
+			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
 			value=$(cat "${__object:?}/parameter/value")
 			uci_cmd "set '%s'='%s'" "${config}" "${value}"
 		fi
@@ -74,6 +78,7 @@ in
 			exit 0
 		fi
 
+		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
 		uci_cmd "delete '%s'" "${config}"
 		;;
 	(*)

From 3a3be3631010b1bf17d084c775e306f11429f5d9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 12 Jun 2020 19:53:26 +0200
Subject: [PATCH 243/527] [type/__uci_commit] Send message on commit of a
 transaction

---
 cdist/conf/type/__uci_commit/gencode-remote | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index ac74e5e4..0332d7e0 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,6 +23,8 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
+printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
+
 cat <<'EOF'
 rollback() {
 	uci changes \

From e7369a1f9957ac31830a69de8101c434666da5da Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 17 Jun 2020 13:00:24 +0200
Subject: [PATCH 244/527] [type/__uci_commit] Abort if uncommited changes are
 present on the target

---
 cdist/conf/type/__uci_commit/explorer/changes | 22 ++++++++++++++++
 cdist/conf/type/__uci_commit/gencode-remote   | 25 +++++++++++++------
 2 files changed, 39 insertions(+), 8 deletions(-)
 create mode 100644 cdist/conf/type/__uci_commit/explorer/changes

diff --git a/cdist/conf/type/__uci_commit/explorer/changes b/cdist/conf/type/__uci_commit/explorer/changes
new file mode 100644
index 00000000..2690def7
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/explorer/changes
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer outputs the uncommited UCI changes on the target.
+
+uci changes
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index 0332d7e0..dd1d839c 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,20 +23,29 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
+if test -s "${__object:?}/explorer/changes"
+then
+	echo 'Uncommited UCI changes were found on the target:'
+	cat "${__object:?}/explorer/changes"
+	echo
+	echo 'This can be caused by manual changes or due to a previous failed run.'
+	echo 'Please investigate the situation, revert or commit the changes, and try again.'
+	exit 1
+fi >&2
+
 printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
 
-cat <<'EOF'
+cat <<CODE
 rollback() {
-	uci changes \
-	| sed -e 's/\..*$//' -e 's/^-//' \
+	uci changes \\
+	| sed -e 's/\..*\$//' -e 's/^-//' \\
 	| while read -r package
 	  do
-		  uci revert "${package}"
+		  uci revert "\${package}"
 	  done
 }
 
+uci batch <<'EOF' && uci commit || rollback
+$(cat "${batchfile}")
 EOF
-
-echo "uci batch <<'EOF' && uci commit || rollback"
-cat "${batchfile}"
-echo 'EOF'
+CODE

From cc599dab15ed5c8761b3bcdfb113403fa6d4bc77 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 5 Jul 2020 10:25:36 +0200
Subject: [PATCH 245/527] [type/__uci_commit] Move uncommited changes check
 from explorer to code-remote

This is done to prevent false positives/negatives (see NOTE in code)
---
 cdist/conf/type/__uci_commit/explorer/changes | 22 -------------------
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++++++++++-----
 2 files changed, 15 insertions(+), 28 deletions(-)
 delete mode 100644 cdist/conf/type/__uci_commit/explorer/changes

diff --git a/cdist/conf/type/__uci_commit/explorer/changes b/cdist/conf/type/__uci_commit/explorer/changes
deleted file mode 100644
index 2690def7..00000000
--- a/cdist/conf/type/__uci_commit/explorer/changes
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/sh
-#
-# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-# This explorer outputs the uncommited UCI changes on the target.
-
-uci changes
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index dd1d839c..504f7c3b 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,19 +23,28 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
-if test -s "${__object:?}/explorer/changes"
+printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
+
+# NOTE: Uncommited changes are checked in code-remote instead of in an explorer
+#       because in cdist there is no interlocking between explorers and code
+#       execution.
+#       Checking for uncommited changes in an explorer leaves a time slot in
+#       which changes made are not detected by the code.
+#       Furthermore an explorer running concurrently with another transactions
+#       code-remote could lead to a false positive.
+
+cat <<CODE
+changes=\$(uci changes)
+
+if test -n "\${changes}"
 then
 	echo 'Uncommited UCI changes were found on the target:'
-	cat "${__object:?}/explorer/changes"
-	echo
+	printf '%s\n\n' "\${changes}"
 	echo 'This can be caused by manual changes or due to a previous failed run.'
 	echo 'Please investigate the situation, revert or commit the changes, and try again.'
 	exit 1
 fi >&2
 
-printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
-
-cat <<CODE
 rollback() {
 	uci changes \\
 	| sed -e 's/\..*\$//' -e 's/^-//' \\

From 3ef638a61123069953a9e24044b6c94eba0b3b6b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 20 Jul 2020 12:12:15 +0200
Subject: [PATCH 246/527] [type/__uci_commit] Fail when uci(1) reports errors

---
 cdist/conf/type/__uci_commit/gencode-remote | 37 ++++++++++++++++-----
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index 504f7c3b..e5c93966 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -45,16 +45,37 @@ then
 	exit 1
 fi >&2
 
-rollback() {
-	uci changes \\
-	| sed -e 's/\..*\$//' -e 's/^-//' \\
-	| while read -r package
-	  do
-		  uci revert "\${package}"
-	  done
+check_errors() {
+	# reads stdin and forwards non-empty lines to stderr.
+	# returns 0 if stdin is empty, else 1.
+	! grep -e . >&2
 }
 
-uci batch <<'EOF' && uci commit || rollback
+commit() {
+	uci commit
+}
+
+rollback() {
+	echo >&2
+	echo 'An error occurred when trying to commit transaction ${transaction_name}!' >&2
+
+	uci changes \\
+	| sed -e 's/^-//' -e 's/\..*\$//' \\
+	| sort -u \\
+	| while read -r _package
+	  do
+		  uci revert "\${_package}"
+		  echo "\${_package}"  # for logging
+	  done \\
+	| awk '
+	  BEGIN { printf "Reverted changes in: " }
+	  { printf "%s%s", (FNR > 1 ? ", " : ""), \$0 }
+	  END { printf "\\n" }' >&2
+
+	return 1
+}
+
+uci batch <<'EOF' 2>&1 | check_errors && commit || rollback
 $(cat "${batchfile}")
 EOF
 CODE

From 4da39681188493981ec3c55ea918787195655972 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 20:04:04 +0200
Subject: [PATCH 247/527] [type/__uci_section] Add type

---
 cdist/conf/type/__uci_section/explorer/match  |  68 ++++++++++
 .../conf/type/__uci_section/explorer/options  |  28 ++++
 cdist/conf/type/__uci_section/explorer/type   |  25 ++++
 cdist/conf/type/__uci_section/man.rst         |  76 +++++++++++
 cdist/conf/type/__uci_section/manifest        | 122 ++++++++++++++++++
 .../__uci_section/parameter/default/state     |   1 +
 .../parameter/default/transaction             |   1 +
 .../type/__uci_section/parameter/optional     |   4 +
 .../__uci_section/parameter/optional_multiple |   1 +
 9 files changed, 326 insertions(+)
 create mode 100644 cdist/conf/type/__uci_section/explorer/match
 create mode 100644 cdist/conf/type/__uci_section/explorer/options
 create mode 100644 cdist/conf/type/__uci_section/explorer/type
 create mode 100644 cdist/conf/type/__uci_section/man.rst
 create mode 100755 cdist/conf/type/__uci_section/manifest
 create mode 100644 cdist/conf/type/__uci_section/parameter/default/state
 create mode 100644 cdist/conf/type/__uci_section/parameter/default/transaction
 create mode 100644 cdist/conf/type/__uci_section/parameter/optional
 create mode 100644 cdist/conf/type/__uci_section/parameter/optional_multiple

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
new file mode 100644
index 00000000..9cf1ea3e
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -0,0 +1,68 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer the "prefix" of the section matching --match.
+
+squote_values() {
+	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
+	    -e "/='.*'$/"'!{s/=/='\''/;s/$/'\''/}'
+}
+
+RS=$(printf '\036')
+
+if ! test -e "${__object:?}/parameter/match"
+then
+	if echo "${__object_id:?}" | grep -qvE '^[^.]+\.[^.]+$'
+	then
+		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
+		exit 1
+	fi
+
+	# If no --match is given, we take the __object_id as the section identifier.
+	echo "${__object_id:?}"
+	exit 0
+fi
+
+test -s "${__object:?}/parameter/match" \
+&& test -s "${__object:?}/parameter/type" \
+|| {
+	echo 'Parameters --match and --type must be used together.' >&2
+	exit 1
+}
+
+# Find by match
+match=$(cat "${__object:?}/parameter/match")
+sect_type_filter=$(cat "${__object:?}/parameter/type")
+
+package_filter=${sect_type_filter%%.*}
+section_filter=${sect_type_filter##*.}
+regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
+
+matched_sections=$(
+	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
+	| grep -e "${regex}" \
+	| sed -e 's/\.[^.]*=.*$//')
+
+if test "$(echo "${matched_sections}" | wc -l)" -gt 1
+then
+	printf 'Found multiple matching sections:\n%s\n' "${matched_sections}" >&2
+	exit 1
+fi
+
+echo "${matched_sections}"
diff --git a/cdist/conf/type/__uci_section/explorer/options b/cdist/conf/type/__uci_section/explorer/options
new file mode 100644
index 00000000..67b4f83f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/options
@@ -0,0 +1,28 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current options of the configuration section.
+
+RS=$(printf '\036')
+
+section=$("${__type_explorer:?}/match")
+test -n "${section}" || exit 0
+
+uci -s -N -d "${RS}" show "${section}" 2>/dev/null \
+| grep -v -e "^${section}=" || true
diff --git a/cdist/conf/type/__uci_section/explorer/type b/cdist/conf/type/__uci_section/explorer/type
new file mode 100644
index 00000000..1675c2e0
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/type
@@ -0,0 +1,25 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current section type.
+
+section=$("${__type_explorer:?}/match")
+test -n "${section}" || exit 0
+
+uci -s -N get "${section}" 2>/dev/null || true
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
new file mode 100644
index 00000000..3468bfc3
--- /dev/null
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -0,0 +1,76 @@
+cdist-type__uci_section(7)
+==========================
+
+NAME
+----
+cdist-type__uci_section - Manage configuration sections in OpenWrt's
+Unified Configuration Interface (UCI)
+
+
+DESCRIPTION
+-----------
+This cdist type can be used to replace whole configuration sections in OpenWrt's
+UCI system.
+It can be thought of as syntactic sugar for `cdist-type__uci`\ (7), as this type
+will generate the required `__uci` objects to make the section contain exactly
+the options specified via ``--option``.
+
+Since many default UCI sections are unnamed, this type allows to find the
+matching section by one of its options using the ``--match`` parameter.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+match
+    Allows to find a section to "replace" through one of its parameters.
+    The value to this parameter is a ``<option>=<string>`` string.
+option
+    An option that should be present in the section.
+    This parameter can be used multiple times to specify multiple options.
+    The value to this parameter is a ``<option>=<string>`` string.
+
+    Lists can be expressed by repeatedly using the same key.
+state
+    `present` or `absent`, defaults to `present`.
+transaction
+    The name of the transaction this option belongs to.
+    The value will be forwarded to `cdist-type__uci`\ (7).
+type
+    The type of the section in the format: ``<config>.<section-type>``
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # TODO
+    __uci_section ...
+
+
+SEE ALSO
+--------
+:strong:`cdist-type__uci`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
new file mode 100755
index 00000000..e16277ba
--- /dev/null
+++ b/cdist/conf/type/__uci_section/manifest
@@ -0,0 +1,122 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+grep_line() { echo "$2" | grep -qxF "$1"; }
+unquote_lines() {
+	sed -e '/^".*"$/{s/^"//;s/"$//}' \
+	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
+}
+
+append_values() {
+	while read -r _value
+	do
+		set -- "$@" --value "${_value}"
+	done
+	unset _value
+	"$@" </dev/null
+}
+
+section=$(cat "${__object:?}/explorer/match")
+
+state_should=$(cat "${__object:?}/parameter/state")
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+case $state_should
+in
+	(present)
+		test -f "${__object:?}/parameter/type" || {
+			echo 'Parameter --type is required.' >&2
+			exit 1
+		}
+		type_is=$(cat "${__object:?}/explorer/type")
+		type_should=$(cat "${__object:?}/parameter/type")
+
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
+		if test -n "${type_is}"
+		then
+			if test "${type_is}" != "${type_should##*.}"
+			then
+				# Check if section type matches (section exists and --type provided)
+				printf 'Section type "%s" does not match --type "%s".\n' \
+					"${type_is}" "${type_should}" >&2
+				exit 1
+			fi
+			sect_type=${type_is}
+		else
+			sect_type=${type_should##*.}
+		fi
+
+		if test -z "${section}"
+		then
+			# No section exists and --match was used.
+			# So we generate a new section identifier from $__object_id.
+			case ${__object_id:?}
+			in
+				(*.*) section=${__object_id:?} ;;
+				(*) section="${type_should%%.*}.${__object_id:?}" ;;
+			esac
+		fi
+
+		# Make sure the section itself is present
+		__uci "${section}" --state present --transaction "${transaction_name}" \
+			--value "${sect_type}"
+		export require=__uci/"${section}"
+
+		# Delete options not in "should"
+		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		| while read -r optname
+		  do
+			  if ! grep_line "${optname}" "${optnames_should}"
+			  then
+				  __uci "${section}.${optname}" --state absent \
+					  --transaction "${transaction_name}" </dev/null
+			  fi
+		  done
+
+		# Set "should" options
+		echo "${optnames_should}" \
+		| while read -r optname
+		  do
+			  test -n "${optname}" || continue  # ignore empty lines
+
+			  grep "^${optname}=" <"${__object:?}/parameter/option" \
+			  | sed -e 's/^.*=//' \
+			  | unquote_lines \
+			  | append_values \
+				    __uci "${section}.${optname}" --state present \
+					    --transaction "${transaction_name}"
+		  done
+		;;
+	(absent)
+		# if explorer found no section there is nothing to delete
+		test -n "${section}" || exit 0
+
+		__uci "${section}" --state absent --transaction "${transaction_name}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__uci_section/parameter/default/state b/cdist/conf/type/__uci_section/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__uci_section/parameter/default/transaction b/cdist/conf/type/__uci_section/parameter/default/transaction
new file mode 100644
index 00000000..4ad96d51
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/default/transaction
@@ -0,0 +1 @@
+default
diff --git a/cdist/conf/type/__uci_section/parameter/optional b/cdist/conf/type/__uci_section/parameter/optional
new file mode 100644
index 00000000..8b956bae
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/optional
@@ -0,0 +1,4 @@
+match
+state
+transaction
+type
diff --git a/cdist/conf/type/__uci_section/parameter/optional_multiple b/cdist/conf/type/__uci_section/parameter/optional_multiple
new file mode 100644
index 00000000..01925a15
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/optional_multiple
@@ -0,0 +1 @@
+option

From 179815b5e93bd51b6757d92d14470808c5bee746 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 21 Jun 2020 18:22:54 +0200
Subject: [PATCH 248/527] [type/__uci_section] Ignore SC2015 error (notabug)

---
 cdist/conf/type/__uci_section/explorer/match | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index 9cf1ea3e..a3832bff 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -39,8 +39,8 @@ then
 	exit 0
 fi
 
-test -s "${__object:?}/parameter/match" \
-&& test -s "${__object:?}/parameter/type" \
+# shellcheck disable=SC2015
+test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type" \
 || {
 	echo 'Parameters --match and --type must be used together.' >&2
 	exit 1

From d453d964e1e0827dad217e1dd424dbaf82e434fc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 20 Jul 2020 11:21:06 +0200
Subject: [PATCH 249/527] [type/__uci_section] Fix in section matching

---
 cdist/conf/type/__uci_section/explorer/match | 23 +++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index a3832bff..a5619d4e 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -17,7 +17,8 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-# This explorer the "prefix" of the section matching --match.
+# This explorer determines the "prefix" of the --type section matching --match
+# if set, or __object_id otherwise.
 
 squote_values() {
 	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
@@ -28,7 +29,7 @@ RS=$(printf '\036')
 
 if ! test -e "${__object:?}/parameter/match"
 then
-	if echo "${__object_id:?}" | grep -qvE '^[^.]+\.[^.]+$'
+	if ! echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$'
 	then
 		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
 		exit 1
@@ -47,16 +48,28 @@ test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type
 }
 
 # Find by match
-match=$(cat "${__object:?}/parameter/match")
-sect_type_filter=$(cat "${__object:?}/parameter/type")
+# NOTE: Apart from section types all values are printed in single quotes by UCI.
+match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
 
+if ! grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' "${__object:?}/parameter/type"
+then
+	echo 'Section types are a package name and section type separated by a "." (period).' >&2
+	exit 1
+fi
+
+sect_type_filter=$(cat "${__object:?}/parameter/type")
 package_filter=${sect_type_filter%%.*}
-section_filter=${sect_type_filter##*.}
+section_filter=${sect_type_filter#*.}
+# FIXME: Does not work with named sections
 regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
 
 matched_sections=$(
 	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
 	| grep -e "${regex}" \
+	| while read -r _line
+	  do
+		  if test "${_line#*=}" = "${match#*=}"; then echo "${_line}"; fi
+	  done \
 	| sed -e 's/\.[^.]*=.*$//')
 
 if test "$(echo "${matched_sections}" | wc -l)" -gt 1

From f782a5a370a956995f444a677ecffe7f1564764d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 23 Oct 2020 11:45:37 +0200
Subject: [PATCH 250/527] [type/__uci] Refactor to do proper quoting of UCI
 commands

---
 cdist/conf/type/__uci/gencode-local | 40 ++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index ba27dc5e..44bf9c4f 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -18,15 +18,37 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
+in_list() {
+	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
+}
+
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
 
 uci_cmd() {
-	printf 'printf "%s\n"' "$1"
-	shift
-	printf " '%s'" "$@"
-	printf " >>'%s'\n" "${tmpfile}"
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	# Will output a shell command to be executed locally to add the given
+	# command to $tmpfile.
+	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
+	printf ' >>%s\n' "$(quote "${tmpfile}")"
 }
 
+
 config=${__object_id:?}
 
 state_is=$(cat "${__object:?}/explorer/state")
@@ -57,19 +79,19 @@ in
 
 			if test "${state_is}" != 'absent'
 			then
-				printf "uci delete '%s'\n" "${config}"
+				uci_cmd delete "${config}"
 			fi
 
 			while read -r value
 			do
-				uci_cmd "add_list '%s'='%s'" "${config}" "${value}"
+				uci_cmd add_list "${config}"="${value}"
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
 			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
 
 			value=$(cat "${__object:?}/parameter/value")
-			uci_cmd "set '%s'='%s'" "${config}" "${value}"
+			uci_cmd set "${config}"="${value}"
 		fi
 		;;
 	(absent)
@@ -79,7 +101,7 @@ in
 		fi
 
 		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
-		uci_cmd "delete '%s'" "${config}"
+		uci_cmd delete "${config}"
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2

From 3a6b0851459f08ad56f1c6a138b168fedfba993e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 23 Oct 2020 22:03:25 +0200
Subject: [PATCH 251/527] [type/__uci] Check __object_id for syntax errors

---
 cdist/conf/type/__uci/gencode-local | 45 +++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index 44bf9c4f..af91a698 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -48,8 +48,53 @@ uci_cmd() {
 	printf ' >>%s\n' "$(quote "${tmpfile}")"
 }
 
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+uci_validate_tuple() (
+	tok=${1:?}
+	case $tok
+	in
+		(*.*.*)
+			# check option
+			option=${tok##*.}
+			uci_validate_name "${option}" || {
+				printf 'Invalid option: %s\n' "${option}" >&2
+				return 1
+			}
+			tok=${tok%.*}
+			;;
+		(*.*)
+			# no option (section definition)
+			;;
+		(*)
+			printf 'Invalid tuple: %s\n' "$1" >&2
+			return 1
+			;;
+	esac
+
+	case ${tok#*.}
+	in
+		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
+		(*)  section=${tok#*.} ;;
+	esac
+	uci_validate_name "${section}" || {
+		printf 'Invalid section: %s\n' "${1#*.}" >&2
+		return 1
+	}
+
+	config=${tok%%.*}
+	uci_validate_name "${config}" || {
+		printf 'Invalid config: %s\n' "${config}" >&2
+		return 1
+	}
+)
+
 
 config=${__object_id:?}
+uci_validate_tuple "${config}"
 
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")

From c37253b852477a0a22568643bad461d20b58706f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 22 Oct 2020 20:32:48 +0200
Subject: [PATCH 252/527] [type/__uci_section] Check __object_id for syntax
 errors

---
 cdist/conf/type/__uci_section/manifest | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index e16277ba..1a59b970 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -33,6 +33,32 @@ append_values() {
 	"$@" </dev/null
 }
 
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+## Check section name and error if invalid!
+case ${__object_id:?}
+in
+	(*.*)
+		uci_validate_name "${__object_id%%.*}" || {
+			printf 'Invalid package name: %s\n' "${__object_id%%.*}" >&2
+			exit 1
+		}
+		uci_validate_name "${__object_id#*.}" || {
+			printf 'Invalid section name: %s\n' "${__object_id#*.}" >&2
+			exit 1
+		}
+		;;
+	(*)
+		uci_validate_name "${__object_id:?}" || {
+			printf 'Invalid section name: %s\n' "${__object_id:?}" >&2
+			exit 1
+		}
+		;;
+esac
+
 section=$(cat "${__object:?}/explorer/match")
 
 state_should=$(cat "${__object:?}/parameter/state")

From fe26c119b52c3822f009fe9aeb1b52b60139e33a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 10:01:49 +0200
Subject: [PATCH 253/527] [type/__uci*] Update man pages

---
 cdist/conf/type/__uci/man.rst         | 15 ++++----
 cdist/conf/type/__uci_commit/man.rst  | 12 ++++---
 cdist/conf/type/__uci_section/man.rst | 51 ++++++++++++++++++++-------
 3 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 2f64355b..cf184f9d 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -3,23 +3,22 @@ cdist-type__uci(7)
 
 NAME
 ----
-cdist-type__uci - Manage configuration values in OpenWrt's
-Unified Configuration Interface (UCI)
+cdist-type__uci - Manage configuration values in UCI
 
 
 DESCRIPTION
 -----------
-This cdist type can be used to alter configuration options in OpenWrt's UCI
-system.
+This cdist type can be used to alter configuration options in OpenWrt's
+Unified Configuration Interface (UCI) system.
 
-Options can be applied in batches if the `--transaction` parameter is used.
+Options can be applied in batches if the ``--transaction`` parameter is used.
 
 
 REQUIRED PARAMETERS
 -------------------
 value
     The value to be set. Can be used multiple times.
-    This parameter is allowed to be omitted if `--state` is `absent`.
+    This parameter is ignored if ``--state`` is ``absent``.
 
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
@@ -28,10 +27,10 @@ value
 OPTIONAL PARAMETERS
 -------------------
 state
-    `present` or `absent`, defaults to `present`.
+    ``present`` or ``absent``, defaults to ``present``.
 transaction
     The name of the transaction this option belongs to.
-    If none is given: "default" is used.
+    If none is given: ``default`` is used.
 
 
 BOOLEAN PARAMETERS
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
index c55d06e5..ea6611eb 100644
--- a/cdist/conf/type/__uci_commit/man.rst
+++ b/cdist/conf/type/__uci_commit/man.rst
@@ -3,14 +3,16 @@ cdist-type__uci_commit(7)
 
 NAME
 ----
-cdist-type__uci_commit - Commit a UCI transaction.
+cdist-type__uci_commit - Commit UCI transactions
 
 
 DESCRIPTION
 -----------
-This type executes the "uci commit" command on the target.
-It is usually not required to use this type. Use the `--transaction` parameter
-of `cdist-type__uci`\ (7) instead.
+This type executes the ``uci commit`` command on the target with the commands
+queued in a transaction.
+It is usually not required to use this type. Use the ``--transaction`` parameter
+of :strong:`cdist-type__uci`\ (7) and :strong:`cdist-type__uci_section`\ (7)
+instead.
 
 
 REQUIRED PARAMETERS
@@ -42,7 +44,7 @@ EXAMPLES
 
 SEE ALSO
 --------
-:strong:`cdist-type__uci`\ (7)
+:strong:`cdist-type__uci`\ (7), :strong:`cdist-type__uci_section`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 3468bfc3..86729316 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -3,21 +3,23 @@ cdist-type__uci_section(7)
 
 NAME
 ----
-cdist-type__uci_section - Manage configuration sections in OpenWrt's
-Unified Configuration Interface (UCI)
+cdist-type__uci_section - Manage configuration sections in UCI
 
 
 DESCRIPTION
 -----------
 This cdist type can be used to replace whole configuration sections in OpenWrt's
-UCI system.
-It can be thought of as syntactic sugar for `cdist-type__uci`\ (7), as this type
-will generate the required `__uci` objects to make the section contain exactly
-the options specified via ``--option``.
+Unified Configuration Interface (UCI) system.
+It can be thought of as syntactic sugar for :strong:`cdist-type__uci`\ (7),
+as this type will generate the required `__uci` objects to make the section
+contain exactly the options specified using ``--option``.
 
 Since many default UCI sections are unnamed, this type allows to find the
 matching section by one of its options using the ``--match`` parameter.
 
+**NOTE:** Options already present on the target and not listed in ``--option``
+or ``--list`` will be deleted.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -28,18 +30,18 @@ OPTIONAL PARAMETERS
 -------------------
 match
     Allows to find a section to "replace" through one of its parameters.
-    The value to this parameter is a ``<option>=<string>`` string.
+    The value to this parameter is a ``<option>=<value>`` string.
 option
     An option that should be present in the section.
     This parameter can be used multiple times to specify multiple options.
-    The value to this parameter is a ``<option>=<string>`` string.
+    The value to this parameter is a ``<option>=<value>`` string.
 
     Lists can be expressed by repeatedly using the same key.
 state
-    `present` or `absent`, defaults to `present`.
+    ``present`` or ``absent``, defaults to ``present``.
 transaction
     The name of the transaction this option belongs to.
-    The value will be forwarded to `cdist-type__uci`\ (7).
+    The value will be forwarded to :strong:`cdist-type__uci`\ (7).
 type
     The type of the section in the format: ``<config>.<section-type>``
 
@@ -54,13 +56,36 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # TODO
-    __uci_section ...
+    # Configure the dropbear daemon
+    __uci_section dropbear --type dropbear.dropbear --transaction sshd \
+        --match Port=22 --option Port=22 \
+        --option PasswordAuth=off \
+        --option RootPasswordAuth=off
+
+    # Block SSH access from the guest network
+    __uci_section firewall.block_ssh_from_guest --type firewall.rule \
+        --transaction fwrules \
+        --option name='Block-SSH-Access-from-Guest' \
+        --option src='guest' \
+        --option proto='tcp' \
+        --option dest_port='22' \
+        --option target='REJECT'
+
+    # Configure a Wi-Fi access point
+    __uci_section wireless.default_radio0 --type wireless.wifi-iface \
+        --transaction wifi \
+        --option device='radio0' \
+        --option mode='ap' \
+        --option network='wlan' \
+        --option ssid='mywifi' \
+        --option encryption="psk2' \
+        --option key='hunter2'
 
 
 SEE ALSO
 --------
-:strong:`cdist-type__uci`\ (7)
+- https://openwrt.org/docs/guide-user/base-system/uci
+- :strong:`cdist-type__uci`\ (7)
 
 
 AUTHORS

From 0840afce03c50824278d64cd204b86e79077b48e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 12:57:40 +0200
Subject: [PATCH 254/527] [type/__uci] Add --type parameter

---
 cdist/conf/type/__uci/gencode-local      |  54 ++++++++----
 cdist/conf/type/__uci/gencode-remote     | 101 +++++++++++++++++++++++
 cdist/conf/type/__uci/man.rst            |   9 +-
 cdist/conf/type/__uci/parameter/optional |   1 +
 4 files changed, 146 insertions(+), 19 deletions(-)
 create mode 100644 cdist/conf/type/__uci/gencode-remote

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index af91a698..34482a01 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -117,27 +117,45 @@ in
 			exit 0
 		fi
 
-		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
-		then
-			# "should" is a list
-			printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+		# Determine type
+		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
+		case ${type}
+		in
+			(option|list) ;;
+			('')
+				# Guess type by the number of values
+				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
+					 && type=list \
+					 || type=option
+				;;
+			(*)
+				printf 'Invalid --type: %s\n' "${type}" >&2
+				exit 1
+				;;
+		esac
 
-			if test "${state_is}" != 'absent'
-			then
-				uci_cmd delete "${config}"
-			fi
+		case ${type}
+		in
+			(list)
+				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
 
-			while read -r value
-			do
-				uci_cmd add_list "${config}"="${value}"
-			done <"${__object:?}/parameter/value"
-		else
-			# "should" is a scalar
-			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+				if test "${state_is}" != 'absent'
+				then
+					uci_cmd delete "${config}"
+				fi
 
-			value=$(cat "${__object:?}/parameter/value")
-			uci_cmd set "${config}"="${value}"
-		fi
+				while read -r value
+				do
+					uci_cmd add_list "${config}"="${value}"
+				done <"${__object:?}/parameter/value"
+				;;
+			(option)
+				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
+				value=$(cat "${__object:?}/parameter/value")
+				uci_cmd set "${config}"="${value}"
+				;;
+		esac
 		;;
 	(absent)
 		if in_list "${state_is}" 'absent'
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
new file mode 100644
index 00000000..348d15a1
--- /dev/null
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -0,0 +1,101 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+config=${__object_id:?}
+uci_validate_tuple "${config}"
+
+
+case ${state_should}
+in
+	(present)
+		if in_list "${state_is}" 'present' 'rearranged'
+		then
+			# NOTE: order is ignored so rearranged is also fine.
+			exit 0
+		fi
+
+		# Determine type
+		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
+		case ${type}
+		in
+			(option|list) ;;
+			('')
+				# Guess type by the number of values
+				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
+					 && type=list \
+					 || type=option
+				;;
+			(*)
+				printf 'Invalid --type: %s\n' "${type}" >&2
+				exit 1
+				;;
+		esac
+
+		case ${type}
+		in
+			(list)
+				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+
+				if test "${state_is}" != 'absent'
+				then
+					uci_cmd delete "${config}"
+				fi
+
+				while read -r value
+				do
+					uci_cmd add_list "${config}"="${value}"
+				done <"${__object:?}/parameter/value"
+				;;
+			(option)
+				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
+				value=$(cat "${__object:?}/parameter/value")
+				uci_cmd set "${config}"="${value}"
+				;;
+		esac
+		;;
+	(absent)
+		if in_list "${state_is}" 'absent'
+		then
+			exit 0
+		fi
+
+		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
+		uci_cmd delete "${config}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index cf184f9d..9d33d618 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -31,6 +31,10 @@ state
 transaction
     The name of the transaction this option belongs to.
     If none is given: ``default`` is used.
+type
+    If the type should generate an option or a list.
+    One of: ``option`` or ``list``.
+    Defaults to auto-detect based on the number of ``--value`` parameters.
 
 
 BOOLEAN PARAMETERS
@@ -46,10 +50,13 @@ EXAMPLES
     # Set the system hostname
     __uci system.@system[0].hostname --value 'OpenWrt'
 
+    # Set DHCP option 252: tell DHCP clients to not ask for proxy information.
+    __uci dhcp.lan.dhcp_option --type list --value '252,"\n"'
+
     # Enable NTP and NTPd (in one transaction)
     __uci system.ntp.enabled --value 1 --transaction ntp
     __uci system.ntp.enable_server --value 1 --transaction ntp
-    __uci system.ntp.server --transaction ntp \
+    __uci system.ntp.server --type list --transaction ntp \
         --value '0.openwrt.pool.ntp.org' \
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
index ddbbba16..2703b3df 100644
--- a/cdist/conf/type/__uci/parameter/optional
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -1,2 +1,3 @@
 state
 transaction
+type

From 49e867fab4c1bcb6549079e735abec2149e42cf6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 15:59:25 +0200
Subject: [PATCH 255/527] [type/__uci_section] Add more parameter checks

---
 cdist/conf/type/__uci_section/manifest | 42 +++++++++++++++++++-------
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 1a59b970..b5fa4a3b 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -19,11 +19,15 @@
 #
 
 grep_line() { echo "$2" | grep -qxF "$1"; }
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
+}
+validate_options() { grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"; }
 unquote_lines() {
 	sed -e '/^".*"$/{s/^"//;s/"$//}' \
 	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
 }
-
 append_values() {
 	while read -r _value
 	do
@@ -32,12 +36,23 @@ append_values() {
 	unset _value
 	"$@" </dev/null
 }
-
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+print_errors() {
+	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
+		BEGIN {
+			if (getline) {
+				print prefix
+				print
+				rc = 1
+			}
+		}
+		{ print }
+		END {
+			if (rc && suffix) print suffix
+			exit rc
+		}' >&2
 }
 
+
 ## Check section name and error if invalid!
 case ${__object_id:?}
 in
@@ -74,12 +89,6 @@ in
 		type_is=$(cat "${__object:?}/explorer/type")
 		type_should=$(cat "${__object:?}/parameter/type")
 
-		if test -f "${__object:?}/parameter/option"
-		then
-			optnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
-		fi
-
 		if test -n "${type_is}"
 		then
 			if test "${type_is}" != "${type_should##*.}"
@@ -105,6 +114,17 @@ in
 			esac
 		fi
 
+		# Check options for syntax errors
+		validate_options "${__object:?}/parameter/object" \
+		| print_errors 'Found erroneous options in arguments:'
+
+		# Collect option names
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
 		# Make sure the section itself is present
 		__uci "${section}" --state present --transaction "${transaction_name}" \
 			--value "${sect_type}"

From b99ca3cbdf7c761db05598e31e9788e5e4e81cc6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 16:01:02 +0200
Subject: [PATCH 256/527] [type/__uci_section] Split up --option and --list

---
 cdist/conf/type/__uci_section/man.rst         | 19 +++++++-
 cdist/conf/type/__uci_section/manifest        | 48 ++++++++++++-------
 .../__uci_section/parameter/optional_multiple |  1 +
 3 files changed, 50 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 86729316..b8829433 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -28,15 +28,21 @@ None.
 
 OPTIONAL PARAMETERS
 -------------------
+list
+    An option that is part of a list and should be present in the section (as
+    part of a list).  Lists with multiple options can be expressed by using the
+    same ``<option>`` repeatedly.
+
+    The value to this parameter is a ``<option>=<value>`` string.
 match
     Allows to find a section to "replace" through one of its parameters.
+
     The value to this parameter is a ``<option>=<value>`` string.
 option
     An option that should be present in the section.
     This parameter can be used multiple times to specify multiple options.
-    The value to this parameter is a ``<option>=<value>`` string.
 
-    Lists can be expressed by repeatedly using the same key.
+    The value to this parameter is a ``<option>=<value>`` string.
 state
     ``present`` or ``absent``, defaults to ``present``.
 transaction
@@ -62,6 +68,15 @@ EXAMPLES
         --option PasswordAuth=off \
         --option RootPasswordAuth=off
 
+    # Define a firewall zone comprised of lan and wlan networks
+    __uci_section firewall.internal --type firewall.zone --transaction fw \
+        --option name='internal' \
+        --list network='lan' \
+        --list network='wlan' \
+        --option input='ACCEPT' \
+        --option output='ACCEPT' \
+        --option forward='ACCEPT'
+
     # Block SSH access from the guest network
     __uci_section firewall.block_ssh_from_guest --type firewall.rule \
         --transaction fwrules \
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index b5fa4a3b..a7865315 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,7 +18,14 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-grep_line() { echo "$2" | grep -qxF "$1"; }
+grep_line() { { shift; printf '%s\n' "$@"; } | grep -qxF "$1"; }
+prefix_lines() {
+	while test $# -gt 0
+	do
+		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
+		shift; shift
+	done
+}
 uci_validate_name() {
 	# like util.c uci_validate_name()
 	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
@@ -115,14 +122,25 @@ in
 		fi
 
 		# Check options for syntax errors
-		validate_options "${__object:?}/parameter/object" \
+		validate_options "${__object:?}/parameter/list" "${__object:?}/parameter/object" \
 		| print_errors 'Found erroneous options in arguments:'
 
 		# Collect option names
+		if test -f "${__object:?}/parameter/list"
+		then
+			listnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
+		fi
+
 		if test -f "${__object:?}/parameter/option"
 		then
-			optnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+			optnames_should=$(sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort)
+
+			echo "${optnames_should}" \
+			| uniq -d \
+			| print_errors \
+				  'Found duplicate --options:' \
+				  "$(printf '\nUse --list for lists, instead.')"
 		fi
 
 		# Make sure the section itself is present
@@ -130,29 +148,27 @@ in
 			--value "${sect_type}"
 		export require=__uci/"${section}"
 
-		# Delete options not in "should"
+		# Delete options/lists not in "should"
 		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
 		| while read -r optname
 		  do
-			  if ! grep_line "${optname}" "${optnames_should}"
-			  then
-				  __uci "${section}.${optname}" --state absent \
-					  --transaction "${transaction_name}" </dev/null
-			  fi
+			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
+				  __uci "${section}.${optname}" --state absent --transaction "${transaction_name}"
+			  } </dev/null
 		  done
 
 		# Set "should" options
-		echo "${optnames_should}" \
-		| while read -r optname
+		prefix_lines option "${optnames_should}" list "${listnames_should}" \
+		| while read -r _type _optname
 		  do
-			  test -n "${optname}" || continue  # ignore empty lines
+			  test -n "${_type}${_optname}" || continue  # ignore empty lines
 
-			  grep "^${optname}=" <"${__object:?}/parameter/option" \
+			  grep "^${_optname}=" <"${__object:?}/parameter/${_type}" \
 			  | sed -e 's/^.*=//' \
 			  | unquote_lines \
 			  | append_values \
-				    __uci "${section}.${optname}" --state present \
-					    --transaction "${transaction_name}"
+				    __uci "${section}.${_optname}" --state present \
+					    --type "${_type}" --transaction "${transaction_name}"
 		  done
 		;;
 	(absent)
diff --git a/cdist/conf/type/__uci_section/parameter/optional_multiple b/cdist/conf/type/__uci_section/parameter/optional_multiple
index 01925a15..98af35e6 100644
--- a/cdist/conf/type/__uci_section/parameter/optional_multiple
+++ b/cdist/conf/type/__uci_section/parameter/optional_multiple
@@ -1 +1,2 @@
+list
 option

From 8728817af6ae1fa801a39456df3f84d420f87744 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 21:11:50 +0200
Subject: [PATCH 257/527] [type/__uci] Unquote UCI reported values

Without unquoting values printed in single quotes by UCI would always lead to
the state explorer reporting "different".
---
 cdist/conf/type/__uci/explorer/state | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
index 6fb4b173..d7363dbf 100644
--- a/cdist/conf/type/__uci/explorer/state
+++ b/cdist/conf/type/__uci/explorer/state
@@ -51,21 +51,34 @@ fi
 # strip off trailing newline
 printf '%s' "${values_is}" \
 | awk '
+function unquote(s) {
+	# simplified dequoting of single quoted strings
+	if (s ~ /^'\''.*'\''$/) {
+		s = substr(s, 2, length(s) - 2)
+		sub(/'"'\\\\''"'/, "'\''", s)
+	}
+	return s
+}
+
 BEGIN {
 	state = "present"  # assume all is fine
 }
 NR == FNR {
 	# memoize "should" state
 	should[FNR] = $0
+	should_count++
 
 	# go to next line (important!)
 	next
 }
 
 # compare "is" state
+
+{ $0 = unquote($0) }
+
 $0 == should[FNR] { next }
 
-FNR > length(should) {
+FNR > should_count {
 	# there are more "is" records than "should" -> definitely different
 	state = "different"
 	exit
@@ -87,7 +100,7 @@ FNR > length(should) {
 }
 
 END {
-	if (FNR < length(should)) {
+	if (FNR < should_count) {
 		# "is" was shorter than "should" -> different
 		state = "different"
 	}

From 4aebb1f12756593408d6f012adb923e4c9b81e1e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 21:18:22 +0200
Subject: [PATCH 258/527] [type/__uci*] Update man.rst regarding quoting
 requirements

---
 cdist/conf/type/__uci/man.rst         | 2 ++
 cdist/conf/type/__uci_section/man.rst | 8 ++++++++
 2 files changed, 10 insertions(+)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 9d33d618..c7f5f4f0 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -23,6 +23,8 @@ value
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
 
+    Values do not need special quoting for UCI. The only requirement is that the
+    value is passed to the type as a single shell argument.
 
 OPTIONAL PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index b8829433..753d00b3 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -34,6 +34,10 @@ list
     same ``<option>`` repeatedly.
 
     The value to this parameter is a ``<option>=<value>`` string.
+
+    ``<value>`` does not need special quoting for UCI.
+    The only requirement is that the value is passed to the type as a single
+    shell argument.
 match
     Allows to find a section to "replace" through one of its parameters.
 
@@ -43,6 +47,10 @@ option
     This parameter can be used multiple times to specify multiple options.
 
     The value to this parameter is a ``<option>=<value>`` string.
+
+    ``<value>`` does not need special quoting for UCI.
+    The only requirement is that the value is passed to the type as a single
+    shell argument.
 state
     ``present`` or ``absent``, defaults to ``present``.
 transaction

From 63d41a1053bd8b03ddda130d7129d233f97ac671 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 21:10:27 +0100
Subject: [PATCH 259/527] [type/__uci_section] Improve --match support with
 existing named sections

Use section if named section exists without --match option (e.g. empty section).
---
 cdist/conf/type/__uci_section/explorer/match | 80 +++++++++++++-------
 1 file changed, 51 insertions(+), 29 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index a5619d4e..0768e404 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -20,62 +20,84 @@
 # This explorer determines the "prefix" of the --type section matching --match
 # if set, or __object_id otherwise.
 
+RS=$(printf '\036')
+NL=$(printf '\n '); NL=${NL% }
+
 squote_values() {
 	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
 	    -e "/='.*'$/"'!{s/=/='\''/;s/$/'\''/}'
 }
+count_lines() (
+	IFS=${NL?}
+	# shellcheck disable=SC2048,SC2086
+	set -f -- $*; echo $#
+)
 
-RS=$(printf '\036')
-
-if ! test -e "${__object:?}/parameter/match"
-then
-	if ! echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$'
-	then
-		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
-		exit 1
-	fi
+echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' || {
+	echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
+	exit 1
+}
 
+test -s "${__object:?}/parameter/match" || {
 	# If no --match is given, we take the __object_id as the section identifier.
 	echo "${__object_id:?}"
 	exit 0
-fi
-
-# shellcheck disable=SC2015
-test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type" \
-|| {
+}
+test -s "${__object:?}/parameter/type" || {
 	echo 'Parameters --match and --type must be used together.' >&2
 	exit 1
 }
 
-# Find by match
-# NOTE: Apart from section types all values are printed in single quotes by UCI.
-match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
-
-if ! grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' "${__object:?}/parameter/type"
-then
+sect_type_param=$(cat "${__object:?}/parameter/type")
+expr "${sect_type_param}" : '[^.]\{1,\}\.[^.]\{1,\}$' >/dev/null 2>&1 || {
 	echo 'Section types are a package name and section type separated by a "." (period).' >&2
 	exit 1
+}
+package_filter=${sect_type_param%%.*}
+section_filter=${sect_type_param#*.}
+
+# Find by --match
+# NOTE: Apart from section types all values are printed in single quotes by uci show.
+match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
+
+if uci -s -N get "${__object_id:?}" >/dev/null 2>&1
+then
+	# Named section exists: ensure if --match applies to it
+	# if the "matched" option does not exist (e.g. empty section) we use the
+	# section unconditionally.
+	if match_value_is=$(uci -s -N get "${__object_id:?}.${match%%=*}" 2>/dev/null)
+	then
+		match_value_should=$(expr "${match}" : ".*='\\(.*\\)'$")
+
+		test "${match_value_is}" = "${match_value_should}" || {
+			printf 'Named section "%s" does not match --match "%s"\n' \
+				   "${__object_id:?}" "${match}" >&2
+			exit 1
+		}
+	fi
+
+	echo "${__object_id:?}"
+	exit 0
 fi
 
-sect_type_filter=$(cat "${__object:?}/parameter/type")
-package_filter=${sect_type_filter%%.*}
-section_filter=${sect_type_filter#*.}
-# FIXME: Does not work with named sections
-regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
+# No correctly named section exists already: find one to which --match applies
+regex="^${package_filter}\\.@${section_filter}\\[[0-9]\\{1,\\}\\]\\.${match%%=*}="
 
 matched_sections=$(
 	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
 	| grep -e "${regex}" \
 	| while read -r _line
 	  do
-		  if test "${_line#*=}" = "${match#*=}"; then echo "${_line}"; fi
+		  if test "${_line#*=}" = "${match#*=}"
+		  then
+			  echo "${_line}"
+		  fi
 	  done \
 	| sed -e 's/\.[^.]*=.*$//')
 
-if test "$(echo "${matched_sections}" | wc -l)" -gt 1
-then
+test "$(count_lines "${matched_sections}")" -le 1 || {
 	printf 'Found multiple matching sections:\n%s\n' "${matched_sections}" >&2
 	exit 1
-fi
+}
 
 echo "${matched_sections}"

From 7b301195040c4861dbd15c5f2514c4e9eeadbe61 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Oct 2020 18:52:43 +0100
Subject: [PATCH 260/527] [type/__uci] Externalise functions to separate file

---
 cdist/conf/type/__uci/files/functions.sh | 75 +++++++++++++++++++++
 cdist/conf/type/__uci/gencode-local      | 85 ++----------------------
 cdist/conf/type/__uci/manifest           |  4 +-
 3 files changed, 85 insertions(+), 79 deletions(-)
 create mode 100644 cdist/conf/type/__uci/files/functions.sh

diff --git a/cdist/conf/type/__uci/files/functions.sh b/cdist/conf/type/__uci/files/functions.sh
new file mode 100644
index 00000000..4ddb82e0
--- /dev/null
+++ b/cdist/conf/type/__uci/files/functions.sh
@@ -0,0 +1,75 @@
+# -*- mode: sh; indent-tabs-mode: t -*-
+
+in_list() {
+	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
+}
+
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
+
+uci_cmd() {
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	# Will output a shell command to be executed locally to add the given
+	# command to $tmpfile.
+	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
+	printf ' >>%s\n' "$(quote "${tmpfile}")"
+}
+
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+uci_validate_tuple() (
+	tok=${1:?}
+	case $tok
+	in
+		(*.*.*)
+			# check option
+			option=${tok##*.}
+			uci_validate_name "${option}" || {
+				printf 'Invalid option: %s\n' "${option}" >&2
+				return 1
+			}
+			tok=${tok%.*}
+			;;
+		(*.*)
+			# no option (section definition)
+			;;
+		(*)
+			printf 'Invalid tuple: %s\n' "$1" >&2
+			return 1
+			;;
+	esac
+
+	case ${tok#*.}
+	in
+		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
+		(*)  section=${tok#*.} ;;
+	esac
+	uci_validate_name "${section}" || {
+		printf 'Invalid section: %s\n' "${1#*.}" >&2
+		return 1
+	}
+
+	config=${tok%%.*}
+	uci_validate_name "${config}" || {
+		printf 'Invalid config: %s\n' "${config}" >&2
+		return 1
+	}
+)
diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index 34482a01..b7664e85 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -18,84 +18,6 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() {
-	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
-}
-
-quote() {
-	for _arg
-	do
-		shift
-		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
-		then
-			# needs quoting
-			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
-		else
-			set -- "$@" "${_arg}"
-		fi
-	done
-	unset _arg
-
-	# NOTE: Use printf because POSIX echo interprets escape sequences
-	printf '%s' "$*"
-}
-
-uci_cmd() {
-	# Usage: uci_cmd [UCI ARGUMENTS]...
-	# Will output a shell command to be executed locally to add the given
-	# command to $tmpfile.
-	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
-	printf ' >>%s\n' "$(quote "${tmpfile}")"
-}
-
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
-}
-
-uci_validate_tuple() (
-	tok=${1:?}
-	case $tok
-	in
-		(*.*.*)
-			# check option
-			option=${tok##*.}
-			uci_validate_name "${option}" || {
-				printf 'Invalid option: %s\n' "${option}" >&2
-				return 1
-			}
-			tok=${tok%.*}
-			;;
-		(*.*)
-			# no option (section definition)
-			;;
-		(*)
-			printf 'Invalid tuple: %s\n' "$1" >&2
-			return 1
-			;;
-	esac
-
-	case ${tok#*.}
-	in
-		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
-		(*)  section=${tok#*.} ;;
-	esac
-	uci_validate_name "${section}" || {
-		printf 'Invalid section: %s\n' "${1#*.}" >&2
-		return 1
-	}
-
-	config=${tok%%.*}
-	uci_validate_name "${config}" || {
-		printf 'Invalid config: %s\n' "${config}" >&2
-		return 1
-	}
-)
-
-
-config=${__object_id:?}
-uci_validate_tuple "${config}"
-
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
@@ -107,6 +29,13 @@ mkdir -p "${tmpdir}"
 
 tmpfile="${tmpdir}/${transaction_name}.txt"
 
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+
+config=${__object_id:?}
+uci_validate_tuple "${config}"
+
 
 case ${state_should}
 in
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index e56462e5..d0c8239e 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,7 +18,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; } }
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
 
 os=$(cat "${__global:?}/explorer/os")
 

From a6c37095f1eea72acbebebfa88bdf84f65f61f51 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Oct 2020 18:50:34 +0100
Subject: [PATCH 261/527] [type/__uci_section] Externalise functions to
 separate file

---
 .../type/__uci_section/files/functions.sh     | 52 +++++++++++++++++++
 cdist/conf/type/__uci_section/manifest        | 42 +--------------
 2 files changed, 54 insertions(+), 40 deletions(-)
 create mode 100644 cdist/conf/type/__uci_section/files/functions.sh

diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
new file mode 100644
index 00000000..9f510958
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -0,0 +1,52 @@
+# -*- mode: sh; indent-tabs-mode: t -*-
+
+append_values() {
+	while read -r _value
+	do
+		set -- "$@" --value "${_value}"
+	done
+	unset _value
+	"$@" </dev/null
+}
+
+grep_line() {
+	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
+}
+
+prefix_lines() {
+	while test $# -gt 0
+	do
+		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
+		shift; shift
+	done
+}
+
+print_errors() {
+	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
+		BEGIN {
+			if (getline) {
+				print prefix
+				print
+				rc = 1
+			}
+		}
+		{ print }
+		END {
+			if (rc && suffix) print suffix
+			exit rc
+		}' >&2
+}
+
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
+}
+
+unquote_lines() {
+	sed -e '/^".*"$/{s/^"//;s/"$//}' \
+	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
+}
+
+validate_options() {
+	grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"
+}
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index a7865315..208a2c4e 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,46 +18,8 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-grep_line() { { shift; printf '%s\n' "$@"; } | grep -qxF "$1"; }
-prefix_lines() {
-	while test $# -gt 0
-	do
-		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
-		shift; shift
-	done
-}
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
-}
-validate_options() { grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"; }
-unquote_lines() {
-	sed -e '/^".*"$/{s/^"//;s/"$//}' \
-	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
-}
-append_values() {
-	while read -r _value
-	do
-		set -- "$@" --value "${_value}"
-	done
-	unset _value
-	"$@" </dev/null
-}
-print_errors() {
-	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
-		BEGIN {
-			if (getline) {
-				print prefix
-				print
-				rc = 1
-			}
-		}
-		{ print }
-		END {
-			if (rc && suffix) print suffix
-			exit rc
-		}' >&2
-}
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
 
 
 ## Check section name and error if invalid!

From c1ae3ccb2f725d0d94eaec9eeae7c8f5f98a5e45 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:13:06 +0100
Subject: [PATCH 262/527] [type/__uci*] Remove public-facing transaction
 "interface"

---
 cdist/conf/type/__uci/man.rst          | 13 ++++---------
 cdist/conf/type/__uci_section/man.rst  |  9 ++-------
 cdist/conf/type/__uci_section/manifest | 10 ++++------
 3 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index c7f5f4f0..8c3f79f1 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -11,8 +11,6 @@ DESCRIPTION
 This cdist type can be used to alter configuration options in OpenWrt's
 Unified Configuration Interface (UCI) system.
 
-Options can be applied in batches if the ``--transaction`` parameter is used.
-
 
 REQUIRED PARAMETERS
 -------------------
@@ -30,9 +28,6 @@ OPTIONAL PARAMETERS
 -------------------
 state
     ``present`` or ``absent``, defaults to ``present``.
-transaction
-    The name of the transaction this option belongs to.
-    If none is given: ``default`` is used.
 type
     If the type should generate an option or a list.
     One of: ``option`` or ``list``.
@@ -55,10 +50,10 @@ EXAMPLES
     # Set DHCP option 252: tell DHCP clients to not ask for proxy information.
     __uci dhcp.lan.dhcp_option --type list --value '252,"\n"'
 
-    # Enable NTP and NTPd (in one transaction)
-    __uci system.ntp.enabled --value 1 --transaction ntp
-    __uci system.ntp.enable_server --value 1 --transaction ntp
-    __uci system.ntp.server --type list --transaction ntp \
+    # Enable NTP and NTPd (each is applied individually)
+    __uci system.ntp.enabled --value 1
+    __uci system.ntp.enable_server --value 1
+    __uci system.ntp.server --type list \
         --value '0.openwrt.pool.ntp.org' \
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 753d00b3..a0ab78e8 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -53,9 +53,6 @@ option
     shell argument.
 state
     ``present`` or ``absent``, defaults to ``present``.
-transaction
-    The name of the transaction this option belongs to.
-    The value will be forwarded to :strong:`cdist-type__uci`\ (7).
 type
     The type of the section in the format: ``<config>.<section-type>``
 
@@ -71,13 +68,13 @@ EXAMPLES
 .. code-block:: sh
 
     # Configure the dropbear daemon
-    __uci_section dropbear --type dropbear.dropbear --transaction sshd \
+    __uci_section dropbear --type dropbear.dropbear \
         --match Port=22 --option Port=22 \
         --option PasswordAuth=off \
         --option RootPasswordAuth=off
 
     # Define a firewall zone comprised of lan and wlan networks
-    __uci_section firewall.internal --type firewall.zone --transaction fw \
+    __uci_section firewall.internal --type firewall.zone \
         --option name='internal' \
         --list network='lan' \
         --list network='wlan' \
@@ -87,7 +84,6 @@ EXAMPLES
 
     # Block SSH access from the guest network
     __uci_section firewall.block_ssh_from_guest --type firewall.rule \
-        --transaction fwrules \
         --option name='Block-SSH-Access-from-Guest' \
         --option src='guest' \
         --option proto='tcp' \
@@ -96,7 +92,6 @@ EXAMPLES
 
     # Configure a Wi-Fi access point
     __uci_section wireless.default_radio0 --type wireless.wifi-iface \
-        --transaction wifi \
         --option device='radio0' \
         --option mode='ap' \
         --option network='wlan' \
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 208a2c4e..41e30a07 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -46,7 +46,6 @@ esac
 section=$(cat "${__object:?}/explorer/match")
 
 state_should=$(cat "${__object:?}/parameter/state")
-transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case $state_should
 in
@@ -106,8 +105,7 @@ in
 		fi
 
 		# Make sure the section itself is present
-		__uci "${section}" --state present --transaction "${transaction_name}" \
-			--value "${sect_type}"
+		__uci "${section}" --state present --value "${sect_type}"
 		export require=__uci/"${section}"
 
 		# Delete options/lists not in "should"
@@ -115,7 +113,7 @@ in
 		| while read -r optname
 		  do
 			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
-				  __uci "${section}.${optname}" --state absent --transaction "${transaction_name}"
+				  __uci "${section}.${optname}" --state absent
 			  } </dev/null
 		  done
 
@@ -130,14 +128,14 @@ in
 			  | unquote_lines \
 			  | append_values \
 				    __uci "${section}.${_optname}" --state present \
-					    --type "${_type}" --transaction "${transaction_name}"
+					    --type "${_type}"
 		  done
 		;;
 	(absent)
 		# if explorer found no section there is nothing to delete
 		test -n "${section}" || exit 0
 
-		__uci "${section}" --state absent --transaction "${transaction_name}"
+		__uci "${section}" --state absent
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2

From e264fb004f797efc540f74191a0e9630c66dd819 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:27:07 +0100
Subject: [PATCH 263/527] [type/__uci] Convert to immediate remote execution

---
 cdist/conf/type/__uci/files/functions.sh |   6 +-
 cdist/conf/type/__uci/gencode-local      | 102 -----------------------
 cdist/conf/type/__uci/gencode-remote     |   8 --
 3 files changed, 2 insertions(+), 114 deletions(-)
 delete mode 100755 cdist/conf/type/__uci/gencode-local
 mode change 100644 => 100755 cdist/conf/type/__uci/gencode-remote

diff --git a/cdist/conf/type/__uci/files/functions.sh b/cdist/conf/type/__uci/files/functions.sh
index 4ddb82e0..277f648c 100644
--- a/cdist/conf/type/__uci/files/functions.sh
+++ b/cdist/conf/type/__uci/files/functions.sh
@@ -24,10 +24,8 @@ quote() {
 
 uci_cmd() {
 	# Usage: uci_cmd [UCI ARGUMENTS]...
-	# Will output a shell command to be executed locally to add the given
-	# command to $tmpfile.
-	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
-	printf ' >>%s\n' "$(quote "${tmpfile}")"
+	mkdir -p "${__object:?}/files"
+	printf '%s\n' "$(quote "$@")" >>"${__object:?}/files/uci_batch.txt"
 }
 
 uci_validate_name() {
diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
deleted file mode 100755
index b7664e85..00000000
--- a/cdist/conf/type/__uci/gencode-local
+++ /dev/null
@@ -1,102 +0,0 @@
-#!/bin/sh -e
-#
-# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-state_is=$(cat "${__object:?}/explorer/state")
-state_should=$(cat "${__object:?}/parameter/state")
-
-transaction_name=$(cat "${__object:?}/parameter/transaction")
-
-tmpdir="${__global:?}/tmp/__uci"
-# HACK
-mkdir -p "${tmpdir}"
-
-tmpfile="${tmpdir}/${transaction_name}.txt"
-
-# shellcheck source=files/functions.sh
-. "${__type:?}/files/functions.sh"
-
-
-config=${__object_id:?}
-uci_validate_tuple "${config}"
-
-
-case ${state_should}
-in
-	(present)
-		if in_list "${state_is}" 'present' 'rearranged'
-		then
-			# NOTE: order is ignored so rearranged is also fine.
-			exit 0
-		fi
-
-		# Determine type
-		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
-		case ${type}
-		in
-			(option|list) ;;
-			('')
-				# Guess type by the number of values
-				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
-					 && type=list \
-					 || type=option
-				;;
-			(*)
-				printf 'Invalid --type: %s\n' "${type}" >&2
-				exit 1
-				;;
-		esac
-
-		case ${type}
-		in
-			(list)
-				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
-
-				if test "${state_is}" != 'absent'
-				then
-					uci_cmd delete "${config}"
-				fi
-
-				while read -r value
-				do
-					uci_cmd add_list "${config}"="${value}"
-				done <"${__object:?}/parameter/value"
-				;;
-			(option)
-				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
-
-				value=$(cat "${__object:?}/parameter/value")
-				uci_cmd set "${config}"="${value}"
-				;;
-		esac
-		;;
-	(absent)
-		if in_list "${state_is}" 'absent'
-		then
-			exit 0
-		fi
-
-		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
-		uci_cmd delete "${config}"
-		;;
-	(*)
-		printf 'Invalid --state: %s\n' "${state_should}" >&2
-		exit 1
-		;;
-esac
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
old mode 100644
new mode 100755
index 348d15a1..a15f50a6
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -91,11 +91,3 @@ in
 		exit 1
 		;;
 esac
-
-if test -s "${__object:?}/files/uci_batch.txt"
-then
-	cat "${__type:?}/files/uci_apply.sh"
-	printf "uci_apply <<'EOF'\n"
-	cat "${__object:?}/files/uci_batch.txt"
-	printf '\nEOF\n'
-fi

From dfe9e08c280a2a3bc0ded15069b7d60df5a757f8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:06:05 +0100
Subject: [PATCH 264/527] [type/__uci_commit] Delete type

---
 cdist/conf/type/__uci/man.rst               |  2 -
 cdist/conf/type/__uci/manifest              | 18 -----
 cdist/conf/type/__uci_commit/gencode-remote | 81 ---------------------
 cdist/conf/type/__uci_commit/man.rst        | 60 ---------------
 cdist/conf/type/__uci_commit/nonparallel    |  0
 5 files changed, 161 deletions(-)
 delete mode 100755 cdist/conf/type/__uci_commit/gencode-remote
 delete mode 100644 cdist/conf/type/__uci_commit/man.rst
 delete mode 100644 cdist/conf/type/__uci_commit/nonparallel

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 8c3f79f1..81a53473 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -58,13 +58,11 @@ EXAMPLES
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
         --value '3.openwrt.pool.ntp.org'
-    export require=__uci_commit/ntp
 
 
 SEE ALSO
 --------
 - https://openwrt.org/docs/guide-user/base-system/uci
-- :strong:`cdist-type__uci_commit`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index d0c8239e..26920011 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,15 +18,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
-. "${__type:?}/files/functions.sh"
-
-
 os=$(cat "${__global:?}/explorer/os")
 
-state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
-transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case ${os}
 in
@@ -40,8 +34,6 @@ in
 		;;
 esac
 
-changes_required=false
-
 case ${state_should}
 in
 	(present)
@@ -49,21 +41,11 @@ in
 			echo 'The parameter --value is required.' >&2
 			exit 1
 		}
-
-		# NOTE: order is ignored so rearranged is also fine.
-		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
 		;;
 	(absent)
-		in_list "${state_is}" 'absent' || changes_required=true
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
 		exit 1
 		;;
 esac
-
-if ${changes_required}
-then
-	# Make sure the changes are being committed
-	require=${__object_name:?} __uci_commit "${transaction_name}"
-fi
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
deleted file mode 100755
index e5c93966..00000000
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/bin/sh -e
-#
-# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-transaction_name=${__object_id:?}
-batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
-
-test -s "${batchfile}" || exit 0
-
-printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
-
-# NOTE: Uncommited changes are checked in code-remote instead of in an explorer
-#       because in cdist there is no interlocking between explorers and code
-#       execution.
-#       Checking for uncommited changes in an explorer leaves a time slot in
-#       which changes made are not detected by the code.
-#       Furthermore an explorer running concurrently with another transactions
-#       code-remote could lead to a false positive.
-
-cat <<CODE
-changes=\$(uci changes)
-
-if test -n "\${changes}"
-then
-	echo 'Uncommited UCI changes were found on the target:'
-	printf '%s\n\n' "\${changes}"
-	echo 'This can be caused by manual changes or due to a previous failed run.'
-	echo 'Please investigate the situation, revert or commit the changes, and try again.'
-	exit 1
-fi >&2
-
-check_errors() {
-	# reads stdin and forwards non-empty lines to stderr.
-	# returns 0 if stdin is empty, else 1.
-	! grep -e . >&2
-}
-
-commit() {
-	uci commit
-}
-
-rollback() {
-	echo >&2
-	echo 'An error occurred when trying to commit transaction ${transaction_name}!' >&2
-
-	uci changes \\
-	| sed -e 's/^-//' -e 's/\..*\$//' \\
-	| sort -u \\
-	| while read -r _package
-	  do
-		  uci revert "\${_package}"
-		  echo "\${_package}"  # for logging
-	  done \\
-	| awk '
-	  BEGIN { printf "Reverted changes in: " }
-	  { printf "%s%s", (FNR > 1 ? ", " : ""), \$0 }
-	  END { printf "\\n" }' >&2
-
-	return 1
-}
-
-uci batch <<'EOF' 2>&1 | check_errors && commit || rollback
-$(cat "${batchfile}")
-EOF
-CODE
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
deleted file mode 100644
index ea6611eb..00000000
--- a/cdist/conf/type/__uci_commit/man.rst
+++ /dev/null
@@ -1,60 +0,0 @@
-cdist-type__uci_commit(7)
-=========================
-
-NAME
-----
-cdist-type__uci_commit - Commit UCI transactions
-
-
-DESCRIPTION
------------
-This type executes the ``uci commit`` command on the target with the commands
-queued in a transaction.
-It is usually not required to use this type. Use the ``--transaction`` parameter
-of :strong:`cdist-type__uci`\ (7) and :strong:`cdist-type__uci_section`\ (7)
-instead.
-
-
-REQUIRED PARAMETERS
--------------------
-None.
-
-
-OPTIONAL PARAMETERS
--------------------
-None.
-
-
-BOOLEAN PARAMETERS
-------------------
-None.
-
-
-EXAMPLES
---------
-
-.. code-block:: sh
-
-    # Commit the default transaction
-    __uci_commit default
-
-    # Commit another transaction
-    __uci_commit my_transaction
-
-
-SEE ALSO
---------
-:strong:`cdist-type__uci`\ (7), :strong:`cdist-type__uci_section`\ (7)
-
-
-AUTHORS
--------
-Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
-
-
-COPYING
--------
-Copyright \(C) 2020 Dennis Camera. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_commit/nonparallel b/cdist/conf/type/__uci_commit/nonparallel
deleted file mode 100644
index e69de29b..00000000

From ec984f81b5a1db6b7f3e54255b570e07cf5282f9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:36:10 +0100
Subject: [PATCH 265/527] [type/__uci] Delete --transaction parameter

---
 cdist/conf/type/__uci/parameter/default/transaction | 1 -
 cdist/conf/type/__uci/parameter/optional            | 1 -
 2 files changed, 2 deletions(-)
 delete mode 100644 cdist/conf/type/__uci/parameter/default/transaction

diff --git a/cdist/conf/type/__uci/parameter/default/transaction b/cdist/conf/type/__uci/parameter/default/transaction
deleted file mode 100644
index 4ad96d51..00000000
--- a/cdist/conf/type/__uci/parameter/default/transaction
+++ /dev/null
@@ -1 +0,0 @@
-default
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
index 2703b3df..d9080e3a 100644
--- a/cdist/conf/type/__uci/parameter/optional
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -1,3 +1,2 @@
 state
-transaction
 type

From 3e5f18d409e448f69e4f259fc451d4a0e5a7a548 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 18:42:50 +0100
Subject: [PATCH 266/527] [type/__uci] Apply all commands in a single batch

---
 cdist/conf/type/__uci/files/uci_apply.sh | 43 ++++++++++++++++++++++++
 cdist/conf/type/__uci/gencode-remote     |  8 +++++
 cdist/conf/type/__uci/nonparallel        |  0
 3 files changed, 51 insertions(+)
 create mode 100644 cdist/conf/type/__uci/files/uci_apply.sh
 create mode 100644 cdist/conf/type/__uci/nonparallel

diff --git a/cdist/conf/type/__uci/files/uci_apply.sh b/cdist/conf/type/__uci/files/uci_apply.sh
new file mode 100644
index 00000000..63f94290
--- /dev/null
+++ b/cdist/conf/type/__uci/files/uci_apply.sh
@@ -0,0 +1,43 @@
+changes=$(uci changes)
+
+if test -n "${changes}"
+then
+	echo 'Uncommited UCI changes were found on the target:'
+	printf '%s\n\n' "${changes}"
+	echo 'This can be caused by manual changes or due to a previous failed run.'
+	echo 'Please investigate the situation, revert or commit the changes, and try again.'
+	exit 1
+fi >&2
+
+check_errors() {
+	# reads stdin and forwards non-empty lines to stderr.
+	# returns 0 if stdin is empty, else 1.
+	! grep -e . >&2
+}
+
+commit() {
+	uci commit
+}
+
+rollback() {
+	printf '\nAn error occurred when trying to commit UCI transaction!\n' >&2
+
+	uci changes \
+	| sed -e 's/^-//' -e 's/\..*\$//' \
+	| sort -u \
+	| while read -r _package
+	  do
+		  uci revert "${_package}"
+		  echo "${_package}"  # for logging
+	  done \
+	| awk '
+	  BEGIN { printf "Reverted changes in: " }
+	  { printf "%s%s", (FNR > 1 ? ", " : ""), $0 }
+	  END { printf "\n" }' >&2
+
+	return 1
+}
+
+uci_apply() {
+	uci batch 2>&1 | check_errors && commit || rollback
+}
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
index a15f50a6..348d15a1 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -91,3 +91,11 @@ in
 		exit 1
 		;;
 esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci/nonparallel b/cdist/conf/type/__uci/nonparallel
new file mode 100644
index 00000000..e69de29b

From 9d40500570875f29aa48784c6637683aa31afe4e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 20:40:29 +0100
Subject: [PATCH 267/527] [type/__uci_section] Apply all commands in a single
 batch

---
 .../type/__uci_section/files/functions.sh     |  31 +++-
 .../type/__uci_section/files/uci_apply.sh     |   1 +
 cdist/conf/type/__uci_section/gencode-remote  | 146 ++++++++++++++++++
 cdist/conf/type/__uci_section/manifest        |  76 ++-------
 cdist/conf/type/__uci_section/nonparallel     |   0
 5 files changed, 180 insertions(+), 74 deletions(-)
 create mode 120000 cdist/conf/type/__uci_section/files/uci_apply.sh
 create mode 100755 cdist/conf/type/__uci_section/gencode-remote
 create mode 100644 cdist/conf/type/__uci_section/nonparallel

diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
index 9f510958..17c64bbc 100644
--- a/cdist/conf/type/__uci_section/files/functions.sh
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -1,13 +1,6 @@
 # -*- mode: sh; indent-tabs-mode: t -*-
 
-append_values() {
-	while read -r _value
-	do
-		set -- "$@" --value "${_value}"
-	done
-	unset _value
-	"$@" </dev/null
-}
+NL=$(printf '\n '); NL=${NL% }
 
 grep_line() {
 	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
@@ -37,6 +30,28 @@ print_errors() {
 		}' >&2
 }
 
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+	printf '%s' "$*"
+}
+
+uci_cmd() {
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	mkdir -p "${__object:?}/files"
+	printf '%s\n' "$(quote "$@")" >>"${__object:?}/files/uci_batch.txt"
+}
+
 uci_validate_name() {
 	# like util.c uci_validate_name()
 	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
diff --git a/cdist/conf/type/__uci_section/files/uci_apply.sh b/cdist/conf/type/__uci_section/files/uci_apply.sh
new file mode 120000
index 00000000..4209151f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/uci_apply.sh
@@ -0,0 +1 @@
+../../__uci/files/uci_apply.sh
\ No newline at end of file
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
new file mode 100755
index 00000000..05bca0a7
--- /dev/null
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -0,0 +1,146 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+
+section=$(cat "${__object:?}/explorer/match")
+
+state_is=$(test -s "${__object:?}/explorer/type" && echo present || echo absent)
+state_should=$(cat "${__object:?}/parameter/state")
+
+case $state_should
+in
+	(present)
+		test -f "${__object:?}/parameter/type" || {
+			echo 'Parameter --type is required.' >&2
+			exit 1
+		}
+
+		type_is=$(cat "${__object:?}/explorer/type")
+		type_should=$(cat "${__object:?}/parameter/type")
+
+		if test -n "${type_is}"
+		then
+			sect_type=${type_is}
+		else
+			sect_type=${type_should##*.}
+		fi
+
+		if test -z "${section}"
+		then
+			# No section exists and --match was used.
+			# So we generate a new section identifier from $__object_id.
+			case ${__object_id:?}
+			in
+				(*.*) section=${__object_id:?} ;;
+				(*) section="${type_should%%.*}.${__object_id:?}" ;;
+			esac
+		fi
+
+		# Collect option names
+		if test -f "${__object:?}/parameter/list"
+		then
+			listnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
+		fi
+
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
+		# Make sure the section itself is present
+		if test "${state_is}" = absent
+		then
+			printf 'set %s\n' "${section}" >>"${__messages_out:?}"
+			# shellcheck disable=SC2140
+			uci_cmd set "${section}"="${sect_type}"
+		fi
+
+		# Delete options/lists not in "should"
+		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		| while read -r _optname
+		  do
+			  grep_line "${_optname}" "${listnames_should}" "${optnames_should}" || {
+				  printf 'delete %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+				  uci_cmd delete "${section}.${_optname}"
+			  } </dev/null
+		  done
+
+		# Set "should" options
+		prefix_lines option "${optnames_should}" list "${listnames_should}" \
+		| while read -r _type _optname
+		  do
+			  test -n "${_type}${_optname}" || continue  # ignore empty lines
+
+			  case $_type
+			  in
+				  (list)
+					  printf 'set_list %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  if grep -q -e "^${_optname}=" "${__object:?}/explorer/options"
+					  then
+						  uci_cmd delete "${section}.${_optname}"
+					  fi
+
+					  grep "^${_optname}=" "${__object:?}/parameter/list" \
+					  | sed -e 's/^.*=//' \
+					  | unquote_lines \
+					  | while read -r _value
+					    do
+						    # shellcheck disable=SC2140
+						    uci_cmd add_list "${section}.${_optname}"="${_value}"
+					    done
+					  ;;
+				  (option)
+					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  # shellcheck disable=SC2140
+					  uci_cmd set "${section}.${_optname}"="$(
+						  grep "^${_optname}=" "${__object:?}/parameter/option" \
+						  | sed -e 's/^.*=//' \
+						  | unquote_lines \
+						  | head -n 1)"
+					  ;;
+			  esac
+		  done
+		;;
+	(absent)
+		if test "${state_is}" = absent
+		then
+			# if explorer found no section there is nothing to delete
+			exit 0
+		fi
+
+		printf 'delete %s\n' "${section}" >>"${__messages_out:?}"
+		uci_cmd delete "${section}"
+		;;
+esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 41e30a07..7e23e443 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -43,8 +43,6 @@ in
 		;;
 esac
 
-section=$(cat "${__object:?}/explorer/match")
-
 state_should=$(cat "${__object:?}/parameter/state")
 
 case $state_should
@@ -57,85 +55,31 @@ in
 		type_is=$(cat "${__object:?}/explorer/type")
 		type_should=$(cat "${__object:?}/parameter/type")
 
-		if test -n "${type_is}"
+		if test -n "${type_is}" && test "${type_is}" != "${type_should##*.}"
 		then
-			if test "${type_is}" != "${type_should##*.}"
-			then
-				# Check if section type matches (section exists and --type provided)
-				printf 'Section type "%s" does not match --type "%s".\n' \
-					"${type_is}" "${type_should}" >&2
-				exit 1
-			fi
-			sect_type=${type_is}
-		else
-			sect_type=${type_should##*.}
-		fi
-
-		if test -z "${section}"
-		then
-			# No section exists and --match was used.
-			# So we generate a new section identifier from $__object_id.
-			case ${__object_id:?}
-			in
-				(*.*) section=${__object_id:?} ;;
-				(*) section="${type_should%%.*}.${__object_id:?}" ;;
-			esac
+			# Check if section type matches (section exists and --type provided)
+			printf 'Section type "%s" does not match --type "%s".\n' \
+				"${type_is}" "${type_should}" >&2
+			exit 1
 		fi
 
 		# Check options for syntax errors
 		validate_options "${__object:?}/parameter/list" "${__object:?}/parameter/object" \
 		| print_errors 'Found erroneous options in arguments:'
 
-		# Collect option names
-		if test -f "${__object:?}/parameter/list"
+		# Check for duplicate option names
+		if test -s "${__object:?}/parameter/option"
 		then
-			listnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
-		fi
-
-		if test -f "${__object:?}/parameter/option"
-		then
-			optnames_should=$(sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort)
-
-			echo "${optnames_should}" \
+			sed -e 's/=.*$//' "${__object:?}/parameter/option" \
+			| sort \
 			| uniq -d \
 			| print_errors \
 				  'Found duplicate --options:' \
 				  "$(printf '\nUse --list for lists, instead.')"
 		fi
-
-		# Make sure the section itself is present
-		__uci "${section}" --state present --value "${sect_type}"
-		export require=__uci/"${section}"
-
-		# Delete options/lists not in "should"
-		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
-		| while read -r optname
-		  do
-			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
-				  __uci "${section}.${optname}" --state absent
-			  } </dev/null
-		  done
-
-		# Set "should" options
-		prefix_lines option "${optnames_should}" list "${listnames_should}" \
-		| while read -r _type _optname
-		  do
-			  test -n "${_type}${_optname}" || continue  # ignore empty lines
-
-			  grep "^${_optname}=" <"${__object:?}/parameter/${_type}" \
-			  | sed -e 's/^.*=//' \
-			  | unquote_lines \
-			  | append_values \
-				    __uci "${section}.${_optname}" --state present \
-					    --type "${_type}"
-		  done
 		;;
 	(absent)
-		# if explorer found no section there is nothing to delete
-		test -n "${section}" || exit 0
-
-		__uci "${section}" --state absent
+		:
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
diff --git a/cdist/conf/type/__uci_section/nonparallel b/cdist/conf/type/__uci_section/nonparallel
new file mode 100644
index 00000000..e69de29b

From ade69729ddd64938ef69c08a522a85ec825628e9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 31 Oct 2020 18:18:15 +0100
Subject: [PATCH 268/527] [type/__uci_section] Only generate UCI commands if
 state differs

---
 .../conf/type/__uci_section/explorer/options  | 22 ++++-
 .../type/__uci_section/files/functions.sh     |  8 --
 .../type/__uci_section/files/option_state.awk | 91 +++++++++++++++++++
 cdist/conf/type/__uci_section/gencode-remote  | 68 ++++++++++----
 4 files changed, 160 insertions(+), 29 deletions(-)
 create mode 100644 cdist/conf/type/__uci_section/files/option_state.awk

diff --git a/cdist/conf/type/__uci_section/explorer/options b/cdist/conf/type/__uci_section/explorer/options
index 67b4f83f..e1e60668 100644
--- a/cdist/conf/type/__uci_section/explorer/options
+++ b/cdist/conf/type/__uci_section/explorer/options
@@ -25,4 +25,24 @@ section=$("${__type_explorer:?}/match")
 test -n "${section}" || exit 0
 
 uci -s -N -d "${RS}" show "${section}" 2>/dev/null \
-| grep -v -e "^${section}=" || true
+| awk -v VSEP="${RS}" '
+  {
+	  # Strip off the config and section parts
+	  is_opt = sub(/^([^.]*\.){2}/, "")
+
+	  if (!is_opt) {
+		  # this line represents the section -> skip
+		  next
+	  }
+
+	  if (index($0, VSEP)) {
+		  # Put values each on a line, like --option and --list parameters
+		  opt = substr($0, 1, index($0, "=") - 1)
+		  split(substr($0, length(opt) + 2), values, VSEP)
+		  for (i in values) {
+			  printf "%s=%s\n", opt, values[i]
+		  }
+	  } else {
+		  print
+	  }
+  }'
diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
index 17c64bbc..60cb9148 100644
--- a/cdist/conf/type/__uci_section/files/functions.sh
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -6,14 +6,6 @@ grep_line() {
 	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
 }
 
-prefix_lines() {
-	while test $# -gt 0
-	do
-		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
-		shift; shift
-	done
-}
-
 print_errors() {
 	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
 		BEGIN {
diff --git a/cdist/conf/type/__uci_section/files/option_state.awk b/cdist/conf/type/__uci_section/files/option_state.awk
new file mode 100644
index 00000000..97cd94fb
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/option_state.awk
@@ -0,0 +1,91 @@
+# -*- mode: awk; indent-tabs-mode:t -*-
+# Usage: awk -f option_state.awk option_type option_name
+# e.g. awk -f option_state.awk option title
+#      awk -f option_state.awk list entry
+
+function unquote(s) {
+	# simplified dequoting of single quoted strings
+	if (s ~ /^'.*'$/) {
+		s = substr(s, 2, length(s) - 2)
+		sub(/'\\''/, "'", s)
+	}
+	return s
+}
+
+function valueof(line) {
+	if (line !~ /^[[:alpha:]_]+=/) return 0
+	return unquote(substr(line, index(line, "=") + 1))
+}
+
+BEGIN {
+	__object = ENVIRON["__object"]
+	if (!__object) exit 1
+
+	opttype = ARGV[1]
+	optname = ARGV[2]
+
+	if (opttype !~ /^(option|list)/ || !optname) {
+		print "invalid"
+		exit (e=1)
+	}
+
+	ARGV[1] = __object "/parameter/" opttype
+	ARGV[2] = __object "/explorer/options"
+
+	state = "present"
+}
+
+NR == FNR {
+	# memoize "should" state
+	if (index($0, optname "=") == 1) {
+		should[++should_count] = valueof($0)
+	}
+
+	# go to next line (important!)
+	next
+}
+
+{
+	# compare "is" state
+	if (index($0, optname "=") != 1)
+		next
+	++is_count
+
+	v = valueof($0)
+
+	if (v == should[is_count]) {
+		# looks good, but can't say definitely just from this line
+	} else if (is_count > should_count) {
+		# there are more "is" records than "should" -> definitely different
+		state = "different"
+		exit
+	} else {
+		# see if we can find the "is" value somewhere in "should"
+		for (i in should) {
+			if (v == should[i]) {
+				# value found -> could be rearranged
+				# FIXME: Duplicate values are not properly handled here. Do they matter?
+				state = "rearranged"
+				next
+			}
+		}
+
+		# "is" value could not be found in "should" -> definitely different
+		state = "different"
+		exit
+	}
+}
+
+END {
+	if (e) exit
+
+	if (!is_count) {
+		# no "is" values -> absent
+		state = "absent"
+	} else if (is_count < should_count) {
+		# "is" was shorter than "should" -> different
+		state = "different"
+	}
+
+	print state
+}
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
index 05bca0a7..285659b0 100755
--- a/cdist/conf/type/__uci_section/gencode-remote
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -70,7 +70,8 @@ in
 		fi
 
 		# Make sure the section itself is present
-		if test "${state_is}" = absent
+		if test "${state_is}" = absent \
+			|| test "${type_is}" != "${type_should#*.}"
 		then
 			printf 'set %s\n' "${section}" >>"${__messages_out:?}"
 			# shellcheck disable=SC2140
@@ -78,7 +79,7 @@ in
 		fi
 
 		# Delete options/lists not in "should"
-		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		sed -e 's/=.*$//' "${__object:?}/explorer/options" \
 		| while read -r _optname
 		  do
 			  grep_line "${_optname}" "${listnames_should}" "${optnames_should}" || {
@@ -87,18 +88,55 @@ in
 			  } </dev/null
 		  done
 
-		# Set "should" options
-		prefix_lines option "${optnames_should}" list "${listnames_should}" \
-		| while read -r _type _optname
-		  do
-			  test -n "${_type}${_optname}" || continue  # ignore empty lines
+		opt_proc_error() {
+			printf 'An error occurred during processing of option %s\n' "${1:?}" >&2
+			exit 1
+		}
 
-			  case $_type
+		# Set "should" options
+		echo "${optnames_should}" \
+		| grep -e . \
+		| while read -r _optname
+		  do
+			  _opt_state=$(awk -f "${__type:?}/files/option_state.awk" option "${_optname}") \
+				  || opt_proc_error "${_optname}"
+			  case ${_opt_state}
 			  in
-				  (list)
+				  (invalid)
+					  opt_proc_error "${_optname}"
+					  ;;
+				  (present)
+					  ;;
+				  (*)
+					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  # shellcheck disable=SC2140
+					  uci_cmd set "${section}.${_optname}"="$(
+						  grep -e "^${_optname}=" "${__object:?}/parameter/option" \
+						  | sed -e 's/^.*=//' \
+						  | unquote_lines \
+						  | head -n 1)"
+					  ;;
+			  esac
+		  done
+
+		echo "${listnames_should}" \
+		| grep -e . \
+		| while read -r _optname
+		  do
+			  _list_state=$(awk -f "${__type:?}/files/option_state.awk" list "${_optname}") \
+				  || opt_proc_error "${_optname}"
+			  case ${_list_state}
+			  in
+				  (invalid)
+					  opt_proc_error "${_optname}"
+					  ;;
+				  (present)
+					  ;;
+				  (*)
 					  printf 'set_list %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
 
-					  if grep -q -e "^${_optname}=" "${__object:?}/explorer/options"
+					  if test "${_list_state}" != absent
 					  then
 						  uci_cmd delete "${section}.${_optname}"
 					  fi
@@ -112,16 +150,6 @@ in
 						    uci_cmd add_list "${section}.${_optname}"="${_value}"
 					    done
 					  ;;
-				  (option)
-					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
-
-					  # shellcheck disable=SC2140
-					  uci_cmd set "${section}.${_optname}"="$(
-						  grep "^${_optname}=" "${__object:?}/parameter/option" \
-						  | sed -e 's/^.*=//' \
-						  | unquote_lines \
-						  | head -n 1)"
-					  ;;
 			  esac
 		  done
 		;;

From 2be8c634584561158960fc5cf2f363f3f5c4cc4c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 3 Nov 2020 06:43:57 +0100
Subject: [PATCH 269/527] pycodestyle fixes

---
 cdist/argparse.py         |  2 +-
 cdist/scan/commandline.py |  2 +-
 cdist/scan/scan.py        | 12 ++++++++----
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index ff195e8c..88759d7b 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -473,7 +473,7 @@ def get_parsers():
 
     # Scan = config + further
     parser['scan'] = parser['sub'].add_parser('scan', add_help=False,
-                                                 parents=[parser['config']])
+                                              parents=[parser['config']])
 
     parser['scan'] = parser['sub'].add_parser(
             'scan', parents=[parser['loglevel'],
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 0fb718e5..eca4cf13 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -37,7 +37,7 @@ def commandline(args):
 
     if not args.mode:
         # By default scan and trigger, but do not call any action
-        args.mode = ['scan', 'trigger' ]
+        args.mode = ['scan', 'trigger', ]
 
     if 'trigger' in args.mode:
         t = scan.Trigger(interfaces=args.interfaces)
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index fcbf1899..0ce4dff3 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -30,9 +30,12 @@
 # Scanner logic
 #  - save results to configdir:
 #     basedir = ~/.cdist/scan/<ipv6-address>
-#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time or similar
-#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
-#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time
+#           or similar
+#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record
+#           unix time or similar
+#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record
+#           unix time or similar
 #
 #
 #
@@ -58,6 +61,7 @@ import datetime
 
 log = logging.getLogger("scan")
 
+
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
@@ -91,6 +95,7 @@ class Trigger(object):
         log.debug(f"Sending request on {interface}")
         send(packet, verbose=self.verbose)
 
+
 class Scanner(object):
     """
     Scan for replies of hosts, maintain the up-to-date database
@@ -134,7 +139,6 @@ class Scanner(object):
               prn=self.handle_pkg)
 
 
-
 if __name__ == '__main__':
     t = Trigger(interfaces=["wlan0"])
     t.start()

From df881c0f9859759c10339e9d4a5ac24ff77fbe1a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 4 Nov 2020 08:34:17 +0100
Subject: [PATCH 270/527] [type/__file] Fix --state pre-exists also for
 non-dry-runs

---
 cdist/conf/type/__file/gencode-remote | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index 2675b03a..f7a528fd 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -95,6 +95,10 @@ case "$state_should" in
         fi
     ;;
 
+    pre-exists)
+        :
+        ;;
+
     *)
         echo "Unknown state: $state_should" >&2
         exit 1

From d28a70a73c2affcf3268623a51df2b7a6cdd534a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 6 Nov 2020 08:32:40 +0100
Subject: [PATCH 271/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 5977d80e..335a0f9f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,8 @@ next:
 	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 	* Type __file: Fix state pre-exists (Dennis Camera)
+	* Type __apt_norecommends: Use 00InstallRecommends file as debian-installer does (Dennis Camera)
+	* Core: Introduce scanner (noninvasive, beta) (Nico Schottelius)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 0f1df5ef68a60c5b8077f437bf0d07f1f79788e7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:07:58 +0100
Subject: [PATCH 272/527] Fix shellcheck source directives

---
 cdist/conf/type/__uci/gencode-remote         | 2 +-
 cdist/conf/type/__uci_section/gencode-remote | 2 +-
 cdist/conf/type/__uci_section/manifest       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
index 348d15a1..70a3d3e0 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 state_is=$(cat "${__object:?}/explorer/state")
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
index 285659b0..50fdfa4e 100755
--- a/cdist/conf/type/__uci_section/gencode-remote
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci_section/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 7e23e443..0325b92c 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci_section/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 

From c7c3075f629191ed7e05a76395b77dce01e2928f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:10:14 +0100
Subject: [PATCH 273/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 335a0f9f..40b816a6 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -13,6 +13,7 @@ next:
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 	* Type __file: Fix state pre-exists (Dennis Camera)
 	* Type __apt_norecommends: Use 00InstallRecommends file as debian-installer does (Dennis Camera)
+	* New types: __uci, __uci_section (Dennis Camera)
 	* Core: Introduce scanner (noninvasive, beta) (Nico Schottelius)
 
 6.8.0: 2020-09-11

From 348c6eedc903ecd28b02c5dda3821e71ef255a4f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:12:20 +0100
Subject: [PATCH 274/527] Release 6.9.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 40b816a6..c47123d9 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.0: 2020-11-07
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)

From 10abe514b8fb7e6d3ce3587a55773a21e78ce9e6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 11 Jun 2020 21:27:53 +0200
Subject: [PATCH 275/527] [type/__hostname] Add support for OpenWrt

---
 cdist/conf/type/__hostname/gencode-remote | 3 +++
 cdist/conf/type/__hostname/manifest       | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index ae224611..02afcbfb 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -69,6 +69,9 @@ in
     centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void)
         echo "hostname '$name_should'"
     ;;
+    openwrt)
+        echo "echo '$name_should' >/proc/sys/kernel/hostname"
+    ;;
     macosx)
         echo "scutil --set HostName '$name_should'"
     ;;
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index e1e356a0..bf8a331c 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -149,6 +149,10 @@ in
     openbsd)
         echo "$name_should" | __file /etc/myname --source -
     ;;
+    openwrt)
+        __uci system.@system[0].hostname --value "$name_should"
+            # --transaction hostname
+    ;;
     slackware)
         # We write the FQDN into /etc/HOSTNAME.  But /etc/rc.d/rc.M will only
         # read the first component from this file and set it as the running

From b0f3bb3350be9575c1632c6e3f669e7ede431020 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 7 Nov 2020 16:49:09 +0100
Subject: [PATCH 276/527] New type __dpkg_architecture

This type handles foreign architectures added to dpkg.
---
 .../__dpkg_architecture/explorer/architecture |   8 ++
 .../explorer/foreign-architectures            |   8 ++
 .../type/__dpkg_architecture/gencode-remote   |  64 +++++++++++
 cdist/conf/type/__dpkg_architecture/man.rst   | 103 ++++++++++++++++++
 .../parameter/default/state                   |   1 +
 .../__dpkg_architecture/parameter/optional    |   1 +
 6 files changed, 185 insertions(+)
 create mode 100755 cdist/conf/type/__dpkg_architecture/explorer/architecture
 create mode 100755 cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
 create mode 100755 cdist/conf/type/__dpkg_architecture/gencode-remote
 create mode 100644 cdist/conf/type/__dpkg_architecture/man.rst
 create mode 100644 cdist/conf/type/__dpkg_architecture/parameter/default/state
 create mode 100644 cdist/conf/type/__dpkg_architecture/parameter/optional

diff --git a/cdist/conf/type/__dpkg_architecture/explorer/architecture b/cdist/conf/type/__dpkg_architecture/explorer/architecture
new file mode 100755
index 00000000..4478bc6e
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/explorer/architecture
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+# __dpkg_architecture/explorer/architecture
+
+# Get the main architecture of this machine
+
+
+# print or die in the gencode-remote
+dpkg --print-architecture || true
diff --git a/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
new file mode 100755
index 00000000..ff01672a
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+# __dpkg_architecture/explorer/foreign-architectures
+
+# Print all additional architectures
+
+
+# print or die in the gencode-remote
+dpkg --print-foreign-architectures || true
diff --git a/cdist/conf/type/__dpkg_architecture/gencode-remote b/cdist/conf/type/__dpkg_architecture/gencode-remote
new file mode 100755
index 00000000..adf9d655
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/gencode-remote
@@ -0,0 +1,64 @@
+#!/bin/sh -e
+# __dpkg_architecture/gencode-remote
+
+
+# Get parameter and explorer
+state_should="$(cat "$__object/parameter/state")"
+arch_wanted="$__object_id"
+main_arch="$(cat "$__object/explorer/architecture")"
+
+# Exit here if dpkg do not work (empty explorer)
+if [ -z "$main_arch" ]; then
+    echo "dpkg is not available or unable to detect a architecture!" >&2
+    exit 1
+fi
+
+
+# Check if requested architecture is the main one
+if [ "$arch_wanted" = "$main_arch" ]; then
+    # higher than present; we can not remove it
+    state_is="present"
+    caution="yes"
+
+# Check if the architecture not already used
+elif grep -qFx "$arch_wanted" "$__object/explorer/foreign-architectures"; then
+    state_is="present"
+
+# arch does not exist
+else
+    state_is="absent"
+fi
+
+
+# Check what to do
+if [ "$state_is" != "$state_should" ]; then
+    case "$state_should" in
+        present)
+            # print add code
+            printf "dpkg --add-architecture '%s'\n" "$arch_wanted"
+            # updating the index to make the new architecture available
+            echo "apt update"
+
+            echo added >> "$__messages_out"
+            ;;
+
+        absent)
+            if [ "$caution" ]; then
+                printf "can not remove the main arch '%s' of the system!\n" "$main_arch" >&2
+                exit 1
+            fi
+
+            # removing all existing packages for the architecture
+            printf "apt purge '.*:%s'\n" "$arch_wanted"
+            # print remove code
+            printf "dpkg --remove-architecture '%s'\n" "$arch_wanted"
+
+            echo removed >> "$__messages_out"
+            ;;
+
+        *)
+            printf "state '%s' is unknown!\n" "$state_should" >&2
+            exit 1
+            ;;
+    esac
+fi
diff --git a/cdist/conf/type/__dpkg_architecture/man.rst b/cdist/conf/type/__dpkg_architecture/man.rst
new file mode 100644
index 00000000..fa196229
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/man.rst
@@ -0,0 +1,103 @@
+cdist-type__dpkg_architecture(7)
+================================
+
+NAME
+----
+cdist-type__dpkg_architecture - Handles foreign architectures on debian-like
+systems managed by `dpkg`
+
+
+DESCRIPTION
+-----------
+This type handles foreign architectures on systems managed by
+:strong:`dpkg`\ (1). The object id is the name of the architecture accepted by
+`dpkg`, which should be added or removed.
+
+If the architecture is not setup on the system, it adds a new architecture as a
+new foreign architecture in `dpkg`. Then, it updates the apt package index to
+make packages from the new architecture available.
+
+If the architecture should be removed, it will remove it if it is not the base
+architecture on where the system was installed on. Before it, it will purge
+every package based on the "to be removed" architecture via `apt` to be able to
+remove the selected architecture.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    ``present`` or ``absent``. Defaults to ``present``.
+
+
+MESSAGES
+--------
+added
+   Added the specified architecture
+
+removed
+   Removed the specified architecture
+
+
+ABORTS
+------
+Aborts in the following cases:
+
+If :strong:`dpkg`\ (1) is not available. It will abort with a proper error
+message.
+
+If the architecture is the same as the base architecture the system is build
+upon it (returned by ``dpkg --print-architecture``) and it should be removed.
+
+It will fail if it can not execute :strong:`apt`\ (8). It is assumed that it is
+already installed.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+  # add i386 (32 bit) architecture
+  __dpkg_architecture i386
+
+  # remove it again :)
+  __dpkg_architecture i386 --state absent
+
+
+SEE ALSO
+--------
+`Multiarch on Debian systems <https://wiki.debian.org/Multiarch>`_
+
+`How to setup multiarch on Debian <https://wiki.debian.org/Multiarch/HOWTO>`_
+
+:strong:`dpkg`\ (1)
+:strong:`cdist-type__package_dpkg`\ (7)
+:strong:`cdist-type__package_apt`\ (7)
+
+Useful commands:
+
+.. code-block:: sh
+
+   # base architecture installed on this system
+   dpkg --print-architecture
+
+   # extra architectures added
+   dpkg --print-foreign-architectures
+
+
+AUTHORS
+-------
+Matthias Stecher <matthiasstecher at gmx.de>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Matthias Stecher. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+ublished by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__dpkg_architecture/parameter/default/state b/cdist/conf/type/__dpkg_architecture/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__dpkg_architecture/parameter/optional b/cdist/conf/type/__dpkg_architecture/parameter/optional
new file mode 100644
index 00000000..ff72b5c7
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/parameter/optional
@@ -0,0 +1 @@
+state

From 7777580d8ff94f2b0725dbbcef84d8957998acb6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 7 Nov 2020 20:56:17 +0100
Subject: [PATCH 277/527] __dpkg_architecture: add copyright headers

---
 .../__dpkg_architecture/explorer/architecture  | 18 ++++++++++++++++++
 .../explorer/foreign-architectures             | 18 ++++++++++++++++++
 .../type/__dpkg_architecture/gencode-remote    | 18 ++++++++++++++++++
 3 files changed, 54 insertions(+)

diff --git a/cdist/conf/type/__dpkg_architecture/explorer/architecture b/cdist/conf/type/__dpkg_architecture/explorer/architecture
index 4478bc6e..03e7e386 100755
--- a/cdist/conf/type/__dpkg_architecture/explorer/architecture
+++ b/cdist/conf/type/__dpkg_architecture/explorer/architecture
@@ -1,5 +1,23 @@
 #!/bin/sh -e
 # __dpkg_architecture/explorer/architecture
+#
+# 2020 Matthias Stecher <matthiasstecher at gmx.de>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 # Get the main architecture of this machine
 
diff --git a/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
index ff01672a..a150d307 100755
--- a/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
+++ b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
@@ -1,5 +1,23 @@
 #!/bin/sh -e
 # __dpkg_architecture/explorer/foreign-architectures
+#
+# 2020 Matthias Stecher <matthiasstecher at gmx.de>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 # Print all additional architectures
 
diff --git a/cdist/conf/type/__dpkg_architecture/gencode-remote b/cdist/conf/type/__dpkg_architecture/gencode-remote
index adf9d655..47fb24e7 100755
--- a/cdist/conf/type/__dpkg_architecture/gencode-remote
+++ b/cdist/conf/type/__dpkg_architecture/gencode-remote
@@ -1,5 +1,23 @@
 #!/bin/sh -e
 # __dpkg_architecture/gencode-remote
+#
+# 2020 Matthias Stecher <matthiasstecher at gmx.de>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 
 # Get parameter and explorer

From 91bcc2a2939508b8722c2a148c89c92aaa4f830f Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 7 Nov 2020 21:03:38 +0100
Subject: [PATCH 278/527] __dpkg_architecture: make type nonparallel

I think it's not good that dpkg or apt is running in parallel.
---
 cdist/conf/type/__dpkg_architecture/nonparallel | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 cdist/conf/type/__dpkg_architecture/nonparallel

diff --git a/cdist/conf/type/__dpkg_architecture/nonparallel b/cdist/conf/type/__dpkg_architecture/nonparallel
new file mode 100644
index 00000000..e69de29b

From 9fc6ee0948347a0de20e633ce4a0ee358dc92c00 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 8 Nov 2020 13:14:13 +0100
Subject: [PATCH 279/527] __package_apt: add --install-recommends parameter

For a good reason, __package_apt doesn't install recommended packages as
default. But the option --install-recommends comes handy if you want to
install a package where you want to install all recommended packages
(and not to install all of them separately).

Also, the manpage now explains that the type won't install recommended
packages by default.
---
 cdist/conf/type/__package_apt/gencode-remote    | 17 ++++++++++++-----
 cdist/conf/type/__package_apt/man.rst           | 15 +++++++++++++--
 cdist/conf/type/__package_apt/parameter/boolean |  1 +
 3 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote
index f3d91566..fbfca330 100755
--- a/cdist/conf/type/__package_apt/gencode-remote
+++ b/cdist/conf/type/__package_apt/gencode-remote
@@ -42,6 +42,13 @@ else
    target_release=""
 fi
 
+if [ -f "$__object/parameter/install-recommends" ]; then
+    # required if __apt_norecommends is used
+    recommendsparam="-o APT::Install-Recommends=1"
+else
+    recommendsparam="-o APT::Install-Recommends=0"
+fi
+
 if [ -f "$__object/parameter/purge-if-absent" ]; then
 	purgeparam="--purge"
 else
@@ -62,16 +69,16 @@ case "$state_is" in
     ;;
 esac
 
-# Hint if we need to avoid questions at some point:
-# DEBIAN_PRIORITY=critical can reduce the number of questions
-aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o APT::Install-Recommends=0 -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
-
 if [ "$state_is" = "$state_should" ]; then
     if [ -z "$version" ] || [ "$version" = "$version_is" ]; then
         exit 0;
     fi
 fi
 
+# Hint if we need to avoid questions at some point:
+# DEBIAN_PRIORITY=critical can reduce the number of questions
+aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
+
 case "$state_should" in
     present)
         # following is bit ugly, but important hack.
@@ -85,7 +92,7 @@ EOF
         if [ -n "$version" ]; then
             name="${name}=${version}"
         fi
-        echo "$aptget install $target_release '$name'"
+        echo "$aptget $recommendsparam install $target_release '$name'"
         echo "installed" >> "$__messages_out"
     ;;
     absent)
diff --git a/cdist/conf/type/__package_apt/man.rst b/cdist/conf/type/__package_apt/man.rst
index a1691eac..4e6101a5 100644
--- a/cdist/conf/type/__package_apt/man.rst
+++ b/cdist/conf/type/__package_apt/man.rst
@@ -9,7 +9,9 @@ cdist-type__package_apt - Manage packages with apt-get
 DESCRIPTION
 -----------
 apt-get is usually used on Debian and variants (like Ubuntu) to
-manage packages.
+manage packages. The package will be installed without recommended
+or suggested packages. If such packages are required, install them
+separatly or use the parameter ``--install-recommends``.
 
 This type will also update package index, if it is older
 than one day, to avoid missing package error messages.
@@ -23,7 +25,7 @@ None
 OPTIONAL PARAMETERS
 -------------------
 name
-   If supplied, use the name and not the object id as the package name.
+    If supplied, use the name and not the object id as the package name.
 
 state
     Either "present" or "absent", defaults to "present"
@@ -39,6 +41,15 @@ version
 
 BOOLEAN PARAMETERS
 ------------------
+install-recommends
+    If the package will be installed, it also installs recommended packages
+    with it. It will not install recommended packages if the original package
+    is already installed.
+
+    In most cases, it is recommended to install recommended packages separatly
+    to control which additional packages will be installed to avoid useless
+    installed packages.
+
 purge-if-absent
     If this parameter is given when state is `absent`, the package is
     purged from the system (using `--purge`).
diff --git a/cdist/conf/type/__package_apt/parameter/boolean b/cdist/conf/type/__package_apt/parameter/boolean
index f9a0f6b0..a2e433f3 100644
--- a/cdist/conf/type/__package_apt/parameter/boolean
+++ b/cdist/conf/type/__package_apt/parameter/boolean
@@ -1 +1,2 @@
+install-recommends
 purge-if-absent

From fded60bd0f4ca7b6b9047c9f3b1470fdbd20c263 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 8 Nov 2020 13:27:01 +0100
Subject: [PATCH 280/527] ++changelog

---
 docs/changelog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index c47123d9..f271142a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,12 @@
 Changelog
 ---------
 
+next:
+	* Type __file: Fix state pre-exists (Dennis Camera)
+	* Type __hostname: Add support for OpenWrt (Dennis Camera)
+	* New type: __dpkg_architecture (Matthias Stecher)
+	* Type __package_apt: Add --install-recommends parameter (Matthias Stecher)
+
 6.9.0: 2020-11-07
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)

From d2506ac04eac987caf0fe0c33f48866ee6e43c0d Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 8 Nov 2020 13:31:57 +0100
Subject: [PATCH 281/527] Release 6.9.1

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index f271142a..b6b8c22b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)
 	* Type __hostname: Add support for OpenWrt (Dennis Camera)
 	* New type: __dpkg_architecture (Matthias Stecher)

From a95eab77a5606f2552ac28c0b817945f72538c97 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 8 Nov 2020 15:18:04 +0100
Subject: [PATCH 282/527] __locale: add state explorer

.. so it doesn't execute code all the time.
---
 cdist/conf/type/__locale/explorer/state | 36 +++++++++++++++++++++++++
 cdist/conf/type/__locale/gencode-remote | 15 ++++++++---
 2 files changed, 47 insertions(+), 4 deletions(-)
 create mode 100755 cdist/conf/type/__locale/explorer/state

diff --git a/cdist/conf/type/__locale/explorer/state b/cdist/conf/type/__locale/explorer/state
new file mode 100755
index 00000000..4494fcbc
--- /dev/null
+++ b/cdist/conf/type/__locale/explorer/state
@@ -0,0 +1,36 @@
+#!/bin/sh -e
+# __locale/explorer/state
+#
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Check if the locale is already installed on the system.
+# Outputs 'present' or 'absent' depending if the locale exists.
+#
+
+
+# Get user-defined locale
+# locale name is echoed differently than the user propably set it (for UTF-8)
+locale="$(echo "$__object_id" | sed 's/UTF-8/utf8/')"
+
+# Check if the given locale exists on the system
+if localedef --list-archive | grep -qFx "$locale"; then
+    echo present
+else
+    echo absent
+fi
diff --git a/cdist/conf/type/__locale/gencode-remote b/cdist/conf/type/__locale/gencode-remote
index 1feb9884..4639cef8 100755
--- a/cdist/conf/type/__locale/gencode-remote
+++ b/cdist/conf/type/__locale/gencode-remote
@@ -23,6 +23,15 @@
 
 locale="$__object_id"
 
+state_is=$(cat "$__object/explorer/state")
+state_should=$(cat "$__object/parameter/state")
+
+# short circuit if there is nothing to do
+if [ "$state_is" = "$state_should" ]; then
+    exit 0
+fi
+
+
 # Hardcoded, create a pull request with
 # branching on $os in case it is at another location
 alias=/usr/share/locale/locale.alias
@@ -35,8 +44,6 @@ charmap=$(echo "$locale" | cut -d . -f 2)
 # W-T-F!
 locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
 
-state=$(cat "$__object/parameter/state")
-
 os=$(cat "$__global/explorer/os")
 
 # Nothing to be done on alpine
@@ -46,7 +53,7 @@ case "$os" in
         ;;
 esac
 
-case "$state" in
+case "$state_should" in
     present)
         echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
     ;;
@@ -54,7 +61,7 @@ case "$state" in
         echo localedef --delete-from-archive "$locale_remove"
     ;;
     *)
-        echo "Unsupported state: $state" >&2
+        echo "Unsupported state: $state_should" >&2
         exit 1
     ;;
 esac

From 792b4b10762c094d861282e78953dfd29b203a41 Mon Sep 17 00:00:00 2001
From: ssrq <ungleich@ssrq-sds-fds.ch>
Date: Mon, 9 Nov 2020 12:08:54 +0100
Subject: [PATCH 283/527] Add missing 'config' command

---
 docs/src/cdist-best-practice.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/src/cdist-best-practice.rst b/docs/src/cdist-best-practice.rst
index 39ec453e..f7677276 100644
--- a/docs/src/cdist-best-practice.rst
+++ b/docs/src/cdist-best-practice.rst
@@ -200,15 +200,15 @@ of cdist:
 .. code-block:: sh
 
     # Singleton type without parameter
-    echo __ungleich_munin_server | cdist --initial-manifest - munin.panter.ch
+    echo __ungleich_munin_server | cdist config --initial-manifest - munin.panter.ch
 
     # Singleton type with parameter
     echo __ungleich_munin_node --allow 1.2.3.4 | \
-        cdist --initial-manifest - rails-19.panter.ch
+        cdist config --initial-manifest - rails-19.panter.ch
 
     # Normal type
     echo __file /tmp/stdintest --mode 0644 | \
-        cdist --initial-manifest - cdist-dev-01.ungleich.ch
+        cdist config --initial-manifest - cdist-dev-01.ungleich.ch
 
 
 Other content in cdist repository

From ba906510529549446952bb3abfd8b49b05da7ca0 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 11 Nov 2020 07:49:32 +0100
Subject: [PATCH 284/527] ++changelog

---
 docs/changelog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index b6b8c22b..9fa0926c 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,11 @@
 Changelog
 ---------
 
+next:
+	* Documentation: Fix examples in best practice (Dennis Camera)
+	* Type __locale: Add state explorer (Matthias Stecher)
+	* Core: Reorganize scripts, version generation (Ander Punnar)
+
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)
 	* Type __hostname: Add support for OpenWrt (Dennis Camera)

From 3e48ef9e11795bf1c8a59ae202ed28f58cff1753 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:09:26 +0100
Subject: [PATCH 285/527] [type/__hostname] Lint

- Error if expected environment variables are unset
- Always wrap variable expansions in {}
---
 cdist/conf/type/__hostname/gencode-remote |  82 ++++++------
 cdist/conf/type/__hostname/manifest       | 147 +++++++++++-----------
 2 files changed, 114 insertions(+), 115 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 02afcbfb..9447daa9 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -20,26 +20,27 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$(cat "$__global/explorer/os")
-name_running=$(cat "$__global/explorer/hostname")
-has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
+os=$(cat "${__global:?}/explorer/os")
+name_running=$(cat "${__global:?}/explorer/hostname")
+has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl")
 
 
-if test -s "$__object/parameter/name"
+if test -s "${__object:?}/parameter/name"
 then
-    name_should=$(cat "$__object/parameter/name")
+    name_should=$(cat "${__object:?}/parameter/name")
 else
-    case $os
+    case ${os}
     in
         # RedHat-derivatives and BSDs
-        centos|fedora|redhat|scientific|freebsd|macosx|netbsd|openbsd)
+        (centos|fedora|redhat|scientific|freebsd|macosx|netbsd|openbsd)
             # Hostname is FQDN
-            name_should="${__target_host}"
-        ;;
-        *)
+            name_should=${__target_host:?}
+            ;;
+        (*)
             # Hostname is only first component of FQDN
-            name_should="${__target_host%%.*}"
-        ;;
+            name_should=${__target_host:?}
+            name_should=${name_should%%.*}
+            ;;
     esac
 fi
 
@@ -47,46 +48,46 @@ fi
 ################################################################################
 # Check if the (running) hostname is already correct
 #
-test "$name_running" != "$name_should" || exit 0
+test "${name_running}" != "${name_should}" || exit 0
 
 
 ################################################################################
 # Setup hostname
 #
-echo 'changed' >>"$__messages_out"
+echo 'changed' >>"${__messages_out:?}"
 
 # Use the good old way to set the hostname.
-case $os
+case ${os}
 in
-    alpine|debian|devuan|ubuntu)
+    (alpine|debian|devuan|ubuntu)
         echo 'hostname -F /etc/hostname'
-    ;;
-    archlinux)
+        ;;
+    (archlinux)
         echo 'command -v hostnamectl >/dev/null 2>&1' \
-            "&& hostnamectl set-hostname '$name_should'" \
-            "|| hostname '$name_should'"
-    ;;
-    centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void)
-        echo "hostname '$name_should'"
-    ;;
-    openwrt)
-        echo "echo '$name_should' >/proc/sys/kernel/hostname"
-    ;;
-    macosx)
-        echo "scutil --set HostName '$name_should'"
-    ;;
-    solaris)
-        echo "uname -S '$name_should'"
-    ;;
-    slackware|suse|opensuse-leap)
+            "&& hostnamectl set-hostname '${name_should}'" \
+            "|| hostname '${name_should}'"
+        ;;
+    (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void)
+        echo "hostname '${name_should}'"
+        ;;
+    (openwrt)
+        echo "echo '${name_should}' >/proc/sys/kernel/hostname"
+        ;;
+    (macosx)
+        echo "scutil --set HostName '${name_should}'"
+        ;;
+    (solaris)
+        echo "uname -S '${name_should}'"
+        ;;
+    (slackware|suse|opensuse-leap)
         # We do not read from /etc/HOSTNAME, because the running
         # hostname is the first component only while the file contains
         # the FQDN.
-        echo "hostname '$name_should'"
-    ;;
-    *)
+        echo "hostname '${name_should}'"
+        ;;
+    (*)
         # Fall back to set the hostname using hostnamectl, if available.
-        if test -n "$has_hostnamectl"
+        if test -n "${has_hostnamectl}"
         then
             # Don't use hostnamectl as the primary means to set the hostname for
             # systemd systems, because it cannot be trusted to work reliably and
@@ -97,7 +98,8 @@ in
             echo "test \"\$(hostname)\" = \"\$(cat /etc/hostname)\"" \
                  " || hostname -F /etc/hostname"
         else
-            printf "echo 'Unsupported OS: %s' >&2\nexit 1\n" "$os"
+            printf "echo 'Unsupported OS: %s' >&2\n" "${os}"
+            printf 'exit 1\n'
         fi
-    ;;
+        ;;
 esac
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index bf8a331c..125bb2fb 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -20,69 +20,65 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-not_supported() {
-    echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
-    echo "Please contribute an implementation for it if you can." >&2
-    exit 1
-}
-
 set_hostname_systemd() {
     echo "$1" | __file /etc/hostname --source -
 }
 
-os=$(cat "$__global/explorer/os")
-os_version=$(cat "$__global/explorer/os_version")
-os_major=$(echo "$os_version" | grep -o '^[0-9][0-9]*' || true)
+os=$(cat "${__global:?}/explorer/os")
+os_version=$(cat "${__global:?}/explorer/os_version")
+os_major=$(echo "${os_version}" | grep -o '^[0-9][0-9]*' || true)
 
-max_len=$(cat "$__object/explorer/max_len")
-has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
+max_len=$(cat "${__object:?}/explorer/max_len")
+has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl")
 
-if test -s "$__object/parameter/name"
+if test -s "${__object:?}/parameter/name"
 then
-    name_should=$(cat "$__object/parameter/name")
+    name_should=$(cat "${__object:?}/parameter/name")
 else
-    case $os
+    case ${os}
     in
         # RedHat-derivatives and BSDs
-        centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware)
+        (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware)
             # Hostname is FQDN
-            name_should="${__target_host}"
-        ;;
-        suse|opensuse-leap)
+            name_should=${__target_host:?}
+            ;;
+        (suse|opensuse-leap)
+            name_should=${__target_host:?}
+
             # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
             # systemd does not. The running hostname is the first
             # component in both cases.
             # In versions before 15.x, the FQDN is stored in /etc/hostname.
-            if test -n "$has_hostnamectl" && test "$os_major" -ge 15 \
-                    && test "$os_major" -ne 42
+            if test -n "${has_hostnamectl}" \
+                    && test "${os_major}" -ge 15 \
+                    && test "${os_major}" -ne 42
             then
-                name_should="${__target_host%%.*}"
-            else
-                name_should="${__target_host}"
+                name_should=${name_should%%.*}
             fi
-        ;;
+            ;;
         *)
             # Hostname is only first component of FQDN on all other systems.
-            name_should="${__target_host%%.*}"
-        ;;
+            name_should=${__target_host:?}
+            name_should=${name_should%%.*}
+            ;;
     esac
 fi
 
-if test -n "$max_len" && test "$(printf '%s' "$name_should" | wc -c)" -gt "$max_len"
+if test -n "${max_len}" && test "$(printf '%s' "${name_should}" | wc -c)" -gt "${max_len}"
 then
     printf "Host name too long. Up to %u characters allowed.\n" "${max_len}" >&2
     exit 1
 fi
 
-case $os
+case ${os}
 in
-    alpine|debian|devuan|ubuntu|void)
-        echo "$name_should" | __file /etc/hostname --source -
-    ;;
-    archlinux)
-        if test -n "$has_hostnamectl"
+    (alpine|debian|devuan|ubuntu|void)
+        echo "${name_should}" | __file /etc/hostname --source -
+        ;;
+    (archlinux)
+        if test -n "${has_hostnamectl}"
         then
-            set_hostname_systemd "$name_should"
+            set_hostname_systemd "${name_should}"
         else
             echo 'Ancient ArchLinux variants without hostnamectl are not supported.' >&2
             exit 1
@@ -97,8 +93,8 @@ in
             #     --value "\"$name_should\""
         fi
         ;;
-    centos|fedora|redhat|scientific)
-        if test -z "$has_hostnamectl"
+    (centos|fedora|redhat|scientific)
+        if test -z "${has_hostnamectl}"
         then
             # Only write to /etc/sysconfig/network on non-systemd versions.
             # On systemd-based versions this entry is ignored.
@@ -106,63 +102,62 @@ in
                 --file /etc/sysconfig/network \
                 --delimiter '=' --exact_delimiter \
                 --key HOSTNAME \
-                --value "\"$name_should\""
+                --value "\"${name_should}\""
         else
-            set_hostname_systemd "$name_should"
+            set_hostname_systemd "${name_should}"
         fi
-    ;;
-    gentoo)
+        ;;
+    (gentoo)
         # Only write to /etc/conf.d/hostname on OpenRC-based installations.
         # On systemd use hostnamectl(1) in gencode-remote.
-        if test -z "$has_hostnamectl"
+        if test -z "${has_hostnamectl}"
         then
             __key_value '/etc/conf.d/hostname:hostname' \
                 --file /etc/conf.d/hostname \
                 --delimiter '=' --exact_delimiter \
                 --key 'hostname' \
-                --value "\"$name_should\""
+                --value "\"${name_should}\""
         else
             set_hostname_systemd "$name_should"
         fi
-    ;;
-    freebsd)
+        ;;
+    (freebsd)
         __key_value '/etc/rc.conf:hostname' \
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
-            --value "\"$name_should\""
-    ;;
-    macosx)
+            --value "\"${name_should}\""
+        ;;
+    (macosx)
         # handled in gencode-remote
-        :
-    ;;
-    netbsd)
+        ;;
+    (netbsd)
         __key_value '/etc/rc.conf:hostname' \
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
-            --value "\"$name_should\""
+            --value "\"${name_should}\""
 
         # To avoid confusion, ensure that the hostname is only stored once.
         __file /etc/myname --state absent
-    ;;
-    openbsd)
-        echo "$name_should" | __file /etc/myname --source -
-    ;;
-    openwrt)
-        __uci system.@system[0].hostname --value "$name_should"
+        ;;
+    (openbsd)
+        echo "${name_should}" | __file /etc/myname --source -
+        ;;
+    (openwrt)
+        __uci system.@system[0].hostname --value "${name_should}"
             # --transaction hostname
-    ;;
-    slackware)
+        ;;
+    (slackware)
         # We write the FQDN into /etc/HOSTNAME.  But /etc/rc.d/rc.M will only
         # read the first component from this file and set it as the running
         # hostname on boot.
-        echo "$name_should" | __file /etc/HOSTNAME --source -
-    ;;
-    solaris)
-        echo "$name_should" | __file /etc/nodename --source -
-    ;;
-    suse|opensuse-leap)
+        echo "${name_should}" | __file /etc/HOSTNAME --source -
+        ;;
+    (solaris)
+        echo "${name_should}" | __file /etc/nodename --source -
+        ;;
+    (suse|opensuse-leap)
         # Modern SuSE provides /etc/HOSTNAME as a symlink for
         # backwards-compatibility. Unfortunately it cannot be used
         # here as __file does not follow the symlink.
@@ -171,23 +166,25 @@ in
         # not work correctly on openSUSE 12.x which provides
         # hostnamectl but not /etc/hostname.
 
-        if test -n "$has_hostnamectl" -a "$os_major" -gt 12
+        if test -n "${has_hostnamectl}" -a "${os_major}" -gt 12
         then
-            hostname_file='/etc/hostname'
+            hostname_file=/etc/hostname
         else
-            hostname_file='/etc/HOSTNAME'
+            hostname_file=/etc/HOSTNAME
         fi
 
-        echo "$name_should" | __file "$hostname_file" --source -
-    ;;
-    *)
+        echo "${name_should}" | __file "${hostname_file}" --source -
+        ;;
+    (*)
         # On other operating systems we fall back to systemd's
         # hostnamectl if available…
-        if test -n "$has_hostnamectl"
+        if test -n "${has_hostnamectl}"
         then
-            set_hostname_systemd "$name_should"
+            set_hostname_systemd "${name_should}"
         else
-            not_supported
+            echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
+            echo "Please contribute an implementation for it if you can." >&2
+            exit 1
         fi
-    ;;
+        ;;
 esac

From 702f3eba4f8f88b634a010b377d077d501a6f0e9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:15:02 +0100
Subject: [PATCH 286/527] [type/__hostname] Remove opensuse-leap OS string
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

everything should be suse now…
---
 cdist/conf/type/__hostname/gencode-remote | 2 +-
 cdist/conf/type/__hostname/manifest       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 9447daa9..c1a97ac8 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -79,7 +79,7 @@ in
     (solaris)
         echo "uname -S '${name_should}'"
         ;;
-    (slackware|suse|opensuse-leap)
+    (slackware|suse)
         # We do not read from /etc/HOSTNAME, because the running
         # hostname is the first component only while the file contains
         # the FQDN.
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 125bb2fb..d044072b 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -42,7 +42,7 @@ else
             # Hostname is FQDN
             name_should=${__target_host:?}
             ;;
-        (suse|opensuse-leap)
+        (suse)
             name_should=${__target_host:?}
 
             # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
@@ -157,7 +157,7 @@ in
     (solaris)
         echo "${name_should}" | __file /etc/nodename --source -
         ;;
-    (suse|opensuse-leap)
+    (suse)
         # Modern SuSE provides /etc/HOSTNAME as a symlink for
         # backwards-compatibility. Unfortunately it cannot be used
         # here as __file does not follow the symlink.

From 87a0d91587568594a616c7f3588bc74106312761 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:20:18 +0100
Subject: [PATCH 287/527] [type/__hostname] Fix OS version detection for SuSE
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

everything should be suse now…
---
 cdist/conf/type/__hostname/manifest | 39 ++++++++++++++++-------------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index d044072b..b80aa2ef 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -25,8 +25,6 @@ set_hostname_systemd() {
 }
 
 os=$(cat "${__global:?}/explorer/os")
-os_version=$(cat "${__global:?}/explorer/os_version")
-os_major=$(echo "${os_version}" | grep -o '^[0-9][0-9]*' || true)
 
 max_len=$(cat "${__object:?}/explorer/max_len")
 has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl")
@@ -38,24 +36,10 @@ else
     case ${os}
     in
         # RedHat-derivatives and BSDs
-        (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware)
+        (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware|suse)
             # Hostname is FQDN
             name_should=${__target_host:?}
             ;;
-        (suse)
-            name_should=${__target_host:?}
-
-            # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
-            # systemd does not. The running hostname is the first
-            # component in both cases.
-            # In versions before 15.x, the FQDN is stored in /etc/hostname.
-            if test -n "${has_hostnamectl}" \
-                    && test "${os_major}" -ge 15 \
-                    && test "${os_major}" -ne 42
-            then
-                name_should=${name_should%%.*}
-            fi
-            ;;
         *)
             # Hostname is only first component of FQDN on all other systems.
             name_should=${__target_host:?}
@@ -158,6 +142,27 @@ in
         echo "${name_should}" | __file /etc/nodename --source -
         ;;
     (suse)
+        if test -s "${__global:?}/explorer/os_release"
+        then
+            # shellcheck source=/dev/null
+            os_version=$(. "${__global:?}/explorer/os_release" && echo "${VERSION}")
+        else
+            os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global:?}/explorer/os_version")
+        fi
+        os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
+
+        # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
+        # systemd does not. The running hostname is the first
+        # component in both cases.
+        # In versions before 15.x, the FQDN is stored in /etc/hostname.
+        if test -n "${has_hostnamectl}" \
+                && test "${os_major}" -ge 15 \
+                && test "${os_major}" -ne 42
+        then
+            # strip away everything but the first part from $name_should
+            name_should=${name_should%%.*}
+        fi
+
         # Modern SuSE provides /etc/HOSTNAME as a symlink for
         # backwards-compatibility. Unfortunately it cannot be used
         # here as __file does not follow the symlink.

From 21dd500c05c91c4927b2a4166690da65b1c98f97 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:44:44 +0100
Subject: [PATCH 288/527] Make pycodestyle pipeline happy

---
 cdist/__init__.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index 1c60ae0f..44366cd0 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -33,14 +33,15 @@ try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
-    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+    cdist_dir = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), os.pardir))
     if os.path.isdir(os.path.join(cdist_dir, '.git')):
         try:
             VERSION = subprocess.check_output(
                 ['git', 'describe', '--always'],
                 cwd=cdist_dir,
                 universal_newlines=True)
-        except:
+        except Exception:
             pass
 
 BANNER = """

From e2d4f8037ab7d30187e4d153aa25ef36e8afc91c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:45:05 +0100
Subject: [PATCH 289/527] [bin/cdist-build-helper] Fix paths to ex scripts/
 scripts

---
 bin/cdist-build-helper | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper
index bdef0dbb..fb8a67a4 100755
--- a/bin/cdist-build-helper
+++ b/bin/cdist-build-helper
@@ -45,7 +45,7 @@ usage() {
         shellcheck-manifests
         shellcheck-local-gencodes
         shellcheck-remote-gencodes
-        shellcheck-scripts
+        shellcheck-bin
         shellcheck-gencodes
         shellcheck-types
         shellcheck
@@ -100,7 +100,7 @@ case "$option" in
                     if (\$0 ~ /^$end/) {
                         exit
                     } else {
-                        print \$0 
+                        print \$0
                     }
                 }
             }" "$basedir/docs/changelog"
@@ -135,7 +135,7 @@ case "$option" in
 
         version=$1; shift
 
-        ( 
+        (
         cat << eof
 Subject: cdist $version has been released
 
@@ -336,7 +336,7 @@ eof
         make docs-clean
         make docs
 
-        ############################################################# 
+        #############################################################
         # Everything green, let's do the release
 
         # Tag the current commit
@@ -405,7 +405,7 @@ eof
     ;;
 
     pycodestyle|pep8)
-        pycodestyle "${basedir}" "${basedir}/scripts/cdist"
+        pycodestyle "${basedir}" "${basedir}/bin/cdist"
     ;;
 
     check-pycodestyle)
@@ -460,9 +460,10 @@ eof
         test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
-    shellcheck-scripts)
+    # NOTE: shellcheck-scripts is kept for compatibility
+    shellcheck-bin|shellcheck-scripts)
         # shellcheck disable=SC2086
-        ${SHELLCHECKCMD} scripts/cdist-dump scripts/cdist-new-type > "${SHELLCHECKTMP}"
+        ${SHELLCHECKCMD} bin/cdist-dump bin/cdist-new-type > "${SHELLCHECKTMP}"
         test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
@@ -480,7 +481,7 @@ eof
     shellcheck)
         "$0" shellcheck-global-explorers || exit 1
         "$0" shellcheck-types || exit 1
-        "$0" shellcheck-scripts || exit 1
+        "$0" shellcheck-bin || exit 1
     ;;
 
     shellcheck-type-files)

From f82e0167aaf295b8451a07fa931506a1290efcdc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:49:04 +0100
Subject: [PATCH 290/527] [.gitlab-ci.yml] Make version before other targets

---
 .gitlab-ci.yml | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e48355ea..a4bc67aa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,20 +1,23 @@
+---
+image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
+
 stages:
     - test
 
-image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
+before_script:
+    - ./bin/cdist-build-helper version
 
-unit_tests:
+shellcheck:
     stage: test
     script:
-        - ./bin/cdist-build-helper version
-        - ./bin/cdist-build-helper test
+        - ./bin/cdist-build-helper shellcheck
 
 pycodestyle:
     stage: test
     script:
         - ./bin/cdist-build-helper pycodestyle
 
-shellcheck:
+unit_tests:
     stage: test
     script:
-        - ./bin/cdist-build-helper shellcheck
+        - ./bin/cdist-build-helper test

From 0ee3fda94deff1461ac488d218a2af98057675c5 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 15:04:51 +0100
Subject: [PATCH 291/527] Fix paths to cdist executable

---
 cdist/integration.py          | 5 ++---
 cdist/test/__init__.py        | 2 +-
 cdist/test/config/__init__.py | 4 ++--
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/cdist/integration.py b/cdist/integration.py
index 03e4167d..17b65f09 100644
--- a/cdist/integration.py
+++ b/cdist/integration.py
@@ -54,13 +54,12 @@ _mydir = os.path.dirname(__file__)
 def find_cdist_exec():
     """Search cdist executable starting from local lib directory.
 
-    Detect if ../scripts/cdist (from local lib direcotry) exists and
+    Detect if ../bin/cdist (from local lib directory) exists and
     if it is executable. If not then try to find cdist exec path in
     os.get_exec_path() entries. If no cdist path is found rasie
     cdist.Error.
     """
-    cdist_path = os.path.abspath(os.path.join(_mydir, '..', 'scripts',
-                                              'cdist'))
+    cdist_path = os.path.abspath(os.path.join(_mydir, '..', 'bin', 'cdist'))
     if os.access(cdist_path, os.X_OK):
         return cdist_path
     cdist_path = find_cdist_exec_in_path()
diff --git a/cdist/test/__init__.py b/cdist/test/__init__.py
index faa3686a..16a311e4 100644
--- a/cdist/test/__init__.py
+++ b/cdist/test/__init__.py
@@ -26,7 +26,7 @@ import tempfile
 cdist_base_path = os.path.abspath(
     os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../"))
 
-cdist_exec_path = os.path.join(cdist_base_path, "scripts/cdist")
+cdist_exec_path = os.path.join(cdist_base_path, "bin/cdist")
 
 global_fixtures_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                    "fixtures"))
diff --git a/cdist/test/config/__init__.py b/cdist/test/config/__init__.py
index 0ed614b1..c405207c 100644
--- a/cdist/test/config/__init__.py
+++ b/cdist/test/config/__init__.py
@@ -202,7 +202,7 @@ class ConfigRunTestCase(test.CdistTestCase):
             host_dir_name=self.hostdir,
             # exec_path can not derivated from sys.argv in case of unittest
             exec_path=os.path.abspath(os.path.join(
-                my_dir, '../../../scripts/cdist')),
+                my_dir, '../../../bin/cdist')),
             initial_manifest=os.path.join(fixtures,
                                           'manifest/dryrun_manifest'),
             add_conf_dirs=[fixtures])
@@ -219,7 +219,7 @@ class ConfigRunTestCase(test.CdistTestCase):
             base_root_path=self.host_base_path,
             host_dir_name=self.hostdir,
             exec_path=os.path.abspath(os.path.join(
-                my_dir, '../../../scripts/cdist')),
+                my_dir, '../../../bin/cdist')),
             initial_manifest=os.path.join(
                 fixtures, 'manifest/init-deps-resolver'),
             add_conf_dirs=[fixtures])

From c39eb1dbce281ed6f64f804a16fb28e5c8dceccd Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 15:16:33 +0100
Subject: [PATCH 292/527] [cdist.emulator] Fix setting of log level (tests OK)

---
 cdist/emulator.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cdist/emulator.py b/cdist/emulator.py
index 60b94880..a2bdc3d4 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -120,17 +120,18 @@ class Emulator:
                 level = logging.WARNING
         else:
             level = logging.WARNING
+        self.log = logging.getLogger(self.target_host[0])
         try:
             logging.root.setLevel(level)
+            self.log.setLevel(level)
         except (ValueError, TypeError):
             # if invalid __cdist_log_level value
             logging.root.setLevel(logging.WARNING)
+            self.log.setLevel(logging.WARNING)
 
         colored_log = self.env.get('__cdist_colored_log', 'false')
         cdist.log.CdistFormatter.USE_COLORS = colored_log == 'true'
 
-        self.log = logging.getLogger(self.target_host[0])
-
     def commandline(self):
         """Parse command line"""
 

From 2f70a8b540f8d4c283029207e98f7883557338f0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 15:25:46 +0100
Subject: [PATCH 293/527] [bin/cdist-build-helper] Keep going in shellcheck
 targets

---
 bin/cdist-build-helper | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper
index fb8a67a4..0380b3f8 100755
--- a/bin/cdist-build-helper
+++ b/bin/cdist-build-helper
@@ -468,20 +468,26 @@ eof
     ;;
 
     shellcheck-gencodes)
-        "$0" shellcheck-local-gencodes || exit 1
-        "$0" shellcheck-remote-gencodes || exit 1
+        errors=false
+        "$0" shellcheck-local-gencodes || errors=true
+        "$0" shellcheck-remote-gencodes || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck-types)
-        "$0" shellcheck-type-explorers || exit 1
-        "$0" shellcheck-manifests || exit 1
-        "$0" shellcheck-gencodes || exit 1
+        errors=false
+        "$0" shellcheck-type-explorers || errors=true
+        "$0" shellcheck-manifests || errors=true
+        "$0" shellcheck-gencodes || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck)
-        "$0" shellcheck-global-explorers || exit 1
-        "$0" shellcheck-types || exit 1
-        "$0" shellcheck-bin || exit 1
+        errors=false
+        "$0" shellcheck-global-explorers || errors=true
+        "$0" shellcheck-types || errors=true
+        "$0" shellcheck-bin || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck-type-files)
@@ -491,8 +497,10 @@ eof
     ;;
 
     shellcheck-with-files)
-        "$0" shellcheck || exit 1
-        "$0" shellcheck-type-files || exit 1
+        errors=false
+        "$0" shellcheck || errors=true
+        "$0" shellcheck-type-files || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck-build-helper)

From ebf471e8d0216cc81928fcdff1fe159584187808 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 13 Nov 2020 02:32:45 +0100
Subject: [PATCH 294/527] [type/__hwclock] Add new type

---
 .../conf/type/__hwclock/explorer/adjtime_mode |  28 +++
 .../__hwclock/explorer/timedatectl_localrtc   |  27 +++
 cdist/conf/type/__hwclock/gencode-remote      |  62 +++++
 cdist/conf/type/__hwclock/man.rst             |  63 +++++
 cdist/conf/type/__hwclock/manifest            | 222 ++++++++++++++++++
 cdist/conf/type/__hwclock/parameter/required  |   1 +
 cdist/conf/type/__hwclock/singleton           |   0
 7 files changed, 403 insertions(+)
 create mode 100755 cdist/conf/type/__hwclock/explorer/adjtime_mode
 create mode 100755 cdist/conf/type/__hwclock/explorer/timedatectl_localrtc
 create mode 100755 cdist/conf/type/__hwclock/gencode-remote
 create mode 100644 cdist/conf/type/__hwclock/man.rst
 create mode 100755 cdist/conf/type/__hwclock/manifest
 create mode 100644 cdist/conf/type/__hwclock/parameter/required
 create mode 100644 cdist/conf/type/__hwclock/singleton

diff --git a/cdist/conf/type/__hwclock/explorer/adjtime_mode b/cdist/conf/type/__hwclock/explorer/adjtime_mode
new file mode 100755
index 00000000..2b27bedc
--- /dev/null
+++ b/cdist/conf/type/__hwclock/explorer/adjtime_mode
@@ -0,0 +1,28 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Prints the clock mode read from the /etc/adjtime file, if present.
+#
+
+# not all operating systems use an adjfile
+test -f /etc/adjtime || exit 0
+
+# 3rd line is clock mode
+# adjtime(5) https://man7.org/linux/man-pages/man5/adjtime.5.html
+sed -n 3p /etc/adjtime
diff --git a/cdist/conf/type/__hwclock/explorer/timedatectl_localrtc b/cdist/conf/type/__hwclock/explorer/timedatectl_localrtc
new file mode 100755
index 00000000..8239122e
--- /dev/null
+++ b/cdist/conf/type/__hwclock/explorer/timedatectl_localrtc
@@ -0,0 +1,27 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Prints the LocalRTC property using timedatectl on systemd-based systems.
+#
+
+command -v timedatectl >/dev/null 2>&1 || exit 0
+
+# NOTE: Older versions of timedatectl do not support `timedatectl show'
+timedatectl --no-pager status \
+| awk -F': ' '$1 ~ "RTC in local TZ$" { sub(/[ \t]*$/, "", $2); print $2 }'
diff --git a/cdist/conf/type/__hwclock/gencode-remote b/cdist/conf/type/__hwclock/gencode-remote
new file mode 100755
index 00000000..5995fb23
--- /dev/null
+++ b/cdist/conf/type/__hwclock/gencode-remote
@@ -0,0 +1,62 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+mode=$(cat "${__object:?}/parameter/mode")
+
+timedatectl_localrtc=$(cat "${__object:?}/explorer/timedatectl_localrtc")
+adjtime_mode=$(cat "${__object:?}/explorer/adjtime_mode")
+
+
+case ${mode}
+in
+	(localtime)
+		adjtime_str=LOCAL
+		local_rtc_str=yes
+		;;
+	(UTC|utc)
+		adjtime_str=UTC
+		local_rtc_str=no
+		;;
+	(*)
+		printf 'Invalid value for --mode: %s\n' "${mode}" >&2
+		printf 'Acceptable values are: localtime, utc.\n' >&2
+		exit 1
+esac
+
+
+if test -n "${timedatectl_localrtc}"
+then
+	# systemd
+	timedatectl_should=${local_rtc_str}
+	if test "${timedatectl_localrtc}" != "${timedatectl_should}"
+	then
+		printf 'timedatectl set-local-rtc %s\n' "${timedatectl_should}"
+	fi
+elif test -n "${adjtime_mode}"
+then
+	# others (update /etc/adjtime if present)
+	if test "${adjtime_mode}" != "${adjtime_str}"
+	then
+		# Update /etc/adjtime (3rd line is clock mode)
+		# adjtime(5) https://man7.org/linux/man-pages/man5/adjtime.5.html
+		# FIXME: Should maybe add third line if adjfile only contains two lines
+		printf "sed -i '3c\\\\\\n%s\\n' /etc/adjtime\\n" "${adjtime_str}"
+	fi
+fi
diff --git a/cdist/conf/type/__hwclock/man.rst b/cdist/conf/type/__hwclock/man.rst
new file mode 100644
index 00000000..65eb648f
--- /dev/null
+++ b/cdist/conf/type/__hwclock/man.rst
@@ -0,0 +1,63 @@
+cdist-type__hwclock(7)
+======================
+
+NAME
+----
+cdist-type__hwclock - Manage the hardware real time clock.
+
+
+DESCRIPTION
+-----------
+This type can be used to control how the hardware clock is used by the operating
+system.
+
+
+REQUIRED PARAMETERS
+-------------------
+mode
+    What mode the hardware clock is in.
+
+    Acceptable values:
+
+    localtime
+        The hardware clock is set to local time (common for systems also running
+        Windows.)
+    UTC
+        The hardware clock is set to UTC (common on UNIX systems.)
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Make the operating system treat the time read from the hwclock as UTC.
+    __hwclock --mode UTC
+
+
+SEE ALSO
+--------
+:strong:`hwclock`\ (8)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__hwclock/manifest b/cdist/conf/type/__hwclock/manifest
new file mode 100755
index 00000000..7d9ab88f
--- /dev/null
+++ b/cdist/conf/type/__hwclock/manifest
@@ -0,0 +1,222 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# TODO: Consider supporting BADYEAR
+
+os=$(cat "${__global:?}/explorer/os")
+mode=$(cat "${__object:?}/parameter/mode")
+
+has_systemd_timedatectl=$(test -s "${__object:?}/explorer/timedatectl_localrtc" && echo true || echo false)
+
+
+case ${mode}
+in
+	(localtime)
+		local_clock=true
+		;;
+	(UTC|utc)
+		local_clock=false
+		;;
+	(*)
+		printf 'Invalid value for --mode: %s\n' "${mode}" >&2
+		printf 'Acceptable values are: UTC, localtime.\n' >&2
+		exit 1
+esac
+
+
+case ${os}
+in
+	(alpine|gentoo)
+		if ! $has_systemd_timedatectl
+		then
+			# NOTE: Gentoo also supports systemd, in which case /etc/conf.d is
+			#       not used. So we check for systemd presence here and only
+			#       update /etc/conf.d if systemd is not installed.
+			# https://wiki.gentoo.org/wiki/System_time#Hardware_clock
+
+			export CDIST_ORDER_DEPENDENCY=true
+			__file /etc/conf.d/hwclock --state present \
+				--owner root --group root --mode 0644
+			__key_value /etc/conf.d/hwclock:clock \
+				--file /etc/conf.d/hwclock \
+				--key clock \
+				--delimiter '=' --exact_delimiter \
+				--value "\"$($local_clock && echo local || echo UTC)\""
+			unset CDIST_ORDER_DEPENDENCY
+		fi
+		;;
+	(centos|fedora|redhat|scientific)
+		os_version=$(cat "${__global:?}/explorer/os_version")
+		os_major=$(expr "${os_version}" : '.* release \([0-9]*\)')
+		case ${os}
+		in
+			(centos|scientific)
+				update_sysconfig=$(test "${os_major}" -lt 6 && echo true || echo false)
+				;;
+			(fedora)
+				update_sysconfig=$(test "${os_major}" -lt 10 && echo true || echo false)
+				;;
+			(redhat|*)
+				case ${os_version}
+				in
+					('Red Hat Enterprise Linux'*)
+						update_sysconfig=$(test "${os_major}" -lt 6 && echo true || echo false)
+						;;
+					('Red Hat Linux'*)
+						update_sysconfig=true
+						;;
+					(*)
+						printf 'Could not determine Red Hat distribution.\n' >&2
+						printf "Please contribute an implementation for it if you can.\n" >&2
+						exit 1
+						;;
+				esac
+				;;
+		esac
+
+		if ${update_sysconfig:?}
+		then
+			export CDIST_ORDER_DEPENDENCY=true
+			__file /etc/sysconfig/clock --state present \
+				--owner root --group root --mode 0644
+			__key_value /etc/sysconfig/clock:UTC \
+				--file /etc/sysconfig/clock \
+				--key UTC \
+				--delimiter '=' --exact_delimiter \
+				--value "$($local_clock && echo false || echo true)"
+			unset CDIST_ORDER_DEPENDENCY
+		fi
+		;;
+	(debian|devuan|ubuntu)
+		os_major=$(sed 's/[^0-9].*$//' "${__global:?}/explorer/os_version")
+
+		case ${os}
+		in
+			(debian)
+				if test "${os_major}" -ge 7
+				then
+					update_rcS=false
+				elif test "${os_major}" -ge 3
+				then
+					update_rcS=true
+				else
+					# Debian 2.2 should be supportable using rcS.
+					# Debian 2.1 uses the ancient GMT key.
+					# Debian 1.3 does not have rcS.
+					printf "Your operating system (Debian %s) is currently not supported by this type (%s)\n" \
+						"$(cat "${__global:?}/explorer/os_version")" "${__type##*/}" >&2
+					printf "Please contribute an implementation for it if you can.\n" >&2
+					exit 1
+				fi
+				;;
+			(devuan)
+				update_rcS=false
+				;;
+			(ubuntu)
+				update_rcS=$(test "${os_major}" -lt 16 && echo true || echo false)
+				;;
+		esac
+
+		if ${update_rcS}
+		then
+			export CDIST_ORDER_DEPENDENCY=true
+			__file /etc/default/rcS --state present \
+				--owner root --group root --mode 0644
+			__key_value /etc/default/rcS:UTC \
+				--file /etc/default/rcS \
+				--key UTC \
+				--delimiter '=' --exact_delimiter \
+				--value "$($local_clock && echo no || echo yes)"
+			unset CDIST_ORDER_DEPENDENCY
+		fi		  
+		;;
+	(freebsd)
+		# cf. adjkerntz(8)
+		__file /etc/wall_cmos_clock \
+			--state "$($local_clock && echo present || echo absent)" \
+			--owner root --group wheel --mode 0444
+		;;
+	(netbsd)
+		# https://wiki.netbsd.org/guide/boot/#index9h2
+		__key_value /etc/rc.conf:rtclocaltime \
+			--file /etc/rc.conf \
+			--key rtclocaltime \
+			--delimiter '=' --exact_delimiter \
+			--value "$($local_clock && echo YES || echo NO)"
+		;;
+	(slackware)
+		__file /etc/hardwareclock --owner root --group root --mode 0644 \
+			--source - <<-EOF
+		# /etc/hardwareclock
+		#
+		# Tells how the hardware clock time is stored.
+		# This file is managed by cdist.
+
+		$($local_clock && echo localtime || echo UTC)
+		EOF
+		;;
+	(suse)
+		if test -s "${__global:?}/explorer/os_release"
+		then
+			# shellcheck source=/dev/null
+			os_version=$(. "${__global:?}/explorer/os_release" && echo "${VERSION}")
+		else
+			os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global:?}/explorer/os_version")
+		fi
+		os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
+
+		# TODO: Consider using `yast2 timezone set hwclock' instead
+		if expr "${os_major}" \< 12
+		then
+			# Starting with SuSE 12 (first systemd-based version)
+			# /etc/sysconfig/clock does not contain the HWCLOCK line
+			# anymore.
+			# With SuSE 13, it has been reduced to TIMEZONE configuration.
+			__key_value /etc/sysconfig/clock:HWCLOCK \
+				--file /etc/sysconfig/clock \
+				--delimiter '=' --exact_delimiter \
+				--key HWCLOCK \
+				--value "$($local_clock && echo '"--localtime"' || echo '"-u"')"
+		fi
+		;;
+	(void)
+		export CDIST_ORDER_DEPENDENCY=true
+		__file /etc/rc.conf \
+			--owner root --group root --mode 0644 \
+			--state present
+		__key_value /etc/rc.conf:HARDWARECLOCK \
+			--file /etc/rc.conf \
+			--delimiter '=' --exact_delimiter \
+			--key HARDWARECLOCK \
+			--value "\"$($local_clock && echo localtime || echo UTC)\""
+		unset CDIST_ORDER_DEPENDENCY
+		;;
+	(*)
+		if ! $has_systemd_timedatectl
+		then
+			printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2
+			printf "Please contribute an implementation for it if you can.\n" >&2
+			exit 1
+		fi
+		;;
+esac
+
+# NOTE: timedatectl set-local-rtc for systemd is in gencode-remote
+# NOTE: /etc/adjtime is also updated in gencode-remote
diff --git a/cdist/conf/type/__hwclock/parameter/required b/cdist/conf/type/__hwclock/parameter/required
new file mode 100644
index 00000000..17ab372f
--- /dev/null
+++ b/cdist/conf/type/__hwclock/parameter/required
@@ -0,0 +1 @@
+mode
diff --git a/cdist/conf/type/__hwclock/singleton b/cdist/conf/type/__hwclock/singleton
new file mode 100644
index 00000000..e69de29b

From a07a45887136105cc8be316307805ceaecdd4cf3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 13 Nov 2020 06:43:01 +0100
Subject: [PATCH 295/527] ++changelog

---
 docs/changelog | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 9fa0926c..ff411a46 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,7 +4,8 @@ Changelog
 next:
 	* Documentation: Fix examples in best practice (Dennis Camera)
 	* Type __locale: Add state explorer (Matthias Stecher)
-	* Core: Reorganize scripts, version generation (Ander Punnar)
+	* Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera)
+	* New type: __hwclock (Dennis Camera)
 
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)

From 50927527863fd382f5200287a808d8b36028ff87 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 14 Nov 2020 09:59:30 +0100
Subject: [PATCH 296/527] Update build helper script in .gitattributes

---
 .gitattributes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitattributes b/.gitattributes
index 45c10d7b..01d20f30 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -4,5 +4,5 @@
 docs/speeches export-ignore
 docs/video export-ignore
 docs/src/man7 export-ignore
-bin/build-helper export-ignore
+bin/cdist-build-helper export-ignore
 README-maintainers export-ignore

From 76aa00b12e9b675c797e997a06ec7597c53a85f3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 14 Nov 2020 10:13:58 +0100
Subject: [PATCH 297/527] Fix importing cdist module

Resolve #845.
---
 bin/cdist | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index 1f92f157..ddaffa7f 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -25,25 +25,23 @@ import logging
 import os
 import sys
 
-# try to import cdist and if that fails,
-# then add this file's parent dir to
-# module search path and try again.
-try:
-    import cdist
-except ModuleNotFoundError:
-    cdist_dir = os.path.realpath(
-        os.path.join(
-            os.path.dirname(os.path.realpath(__file__)),
-            os.pardir))
+# See if this file's parent is cdist module
+# and if so add it to module search path.
+cdist_dir = os.path.realpath(
+    os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        os.pardir))
+cdist_init_dir = os.path.join(cdist_dir, 'cdist', '__init__.py')
+if os.path.exists(cdist_init_dir):
     sys.path.insert(0, cdist_dir)
-    import cdist
 
-import cdist.argparse
-import cdist.banner
-import cdist.config
-import cdist.install
-import cdist.shell
-import cdist.inventory
+import cdist            # noqa 402
+import cdist.argparse   # noqa 402
+import cdist.banner     # noqa 402
+import cdist.config     # noqa 402
+import cdist.install    # noqa 402
+import cdist.shell      # noqa 402
+import cdist.inventory  # noqa 402
 
 
 def commandline():

From f75d4772090d3a9cfd798dd193cd324083a9b3a7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 13:45:12 +0100
Subject: [PATCH 298/527] Deprecate __locale and replace with __localedef

---
 cdist/conf/type/__locale/deprecated           |  1 +
 cdist/conf/type/__localedef/gencode-remote    | 60 +++++++++++++++++++
 cdist/conf/type/__localedef/man.rst           | 55 +++++++++++++++++
 cdist/conf/type/__localedef/manifest          | 41 +++++++++++++
 .../type/__localedef/parameter/default/state  |  1 +
 .../conf/type/__localedef/parameter/optional  |  1 +
 6 files changed, 159 insertions(+)
 create mode 100644 cdist/conf/type/__locale/deprecated
 create mode 100755 cdist/conf/type/__localedef/gencode-remote
 create mode 100644 cdist/conf/type/__localedef/man.rst
 create mode 100755 cdist/conf/type/__localedef/manifest
 create mode 100644 cdist/conf/type/__localedef/parameter/default/state
 create mode 100644 cdist/conf/type/__localedef/parameter/optional

diff --git a/cdist/conf/type/__locale/deprecated b/cdist/conf/type/__locale/deprecated
new file mode 100644
index 00000000..5a06b28e
--- /dev/null
+++ b/cdist/conf/type/__locale/deprecated
@@ -0,0 +1 @@
+This type is deprecated. Please use __localedef instead.
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
new file mode 100755
index 00000000..1feb9884
--- /dev/null
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -0,0 +1,60 @@
+#!/bin/sh -e
+#
+# 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Let localedef do the magic
+#
+
+locale="$__object_id"
+
+# Hardcoded, create a pull request with
+# branching on $os in case it is at another location
+alias=/usr/share/locale/locale.alias
+
+input=$(echo "$locale" | cut -d . -f 1)
+charmap=$(echo "$locale" | cut -d . -f 2)
+
+# Adding locale? The name is de_CH.UTF-8
+# Removing locale? The name is de_CH.utf8.
+# W-T-F!
+locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
+
+state=$(cat "$__object/parameter/state")
+
+os=$(cat "$__global/explorer/os")
+
+# Nothing to be done on alpine
+case "$os" in
+    alpine)
+        exit 0
+        ;;
+esac
+
+case "$state" in
+    present)
+        echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
+    ;;
+    absent)
+        echo localedef --delete-from-archive "$locale_remove"
+    ;;
+    *)
+        echo "Unsupported state: $state" >&2
+        exit 1
+    ;;
+esac
diff --git a/cdist/conf/type/__localedef/man.rst b/cdist/conf/type/__localedef/man.rst
new file mode 100644
index 00000000..0ccf484a
--- /dev/null
+++ b/cdist/conf/type/__localedef/man.rst
@@ -0,0 +1,55 @@
+cdist-type__localedef(7)
+========================
+
+NAME
+----
+cdist-type__localedef - Define and remove system locales
+
+
+DESCRIPTION
+-----------
+This cdist type allows you to define locales on the system using
+:strong:`localedef`\ (1) or remove them.
+On systems that don't support definition of new locales, the type will raise an
+error.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+   ``present`` or ``absent``. Defaults to ``present``.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Add locale de_CH.UTF-8
+    __localedef de_CH.UTF-8
+
+    # Same as above, but more explicit
+    __localedef de_CH.UTF-8 --state present
+
+    # Remove colourful British English
+    __localedef en_GB.UTF-8 --state absent
+
+
+SEE ALSO
+--------
+:strong:`locale`\ (1),
+:strong:`localedef`\ (1),
+:strong:`cdist-type__locale_system`\ (7)
+
+
+AUTHORS
+-------
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+
+
+COPYING
+-------
+Copyright \(C) 2013-2019 Nico Schottelius, 2020 Dennis Camera. Free use of this
+software is granted under the terms of the GNU General Public License version 3
+or later (GPLv3+).
diff --git a/cdist/conf/type/__localedef/manifest b/cdist/conf/type/__localedef/manifest
new file mode 100755
index 00000000..9f1e17ac
--- /dev/null
+++ b/cdist/conf/type/__localedef/manifest
@@ -0,0 +1,41 @@
+#!/bin/sh -e
+#
+# 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2015 David Hürlimann (david at ungleich.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Install required packages
+#
+
+os=$(cat "$__global/explorer/os")
+
+
+case "$os" in
+    debian|devuan)
+        # Debian needs a seperate package
+        __package locales --state present
+    ;;
+    archlinux|suse|ubuntu|scientific|centos|alpine)
+        :
+    ;;
+    *)
+        echo "Sorry, do not know how to handle os: $os" >&2
+        echo "Please edit the type ${__type##*/} to fix this." >&2
+        exit 1
+    ;;
+esac
diff --git a/cdist/conf/type/__localedef/parameter/default/state b/cdist/conf/type/__localedef/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__localedef/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__localedef/parameter/optional b/cdist/conf/type/__localedef/parameter/optional
new file mode 100644
index 00000000..ff72b5c7
--- /dev/null
+++ b/cdist/conf/type/__localedef/parameter/optional
@@ -0,0 +1 @@
+state

From 54e689f7c2b3bc25aedb2aaf9f5aece289ec5b66 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 13:49:04 +0100
Subject: [PATCH 299/527] [type/__localedef] Add state explorer

---
 cdist/conf/type/__localedef/explorer/state | 71 ++++++++++++++++++++++
 cdist/conf/type/__localedef/gencode-remote | 13 +++-
 2 files changed, 82 insertions(+), 2 deletions(-)
 create mode 100755 cdist/conf/type/__localedef/explorer/state

diff --git a/cdist/conf/type/__localedef/explorer/state b/cdist/conf/type/__localedef/explorer/state
new file mode 100755
index 00000000..d8db29c5
--- /dev/null
+++ b/cdist/conf/type/__localedef/explorer/state
@@ -0,0 +1,71 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer determines if the locale is defined on the target system.
+# Will print nothing on error.
+#
+# Possible output:
+#  present:
+#    the main locale (and possibly aliases) is present
+#  absent:
+#    neither the main locale nor any aliases are present
+#
+
+command -v locale >/dev/null 2>&1 || exit 0
+
+locales=$(locale -a)
+
+parse_locale() {
+	# This function will split locales into their parts. Locale strings are
+	# usually of the form: [language[_territory][.codeset][@modifier]]
+	# For simplicity, language and territory are not separated by this function.
+	# Old Linux systems were also using "english" or "german" as locale strings.
+	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
+	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
+	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
+	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
+}
+
+format_locale() {
+	# Usage: format_locale language codeset modifier
+	printf '%s' "$1"
+	test -z "$2" || printf '.%s' "$2"
+	test -z "$3" || printf '@%s' "$3"
+	printf '\n'
+}
+
+gnu_normalize_codeset() {
+	# reimplementation of glibc/locale/programs/localedef.c normalize_codeset()
+	echo "$*" | tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]'
+}
+
+locale_available() (
+	echo "${locales}" | grep -qxF "$1" || {
+		# glibc uses "normalized" locale names in archives.
+		# If a locale is stored in an archive, the normalized name will be
+		# printed by locale, so that needs to be checked, too.
+		localename=$(
+			parse_locale "$1" _lang _codeset _modifier \
+			&& format_locale "${_lang:?}" "$(gnu_normalize_codeset "${_codeset?}")" \
+			   "${_modifier?}")
+		echo "${locales}" | grep -qxF "${localename}"
+	}
+)
+
+locale_available "${__object_id:?}" && echo present || echo absent
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 1feb9884..9ea3f6f1 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -35,7 +35,8 @@ charmap=$(echo "$locale" | cut -d . -f 2)
 # W-T-F!
 locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
 
-state=$(cat "$__object/parameter/state")
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 
 os=$(cat "$__global/explorer/os")
 
@@ -46,7 +47,15 @@ case "$os" in
         ;;
 esac
 
-case "$state" in
+# NOTE: If state explorer fails (e.g. locale(1) missing), the following check
+#       will always fail and let definition/removal run.
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
+
+case ${state_should}
+in
     present)
         echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
     ;;

From cc29e54b851212568c3f6eddf4bf0a64c1eabb0d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 23 Jul 2020 23:20:39 +0200
Subject: [PATCH 300/527] [type/__localedef] Differentiate between OSes and
 better handling of normalized locale names

---
 cdist/conf/type/__localedef/gencode-remote | 117 +++++++++++++++------
 1 file changed, 82 insertions(+), 35 deletions(-)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 9ea3f6f1..b9cc68e8 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,35 +18,16 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+# Manage system locales using localedef(1).
 #
-# Let localedef do the magic
-#
-
-locale="$__object_id"
-
-# Hardcoded, create a pull request with
-# branching on $os in case it is at another location
-alias=/usr/share/locale/locale.alias
-
-input=$(echo "$locale" | cut -d . -f 1)
-charmap=$(echo "$locale" | cut -d . -f 2)
-
-# Adding locale? The name is de_CH.UTF-8
-# Removing locale? The name is de_CH.utf8.
-# W-T-F!
-locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
 
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
-os=$(cat "$__global/explorer/os")
-
-# Nothing to be done on alpine
-case "$os" in
-    alpine)
-        exit 0
-        ;;
-esac
+test "${state_should}" = 'present' -o "${state_should}" = 'absent' || {
+	printf 'Invalid state: %s\n' "${state_should}" >&2
+	exit 1
+}
 
 # NOTE: If state explorer fails (e.g. locale(1) missing), the following check
 #       will always fail and let definition/removal run.
@@ -54,16 +36,81 @@ then
 	exit 0
 fi
 
-case ${state_should}
+locale=${__object_id:?}
+os=$(cat "${__global:?}/explorer/os")
+
+if expr "${locale}" : '.*/' >/dev/null
+then
+	printf 'Paths as locales are not supported.\n' >&2
+	printf '__object_id is: %s\n' "${locale}" >&2
+	exit 1
+fi
+
+parse_locale() {
+	# This function will split locales into their parts. Locale strings are
+	# usually of the form: [language[_territory][.codeset][@modifier]]
+	# For simplicity, language and territory are not separated by this function.
+	# Old Linux systems were also using "english" or "german" as locale strings.
+	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
+	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
+	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
+	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
+}
+
+format_locale() {
+	# Usage: format_locale language codeset modifier
+	printf '%s' "$1"
+	test -z "$2" || printf '.%s' "$2"
+	test -z "$3" || printf '@%s' "$3"
+	printf '\n'
+}
+
+gnu_normalize_codeset() {
+	echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]'
+}
+
+
+: "${lang=}" "${codeset=}" "${modifier=}"  # declare variables for shellcheck
+parse_locale "${locale}" lang codeset modifier
+
+
+case ${os}
 in
-    present)
-        echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
-    ;;
-    absent)
-        echo localedef --delete-from-archive "$locale_remove"
-    ;;
-    *)
-        echo "Unsupported state: $state" >&2
-        exit 1
-    ;;
+	(alpine|openwrt)
+		printf '%s does not support locales.\n' "${os}" >&2
+		exit 1
+		;;
+	(archlinux|debian|devuan|ubuntu|suse|centos|fedora|redhat|scientific)
+		# FIXME: The code below only works for glibc-based installations.
+
+		# NOTE: Hardcoded, create a pull request in case it is at another
+		#       location for some opther distro.
+		# NOTE: locale.alias can be symlinked (e.g. Debian)
+		aliasfile='/usr/share/locale/locale.alias'
+
+		case ${state_should}
+		in
+			(present)
+				input=$(format_locale "${lang}" '' "${modifier}")
+				cat <<-EOF
+				set --
+				if test -e '${aliasfile}'
+				then
+					set -- -A '${aliasfile}'
+				fi
+
+				localedef -i '${input}' -f '${codeset}' "\$@" '${locale}'
+				EOF
+				;;
+			(absent)
+				localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}")
+				printf "localedef --delete-from-archive '%s'\n" "${localename}"
+				;;
+		esac
+		;;
+	(*)
+		echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
+		echo "Please contribute an implementation for it if you can." >&2
+		exit 1
+		;;
 esac

From f44888f192d78d8ce1e3583f70f94b0c2039c1e2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 14:02:08 +0100
Subject: [PATCH 301/527] [type/__localedef] Only install dependencies in
 manifest. OS checking moved to gencode-remote

---
 cdist/conf/type/__localedef/manifest | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__localedef/manifest b/cdist/conf/type/__localedef/manifest
index 9f1e17ac..3ab3ad8c 100755
--- a/cdist/conf/type/__localedef/manifest
+++ b/cdist/conf/type/__localedef/manifest
@@ -2,6 +2,7 @@
 #
 # 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
 # 2015 David Hürlimann (david at ungleich.ch)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,24 +19,12 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# Install required packages
+# Install required packages.
 #
 
-os=$(cat "$__global/explorer/os")
-
-
-case "$os" in
-    debian|devuan)
-        # Debian needs a seperate package
-        __package locales --state present
-    ;;
-    archlinux|suse|ubuntu|scientific|centos|alpine)
-        :
-    ;;
-    *)
-        echo "Sorry, do not know how to handle os: $os" >&2
-        echo "Please edit the type ${__type##*/} to fix this." >&2
-        exit 1
-    ;;
+case $(cat "${__global:?}/explorer/os")
+in
+	(debian|devuan)
+		__package_apt locales --state present
+		;;
 esac

From dcef2c19f5b87e3a9594f01fefab66cc7488dce7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 14:07:42 +0100
Subject: [PATCH 302/527] [type/__localedef] Add support for FreeBSD

---
 cdist/conf/type/__localedef/gencode-remote | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index b9cc68e8..80e7eb1c 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -108,6 +108,25 @@ in
 				;;
 		esac
 		;;
+	(freebsd)
+		case ${state_should}
+		in
+			(present)
+				if expr "$(grep -oe '^[0-9]*' "${__global:?}/explorer/os_version")" '>=' 11 >/dev/null
+				then
+					# localedef(1) is available with FreeBSD >= 11
+					printf "localedef -i '%s' -f '%s' '%s'\n" "${input}" "${codeset}" "${locale}"
+				else
+					printf 'localedef(1) was added to FreeBSD starting with version 11.\n' >&2
+					printf 'Please upgrade your FreeBSD installation to use %s.\n' "${__type##*/}" >&2
+					exit 1
+				fi
+				;;
+			(absent)
+				printf "rm -R '/usr/share/locale/%s'\n" "${locale}"
+				;;
+		esac
+		;;
 	(*)
 		echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
 		echo "Please contribute an implementation for it if you can." >&2

From c1c60e3374e0cb93ea182c4cbcb813150756c3e4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 15:24:46 +0100
Subject: [PATCH 303/527] [type/__localedef] Blacklist OpenBSD and NetBSD

---
 cdist/conf/type/__localedef/gencode-remote | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 80e7eb1c..af1a77f7 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -127,6 +127,12 @@ in
 				;;
 		esac
 		;;
+	(netbsd|openbsd)
+		# NetBSD/OpenBSD are missing localedef(1).
+		# We also do not delete defined locales because they can't be recreated.
+		echo "${os} is lacking localedef(1). Locale management unavailable." >&2
+		exit 1
+		;;
 	(*)
 		echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
 		echo "Please contribute an implementation for it if you can." >&2

From 575bb62dc507fb8c294319f8db20ba45595e6869 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 13 Nov 2020 18:42:04 +0100
Subject: [PATCH 304/527] [type/__localedef] Externalise functions to separate
 files

---
 .../conf/type/__localedef/files/lib/glibc.sh  |  5 ++++
 .../conf/type/__localedef/files/lib/locale.sh | 20 +++++++++++++
 cdist/conf/type/__localedef/gencode-remote    | 29 ++++---------------
 3 files changed, 30 insertions(+), 24 deletions(-)
 create mode 100644 cdist/conf/type/__localedef/files/lib/glibc.sh
 create mode 100644 cdist/conf/type/__localedef/files/lib/locale.sh

diff --git a/cdist/conf/type/__localedef/files/lib/glibc.sh b/cdist/conf/type/__localedef/files/lib/glibc.sh
new file mode 100644
index 00000000..6ace80d4
--- /dev/null
+++ b/cdist/conf/type/__localedef/files/lib/glibc.sh
@@ -0,0 +1,5 @@
+# -*- mode: sh; indent-tabs-mode: t -*-
+
+gnu_normalize_codeset() {
+	echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]'
+}
diff --git a/cdist/conf/type/__localedef/files/lib/locale.sh b/cdist/conf/type/__localedef/files/lib/locale.sh
new file mode 100644
index 00000000..b5e61374
--- /dev/null
+++ b/cdist/conf/type/__localedef/files/lib/locale.sh
@@ -0,0 +1,20 @@
+# -*- mode: sh; indent-tabs-mode:t -*-
+
+parse_locale() {
+	# This function will split locales into their parts. Locale strings are
+	# usually of the form: [language[_territory][.codeset][@modifier]]
+	# For simplicity, language and territory are not separated by this function.
+	# Old Linux systems were also using "english" or "german" as locale strings.
+	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
+	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
+	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
+	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
+}
+
+format_locale() {
+	# Usage: format_locale language codeset modifier
+	printf '%s' "$1"
+	test -z "$2" || printf '.%s' "$2"
+	test -z "$3" || printf '@%s' "$3"
+	printf '\n'
+}
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index af1a77f7..17941c63 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -21,6 +21,11 @@
 # Manage system locales using localedef(1).
 #
 
+# shellcheck source=cdist/conf/type/__localedef/files/lib/locale.sh
+. "${__type:?}/files/lib/locale.sh"
+# shellcheck source=cdist/conf/type/__localedef/files/lib/glibc.sh
+. "${__type:?}/files/lib/glibc.sh"
+
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
@@ -46,30 +51,6 @@ then
 	exit 1
 fi
 
-parse_locale() {
-	# This function will split locales into their parts. Locale strings are
-	# usually of the form: [language[_territory][.codeset][@modifier]]
-	# For simplicity, language and territory are not separated by this function.
-	# Old Linux systems were also using "english" or "german" as locale strings.
-	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
-	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
-	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
-	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
-}
-
-format_locale() {
-	# Usage: format_locale language codeset modifier
-	printf '%s' "$1"
-	test -z "$2" || printf '.%s' "$2"
-	test -z "$3" || printf '@%s' "$3"
-	printf '\n'
-}
-
-gnu_normalize_codeset() {
-	echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]'
-}
-
-
 : "${lang=}" "${codeset=}" "${modifier=}"  # declare variables for shellcheck
 parse_locale "${locale}" lang codeset modifier
 

From eeb98719197a09691af60fa26c0b64b3aaf8960e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 14 Nov 2020 09:54:01 +0100
Subject: [PATCH 305/527] [type/__localedef] glibc: Also delete aliases when
 removing a locale

---
 cdist/conf/type/__localedef/gencode-remote | 15 +++++++++++++--
 cdist/conf/type/__localedef/man.rst        |  5 +++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 17941c63..d7fd6942 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -84,8 +84,19 @@ in
 				EOF
 				;;
 			(absent)
-				localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}")
-				printf "localedef --delete-from-archive '%s'\n" "${localename}"
+				main_localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}")
+
+				cat <<-EOF
+				while read -r _alias _localename
+				do
+				    if test "\${_localename}" = '$(format_locale "${lang}" "${codeset}")'
+				    then
+				        localedef --delete-from-archive "\${_alias}"
+				    fi
+				done <'${aliasfile}'
+
+				localedef --delete-from-archive '${main_localename}'
+				EOF
 				;;
 		esac
 		;;
diff --git a/cdist/conf/type/__localedef/man.rst b/cdist/conf/type/__localedef/man.rst
index 0ccf484a..454ce9d1 100644
--- a/cdist/conf/type/__localedef/man.rst
+++ b/cdist/conf/type/__localedef/man.rst
@@ -13,6 +13,11 @@ This cdist type allows you to define locales on the system using
 On systems that don't support definition of new locales, the type will raise an
 error.
 
+**NB:** This type respects the glibc ``locale.alias`` file,
+i.e. it defines alias locales or deletes aliases of a locale when it is removed.
+It is not possible, however, to use alias names to define locales or only remove
+certain aliases of a locale.
+
 
 OPTIONAL PARAMETERS
 -------------------

From 87faffd8750cac94190707c2b37233229c3fbde2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 14 Nov 2020 10:41:50 +0100
Subject: [PATCH 306/527] [type/__localdef] Also check for aliases in state
 explorer

---
 cdist/conf/type/__localedef/explorer/state | 31 +++++++++++++++++++++-
 cdist/conf/type/__localedef/gencode-remote |  7 +++--
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__localedef/explorer/state b/cdist/conf/type/__localedef/explorer/state
index d8db29c5..3ba57661 100755
--- a/cdist/conf/type/__localedef/explorer/state
+++ b/cdist/conf/type/__localedef/explorer/state
@@ -25,8 +25,14 @@
 #    the main locale (and possibly aliases) is present
 #  absent:
 #    neither the main locale nor any aliases are present
+#  alias-present:
+#    the main locale is absent, but at least one of its aliases is present
 #
 
+# Hardcoded, create a pull request in case it is at another location for
+# some other distro. (cf. gencode-remote)
+aliasfile='/usr/share/locale/locale.alias'
+
 command -v locale >/dev/null 2>&1 || exit 0
 
 locales=$(locale -a)
@@ -68,4 +74,27 @@ locale_available() (
 	}
 )
 
-locale_available "${__object_id:?}" && echo present || echo absent
+if locale_available "${__object_id:?}"
+then
+	echo present
+else
+	# NOTE: locale.alias can be symlinked.
+	if test -e "${aliasfile}"
+	then
+		# Check if one of the aliases of the locale is defined
+		baselocale=$(
+			parse_locale "${__object_id:?}" _lang _codeset _modifiers \
+			&& format_locale "${_lang}" "${_codeset}")
+		while read -r _alias _localename
+		do
+			if test "${_localename}" = "${baselocale}" \
+				&& echo "${locales}" | grep -qxF "${_alias}"
+			then
+				echo alias-present
+				exit 0
+			fi
+		done <"${aliasfile}"
+	fi
+
+	echo absent
+fi
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index d7fd6942..4538151f 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -94,9 +94,12 @@ in
 				        localedef --delete-from-archive "\${_alias}"
 				    fi
 				done <'${aliasfile}'
-
-				localedef --delete-from-archive '${main_localename}'
 				EOF
+
+				if test "${state_is}" = present
+				then
+					printf "localedef --delete-from-archive '%s'\n" "${main_localename}"
+				fi
 				;;
 		esac
 		;;

From 9d4f69250ea0e6bb733cde1e42183480f48b5d8f Mon Sep 17 00:00:00 2001
From: ssrq <ungleich@ssrq-sds-fds.ch>
Date: Thu, 19 Nov 2020 19:33:47 +0100
Subject: [PATCH 307/527] __sshd config: New type

---
 cdist/conf/type/__sshd_config/explorer/state  | 121 ++++++++
 .../files/update_sshd_config.awk              | 293 ++++++++++++++++++
 cdist/conf/type/__sshd_config/gencode-remote  |  97 ++++++
 cdist/conf/type/__sshd_config/man.rst         |  94 ++++++
 cdist/conf/type/__sshd_config/manifest        |  48 +++
 .../type/__sshd_config/parameter/default/file |   1 +
 .../__sshd_config/parameter/default/state     |   1 +
 .../type/__sshd_config/parameter/optional     |   4 +
 .../__sshd_config/parameter/optional_multiple |   1 +
 9 files changed, 660 insertions(+)
 create mode 100644 cdist/conf/type/__sshd_config/explorer/state
 create mode 100644 cdist/conf/type/__sshd_config/files/update_sshd_config.awk
 create mode 100755 cdist/conf/type/__sshd_config/gencode-remote
 create mode 100644 cdist/conf/type/__sshd_config/man.rst
 create mode 100755 cdist/conf/type/__sshd_config/manifest
 create mode 100644 cdist/conf/type/__sshd_config/parameter/default/file
 create mode 100644 cdist/conf/type/__sshd_config/parameter/default/state
 create mode 100644 cdist/conf/type/__sshd_config/parameter/optional
 create mode 100644 cdist/conf/type/__sshd_config/parameter/optional_multiple

diff --git a/cdist/conf/type/__sshd_config/explorer/state b/cdist/conf/type/__sshd_config/explorer/state
new file mode 100644
index 00000000..75c68b8a
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/explorer/state
@@ -0,0 +1,121 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Determines the current state of the config option.
+# Possible output:
+#  - present: "should" option present in config file
+#  - default: the "should" option is the default -> don’t know if present
+#  - absent: no such option present in config file
+#
+
+joinlines() { sed -n -e H -e "\${x;s/^\\n//;s/\\n/${1:?}/g;p;}"; }
+trlower() { tr '[:upper:]' '[:lower:]'; }
+tolower() { printf '%s' "$*" | trlower; }
+
+default_value() {
+	sshd -T -f /dev/null -C "$(make_conn_spec)" \
+	| sed -n -e 's/^'"$(tolower "${1:?}")"'[[:blank:]]\{1,\}//p'
+}
+
+make_conn_spec() {
+	if test -s "${__object:?}/parameter/match"
+	then
+		_match_file="${__object:?}/parameter/match"
+	else
+		_match_file='/dev/null'
+	fi
+
+	for _kw in \
+		addr=Address \
+		user=User \
+		host=Host \
+		laddr=LocalAddress \
+		lport=LocalPort \
+		rdomain=RDomain
+	do
+		_specname=${_kw%%=*}
+		_confname=$(tolower "${_kw#*=}")
+		while read -r _k _v
+		do
+			if test "$(tolower "${_k}")" = "${_confname}"
+			then
+				printf '%s=%s\n' "${_specname}" "${_v}"
+				continue 2
+			fi
+		done <"${_match_file}"
+
+		# NOTE: Print test spec even for empty keys to suppress errors like:
+		#  'Match User' in configuration but 'user' not in connection test specification.
+		# except lport:
+		#  Invalid port '' in test mode specification lport=
+		test "${_specname}" = 'lport' || printf '%s=\n' "${_specname}"
+	done \
+	| joinlines ','
+	unset _match_file
+}
+
+sshd_config_file=$(cat "${__object:?}/parameter/file")
+state_should=$(cat "${__object:?}/parameter/state")
+
+if test -s "${__object:?}/parameter/option"
+then
+	option_name=$(cat "${__object:?}/parameter/option")
+else
+	option_name=${__object_id:?}
+fi
+
+value_should=$(cat "${__object:?}/parameter/value" 2>/dev/null) \
+|| test "${state_should}" = absent || exit 0  # param optional if --state absent
+
+command -v sshd >/dev/null 2>&1 || {
+	echo 'Cannot find sshd.' >&2
+	exit 1
+}
+
+test -e "${sshd_config_file}" || {
+	echo 'absent'
+	exit 0
+}
+
+value_is=$(
+	sshd -T -f "${sshd_config_file}" -C "$(make_conn_spec)" \
+	| sed -n -e 's/^'"$(tolower "${option_name}")"'[[:blank:]]\{1,\}//p')
+
+if printf '%s\n' "${value_is}" | {
+		if test -n "${value_should}"
+		then
+			grep -q -x -F "${value_should}"
+		else
+			# if no value provided, assume "any" value
+			grep -q -e .
+		fi
+	}
+then
+	if default_value "${option_name}" | grep -q -x -F "${value_is}"
+	then
+		# Might produce false positives for default values.
+		# TODO: Manual checking should be done, but for simplicity, this case is
+		#       currently ignored here.
+		echo default
+	else
+		echo present
+	fi
+else
+	echo absent
+fi
diff --git a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
new file mode 100644
index 00000000..d0bc2b4b
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
@@ -0,0 +1,293 @@
+# -*- mode: awk; indent-tabs-mode: t -*-
+
+function usage() {
+	print_err("Usage: awk -f update_sshd_config.awk -- -o set|unset [-m 'User git'] -l 'X11Forwarding no' /etc/ssh/sshd_config")
+}
+
+function print_err(s) { print s | "cat >&2" }
+
+function alength(a,    i) {
+	for (i = 0; (i + 1) in a; ++i);
+	return i
+}
+
+function join(sep, a, i,    s) {
+	for (i = i ? i : 1; i in a; i++)
+		s = s sep a[i]
+	return substr(s, 2)
+}
+
+function getopt(opts, argv, target, files,    i, c, lv, idx, nf) {
+	# trivial getopt(3) implementation; only basic functionality
+	if (argv[1] == "--") i++
+	for (i += 1; i in argv; i++) {
+		if (lv) { target[c] = argv[i]; lv = 0; continue }
+		if (argv[i] ~ /^-/) {
+			c = substr(argv[i], 2, 1)
+			idx = index(opts, c)
+			if (!idx) {
+				print_err(sprintf("invalid option -%c\n", c))
+				continue
+			}
+			if (substr(opts, idx + 1, 1) == ":") {
+				# option takes argument
+				if (length(argv[i]) > 2)
+					target[c] = substr(argv[i], 3)
+				else
+					lv = 1
+			} else {
+				target[c] = 1
+			}
+		} else
+			files[++nf] = argv[i]
+	}
+}
+
+# tokenise configuration line
+# this function mimics the counterpart in OpenSSH (misc.c)
+# but it returns two (next token SUBSEP rest) because I didn’t want to have to
+# simulate any pointer magic.
+function strdelim_internal(s, split_equals,    old) {
+	if (!s)
+		return ""
+
+	old = s
+
+	if (!match(s, WHITESPACE "|" QUOTE "" (split_equals ? "|" EQUALS : "")))
+		return s
+
+	s = substr(s, RSTART)
+	old = substr(old, 1, RSTART - 1)
+
+	if (s ~ "^" QUOTE) {
+		old = substr(old, 2)
+
+		# Find matching quote
+		if (match(s, QUOTE)) {
+			old = substr(old, 1, RSTART)
+			# s = substr()
+			if (match(s, "^" WHITESPACE "*"))
+				s = substr(s, RLENGTH)
+			return old
+		} else {
+			# no matching quote
+			return ""
+		}
+	}
+
+	if (match(s, "^" WHITESPACE "+")) {
+		sub("^" WHITESPACE "+", "", s)
+		if (split_equals)
+			sub(EQUALS WHITESPACE "*", "", s)
+	} else if (s ~ "^" EQUALS) {
+		s = substr(s, 2)
+	}
+
+	return old SUBSEP s
+}
+function strdelim(s) { return strdelim_internal(s, 1) }
+function strdelimw(s) { return strdelim_internal(s, 0) }
+
+function singleton_option(opt) {
+	return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|authenticationmethods|authorizedkeysfile|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/
+}
+
+function print_update() {
+	if (mode) {
+		if (match_only) printf "\t"
+		printf "%s\n", line_should
+		updated = 1
+	}
+}
+
+BEGIN {
+	FS = "\n"  # disable field splitting
+
+	WHITESPACE = "[ \t]"  # servconf.c, misc.c:strdelim_internal (without line breaks, cf. bugs)
+	QUOTE = "[\"]"  # misc.c:strdelim_internal
+	EQUALS = "[=]"
+
+	split("", opts)
+	split("", files)
+	getopt("ho:l:m:", ARGV, opts, files)
+
+	if (opts["h"]) { usage(); exit (e="0") }
+
+	line_should = opts["l"]
+	match_only = opts["m"]
+	num_files = alength(files)
+
+	if (num_files != 1 || !opts["o"] || !line_should) {
+		usage()
+		exit (e=126)
+	}
+
+	if (opts["o"] == "set") {
+		mode = 1
+	} else if (opts["o"] == "unset") {
+		mode = 0
+	} else {
+		print_err(sprintf("invalid mode %s\n", mode))
+		exit (e=1)
+	}
+
+	if (mode) {
+		# loop over sshd_config twice!
+		ARGV[2] = ARGV[1] = files[1]
+		ARGC = 3
+	} else {
+		# only loop once
+		ARGV[1] = files[1]
+		ARGC = 2
+	}
+
+	split(strdelim(line_should), should, SUBSEP)
+	option_should = tolower(should[1])
+	value_should = should[2]
+}
+
+{
+	line = $0
+
+	# Strip trailing whitespace. Allow \f (form feed) at EOL only
+	sub("(" WHITESPACE "|\f)*$", "", line)
+
+	# Strip leading whitespace
+	sub("^" WHITESPACE "*", "", line)
+
+	if (match(line, "^#" WHITESPACE "*")) {
+		prefix = substr(line, RSTART, RLENGTH)
+		line = substr(line, RSTART + RLENGTH)
+	} else {
+		prefix = ""
+	}
+
+	line_type = "invalid"
+	option_is = value_is = ""
+
+	if (line) {
+		split(strdelim(line), toks, SUBSEP)
+
+		if (tolower(toks[1]) == "match") {
+			MATCH = (prefix ~ /^#/ ? "#" : "") join(" ", toks, 2)
+			line_type = "match"
+		} else if (toks[1] ~ /^[A-Za-z][A-Za-z0-9]+$/) {
+			# This could be an option line
+			line_type = "option"
+			option_is = tolower(toks[1])
+			value_is = toks[2]
+		}
+	} else {
+		line_type = "empty"
+	}
+}
+
+# mode: unset
+
+!mode {
+	# delete matching config
+	if (prefix !~ /^#/)
+		if (MATCH == match_only && option_is == option_should)
+			if (!value_should || value_should == value_is)
+				next
+
+	print
+	next
+}
+
+
+# mode: set
+
+mode && NR == FNR {
+	if (line_type == "option") {
+		if (MATCH !~ /^#/) {
+			if (prefix ~ /^#/) {
+				# comment line
+				last_occ[MATCH, "#" option_is] = FNR
+			} else {
+				# option line
+				last_occ[MATCH, option_is] = FNR
+			}
+			last_occ[MATCH] = FNR
+		}
+	} else if (line_type == "invalid" && !prefix) {
+		# INVALID LINE
+		print_err(sprintf("%s: syntax error on line %u\n", ARGV[0], FNR))
+	}
+
+	next
+}
+
+# before second pass prepare hashes containing location information to be used
+# in the second pass.
+mode && NR > FNR && FNR == 1 {
+	# First we drop the locations of commented-out options if a non-commented
+	# option is available. If a non-commented option is available, we will
+	# append new config options there to have them all at one place.
+	for (k in last_occ) {
+		if (k ~ /^#/) {
+			# delete entries of commented out match blocks
+			delete last_occ[k]
+			continue
+		}
+
+		split(k, parts, SUBSEP)
+
+		if (parts[2] ~ /^#/ && ((parts[1], substr(parts[2], 2)) in last_occ))
+			delete last_occ[k]
+	}
+
+	# Reverse the option => line mapping. The line_map allows for easier lookups
+	# in the second pass.
+	# We only keep options, not top-level keywords, because we can only have
+	# one entry per line and there are conflicts with last lines of "sections".
+	for (k in last_occ) {
+		if (!index(k, SUBSEP)) continue
+		line_map[last_occ[k]] = k
+	}
+}
+
+# Second pass
+mode && line_map[FNR] == match_only SUBSEP option_should && !updated {
+	split(line_map[FNR], parts, SUBSEP)
+
+	# If option allows multiple values, print current value
+	if (!singleton_option(parts[2])) {
+		if (value_should != value_is)
+			print
+	}
+
+	print_update()
+
+	next
+}
+
+mode { print }
+
+# Is a comment option
+mode && line_map[FNR] == match_only SUBSEP "#" option_should && !updated {
+	print_update()
+}
+
+# Last line of the should match section
+mode && last_occ[match_only] == FNR && !updated {
+	# NOTE: Inserting empty lines is only cosmetic.  It is only done if
+	#       different options are next to each other and not in a match block
+	#       (match blocks are usually not in the default config and thus don’t
+	#       contain commented blocks.)
+	if (line && option_is != option_should && !MATCH)
+		print ""
+	print_update()
+}
+
+END {
+	if (e) exit e
+
+	if (mode && !updated) {
+		if (match_only && MATCH != match_only) {
+			printf "\nMatch %s\n", match_only
+		}
+
+		print_update()
+	}
+}
diff --git a/cdist/conf/type/__sshd_config/gencode-remote b/cdist/conf/type/__sshd_config/gencode-remote
new file mode 100755
index 00000000..0b44dfa7
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/gencode-remote
@@ -0,0 +1,97 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+joinlines() { sed -n -e H -e "\${x;s/^\\n//;s/\\n/${1:?}/g;p;}"; }
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+if test "${state_is}" = "${state_should}" -o "${state_is}" = 'default'
+then
+	# nothing to do (if the value is the default, ignore its state)
+	exit 0
+fi
+
+case ${state_should}
+in
+	(present)
+		mode='set'
+		;;
+	(absent)
+		mode='unset'
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
+
+sshd_config_file=$(cat "${__object:?}/parameter/file")
+
+quote() { printf "'%s'" "$(printf '%s' "$*" | sed -e "s/'/'\\\\''/g")"; }
+drop_awk_comments() { quote "$(sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@")"; }
+
+# Ensure the sshd_config file is there
+cat <<EOF
+test -e $(quote "${sshd_config_file}") || {
+	 : >$(quote "${sshd_config_file}")
+	 chown 0:0 $(quote "${sshd_config_file}")
+	 chmod 0644 $(quote "${sshd_config_file}")
+}
+
+EOF
+
+match_only=
+if test -s "${__object:?}/parameter/match"
+then
+	match_only=$(joinlines ' ' <"${__object:?}/parameter/match")
+fi
+
+if test -s "${__object:?}/parameter/option"
+then
+	option_line=$(cat "${__object:?}/parameter/option")
+else
+	option_line=${__object_id:?}
+fi
+
+if test -s "${__object:?}/parameter/value"
+then
+	option_line="${option_line} $(cat "${__object:?}/parameter/value")"
+fi
+
+# Send message on config update
+printf '%s%s %s\n' "${mode}" "${match_only:+ [${match_only}]}" \
+	"${option_line}" >>"${__messages_out:?}"
+
+# Update sshd_config (remote code)
+cat <<EOF
+awk $(drop_awk_comments "${__type:?}/files/update_sshd_config.awk") \\
+	-o ${mode} \\
+	-m $(quote "${match_only}") \\
+	-l $(quote "${option_line}") \\
+	$(quote "${sshd_config_file}") >$(quote "${sshd_config_file}.tmp") \\
+|| exit
+
+cmp -s $(quote "${sshd_config_file}") $(quote "${sshd_config_file}.tmp") || {
+	sshd -t -f $(quote "${sshd_config_file}.tmp") \\
+	&& cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}")
+}
+rm -f $(quote "${sshd_config_file}.tmp")
+EOF
diff --git a/cdist/conf/type/__sshd_config/man.rst b/cdist/conf/type/__sshd_config/man.rst
new file mode 100644
index 00000000..8b0069ac
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/man.rst
@@ -0,0 +1,94 @@
+cdist-type__sshd_config(7)
+==========================
+
+NAME
+----
+cdist-type__sshd_config - Manage options in sshd_config
+
+
+DESCRIPTION
+-----------
+This space intentionally left blank.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+file
+    The path to the sshd_config file to edit.
+    Defaults to ``/etc/ssh/sshd_config``.
+match
+    Restrict this option to apply only for certain connections.
+    Allowed values are what would be allowed to be written after a ``Match``
+    keyword in ``sshd_config``, e.g. ``--match 'User anoncvs'``.
+
+    Can be used multiple times. All of the values are ANDed together.
+option
+    The name of the option to manipulate. Defaults to ``__object_id``.
+state
+    Can be:
+
+    - ``present``: ensure a matching config line is present (or the default
+      value).
+    - ``absent``: ensure no matching config line is present.
+value
+    The option's value to be assigned to the option (if ``--state present``) or
+    removed (if ``--state absent``).
+
+    This option is required if ``--state present``. If not specified and
+    ``--state absent``, all values for the given option are removed.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Disallow root logins with password
+    __sshd_config PermitRootLogin --value without-password
+
+    # Disallow password-based authentication
+    __sshd_config PasswordAuthentication --value no
+
+    # Accept the EDITOR environment variable
+    __sshd_config AcceptEnv:EDITOR --option AcceptEnv --value EDITOR
+
+    # Force command for connections as git user
+    __sshd_config git@ForceCommand --match 'User git' --option ForceCommand \
+        --value 'cd ~git && exec git-shell ${SSH_ORIGINAL_COMMAND:+-c "${SSH_ORIGINAL_COMMAND}"}'
+
+
+SEE ALSO
+--------
+:strong:`sshd_config`\ (5)
+
+
+BUGS
+----
+- This type assumes a nicely formatted config file,
+  i.e. no config options spanning multiple lines.
+- ``Include`` directives are ignored.
+- Config options are not added/removed to/from the config file if their value is
+  the default value.
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest
new file mode 100755
index 00000000..566bde90
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/manifest
@@ -0,0 +1,48 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$(cat "${__global:?}/explorer/os")
+
+state_should=$(cat "${__object:?}/parameter/state")
+
+case ${os}
+in
+	(alpine|centos|fedora|redhat|scientific|debian|devuan|ubuntu)
+		if test "${state_should}" != 'absent'
+		then
+			__package openssh-server --state present
+		fi
+		;;
+	(archlinux|gentoo|slackware|suse)
+		if test "${state_should}" != 'absent'
+		then
+			__package openssh --state present
+		fi
+		;;
+	(freebsd|netbsd|openbsd)
+		# whitelist
+		;;
+	(*)
+		printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \
+			"${os}" "${__type##*/}" >&2
+		printf 'Please contribute an implementation for it if you can.\n' >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__sshd_config/parameter/default/file b/cdist/conf/type/__sshd_config/parameter/default/file
new file mode 100644
index 00000000..d8ea5dfc
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/default/file
@@ -0,0 +1 @@
+/etc/ssh/sshd_config
diff --git a/cdist/conf/type/__sshd_config/parameter/default/state b/cdist/conf/type/__sshd_config/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__sshd_config/parameter/optional b/cdist/conf/type/__sshd_config/parameter/optional
new file mode 100644
index 00000000..922ab093
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/optional
@@ -0,0 +1,4 @@
+file
+option
+state
+value
diff --git a/cdist/conf/type/__sshd_config/parameter/optional_multiple b/cdist/conf/type/__sshd_config/parameter/optional_multiple
new file mode 100644
index 00000000..02b1d1a9
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/optional_multiple
@@ -0,0 +1 @@
+match

From 82eadb69948e631041263938994be5aeae161ab0 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 19 Nov 2020 19:34:43 +0100
Subject: [PATCH 308/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index ff411a46..331acbeb 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,8 @@ next:
 	* Type __locale: Add state explorer (Matthias Stecher)
 	* Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera)
 	* New type: __hwclock (Dennis Camera)
+	* Type __hostname: Fix guessing SuSE OS version (Dennis Camera)
+	* New type: __sshd_config (Dennis Camera)
 
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)

From 803a9d62a7a6b7fd11844f6ea273f4b692c2334e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 20 Nov 2020 19:46:03 +0100
Subject: [PATCH 309/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 331acbeb..cd5dbb4f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,8 @@ next:
 	* New type: __hwclock (Dennis Camera)
 	* Type __hostname: Fix guessing SuSE OS version (Dennis Camera)
 	* New type: __sshd_config (Dennis Camera)
+	* New type: __localedef (Dennis Camera)
+	* Type __locale: Deprecate in favor of __localedef (Dennis Camera)
 
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)

From 23e0da521c785715357b27d66007bd03fe173bf3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 20 Nov 2020 19:46:55 +0100
Subject: [PATCH 310/527] Release 6.9.2

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index cd5dbb4f..9a91bf57 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.2: 2020-11-20
 	* Documentation: Fix examples in best practice (Dennis Camera)
 	* Type __locale: Add state explorer (Matthias Stecher)
 	* Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera)

From 84172550df9e1800bc795dc51da01c9d3a0be9e0 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 30 Nov 2020 20:29:51 +0100
Subject: [PATCH 311/527] __iptables*: add IPv6 support

Because it currently only support IPv4. To implement this, it falls back
to IPv4 for backward compatibilty, but now supports rules for IPv6 and
both protocols at the same time.
---
 .../type/__iptables_apply/files/init-script   | 84 +++++++++++++++----
 cdist/conf/type/__iptables_rule/man.rst       | 38 ++++++++-
 cdist/conf/type/__iptables_rule/manifest      | 40 ++++++++-
 .../type/__iptables_rule/parameter/boolean    |  3 +
 4 files changed, 140 insertions(+), 25 deletions(-)
 create mode 100644 cdist/conf/type/__iptables_rule/parameter/boolean

diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script
index d9c79ef7..196f019b 100644
--- a/cdist/conf/type/__iptables_apply/files/init-script
+++ b/cdist/conf/type/__iptables_apply/files/init-script
@@ -1,6 +1,27 @@
 #!/bin/sh
-# Nico Schottelius
-# Zürisee, Mon Sep  2 18:38:27 CEST 2013
+#
+# 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is distributed with cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Originally written by:
+#   Nico Schottelius
+#   Zürisee, Mon Sep  2 18:38:27 CEST 2013
 #
 ### BEGIN INIT INFO
 # Provides:          iptables
@@ -15,33 +36,60 @@
 ### END INIT INFO
 
 
+# Read files and execute the content with the given commands
+#
+# Arguments:
+#  1:    Directory
+#  2..n: Commands which should be used to execute the file content
+gothrough() {
+    cd "$1" || return
+    shift
+
+    # iterate through all rules and continue if it's not a file
+    for rule in *; do
+        [ -f "$rule" ] || continue
+        echo "Appling iptables rule $rule ..."
+
+        # execute it with all commands specificed
+        ruleparam="$(cat "$rule")"
+        for cmd in "$@"; do
+            # Command and Rule should be split.
+            # shellcheck disable=SC2046
+            command $cmd $ruleparam
+        done
+    done
+}
+
+# Shortcut for iptables command to do IPv4 and v6
+iptables() {
+    command iptables "$@"
+    command ip6tables "$@"
+}
+
 basedir=/etc/iptables.d
-status="${basedir}/.pre-start"
+status4="${basedir}/.pre-start"
+status6="${basedir}/.pre-start6"
 
 case $1 in
     start)
         # Save status
-        iptables-save > "$status"
+        iptables-save > "$status4"
+        ip6tables-save > "$status6"
 
         # Apply our ruleset
-        cd "$basedir" || exit
-        count="$(find . ! -name . -prune | wc -l)"
-
-        # Only do something if there are rules
-        if [ "$count" -ge 1 ]; then
-            for rule in *; do
-                echo "Applying iptables rule $rule ..."
-                # Rule should be split.
-                # shellcheck disable=SC2046
-                iptables $(cat "$rule")
-            done
-        fi
+        gothrough "$basedir"     iptables
+        #gothrough "$basedir/v4"  iptables  # conflicts with $basedir
+        gothrough "$basedir/v6"  ip6tables
+        gothrough "$basedir/all" iptables  ip6tables
     ;;
 
     stop)
         # Restore from status before, if there is something to restore
-        if [ -f "$status" ]; then
-            iptables-restore < "$status"
+        if [ -f "$status4" ]; then
+            iptables-restore < "$status4"
+        fi
+        if [ -f "$status6" ]; then
+            ip6tables-restore < "$status6"
         fi
     ;;
     restart)
diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst
index 92d8859f..75d0740b 100644
--- a/cdist/conf/type/__iptables_rule/man.rst
+++ b/cdist/conf/type/__iptables_rule/man.rst
@@ -25,6 +25,24 @@ state
    'present' or 'absent', defaults to 'present'
 
 
+BOOLEAN PARAMETERS
+------------------
+All rules without any of this parameter will be threaten like ``--v4`` because
+of backward compatibility.
+
+v4
+    Explicitly set it as rule for IPv4. If IPv6 is set, too, it will be
+    threaten like ``--all``. Will be the default if nothing else is set.
+
+v6
+    Explicitly set it as rule for IPv6. If IPv4 is set, too, it will be
+    threaten like ``--all``.
+
+all
+    Set the rule for both IPv4 and IPv6. It will be saved separately from the
+    other rules.
+
+
 EXAMPLES
 --------
 
@@ -48,6 +66,16 @@ EXAMPLES
         --state absent
 
 
+    # IPv4-only rule for ICMPv4
+    __iptables_rule icmp-v4 --v4 --rule "-A INPUT -p icmp -j ACCEPT"
+    # IPv6-only rule for ICMPv6
+    __iptables_rule icmp-v6 --v6 --rule "-A INPUT -p icmpv6 -j ACCEPT"
+
+    # doing something for the dual stack
+    __iptables_rule fwd-eth0-eth1 --v4 --v6 --rule "-A INPUT -i eth0 -o eth1 -j ACCEPT"
+    __iptables_rule fwd-eth1-eth0 --all --rule "-A -o eth1 -i eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT"
+
+
 SEE ALSO
 --------
 :strong:`cdist-type__iptables_apply`\ (7), :strong:`iptables`\ (8)
@@ -56,11 +84,13 @@ SEE ALSO
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2013 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2013 Nico Schottelius.
+Copyright \(C) 2020 Matthias Stecher.
+You can redistribute it and/or modify it under the terms of the GNU
+General Public License as published by the Free Software Foundation,
+either version 3 of the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest
index ed78787f..27d00024 100755
--- a/cdist/conf/type/__iptables_rule/manifest
+++ b/cdist/conf/type/__iptables_rule/manifest
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
 #
 # This file is part of cdist.
 #
@@ -24,12 +25,36 @@ base_dir=/etc/iptables.d
 name="$__object_id"
 state="$(cat "$__object/parameter/state")"
 
+if [ -f "$__object/parameter/v4" ]; then
+    only_v4="yes"
+    # $specific_dir is $base_dir
+fi
+if [ -f "$__object/parameter/v6" ]; then
+    only_v6="yes"
+    specific_dir="$base_dir/v6"
+fi
+# If rules should be set for both protocols
+if ([ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]) \
+    || [ -f "$__object/parameter/all" ]; then
+
+    # all to a specific directory
+    specific_dir="$base_dir/all"
+fi
+
+# set rule directory based on if it's the base or subdirectory
+rule_dir="${specific_dir:-$base_dir}"
+
 ################################################################################
 # Basic setup
 #
 
 __directory "$base_dir" --state present
 
+# sub-directory if required
+if [ "$specific_dir" ]; then
+    require="__directory/$base_dir" __directory "$specific_dir" --state present
+fi
+
 # Have apply do the real job
 require="$__object_name" __iptables_apply
 
@@ -37,6 +62,15 @@ require="$__object_name" __iptables_apply
 # The rule
 #
 
-require="__directory/$base_dir" __file "$base_dir/${name}" \
-    --source "$__object/parameter/rule" \
-    --state "$state"
+for dir in "$base_dir" "$base_dir/v6" "$base_dir/all"; do
+    # defaults to absent except the directory that should contain the file
+    if [ "$rule_dir" = "$dir" ]; then
+        curr_state="$state"
+    else
+        curr_state="absent"
+    fi
+
+    require="__directory/$rule_dir" __file "$dir/$name" \
+        --source "$__object/parameter/rule" \
+        --state "$curr_state"
+done
diff --git a/cdist/conf/type/__iptables_rule/parameter/boolean b/cdist/conf/type/__iptables_rule/parameter/boolean
new file mode 100644
index 00000000..76882272
--- /dev/null
+++ b/cdist/conf/type/__iptables_rule/parameter/boolean
@@ -0,0 +1,3 @@
+all
+v4
+v6

From f568462e4981c8b6437c29f96963c3d8e7bed742 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 2 Dec 2020 17:48:41 +0100
Subject: [PATCH 312/527] __iptables_rule: fix shellcheck SC2235

---
 cdist/conf/type/__iptables_rule/manifest | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest
index 27d00024..d4394c25 100755
--- a/cdist/conf/type/__iptables_rule/manifest
+++ b/cdist/conf/type/__iptables_rule/manifest
@@ -34,8 +34,8 @@ if [ -f "$__object/parameter/v6" ]; then
     specific_dir="$base_dir/v6"
 fi
 # If rules should be set for both protocols
-if ([ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]) \
-    || [ -f "$__object/parameter/all" ]; then
+if { [ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]; } ||
+    [ -f "$__object/parameter/all" ]; then
 
     # all to a specific directory
     specific_dir="$base_dir/all"

From bee255c1ae0039e3d532317007f06fcd2e64e3bc Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 2 Dec 2020 18:04:50 +0100
Subject: [PATCH 313/527] __iptables_apply: man updates

---
 .../type/__iptables_apply/files/init-script   |  1 +
 cdist/conf/type/__iptables_apply/man.rst      | 21 +++++++++++++------
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script
index 196f019b..7faa2f92 100644
--- a/cdist/conf/type/__iptables_apply/files/init-script
+++ b/cdist/conf/type/__iptables_apply/files/init-script
@@ -61,6 +61,7 @@ gothrough() {
 }
 
 # Shortcut for iptables command to do IPv4 and v6
+# only applies to the "reset" target
 iptables() {
     command iptables "$@"
     command ip6tables "$@"
diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst
index 76e1f6bf..db0e7869 100644
--- a/cdist/conf/type/__iptables_apply/man.rst
+++ b/cdist/conf/type/__iptables_apply/man.rst
@@ -10,7 +10,14 @@ DESCRIPTION
 -----------
 This cdist type deploys an init script that triggers
 the configured rules and also re-applies them on
-configuration.
+configuration. Rules are written from __iptables_rule
+into the folder ``/etc/iptables.d/``.
+
+It reads all rules from the base folder as rules for IPv4.
+Rules in the subfolder ``v6/`` are IPv6 rules. Rules in
+the subfolder ``all/`` are applied to both rule tables. All
+files contain the arguments for a single ``iptables`` and/or
+``ip6tables`` command.
 
 
 REQUIRED PARAMETERS
@@ -24,7 +31,7 @@ None
 EXAMPLES
 --------
 
-None (__iptables_apply is used by __iptables_rule)
+None (__iptables_apply is used by __iptables_rule automaticly)
 
 
 SEE ALSO
@@ -35,11 +42,13 @@ SEE ALSO
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2013 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2013 Nico Schottelius.
+Copyright \(C) 2020 Matthias Stecher.
+You can redistribute it and/or modify it under the terms of the GNU
+General Public License as published by the Free Software Foundation,
+either version 3 of the License, or (at your option) any later version.

From a1db5c3d0e7b5899f1a877c5bab28ac4d4796f8c Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 2 Dec 2020 18:22:31 +0100
Subject: [PATCH 314/527] __iptables*: Update manpages for execution order

To make some thinks clear if someone needs it ..
---
 cdist/conf/type/__iptables_apply/man.rst | 10 ++++++++++
 cdist/conf/type/__iptables_rule/man.rst  |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst
index db0e7869..4109e789 100644
--- a/cdist/conf/type/__iptables_apply/man.rst
+++ b/cdist/conf/type/__iptables_apply/man.rst
@@ -19,6 +19,16 @@ the subfolder ``all/`` are applied to both rule tables. All
 files contain the arguments for a single ``iptables`` and/or
 ``ip6tables`` command.
 
+Rules are applied in the following order:
+1. All IPv4 rules
+2. All IPv6 rules
+2. All rules that should be applied to both tables
+
+The order of the rules that will be applied are definite
+from the result the shell glob returns, which should be
+alphabetical. If rules must be applied in a special order,
+prefix them with a number like ``02-some-rule``.
+
 
 REQUIRED PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst
index 75d0740b..86d38a34 100644
--- a/cdist/conf/type/__iptables_rule/man.rst
+++ b/cdist/conf/type/__iptables_rule/man.rst
@@ -11,6 +11,10 @@ DESCRIPTION
 This cdist type allows you to manage iptable rules
 in a distribution independent manner.
 
+See :strong:`cdist-type__iptables_apply`\ (7) for the
+execution order of these rules. It will be executed
+automaticly to apply all rules non-volaite.
+
 
 REQUIRED PARAMETERS
 -------------------

From 1055e92545e74cf7845ede8094e783b4103efbcc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 2 Dec 2020 19:54:41 +0100
Subject: [PATCH 315/527] [setup.py] Add cdist.scan to packages

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 858c2c17..bfc8b495 100644
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@ os.chdir(cur)
 
 setup(
     name="cdist",
-    packages=["cdist", "cdist.core", "cdist.exec", "cdist.util", ],
+    packages=["cdist", "cdist.core", "cdist.exec", "cdist.scan", "cdist.util"],
     package_data={'cdist': package_data},
     scripts=["bin/cdist", "bin/cdist-dump", "bin/cdist-new-type"],
     version=cdist.version.VERSION,

From c7fa2efe6b780d63a779eed310bfa4ebf926d27e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 4 Dec 2020 15:30:08 +0100
Subject: [PATCH 316/527] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 9a91bf57..ad0aa75d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)
+
 6.9.2: 2020-11-20
 	* Documentation: Fix examples in best practice (Dennis Camera)
 	* Type __locale: Add state explorer (Matthias Stecher)

From d44b5cfdc990ce020ffe3af5085bbb93b8ce64a8 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 4 Dec 2020 15:31:35 +0100
Subject: [PATCH 317/527] Release 6.9.3

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index ad0aa75d..6c272fd7 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)
 
 6.9.2: 2020-11-20

From ba7d16a155cef46230d1dc650119f3d542a7f7f4 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 4 Dec 2020 17:57:55 +0100
Subject: [PATCH 318/527] __iptables_*: correct manpage spelling

---
 cdist/conf/type/__iptables_apply/man.rst | 2 +-
 cdist/conf/type/__iptables_rule/man.rst  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst
index 4109e789..3bef92cc 100644
--- a/cdist/conf/type/__iptables_apply/man.rst
+++ b/cdist/conf/type/__iptables_apply/man.rst
@@ -41,7 +41,7 @@ None
 EXAMPLES
 --------
 
-None (__iptables_apply is used by __iptables_rule automaticly)
+None (__iptables_apply is used by __iptables_rule automatically)
 
 
 SEE ALSO
diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst
index 86d38a34..afb71e01 100644
--- a/cdist/conf/type/__iptables_rule/man.rst
+++ b/cdist/conf/type/__iptables_rule/man.rst
@@ -31,7 +31,7 @@ state
 
 BOOLEAN PARAMETERS
 ------------------
-All rules without any of this parameter will be threaten like ``--v4`` because
+All rules without any of these parameters will be treated like ``--v4`` because
 of backward compatibility.
 
 v4

From 2d19856840400af57bd6667cc868ab314a3e07c2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 4 Dec 2020 18:26:03 +0100
Subject: [PATCH 319/527] [type/__package_pkgng_freebsd] Set ASSUME_ALWAYS_YES
 instead of -y

---
 cdist/conf/type/__package_pkgng_freebsd/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
index b5944177..05ba4cb2 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
+++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
@@ -75,7 +75,7 @@ execcmd(){
    esac
 
    if [ -z "${pkg_bootstrapped}" ]; then
-       echo "pkg bootstrap -y >/dev/null 2>&1"
+       echo "ASSUME_ALWAYS_YES=yes pkg bootstrap >/dev/null 2>&1"
    fi
 
    echo "$_cmd >/dev/null 2>&1"   # Silence the output of the command

From 087be130fa67d3fe195387ae0ab079f39c5066e1 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 4 Dec 2020 19:23:49 +0100
Subject: [PATCH 320/527] __iptables_apply: shorten copyright header

Do we need all the copyright header or is this sufficient? The licence
is given for cdist, but not on the target host. But it should be clear
anyway.
---
 .../type/__iptables_apply/files/init-script   | 34 ++++++-------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script
index 7faa2f92..e42017ae 100644
--- a/cdist/conf/type/__iptables_apply/files/init-script
+++ b/cdist/conf/type/__iptables_apply/files/init-script
@@ -1,28 +1,4 @@
 #!/bin/sh
-#
-# 2013 Nico Schottelius (nico-cdist at schottelius.org)
-# 2020 Matthias Stecher (matthiasstecher at gmx.de)
-#
-# This file is distributed with cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Originally written by:
-#   Nico Schottelius
-#   Zürisee, Mon Sep  2 18:38:27 CEST 2013
-#
 ### BEGIN INIT INFO
 # Provides:          iptables
 # Required-Start:    $local_fs $remote_fs
@@ -35,6 +11,16 @@
 #                    and saves/restores previous status
 ### END INIT INFO
 
+# Originally written by:
+#   Nico Schottelius
+#   Zürisee, Mon Sep  2 18:38:27 CEST 2013
+#
+# 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is distributed with cdist and licenced under the
+# GNU GPLv3+ WITHOUT ANY WARRANTY.
+
 
 # Read files and execute the content with the given commands
 #

From 3930f69456fd5a0d108a107e4ee61d87c9a73a56 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 6 Dec 2020 16:45:58 +0100
Subject: [PATCH 321/527] __block: fix escaping in here-doc

This changes the here-document to do not interpret any shell-things. It
also single-quotes some more strings that are printed to code-remote.

Fixes #838
---
 cdist/conf/type/__block/gencode-remote | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__block/gencode-remote b/cdist/conf/type/__block/gencode-remote
index 1f5cc033..acdb3286 100755
--- a/cdist/conf/type/__block/gencode-remote
+++ b/cdist/conf/type/__block/gencode-remote
@@ -46,10 +46,10 @@ fi
 
 remove_block() {
    cat << DONE
-tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
+tmpfile=\$(mktemp '${file}.cdist.XXXXXXXXXX')
 # preserve ownership and permissions of existing file
-if [ -f "$file" ]; then
-   cp -p "$file" "\$tmpfile"
+if [ -f '$file' ]; then
+   cp -p '$file' "\$tmpfile"
 fi
 awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
 {
@@ -63,8 +63,8 @@ awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
    } else {
       print
    }
-}' "$file" > "\$tmpfile"
-mv -f "\$tmpfile" "$file"
+}' '$file' > "\$tmpfile"
+mv -f "\$tmpfile" '$file'
 DONE
 }
 
@@ -77,7 +77,7 @@ case "$state_should" in
          echo add >> "$__messages_out"
       fi
       cat << DONE
-cat >> "$file" << ${__type##*/}_DONE
+cat >> '$file' << '${__type##*/}_DONE'
 $(cat "$block")
 ${__type##*/}_DONE
 DONE

From bed08c2c5c71ec4252743e0b64b75e7d8fedbd92 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sun, 6 Dec 2020 20:24:00 +0100
Subject: [PATCH 322/527] Deal with deprecation of imp module.

importlib has been a thing since Python 3.1, and imp has been deprecated since
3.4.

Insert random complaint here about not being able to use f-strings because they
were introduced in Python 3.6 and apparently we support Python 3.5 >,<.

Output diff before to after for ./bin/cdist-build-helper test (on heavy load):
```
1,2d0
< /usr/home/evilham/s/cdist/cdist/cdist/test/__main__.py:23: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
<   import imp
72c70
< ERROR: cdisttesthost: __file/tmp/foobar requires object __file without object id. Defined at /tmp/tmp.cdist.test.g87lx7c8/tmp.cdist.test.6ramsakx
---
> ERROR: cdisttesthost: __file/tmp/foobar requires object __file without object id. Defined at /tmp/tmp.cdist.test.aqdf6vjz/tmp.cdist.test.jgv3udel
76c74
< test_nonexistent_type_requirement (cdist.test.emulator.EmulatorTestCase) ... ERROR: cdisttesthost: __file/tmp/foobar requires object __does-not-exist/some-id, but type __does-not-exist does not exist. Defined at /tmp/tmp.cdist.test.mma5j8ln/tmp.cdist.test.3zg4by4d
---
> test_nonexistent_type_requirement (cdist.test.emulator.EmulatorTestCase) ... ERROR: cdisttesthost: __file/tmp/foobar requires object __does-not-exist/some-id, but type __does-not-exist does not exist. Defined at /tmp/tmp.cdist.test.t8d6ockr/tmp.cdist.test.uimxurg9
86c84
< test_initial_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running initial manifest /tmp/tmp.cdist.test.uvid60ij/759547ff4356de6e3d9e08522b0d0807/data/conf/manifest/dump_environment
---
> test_initial_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running initial manifest /tmp/tmp.cdist.test._cttcnrj/759547ff4356de6e3d9e08522b0d0807/data/conf/manifest/dump_environment
89c87
< test_type_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running type manifest /tmp/tmp.cdist.test.k1i2onpb/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__dump_environment/manifest for object __dump_environment/whatever
---
> test_type_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running type manifest /tmp/tmp.cdist.test.ukr7lrzd/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__dump_environment/manifest for object __dump_environment/whatever
272c270
< Ran 225 tests in 44.457s
---
> Ran 225 tests in 43.750s
```
---
 cdist/test/__main__.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cdist/test/__main__.py b/cdist/test/__main__.py
index c8c7df3b..8049c752 100644
--- a/cdist/test/__main__.py
+++ b/cdist/test/__main__.py
@@ -20,7 +20,7 @@
 #
 #
 
-import imp
+import importlib
 import os
 import sys
 import unittest
@@ -37,8 +37,9 @@ for possible_test in os.listdir(base_dir):
 
 suites = []
 for test_module in test_modules:
-    module_parameters = imp.find_module(test_module, [base_dir])
-    module = imp.load_module("cdist.test." + test_module, *module_parameters)
+    module_spec = importlib.util.find_spec("cdist.test.{}".format(test_module))
+    module = importlib.util.module_from_spec(module_spec)
+    module_spec.loader.exec_module(module)
 
     suite = unittest.defaultTestLoader.loadTestsFromModule(module)
     # print("Got suite: " + suite.__str__())

From 29662961731ef07fbc20213d0d87da1469f0d27c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 7 Dec 2020 19:47:52 +0100
Subject: [PATCH 323/527] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 6c272fd7..8b35901b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
+	* Core: Deal with deprecated imp in unit tests (Evil Ham)
+
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)
 

From c5ca4cd2e13516dfb55371c1600e32297c3343e9 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 7 Dec 2020 19:59:05 +0100
Subject: [PATCH 324/527] __block: securly quote via the quote function

Because the function already exists, it will be used for the file to be
changed, too. Therefor, no quotes are required for that value.

The prefix and suffix match was also improved: There is no regex check
any more (the regex did checked the whole line); instead it will do it
simple.
---
 cdist/conf/type/__block/gencode-remote | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__block/gencode-remote b/cdist/conf/type/__block/gencode-remote
index acdb3286..7a1f4064 100755
--- a/cdist/conf/type/__block/gencode-remote
+++ b/cdist/conf/type/__block/gencode-remote
@@ -46,28 +46,29 @@ fi
 
 remove_block() {
    cat << DONE
-tmpfile=\$(mktemp '${file}.cdist.XXXXXXXXXX')
+tmpfile=\$(mktemp ${quoted_file}.cdist.XXXXXXXXXX)
 # preserve ownership and permissions of existing file
-if [ -f '$file' ]; then
-   cp -p '$file' "\$tmpfile"
+if [ -f $quoted_file ]; then
+   cp -p $quoted_file "\$tmpfile"
 fi
-awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
+awk -v prefix=$(quote "$prefix") -v suffix=$(quote "$suffix") '
 {
-   if (match(\$0,prefix)) {
+   if (\$0 == prefix) {
       triggered=1
    }
    if (triggered) {
-      if (match(\$0,suffix)) {
+      if (\$0 == suffix) {
          triggered=0
       }
    } else {
       print
    }
-}' '$file' > "\$tmpfile"
-mv -f "\$tmpfile" '$file'
+}' $quoted_file > "\$tmpfile"
+mv -f "\$tmpfile" $quoted_file
 DONE
 }
 
+quoted_file="$(quote "$file")"
 case "$state_should" in
    present)
       if [ "$state_is" = "changed" ]; then
@@ -77,7 +78,7 @@ case "$state_should" in
          echo add >> "$__messages_out"
       fi
       cat << DONE
-cat >> '$file' << '${__type##*/}_DONE'
+cat >> $quoted_file << '${__type##*/}_DONE'
 $(cat "$block")
 ${__type##*/}_DONE
 DONE

From 14c81d6c7e20a2ed04c7318f6e8ecb51098390e8 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 8 Dec 2020 07:16:26 +0100
Subject: [PATCH 325/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8b35901b..4be08ae9 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
+	* Type __iptables: Add IPv6 support (Matthias Stecher)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From a58f5ffa7f59a7e28ba0a414097f154e4a497e88 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 8 Dec 2020 19:36:44 +0100
Subject: [PATCH 326/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 4be08ae9..3d6084f1 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
 	* Type __iptables: Add IPv6 support (Matthias Stecher)
+	* Type __block: Fix escaping in here-doc (Matthias Stecher)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From a5169ad858a4d6e9c378184db5736f55a86306e0 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 10 Dec 2020 21:24:26 +0100
Subject: [PATCH 327/527] new type __debian_backports

This new type will setup the backports distribution for the current
Debian release.
---
 cdist/conf/type/__debian_backports/man.rst    | 90 +++++++++++++++++++
 cdist/conf/type/__debian_backports/manifest   | 59 ++++++++++++
 .../parameter/default/mirror                  |  1 +
 .../parameter/default/state                   |  1 +
 .../__debian_backports/parameter/optional     |  2 +
 cdist/conf/type/__debian_backports/singleton  |  0
 6 files changed, 153 insertions(+)
 create mode 100644 cdist/conf/type/__debian_backports/man.rst
 create mode 100755 cdist/conf/type/__debian_backports/manifest
 create mode 100644 cdist/conf/type/__debian_backports/parameter/default/mirror
 create mode 100644 cdist/conf/type/__debian_backports/parameter/default/state
 create mode 100644 cdist/conf/type/__debian_backports/parameter/optional
 create mode 100644 cdist/conf/type/__debian_backports/singleton

diff --git a/cdist/conf/type/__debian_backports/man.rst b/cdist/conf/type/__debian_backports/man.rst
new file mode 100644
index 00000000..ba353f4e
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/man.rst
@@ -0,0 +1,90 @@
+cdist-type__debian_backports(7)
+===============================
+
+NAME
+----
+cdist-type__debian_backports - Install backports for Debain systems
+
+
+DESCRIPTION
+-----------
+This singleton type installs backports for the current Debian version.
+It aborts if backports are not supported for the specified os or no
+version codename could be fetched (like Debian unstable).
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    Represents the state of the backports repository. ``present`` or
+    ``absent``, defaults to ``present``.
+
+    Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
+
+mirror
+    The mirror to fetch the backports from. Will defaults to the Debian default
+    `<http://deb.debian.org/debian/>`_.
+
+    Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+MESSAGES
+--------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+   # setup the backports
+   __debian_backports
+   __debian_backports --state absent
+   __debian_backports --state present --mirror "http://ftp.de.debian.org/debian/"
+
+   # update
+   require="__debian_backports" __apt_update_index
+
+   # install a backports package
+   # currently for the buster release backports
+   require="__apt_update_index" __package_apt wireguard \
+        --target-release buster-backports
+
+
+ABORTS
+------
+Aborts if the detected os is not Debian.
+
+Aborts if no distribuition codename could be detected. This is common for the
+unstable distribution, but there is no backports repository for it already.
+
+
+SEE ALSO
+--------
+`Official Debian Backports site <https://backports.debian.org/>`_
+
+:strong:`cdist-type__apt_source`\ (7)
+
+
+AUTHORS
+-------
+Matthias Stecher <matthiasstecher at gmx.de>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Matthias Stecher. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__debian_backports/manifest
new file mode 100755
index 00000000..29bf9a43
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/manifest
@@ -0,0 +1,59 @@
+#!/bin/sh -e
+# __debian_backports/manifest
+#
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Enables/disables backports repository. Utilies __apt_source for it.
+#
+
+
+# detect backport distribution
+os="$(cat "$__global/explorer/os")"
+case "$os" in
+    debian)
+        # distribution codename from /etc/os-release
+        # lsb_release may not be given in all debian installations
+        dist="$(
+            . "$__global/explorer/os-release"
+            printf "%s" "$VERSION_CODENAME"
+        )"
+        ;;
+    *)
+        printf "Backports for %s are not supported!\n" "$os" >&2
+        exit 1
+        ;;
+esac
+
+# error if no codename given (e.g. on Debian unstable)
+if [ -z "$dist" ]; then
+    printf "No backports for unkown version of distribution %s!\n" "$os" >&2
+    exit 1
+fi
+
+
+# parameters
+state="$(cat "$__object/parameter/state")"
+mirror="$(cat "$__object/parameter/mirror")"
+
+# install the given backports repository
+__apt_source "${dist}-backports" \
+    --state "$state" \
+    --distribution "${dist}-backports" \
+    --component main \
+    --uri "$mirror"
diff --git a/cdist/conf/type/__debian_backports/parameter/default/mirror b/cdist/conf/type/__debian_backports/parameter/default/mirror
new file mode 100644
index 00000000..0965ef04
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/parameter/default/mirror
@@ -0,0 +1 @@
+http://deb.debian.org/debian/
diff --git a/cdist/conf/type/__debian_backports/parameter/default/state b/cdist/conf/type/__debian_backports/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__debian_backports/parameter/optional b/cdist/conf/type/__debian_backports/parameter/optional
new file mode 100644
index 00000000..4b05c235
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/parameter/optional
@@ -0,0 +1,2 @@
+state
+mirror
diff --git a/cdist/conf/type/__debian_backports/singleton b/cdist/conf/type/__debian_backports/singleton
new file mode 100644
index 00000000..e69de29b

From 0d96b31b5696f95b559450e17c71914d0746c0ce Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 11 Dec 2020 18:13:44 +0100
Subject: [PATCH 328/527] __debian_backports: pass shellcheck for sourced file

Because the sourced explorer can't be detected by shellcheck, it will be
completely disabled. Changing the path to /etc/os-release isn't
deterministic either.

The shellcheck wiki page suggests to use `source=/dev/null` instead of
`disable=SC1090`, but it was choosen to completely avoid that check ..
---
 cdist/conf/type/__debian_backports/manifest | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__debian_backports/manifest
index 29bf9a43..661e5281 100755
--- a/cdist/conf/type/__debian_backports/manifest
+++ b/cdist/conf/type/__debian_backports/manifest
@@ -30,6 +30,7 @@ case "$os" in
         # distribution codename from /etc/os-release
         # lsb_release may not be given in all debian installations
         dist="$(
+            # shellcheck disable=SC1090
             . "$__global/explorer/os-release"
             printf "%s" "$VERSION_CODENAME"
         )"

From bc2948a8a552e3bda3ab9ae8e8e199e831d81af2 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 11 Dec 2020 19:37:53 +0100
Subject: [PATCH 329/527] ++scan stuff

---
 cdist/scan/scan.py           | 15 +++++++++++++++
 docs/dev/logs/2020-10-29.org | 10 +++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index fcbf1899..00858581 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -56,6 +56,8 @@ from scapy.all import *
 # Datetime overwrites scapy.all.datetime - needs to be imported AFTER
 import datetime
 
+import cdist.config
+
 log = logging.getLogger("scan")
 
 class Trigger(object):
@@ -120,6 +122,19 @@ class Scanner(object):
             with open(fname, "w") as fd:
                 fd.write(f"{now}\n")
 
+    def config(self):
+        """
+        Configure a host
+
+        - Assume we are only called if necessary
+        - However we need to ensure to not run in parallel
+        - Maybe keep dict storing per host processes
+        - Save the result
+        - Save the output -> probably aligned to config mode
+
+        """
+
+
     def start(self):
         self.process = Process(target=self.scan)
         self.process.start()
diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org
index 4461be8c..03d6b3f4 100644
--- a/docs/dev/logs/2020-10-29.org
+++ b/docs/dev/logs/2020-10-29.org
@@ -54,4 +54,12 @@ VERBOSE: scan: Host fe80::f29f:c2ff:fe7c:275e is alive
 VERBOSE: scan: Host fe80::ba69:f4ff:fec5:8db7 is alive
 VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f863 is alive
 VERBOSE: scan: Host fe80::21b:fcff:feee:f4bc is alive
-...
+** Better usage -> saving the env
+    sudo -E cdist scan -b -I wlan0 -vv
+** TODO Implement actual configuration step
+   - Also serves as a nice PoC
+   - Might need to escape literal IPv6 addresses for scp
+** TODO Define how to map link local address to something useful
+   - via reverse DNS?
+   - via link local in manifest?
+** TODO define ignorehosts?

From c4d19a23193ec13eda96a3d7536ba338d0801825 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 09:36:17 +0100
Subject: [PATCH 330/527] __debian_backports -> __apt_backports; add wider os
 support

As discussed in the chat, this type now supports a broader list of OSes
which it supports backports for. Because of this, it was renamed to
something more generic. "apt" should fit in.
---
 .../man.rst                                   | 27 ++++++++----
 .../manifest                                  | 41 ++++++++++++++-----
 .../parameter/default/state                   |  0
 .../parameter/optional                        |  0
 .../singleton                                 |  0
 .../parameter/default/mirror                  |  1 -
 6 files changed, 49 insertions(+), 20 deletions(-)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/man.rst (66%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/manifest (59%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/parameter/default/state (100%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/parameter/optional (100%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/singleton (100%)
 delete mode 100644 cdist/conf/type/__debian_backports/parameter/default/mirror

diff --git a/cdist/conf/type/__debian_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
similarity index 66%
rename from cdist/conf/type/__debian_backports/man.rst
rename to cdist/conf/type/__apt_backports/man.rst
index ba353f4e..7d269fbb 100644
--- a/cdist/conf/type/__debian_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -3,13 +3,13 @@ cdist-type__debian_backports(7)
 
 NAME
 ----
-cdist-type__debian_backports - Install backports for Debain systems
+cdist-type__apt_backports - Install backports
 
 
 DESCRIPTION
 -----------
-This singleton type installs backports for the current Debian version.
-It aborts if backports are not supported for the specified os or no
+This singleton type installs backports for the current OS release.
+It aborts if backports are not supported for the specified OS or no
 version codename could be fetched (like Debian unstable).
 
 
@@ -27,8 +27,8 @@ state
     Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
 
 mirror
-    The mirror to fetch the backports from. Will defaults to the Debian default
-    `<http://deb.debian.org/debian/>`_.
+    The mirror to fetch the backports from. Will defaults to the generic
+    mirror of the current OS.
 
     Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
 
@@ -49,12 +49,12 @@ EXAMPLES
 .. code-block:: sh
 
    # setup the backports
-   __debian_backports
-   __debian_backports --state absent
-   __debian_backports --state present --mirror "http://ftp.de.debian.org/debian/"
+   __apt_backports
+   __apt_backports --state absent
+   __apt_backports --state present --mirror "http://ftp.de.debian.org/debian/"
 
    # update
-   require="__debian_backports" __apt_update_index
+   require="__apt_backports" __apt_update_index
 
    # install a backports package
    # currently for the buster release backports
@@ -70,6 +70,15 @@ Aborts if no distribuition codename could be detected. This is common for the
 unstable distribution, but there is no backports repository for it already.
 
 
+CAVEATS
+-------
+For Ubuntu, it setup all componenents for the backports repository: ``main``,
+``restricted``, ``universe`` and ``multiverse``. The user may not want to
+install proprietary packages, which will only be installed if the user
+explicitly uses the backports target-release. The user may change this behavior
+to install backports packages without the need of explicitly select it.
+
+
 SEE ALSO
 --------
 `Official Debian Backports site <https://backports.debian.org/>`_
diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__apt_backports/manifest
similarity index 59%
rename from cdist/conf/type/__debian_backports/manifest
rename to cdist/conf/type/__apt_backports/manifest
index 661e5281..e5358dea 100755
--- a/cdist/conf/type/__debian_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -1,5 +1,5 @@
 #!/bin/sh -e
-# __debian_backports/manifest
+# __apt_backports/manifest
 #
 # 2020 Matthias Stecher (matthiasstecher at gmx.de)
 #
@@ -23,18 +23,34 @@
 #
 
 
+# Get the distribution codename by /etc/os-release.
+#  is already executed in a subshell by string substitution
+#  lsb_release may not be given in all installations
+codename_os_release() {
+    # shellcheck disable=SC1090
+    . "$__global/explorer/os-release"
+    printf "%s" "$VERSION_CODENAME"
+}
+
 # detect backport distribution
 os="$(cat "$__global/explorer/os")"
 case "$os" in
     debian)
-        # distribution codename from /etc/os-release
-        # lsb_release may not be given in all debian installations
-        dist="$(
-            # shellcheck disable=SC1090
-            . "$__global/explorer/os-release"
-            printf "%s" "$VERSION_CODENAME"
-        )"
+        dist="$( codename_os_release )"
+        components="main"
+        mirror="http://deb.debian.org/debian/"
         ;;
+    devuan)
+        dist="$( codename_os_release )"
+        components="main"
+        mirror="http://deb.devuan.org/merged"
+        ;;
+    ubuntu)
+        dist="$( codename_os_release )"
+        components="main restricted universe multiverse"
+        mirror="http://archive.ubuntu.com/ubuntu"
+        ;;
+
     *)
         printf "Backports for %s are not supported!\n" "$os" >&2
         exit 1
@@ -50,11 +66,16 @@ fi
 
 # parameters
 state="$(cat "$__object/parameter/state")"
-mirror="$(cat "$__object/parameter/mirror")"
+
+# mirror already set for the os, only override user-values
+if [ -f "$__object/parameter/mirror" ]; then
+    mirror="$(cat "$__object/parameter/mirror")"
+fi
+
 
 # install the given backports repository
 __apt_source "${dist}-backports" \
     --state "$state" \
     --distribution "${dist}-backports" \
-    --component main \
+    --component "$components" \
     --uri "$mirror"
diff --git a/cdist/conf/type/__debian_backports/parameter/default/state b/cdist/conf/type/__apt_backports/parameter/default/state
similarity index 100%
rename from cdist/conf/type/__debian_backports/parameter/default/state
rename to cdist/conf/type/__apt_backports/parameter/default/state
diff --git a/cdist/conf/type/__debian_backports/parameter/optional b/cdist/conf/type/__apt_backports/parameter/optional
similarity index 100%
rename from cdist/conf/type/__debian_backports/parameter/optional
rename to cdist/conf/type/__apt_backports/parameter/optional
diff --git a/cdist/conf/type/__debian_backports/singleton b/cdist/conf/type/__apt_backports/singleton
similarity index 100%
rename from cdist/conf/type/__debian_backports/singleton
rename to cdist/conf/type/__apt_backports/singleton
diff --git a/cdist/conf/type/__debian_backports/parameter/default/mirror b/cdist/conf/type/__debian_backports/parameter/default/mirror
deleted file mode 100644
index 0965ef04..00000000
--- a/cdist/conf/type/__debian_backports/parameter/default/mirror
+++ /dev/null
@@ -1 +0,0 @@
-http://deb.debian.org/debian/

From 49aec0b5e40ba05e7703565731aae6b0a9e5936a Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 09:40:47 +0100
Subject: [PATCH 331/527] __apt_backports: list supported OSes

The manpage now lists all OSes where this type supports backports.
---
 cdist/conf/type/__apt_backports/man.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
index 7d269fbb..fd311361 100644
--- a/cdist/conf/type/__apt_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -12,6 +12,12 @@ This singleton type installs backports for the current OS release.
 It aborts if backports are not supported for the specified OS or no
 version codename could be fetched (like Debian unstable).
 
+It supports backports from following OSes:
+
+- Debian
+- Devuan
+- Ubuntu
+
 
 REQUIRED PARAMETERS
 -------------------

From fafa3d9ea55f38ae0a350a463d64b7ef1d70c7bb Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 10:00:23 +0100
Subject: [PATCH 332/527] __apt_backports: update index if required

This type now automatically calls the type __apt_update_index to update
the package index if something changed.
---
 cdist/conf/type/__apt_backports/man.rst  | 13 +++++++------
 cdist/conf/type/__apt_backports/manifest |  5 ++++-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
index fd311361..c578ce6b 100644
--- a/cdist/conf/type/__apt_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -9,8 +9,11 @@ cdist-type__apt_backports - Install backports
 DESCRIPTION
 -----------
 This singleton type installs backports for the current OS release.
-It aborts if backports are not supported for the specified OS or no
-version codename could be fetched (like Debian unstable).
+It aborts if backports are not supported for the specified OS or
+no version codename could be fetched (like Debian unstable).
+
+The package index will be automatically updated by the type
+:strong:`cdist-type__apt_update_index`\ (7) if required.
 
 It supports backports from following OSes:
 
@@ -59,12 +62,9 @@ EXAMPLES
    __apt_backports --state absent
    __apt_backports --state present --mirror "http://ftp.de.debian.org/debian/"
 
-   # update
-   require="__apt_backports" __apt_update_index
-
    # install a backports package
    # currently for the buster release backports
-   require="__apt_update_index" __package_apt wireguard \
+   require="__apt_backports" __package_apt wireguard \
         --target-release buster-backports
 
 
@@ -90,6 +90,7 @@ SEE ALSO
 `Official Debian Backports site <https://backports.debian.org/>`_
 
 :strong:`cdist-type__apt_source`\ (7)
+:strong:`cdist-type__apt_update_index`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index e5358dea..c490a103 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -19,7 +19,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Enables/disables backports repository. Utilies __apt_source for it.
+# Enables/disables backports repository. Utilises __apt_source for it.
 #
 
 
@@ -79,3 +79,6 @@ __apt_source "${dist}-backports" \
     --distribution "${dist}-backports" \
     --component "$components" \
     --uri "$mirror"
+
+# update the index if the source changed
+require="__apt_source/${dist}-backports" __apt_update_index

From 645734c62959c694597826598be3b34c0edc79d2 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 12 Dec 2020 12:15:17 +0100
Subject: [PATCH 333/527] [explorer/os_version] Improve FreeBSD support.

It looks like uname -r is not the most reliable way to get the target patch
level for the target system.

For more information see:
https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
---
 cdist/conf/explorer/os_version | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index a7b1d3bc..3b02dedd 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -70,6 +70,11 @@ case "$("$__explorer/os")" in
    macosx)
       sw_vers -productVersion
    ;;
+   freebsd)
+      # Apparently uname -r is not a reliable way to get the patch level.
+      # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
+      freebsd-version
+   ;;
    *bsd|solaris)
       uname -r
    ;;

From fca35fc858d09fc649dc5e7f0964cef4af9e09f0 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 17:29:58 +0100
Subject: [PATCH 334/527] __apt_backports: fix explorer call

s/-/_/ because the explorers are following an other convention :-)
---
 cdist/conf/type/__apt_backports/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index c490a103..c2943725 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -28,7 +28,7 @@
 #  lsb_release may not be given in all installations
 codename_os_release() {
     # shellcheck disable=SC1090
-    . "$__global/explorer/os-release"
+    . "$__global/explorer/os_release"
     printf "%s" "$VERSION_CODENAME"
 }
 

From 27aca06fb893c84601e14bc30890ea6be9300dcd Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 17:34:51 +0100
Subject: [PATCH 335/527] __apt_backports: undo __apt_update_index call

Becuase it is already done by __apt_source.
---
 cdist/conf/type/__apt_backports/man.rst  | 4 +---
 cdist/conf/type/__apt_backports/manifest | 3 ---
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
index c578ce6b..7036fb84 100644
--- a/cdist/conf/type/__apt_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -12,8 +12,7 @@ This singleton type installs backports for the current OS release.
 It aborts if backports are not supported for the specified OS or
 no version codename could be fetched (like Debian unstable).
 
-The package index will be automatically updated by the type
-:strong:`cdist-type__apt_update_index`\ (7) if required.
+The package index will be automatically updated if required.
 
 It supports backports from following OSes:
 
@@ -90,7 +89,6 @@ SEE ALSO
 `Official Debian Backports site <https://backports.debian.org/>`_
 
 :strong:`cdist-type__apt_source`\ (7)
-:strong:`cdist-type__apt_update_index`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index c2943725..bc47d8de 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -79,6 +79,3 @@ __apt_source "${dist}-backports" \
     --distribution "${dist}-backports" \
     --component "$components" \
     --uri "$mirror"
-
-# update the index if the source changed
-require="__apt_source/${dist}-backports" __apt_update_index

From 71f22831175ec41e9348a759fcdaf0eceb1a2a52 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 13 Dec 2020 16:03:39 +0100
Subject: [PATCH 336/527] ++changelog

---
 docs/changelog | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 3d6084f1..290f94d0 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -2,10 +2,12 @@ Changelog
 ---------
 
 next:
-	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
+	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
 	* Type __iptables: Add IPv6 support (Matthias Stecher)
 	* Type __block: Fix escaping in here-doc (Matthias Stecher)
+	* Explorer os_version: Improve FreeBSD support (Evil Ham)
+	* New type: __apt_backports (Matthias Stecher)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From 932e2496ed87b830a468a263cf1b1f5e6a88e2ef Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 15 Dec 2020 18:40:39 +0100
Subject: [PATCH 337/527] [type/__postgres_role] Lint

---
 .../conf/type/__postgres_role/explorer/state  | 32 ++++----
 .../conf/type/__postgres_role/gencode-remote  | 78 ++++++++++---------
 2 files changed, 58 insertions(+), 52 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index c8e1fa9d..110d29d5 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -11,32 +11,32 @@
 #
 # cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$("${__explorer}/os")"
+case $("${__explorer:?}/os")
 in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
+	(netbsd)
+		postgres_user='pgsql'
+		;;
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(*)
+		postgres_user='postgres'
+		;;
 esac
 
 
-name="$__object_id"
+rolename=${__object_id:?}
 
-if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='$name'\"")"
+if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='${rolename}'\"")"
 then
-    echo 'present'
+	echo 'present'
 else
-    echo 'absent'
+	echo 'absent'
 fi
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 282294c9..4fb761ed 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -11,55 +11,61 @@
 #
 # cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
+case $(cat "${__global:?}/explorer/os")
 in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
+	(netbsd)
+		postgres_user='pgsql'
+		;;
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(*)
+		postgres_user='postgres'
+		;;
 esac
 
 
-name="$__object_id"
-state_is="$(cat "$__object/explorer/state")"
-state_should="$(cat "$__object/parameter/state")"
+rolename=${__object_id:?}
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 
-[ "$state_is" = "$state_should" ] && exit 0
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
 
-case "$state_should" in
-    present)
-        if [ -f "$__object/parameter/password" ]; then
-            password="$(cat "$__object/parameter/password")"
-        fi
-        booleans=""
-        for boolean in login createdb createrole superuser; do
-            if [ ! -f "$__object/parameter/$boolean" ]; then
-                boolean="no${boolean}"
-            fi
-            upper=$(echo $boolean | tr '[:lower:]' '[:upper:]')
-            booleans="$booleans $upper"
-        done
+case ${state_should}
+in
+	(present)
+		if test -f "${__object:?}/parameter/password"
+		then
+			password=$(cat "${__object:?}/parameter/password")
+		fi
+		booleans=
+		for boolean in login createdb createrole superuser
+		do
+			if test ! -f "${__object:?}/parameter/${boolean}"
+			then
+				boolean="no${boolean}"
+			fi
+			booleans="${booleans} $(echo ${boolean} | tr '[:lower:]' '[:upper:]')"
+		done
 
-        [ -n "$password" ] && password="PASSWORD '$password'"
-        cat << EOF
-su - '$postgres_user' -c "psql postgres -wc \"CREATE ROLE \\\\\"$name\\\\\" WITH $password $booleans;\""
+		[ -n "${password}" ] && password="PASSWORD '${password}'"
+		cat << EOF
+su - '${postgres_user}' -c "psql postgres -wc 'CREATE ROLE \\"${rolename}\\" WITH ${password} ${booleans};'"
 EOF
-    ;;
-    absent)
-        cat << EOF
-su - '$postgres_user' -c "dropuser \"$name\""
+		;;
+	(absent)
+		cat << EOF
+su - '${postgres_user}' -c "dropuser '${rolename}'"
 EOF
-    ;;
+		;;
 esac

From c36df82882fedb6ce3630e4718c46992a7c153a0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 15 Dec 2020 21:05:55 +0100
Subject: [PATCH 338/527] [type/__postgres_role] ALTER ROLE when parameters
 change

---
 .../conf/type/__postgres_role/explorer/state  | 42 +++++++++++-
 .../conf/type/__postgres_role/gencode-remote  | 65 +++++++++++++++----
 2 files changed, 91 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 110d29d5..033afcb2 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -31,11 +32,48 @@ in
 		;;
 esac
 
-
 rolename=${__object_id:?}
 
-if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='${rolename}'\"")"
+role_properties=$(
+	cmd=$(printf "psql -F '\034' -R '\036' -wAc \"SELECT * FROM pg_roles WHERE rolname='%s'\"" "${rolename}")
+	su -l "${postgres_user}" -c "${cmd}" \
+	| awk '
+	  BEGIN { RS = "\036"; FS = "\034" }
+	  /^\([0-9]+ rows?\)/ { exit }
+	  NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next }
+	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }
+	  '
+)
+
+if test -n "${role_properties}"
 then
+	# Check if the user's properties match the parameters
+	for prop in login createdb createrole superuser
+	do
+		bool_should=$(test -f "${__object:?}/parameter/${prop}" && echo 't' || echo 'f')
+		bool_is=$(
+			printf '%s\n' "${role_properties}" |
+			awk -F '=' -v key="${prop}" '
+			BEGIN {
+				if (key == "login")
+					key = "canlogin"
+				else if (key == "superuser")
+					key = "super"
+				key = "rol" key
+			}
+			$1 == key {
+				sub(/^[^=]*=/, "")
+				print
+			}
+			'
+		)
+
+		test "${bool_is}" = "${bool_should}" || {
+			echo 'different'
+			exit 0
+		}
+	done
+
 	echo 'present'
 else
 	echo 'absent'
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 4fb761ed..7837976c 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,6 +19,15 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+quote() {
+	if test $# -gt 0
+	then
+		printf '%s' "$*"
+	else
+		cat -
+	fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
+}
+
 case $(cat "${__global:?}/explorer/os")
 in
 	(netbsd)
@@ -44,28 +54,55 @@ fi
 case ${state_should}
 in
 	(present)
-		if test -f "${__object:?}/parameter/password"
+		if test -s "${__object:?}/parameter/password"
 		then
-			password=$(cat "${__object:?}/parameter/password")
+			quoted_password=$(
+				delim='$$'
+				while grep -q -F "${delim}" "${__object:?}/parameter/password"
+				do
+					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
+				done
+
+				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
+				# shellcheck disable=SC2016
+				printf '%s%s%s' "${delim}" "${raw_passwd%?.}" "${delim}"
+			)
 		fi
+
 		booleans=
 		for boolean in login createdb createrole superuser
 		do
-			if test ! -f "${__object:?}/parameter/${boolean}"
-			then
-				boolean="no${boolean}"
-			fi
-			booleans="${booleans} $(echo ${boolean} | tr '[:lower:]' '[:upper:]')"
+			booleans="${booleans}${booleans:+ }$(
+				if test -f "${__object:?}/parameter/${boolean}"
+				then
+					echo "${boolean}"
+				else
+					echo "no${boolean}"
+				fi \
+				| tr '[:lower:]' '[:upper:]')"
 		done
 
-		[ -n "${password}" ] && password="PASSWORD '${password}'"
-		cat << EOF
-su - '${postgres_user}' -c "psql postgres -wc 'CREATE ROLE \\"${rolename}\\" WITH ${password} ${booleans};'"
-EOF
+		case ${state_is}
+		in
+			(absent)
+				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				;;
+			(different)
+				query=$(printf 'ALTER ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				;;
+			(*)
+				exit 1	# TODO: error msg
+				;;
+		esac
+
+		psql_cmd=$(printf 'psql postgres -wc %s' "$(quote "${query}")" | quote)
+		printf "su -l '%s' -c %s\\n" "${postgres_user}" "${psql_cmd}"
 		;;
 	(absent)
-		cat << EOF
-su - '${postgres_user}' -c "dropuser '${rolename}'"
-EOF
+		printf "su -l '%s' -c 'dropuser '\\\\'%s\\\\'\\n" \
+			"${postgres_user}" \
+			"$(quote "${rolename}")"
 		;;
 esac

From 7b7ca4d385ccd025cbefced58419468651652da8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 16 Dec 2020 19:06:50 +0100
Subject: [PATCH 339/527] [type/__postgres_role] Handle password changes

---
 .../conf/type/__postgres_role/explorer/state  | 76 +++++++++++++++++--
 .../conf/type/__postgres_role/gencode-remote  | 23 ++++--
 2 files changed, 88 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 033afcb2..4aadbdff 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -34,9 +34,23 @@ esac
 
 rolename=${__object_id:?}
 
+
+psql_query() {
+	su -l "${postgres_user}" -c "$(
+		printf "psql -F '\034' -R '\036' -wAc '%s'" \
+		"$(printf %s "$*" | sed "s/'/'\\\\''/g")"
+	)"
+}
+
+password_check_login() (
+	PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .)
+	PGPASSWORD=${PGPASSWORD%?.}
+	export PGPASSWORD
+	psql -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
+)
+
 role_properties=$(
-	cmd=$(printf "psql -F '\034' -R '\036' -wAc \"SELECT * FROM pg_roles WHERE rolname='%s'\"" "${rolename}")
-	su -l "${postgres_user}" -c "${cmd}" \
+	psql_query "SELECT * FROM pg_roles WHERE rolname = '${rolename}'" \
 	| awk '
 	  BEGIN { RS = "\036"; FS = "\034" }
 	  /^\([0-9]+ rows?\)/ { exit }
@@ -69,12 +83,62 @@ then
 		)
 
 		test "${bool_is}" = "${bool_should}" || {
-			echo 'different'
-			exit 0
+			state='different properties'
 		}
 	done
 
-	echo 'present'
+	# Check password
+	passwd_stored=$(
+		psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \
+		| awk 'BEGIN { RS = "\036" } NR == 2'
+		printf .
+	)
+	passwd_stored=${passwd_stored%?.}
+
+	passwd_should=$(cat "${__object}/parameter/password"; printf .)
+	passwd_should=${passwd_should%?.}
+
+	if expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
+	then
+		# SCRAM-SHA-256 "encrypted" password
+		# NOTE: There is currently no easy way to check SCRAM passwords
+		password_check_login || state="${state:-different} password"
+	elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null
+	then
+		# MD5 "encrypted" password
+		if command -v md5sum >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| md5sum - | sed -e 's/[^0-9a-f]*$//')
+		elif command -v gmd5sum >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| gmd5sum - | sed -e 's/[^0-9a-f]*$//')
+		elif command -v openssl >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| openssl dgst -md5 | sed 's/^.* //')
+		fi
+
+		if test -n "${should_md5}"
+		then
+			test "${passwd_stored}" = "md5${should_md5}" \
+			|| state="${state:-different} password"
+		else
+			password_check_login || state="${state:-different} password"
+		fi
+	else
+		# unencrypted password (unsupported since PostgreSQL 10)
+		test "${passwd_stored}" = "${passwd_should}" \
+		|| state="${state:-different} password"
+	fi
+
+	test -n "${state}" || state='present'
 else
-	echo 'absent'
+	state='absent'
 fi
+
+echo "${state}"
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 7837976c..fd590a4b 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -58,7 +58,9 @@ in
 		then
 			quoted_password=$(
 				delim='$$'
-				while grep -q -F "${delim}" "${__object:?}/parameter/password"
+				# NOTE: Strip away trailing $ because with it the check breaks
+				#       if the password ends with $ + random value.
+				while grep -q -F "${delim%$}" "${__object:?}/parameter/password"
 				do
 					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
 				done
@@ -88,12 +90,23 @@ in
 				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
 					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
 				;;
-			(different)
-				query=$(printf 'ALTER ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+			(different*)
+				query="ALTER ROLE \"${rolename}\" WITH"
+
+				if expr "${state_is}" : 'different.*properties' >/dev/null
+				then
+					query="${query} ${booleans}"
+				fi
+				if expr "${state_is}" : 'different.*password' >/dev/null
+				then
+					query="${query} PASSWORD ${quoted_password:-NULL}"
+				fi
+
+				query="${query};"
 				;;
 			(*)
-				exit 1	# TODO: error msg
+				printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2
+				exit 1
 				;;
 		esac
 

From 4859c27900d125841c703e0270dd8c0eae05d504 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 16:57:03 +0100
Subject: [PATCH 340/527] [type/__postgres_role] Refactor gencode-remote

---
 .../conf/type/__postgres_role/gencode-remote  | 65 ++++++++++---------
 1 file changed, 34 insertions(+), 31 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index fd590a4b..540eb606 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -51,9 +51,6 @@ then
 	exit 0
 fi
 
-case ${state_should}
-in
-	(present)
 		if test -s "${__object:?}/parameter/password"
 		then
 			quoted_password=$(
@@ -64,6 +61,11 @@ in
 				do
 					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
 				done
+psql_query() {
+	printf 'su -l %s -c %s\n' \
+		"$(quote "${postgres_user}")" \
+		"$(quote "psql postgres -wc $(quote "$1")")"
+}
 
 				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
 				# shellcheck disable=SC2016
@@ -71,51 +73,52 @@ in
 			)
 		fi
 
-		booleans=
-		for boolean in login createdb createrole superuser
-		do
-			booleans="${booleans}${booleans:+ }$(
-				if test -f "${__object:?}/parameter/${boolean}"
-				then
-					echo "${boolean}"
-				else
-					echo "no${boolean}"
-				fi \
-				| tr '[:lower:]' '[:upper:]')"
-		done
+role_properties_should() {
+	_props=
+	for _prop in login createdb createrole superuser
+	do
+		_props="${_props}${_props:+ }$(
+			if test -f "${__object:?}/parameter/${_prop}"
+			then
+				echo "${_prop}"
+			else
+				echo "no${_prop}"
+			fi \
+			| tr '[:lower:]' '[:upper:]')"
+	done
+	printf '%s\n' "${_props}"
+	unset _prop _props
+}
 
+case ${state_should}
+in
+	(present)
 		case ${state_is}
 		in
 			(absent)
-				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				psql_query "$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "$(role_properties_should)" "${quoted_password:-NULL}")"
 				;;
 			(different*)
-				query="ALTER ROLE \"${rolename}\" WITH"
-
 				if expr "${state_is}" : 'different.*properties' >/dev/null
 				then
-					query="${query} ${booleans}"
-				fi
-				if expr "${state_is}" : 'different.*password' >/dev/null
-				then
-					query="${query} PASSWORD ${quoted_password:-NULL}"
+					psql_query "ALTER ROLE \"${rolename}\" WITH $(role_properties_should);"
 				fi
 
-				query="${query};"
+				if expr "${state_is}" : 'different.*password' >/dev/null
+				then
+					psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD ${quoted_password:-NULL};"
+				fi
 				;;
 			(*)
 				printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2
 				exit 1
 				;;
 		esac
-
-		psql_cmd=$(printf 'psql postgres -wc %s' "$(quote "${query}")" | quote)
-		printf "su -l '%s' -c %s\\n" "${postgres_user}" "${psql_cmd}"
 		;;
 	(absent)
-		printf "su -l '%s' -c 'dropuser '\\\\'%s\\\\'\\n" \
-			"${postgres_user}" \
-			"$(quote "${rolename}")"
+		printf 'su -l %s -c %s\n' \
+			"$(quote "${postgres_user}")" \
+			"$(quote "dropuser $(quote "${rolename}")")"
 		;;
 esac

From 1180f13ed6c12e0277bebdf32bdd9840320d0fb9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 16:58:32 +0100
Subject: [PATCH 341/527] [type/__postgres_role] Fix setting password

We need to make sure that the password does not end up in ~/.psql_history.
---
 .../conf/type/__postgres_role/gencode-remote  | 38 ++++++++++---------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 540eb606..15c3e692 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -51,27 +51,29 @@ then
 	exit 0
 fi
 
-		if test -s "${__object:?}/parameter/password"
-		then
-			quoted_password=$(
-				delim='$$'
-				# NOTE: Strip away trailing $ because with it the check breaks
-				#       if the password ends with $ + random value.
-				while grep -q -F "${delim%$}" "${__object:?}/parameter/password"
-				do
-					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
-				done
 psql_query() {
 	printf 'su -l %s -c %s\n' \
 		"$(quote "${postgres_user}")" \
 		"$(quote "psql postgres -wc $(quote "$1")")"
 }
 
-				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
-				# shellcheck disable=SC2016
-				printf '%s%s%s' "${delim}" "${raw_passwd%?.}" "${delim}"
-			)
-		fi
+psql_set_password() {
+	# NOTE: Always make sure that the password does not end up in psql_history!
+	if test -s "${__object:?}/parameter/password"
+	then
+		cat <<-EOF
+		exec 3< "\${__object:?}/parameter/password"
+		su -l '${postgres_user}' -c 'psql -q postgres -w' <<'SQL'
+		\set HISTFILE /dev/null
+		\set pw \`cat <&3\`
+		ALTER ROLE "${rolename}" WITH PASSWORD :'pw';
+		SQL
+		exec 3<&-
+		EOF
+	else
+		psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD NULL;"
+	fi
+}
 
 role_properties_should() {
 	_props=
@@ -96,8 +98,8 @@ in
 		case ${state_is}
 		in
 			(absent)
-				psql_query "$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "$(role_properties_should)" "${quoted_password:-NULL}")"
+				psql_query "CREATE ROLE \"${rolename}\" WITH $(role_properties_should);"
+				psql_set_password
 				;;
 			(different*)
 				if expr "${state_is}" : 'different.*properties' >/dev/null
@@ -107,7 +109,7 @@ in
 
 				if expr "${state_is}" : 'different.*password' >/dev/null
 				then
-					psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD ${quoted_password:-NULL};"
+					psql_set_password
 				fi
 				;;
 			(*)

From 99d82fd0d5ab6d6ccd95fe8a7d3b24f5d82a4cce Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 17:05:58 +0100
Subject: [PATCH 342/527] [type/__postgres_role] Always set psql -q

---
 cdist/conf/type/__postgres_role/explorer/state | 4 ++--
 cdist/conf/type/__postgres_role/gencode-remote | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 4aadbdff..77779662 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -37,7 +37,7 @@ rolename=${__object_id:?}
 
 psql_query() {
 	su -l "${postgres_user}" -c "$(
-		printf "psql -F '\034' -R '\036' -wAc '%s'" \
+		printf "psql -q -F '\034' -R '\036' -wAc '%s'" \
 		"$(printf %s "$*" | sed "s/'/'\\\\''/g")"
 	)"
 }
@@ -46,7 +46,7 @@ password_check_login() (
 	PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .)
 	PGPASSWORD=${PGPASSWORD%?.}
 	export PGPASSWORD
-	psql -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
+	psql -q -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
 )
 
 role_properties=$(
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 15c3e692..877c8135 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -54,7 +54,7 @@ fi
 psql_query() {
 	printf 'su -l %s -c %s\n' \
 		"$(quote "${postgres_user}")" \
-		"$(quote "psql postgres -wc $(quote "$1")")"
+		"$(quote "psql postgres -q -w -c $(quote "$1")")"
 }
 
 psql_set_password() {
@@ -63,7 +63,7 @@ psql_set_password() {
 	then
 		cat <<-EOF
 		exec 3< "\${__object:?}/parameter/password"
-		su -l '${postgres_user}' -c 'psql -q postgres -w' <<'SQL'
+		su -l '${postgres_user}' -c 'psql -q -w postgres' <<'SQL'
 		\set HISTFILE /dev/null
 		\set pw \`cat <&3\`
 		ALTER ROLE "${rolename}" WITH PASSWORD :'pw';

From 8dc2c4207cdd826b9e64cc91cfc7c918cddd5f35 Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Fri, 18 Dec 2020 11:16:28 +0100
Subject: [PATCH 343/527] Added optional dirmode parameter to set the mode of
 (optional) the directory.

---
 cdist/conf/type/__dot_file/man.rst                   | 3 +++
 cdist/conf/type/__dot_file/manifest                  | 2 ++
 cdist/conf/type/__dot_file/parameter/default/dirmode | 1 +
 cdist/conf/type/__dot_file/parameter/optional        | 1 +
 4 files changed, 7 insertions(+)
 create mode 100644 cdist/conf/type/__dot_file/parameter/default/dirmode

diff --git a/cdist/conf/type/__dot_file/man.rst b/cdist/conf/type/__dot_file/man.rst
index ae65eb95..ba7621a1 100644
--- a/cdist/conf/type/__dot_file/man.rst
+++ b/cdist/conf/type/__dot_file/man.rst
@@ -25,6 +25,9 @@ user
 OPTIONAL PARAMETERS
 -------------------
 
+dirmode
+    forwarded to :strong:`__directory` type as mode
+
 mode
     forwarded to :strong:`__file` type
 
diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest
index 5e4957e5..02dadf05 100755
--- a/cdist/conf/type/__dot_file/manifest
+++ b/cdist/conf/type/__dot_file/manifest
@@ -19,6 +19,7 @@ set -eu
 user="$(cat "${__object}/parameter/user")"
 home="$(cat "${__object}/explorer/home")"
 primary_group="$(cat "${__object}/explorer/primary_group")"
+dirmode="$(cat "${__object}/parameter/dirmode")"
 
 # Create parent directory. Type __directory has flag 'parents', but it
 # will leave us with root-owned directory in user home, which is not
@@ -36,6 +37,7 @@ export CDIST_ORDER_DEPENDENCY
 for dir ; do
 	__directory "${home}/${dir}"   \
 	    --group "${primary_group}" \
+            --mode "${dirmode}" \
 	    --owner "${user}"
 done
 
diff --git a/cdist/conf/type/__dot_file/parameter/default/dirmode b/cdist/conf/type/__dot_file/parameter/default/dirmode
new file mode 100644
index 00000000..e9745d1f
--- /dev/null
+++ b/cdist/conf/type/__dot_file/parameter/default/dirmode
@@ -0,0 +1 @@
+0700
diff --git a/cdist/conf/type/__dot_file/parameter/optional b/cdist/conf/type/__dot_file/parameter/optional
index ccab9fa6..9f7f83fb 100644
--- a/cdist/conf/type/__dot_file/parameter/optional
+++ b/cdist/conf/type/__dot_file/parameter/optional
@@ -1,3 +1,4 @@
 state
 mode
 source
+dirmode

From 4bae2863dbc1f6b60b8e797c2f121e1616a94c94 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 18 Dec 2020 12:54:33 +0100
Subject: [PATCH 344/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 290f94d0..3a623236 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __block: Fix escaping in here-doc (Matthias Stecher)
 	* Explorer os_version: Improve FreeBSD support (Evil Ham)
 	* New type: __apt_backports (Matthias Stecher)
+	* Type __dot_file: Add dirmode parameter (Mark Verboom)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From 7cf85c465980c74049f8e0258758c0fd8ea178b2 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Dec 2020 19:21:51 +0100
Subject: [PATCH 345/527] Release 6.9.4

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 3a623236..35953a88 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
 	* Type __iptables: Add IPv6 support (Matthias Stecher)

From 766198912d558e4630b258bba62e78587b3f0efe Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 5 Jan 2021 15:50:21 +0100
Subject: [PATCH 346/527] [type/__sshd_config] Produce error if invalid config
 file is generated

Previously, cdist would silently swallow the error (no invalid config file was
generated).

Reason: `set -e` does not exit if a command in a sub-command group fails,
it merely returns with a non-zero exit status.

e.g. the following snippet does not abort the script if sshd -t returns with a
non-zero exit status:

    set -e
    cmp -s old new || {
        # check config file and update it
        sshd -t -f new \
        && cat new >old
    }

or compressed:

    set -e
    false || { false && true; }
    echo $?
    # prints 1
---
 cdist/conf/type/__sshd_config/gencode-remote | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sshd_config/gencode-remote b/cdist/conf/type/__sshd_config/gencode-remote
index 0b44dfa7..275db4aa 100755
--- a/cdist/conf/type/__sshd_config/gencode-remote
+++ b/cdist/conf/type/__sshd_config/gencode-remote
@@ -91,7 +91,8 @@ awk $(drop_awk_comments "${__type:?}/files/update_sshd_config.awk") \\
 
 cmp -s $(quote "${sshd_config_file}") $(quote "${sshd_config_file}.tmp") || {
 	sshd -t -f $(quote "${sshd_config_file}.tmp") \\
-	&& cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}")
+	&& cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}") \\
+	|| exit  # stop if sshd_config file check fails
 }
 rm -f $(quote "${sshd_config_file}.tmp")
 EOF

From 8753b7eedf022a052ddc0b27fbc58ebf8fd638e1 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 5 Jan 2021 15:57:39 +0100
Subject: [PATCH 347/527] [type/__sshd_config] Make AuthenticationMethods and
 AuthorizedKeysFile singleton options

They were incorrectly treated as non-singleton options before.

cf. https://github.com/openssh/openssh-portable/blob/V_8_4/servconf.c#L2273
and https://github.com/openssh/openssh-portable/blob/V_8_4/servconf.c#L1899 resp.
---
 cdist/conf/type/__sshd_config/files/update_sshd_config.awk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
index d0bc2b4b..f7f30e87 100644
--- a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
+++ b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
@@ -89,7 +89,7 @@ function strdelim(s) { return strdelim_internal(s, 1) }
 function strdelimw(s) { return strdelim_internal(s, 0) }
 
 function singleton_option(opt) {
-	return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|authenticationmethods|authorizedkeysfile|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/
+	return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/
 }
 
 function print_update() {

From bd8ab8f26fdc242f2eb77e58049d640f96d51a69 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 5 Jan 2021 17:00:55 +0100
Subject: [PATCH 348/527] [type/__sshd_config] Document "bug" in state explorer

---
 cdist/conf/type/__sshd_config/man.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cdist/conf/type/__sshd_config/man.rst b/cdist/conf/type/__sshd_config/man.rst
index 8b0069ac..c8e6b8ad 100644
--- a/cdist/conf/type/__sshd_config/man.rst
+++ b/cdist/conf/type/__sshd_config/man.rst
@@ -79,6 +79,10 @@ BUGS
 - ``Include`` directives are ignored.
 - Config options are not added/removed to/from the config file if their value is
   the default value.
+- | The explorer will incorrectly report ``absent`` if OpenSSH internally
+    transforms one value to another (e.g. ``permitrootlogin prohibit-password``
+    is transformed to ``permitrootlogin without-password``).
+  | Workaround: Use the value that OpenSSH uses internally.
 
 
 AUTHORS

From c819548343ad7bb24d023f94a32575e1b2520328 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 11 Jan 2021 09:50:12 +0100
Subject: [PATCH 349/527] Fix debug parameter

-d was removed from cdist in favor of mulitple -v and -l parameters, but
-d was not removed from preos.

Resolve #849.
---
 cdist/preos/debootstrap/files/code | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/preos/debootstrap/files/code b/cdist/preos/debootstrap/files/code
index 9e37003b..d836848c 100755
--- a/cdist/preos/debootstrap/files/code
+++ b/cdist/preos/debootstrap/files/code
@@ -22,7 +22,7 @@ set -e
 if [ "${debug}" ]
 then
     set -x
-    cdist_params="${cdist_params} -d"
+    cdist_params="${cdist_params} -l 3"
 fi
 
 bootstrap_dir="${target_dir}"

From 2954347771e00610a084c7b033016765f7a3d6d7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 14 Jan 2021 13:46:40 +0100
Subject: [PATCH 350/527] [type/__postgres_role] Add note regarding empty
 passwords

---
 cdist/conf/type/__postgres_role/gencode-remote | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 877c8135..d7631fbd 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -59,6 +59,8 @@ psql_query() {
 
 psql_set_password() {
 	# NOTE: Always make sure that the password does not end up in psql_history!
+	# NOTE: Never set an empty string as the password, because they can be
+	#       interpreted differently by different tooling.
 	if test -s "${__object:?}/parameter/password"
 	then
 		cat <<-EOF

From 6e9b13d94962e1bfb365efa849ebefc32854364b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 18 Jan 2021 06:22:32 +0100
Subject: [PATCH 351/527] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 35953a88..b2b35616 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
+	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
+
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)

From 92a50da4873ff4bfa85dde3b85e272704cda02c7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 18 Jan 2021 06:28:09 +0100
Subject: [PATCH 352/527] Fix pycodestyle issues

---
 cdist/scan/scan.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 5ca8fae2..b1d0e9e1 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -139,7 +139,6 @@ class Scanner(object):
 
         """
 
-
     def start(self):
         self.process = Process(target=self.scan)
         self.process.start()

From 35cde3e666c229ceb94a1c5074083f7ba905768a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 13:09:29 +0100
Subject: [PATCH 353/527] [type/__postgres_role] Fix state explorer when stored
 password is empty

---
 cdist/conf/type/__postgres_role/explorer/state | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 77779662..34069de9 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -95,13 +95,20 @@ then
 	)
 	passwd_stored=${passwd_stored%?.}
 
-	passwd_should=$(cat "${__object}/parameter/password"; printf .)
+	if test -f "${__object:?}/parameter/password"
+	then
+		passwd_should=$(cat "${__object:?}/parameter/password"; printf .)
+	fi
 	passwd_should=${passwd_should%?.}
 
-	if expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
+	if test -z "${passwd_stored}"
+	then
+		test -z "${passwd_should}" || state="${state:-different} password"
+	elif expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
 	then
 		# SCRAM-SHA-256 "encrypted" password
-		# NOTE: There is currently no easy way to check SCRAM passwords
+		# NOTE: There is currently no easy way to check SCRAM passwords without
+		#       logging in
 		password_check_login || state="${state:-different} password"
 	elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null
 	then

From c51d68a7375915154a90c22e716e832f635ca878 Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Thu, 3 Dec 2020 18:48:04 +0100
Subject: [PATCH 354/527] [type/__postgres_conf] New type based on ALTER SYSTEM
 command

---
 .../conf/type/__postgres_conf/gencode-remote  | 92 +++++++++++++++++++
 cdist/conf/type/__postgres_conf/man.rst       | 55 +++++++++++
 .../__postgres_conf/parameter/default/state   |  1 +
 .../type/__postgres_conf/parameter/optional   |  2 +
 4 files changed, 150 insertions(+)
 create mode 100755 cdist/conf/type/__postgres_conf/gencode-remote
 create mode 100644 cdist/conf/type/__postgres_conf/man.rst
 create mode 100644 cdist/conf/type/__postgres_conf/parameter/default/state
 create mode 100644 cdist/conf/type/__postgres_conf/parameter/optional

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
new file mode 100755
index 00000000..a09e1873
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -0,0 +1,92 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+# 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$(cat "${__global}/explorer/os")
+state_should=$(cat "${__object}/parameter/state")
+
+if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ]
+then
+	echo "Invalid state '${state_should}'." \
+	     'Only "present" and "absent" are acceptable' >&2
+	exit 1
+fi
+
+if [ "${state_should}" = 'present' ] && [ ! -f "${__object}/parameter/value" ]
+then
+	echo 'Missing required parameter "value"' >&2
+	exit 1
+fi
+
+# Parameters
+conf_name="${__object_id}"
+if [ -f "${__object}/parameter/value" ]
+then
+	conf_value=$(cat "${__object}/parameter/value")
+fi
+
+if [ "${state_should}" = 'present' ]
+then
+	set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'"
+	check_command="SHOW ${conf_name}"
+else
+	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
+fi
+
+case $os
+in
+	openbsd|devuan)
+		case $os
+		in
+			openbsd)
+				postgres_user='_postgresql'
+				restart_command='/etc/rc.d/postgresql restart'
+				;;
+			devuan)
+				postgres_user='postgres'
+				restart_command='/etc/init.d/postgresql restart'
+				;;
+		esac
+		# needs two separate psql commands because ALTER SYSTEM
+		# cannot run inside a transaction block
+		cat <<-EOF
+		su - ${postgres_user} -c "psql postgres -twAc \
+			\"${set_command}\""
+		su - ${postgres_user} -c "psql postgres -twAc \
+			\"SELECT pg_reload_conf()\""
+		EOF
+		# check success (makes only sense if setting to a non-default value)
+		# and restart server if needed
+		if [ "${state_should}" = 'present' ]
+		then
+			cat <<-EOF
+			if [ \$(su - ${postgres_user} -c "psql postgres -twAc \"SHOW ${conf_name}\"") != '${conf_value}' ]
+			then
+				${restart_command}
+			fi
+			EOF
+		fi
+		;;
+	*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst
new file mode 100644
index 00000000..76016c6c
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/man.rst
@@ -0,0 +1,55 @@
+cdist-type__postgres_conf(7)
+============================
+
+Configure a PostgreSQL server.
+
+NOTE: This type might need to be run multiple times to apply all bits of the
+configuration due to ordering requirements.
+
+SSRQ <cdist--@--ssrq-sds-fds.ch>
+
+
+DESCRIPTION
+-----------
+Configure a PostgreSQL server using ALTER SYSTEM.
+
+
+REQUIRED PARAMETERS
+-------------------
+value
+    The value to setup (can be omitted when state is set to "absent").
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    "present" or "absent". Defaults to "present".
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+	# set timezone
+    __postgres_conf timezone --value Europe/Zurich
+
+    # reset maximum number of concurrent connections to default (normally 100)
+    __postgres_conf max_connections --state absent
+
+
+SEE ALSO
+--------
+- `cdist-type(7) <cdist-type.html>`_
+
+
+COPYING
+-------
+Copyright \(C) 2020 SSRQ (www.ssrq-sds-fds.ch).
+Free use of this software is granted under the terms
+of the GNU General Public License version 3 (GPLv3).
diff --git a/cdist/conf/type/__postgres_conf/parameter/default/state b/cdist/conf/type/__postgres_conf/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__postgres_conf/parameter/optional b/cdist/conf/type/__postgres_conf/parameter/optional
new file mode 100644
index 00000000..d0460d86
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/parameter/optional
@@ -0,0 +1,2 @@
+state
+value

From 534d5f6bb5dd3d8e9cb2256e75ed182d6530a892 Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Fri, 4 Dec 2020 14:31:04 +0100
Subject: [PATCH 355/527] [type/__postgres_conf] Fix errors found by ShellCheck

---
 cdist/conf/type/__postgres_conf/gencode-remote | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index a09e1873..e25514d0 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -20,8 +20,8 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$(cat "${__global}/explorer/os")
-state_should=$(cat "${__object}/parameter/state")
+os=$(cat "${__global:?}/explorer/os")
+state_should=$(cat "${__object:?}/parameter/state")
 
 if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ]
 then
@@ -37,7 +37,7 @@ then
 fi
 
 # Parameters
-conf_name="${__object_id}"
+conf_name="${__object_id:?}"
 if [ -f "${__object}/parameter/value" ]
 then
 	conf_value=$(cat "${__object}/parameter/value")
@@ -46,7 +46,6 @@ fi
 if [ "${state_should}" = 'present' ]
 then
 	set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'"
-	check_command="SHOW ${conf_name}"
 else
 	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
 fi

From 50bcd951055d7599ec340b4dea80a3a5d11dd9b0 Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Thu, 10 Dec 2020 11:45:32 +0100
Subject: [PATCH 356/527] [type/__postgres_conf] Remove faulty quotes

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index e25514d0..8ccb3b42 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -45,7 +45,7 @@ fi
 
 if [ "${state_should}" = 'present' ]
 then
-	set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'"
+	set_command="ALTER SYSTEM SET ${conf_name} = ${conf_value}"
 else
 	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
 fi

From b4060720dc3c7cc58aa992285877e4c009caa6be Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Fri, 11 Dec 2020 11:12:16 +0100
Subject: [PATCH 357/527] [type/__postgres_conf] Fix psql options for ALTER
 command

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 8ccb3b42..7d86028e 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -67,7 +67,7 @@ in
 		# needs two separate psql commands because ALTER SYSTEM
 		# cannot run inside a transaction block
 		cat <<-EOF
-		su - ${postgres_user} -c "psql postgres -twAc \
+		su - ${postgres_user} -c "psql postgres -qwc \
 			\"${set_command}\""
 		su - ${postgres_user} -c "psql postgres -twAc \
 			\"SELECT pg_reload_conf()\""

From 1b49fec972ce5f486f2794571c8e892d1fa6cb51 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 19:15:49 +0100
Subject: [PATCH 358/527] [type/__postgres_conf] Refactor

---
 .../type/__postgres_conf/explorer/context     |  42 ++++++
 .../conf/type/__postgres_conf/explorer/state  |  60 ++++++++
 .../conf/type/__postgres_conf/gencode-remote  | 135 ++++++++++--------
 3 files changed, 180 insertions(+), 57 deletions(-)
 create mode 100644 cdist/conf/type/__postgres_conf/explorer/context
 create mode 100644 cdist/conf/type/__postgres_conf/explorer/state

diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context
new file mode 100644
index 00000000..3a2fa504
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/explorer/context
@@ -0,0 +1,42 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Returns the "context" of the configuration setting.
+# cf. also https://www.postgresql.org/docs/10/view-pg-settings.html
+
+os=$("${__explorer:?}/os")
+
+case ${os}
+in
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(devuan)
+		postgres_user='postgres'
+		;;
+	(*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
+
+conf_name=${__object_id:?}
+
+su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE name = '${conf_name}'\""
diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
new file mode 100644
index 00000000..da904b56
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -0,0 +1,60 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$("${__explorer:?}/os")
+
+case ${os}
+in
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(devuan)
+		postgres_user='postgres'
+		;;
+	(*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
+
+conf_name=${__object_id:?}
+
+if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
+	| cmp -s "${__object:?}/parameter/value" -
+then
+	echo present
+else
+	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT source FROM pg_settings WHERE name = '${conf_name}'\"")
+	in
+		('')
+			# invalid configuration parameter
+			# (error message was already printed by SHOW command above.
+			#  Yes, it's a hack)
+			exit 1
+			;;
+		(default)
+			echo absent
+			;;
+		(*)
+			echo different
+			;;
+	esac
+fi
diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 7d86028e..998b8582 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -1,7 +1,7 @@
 #!/bin/sh -e
 # -*- mode: sh; indent-tabs-mode: t -*-
 #
-# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+# 2019-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 # 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
@@ -21,71 +21,92 @@
 #
 
 os=$(cat "${__global:?}/explorer/os")
+state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
-if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ]
+conf_name=${__object_id:?}
+
+if test "${state_is}" = "${state_should}"
 then
-	echo "Invalid state '${state_should}'." \
-	     'Only "present" and "absent" are acceptable' >&2
-	exit 1
+	exit 0
 fi
 
-if [ "${state_should}" = 'present' ] && [ ! -f "${__object}/parameter/value" ]
-then
-	echo 'Missing required parameter "value"' >&2
-	exit 1
-fi
-
-# Parameters
-conf_name="${__object_id:?}"
-if [ -f "${__object}/parameter/value" ]
-then
-	conf_value=$(cat "${__object}/parameter/value")
-fi
-
-if [ "${state_should}" = 'present' ]
-then
-	set_command="ALTER SYSTEM SET ${conf_name} = ${conf_value}"
-else
-	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
-fi
-
-case $os
-in
-	openbsd|devuan)
-		case $os
-		in
-			openbsd)
-				postgres_user='_postgresql'
-				restart_command='/etc/rc.d/postgresql restart'
-				;;
-			devuan)
-				postgres_user='postgres'
-				restart_command='/etc/init.d/postgresql restart'
-				;;
-		esac
-		# needs two separate psql commands because ALTER SYSTEM
-		# cannot run inside a transaction block
-		cat <<-EOF
-		su - ${postgres_user} -c "psql postgres -qwc \
-			\"${set_command}\""
-		su - ${postgres_user} -c "psql postgres -twAc \
-			\"SELECT pg_reload_conf()\""
-		EOF
-		# check success (makes only sense if setting to a non-default value)
-		# and restart server if needed
-		if [ "${state_should}" = 'present' ]
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
 		then
-			cat <<-EOF
-			if [ \$(su - ${postgres_user} -c "psql postgres -twAc \"SHOW ${conf_name}\"") != '${conf_value}' ]
-			then
-				${restart_command}
-			fi
-			EOF
+			# needs quoting
+			set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'"
+		else
+			set -- "$@" "${_arg}"
 		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
+
+case ${os}
+in
+	(openbsd)
+		postgres_user='_postgresql'
+		restart_command='/etc/rc.d/postgresql restart'
 		;;
-	*)
+	(devuan)
+		postgres_user='postgres'
+		restart_command='/etc/init.d/postgresql restart'
+		;;
+	(*)
 		echo "Unsupported OS: ${os}" >&2
 		exit 1
 		;;
 esac
+
+
+psql_cmd() {
+	printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")"
+}
+
+case ${state_should}
+in
+	(present)
+		test -s "${__object:?}/parameter/value" || {
+			echo 'Missing required parameter --value' >&2
+			exit 1
+		}
+
+		cat <<-EOF
+		exec 3< "\${__object:?}/parameter/value"
+		$(psql_cmd postgres -tAw) <<'SQL'
+		\\set conf_value \`cat <&3\`
+		ALTER SYSTEM SET ${conf_name} = :'conf_value';
+		SELECT pg_reload_conf();
+		SQL
+		exec 3<&-
+		EOF
+		;;
+	(absent)
+		psql_cmd postgres -qwc "ALTER SYSTEM SET ${conf_name} TO DEFAULT"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		printf 'Only "present" and "absent" are acceptable.\n' >&2
+		exit 1
+		;;
+esac
+
+# check success (makes only sense if setting to a non-default value)
+# and restart server if needed
+if test "${state_should}" = 'present'
+then
+	cat <<-EOF
+
+	$(psql_cmd postgres -twAc "SHOW ${conf_name}") \\
+	| cmp -s "\${__object:?}/parameter/value" - || {
+		${restart_command}
+	}
+	EOF
+fi

From 803367b316f5c72c51153172a41b7a745f2d2d83 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 19:29:42 +0100
Subject: [PATCH 359/527] [type/__postgres_conf] Fix default detection when
 default is also set in config file

e.g. port is usually also set to the default value in postgresql.conf
---
 cdist/conf/type/__postgres_conf/explorer/state | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index da904b56..400b27e8 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -42,7 +42,7 @@ if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
 then
 	echo present
 else
-	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT source FROM pg_settings WHERE name = '${conf_name}'\"")
+	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE name = '${conf_name}'\"")
 	in
 		('')
 			# invalid configuration parameter

From 891c98567e1d94577dc4f20d3ec9f73f4927fd24 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 19:39:56 +0100
Subject: [PATCH 360/527] [type/__postgres_conf] Compare configuration
 parameter names case insensitively

---
 cdist/conf/type/__postgres_conf/explorer/context | 3 ++-
 cdist/conf/type/__postgres_conf/explorer/state   | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context
index 3a2fa504..def5f676 100644
--- a/cdist/conf/type/__postgres_conf/explorer/context
+++ b/cdist/conf/type/__postgres_conf/explorer/context
@@ -39,4 +39,5 @@ esac
 
 conf_name=${__object_id:?}
 
-su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE name = '${conf_name}'\""
+# NOTE: SHOW/SET are case-insentitive, so this command should also be.
+su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE lower(name) = lower('${conf_name}')\""
diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 400b27e8..a4930296 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -42,7 +42,8 @@ if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
 then
 	echo present
 else
-	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE name = '${conf_name}'\"")
+	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
+	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"")
 	in
 		('')
 			# invalid configuration parameter

From 5051d4f40b7097f58431ce334f9ddf5b171dbc7f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 16:36:44 +0100
Subject: [PATCH 361/527] [type/__postgres_conf] Catch invalid values

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 998b8582..cc9b4e99 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -80,7 +80,7 @@ in
 
 		cat <<-EOF
 		exec 3< "\${__object:?}/parameter/value"
-		$(psql_cmd postgres -tAw) <<'SQL'
+		$(psql_cmd postgres -tAw -v ON_ERROR_STOP=on) <<'SQL'
 		\\set conf_value \`cat <&3\`
 		ALTER SYSTEM SET ${conf_name} = :'conf_value';
 		SELECT pg_reload_conf();

From 0f2ff477381d67de27149a92744f873322e44fd7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 16:37:43 +0100
Subject: [PATCH 362/527] [type/__postgres_conf] Restart PostgreSQL server
 based on pending_restart column of pg_settings

---
 .../type/__postgres_conf/explorer/context     | 43 -------------------
 .../conf/type/__postgres_conf/gencode-remote  | 16 +++----
 2 files changed, 6 insertions(+), 53 deletions(-)
 delete mode 100644 cdist/conf/type/__postgres_conf/explorer/context

diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context
deleted file mode 100644
index def5f676..00000000
--- a/cdist/conf/type/__postgres_conf/explorer/context
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/sh -e
-# -*- mode: sh; indent-tabs-mode: t -*-
-#
-# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-# Returns the "context" of the configuration setting.
-# cf. also https://www.postgresql.org/docs/10/view-pg-settings.html
-
-os=$("${__explorer:?}/os")
-
-case ${os}
-in
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(devuan)
-		postgres_user='postgres'
-		;;
-	(*)
-		echo "Unsupported OS: ${os}" >&2
-		exit 1
-		;;
-esac
-
-conf_name=${__object_id:?}
-
-# NOTE: SHOW/SET are case-insentitive, so this command should also be.
-su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE lower(name) = lower('${conf_name}')\""
diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index cc9b4e99..fa930cc4 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -98,15 +98,11 @@ in
 		;;
 esac
 
-# check success (makes only sense if setting to a non-default value)
-# and restart server if needed
-if test "${state_should}" = 'present'
-then
-	cat <<-EOF
+# Restart PostgreSQL server if required to apply new configuration value
+cat <<EOF
 
-	$(psql_cmd postgres -twAc "SHOW ${conf_name}") \\
-	| cmp -s "\${__object:?}/parameter/value" - || {
-		${restart_command}
-	}
-	EOF
+if test 't' = "\$($(psql_cmd postgres -twAc "SELECT pending_restart FROM pg_settings WHERE lower(name) = lower('${conf_name}')"))"
+then
+	${restart_command}
 fi
+EOF

From 3f605c31ac7538d188a11b5d9d84cf2ed485bf23 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 17:18:27 +0100
Subject: [PATCH 363/527] [type/__postgres_conf] Add support for more init
 systems to restart service

---
 .../conf/type/__postgres_conf/gencode-remote  | 35 +++++++++++++++++--
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index fa930cc4..15d5bb3e 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -53,11 +53,9 @@ case ${os}
 in
 	(openbsd)
 		postgres_user='_postgresql'
-		restart_command='/etc/rc.d/postgresql restart'
 		;;
 	(devuan)
 		postgres_user='postgres'
-		restart_command='/etc/init.d/postgresql restart'
 		;;
 	(*)
 		echo "Unsupported OS: ${os}" >&2
@@ -103,6 +101,37 @@ cat <<EOF
 
 if test 't' = "\$($(psql_cmd postgres -twAc "SELECT pending_restart FROM pg_settings WHERE lower(name) = lower('${conf_name}')"))"
 then
-	${restart_command}
+	$(
+		init=$(cat "${__global:?}/explorer/init")
+		case ${init}
+		in
+			(systemd)
+				echo 'systemctl restart postgresql.service'
+				;;
+			(*openrc*)
+				echo 'rc-service postgresql restart'
+				;;
+			(sysvinit)
+				echo '/etc/init.d/postgresql restart'
+				;;
+			(init)
+				case $(cat "${__global:?}/explorer/kernel_name")
+				in
+					(FreeBSD)
+						echo 'service postgresql restart'
+						;;
+					(OpenBSD|NetBSD)
+						echo '/etc/rc.d/postgresql restart'
+						;;
+					(*)
+						echo "Unsupported operating system. Don't know how to restart services." >&2
+						exit 1
+				esac
+				;;
+			(*)
+				printf "Don't know how to restart services with your init (%s)\n" "${init}" >&2
+				exit 1
+		esac
+	)
 fi
 EOF

From 4967c7ebbb7e98af6aa2c7f8bc91511bcbbf9820 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 17:20:27 +0100
Subject: [PATCH 364/527] [type/__postgres_conf] Silence psql output

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 15d5bb3e..9b8227ee 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -78,7 +78,7 @@ in
 
 		cat <<-EOF
 		exec 3< "\${__object:?}/parameter/value"
-		$(psql_cmd postgres -tAw -v ON_ERROR_STOP=on) <<'SQL'
+		$(psql_cmd postgres -tAwq -o /dev/null -v ON_ERROR_STOP=on) <<'SQL'
 		\\set conf_value \`cat <&3\`
 		ALTER SYSTEM SET ${conf_name} = :'conf_value';
 		SELECT pg_reload_conf();

From f9ebb4333c85e41439635cb00a40c3e07d8a3683 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 17:21:50 +0100
Subject: [PATCH 365/527] [type/__postgres_conf] Add NetBSD PostgreSQL UNIX
 user

---
 cdist/conf/type/__postgres_conf/gencode-remote | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 9b8227ee..8f8a2bfe 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -51,15 +51,14 @@ quote() {
 
 case ${os}
 in
+	(netbsd)
+		postgres_user='pgsql'
+		;;
 	(openbsd)
 		postgres_user='_postgresql'
 		;;
-	(devuan)
-		postgres_user='postgres'
-		;;
 	(*)
-		echo "Unsupported OS: ${os}" >&2
-		exit 1
+		postgres_user='postgres'
 		;;
 esac
 

From 6b18cace759d1345cec6c2168c3bd8f238f23bca Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 21 Jan 2021 19:29:07 +0100
Subject: [PATCH 366/527] [type/__postgres_conf] Catch connection errors early

---
 cdist/conf/type/__postgres_conf/explorer/state | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index a4930296..589925de 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -37,6 +37,11 @@ esac
 
 conf_name=${__object_id:?}
 
+su - "${postgres_user}" -c 'psql postgres -c "SELECT 1"' >/dev/null || {
+	echo 'Connection to PostgreSQL server failed' >&2
+	exit 1
+}
+
 if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
 	| cmp -s "${__object:?}/parameter/value" -
 then

From 8eccacec59b0badd7a6e2a40a27f02f0ff355ccf Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 4 Feb 2021 19:09:26 +0100
Subject: [PATCH 367/527] __package_pip: add optional dependencies

This is a poor implementation of optional dependencies for pip packages.
It ensures to install them if the package will be installed, but does
not take into account if they must be added/removed after the package is
already installed. Also, it will not be autoremoved, as all dependencies
will not be removed.
---
 cdist/conf/type/__package_pip/gencode-remote        | 13 +++++++++++++
 cdist/conf/type/__package_pip/man.rst               | 13 +++++++++++++
 .../type/__package_pip/parameter/optional_multiple  |  1 +
 3 files changed, 27 insertions(+)
 create mode 100644 cdist/conf/type/__package_pip/parameter/optional_multiple

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index a1375c2d..58c89040 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -56,6 +56,19 @@ fi
 
 case "$state_should" in
     present)
+        if [ -f "$__object/parameter/extra" ]
+        then
+            while read extra
+            do
+                if [ "$extras" ]; then
+                    extras="$extras,$extra"
+                else
+                    extras="$extra"
+                fi
+            done < "$__object/parameter/extra"
+            name="${name}[${extras}]"
+        fi
+
         if [ "$runas" ]
         then
             echo "su -c '$pip install -q $name' $runas"
diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index 234ceee2..64ad0358 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -22,6 +22,11 @@ OPTIONAL PARAMETERS
 name
     If supplied, use the name and not the object id as the package name.
 
+extra
+    Extra optional dependencies which should be installed along the selected
+    package. Can be specified multiple times. Will only be applied if the
+    package actually will be installed, but will not explicitly checked.
+
 pip
     Instead of using pip from PATH, use the specific pip path.
 
@@ -46,6 +51,14 @@ EXAMPLES
     # Use pip in a virtualenv located at /foo/shinken_virtualenv as user foo
     __package_pip pyro --state present --pip /foo/shinken_virtualenv/bin/pip --runas foo
 
+    # Install package with optional dependencies
+    __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails
+    # or do a little cheating
+    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails
+
+    # or take all extras
+    __package_pip mautrix-telegram --extra all
+
 
 SEE ALSO
 --------
diff --git a/cdist/conf/type/__package_pip/parameter/optional_multiple b/cdist/conf/type/__package_pip/parameter/optional_multiple
new file mode 100644
index 00000000..0f228715
--- /dev/null
+++ b/cdist/conf/type/__package_pip/parameter/optional_multiple
@@ -0,0 +1 @@
+extra

From 73a03d75d7d23fea292b5fd60aeb059d8c72c4ba Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 4 Feb 2021 19:18:02 +0100
Subject: [PATCH 368/527] __package_pip: fix shellcheck

---
 cdist/conf/type/__package_pip/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 58c89040..925eaf24 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -58,7 +58,7 @@ case "$state_should" in
     present)
         if [ -f "$__object/parameter/extra" ]
         then
-            while read extra
+            while read -r extra
             do
                 if [ "$extras" ]; then
                     extras="$extras,$extra"

From cda17be38a1030742ed224b2536fca517bf8c09b Mon Sep 17 00:00:00 2001
From: ssrq <ungleich@ssrq-sds-fds.ch>
Date: Mon, 8 Feb 2021 08:27:03 +0100
Subject: [PATCH 369/527] [explorer/memory] Clean up, return kiB for all
 systems, add SunOS

BSDs were MiB before.
---
 cdist/conf/explorer/memory | 85 ++++++++++++++++++++++++++++++--------
 1 file changed, 68 insertions(+), 17 deletions(-)

diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory
index 5ea15ada..63aba9c6 100755
--- a/cdist/conf/explorer/memory
+++ b/cdist/conf/explorer/memory
@@ -1,8 +1,9 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2014 Daniel Heule  (hda at sfs.biz)
 # 2014 Thomas Oettli (otho at sfs.biz)
 # Copyright 2017, Philippe Gregoire <pg@pgregoire.xyz>
+# 2020 Dennis Camera <dennis.camera at ssrq-sds-fds.ch>
 #
 # This file is part of cdist.
 #
@@ -19,24 +20,74 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
+# Returns the amount of memory physically installed in the system, or if that
+# cannot be determined the amount available to the operating system kernel,
+# in kibibytes (kiB).
 
-# FIXME: other system types (not linux ...)
+str2bytes() {
+	awk -F' ' '
+	$2 ==   "B" || !$2 { print $1 }
+	$2 ==  "kB" { print $1 * 1000 }
+	$2 ==  "MB" { print $1 * 1000 * 1000 }
+	$2 ==  "GB" { print $1 * 1000 * 1000 * 1000 }
+	$2 ==  "TB" { print $1 * 1000 * 1000 * 1000 * 1000 }
+	$2 == "kiB" { print $1 * 1024 }
+	$2 == "MiB" { print $1 * 1024 * 1024 }
+	$2 == "GiB" { print $1 * 1024 * 1024 * 1024 }
+	$2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }'
+}
 
-os=$("$__explorer/os")
-case "$os" in
-    "macosx")
-        echo "$(sysctl -n hw.memsize)/1024" | bc
-    ;;
+bytes2kib() {
+	set -- "$(cat)"
+	test "$1" -gt 0 && echo $(($1 / 1024))
+}
 
-    *"bsd")
-        PATH=$(getconf PATH)
-        echo "$(sysctl -n hw.physmem) / 1048576" | bc
-    ;;
 
-    *)
-    if [ -r /proc/meminfo ]; then
-        grep "MemTotal:" /proc/meminfo | awk '{print $2}'
-    fi
-    ;;
+case $(uname -s)
+in
+	(Darwin)
+		sysctl -n hw.memsize | bytes2kib
+		;;
+	(FreeBSD)
+		sysctl -n hw.realmem | bytes2kib
+		;;
+	(NetBSD|OpenBSD)
+		# NOTE: This reports "usable" memory, not physically installed memory.
+		command -p sysctl -n hw.physmem | bytes2kib
+		;;
+	(SunOS)
+		# Make sure that awk from xpg4 is used for the scripts to work
+		export PATH="/usr/xpg4/bin:${PATH}"
+		prtconf \
+		| awk -F ': ' '
+		  $1 == "Memory size" { sub(/Megabytes/, "MiB", $2); print $2 }
+		  /^$/ { exit }' \
+		| str2bytes \
+		| bytes2kib
+		;;
+	(Linux)
+		if test -d /sys/devices/system/memory
+		then
+			# Use memory blocks if the architecture (e.g. x86, PPC64, s390)
+			# supports them (they denote physical memory)
+			num_mem_blocks=$(cat /sys/devices/system/memory/memory[0-9]*/state | grep -cxF online)
+			mem_block_size=$(cat /sys/devices/system/memory/block_size_bytes)
+
+			echo $((num_mem_blocks * 0x$mem_block_size)) | bytes2kib && exit
+		fi
+		if test -r /proc/meminfo
+		then
+			# Fall back to meminfo file on other architectures (e.g. ARM, MIPS,
+			# PowerPC)
+			# NOTE: This is "usable" memory, not physically installed memory.
+			awk -F ': +' '$1 == "MemTotal" { sub(/B$/, "iB", $2); print $2 }' /proc/meminfo \
+			| str2bytes \
+			| bytes2kib
+		fi
+		;;
+	(*)
+		printf "Your kernel (%s) is currently not supported by the memory explorer\n" "$(uname -s)" >&2
+		printf "Please contribute an implementation for it if you can.\n" >&2
+		exit 1
+		;;
 esac

From 65a6a2ed52c56f553d65ad0aa0aaba3ee4579e3c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 8 Feb 2021 08:28:31 +0100
Subject: [PATCH 370/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index b2b35616..52686617 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
+	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)

From bc145bbc27a45e382adb744e6774fb803f4fda0a Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 9 Feb 2021 19:58:47 +0100
Subject: [PATCH 371/527] [__letsencrypt_cert] Fix various issues with hooks.

Closes #853, see issue for full description / discussion.

Short summary:
- There was about 6.53% chances of `--renewal-hook` not being applied
- Using --automatic-renewal in one cert and not in another was an error.
- It was not possible to use different hooks for different certificates.
- FreeBSD support was utterly broken.
---
 cdist/conf/type/__letsencrypt_cert/man.rst    |  84 +++++++--
 cdist/conf/type/__letsencrypt_cert/manifest   | 171 ++++++++++++++----
 .../parameter/deprecated/automatic-renewal    |   2 +
 .../parameter/deprecated/renew-hook           |   2 +
 .../parameter/optional_multiple               |   3 +
 5 files changed, 220 insertions(+), 42 deletions(-)
 mode change 100755 => 100644 cdist/conf/type/__letsencrypt_cert/manifest
 create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal
 create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook

diff --git a/cdist/conf/type/__letsencrypt_cert/man.rst b/cdist/conf/type/__letsencrypt_cert/man.rst
index 85eb88ea..43be8424 100644
--- a/cdist/conf/type/__letsencrypt_cert/man.rst
+++ b/cdist/conf/type/__letsencrypt_cert/man.rst
@@ -1,16 +1,33 @@
 cdist-type__letsencrypt_cert(7)
 ===============================
 
+
 NAME
 ----
 
 cdist-type__letsencrypt_cert - Get an SSL certificate from Let's Encrypt
 
+
 DESCRIPTION
 -----------
 
 Automatically obtain a Let's Encrypt SSL certificate using Certbot.
 
+This type attempts to setup automatic renewals always. In many Linux
+distributions, that is the case out of the box, see:
+https://certbot.eff.org/docs/using.html#automated-renewals
+
+For Alpine Linux and Arch Linux, we setup a system-wide cronjob that
+attempts to renew certificates daily.
+
+If you are using FreeBSD, we configure periodic(8) as recommended by
+the port mantainer, so there will be a weekly attempt at renewal.
+
+If your OS is not mentioned here or on Certbot's docs as having
+support for automated renewals, please make sure you check your OS
+and possibly patch this type so the system-wide cronjob is installed.
+
+
 REQUIRED PARAMETERS
 -------------------
 
@@ -21,6 +38,7 @@ object id
 admin-email
     Where to send Let's Encrypt emails like "certificate needs renewal".
 
+
 OPTIONAL PARAMETERS
 -------------------
 
@@ -36,25 +54,68 @@ webroot
     The path to your webroot, as set up in your webserver config. If this
     parameter is not present, Certbot will be run in standalone mode.
 
+
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
 
-renew-hook
-    Renew hook command directly passed to Certbot in cron job.
-
 domain
     Domains to be included in the certificate. When specified then object id
     is not used as a domain.
 
+deploy-hook
+    Command to be executed only when the certificate associated with this
+    ``$__object_id`` is issued or renewed.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    For this command, the
+    shell variable ``$RENEWED_LINEAGE`` will point to the
+    config live subdirectory (for example,
+    ``/etc/letsencrypt/live/${__object_id}``) containing the
+    new certificates and keys; the shell variable
+    ``$RENEWED_DOMAINS`` will contain a space-delimited list
+    of renewed certificate domains (for example,
+    ``example.com www.example.com``)
+
+pre-hook
+    Command to be run in a shell before obtaining any
+    certificates.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    Note these run regardless of which certificate is attempted, you may want to
+    manage these system-wide hooks with ``__file`` in
+    ``/etc/letsencrypt/renewal-hooks/pre/``.
+
+    Intended primarily for renewal, where it
+    can be used to temporarily shut down a webserver that
+    might conflict with the standalone plugin. This will
+    only be called if a certificate is actually to be
+    obtained/renewed.
+
+post-hook
+    Command to be run in a shell after attempting to
+    obtain/renew certificates.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    Note these run regardless of which certificate was attempted, you may want to
+    manage these system-wide hooks with ``__file`` in
+    ``/etc/letsencrypt/renewal-hooks/post/``.
+
+    Can be used to deploy
+    renewed certificates, or to restart any servers that
+    were stopped by --pre-hook. This is only run if an
+    attempt was made to obtain/renew a certificate.
+
+
 BOOLEAN PARAMETERS
 ------------------
 
-automatic-renewal
-    Install a cron job, which attempts to renew certificates daily.
-
 staging
     Obtain a test certificate from a staging server.
 
+
 MESSAGES
 --------
 
@@ -67,6 +128,7 @@ create
 remove
     Certificate was removed.
 
+
 EXAMPLES
 --------
 
@@ -75,8 +137,7 @@ EXAMPLES
     # use object id as domain
     __letsencrypt_cert example.com \
         --admin-email root@example.com \
-        --automatic-renewal \
-        --renew-hook "service nginx reload" \
+        --deploy-hook "service nginx reload" \
         --webroot /data/letsencrypt/root
 
 .. code-block:: sh
@@ -85,11 +146,10 @@ EXAMPLES
     # and example.com needs to be included again with domain parameter
     __letsencrypt_cert example.com \
         --admin-email root@example.com \
-        --automatic-renewal \
         --domain example.com \
         --domain foo.example.com \
         --domain bar.example.com \
-        --renew-hook "service nginx reload" \
+        --deploy-hook "service nginx reload" \
         --webroot /data/letsencrypt/root
 
 AUTHORS
@@ -99,11 +159,13 @@ AUTHORS
 | Kamila Součková <kamila--@--ksp.sk>
 | Darko Poljak <darko.poljak--@--gmail.com>
 | Ľubomír Kučera <lubomir.kucera.jr at gmail.com>
+| Evilham <contact@evilham.com>
+
 
 COPYING
 -------
 
-Copyright \(C) 2017-2018 Nico Schottelius, Kamila Součková, Darko Poljak and
+Copyright \(C) 2017-2021 Nico Schottelius, Kamila Součková, Darko Poljak and
 Ľubomír Kučera. You can redistribute it and/or modify it under the terms of
 the GNU General Public License as published by the Free Software Foundation,
 either version 3 of the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
old mode 100755
new mode 100644
index b4464366..9c8cc043
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -1,18 +1,20 @@
 #!/bin/sh
 
 certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")"
+state=$(cat "${__object}/parameter/state")
+os="$(cat "${__global:?}/explorer/os")"
 
 if [ -z "${certbot_fullpath}" ]; then
-	os="$(cat "${__global:?}/explorer/os")"
 	os_version="$(cat "${__global}/explorer/os_version")"
-
+	# Use this, very common value, as a default. It is OS-dependent
+	certbot_fullpath="/usr/bin/certbot"
 	case "$os" in
-        archlinux)
-            __package certbot
-            ;;
-        alpine)
-            __package certbot
-            ;;
+		archlinux)
+			__package certbot
+		;;
+		alpine)
+			__package certbot
+		;;
 		debian)
 			case "$os_version" in
 				8*)
@@ -48,9 +50,7 @@ if [ -z "${certbot_fullpath}" ]; then
 					exit 1
 				;;
 			esac
-
-			certbot_fullpath=/usr/bin/certbot
-			;;
+		;;
 		devuan)
 			case "$os_version" in
 				jessie)
@@ -83,17 +83,14 @@ if [ -z "${certbot_fullpath}" ]; then
 					exit 1
 				;;
 			esac
-
-			certbot_fullpath=/usr/bin/certbot
 		;;
 		freebsd)
-			__package py27-certbot
-
-			certbot_fullpath=/usr/local/bin/certbot
+			__package py37-certbot
+			certbot_fullpath="/usr/local/bin/certbot"
 		;;
 		ubuntu)
-            __package certbot
-            ;;
+			__package certbot
+		;;
 		*)
 			echo "Unsupported os: $os" >&2
 			exit 1
@@ -101,18 +98,130 @@ if [ -z "${certbot_fullpath}" ]; then
 	esac
 fi
 
-if [ -f "${__object}/parameter/automatic-renewal" ]; then
-	renew_hook_param="${__object}/parameter/renew-hook"
-	renew_hook=""
-	if [ -f "${renew_hook_param}" ]; then
-		while read -r hook; do
-			renew_hook="${renew_hook} --renew-hook \"${hook}\""
-		done < "${renew_hook_param}"
-	fi
+# Other OS-dependent values that we want to set every time
+LE_DIR="/etc/letsencrypt"
+certbot_cronjob_state="absent"
+case "$os" in
+	archlinux|alpine)
+		certbot_cronjob_state="present"
+	;;
+	freebsd)
+		LE_DIR="/usr/local/etc/letsencrypt"
+		# FreeBSD uses periodic(8) instead of crontabs for this
+		__line "periodic.conf_weekly_certbot" \
+			--file "/etc/periodic.conf" \
+			--regex "^(#[[:space:]]*)?weekly_certbot_enable=.*" \
+			--state "replace" \
+			--line 'weekly_certbot_enable="YES"'
+	;;
+	*)
+	;;
+esac
 
-	__cron letsencrypt-certbot  \
-		--user root \
-		--command "${certbot_fullpath} renew -q ${renew_hook}" \
-		--hour 0 \
-		--minute 47
+# This is only necessary in certain OS
+__cron letsencrypt-certbot \
+	--user root \
+	--command "${certbot_fullpath} renew -q" \
+	--hour 0 \
+	--minute 47 \
+	--state "${certbot_cronjob_state}"
+
+# Ensure hook directories
+HOOKS_DIR="${LE_DIR}/renewal-hooks"
+__directory "${LE_DIR}" --mode 0755
+require="__directory/${LE_DIR}" __directory "${HOOKS_DIR}" --mode 0755
+
+if [ -f "${__object}/parameter/domain" ]; then
+	domains="$(sort "${__object}/parameter/domain")"
+else
+	domains="${__object_id}"
 fi
+
+# Install hooks as needed
+for hook in deploy pre post; do
+	# Using something unique and specific to this object
+	hook_file="${HOOKS_DIR}/${hook}/${__object_id}.cdist.sh"
+	# Reasonable defaults
+	hook_source="${__object}/parameter/${hook}-hook"
+	hook_state="absent"
+	hook_contents_head="#!/bin/sh -e"
+	hook_contents_logic=""
+	hook_contents_tail=""
+
+	# Backwards compatibility
+	# Remove this when renew-hook is removed
+	# Falling back to renew-hook if deploy-hook is not passed
+	if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then
+		hook_source="${__object}/parameter/renew-hook"
+	fi
+	if [ "${state}" = "present" ] && \
+		[ -f "${hook_source}" ]; then
+		# This hook is to be installed, let's generate it with some
+		# safety boilerplate
+		# Since certbot runs all hooks for all renewal processes
+		# (at each state for deploy, pre, post), it is up to us to
+		# differentiate whether or not the hook must run
+		hook_state="present"
+		hook_contents_head="$(cat <<EOF
+#!/bin/sh -e
+#
+# Managed remotely with https://cdi.st
+#
+# Domains for which this hook is supposed to apply
+lineage="${LE_DIR}/live/${__object_id}"
+domains="\$(cat <<eof
+${domains}
+eof
+)"
+EOF
+)"
+		case "${hook}" in
+			pre|post)
+				# Certbot is kind of terrible, we have
+				# no way of knowing what domain/lineage the
+				# hook is running for
+				hook_contents_logic="$(cat <<EOF
+# pre/post-hooks apply always due to a certbot limitation
+APPLY_HOOK="YES"
+EOF
+)"
+			;;
+			deploy)
+				hook_contents_logic="$(cat <<EOF
+# certbot defines these:
+# RENEWED_DOMAINS: DOMAIN1,DOMAIN2
+# RENEWED_LINEAGE: /etc/letsencrypt/live/__object_id
+# It feels more stable to use RENEWED_LINEAGE
+if [ "\${lineage}" = "\${RENEWED_LINEAGE}" ]; then
+	APPLY_HOOK="YES"
+fi
+EOF
+)"
+			;;
+			*)
+				echo "Unknown hook '${hook}'" >> /dev/stderr
+				exit 1
+			;;
+		esac
+
+		hook_contents_tail="$(cat <<EOF
+if [ -n "\${APPLY_HOOK}" ]; then
+$(sed -e 's/^/	//' "${hook_source}")
+fi
+EOF
+)"
+	fi
+	# Ensure hook directory exists
+	require="__directory/${HOOKS_DIR}" __directory "${HOOKS_DIR}/${hook}" \
+		--mode 0755
+	require="__directory/${HOOKS_DIR}/${hook}" __file "${hook_file}" \
+		--mode 0555 \
+		--source '-' \
+		--state "${hook_state}" <<EOF
+${hook_contents_head}
+
+${hook_contents_logic}
+
+${hook_contents_tail}
+EOF
+done
diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal
new file mode 100644
index 00000000..dbfc5fa7
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal
@@ -0,0 +1,2 @@
+Deprecated in favour of consistent behaviour. It has no effect, see:
+https://code.ungleich.ch/ungleich-public/cdist/-/issues/853
diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook
new file mode 100644
index 00000000..be4a3460
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook
@@ -0,0 +1,2 @@
+This parameter has been deprecated in favour of --deploy-hook.
+See: https://code.ungleich.ch/ungleich-public/cdist/-/issues/853
diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple b/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple
index 0e866d45..bebb19bd 100644
--- a/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple
+++ b/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple
@@ -1,2 +1,5 @@
+deploy-hook
 domain
+post-hook
+pre-hook
 renew-hook

From e49da474c4e92775477c132833fb8c517db6921a Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 9 Feb 2021 20:29:17 +0100
Subject: [PATCH 372/527] [__letsencrypt_cert] Remove problematic trailing
 slash in sed.

Happy fingers are happy and like adding slashes places.
---
 cdist/conf/type/__letsencrypt_cert/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 9c8cc043..04edea1e 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -206,7 +206,7 @@ EOF
 
 		hook_contents_tail="$(cat <<EOF
 if [ -n "\${APPLY_HOOK}" ]; then
-$(sed -e 's/^/	//' "${hook_source}")
+$(sed -e 's/^/	/' "${hook_source}")
 fi
 EOF
 )"

From b832af5e3bce7d631dc1aafd6c4d9c2786391684 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 9 Feb 2021 20:53:58 +0100
Subject: [PATCH 373/527] [__letsencrypt_cert] Don't mess with user script
 indentation

This could break in odd ways if they passed sth like:
cat <<eof
bla bla
eof
---
 cdist/conf/type/__letsencrypt_cert/manifest | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 04edea1e..84221a80 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -206,7 +206,8 @@ EOF
 
 		hook_contents_tail="$(cat <<EOF
 if [ -n "\${APPLY_HOOK}" ]; then
-$(sed -e 's/^/	/' "${hook_source}")
+# Messing with indentation can eff up the users' scripts, let's not
+$(cat "${hook_source}")
 fi
 EOF
 )"

From aa80c09c803e132c3a05a642e9ada82ccb0e7dd8 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Wed, 10 Feb 2021 10:10:21 +0100
Subject: [PATCH 374/527] [__letsencrypt_cert] Move hook contents generation
 out of manifest

While there address some minor issues in the comments in the hook contents.
---
 .../type/__letsencrypt_cert/files/gen_hook.sh | 84 +++++++++++++++++++
 cdist/conf/type/__letsencrypt_cert/manifest   | 78 +----------------
 2 files changed, 88 insertions(+), 74 deletions(-)
 create mode 100644 cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh

diff --git a/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh b/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh
new file mode 100644
index 00000000..81ea4856
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh
@@ -0,0 +1,84 @@
+#!/bin/sh -e
+
+# It is expected that this defines hook_contents
+
+# Reasonable defaults
+hook_source="${__object}/parameter/${hook}-hook"
+hook_state="absent"
+hook_contents_head="#!/bin/sh -e"
+hook_contents_logic=""
+hook_contents_tail=""
+
+# Backwards compatibility
+# Remove this when renew-hook is removed
+# Falling back to renew-hook if deploy-hook is not passed
+if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then
+	hook_source="${__object}/parameter/renew-hook"
+fi
+if [ "${state}" = "present" ] && \
+	[ -f "${hook_source}" ]; then
+	# This hook is to be installed, let's generate it with some
+	# safety boilerplate
+	# Since certbot runs all hooks for all renewal processes
+	# (at each state for deploy, pre, post), it is up to us to
+	# differentiate whether or not the hook must run
+	hook_state="present"
+	hook_contents_head="$(cat <<EOF
+#!/bin/sh -e
+#
+# Managed remotely with https://cdi.st
+#
+# Domains for which this hook is supposed to apply
+lineage="${LE_DIR}/live/${__object_id}"
+domains="\$(cat <<eof
+${domains}
+eof
+)"
+EOF
+)"
+	case "${hook}" in
+		pre|post)
+			# Certbot is kind of terrible, we have
+			# no way of knowing what domain/lineage the
+			# hook is running for
+			hook_contents_logic="$(cat <<EOF
+# pre/post-hooks apply always due to a certbot limitation
+APPLY_HOOK="YES"
+EOF
+)"
+		;;
+		deploy)
+			hook_contents_logic="$(cat <<EOF
+# certbot defines these environment variables:
+# RENEWED_DOMAINS="DOMAIN1 DOMAIN2"
+# RENEWED_LINEAGE="/etc/letsencrypt/live/__object_id"
+# It feels more stable to use RENEWED_LINEAGE
+if [ "\${lineage}" = "\${RENEWED_LINEAGE}" ]; then
+	APPLY_HOOK="YES"
+fi
+EOF
+)"
+		;;
+		*)
+			echo "Unknown hook '${hook}'" >> /dev/stderr
+			exit 1
+		;;
+	esac
+
+	hook_contents_tail="$(cat <<EOF
+if [ -n "\${APPLY_HOOK}" ]; then
+	# Messing with indentation can eff up the users' scripts, let's not
+$(cat "${hook_source}")
+fi
+EOF
+)"
+fi
+
+hook_contents="$(cat <<EOF
+${hook_contents_head}
+
+${hook_contents_logic}
+
+${hook_contents_tail}
+EOF
+)"
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 84221a80..1df3574a 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -141,77 +141,11 @@ fi
 for hook in deploy pre post; do
 	# Using something unique and specific to this object
 	hook_file="${HOOKS_DIR}/${hook}/${__object_id}.cdist.sh"
-	# Reasonable defaults
-	hook_source="${__object}/parameter/${hook}-hook"
-	hook_state="absent"
-	hook_contents_head="#!/bin/sh -e"
-	hook_contents_logic=""
-	hook_contents_tail=""
 
-	# Backwards compatibility
-	# Remove this when renew-hook is removed
-	# Falling back to renew-hook if deploy-hook is not passed
-	if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then
-		hook_source="${__object}/parameter/renew-hook"
-	fi
-	if [ "${state}" = "present" ] && \
-		[ -f "${hook_source}" ]; then
-		# This hook is to be installed, let's generate it with some
-		# safety boilerplate
-		# Since certbot runs all hooks for all renewal processes
-		# (at each state for deploy, pre, post), it is up to us to
-		# differentiate whether or not the hook must run
-		hook_state="present"
-		hook_contents_head="$(cat <<EOF
-#!/bin/sh -e
-#
-# Managed remotely with https://cdi.st
-#
-# Domains for which this hook is supposed to apply
-lineage="${LE_DIR}/live/${__object_id}"
-domains="\$(cat <<eof
-${domains}
-eof
-)"
-EOF
-)"
-		case "${hook}" in
-			pre|post)
-				# Certbot is kind of terrible, we have
-				# no way of knowing what domain/lineage the
-				# hook is running for
-				hook_contents_logic="$(cat <<EOF
-# pre/post-hooks apply always due to a certbot limitation
-APPLY_HOOK="YES"
-EOF
-)"
-			;;
-			deploy)
-				hook_contents_logic="$(cat <<EOF
-# certbot defines these:
-# RENEWED_DOMAINS: DOMAIN1,DOMAIN2
-# RENEWED_LINEAGE: /etc/letsencrypt/live/__object_id
-# It feels more stable to use RENEWED_LINEAGE
-if [ "\${lineage}" = "\${RENEWED_LINEAGE}" ]; then
-	APPLY_HOOK="YES"
-fi
-EOF
-)"
-			;;
-			*)
-				echo "Unknown hook '${hook}'" >> /dev/stderr
-				exit 1
-			;;
-		esac
+	# This defines hook_contents
+	# shellcheck source=cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh
+	. "${__type}/files/gen_hook.sh"
 
-		hook_contents_tail="$(cat <<EOF
-if [ -n "\${APPLY_HOOK}" ]; then
-# Messing with indentation can eff up the users' scripts, let's not
-$(cat "${hook_source}")
-fi
-EOF
-)"
-	fi
 	# Ensure hook directory exists
 	require="__directory/${HOOKS_DIR}" __directory "${HOOKS_DIR}/${hook}" \
 		--mode 0755
@@ -219,10 +153,6 @@ EOF
 		--mode 0555 \
 		--source '-' \
 		--state "${hook_state}" <<EOF
-${hook_contents_head}
-
-${hook_contents_logic}
-
-${hook_contents_tail}
+${hook_contents}
 EOF
 done

From 4717e5ceff078111748824e4fb365df7979e5fe7 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 10:31:07 +0100
Subject: [PATCH 375/527] __package_pip: add extras explorer

The two new explorers detect all installed extras for this package.
---
 .../type/__package_pip/explorer/distinfo-dir  | 45 +++++++++++++++++++
 cdist/conf/type/__package_pip/explorer/extras | 26 +++++++++++
 cdist/conf/type/__package_pip/explorer/state  |  0
 3 files changed, 71 insertions(+)
 create mode 100755 cdist/conf/type/__package_pip/explorer/distinfo-dir
 create mode 100755 cdist/conf/type/__package_pip/explorer/extras
 mode change 100644 => 100755 cdist/conf/type/__package_pip/explorer/state

diff --git a/cdist/conf/type/__package_pip/explorer/distinfo-dir b/cdist/conf/type/__package_pip/explorer/distinfo-dir
new file mode 100755
index 00000000..18e169ae
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/distinfo-dir
@@ -0,0 +1,45 @@
+#!/bin/sh
+#
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+nameparam="$__object/parameter/name"
+if [ -f "$nameparam" ]; then
+    name=$(cat "$nameparam")
+else
+    name="$__object_id"
+fi
+
+pipparam="$__object/parameter/pip"
+if [ -f "$pipparam" ]; then
+    pip=$(cat "$pipparam")
+else
+    pip="$( "$__type_explorer/pip" )"
+fi
+
+
+if command -v "$pip" >/dev/null 2>&1; then
+    # assemble the path where pip stores all pip package info
+    "$pip" show "$name" \
+        | awk -F': ' '
+            $1 == "Name" {name=$2; gsub(/-/,"_",name); next}
+            $1 == "Version" {version=$2; next}
+            $1 == "Location" {location=$2; next}
+            END {if (version != "") printf "%s/%s-%s.dist-info", location, name, version}'
+fi
diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
new file mode 100755
index 00000000..92fc4732
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -0,0 +1,26 @@
+#!/bin/sh
+#
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+distinfo_dir="$("$__type_explorer/distinfo-dir")"
+if [ "$distinfo_dir" ]; then
+    # output all extras that are installed
+    awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA"
+fi
diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state
old mode 100644
new mode 100755

From 8dc6ab97384f2240e9218fbd4463ba12b004f44c Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 13:49:53 +0100
Subject: [PATCH 376/527] __package_pip: install not found extras

Compares the explorer against the parameters and install those extras
that are not already installed.
---
 cdist/conf/type/__package_pip/gencode-remote | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 925eaf24..ad327e37 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -2,6 +2,7 @@
 #
 # 2012 Nico Schottelius (nico-cdist at schottelius.org)
 # 2016 Darko Poljak (darko.poljak at gmail.com)
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
 #
 # This file is part of cdist.
 #
@@ -58,14 +59,17 @@ case "$state_should" in
     present)
         if [ -f "$__object/parameter/extra" ]
         then
-            while read -r extra
-            do
-                if [ "$extras" ]; then
-                    extras="$extras,$extra"
-                else
-                    extras="$extra"
-                fi
-            done < "$__object/parameter/extra"
+            mkdir "$__object/files"
+            # sort and generalize all extras
+            sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params"
+            sort "$__object/explorer/extras" > "$__object/files/extras-explorer"
+
+            # assemble extras
+            # all extras are passed to pip in a comma-separated list
+            extras="$(comm -23 \
+                "$__object/files/extras-params" "$__object/files/extras-explorer" \
+                | sed ':a; $!N; s/\n/,/; ta'  # loops through all input lines and add commas between them
+            )"
             name="${name}[${extras}]"
         fi
 

From 2db0ef7c98226661e5e4b55c89180b31ba65b8bc Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 22:53:26 +0100
Subject: [PATCH 377/527] __package_pip: updating real detection of extras

As the previous detection took the wrong values, this explorer now
checks if packages for an extra are installed or not. If not, the extra
is not installed.

Based on the information of the explorer, it will install the package
again with the absent extras.
---
 cdist/conf/type/__package_pip/explorer/extras | 37 +++++++++++++++++--
 cdist/conf/type/__package_pip/gencode-remote  | 21 ++++-------
 2 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 92fc4732..2f5fcca6 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -17,10 +17,41 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+#
+# Checks if the given extras are really installed or not. It will be
+# done by querring all dependencies for that extra and return it as
+# "to be installed" if no dependency was found.
+#
 
 
 distinfo_dir="$("$__type_explorer/distinfo-dir")"
-if [ "$distinfo_dir" ]; then
-    # output all extras that are installed
-    awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA"
+
+# check if we have something to check
+if [ "$distinfo_dir" ] && [ -s "$__object/parameter/extra" ]
+then
+    # save cause freezing is slow
+    mkdir "$__object/files"
+    pip_freeze="$__object/files/pip-freeze.tmp"
+    pip3 freeze > "$pip_freeze"
+
+    for extra in $(cat "$__object/parameter/extra" | tr ',' '\n')
+    do
+        # create a grep BRE pattern to search all packages
+        grep_pattern="$(
+            awk -F'(: | ; )' -v check="$extra" '
+                $1 == "Requires-Dist" {
+                    split($2, r, " ");
+                    sub("extra == ", "", $3); gsub("'"'"'", "", $3);
+                    if($3 == check) print r[1]
+                }' "$distinfo_dir/METADATA" \
+                | sed ':a; $!N; s/\n/\\|/; ta'
+        )"
+
+        # echo the extra if no packages where found for it
+        # if there is no pattern, we don't need to search ;-)
+        if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze"
+        then
+            echo "$extra"
+        fi
+    done
 fi
diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index ad327e37..9abe28bf 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -26,7 +26,10 @@
 state_is=$(cat "$__object/explorer/state")
 state_should="$(cat "$__object/parameter/state")"
 
-[ "$state_is" = "$state_should" ] && exit 0
+# short circuit if state is the same and no extras to install
+[ "$state_is" = "$state_should" ] && ! [ -s "$__object/explorer/extras" ] \
+    && exit 0
+
 
 nameparam="$__object/parameter/name"
 if [ -f "$nameparam" ]; then
@@ -57,19 +60,11 @@ fi
 
 case "$state_should" in
     present)
-        if [ -f "$__object/parameter/extra" ]
+        if [ -s "$__object/explorer/extras" ]
         then
-            mkdir "$__object/files"
-            # sort and generalize all extras
-            sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params"
-            sort "$__object/explorer/extras" > "$__object/files/extras-explorer"
-
-            # assemble extras
-            # all extras are passed to pip in a comma-separated list
-            extras="$(comm -23 \
-                "$__object/files/extras-params" "$__object/files/extras-explorer" \
-                | sed ':a; $!N; s/\n/,/; ta'  # loops through all input lines and add commas between them
-            )"
+            # all extras are passed to pip in a comma-separated list in the name
+            # sed loops through all input lines and add commas between them
+            extras="$(sed ':a; $!N; s/\n/,/; ta' "$__object/explorer/extras")"
             name="${name}[${extras}]"
         fi
 

From 7398382890a4368e588e700183e6c777e90fb069 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 23:12:10 +0100
Subject: [PATCH 378/527] __package_pip: fix shellcheck

Useless `cat $file`, use `< $file` instead.
---
 cdist/conf/type/__package_pip/explorer/extras | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 2f5fcca6..d57c69db 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -34,7 +34,7 @@ then
     pip_freeze="$__object/files/pip-freeze.tmp"
     pip3 freeze > "$pip_freeze"
 
-    for extra in $(cat "$__object/parameter/extra" | tr ',' '\n')
+    for extra in $(tr ',' '\n' < "$__object/parameter/extra")
     do
         # create a grep BRE pattern to search all packages
         grep_pattern="$(

From a9d7dfb2ed40d0a61978cc706ff469c354a6c8f5 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 12 Feb 2021 09:17:02 +0100
Subject: [PATCH 379/527] __package_pip: split extra 'all' to a list of all
 extras

This will fix if a package will be upgraded from some extras to all
extras. Previously, it will not work because some dependencies of 'all'
are already installed, so the feature 'all' is already installed.

Now, it will use a list of all extras to iterate over them separatly. This
will result it will never install all extras via `[all]`, but rather
`[foo,bar]`.
---
 cdist/conf/type/__package_pip/explorer/extras | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index d57c69db..be945e36 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -34,7 +34,14 @@ then
     pip_freeze="$__object/files/pip-freeze.tmp"
     pip3 freeze > "$pip_freeze"
 
-    for extra in $(tr ',' '\n' < "$__object/parameter/extra")
+    # If all is set, it searches all available extras to separatly check them.
+    # It would work with just 'all' (cause dependencies are given), but will
+    # not update if one extra is already present. Side effect is that it will
+    # not use [all] but instead name all extras seperatly.
+    for extra in $(if grep -qFx all "$__object/parameter/extra";
+        then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA";
+        else tr ',' '\n' < "$__object/parameter/extra";
+        fi)
     do
         # create a grep BRE pattern to search all packages
         grep_pattern="$(

From 951712740f95f8f1277f1295bcd97c0b9bafdc94 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 12 Feb 2021 13:42:51 +0100
Subject: [PATCH 380/527] __package_pip: update man.rst

Adjusted comments for `explorer/extras` and updated the man page for the
new behaviour of updating the extras.
---
 cdist/conf/type/__package_pip/explorer/extras |  7 +++---
 cdist/conf/type/__package_pip/man.rst         | 25 ++++++++++++-------
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index be945e36..8104ed47 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -35,15 +35,16 @@ then
     pip3 freeze > "$pip_freeze"
 
     # If all is set, it searches all available extras to separatly check them.
-    # It would work with just 'all' (cause dependencies are given), but will
-    # not update if one extra is already present. Side effect is that it will
-    # not use [all] but instead name all extras seperatly.
+    # It would work with just 'all' (cause dependencies are specified for
+    # 'all'), but will not update if one extra is already present. Side effect
+    # is that it will not use [all] but instead name all extras seperatly.
     for extra in $(if grep -qFx all "$__object/parameter/extra";
         then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA";
         else tr ',' '\n' < "$__object/parameter/extra";
         fi)
     do
         # create a grep BRE pattern to search all packages
+        # maybe a file full of patterns for -F could be written
         grep_pattern="$(
             awk -F'(: | ; )' -v check="$extra" '
                 $1 == "Requires-Dist" {
diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index 64ad0358..c013695c 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -24,8 +24,14 @@ name
 
 extra
     Extra optional dependencies which should be installed along the selected
-    package. Can be specified multiple times. Will only be applied if the
-    package actually will be installed, but will not explicitly checked.
+    package. Can be specified multiple times. Multiple extra optional
+    dependencies can also be specified in a comma-separated list to provide
+    a more pip-natvie style.
+
+    Extra optional dependencies will be installed even when the base package
+    is already installed. Notice that the type will not remove installed extras
+    that are not explicitly named for the type because pip does not offer a
+    management for orphaned packages and they may be used by other packages.
 
 pip
     Instead of using pip from PATH, use the specific pip path.
@@ -53,8 +59,8 @@ EXAMPLES
 
     # Install package with optional dependencies
     __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails
-    # or do a little cheating
-    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails
+    # the extras can also be specified comma-separated
+    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails --extra postgres
 
     # or take all extras
     __package_pip mautrix-telegram --extra all
@@ -67,12 +73,13 @@ SEE ALSO
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2012 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2012 Nico Schottelius, 2021 Matthias Stecher. You can
+redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either
+version 3 of the License, or (at your option) any later version.

From 2ce1fce7677b64a8d17a86f29f676b1ba8475db2 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 15 Feb 2021 16:17:46 +0100
Subject: [PATCH 381/527] __package_pip: match package names case insensitive

Pip matches them insensitive, so we need to do the same to avoid
problems by saying extras are not installed but already is there in
place.
---
 cdist/conf/type/__package_pip/explorer/extras | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 8104ed47..bbdc17ab 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -57,7 +57,8 @@ then
 
         # echo the extra if no packages where found for it
         # if there is no pattern, we don't need to search ;-)
-        if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze"
+        # pip matches packages case-insensetive, we need to do that, too
+        if [ "$grep_pattern" ] && ! grep -qi "$grep_pattern" "$pip_freeze"
         then
             echo "$extra"
         fi

From 0835f414a5b2ce41309ee9b265cb80abb1362563 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 16 Feb 2021 16:03:23 +0100
Subject: [PATCH 382/527] [type/__postgres_conf] Extract PostgreSQL service
 user detection to separate explorer

---
 .../__postgres_conf/explorer/postgres_user    | 64 +++++++++++++++++++
 .../conf/type/__postgres_conf/explorer/state  | 16 +----
 .../conf/type/__postgres_conf/gencode-remote  | 17 +----
 3 files changed, 67 insertions(+), 30 deletions(-)
 create mode 100644 cdist/conf/type/__postgres_conf/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_conf/explorer/postgres_user b/cdist/conf/type/__postgres_conf/explorer/postgres_user
new file mode 100644
index 00000000..c6582dc4
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/explorer/postgres_user
@@ -0,0 +1,64 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$("${__explorer:?}/os")
+
+case ${os}
+in
+	(alpine)
+		echo 'postgres'
+		;;
+	(centos|rhel|scientific)
+		echo 'postgres'
+		;;
+	(debian|devuan|ubuntu)
+		echo 'postgres'
+		;;
+	(freebsd)
+		test -x /usr/local/etc/rc.d/postgresql || {
+			printf 'could not find postgresql rc script./n' >&2
+			exit 1
+		}
+		pg_status=$(/usr/local/etc/rc.d/postgresql onestatus) || {
+			printf 'postgresql daemon is not running.\n' >&2
+			exit 1
+		}
+		pg_pid=$(printf '%s\n' "${pg_status}" \
+			| sed -n 's/^pg_ctl:.*(PID: *\([0-9]*\))$/\1/p')
+
+		# PostgreSQL < 9.6: pgsql
+		# PostgreSQL >= 9.6: postgres
+		ps -o user -p "${pg_pid}" | sed -n '2p'
+		;;
+	(netbsd)
+		echo 'pgsql'
+		;;
+	(openbsd)
+		echo '_postgresql'
+		;;
+	(suse)
+		echo 'postgres'
+		;;
+	(*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 589925de..de6e8aa0 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -19,21 +19,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$("${__explorer:?}/os")
-
-case ${os}
-in
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(devuan)
-		postgres_user='postgres'
-		;;
-	(*)
-		echo "Unsupported OS: ${os}" >&2
-		exit 1
-		;;
-esac
+postgres_user=$("${__type_explorer:?}/postgres_user")
 
 conf_name=${__object_id:?}
 
diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 8f8a2bfe..d0d247b4 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -20,9 +20,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$(cat "${__global:?}/explorer/os")
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
 conf_name=${__object_id:?}
 
@@ -49,19 +49,6 @@ quote() {
 	printf '%s' "$*"
 }
 
-case ${os}
-in
-	(netbsd)
-		postgres_user='pgsql'
-		;;
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(*)
-		postgres_user='postgres'
-		;;
-esac
-
 
 psql_cmd() {
 	printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")"
@@ -117,7 +104,7 @@ then
 				case $(cat "${__global:?}/explorer/kernel_name")
 				in
 					(FreeBSD)
-						echo 'service postgresql restart'
+						echo '/usr/local/etc/rc.d/postgresql restart'
 						;;
 					(OpenBSD|NetBSD)
 						echo '/etc/rc.d/postgresql restart'

From d1f45d3524f8a025488d0d2f020433b6efb7edb6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 19 Feb 2021 09:03:56 +0100
Subject: [PATCH 383/527] __package_pip: corrected typo in man

.. by fully replacing it with a smaller sentence.
---
 cdist/conf/type/__package_pip/man.rst | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index c013695c..5a2bc673 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -24,9 +24,8 @@ name
 
 extra
     Extra optional dependencies which should be installed along the selected
-    package. Can be specified multiple times. Multiple extra optional
-    dependencies can also be specified in a comma-separated list to provide
-    a more pip-natvie style.
+    package. Can be specified multiple times. Multiple extras can be passed
+    in one `--extra` as a comma-separated list.
 
     Extra optional dependencies will be installed even when the base package
     is already installed. Notice that the type will not remove installed extras

From 5e0572189fb930764318227fa9fd9f4966b0da99 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 22 Feb 2021 09:11:22 +0100
Subject: [PATCH 384/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 52686617..59c58e65 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,8 @@ next:
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
+	* Type __postgres_role: Implement modification of roles (Dennis Camera)
+	* Type __letsencrypt_cert: Fix issues with hooks (Evil Ham)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)

From 22f637c15b6616899c848e2a08c4e1c1b13f0f0b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 23 Feb 2021 06:29:24 +0100
Subject: [PATCH 385/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 59c58e65..88dda0aa 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
 	* Type __postgres_role: Implement modification of roles (Dennis Camera)
 	* Type __letsencrypt_cert: Fix issues with hooks (Evil Ham)
+	* Type __package_pip: Add optional extra dependencies param (Matthias Stecher)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)

From 0734288483700e8e10cebd87c797b625aa83d55e Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Sun, 21 Feb 2021 19:59:57 +0000
Subject: [PATCH 386/527] First draft of __apt_pin

---
 cdist/conf/type/__apt_pin/man.rst             | 53 ++++++++++++++++
 cdist/conf/type/__apt_pin/manifest            | 63 +++++++++++++++++++
 .../type/__apt_pin/parameter/default/package  |  1 +
 .../type/__apt_pin/parameter/default/state    |  1 +
 cdist/conf/type/__apt_pin/parameter/optional  |  2 +
 cdist/conf/type/__apt_pin/parameter/required  |  2 +
 6 files changed, 122 insertions(+)
 create mode 100644 cdist/conf/type/__apt_pin/man.rst
 create mode 100755 cdist/conf/type/__apt_pin/manifest
 create mode 100644 cdist/conf/type/__apt_pin/parameter/default/package
 create mode 100644 cdist/conf/type/__apt_pin/parameter/default/state
 create mode 100644 cdist/conf/type/__apt_pin/parameter/optional
 create mode 100644 cdist/conf/type/__apt_pin/parameter/required

diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst
new file mode 100644
index 00000000..7fcae6f8
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/man.rst
@@ -0,0 +1,53 @@
+cdist-type__apt_pin(7)
+======================
+
+NAME
+----
+cdist-type__apt_pin - TODO
+
+
+DESCRIPTION
+-----------
+This space intentionally left blank.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # TODO
+    __apt_pin
+
+
+SEE ALSO
+--------
+:strong:`TODO`\ (7)
+
+
+AUTHORS
+-------
+Daniel Fancsali <fancsali@gmail.com>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Daniel Fancsali. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
new file mode 100755
index 00000000..8dd9770d
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -0,0 +1,63 @@
+#!/bin/sh -e
+#
+# 2021 Daniel Fancsali (fancsali@gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+os=$(cat "$__global/explorer/os")
+state="$(cat "$__object/parameter/state")"
+package="$(cat "$__object/parameter/package")"
+distribution="$(cat "$__object/parameter/distribution")"
+priority="$(cat "$__object/parameter/priority")"
+
+
+case "$os" in
+   debian|ubuntu|devuan)
+   ;;
+   *)
+      printf "This type is specific to Debian and it's derivatives" >&2
+      printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2
+      exit 1
+   ;;
+esac
+
+if [ "$package" = "*" ]; then
+    name="default"
+
+else
+    name="$__object_id"
+fi
+
+case $distribution in
+    stabletesting|unsatbel|experimental)
+        pin="release a=$distribution"
+        ;;
+    *)
+        pin="release n=$distribution"
+        ;;
+esac
+
+
+__file /etc/apt/preferences.d/$name \
+    --owner root --group root --mode 0644 \
+    --state "$state" \
+    --source - << EOF
+Package: $package
+Pin: $pin
+Pin-Priority: $priority
+EOF
diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package
new file mode 100644
index 00000000..72e8ffc0
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/default/package
@@ -0,0 +1 @@
+*
diff --git a/cdist/conf/type/__apt_pin/parameter/default/state b/cdist/conf/type/__apt_pin/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional
new file mode 100644
index 00000000..52f01fd2
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/optional
@@ -0,0 +1,2 @@
+state
+package
diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required
new file mode 100644
index 00000000..4b4e9741
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/required
@@ -0,0 +1,2 @@
+distribution
+priority

From 1a74470c4d9b30e41a72fbfc084dc54ea44e643b Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Tue, 23 Feb 2021 09:37:36 +0000
Subject: [PATCH 387/527] __apt_pin: Always use $__object_id as preferences.d
 filename

---
 cdist/conf/type/__apt_pin/manifest | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index 8dd9770d..162b523f 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -36,12 +36,7 @@ case "$os" in
    ;;
 esac
 
-if [ "$package" = "*" ]; then
-    name="default"
-
-else
-    name="$__object_id"
-fi
+name="$__object_id"
 
 case $distribution in
     stabletesting|unsatbel|experimental)

From dc66efa690e15ff32d6836e23baa1cbec5eee1ed Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Tue, 23 Feb 2021 11:59:09 +0000
Subject: [PATCH 388/527] Fix shellcheck issues

---
 cdist/conf/type/__apt_pin/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index 162b523f..b1372ad0 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -48,7 +48,7 @@ case $distribution in
 esac
 
 
-__file /etc/apt/preferences.d/$name \
+__file "/etc/apt/preferences.d/$name" \
     --owner root --group root --mode 0644 \
     --state "$state" \
     --source - << EOF

From 60fd7ba1f38382761c366cbc0425ddaec62f1164 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 28 Feb 2021 13:37:23 +0100
Subject: [PATCH 389/527] Release 6.9.5

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 88dda0aa..95e36b88 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)

From 8ef19d47f6b5fb6cdcba7a9a3b8890388dbd7023 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 1 Mar 2021 17:59:25 +0100
Subject: [PATCH 390/527] [type/__pyvenv] Fix example (--user -> --owner)

---
 cdist/conf/type/__pyvenv/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__pyvenv/man.rst b/cdist/conf/type/__pyvenv/man.rst
index 8085ff12..e2e4a1e6 100644
--- a/cdist/conf/type/__pyvenv/man.rst
+++ b/cdist/conf/type/__pyvenv/man.rst
@@ -61,7 +61,7 @@ EXAMPLES
     __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4
 
     # Create python virtualenv for user foo.
-    __pyvenv /home/foo/fooenv --group foo --user foo
+    __pyvenv /home/foo/fooenv --group foo --owner foo
 
     # Create python virtualenv with specific parameters.
     __pyvenv /home/services/djangoenv --venvparams "--copies --system-site-packages"

From e7d33891df945831d54d164eb13f2a9ae4f9dd8e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 2 Mar 2021 09:29:33 +0100
Subject: [PATCH 391/527] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 95e36b88..072fcb6b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __pyvenv: Fix user example in man page (Dennis Camera)
+
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)

From ea0126dd8117c051612e6b6abfe895f5f722c584 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Fri, 5 Mar 2021 16:11:49 +0100
Subject: [PATCH 392/527] Make local state dir available to custom remote
 scripts

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/config.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cdist/config.py b/cdist/config.py
index e84f6f84..19d5bd70 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -420,6 +420,9 @@ class Config:
                 exec_path=sys.argv[0],
                 save_output_streams=args.save_output_streams)
 
+            # Make __global state dir available to custom remote scripts.
+            os.environ['__global'] = local.base_path
+
             remote = cdist.exec.remote.Remote(
                 target_host=target_host,
                 remote_exec=remote_exec,

From ecba284fc8ce68486c5f1f87a863409fcad0f106 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Fri, 5 Mar 2021 16:13:02 +0100
Subject: [PATCH 393/527] changelog++

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 072fcb6b..ad93a595 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
+	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From fb19f342669d18e6decb21ee4cfb2b22e46748d9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 9 Mar 2021 21:15:26 +0100
Subject: [PATCH 394/527] [type/__ssh_authorized_key] Only grep if file exists

---
 cdist/conf/type/__ssh_authorized_key/explorer/entry | 1 +
 cdist/conf/type/__ssh_authorized_key/gencode-remote | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__ssh_authorized_key/explorer/entry b/cdist/conf/type/__ssh_authorized_key/explorer/entry
index ccab0afc..aca0f2b9 100755
--- a/cdist/conf/type/__ssh_authorized_key/explorer/entry
+++ b/cdist/conf/type/__ssh_authorized_key/explorer/entry
@@ -25,6 +25,7 @@ type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ]
 if [ -n "${type_and_key}" ]
 then
     file="$(cat "$__object/parameter/file")"
+    test -e "$file" || exit 0
 
     # get any entries that match the type and key
 
diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote
index f37aa565..61c77fb9 100755
--- a/cdist/conf/type/__ssh_authorized_key/gencode-remote
+++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote
@@ -37,9 +37,9 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
 # preserve ownership and permissions of existing file
 if [ -f "$file" ]; then
    cp -p "$file" "\$tmpfile"
+   grep -v -F -x '$line' '$file' >\$tmpfile
 fi
-grep -v -F -x '$line' '$file' > \$tmpfile || true
-mv -f "\$tmpfile" "$file"
+cat "\$tmpfile" >"$file"
 DONE
 }
 

From 31cc592aa10902c3e83357ee4a0b9300b5e34efa Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 10 Mar 2021 19:25:04 +0100
Subject: [PATCH 395/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index ad93a595..c63b901a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
+	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From e47c4dd8a4cc441b94fe73d362d79525d01a4bb1 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 9 Mar 2021 19:59:05 +0100
Subject: [PATCH 396/527] [type/__sshd_config] Whitelist OpenBMC in manifest

---
 cdist/conf/type/__sshd_config/manifest | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest
index 566bde90..e37afebb 100755
--- a/cdist/conf/type/__sshd_config/manifest
+++ b/cdist/conf/type/__sshd_config/manifest
@@ -39,7 +39,14 @@ in
 	(freebsd|netbsd|openbsd)
 		# whitelist
 		;;
+	(openbmc-phosphor)
+		# whitelist
+		# OpenBMC can be configured with dropbear and OpenSSH.
+		# If dropbear is used, the state explorer will already fail because it
+		# cannot find the sshd binary.
+		;;
 	(*)
+		: "${__type:?}"  # make shellcheck happy
 		printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \
 			"${os}" "${__type##*/}" >&2
 		printf 'Please contribute an implementation for it if you can.\n' >&2

From 10ca1c12fd25a9f2b92cebbc5d375a1723ed5e2f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 12 Mar 2021 08:21:03 +0100
Subject: [PATCH 397/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index c63b901a..42a74d04 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
+	* Type __sshd_config: Whitelist OpenBMC (Dennis Camera)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From 7a0b697f4c394f1ae8a8941a59937007cfe474aa Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 21 Mar 2021 18:10:06 +0100
Subject: [PATCH 398/527] Implement maintaining object relationship graph

For each object maintain parent-child relationship graph, i.e. list of
parent objects ('parents' property) and list of children objects ('children'
property).

Objects without parent(s) are objects specified in init manifest.
Objects without children are object of types that do not reuse other types.
---
 cdist/core/cdist_object.py |  7 +++++++
 cdist/emulator.py          | 19 +++++++++++++++++++
 docs/src/cdist-cache.rst   | 17 +++++++++++++++++
 3 files changed, 43 insertions(+)

diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index 51d61e04..eea2c255 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -247,6 +247,13 @@ class CdistObject:
             lambda obj: os.path.join(obj.absolute_path, 'typeorder'))
     typeorder_dep = fsproperty.FileListProperty(
             lambda obj: os.path.join(obj.absolute_path, 'typeorder_dep'))
+    # objects without parents are objects specified in init manifest
+    parents = fsproperty.FileListProperty(
+            lambda obj: os.path.join(obj.absolute_path, 'parents'))
+    # objects without children are object of types that do not reuse other
+    # types
+    children = fsproperty.FileListProperty(
+            lambda obj: os.path.join(obj.absolute_path, 'children'))
 
     def cleanup(self):
         try:
diff --git a/cdist/emulator.py b/cdist/emulator.py
index a2bdc3d4..f1db862e 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -106,6 +106,7 @@ class Emulator:
             self.save_stdin()
             self.record_requirements()
             self.record_auto_requirements()
+            self.record_parent_child_relationships()
             self.log.trace("Finished %s %s" % (
                 self.cdist_object.path, self.parameters))
 
@@ -420,3 +421,21 @@ class Emulator:
                 self.log.debug("Recording autorequirement %s for %s",
                                current_object.name, parent.name)
                 parent.autorequire.append(current_object.name)
+
+    def record_parent_child_relationships(self):
+        # __object_name is the name of the object whose type manifest is
+        # currently executed
+        __object_name = self.env.get('__object_name', None)
+        if __object_name:
+            # The object whose type manifest is currently run
+            parent = self.cdist_object.object_from_name(__object_name)
+            # The object currently being defined
+            current_object = self.cdist_object
+            if current_object.name not in parent.children:
+                self.log.debug("Recording child %s for %s",
+                               current_object.name, parent.name)
+                parent.children.append(current_object.name)
+            if parent.name not in current_object.parents:
+                self.log.debug("Recording parent %s for %s",
+                               parent.name, current_object.name)
+                current_object.parents.append(parent.name)
diff --git a/docs/src/cdist-cache.rst b/docs/src/cdist-cache.rst
index d2d2d56c..d4159e77 100644
--- a/docs/src/cdist-cache.rst
+++ b/docs/src/cdist-cache.rst
@@ -61,6 +61,14 @@ Object cache overview
 ~~~~~~~~~~~~~~~~~~~~~
 Each object under :strong:`object` directory has its own structure.
 
+autorequire
+    file containing a list of object auto requirements
+
+children
+    file containing a list of object children, i.e. objects of types that this
+    type reuses (along with 'parents' it is used for maintaining parent-child
+    relationship graph)
+
 code-local
     code generated from gencode-local, present only if something is
     generated
@@ -80,6 +88,15 @@ parameter
     directory containing type parameter named files containing parameter
     values   
 
+parents
+    file containing a list of object parents, i.e. objects of types that reuse
+    this type (along with 'children' it is used for maintaining parent-child
+    relationship graph); objects without parents are objects specified in init
+    manifest
+
+require
+    file containing a list of object requirements
+
 source
     this type's source (init manifest)
 

From 167c2ad7ea2b895cdc7d89c72968d88c7d1ab34f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 30 Mar 2021 13:24:56 +0200
Subject: [PATCH 399/527] [type/__git] Fix if --owner / --group is numeric

Before, if --owner and/or --group was numeric, gencode-remote would generate
`chown` code every time.
---
 cdist/conf/type/__git/explorer/group | 20 +++++++++++++++++---
 cdist/conf/type/__git/explorer/owner | 20 +++++++++++++++++---
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group
index 3ddf9656..1365c60d 100644
--- a/cdist/conf/type/__git/explorer/group
+++ b/cdist/conf/type/__git/explorer/group
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id/.git"
+destination="/${__object_id:?}/.git"
 
-stat --print "%G" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((group_gid + 1)) -ge 0
+then
+	group_should=$(cat "${__object:?}/parameter/group")
+
+	if expr "${group_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${group_gid}"
+	else
+		printf '%s\n' "$(id -u -n "${group_gid}")"
+	fi
+fi
diff --git a/cdist/conf/type/__git/explorer/owner b/cdist/conf/type/__git/explorer/owner
index 4c3cd431..4a4d0d13 100644
--- a/cdist/conf/type/__git/explorer/owner
+++ b/cdist/conf/type/__git/explorer/owner
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id/.git"
+destination="/${__object_id:?}/.git"
 
-stat --print "%U" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((owner_uid + 1)) -ge 0
+then
+	owner_should=$(cat "${__object:?}/parameter/owner")
+
+	if expr "${owner_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${owner_uid}"
+	else
+		printf '%s\n' "$(id -u -n "${owner_uid}")"
+	fi
+fi

From 985252585ce83605787d2612fdc2a84b9788d943 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 30 Mar 2021 13:26:21 +0200
Subject: [PATCH 400/527] [type/__pyvenv] Fix if --owner / --group is numeric

Before, if --owner and/or --group was numeric, gencode-remote would generate
`chown` code every time.
---
 cdist/conf/type/__pyvenv/explorer/group | 20 +++++++++++++++++---
 cdist/conf/type/__pyvenv/explorer/owner | 20 +++++++++++++++++---
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group
index a655bda7..f31a1cb7 100755
--- a/cdist/conf/type/__pyvenv/explorer/group
+++ b/cdist/conf/type/__pyvenv/explorer/group
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id"
+destination="/${__object_id:?}"
 
-stat --print "%G" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((group_gid + 1)) -ge 0
+then
+	group_should=$(cat "${__object:?}/parameter/group")
+
+	if expr "${group_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${group_gid}"
+	else
+		printf '%s\n' "$(id -u -n "${group_gid}")"
+	fi
+fi
diff --git a/cdist/conf/type/__pyvenv/explorer/owner b/cdist/conf/type/__pyvenv/explorer/owner
index 8b3c7f8e..ebec751f 100755
--- a/cdist/conf/type/__pyvenv/explorer/owner
+++ b/cdist/conf/type/__pyvenv/explorer/owner
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id"
+destination="/${__object_id:?}"
 
-stat --print "%U" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((owner_uid + 1)) -ge 0
+then
+	owner_should=$(cat "${__object:?}/parameter/owner")
+
+	if expr "${owner_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${owner_uid}"
+	else
+		printf '%s\n' "$(id -u -n "${owner_uid}")"
+	fi
+fi

From 1e765fcab7bae02d6b380ca02e7cb04ec69ebdc5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 31 Mar 2021 07:54:00 +0200
Subject: [PATCH 401/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 42a74d04..03ae7566 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
 	* Type __sshd_config: Whitelist OpenBMC (Dennis Camera)
+	* Core: Maintain object relationship graph in cdist cache (Darko Poljak)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From f984a918b959a46e49fe8456a327eca8d44313de Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 30 Mar 2021 07:56:38 +0200
Subject: [PATCH 402/527] Fix log message string formatting

Use logging message format with args, instead of direct `%` or `str.format`.

Resolve #855.
---
 cdist/argparse.py                      |  8 +--
 cdist/config.py                        | 87 ++++++++++++--------------
 cdist/core/cdist_type.py               |  6 +-
 cdist/core/explorer.py                 | 26 ++++----
 cdist/core/manifest.py                 |  2 +-
 cdist/emulator.py                      | 37 +++++------
 cdist/exec/local.py                    | 16 +++--
 cdist/exec/remote.py                   | 14 ++---
 cdist/install.py                       |  2 +-
 cdist/inventory.py                     | 36 +++++------
 cdist/preos.py                         |  6 +-
 cdist/preos/debootstrap/debootstrap.py | 41 ++++++------
 cdist/scan/scan.py                     |  4 +-
 cdist/util/ipaddr.py                   |  6 +-
 14 files changed, 140 insertions(+), 151 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 88759d7b..cadac39a 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -533,10 +533,10 @@ def parse_and_configure(argv, singleton=True):
 
     log = logging.getLogger("cdist")
 
-    log.verbose("version %s" % cdist.VERSION)
-    log.trace('command line args: {}'.format(cfg.command_line_args))
-    log.trace('configuration: {}'.format(cfg.get_config()))
-    log.trace('configured args: {}'.format(args))
+    log.verbose("version %s", cdist.VERSION)
+    log.trace('command line args: %s', cfg.command_line_args)
+    log.trace('configuration: %s', cfg.get_config())
+    log.trace('configured args: %s', args)
 
     check_beta(vars(args))
 
diff --git a/cdist/config.py b/cdist/config.py
index 19d5bd70..d6fec55f 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -273,15 +273,15 @@ class Config:
                     host_tags = None
             host_base_path, hostdir = cls.create_host_base_dirs(
                 host, base_root_path)
-            log.debug("Base root path for target host \"{}\" is \"{}\"".format(
-                host, host_base_path))
+            log.debug("Base root path for target host \"%s\" is \"%s\"",
+                      host, host_base_path)
 
             hostcnt += 1
             if args.parallel:
                 pargs = (host, host_tags, host_base_path, hostdir, args, True,
                          configuration)
-                log.trace(("Args for multiprocessing operation "
-                           "for host {}: {}".format(host, pargs)))
+                log.trace("Args for multiprocessing operation for host %s: %s",
+                          host, pargs)
                 process_args.append(pargs)
             else:
                 try:
@@ -298,10 +298,10 @@ class Config:
             except cdist.Error:
                 failed_hosts.append(host)
         elif args.parallel:
-            log.trace("Multiprocessing start method is {}".format(
-                multiprocessing.get_start_method()))
-            log.trace(("Starting multiprocessing Pool for {} "
-                       "parallel host operation".format(args.parallel)))
+            log.trace("Multiprocessing start method is %s",
+                      multiprocessing.get_start_method())
+            log.trace("Starting multiprocessing Pool for %d parallel host"
+                      " operation", args.parallel)
 
             results = mp_pool_run(cls.onehost,
                                   process_args,
@@ -396,16 +396,13 @@ class Config:
 
             remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds(
                 args)
-            log.debug("remote_exec for host \"{}\": {}".format(
-                host, remote_exec))
-            log.debug("remote_copy for host \"{}\": {}".format(
-                host, remote_copy))
+            log.debug("remote_exec for host \"%s\": %s", host, remote_exec)
+            log.debug("remote_copy for host \"%s\": %s", host, remote_copy)
 
             family = cls._address_family(args)
-            log.debug("address family: {}".format(family))
+            log.debug("address family: %s", family)
             target_host = cls.resolve_target_addresses(host, family)
-            log.debug("target_host for host \"{}\": {}".format(
-                host, target_host))
+            log.debug("target_host for host \"%s\": %s", host, target_host)
 
             local = cdist.exec.local.Local(
                 target_host=target_host,
@@ -474,8 +471,8 @@ class Config:
         """Do what is most often done: deploy & cleanup"""
         start_time = time.time()
 
-        self.log.info("Starting {} run".format(
-            'dry' if self.dry_run else 'configuration'))
+        self.log.info("Starting %s run",
+                      'dry' if self.dry_run else 'configuration')
 
         self._init_files_dirs()
 
@@ -493,9 +490,9 @@ class Config:
         self._remove_files_dirs()
 
         self.local.save_cache(start_time)
-        self.log.info("Finished {} run in {:.2f} seconds".format(
+        self.log.info("Finished %s run in %.2f seconds",
                       'dry' if self.dry_run else 'successful',
-                      time.time() - start_time))
+                      time.time() - start_time)
 
     def cleanup(self):
         self.log.debug("Running cleanup commands")
@@ -519,8 +516,8 @@ class Config:
                 self.local.object_path, self.local.type_path,
                 self.local.object_marker_name):
             if cdist_object.cdist_type.is_install:
-                self.log.debug(("Running in config mode, ignoring install "
-                                "object: {0}").format(cdist_object))
+                self.log.debug("Running in config mode, ignoring install "
+                               "object: %s", cdist_object)
             else:
                 yield cdist_object
 
@@ -565,8 +562,7 @@ class Config:
         return objects_changed
 
     def _iterate_once_parallel(self):
-        self.log.debug("Iteration in parallel mode in {} jobs".format(
-            self.jobs))
+        self.log.debug("Iteration in parallel mode in %d jobs", self.jobs)
         objects_changed = False
 
         cargo = []
@@ -588,8 +584,8 @@ class Config:
             self.object_prepare(cargo[0])
             objects_changed = True
         elif cargo:
-            self.log.trace("Multiprocessing start method is {}".format(
-                multiprocessing.get_start_method()))
+            self.log.trace("Multiprocessing start method is %s",
+                           multiprocessing.get_start_method())
 
             self.log.trace("Multiprocessing cargo: %s", cargo)
 
@@ -603,9 +599,8 @@ class Config:
                                 "sequentially"))
                 self.explorer.transfer_type_explorers(cargo_types.pop())
             else:
-                self.log.trace(("Starting multiprocessing Pool for {} "
-                                "parallel types explorers transferring".format(
-                                    nt)))
+                self.log.trace("Starting multiprocessing Pool for %d "
+                               "parallel types explorers transferring", nt)
                 args = [
                     (ct, ) for ct in cargo_types
                 ]
@@ -614,8 +609,8 @@ class Config:
                 self.log.trace(("Multiprocessing for parallel transferring "
                                 "types' explorers finished"))
 
-            self.log.trace(("Starting multiprocessing Pool for {} parallel "
-                            "objects preparation".format(n)))
+            self.log.trace("Starting multiprocessing Pool for %d parallel "
+                           "objects preparation", n)
             args = [
                 (c, False, ) for c in cargo
             ]
@@ -667,10 +662,10 @@ class Config:
                 self.object_run(chunk[0])
                 objects_changed = True
             elif chunk:
-                self.log.trace("Multiprocessing start method is {}".format(
-                    multiprocessing.get_start_method()))
-                self.log.trace(("Starting multiprocessing Pool for {} "
-                               "parallel object run".format(n)))
+                self.log.trace("Multiprocessing start method is %s",
+                               multiprocessing.get_start_method())
+                self.log.trace("Starting multiprocessing Pool for %d "
+                               "parallel object run", n)
                 args = [
                     (c, ) for c in chunk
                 ]
@@ -794,9 +789,9 @@ class Config:
     def object_prepare(self, cdist_object, transfer_type_explorers=True):
         """Prepare object: Run type explorer + manifest"""
         self._handle_deprecation(cdist_object)
-        self.log.verbose("Preparing object {}".format(cdist_object.name))
-        self.log.verbose(
-            "Running manifest and explorers for " + cdist_object.name)
+        self.log.verbose("Preparing object %s", cdist_object.name)
+        self.log.verbose("Running manifest and explorers for %s",
+                         cdist_object.name)
         self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
         try:
             self.manifest.run_type_manifest(cdist_object)
@@ -810,13 +805,13 @@ class Config:
     def object_run(self, cdist_object):
         """Run gencode and code for an object"""
         try:
-            self.log.verbose("Running object " + cdist_object.name)
+            self.log.verbose("Running object %s", cdist_object.name)
             if cdist_object.state == core.CdistObject.STATE_DONE:
                 raise cdist.Error(("Attempting to run an already finished "
-                                   "object: %s"), cdist_object)
+                                   "object: {}").format(cdist_object))
 
             # Generate
-            self.log.debug("Generating code for %s" % (cdist_object.name))
+            self.log.debug("Generating code for %s", cdist_object.name)
             cdist_object.code_local = self.code.run_gencode_local(cdist_object)
             cdist_object.code_remote = self.code.run_gencode_remote(
                 cdist_object)
@@ -825,20 +820,20 @@ class Config:
 
             # Execute
             if cdist_object.code_local or cdist_object.code_remote:
-                self.log.info("Processing %s" % (cdist_object.name))
+                self.log.info("Processing %s", cdist_object.name)
             if not self.dry_run:
                 if cdist_object.code_local:
-                    self.log.trace("Executing local code for %s"
-                                   % (cdist_object.name))
+                    self.log.trace("Executing local code for %s",
+                                   cdist_object.name)
                     self.code.run_code_local(cdist_object)
                 if cdist_object.code_remote:
-                    self.log.trace("Executing remote code for %s"
-                                   % (cdist_object.name))
+                    self.log.trace("Executing remote code for %s",
+                                   cdist_object.name)
                     self.code.transfer_code_remote(cdist_object)
                     self.code.run_code_remote(cdist_object)
 
             # Mark this object as done
-            self.log.trace("Finishing run of " + cdist_object.name)
+            self.log.trace("Finishing run of %s", cdist_object.name)
             cdist_object.state = core.CdistObject.STATE_DONE
         except cdist.Error as e:
             raise cdist.CdistObjectError(cdist_object, e)
diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py
index c0329c8a..274de989 100644
--- a/cdist/core/cdist_type.py
+++ b/cdist/core/cdist_type.py
@@ -82,9 +82,9 @@ class CdistType:
                 yield cls(base_path, name)
             except InvalidTypeError as e:
                 # ignore invalid type, log warning and continue
-                msg = "Ignoring invalid type '%s' at '%s' defined at '%s'" % (
-                    e.type_path, e.type_absolute_path, e.source_path)
-                cls.log.warning(msg)
+                cls.log.warning("Ignoring invalid type '%s' at '%s' defined"
+                                " at '%s'", e.type_path, e.type_absolute_path,
+                                e.source_path)
                 # remove invalid from runtime conf dir
                 os.remove(e.type_absolute_path)
 
diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py
index a3baa959..caa12a7d 100644
--- a/cdist/core/explorer.py
+++ b/cdist/core/explorer.py
@@ -131,18 +131,17 @@ class Explorer:
             self._run_global_explorer(explorer, out_path)
 
     def _run_global_explorers_parallel(self, out_path):
-        self.log.debug("Running global explorers in {} parallel jobs".format(
-            self.jobs))
-        self.log.trace("Multiprocessing start method is {}".format(
-            multiprocessing.get_start_method()))
-        self.log.trace(("Starting multiprocessing Pool for global "
-                       "explorers run"))
+        self.log.debug("Running global explorers in %s parallel jobs",
+                       self.jobs)
+        self.log.trace("Multiprocessing start method is %s",
+                       multiprocessing.get_start_method())
+        self.log.trace("Starting multiprocessing Pool for global explorers"
+                       " run")
         args = [
             (e, out_path, ) for e in self.list_global_explorer_names()
         ]
         mp_pool_run(self._run_global_explorer, args, jobs=self.jobs)
-        self.log.trace(("Multiprocessing run for global explorers "
-                        "finished"))
+        self.log.trace("Multiprocessing run for global explorers finished")
 
     # logger is not pickable, so remove it when we pickle
     def __getstate__(self):
@@ -184,15 +183,14 @@ class Explorer:
         in the object.
 
         """
-        self.log.verbose("Running type explorers for {}".format(
-            cdist_object.cdist_type))
+        self.log.verbose("Running type explorers for %s",
+                         cdist_object.cdist_type)
         if transfer_type_explorers:
             self.log.trace("Transferring type explorers for type: %s",
                            cdist_object.cdist_type)
             self.transfer_type_explorers(cdist_object.cdist_type)
         else:
-            self.log.trace(("No need for transferring type explorers for "
-                            "type: %s"),
+            self.log.trace("No need for transferring type explorers for %s",
                            cdist_object.cdist_type)
         self.log.trace("Transferring object parameters for object: %s",
                        cdist_object.name)
@@ -236,8 +234,8 @@ class Explorer:
            remote side."""
         if cdist_type.explorers:
             if cdist_type.name in self._type_explorers_transferred:
-                self.log.trace(("Skipping retransfer of type explorers "
-                                "for: %s"), cdist_type)
+                self.log.trace("Skipping retransfer of type explorers for: %s",
+                               cdist_type)
             else:
                 source = os.path.join(self.local.type_path,
                                       cdist_type.explorer_path)
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 390340d4..3148d66c 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -161,7 +161,7 @@ class Manifest:
             raise NoInitialManifestError(initial_manifest, user_supplied)
 
         message_prefix = "initialmanifest"
-        self.log.verbose("Running initial manifest " + initial_manifest)
+        self.log.verbose("Running initial manifest %s", initial_manifest)
         which = "init"
         if self.local.save_output_streams:
             stderr_path = os.path.join(self.local.stderr_base_path, which)
diff --git a/cdist/emulator.py b/cdist/emulator.py
index f1db862e..f09c282d 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -107,8 +107,8 @@ class Emulator:
             self.record_requirements()
             self.record_auto_requirements()
             self.record_parent_child_relationships()
-            self.log.trace("Finished %s %s" % (
-                self.cdist_object.path, self.parameters))
+            self.log.trace("Finished %s %s", self.cdist_object.path,
+                           self.parameters)
 
     def __init_log(self):
         """Setup logging facility"""
@@ -170,7 +170,7 @@ class Emulator:
 
         # And finally parse/verify parameter
         self.args = parser.parse_args(self.argv[1:])
-        self.log.trace('Args: %s' % self.args)
+        self.log.trace('Args: %s', self.args)
 
     def init_object(self):
         # Initialize object - and ensure it is not in args
@@ -241,8 +241,8 @@ class Emulator:
                 raise cdist.Error(errmsg)
         else:
             if self.cdist_object.exists:
-                self.log.debug(('Object %s override forced with '
-                                'CDIST_OVERRIDE'), self.cdist_object.name)
+                self.log.debug('Object %s override forced with CDIST_OVERRIDE',
+                               self.cdist_object.name)
                 self.cdist_object.create(True)
             else:
                 self.cdist_object.create()
@@ -260,8 +260,8 @@ class Emulator:
             parent = self.cdist_object.object_from_name(__object_name)
             parent.typeorder.append(self.cdist_object.name)
             if self._order_dep_on():
-                self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
-                                ' for %s'), depname, parent.name)
+                self.log.trace('[ORDER_DEP] Adding %s to typeorder dep for %s',
+                               depname, parent.name)
                 parent.typeorder_dep.append(depname)
         elif self._order_dep_on():
             self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
@@ -301,16 +301,14 @@ class Emulator:
         try:
             cdist_object = self.cdist_object.object_from_name(requirement)
         except core.cdist_type.InvalidTypeError as e:
-            self.log.error(("%s requires object %s, but type %s does not"
-                            " exist. Defined at %s" % (
-                                self.cdist_object.name,
-                                requirement, e.name, self.object_source)))
+            self.log.error("%s requires object %s, but type %s does not"
+                           " exist. Defined at %s", self.cdist_object.name,
+                           requirement, e.name, self.object_source)
             raise
         except core.cdist_object.MissingObjectIdError:
-            self.log.error(("%s requires object %s without object id."
-                            " Defined at %s" % (self.cdist_object.name,
-                                                requirement,
-                                                self.object_source)))
+            self.log.error("%s requires object %s without object id."
+                           " Defined at %s", self.cdist_object.name,
+                           requirement, self.object_source)
             raise
 
         self.log.debug("Recording requirement %s for %s",
@@ -380,10 +378,9 @@ class Emulator:
                         self.env['require'] += " " + lastcreatedtype
                 else:
                     self.env['require'] = lastcreatedtype
-                self.log.debug(("Injecting require for "
-                                "CDIST_ORDER_DEPENDENCY: %s for %s"),
-                               lastcreatedtype,
-                               self.cdist_object.name)
+                self.log.debug("Injecting require for"
+                               " CDIST_ORDER_DEPENDENCY: %s for %s",
+                               lastcreatedtype, self.cdist_object.name)
             except IndexError:
                 # if no second last line, we are on the first type,
                 # so do not set a requirement
@@ -391,7 +388,7 @@ class Emulator:
 
         if "require" in self.env:
             requirements = self.env['require']
-            self.log.debug("reqs = " + requirements)
+            self.log.debug("reqs = %s", requirements)
             for requirement in self._parse_require(requirements):
                 # Ignore empty fields - probably the only field anyway
                 if len(requirement) == 0:
diff --git a/cdist/exec/local.py b/cdist/exec/local.py
index e0aab190..6713cd13 100644
--- a/cdist/exec/local.py
+++ b/cdist/exec/local.py
@@ -154,8 +154,8 @@ class Local:
         with open(self.object_marker_file, 'w') as fd:
             fd.write("%s\n" % self.object_marker_name)
 
-        self.log.trace("Object marker %s saved in %s" % (
-            self.object_marker_name, self.object_marker_file))
+        self.log.trace("Object marker %s saved in %s",
+                       self.object_marker_name, self.object_marker_file)
 
     def _init_cache_dir(self, cache_dir):
         home_dir = cdist.home_dir()
@@ -289,14 +289,12 @@ class Local:
         return cache_subpath
 
     def save_cache(self, start_time=time.time()):
-        self.log.trace("cache subpath pattern: {}".format(
-            self.cache_path_pattern))
+        self.log.trace("cache subpath pattern: %s", self.cache_path_pattern)
         cache_subpath = self._cache_subpath(start_time,
                                             self.cache_path_pattern)
-        self.log.debug("cache subpath: {}".format(cache_subpath))
+        self.log.debug("cache subpath: %s", cache_subpath)
         destination = os.path.join(self.cache_path, cache_subpath)
-        self.log.trace(("Saving cache: " + self.base_path + " to " +
-                        destination))
+        self.log.trace("Saving cache %s to %s", self.base_path, destination)
 
         if not os.path.exists(destination):
             shutil.move(self.base_path, destination)
@@ -332,7 +330,7 @@ class Local:
 
         # Iterate over all directories and link the to the output dir
         for conf_dir in self.conf_dirs:
-            self.log.debug("Checking conf_dir %s ..." % (conf_dir))
+            self.log.debug("Checking conf_dir %s ...", conf_dir)
             for sub_dir in CONF_SUBDIRS_LINKED:
                 current_dir = os.path.join(conf_dir, sub_dir)
 
@@ -350,7 +348,7 @@ class Local:
                     if os.path.exists(dst):
                         os.unlink(dst)
 
-                    self.log.trace("Linking %s to %s ..." % (src, dst))
+                    self.log.trace("Linking %s to %s ...", src, dst)
                     try:
                         os.symlink(src, dst)
                     except OSError as e:
diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py
index e5af2f34..ea85da4c 100644
--- a/cdist/exec/remote.py
+++ b/cdist/exec/remote.py
@@ -176,19 +176,19 @@ class Remote:
                 # create archive
                 tarpath, fcnt = autil.tar(source, self.archiving_mode)
                 if tarpath is None:
-                    self.log.trace(("Files count {} is lower than {} limit, "
-                                    "skipping archiving").format(
-                                        fcnt, autil.FILES_LIMIT))
+                    self.log.trace("Files count %d is lower than %d limit, "
+                                   "skipping archiving",
+                                   fcnt, autil.FILES_LIMIT)
                 else:
-                    self.log.trace(("Archiving mode, tarpath: %s, file count: "
-                                    "%s"), tarpath, fcnt)
+                    self.log.trace("Archiving mode, tarpath: %s, file count: "
+                                   "%s", tarpath, fcnt)
                     # get archive name
                     tarname = os.path.basename(tarpath)
                     self.log.trace("Archiving mode tarname: %s", tarname)
                     # archive path at the remote
                     desttarpath = os.path.join(destination, tarname)
-                    self.log.trace(
-                        "Archiving mode desttarpath: %s", desttarpath)
+                    self.log.trace("Archiving mode desttarpath: %s",
+                                   desttarpath)
                     # transfer archive to the remote side
                     self.log.trace("Archiving mode: transferring")
                     self._transfer_file(tarpath, desttarpath)
diff --git a/cdist/install.py b/cdist/install.py
index 7c894fe5..d077baef 100644
--- a/cdist/install.py
+++ b/cdist/install.py
@@ -47,4 +47,4 @@ class Install(cdist.config.Config):
                 yield cdist_object
             else:
                 self.log.debug("Running in install mode, ignoring non install"
-                               "object: {0}".format(cdist_object))
+                               "object: %s", cdist_object)
diff --git a/cdist/inventory.py b/cdist/inventory.py
index 0387f326..106052a2 100644
--- a/cdist/inventory.py
+++ b/cdist/inventory.py
@@ -92,7 +92,7 @@ class Inventory:
         self.init_db()
 
     def init_db(self):
-        self.log.trace("Init db: {}".format(self.db_basedir))
+        self.log.trace("Init db: %s", self.db_basedir)
         if not os.path.exists(self.db_basedir):
             os.makedirs(self.db_basedir, exist_ok=True)
         elif not os.path.isdir(self.db_basedir):
@@ -182,9 +182,9 @@ class Inventory:
         configuration = cfg.get_config(section='GLOBAL')
         determine_default_inventory_dir(args, configuration)
 
-        log.debug("Using inventory: {}".format(args.inventory_dir))
-        log.trace("Inventory args: {}".format(vars(args)))
-        log.trace("Inventory command: {}".format(args.subcommand))
+        log.debug("Using inventory: %s", args.inventory_dir)
+        log.trace("Inventory args: %s", vars(args))
+        log.trace("Inventory command: %s", args.subcommand)
 
         if args.subcommand == "list":
             c = InventoryList(hosts=args.host, istag=args.tag,
@@ -237,16 +237,16 @@ class InventoryList(Inventory):
     def _do_list(self, it_tags, it_hosts, check_func):
         if (it_tags is not None):
             param_tags = set(it_tags)
-            self.log.trace("param_tags: {}".format(param_tags))
+            self.log.trace("param_tags: %s", param_tags)
         else:
             param_tags = set()
         for host in it_hosts:
-            self.log.trace("host: {}".format(host))
+            self.log.trace("host: %s", host)
             tags = self._get_host_tags(host)
             if tags is None:
-                self.log.debug("Host \'{}\' not found, skipped".format(host))
+                self.log.debug("Host \'%s\' not found, skipped", host)
                 continue
-            self.log.trace("tags: {}".format(tags))
+            self.log.trace("tags: %s", tags)
             if check_func(tags, param_tags):
                 yield host, tags
 
@@ -308,11 +308,11 @@ class InventoryHost(Inventory):
 
     def _action(self, host):
         if self.action == "add":
-            self.log.debug("Adding host \'{}\'".format(host))
+            self.log.debug("Adding host \'%s\'", host)
         elif self.action == "del":
-            self.log.debug("Deleting host \'{}\'".format(host))
+            self.log.debug("Deleting host \'%s\'", host)
         hostpath = self._host_path(host)
-        self.log.trace("hostpath: {}".format(hostpath))
+        self.log.trace("hostpath: %s", hostpath)
         if self.action == "add" and not os.path.exists(hostpath):
             self._new_hostpath(hostpath)
         else:
@@ -372,23 +372,23 @@ class InventoryTag(Inventory):
             print("Host \'{}\' does not exist, skipping".format(host),
                   file=sys.stderr)
             return
-        self.log.trace("existing host_tags: {}".format(host_tags))
+        self.log.trace("existing host_tags: %s", host_tags)
         if self.action == "del" and self.all:
             host_tags = set()
         else:
             for tag in self.input_tags:
                 if self.action == "add":
-                    self.log.debug("Adding tag \'{}\' for host \'{}\'".format(
-                        tag, host))
+                    self.log.debug("Adding tag \'%s\' for host \'%s\'",
+                                   tag, host)
                     host_tags.add(tag)
                 elif self.action == "del":
-                    self.log.debug("Deleting tag \'{}\' for host "
-                                   "\'{}\'".format(tag, host))
+                    self.log.debug("Deleting tag \'%s\' for host \'%s\'",
+                                   tag, host)
                     if tag in host_tags:
                         host_tags.remove(tag)
-        self.log.trace("new host tags: {}".format(host_tags))
+        self.log.trace("new host tags: %s", host_tags)
         if not self._write_host_tags(host, host_tags):
-            self.log.trace("{} does not exist, skipped".format(host))
+            self.log.trace("%s does not exist, skipped", host)
 
     def run(self):
         if self.allhosts:
diff --git a/cdist/preos.py b/cdist/preos.py
index f8a5dd67..45711a41 100644
--- a/cdist/preos.py
+++ b/cdist/preos.py
@@ -49,7 +49,7 @@ def scan_preos_dir_plugins(dir):
                 c = cm[1]
                 yield from preos_plugin(c)
         except ImportError as e:
-            log.warning("Cannot import '{}': {}".format(module_name, e))
+            log.warning("Cannot import '%s': %s", module_name, e)
 
 
 def find_preos_plugins():
@@ -102,7 +102,7 @@ class PreOS:
         parser.add_argument('remainder_args', nargs=argparse.REMAINDER)
         args = parser.parse_args(argv[1:])
         cdist.argparse.handle_loglevel(args)
-        log.debug("preos args : {}".format(args))
+        log.debug("preos args : %s", args)
 
         conf_dirs = util.resolve_conf_dirs_from_config_and_args(args)
 
@@ -122,7 +122,7 @@ class PreOS:
                 func_args = [preos, args.remainder_args, ]
             else:
                 func_args = [args.remainder_args, ]
-            log.info("Running preos : {}".format(preos_name))
+            log.info("Running preos : %s", preos_name)
             func(*func_args)
         else:
             raise cdist.Error(
diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py
index a20cdb9c..f1f750ee 100644
--- a/cdist/preos/debootstrap/debootstrap.py
+++ b/cdist/preos/debootstrap/debootstrap.py
@@ -166,7 +166,7 @@ class Debian:
             args.pxe_boot_dir = os.path.realpath(args.pxe_boot_dir)
 
         cdist.argparse.handle_loglevel(args)
-        log.debug("preos: {}, args: {}".format(cls._preos_name, args))
+        log.debug("preos: %s, args: %s", cls._preos_name, args)
         try:
             env = vars(args)
             new_env = {}
@@ -190,27 +190,30 @@ class Debian:
             env = new_env
             env.update(os.environ)
             cls.update_env(env)
-            log.debug("preos: {} env: {}".format(cls._preos_name, env))
+            log.debug("preos: %s env: %s", cls._preos_name, env)
+
+            if log.getEffectiveLevel() <= logging.INFO:
+                info_msg = ["Running preos: {}, suite: {}, arch: {}".format(
+                    cls._preos_name, args.suite, args.arch), ]
+                if args.mirror:
+                    info_msg.append("mirror: {}".format(args.mirror))
+                if args.script:
+                    info_msg.append("script: {}".format(args.script))
+                if args.bootstrap:
+                    info_msg.append("bootstrapping")
+                if args.configure:
+                    info_msg.append("configuring")
+                if args.pxe_boot_dir:
+                    info_msg.append("creating PXE")
+                if args.drive:
+                    info_msg.append("creating bootable drive")
+                log.info(info_msg)
+
             cmd = os.path.join(cls._files_dir, "code")
-            info_msg = ["Running preos: {}, suite: {}, arch: {}".format(
-                cls._preos_name, args.suite, args.arch), ]
-            if args.mirror:
-                info_msg.append("mirror: {}".format(args.mirror))
-            if args.script:
-                info_msg.append("script: {}".format(args.script))
-            if args.bootstrap:
-                info_msg.append("bootstrapping")
-            if args.configure:
-                info_msg.append("configuring")
-            if args.pxe_boot_dir:
-                info_msg.append("creating PXE")
-            if args.drive:
-                info_msg.append("creating bootable drive")
-            log.info(info_msg)
-            log.debug("cmd={}".format(cmd))
+            log.debug("cmd=%s", cmd)
             subprocess.check_call(cmd, env=env, shell=True)
         except subprocess.CalledProcessError as e:
-            log.error("preos {} failed: {}".format(cls._preos_name, e))
+            log.error("preos %s failed: %s", cls._preos_name, e)
 
 
 class Ubuntu(Debian):
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index b1d0e9e1..9a0bcc52 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -94,7 +94,7 @@ class Trigger(object):
 
     def trigger(self, interface):
         packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
-        log.debug(f"Sending request on {interface}")
+        log.debug("Sending request on %s", interface)
         send(packet, verbose=self.verbose)
 
 
@@ -114,7 +114,7 @@ class Scanner(object):
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
             host = pkg['IPv6'].src
-            log.verbose(f"Host {host} is alive")
+            log.verbose("Host %s is alive", host)
 
             dir = os.path.join(self.outdir, host)
             fname = os.path.join(dir, "last_seen")
diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py
index 95ca74ee..d9e5f498 100644
--- a/cdist/util/ipaddr.py
+++ b/cdist/util/ipaddr.py
@@ -42,8 +42,7 @@ def resolve_target_host_name(host, family=0):
         # gethostbyaddr returns triple
         # (hostname, aliaslist, ipaddrlist)
         host_name = socket.gethostbyaddr(ip_addr)[0]
-        log.debug("derived host_name for host \"{}\": {}".format(
-            host, host_name))
+        log.debug("derived host_name for host \"%s\": %s", host, host_name)
     except (socket.gaierror, socket.herror) as e:
         # in case of error provide empty value
         host_name = ''
@@ -54,8 +53,7 @@ def resolve_target_fqdn(host):
     log = logging.getLogger(host)
     try:
         host_fqdn = socket.getfqdn(host)
-        log.debug("derived host_fqdn for host \"{}\": {}".format(
-            host, host_fqdn))
+        log.debug("derived host_fqdn for host \"%s\": %s", host, host_fqdn)
     except socket.herror as e:
         # in case of error provide empty value
         host_fqdn = ''

From 4c2d273f07e6cedb3dcb539d132cb0c3f8176d42 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 30 Mar 2021 07:56:38 +0200
Subject: [PATCH 403/527] Unify string formatting

Use one way of string formatting: replace old `%` style with new `str.format`.

Resolve #855.
---
 cdist/config.py                     |  4 ++--
 cdist/core/cdist_object.py          | 14 +++++++-------
 cdist/core/cdist_type.py            |  4 ++--
 cdist/core/code.py                  |  8 ++++----
 cdist/core/explorer.py              |  6 +++---
 cdist/core/manifest.py              | 11 +++++------
 cdist/emulator.py                   | 12 ++++++------
 cdist/exec/local.py                 | 11 ++++++-----
 cdist/exec/remote.py                | 11 +++++------
 cdist/message.py                    |  2 +-
 cdist/scan/scan.py                  |  2 +-
 cdist/shell.py                      |  2 +-
 cdist/test/cdist_object/__init__.py | 27 +++++++++++++++------------
 cdist/test/emulator/__init__.py     |  4 ++--
 cdist/test/exec/remote.py           |  8 ++++----
 cdist/test/message/__init__.py      |  2 +-
 cdist/util/fsproperty.py            |  4 ++--
 17 files changed, 67 insertions(+), 65 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index d6fec55f..adc460de 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -190,7 +190,7 @@ class Config:
                     fd.write(sys.stdin.read())
             except (IOError, OSError) as e:
                 raise cdist.Error(("Creating tempfile for stdin data "
-                                   "failed: %s" % e))
+                                   "failed: {}").format(e))
 
             args.manifest = initial_manifest_temp_path
             atexit.register(lambda: os.remove(initial_manifest_temp_path))
@@ -764,7 +764,7 @@ class Config:
 
             raise cdist.UnresolvableRequirementsError(
                     ("The requirements of the following objects could not be "
-                     "resolved:\n%s") % ("\n".join(info_string)))
+                     "resolved:\n{}").format("\n".join(info_string)))
 
     def _handle_deprecation(self, cdist_object):
         cdist_type = cdist_object.cdist_type
diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index eea2c255..ccf7392d 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -34,17 +34,17 @@ class IllegalObjectIdError(cdist.Error):
         self.message = message or 'Illegal object id'
 
     def __str__(self):
-        return '%s: %s' % (self.message, self.object_id)
+        return '{}: {}'.format(self.message, self.object_id)
 
 
 class MissingObjectIdError(cdist.Error):
     def __init__(self, type_name):
         self.type_name = type_name
-        self.message = ("Type %s requires object id (is not a "
-                        "singleton type)") % self.type_name
+        self.message = ("Type {} requires object id (is not a "
+                        "singleton type)").format(self.type_name)
 
     def __str__(self):
-        return '%s' % (self.message)
+        return '{}'.format(self.message)
 
 
 class CdistObject:
@@ -142,7 +142,7 @@ class CdistObject:
             if self.object_marker in self.object_id.split(os.sep):
                 raise IllegalObjectIdError(
                         self.object_id, ('object_id may not contain '
-                                         '\'%s\'') % self.object_marker)
+                                         '\'{}\'').format(self.object_marker))
             if '//' in self.object_id:
                 raise IllegalObjectIdError(
                         self.object_id, 'object_id may not contain //')
@@ -189,7 +189,7 @@ class CdistObject:
                               object_id=object_id)
 
     def __repr__(self):
-        return '<CdistObject %s>' % self.name
+        return '<CdistObject {}>'.format(self.name)
 
     def __eq__(self, other):
         """define equality as 'name is the same'"""
@@ -277,7 +277,7 @@ class CdistObject:
                 os.makedirs(path, exist_ok=allow_overwrite)
         except EnvironmentError as error:
             raise cdist.Error(('Error creating directories for cdist object: '
-                               '%s: %s') % (self, error))
+                               '{}: {}').format(self, error))
 
     def requirements_unfinished(self, requirements):
         """Return state whether requirements are satisfied"""
diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py
index 274de989..b68448bc 100644
--- a/cdist/core/cdist_type.py
+++ b/cdist/core/cdist_type.py
@@ -34,7 +34,7 @@ class InvalidTypeError(cdist.Error):
         self.source_path = os.path.realpath(self.type_absolute_path)
 
     def __str__(self):
-        return "Invalid type '%s' at '%s' defined at '%s'" % (
+        return "Invalid type '{}' at '{}' defined at '{}'".format(
                 self.type_path, self.type_absolute_path, self.source_path)
 
 
@@ -109,7 +109,7 @@ class CdistType:
         return cls._instances[name]
 
     def __repr__(self):
-        return '<CdistType %s>' % self.name
+        return '<CdistType {}>'.format(self.name)
 
     def __eq__(self, other):
         return isinstance(other, self.__class__) and self.name == other.name
diff --git a/cdist/core/code.py b/cdist/core/code.py
index 1e9b4f80..12888ba4 100644
--- a/cdist/core/code.py
+++ b/cdist/core/code.py
@@ -122,8 +122,8 @@ class Code:
 
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.cdist_type
-        script = os.path.join(self.local.type_path,
-                              getattr(cdist_type, 'gencode_%s_path' % which))
+        gencode_attr = getattr(cdist_type, 'gencode_{}_path'.format(which))
+        script = os.path.join(self.local.type_path, gencode_attr)
         if os.path.isfile(script):
             env = os.environ.copy()
             env.update(self.env)
@@ -167,8 +167,8 @@ class Code:
 
     def _run_code(self, cdist_object, which, env=None):
         which_exec = getattr(self, which)
-        script = os.path.join(which_exec.object_path,
-                              getattr(cdist_object, 'code_%s_path' % which))
+        code_attr = getattr(cdist_object, 'code_{}_path'.format(which))
+        script = os.path.join(which_exec.object_path, code_attr)
         if which_exec.save_output_streams:
             stderr_path = os.path.join(cdist_object.stderr_path,
                                        'code-' + which)
diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py
index caa12a7d..48414ade 100644
--- a/cdist/core/explorer.py
+++ b/cdist/core/explorer.py
@@ -160,8 +160,8 @@ class Explorer:
         self.remote.transfer(self.local.global_explorer_path,
                              self.remote.global_explorer_path,
                              self.jobs)
-        self.remote.run(["chmod", "0700",
-                         "%s/*" % (self.remote.global_explorer_path)])
+        self.remote.run(["chmod", "0700", "{}/*".format(
+            self.remote.global_explorer_path)])
 
     def run_global_explorer(self, explorer):
         """Run the given global explorer and return it's output."""
@@ -242,7 +242,7 @@ class Explorer:
                 destination = os.path.join(self.remote.type_path,
                                            cdist_type.explorer_path)
                 self.remote.transfer(source, destination)
-                self.remote.run(["chmod", "0700", "%s/*" % (destination)])
+                self.remote.run(["chmod", "0700", "{}/*".format(destination)])
                 self._type_explorers_transferred.append(cdist_type.name)
 
     def transfer_object_parameters(self, cdist_object):
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 3148d66c..09e74dac 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -80,13 +80,12 @@ class NoInitialManifestError(cdist.Error):
 
         if user_supplied:
             if os.path.islink(manifest_path):
-                self.message = "%s: %s -> %s" % (
-                        msg_header, manifest_path,
-                        os.path.realpath(manifest_path))
+                self.message = "{}: {} -> {}".format(
+                    msg_header, manifest_path, os.path.realpath(manifest_path))
             else:
-                self.message = "%s: %s" % (msg_header, manifest_path)
+                self.message = "{}: {}".format(msg_header, manifest_path)
         else:
-            self.message = "%s" % (msg_header)
+            self.message = "{}".format(msg_header)
 
     def __str__(self):
         return repr(self.message)
@@ -107,7 +106,7 @@ class Manifest:
         self._open_logger()
 
         self.env = {
-            'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
+            'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']),
             # for use in type emulator
             '__cdist_type_base_path': self.local.type_path,
             '__global': self.local.base_path,
diff --git a/cdist/emulator.py b/cdist/emulator.py
index f09c282d..c55b47d2 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -36,8 +36,8 @@ from cdist.core.manifest import Manifest
 class MissingRequiredEnvironmentVariableError(cdist.Error):
     def __init__(self, name):
         self.name = name
-        self.message = ("Emulator requires the environment variable %s to be "
-                        "setup" % self.name)
+        self.message = ("Emulator requires the environment variable {} to be "
+                        "setup").format(self.name)
 
     def __str__(self):
         return self.message
@@ -231,13 +231,13 @@ class Emulator:
         if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env:
             obj_params = self._object_params_in_context()
             if obj_params != self.parameters:
-                errmsg = ("Object %s already exists with conflicting "
-                          "parameters:\n%s: %s\n%s: %s" % (
+                errmsg = ("Object {} already exists with conflicting "
+                          "parameters:\n{}: {}\n{}: {}").format(
                               self.cdist_object.name,
                               " ".join(self.cdist_object.source),
                               obj_params,
                               self.object_source,
-                              self.parameters))
+                              self.parameters)
                 raise cdist.Error(errmsg)
         else:
             if self.cdist_object.exists:
@@ -292,7 +292,7 @@ class Emulator:
                         fd.write(chunk)
                         chunk = self._read_stdin()
             except EnvironmentError as e:
-                raise cdist.Error('Failed to read from stdin: %s' % e)
+                raise cdist.Error('Failed to read from stdin: {}'.format(e))
 
     def record_requirement(self, requirement):
         """record requirement and return recorded requirement"""
diff --git a/cdist/exec/local.py b/cdist/exec/local.py
index 6713cd13..370336ad 100644
--- a/cdist/exec/local.py
+++ b/cdist/exec/local.py
@@ -152,7 +152,7 @@ class Local:
 
     def _setup_object_marker_file(self):
         with open(self.object_marker_file, 'w') as fd:
-            fd.write("%s\n" % self.object_marker_name)
+            fd.write("{}\n".format(self.object_marker_name))
 
         self.log.trace("Object marker %s saved in %s",
                        self.object_marker_name, self.object_marker_file)
@@ -184,7 +184,7 @@ class Local:
 
         """
         assert isinstance(command, (list, tuple)), (
-                "list or tuple argument expected, got: %s" % command)
+                "list or tuple argument expected, got: {}".format(command))
 
         quiet = self.quiet_mode or quiet_mode
         do_save_output = save_output and not quiet and self.save_output_streams
@@ -352,8 +352,9 @@ class Local:
                     try:
                         os.symlink(src, dst)
                     except OSError as e:
-                        raise cdist.Error("Linking %s %s to %s failed: %s" % (
-                            sub_dir, src, dst, e.__str__()))
+                        raise cdist.Error(
+                            "Linking {} {} to {} failed: {}".format(
+                                sub_dir, src, dst, e.__str__()))
 
     def _link_types_for_emulator(self):
         """Link emulator to types"""
@@ -366,5 +367,5 @@ class Local:
                 os.symlink(src, dst)
             except OSError as e:
                 raise cdist.Error(
-                        "Linking emulator from %s to %s failed: %s" % (
+                        "Linking emulator from {} to {} failed: {}".format(
                             src, dst, e.__str__()))
diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py
index ea85da4c..af9e70cb 100644
--- a/cdist/exec/remote.py
+++ b/cdist/exec/remote.py
@@ -24,12 +24,10 @@ import os
 import glob
 import subprocess
 import logging
-import multiprocessing
 
 import cdist
 import cdist.exec.util as util
 import cdist.util.ipaddr as ipaddr
-from cdist.mputil import mp_pool_run
 
 
 def _wrap_addr(addr):
@@ -262,9 +260,10 @@ class Remote:
         # remotely in e.g. csh and setting up CDIST_REMOTE_SHELL to e.g.
         # /bin/csh will execute this script in the right way.
         if env:
-            remote_env = [" export %s=%s;" % item for item in env.items()]
-            string_cmd = ("/bin/sh -c '" + " ".join(remote_env) +
-                          " ".join(command) + "'")
+            remote_env = [" export {env[0]}={env[1]};".format(env=item)
+                          for item in env.items()]
+            string_cmd = ("/bin/sh -c '{}{}'").format(" ".join(remote_env),
+                                                      " ".join(command))
             cmd.append(string_cmd)
         else:
             cmd.extend(command)
@@ -278,7 +277,7 @@ class Remote:
 
         """
         assert isinstance(command, (list, tuple)), (
-                "list or tuple argument expected, got: %s" % command)
+                "list or tuple argument expected, got: {}".format(command))
 
         close_stdout = False
         close_stderr = False
diff --git a/cdist/message.py b/cdist/message.py
index ffa8c2bb..0c4e21a6 100644
--- a/cdist/message.py
+++ b/cdist/message.py
@@ -70,7 +70,7 @@ class Message:
 
         with open(self.global_messages, 'a') as fd:
             for line in content:
-                fd.write("%s:%s" % (self.prefix, line))
+                fd.write("{}:{}".format(self.prefix, line))
 
     def merge_messages(self):
         self._merge_messages()
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 9a0bcc52..faee8a56 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -93,7 +93,7 @@ class Trigger(object):
             time.sleep(self.sleeptime)
 
     def trigger(self, interface):
-        packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
+        packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest()
         log.debug("Sending request on %s", interface)
         send(packet, verbose=self.verbose)
 
diff --git a/cdist/shell.py b/cdist/shell.py
index 04a68937..05803556 100644
--- a/cdist/shell.py
+++ b/cdist/shell.py
@@ -65,7 +65,7 @@ class Shell:
     def _init_environment(self):
         self.env = os.environ.copy()
         additional_env = {
-            'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
+            'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']),
             # for use in type emulator
             '__cdist_type_base_path': self.local.type_path,
             '__cdist_manifest': "cdist shell",
diff --git a/cdist/test/cdist_object/__init__.py b/cdist/test/cdist_object/__init__.py
index a9c20cd3..11619e96 100644
--- a/cdist/test/cdist_object/__init__.py
+++ b/cdist/test/cdist_object/__init__.py
@@ -86,8 +86,7 @@ class ObjectClassTestCase(test.CdistTestCase):
 
     def test_create_singleton(self):
         """Check whether creating an object without id (singleton) works"""
-        singleton = self.expected_objects[0].object_from_name(
-                "__test_singleton")
+        self.expected_objects[0].object_from_name("__test_singleton")
         # came here - everything fine
 
     def test_create_singleton_not_singleton_type(self):
@@ -126,16 +125,16 @@ class ObjectIdTestCase(test.CdistTestCase):
 
     def test_object_id_contains_object_marker(self):
         cdist_type = core.CdistType(type_base_path, '__third')
-        illegal_object_id = (
-            'object_id/may/not/contain/%s/anywhere' % OBJECT_MARKER_NAME)
+        illegal_object_id = 'object_id/may/not/contain/{}/anywhere'.format(
+            OBJECT_MARKER_NAME)
         with self.assertRaises(core.IllegalObjectIdError):
             core.CdistObject(cdist_type, self.object_base_path,
                              OBJECT_MARKER_NAME, illegal_object_id)
 
     def test_object_id_contains_object_marker_string(self):
         cdist_type = core.CdistType(type_base_path, '__third')
-        illegal_object_id = (
-            'object_id/may/contain_%s_in_filename' % OBJECT_MARKER_NAME)
+        illegal_object_id = 'object_id/may/contain_{}_in_filename'.format(
+            OBJECT_MARKER_NAME)
         core.CdistObject(cdist_type, self.object_base_path,
                          OBJECT_MARKER_NAME, illegal_object_id)
         # if we get here, the test passed
@@ -195,28 +194,32 @@ class ObjectTestCase(test.CdistTestCase):
 
     def test_path(self):
         self.assertEqual(self.cdist_object.path,
-                         "__third/moon/%s" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}".format(OBJECT_MARKER_NAME))
 
     def test_absolute_path(self):
         self.assertEqual(self.cdist_object.absolute_path,
                          os.path.join(self.object_base_path,
-                                      "__third/moon/%s" % OBJECT_MARKER_NAME))
+                                      "__third/moon/{}".format(
+                                          OBJECT_MARKER_NAME)))
 
     def test_code_local_path(self):
         self.assertEqual(self.cdist_object.code_local_path,
-                         "__third/moon/%s/code-local" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/code-local".format(
+                             OBJECT_MARKER_NAME))
 
     def test_code_remote_path(self):
         self.assertEqual(self.cdist_object.code_remote_path,
-                         "__third/moon/%s/code-remote" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/code-remote".format(
+                             OBJECT_MARKER_NAME))
 
     def test_parameter_path(self):
         self.assertEqual(self.cdist_object.parameter_path,
-                         "__third/moon/%s/parameter" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/parameter".format(
+                             OBJECT_MARKER_NAME))
 
     def test_explorer_path(self):
         self.assertEqual(self.cdist_object.explorer_path,
-                         "__third/moon/%s/explorer" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/explorer".format(OBJECT_MARKER_NAME))
 
     def test_parameters(self):
         expected_parameters = {'planet': 'Saturn', 'name': 'Prometheus'}
diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py
index befd7b57..4b2bc3ba 100644
--- a/cdist/test/emulator/__init__.py
+++ b/cdist/test/emulator/__init__.py
@@ -84,8 +84,8 @@ class EmulatorTestCase(test.CdistTestCase):
 
     def test_illegal_object_id_requirement(self):
         argv = ['__file', '/tmp/foobar']
-        self.env['require'] = (
-                "__file/bad/id/with/%s/inside") % self.local.object_marker_name
+        self.env['require'] = "__file/bad/id/with/{}/inside".format(
+            self.local.object_marker_name)
         emu = emulator.Emulator(argv, env=self.env)
         self.assertRaises(core.IllegalObjectIdError, emu.run)
 
diff --git a/cdist/test/exec/remote.py b/cdist/test/exec/remote.py
index a7fe384d..b23f8447 100644
--- a/cdist/test/exec/remote.py
+++ b/cdist/test/exec/remote.py
@@ -47,8 +47,8 @@ class RemoteTestCase(test.CdistTestCase):
             args = (self.target_host,)
         kwargs.setdefault('base_path', self.base_path)
         user = getpass.getuser()
-        kwargs.setdefault('remote_exec', 'ssh -o User=%s -q' % user)
-        kwargs.setdefault('remote_copy', 'scp -o User=%s -q' % user)
+        kwargs.setdefault('remote_exec', 'ssh -o User={} -q'.format(user))
+        kwargs.setdefault('remote_copy', 'scp -o User={} -q'.format(user))
         if 'stdout_base_path' not in kwargs:
             stdout_path = os.path.join(self.temp_dir, 'stdout')
             os.makedirs(stdout_path, exist_ok=True)
@@ -170,7 +170,7 @@ class RemoteTestCase(test.CdistTestCase):
         r = self.create_remote(remote_exec=remote_exec,
                                remote_copy=remote_copy)
         self.assertEqual(r.run('true', return_output=True),
-                         "%s\n" % self.target_host[0])
+                         "{}\n".format(self.target_host[0]))
 
     def test_run_script_target_host_in_env(self):
         handle, remote_exec_path = self.mkstemp(dir=self.temp_dir)
@@ -185,7 +185,7 @@ class RemoteTestCase(test.CdistTestCase):
         with os.fdopen(handle, "w") as fd:
             fd.writelines(["#!/bin/sh\n", "true"])
         self.assertEqual(r.run_script(script, return_output=True),
-                         "%s\n" % self.target_host[0])
+                         "{}\n".format(self.target_host[0]))
 
     def test_run_script_with_env_target_host_in_env(self):
         handle, script = self.mkstemp(dir=self.temp_dir)
diff --git a/cdist/test/message/__init__.py b/cdist/test/message/__init__.py
index 61cd5d97..55040fc9 100644
--- a/cdist/test/message/__init__.py
+++ b/cdist/test/message/__init__.py
@@ -67,7 +67,7 @@ class MessageTestCase(test.CdistTestCase):
     def test_message_merge_prefix(self):
         """Ensure messages are merged and are prefixed"""
 
-        expectedcontent = "%s:%s" % (self.prefix, self.content)
+        expectedcontent = "{}:{}".format(self.prefix, self.content)
 
         out = self.message.env['__messages_out']
 
diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py
index 1d76fd76..09e9cc19 100644
--- a/cdist/util/fsproperty.py
+++ b/cdist/util/fsproperty.py
@@ -30,7 +30,7 @@ class AbsolutePathRequiredError(cdist.Error):
         self.path = path
 
     def __str__(self):
-        return 'Absolute path required, got: %s' % self.path
+        return 'Absolute path required, got: {}'.format(self.path)
 
 
 class FileList(collections.MutableSequence):
@@ -218,7 +218,7 @@ class FileBasedProperty:
 
     def _get_attribute(self, instance, owner):
         name = self._get_property_name(owner)
-        attribute_name = '__%s' % name
+        attribute_name = '__{}'.format(name)
         if not hasattr(instance, attribute_name):
             path = self._get_path(instance)
             attribute_instance = self.attribute_class(path)

From ab811ad282882558a4639986e75c800e1b63c2e0 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 1 Apr 2021 15:37:11 +0200
Subject: [PATCH 404/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 03ae7566..db60c0c4 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,8 @@ next:
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
 	* Type __sshd_config: Whitelist OpenBMC (Dennis Camera)
 	* Core: Maintain object relationship graph in cdist cache (Darko Poljak)
+	* Type __git: Fix numeric owner and group handling (Dennis Camera)
+	* Type __pyvenv: Fix numeric owner and group handling (Dennis Camera)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From 199effb7ef23980bc69f0233ff6d7ee465623f40 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 6 Apr 2021 19:35:14 +0200
Subject: [PATCH 405/527] Improve unfinished object requirements bool check

When we need only boolean value for unfinished object requirements then
we don't need to determine the whole list of unfinished objects.
---
 cdist/config.py            | 12 +++++++-----
 cdist/core/cdist_object.py | 13 ++++++++++++-
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index adc460de..f7da9b1d 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -537,7 +537,7 @@ class Config:
         objects_changed = False
 
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(
+            if cdist_object.has_requirements_unfinished(
                     cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
@@ -548,7 +548,7 @@ class Config:
                 self.object_prepare(cdist_object)
                 objects_changed = True
 
-            if cdist_object.requirements_unfinished(
+            if cdist_object.has_requirements_unfinished(
                     cdist_object.autorequire):
                 """The previous step created objects we depend on -
                     wait for them
@@ -567,7 +567,8 @@ class Config:
 
         cargo = []
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(cdist_object.requirements):
+            if cdist_object.has_requirements_unfinished(
+                    cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
 
@@ -621,12 +622,13 @@ class Config:
 
         del cargo[:]
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(cdist_object.requirements):
+            if cdist_object.has_requirements_unfinished(
+                    cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
 
             if cdist_object.state == core.CdistObject.STATE_PREPARED:
-                if cdist_object.requirements_unfinished(
+                if cdist_object.has_requirements_unfinished(
                         cdist_object.autorequire):
                     """The previous step created objects we depend on -
                     wait for them
diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index ccf7392d..bb3a65bd 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -280,7 +280,7 @@ class CdistObject:
                                '{}: {}').format(self, error))
 
     def requirements_unfinished(self, requirements):
-        """Return state whether requirements are satisfied"""
+        """Return unsatisfied requirements"""
 
         object_list = []
 
@@ -291,3 +291,14 @@ class CdistObject:
                 object_list.append(cdist_object)
 
         return object_list
+
+    def has_requirements_unfinished(self, requirements):
+        """Return whether requirements are satisfied"""
+
+        for requirement in requirements:
+            cdist_object = self.object_from_name(requirement)
+
+            if cdist_object.state != self.STATE_DONE:
+                return True
+
+        return False

From 750c71fb5a9fcf4a30af0700b7a09e3501988a27 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 7 Apr 2021 09:07:29 +0200
Subject: [PATCH 406/527] Minor refactoring and remove code duplication

---
 cdist/config.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index f7da9b1d..638fdf0e 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -699,20 +699,22 @@ class Config:
             check for cycles.
         '''
         graph = {}
-        for cdist_object in self.object_list():
+
+        def _add_requirements(cdist_object, requirements):
             obj_name = cdist_object.name
             if obj_name not in graph:
                 graph[obj_name] = []
+
+            for requirement in cdist_object.requirements_unfinished(
+                    requirements):
+                graph[obj_name].append(requirement.name)
+
+        for cdist_object in self.object_list():
             if cdist_object.state == cdist_object.STATE_DONE:
                 continue
 
-            for requirement in cdist_object.requirements_unfinished(
-                    cdist_object.requirements):
-                graph[obj_name].append(requirement.name)
-
-            for requirement in cdist_object.requirements_unfinished(
-                    cdist_object.autorequire):
-                graph[obj_name].append(requirement.name)
+            _add_requirements(cdist_object, cdist_object.requirements)
+            _add_requirements(cdist_object, cdist_object.autorequire)
         return graph_check_cycle(graph)
 
     def iterate_until_finished(self):

From d2eec6066876eeb4a295b81a670c70b0ff8ad2a7 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 11 Apr 2021 23:05:48 +0300
Subject: [PATCH 407/527] [__download] make --sum optional

---
 cdist/conf/type/__download/explorer/state     |  6 ++++++
 cdist/conf/type/__download/man.rst            | 15 ++++++---------
 cdist/conf/type/__download/parameter/optional |  1 +
 cdist/conf/type/__download/parameter/required |  1 -
 4 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 00362545..68b517c5 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -8,6 +8,12 @@ then
     exit 0
 fi
 
+if [ ! -f "$__object/parameter/sum" ]
+then
+    echo 'present'
+    exit 0
+fi
+
 sum_should="$( cat "$__object/parameter/sum" )"
 
 if [ -f "$__object/parameter/cmd-sum" ]
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index 54503470..a1278cfb 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -8,9 +8,6 @@ cdist-type__download - Download a file
 
 DESCRIPTION
 -----------
-Destination (``$__object_id``) in target host must be persistent storage
-in order to calculate checksum and decide if file must be (re-)downloaded.
-
 By default type will try to use ``wget``, ``curl`` or ``fetch``.
 If download happens in target (see ``--download``) then type will
 fallback to (and install) ``wget``.
@@ -25,14 +22,14 @@ REQUIRED PARAMETERS
 url
    File's URL.
 
-sum
-   Checksum of file going to be downloaded.
-   By default output of ``cksum`` without filename is expected.
-   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
-
 
 OPTIONAL PARAMETERS
 -------------------
+sum
+   Checksum is used to decide if existing destination file must be redownloaded.
+   By default output of ``cksum`` without filename is expected.
+   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
+
 download
    If ``local`` (default), then download file to local storage and copy
    it to target host. If ``remote``, then download happens in target.
@@ -81,7 +78,7 @@ Ander Punnar <ander-at-kvlt-dot-ee>
 
 COPYING
 -------
-Copyright \(C) 2020 Ander Punnar. You can redistribute it
+Copyright \(C) 2021 Ander Punnar. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index 838e2fbf..d69e083e 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,3 +1,4 @@
+sum
 cmd-get
 cmd-sum
 download
diff --git a/cdist/conf/type/__download/parameter/required b/cdist/conf/type/__download/parameter/required
index 6ea4c38f..96cdd3b9 100644
--- a/cdist/conf/type/__download/parameter/required
+++ b/cdist/conf/type/__download/parameter/required
@@ -1,2 +1 @@
 url
-sum

From 9ec01d9f971339ee67c866c4a248047966f73ce1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 13 Apr 2021 12:22:45 +0200
Subject: [PATCH 408/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index db60c0c4..c8cb44da 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,7 @@ next:
 	* Core: Maintain object relationship graph in cdist cache (Darko Poljak)
 	* Type __git: Fix numeric owner and group handling (Dennis Camera)
 	* Type __pyvenv: Fix numeric owner and group handling (Dennis Camera)
+	* Type __download: Make sum parameter optional (Ander Punnar)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From 92b8942a8c37bb985ef42991f3b7aaef0cd8073f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 09:00:32 +0200
Subject: [PATCH 409/527] [type/__postgres_conf] Add psql_exec function to
 state explorer

---
 cdist/conf/type/__postgres_conf/explorer/state | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index de6e8aa0..1a58751a 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -20,21 +20,24 @@
 #
 
 postgres_user=$("${__type_explorer:?}/postgres_user")
-
 conf_name=${__object_id:?}
 
-su - "${postgres_user}" -c 'psql postgres -c "SELECT 1"' >/dev/null || {
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_exec() {
+	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
+}
+
+psql_exec 'SELECT 1' >/dev/null || {
 	echo 'Connection to PostgreSQL server failed' >&2
 	exit 1
 }
 
-if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
-	| cmp -s "${__object:?}/parameter/value" -
+if psql_exec "SHOW ${conf_name}" | cmp -s "${__object:?}/parameter/value" -
 then
 	echo present
 else
 	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
-	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"")
+	case $(psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')")
 	in
 		('')
 			# invalid configuration parameter

From 2ccc03fef1e28c759a7918fa00ffcb45c76dc57a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 09:02:40 +0200
Subject: [PATCH 410/527] [type/__postgres_conf] Add psql_conf_cmp function to
 state explorer

---
 cdist/conf/type/__postgres_conf/explorer/state | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 1a58751a..be881be6 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -27,12 +27,16 @@ psql_exec() {
 	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
 }
 
+psql_conf_cmp() {
+	test "$(psql_exec "SHOW $1")" = "$2"
+}
+
 psql_exec 'SELECT 1' >/dev/null || {
 	echo 'Connection to PostgreSQL server failed' >&2
 	exit 1
 }
 
-if psql_exec "SHOW ${conf_name}" | cmp -s "${__object:?}/parameter/value" -
+if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
 then
 	echo present
 else

From e0416403c4862cb8bc0f41a7c6f8a33d1a8656a5 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 09:04:07 +0200
Subject: [PATCH 411/527] [type/__postgres_conf] Add psql_conf_source function
 to state explorer

---
 cdist/conf/type/__postgres_conf/explorer/state | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index be881be6..9f54724a 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -27,6 +27,10 @@ psql_exec() {
 	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
 }
 
+psql_conf_source() {
+	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
+	psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')"
+}
 psql_conf_cmp() {
 	test "$(psql_exec "SHOW $1")" = "$2"
 }
@@ -40,8 +44,7 @@ if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
 then
 	echo present
 else
-	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
-	case $(psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')")
+	case $(psql_conf_source "${conf_name}")
 	in
 		('')
 			# invalid configuration parameter

From 12c2995494ce582cb3adf1d942b36f6df035370a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 14:46:36 +0200
Subject: [PATCH 412/527] [type/__postgres_conf] Implement complex state
 compare logic

---
 .../conf/type/__postgres_conf/explorer/state  | 166 +++++++++++++++++-
 1 file changed, 163 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 9f54724a..e76540c5 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -22,6 +22,52 @@
 postgres_user=$("${__type_explorer:?}/postgres_user")
 conf_name=${__object_id:?}
 
+tolower() { printf '%s' "$*" | tr '[:upper:]' '[:lower:]'; }
+
+tobytes() {
+	# NOTE: This function treats everything as base 2.
+	#       It is not compatible with SI units.
+	awk 'BEGIN { FS = "\n" }
+	/TB$/ { $0 = ($0 * 1024) "GB" }
+	/GB$/ { $0 = ($0 * 1024) "MB" }
+	/MB$/ { $0 = ($0 * 1024) "kB" }
+	/kB$/ { $0 = ($0 * 1024) "B" }
+	 /B?$/ { sub(/ *B?$/, "") }
+	($0*1) == $0  # is number
+	' <<-EOF
+	$1
+	EOF
+}
+
+tomillisecs() {
+	awk 'BEGIN { FS = "\n" }
+	    /d$/ { $0 = ($0 * 24) "h" }
+	    /h$/ { $0 = ($0 * 60) "min" }
+	  /min$/ { $0 = ($0 * 60) "s" }
+	/[^m]s$/ { $0 = ($0 * 1000) "ms" }
+	   /ms$/ { $0 *= 1 }
+	($0*1) == $0  # is number
+	' <<-EOF
+	$1
+	EOF
+}
+
+tobool() {
+	# prints either 'on' or 'off'
+	case $(tolower "$1")
+	in
+		(t|true|y|yes|on|1)
+			echo 'on' ;;
+		(f|false|n|no|off|0)
+			echo 'off' ;;
+		(*)
+			printf 'Inavlid bool value: %s\n' "$2" >&2
+			return 1
+			;;
+	esac
+	return 0
+}
+
 quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
 psql_exec() {
 	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
@@ -31,9 +77,123 @@ psql_conf_source() {
 	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
 	psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')"
 }
-psql_conf_cmp() {
-	test "$(psql_exec "SHOW $1")" = "$2"
-}
+psql_conf_cmp() (
+	IFS='|' read -r lower_name vartype setting unit <<-EOF
+	$(psql_exec "SELECT lower(name), vartype, setting, unit FROM pg_settings WHERE lower(name) = lower('$1')")
+	EOF
+
+	should_value=$2
+	is_value=${setting}
+
+	# The following case contains special cases for special settings.
+	case ${lower_name}
+	in
+		(archive_command)
+			if test "${setting}" = '(disabled)'
+			then
+				# DAFUQ PostgreSQL?!
+				# PostgreSQL returns (disabled) if the feature is inactive.
+				# We cannot compare the values unless it is enabled, first.
+				return 0
+			fi
+			;;
+		(archive_mode|backslash_quote|constraint_exclusion|force_parallel_mode|huge_pages|synchronous_commit)
+			# Although only 'on', 'off' are documented, PostgreSQL accepts all
+			# the "likely" variants of "on" and "off".
+			case $(tolower "${should_value}")
+			in
+				(on|off|true|false|yes|no|1|0)
+					should_value=$(tobool "${should_value}")
+					;;
+			esac
+			;;
+	esac
+
+	case ${vartype}
+	in
+		(bool)
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported for vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			should_value=$(tobool "${should_value}")
+
+			test "${is_value}" = "${should_value}"
+			;;
+		(enum)
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported with vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			# NOTE: All enums that are currently defined are lower case, but
+			#       PostgreSQL also accepts upper case spelling.
+			should_value=$(tolower "$2")
+
+			test "${is_value}" = "${should_value}"
+			;;
+		(integer)
+			# split multiples from unit, first (e.g. 8kB -> 8, kB)
+			case ${unit}
+			in
+				([0-9]*)
+					multiple=${unit%%[!0-9]*}
+					unit=${unit##*[0-9 ]}
+					;;
+				(*) multiple=1 ;;
+			esac
+
+			is_value=$((setting * multiple))${unit}
+
+			if expr "${should_value}" : '-\{0,1\}[0-9]*$' >/dev/null
+			then
+				# default unit
+				should_value=$((should_value * multiple))${unit}
+			fi
+
+			# then, do conversion
+			# NOTE: these conversions work for integers only!
+			case ${unit}
+			in
+				(B|[kMGT]B)
+					# bytes
+					is_bytes=$(tobytes "${is_value}")
+					should_bytes=$(tobytes "${should_value}")
+
+					test $((is_bytes)) -eq $((should_bytes))
+					;;
+				(ms|s|min|h|d)
+					# seconds
+					is_ms=$(tomillisecs "${is_value}")
+					should_ms=$(tomillisecs "${should_value}")
+
+					test $((is_ms)) -eq $((should_ms))
+					;;
+				('')
+					# no unit
+					is_int=${is_value}
+					should_int=${should_value}
+
+					test $((is_int)) -eq $((should_int))
+					;;
+			esac
+			;;
+		(real|string)
+			# NOTE: reals could possibly have units, but currently there none.
+
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported with vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			test "${is_value}" = "${should_value}"
+			;;
+	esac
+)
 
 psql_exec 'SELECT 1' >/dev/null || {
 	echo 'Connection to PostgreSQL server failed' >&2

From bef1433ba3adb022f8b50eb9df0fbbbf90757d76 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 14:46:50 +0200
Subject: [PATCH 413/527] [type/__postgres_conf] Accept empty values

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index d0d247b4..27651600 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -57,7 +57,7 @@ psql_cmd() {
 case ${state_should}
 in
 	(present)
-		test -s "${__object:?}/parameter/value" || {
+		test -n "${__object:?}/parameter/value" || {
 			echo 'Missing required parameter --value' >&2
 			exit 1
 		}

From 686e4f0f2d498ee39a69e43f7535171f16f5fcbc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 15:01:38 +0200
Subject: [PATCH 414/527] [type/__postgres_conf] Reverse state logic (decide
 based on source first)

---
 .../conf/type/__postgres_conf/explorer/state  | 39 ++++++++++---------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index e76540c5..4b7b0a43 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -200,23 +200,24 @@ psql_exec 'SELECT 1' >/dev/null || {
 	exit 1
 }
 
-if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
-then
-	echo present
-else
-	case $(psql_conf_source "${conf_name}")
-	in
-		('')
-			# invalid configuration parameter
-			# (error message was already printed by SHOW command above.
-			#  Yes, it's a hack)
-			exit 1
-			;;
-		(default)
-			echo absent
-			;;
-		(*)
+case $(psql_conf_source "${conf_name}")
+in
+	('')
+		printf 'Invalid configuration parameter: %s\n' "${conf_name}" >&2
+		exit 1
+		;;
+	(default)
+		echo absent
+		;;
+	(*)
+		if ! test -f "${__object:?}/parameter/value"
+		then
+			echo present
+		elif psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
+		then
+			echo present
+		else
 			echo different
-			;;
-	esac
-fi
+		fi
+		;;
+esac

From 19bf37be1a8b0b099a7199d1565b74598010f614 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 15:55:52 +0200
Subject: [PATCH 415/527] [type/__postgres_conf] Update man.rst

---
 cdist/conf/type/__postgres_conf/man.rst | 39 ++++++++++++++-----------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst
index 76016c6c..e035f080 100644
--- a/cdist/conf/type/__postgres_conf/man.rst
+++ b/cdist/conf/type/__postgres_conf/man.rst
@@ -1,29 +1,27 @@
 cdist-type__postgres_conf(7)
 ============================
 
-Configure a PostgreSQL server.
-
-NOTE: This type might need to be run multiple times to apply all bits of the
-configuration due to ordering requirements.
-
-SSRQ <cdist--@--ssrq-sds-fds.ch>
+NAME
+----
+cdist-type__postgres_conf - Alter PostgreSQL configuration
 
 
 DESCRIPTION
 -----------
-Configure a PostgreSQL server using ALTER SYSTEM.
+Configure a running PostgreSQL server using ``ALTER SYSTEM``.
 
 
 REQUIRED PARAMETERS
 -------------------
 value
-    The value to setup (can be omitted when state is set to "absent").
+   The value to set (can be omitted if ``--state`` is set to ``absent``).
 
 
 OPTIONAL PARAMETERS
 -------------------
 state
-    "present" or "absent". Defaults to "present".
+   ``present`` or ``absent``.
+   Defaults to ``present``.
 
 
 BOOLEAN PARAMETERS
@@ -36,20 +34,27 @@ EXAMPLES
 
 .. code-block:: sh
 
-	# set timezone
-    __postgres_conf timezone --value Europe/Zurich
+   # set timezone
+   __postgres_conf timezone --value Europe/Zurich
 
-    # reset maximum number of concurrent connections to default (normally 100)
-    __postgres_conf max_connections --state absent
+   # reset maximum number of concurrent connections to default (normally 100)
+   __postgres_conf max_connections --state absent
 
 
 SEE ALSO
 --------
-- `cdist-type(7) <cdist-type.html>`_
+None.
+
+
+AUTHORS
+-------
+Beni Ruef (bernhard.ruef--@--ssrq-sds-fds.ch)
+Dennis Camera (dennis.camera--@--ssrq-sds-fds.ch)
 
 
 COPYING
 -------
-Copyright \(C) 2020 SSRQ (www.ssrq-sds-fds.ch).
-Free use of this software is granted under the terms
-of the GNU General Public License version 3 (GPLv3).
+Copyright \(C) 2019-2021 SSRQ (www.ssrq-sds-fds.ch).
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.

From 1c047353a95e7ac079ccf89af8fc232451b8f891 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 17 Apr 2021 09:57:10 +0200
Subject: [PATCH 416/527] [bin/cdist] Fix Python version check

---
 bin/cdist         | 8 +++++---
 cdist/__init__.py | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index ddaffa7f..adb06a8d 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -72,9 +72,11 @@ def commandline():
 
 
 if __name__ == "__main__":
-    if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION:
-        print('Python >= {} is required on the source host.'.format(
-                cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr)
+    if sys.version_info[:3] < cdist.MIN_SUPPORTED_PYTHON_VERSION:
+        print(
+            'Python >= {} is required on the source host.'.format(
+                ".".join(map(str, cdist.MIN_SUPPORTED_PYTHON_VERSION))),
+            file=sys.stderr)
         sys.exit(1)
 
     exit_code = 0
diff --git a/cdist/__init__.py b/cdist/__init__.py
index 44366cd0..31d49889 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -64,7 +64,7 @@ REMOTE_EXEC = "ssh -o User=root"
 REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
 
 
-MIN_SUPPORTED_PYTHON_VERSION = '3.5'
+MIN_SUPPORTED_PYTHON_VERSION = (3, 5)
 
 
 class Error(Exception):

From 1bb696a4100f161afada9b1e8d8ad2b8ab190f54 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 20 Apr 2021 07:33:07 +0200
Subject: [PATCH 417/527] Release 6.9.6

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index c8cb44da..70d0b960 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)

From acf9bf91f17fb4ad0e5813c6e4b0b40fc7e027b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Thu, 22 Apr 2021 08:55:14 +0200
Subject: [PATCH 418/527] [scanner] error to stderr and exit when scapy is not
 available

---
 cdist/scan/commandline.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index eca4cf13..0d7fb0ca 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -20,6 +20,7 @@
 #
 
 import logging
+import sys
 
 log = logging.getLogger("scan")
 
@@ -31,7 +32,9 @@ def commandline(args):
     try:
         import cdist.scan.scan as scan
     except ModuleNotFoundError:
-        print('cdist scan requires scapy to be installed')
+        print('cdist scan requires scapy to be installed! Exiting.',
+                file=sys.stderr)
+        sys.exit(1)
 
     processes = []
 

From a4464209b6741f2c1764aa432c56cfa8128852a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Thu, 22 Apr 2021 09:29:53 +0200
Subject: [PATCH 419/527] [scanner] add minimal error handling, consolidate CLI
 args processing

---
 cdist/argparse.py         | 10 ++++--
 cdist/scan/commandline.py | 19 ++++++-----
 cdist/scan/scan.py        | 71 +++++++++++----------------------------
 3 files changed, 37 insertions(+), 63 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index cadac39a..153e7864 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -492,12 +492,16 @@ def get_parsers():
         help='Try to configure detected hosts')
     parser['scan'].add_argument(
         '-I', '--interfaces',
-        action='append',  default=[],
+        action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
     parser['scan'].add_argument(
         '-d', '--delay',
-        action='store',  default=3600,
-        help='How long to wait before reconfiguring after last try')
+        action='store',  default=3600, type=int,
+        help='How long (seconds) to wait before reconfiguring after last try')
+    parser['scan'].add_argument(
+        '-t', '--trigger-delay',
+        action='store',  default=5, type=int,
+        help='How long (seconds) to wait between ICMPv6 echo requests')
     parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
 
     for p in parser:
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 0d7fb0ca..dead5292 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -24,26 +24,29 @@ import sys
 
 log = logging.getLogger("scan")
 
-
-# define this outside of the class to not handle scapy import errors by default
+# CLI processing is defined outside of the main scan class to handle
+# non-available optional scapy dependency (instead of crashing mid-flight).
 def commandline(args):
     log.debug(args)
 
+    # Check if we have the optional scapy dependency available.
     try:
         import cdist.scan.scan as scan
     except ModuleNotFoundError:
-        print('cdist scan requires scapy to be installed! Exiting.',
-                file=sys.stderr)
+        log.error('cdist scan requires scapy to be installed. Exiting.')
         sys.exit(1)
 
-    processes = []
-
+    # Default operation mode.
     if not args.mode:
-        # By default scan and trigger, but do not call any action
+        # By default scan and trigger, but do not call any action.
         args.mode = ['scan', 'trigger', ]
 
+    # We run each component in a separate process since they
+    # must not block on each other.
+    processes = []
+
     if 'trigger' in args.mode:
-        t = scan.Trigger(interfaces=args.interfaces)
+        t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay)
         t.start()
         processes.append(t)
         log.debug("Trigger started")
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index faee8a56..633a5c06 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -61,20 +61,22 @@ import datetime
 
 import cdist.config
 
+logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("scan")
 
-
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
     """
 
-    def __init__(self, interfaces=None, verbose=False):
+    def __init__(self, interfaces, sleeptime, verbose=False):
         self.interfaces = interfaces
+
+        # Used by scapy / send in trigger/2.
         self.verbose = verbose
 
-        # Wait 5 seconds before triggering again - FIXME: add parameter
-        self.sleeptime = 5
+        # Delay in seconds between sent ICMPv6EchoRequests.
+        self.sleeptime = sleeptime
 
     def start(self):
         self.processes = []
@@ -93,9 +95,12 @@ class Trigger(object):
             time.sleep(self.sleeptime)
 
     def trigger(self, interface):
-        packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest()
-        log.debug("Sending request on %s", interface)
-        send(packet, verbose=self.verbose)
+        try:
+            log.debug("Sending ICMPv6EchoRequest on %s", interface)
+            packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest()
+            send(packet, verbose=self.verbose)
+        except Exception as e:
+            log.error( "Could not send ICMPv6EchoRequest: %s", e)
 
 
 class Scanner(object):
@@ -103,7 +108,7 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces=None, args=None, outdir=None):
+    def __init__(self, interfaces, args=None, outdir=None):
         self.interfaces = interfaces
 
         if outdir:
@@ -148,47 +153,9 @@ class Scanner(object):
 
     def scan(self):
         log.debug("Scanning - zzzzz")
-        sniff(iface=self.interfaces,
-              filter="icmp6",
-              prn=self.handle_pkg)
-
-
-if __name__ == '__main__':
-    t = Trigger(interfaces=["wlan0"])
-    t.start()
-
-    # Scanner can listen on many interfaces at the same time
-    s = Scanner(interfaces=["wlan0"])
-    s.scan()
-
-    # Join back the trigger processes
-    t.join()
-
-    # Test in my lan shows:
-    # [18:48] bridge:cdist% ls -1d fe80::*
-    # fe80::142d:f0a5:725b:1103
-    # fe80::20d:b9ff:fe49:ac11
-    # fe80::20d:b9ff:fe4c:547d
-    # fe80::219:d2ff:feb2:2e12
-    # fe80::21b:fcff:feee:f446
-    # fe80::21b:fcff:feee:f45c
-    # fe80::21b:fcff:feee:f4b1
-    # fe80::21b:fcff:feee:f4ba
-    # fe80::21b:fcff:feee:f4bc
-    # fe80::21b:fcff:feee:f4c1
-    # fe80::21d:72ff:fe86:46b
-    # fe80::42b0:34ff:fe6f:f6f0
-    # fe80::42b0:34ff:fe6f:f863
-    # fe80::42b0:34ff:fe6f:f9b2
-    # fe80::4a5d:60ff:fea1:e55f
-    # fe80::77a3:5e3f:82cc:f2e5
-    # fe80::9e93:4eff:fe6c:c1f4
-    # fe80::ba69:f4ff:fec5:6041
-    # fe80::ba69:f4ff:fec5:8db7
-    # fe80::bad8:12ff:fe65:313d
-    # fe80::bad8:12ff:fe65:d9b1
-    # fe80::ce2d:e0ff:fed4:2611
-    # fe80::ce32:e5ff:fe79:7ea7
-    # fe80::d66d:6dff:fe33:e00
-    # fe80::e2ff:f7ff:fe00:20e6
-    # fe80::f29f:c2ff:fe7c:275e
+        try:
+            sniff(iface=self.interfaces,
+                  filter="icmp6",
+                  prn=self.handle_pkg)
+        except Exception as e:
+            log.error( "Could not start listener: %s", e)

From bb24d632d62da8390dd1a724fc325a57526da40d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Thu, 22 Apr 2021 10:20:49 +0200
Subject: [PATCH 420/527] [scanner] implement the --list flag

---
 cdist/argparse.py         |  4 +++
 cdist/scan/commandline.py | 63 ++++++++++++++++++++++++++++-----------
 cdist/scan/scan.py        | 17 +++++++++++
 3 files changed, 67 insertions(+), 17 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 153e7864..f390a974 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -486,6 +486,10 @@ def get_parsers():
         '-m', '--mode', help='Which modes should run',
         action='append', default=[],
         choices=['scan', 'trigger'])
+    parser['scan'].add_argument(
+        '--list',
+        action='store_true',
+        help='List the known hosts and exit')
     parser['scan'].add_argument(
         '--config',
         action='store_true',
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index dead5292..331694e4 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -21,26 +21,11 @@
 
 import logging
 import sys
+from datetime import datetime
 
 log = logging.getLogger("scan")
 
-# CLI processing is defined outside of the main scan class to handle
-# non-available optional scapy dependency (instead of crashing mid-flight).
-def commandline(args):
-    log.debug(args)
-
-    # Check if we have the optional scapy dependency available.
-    try:
-        import cdist.scan.scan as scan
-    except ModuleNotFoundError:
-        log.error('cdist scan requires scapy to be installed. Exiting.')
-        sys.exit(1)
-
-    # Default operation mode.
-    if not args.mode:
-        # By default scan and trigger, but do not call any action.
-        args.mode = ['scan', 'trigger', ]
-
+def run(scan, args):
     # We run each component in a separate process since they
     # must not block on each other.
     processes = []
@@ -59,3 +44,47 @@ def commandline(args):
 
     for process in processes:
         process.join()
+
+def list(scan, args):
+    s = scan.Scanner(interfaces=args.interfaces, args=args)
+    hosts = s.list()
+
+    # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
+    # 6 colons.
+    ipv6_max_size = 8 * 4 + 10
+    # We format dates as follow: YYYY-MM-DD HH:MM:SS
+    date_max_size =              8 + 2    + 6 + 2
+
+    print("{} | {}".format(
+        'link-local address'.ljust(ipv6_max_size),
+        'last seen'.ljust(date_max_size)))
+    print('=' * (ipv6_max_size + 3 + date_max_size))
+    for addr in hosts:
+        last_seen = datetime.strftime(
+                datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'),
+                '%Y-%m-%d %H:%M:%S')
+        print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size)))
+
+# CLI processing is defined outside of the main scan class to handle
+# non-available optional scapy dependency (instead of crashing mid-flight).
+def commandline(args):
+    log.debug(args)
+
+    # Check if we have the optional scapy dependency available.
+    try:
+        import cdist.scan.scan as scan
+    except ModuleNotFoundError:
+        log.error('cdist scan requires scapy to be installed. Exiting.')
+        sys.exit(1)
+
+    # Set default operation mode.
+    if not args.mode:
+        # By default scan and trigger, but do not call any action.
+        args.mode = ['scan', 'trigger', ]
+
+    # Print known hosts and exit is --list is specified - do not start
+    # the scanner.
+    if args.list:
+        list(scan, args)
+    else:
+        run(scan, args)
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 633a5c06..f3976370 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -132,6 +132,23 @@ class Scanner(object):
             with open(fname, "w") as fd:
                 fd.write(f"{now}\n")
 
+    def list(self):
+        hosts = dict()
+        for linklocal_addr in os.listdir(self.outdir):
+            workdir = os.path.join(self.outdir, linklocal_addr)
+            # We ignore any (unexpected) file in this directory.
+            if os.path.isdir(workdir):
+                last_seen='-'
+                last_seen_file = os.path.join(workdir, 'last_seen')
+                if os.path.isfile(last_seen_file):
+                    with open(last_seen_file, "r") as fd:
+                        last_seen = fd.readline()
+
+                hosts[linklocal_addr] = {'last_seen': last_seen}
+
+        return hosts
+
+
     def config(self):
         """
         Configure a host

From 13e2ad175f01b6cfbdb21ee50e85b6f3ba6fe750 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Sun, 25 Apr 2021 12:45:34 +0200
Subject: [PATCH 421/527] [scanner] add host class, name mapper and pre-config
 logic

---
 cdist/argparse.py         |   6 +-
 cdist/scan/commandline.py |  37 ++++++++----
 cdist/scan/scan.py        | 122 +++++++++++++++++++++++++++++---------
 3 files changed, 124 insertions(+), 41 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index f390a974..bedb23ac 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -485,7 +485,7 @@ def get_parsers():
     parser['scan'].add_argument(
         '-m', '--mode', help='Which modes should run',
         action='append', default=[],
-        choices=['scan', 'trigger'])
+        choices=['scan', 'trigger', 'config'])
     parser['scan'].add_argument(
         '--list',
         action='store_true',
@@ -498,6 +498,10 @@ def get_parsers():
         '-I', '--interfaces',
         action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
+    parser['scan'].add_argument(
+        '--name-mapper',
+        action='store',  default=None,
+        help='Map addresses to names, required for config mode')
     parser['scan'].add_argument(
         '-d', '--delay',
         action='store',  default=3600, type=int,
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 331694e4..1a0ab0a7 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -37,7 +37,10 @@ def run(scan, args):
         log.debug("Trigger started")
 
     if 'scan' in args.mode:
-        s = scan.Scanner(interfaces=args.interfaces, args=args)
+        s = scan.Scanner(
+                autoconfigure='config' in args.mode,
+                interfaces=args.interfaces,
+                name_mapper=args.name_mapper)
         s.start()
         processes.append(s)
         log.debug("Scanner started")
@@ -46,24 +49,27 @@ def run(scan, args):
         process.join()
 
 def list(scan, args):
-    s = scan.Scanner(interfaces=args.interfaces, args=args)
+    s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper)
     hosts = s.list()
 
     # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
     # 6 colons.
     ipv6_max_size = 8 * 4 + 10
-    # We format dates as follow: YYYY-MM-DD HH:MM:SS
-    date_max_size =              8 + 2    + 6 + 2
+    date_max_size = len(datetime.now().strftime(scan.datetime_format))
+    name_max_size = 25
 
-    print("{} | {}".format(
-        'link-local address'.ljust(ipv6_max_size),
-        'last seen'.ljust(date_max_size)))
-    print('=' * (ipv6_max_size + 3 + date_max_size))
-    for addr in hosts:
-        last_seen = datetime.strftime(
-                datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'),
-                '%Y-%m-%d %H:%M:%S')
-        print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size)))
+    print("{} | {} | {} | {}".format(
+        'name'.ljust(name_max_size),
+        'address'.ljust(ipv6_max_size),
+        'last seen'.ljust(date_max_size),
+        'last configured'.ljust(date_max_size)))
+    print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
+    for host in hosts:
+        print("{} | {} | {} | {}".format(
+            host.name(default='-').ljust(name_max_size),
+            host.address().ljust(ipv6_max_size),
+            host.last_seen().ljust(date_max_size),
+            host.last_configured().ljust(date_max_size)))
 
 # CLI processing is defined outside of the main scan class to handle
 # non-available optional scapy dependency (instead of crashing mid-flight).
@@ -82,6 +88,11 @@ def commandline(args):
         # By default scan and trigger, but do not call any action.
         args.mode = ['scan', 'trigger', ]
 
+    if 'config' in args.mode and args.name_mapper == None:
+        print('--name-mapper must be specified for scanner config mode.',
+                file=sys.stderr)
+        sys.exit(1)
+
     # Print known hosts and exit is --list is specified - do not start
     # the scanner.
     if args.list:
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index f3976370..459138e2 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -63,6 +63,69 @@ import cdist.config
 
 logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("scan")
+datetime_format = '%Y-%m-%d %H:%M:%S'
+
+class Host(object):
+    def __init__(self, addr, outdir, name_mapper=None):
+        self.addr = addr
+        self.workdir = os.path.join(outdir, addr)
+        self.name_mapper = name_mapper
+
+        os.makedirs(self.workdir, exist_ok=True)
+
+    def __get(self, key, default=None):
+        fname = os.path.join(self.workdir, key)
+        value=default
+        if os.path.isfile(fname):
+            with open(fname, "r") as fd:
+                value = fd.readline()
+        return value
+
+    def __set(self, key, value):
+        fname = os.path.join(self.workdir, key)
+        with open(fname, "w") as fd:
+            fd.write(f"{value}")
+
+    def name(self, default=None):
+        if self.name_mapper == None:
+            return default
+
+        fpath = os.path.join(os.getcwd(), self.name_mapper)
+        if os.path.isfile(fpath) and os.access(fpath, os.X_OK):
+             out = subprocess.run([fpath, self.addr], capture_output=True)
+             if out.returncode != 0:
+                 return default
+             else:
+                value = out.stdout.decode()
+                return (None if len(value) == 0 else value)
+        else:
+            return default
+
+    def address(self):
+        return self.addr
+
+    def last_seen(self, default=None):
+        raw = self.__get('last_seen')
+        if raw:
+            return datetime.datetime.strptime(raw, datetime_format)
+        else:
+            return default
+
+    def last_configured(self, default=None):
+        raw = self.__get('last_configured')
+        if raw:
+            return datetime.datetime.strptime(raw, datetime_format)
+        else:
+            return default
+
+    def seen(self):
+        now = datetime.datetime.now().strftime(datetime_format)
+        self.__set('last_seen', now)
+
+    def configure(self):
+        # TODO: configure.
+        now = datetime.datetime.now().strftime(datetime_format)
+        self.__set('last_configured', now)
 
 class Trigger(object):
     """
@@ -108,48 +171,43 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces, args=None, outdir=None):
+    def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None):
         self.interfaces = interfaces
+        self.autoconfigure=autoconfigure
+        self.name_mapper = name_mapper
+        self.config_delay = datetime.timedelta(seconds=3600)
 
         if outdir:
             self.outdir = outdir
         else:
             self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan')
+        os.makedirs(self.outdir, exist_ok=True)
+
+        self.running_configs = {}
 
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
-            host = pkg['IPv6'].src
-            log.verbose("Host %s is alive", host)
+            host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper)
+            if host.name():
+                log.verbose("Host %s (%s) is alive", host.name(), host.address())
+            else:
+                log.verbose("Host %s is alive", host.address())
+            host.seen()
 
-            dir = os.path.join(self.outdir, host)
-            fname = os.path.join(dir, "last_seen")
-
-            now = datetime.datetime.now()
-
-            os.makedirs(dir, exist_ok=True)
-
-            # FIXME: maybe adjust the format so we can easily parse again
-            with open(fname, "w") as fd:
-                fd.write(f"{now}\n")
+            # TODO check last config.
+            if self.autoconfigure and \
+                    host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now():
+                self.config(host)
 
     def list(self):
-        hosts = dict()
-        for linklocal_addr in os.listdir(self.outdir):
-            workdir = os.path.join(self.outdir, linklocal_addr)
-            # We ignore any (unexpected) file in this directory.
-            if os.path.isdir(workdir):
-                last_seen='-'
-                last_seen_file = os.path.join(workdir, 'last_seen')
-                if os.path.isfile(last_seen_file):
-                    with open(last_seen_file, "r") as fd:
-                        last_seen = fd.readline()
-
-                hosts[linklocal_addr] = {'last_seen': last_seen}
+        hosts = []
+        for addr in os.listdir(self.outdir):
+            hosts.append(Host(addr, self.outdir, self.name_mapper))
 
         return hosts
 
 
-    def config(self):
+    def config(self, host):
         """
         Configure a host
 
@@ -158,9 +216,19 @@ class Scanner(object):
         - Maybe keep dict storing per host processes
         - Save the result
         - Save the output -> probably aligned to config mode
-
         """
 
+        if host.name() == None:
+            log.debug("config - could not resolve name for %s, aborting.", host.address())
+            return
+
+        if self.running_configs.get(host.name()) != None:
+            log.debug("config - is already running for %s, aborting.", host.name())
+
+        log.info("config - running against host %s.", host.name())
+        p = host.configure()
+        self.running_configs[host.name()] = p
+
     def start(self):
         self.process = Process(target=self.scan)
         self.process.start()

From 512e9b23c0035b09ac1e7e76f706fee2e2ec481f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 25 Apr 2021 15:53:40 +0200
Subject: [PATCH 422/527] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 70d0b960..c41743cd 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
+
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong

From 6ac8cbf98f5d5018373db6eb9c038ef1c599ff2f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:33:56 +0200
Subject: [PATCH 423/527] [type/__postgres_database] Include postgres_user
 explorer from __postgres_conf

---
 .../explorer/postgres_user                    |  1 +
 .../type/__postgres_database/explorer/state   | 23 +++++--------------
 .../type/__postgres_database/gencode-remote   | 13 +----------
 3 files changed, 8 insertions(+), 29 deletions(-)
 create mode 120000 cdist/conf/type/__postgres_database/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_database/explorer/postgres_user b/cdist/conf/type/__postgres_database/explorer/postgres_user
new file mode 120000
index 00000000..714e7237
--- /dev/null
+++ b/cdist/conf/type/__postgres_database/explorer/postgres_user
@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user
\ No newline at end of file
diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state
index d68d4120..bb044e0e 100755
--- a/cdist/conf/type/__postgres_database/explorer/state
+++ b/cdist/conf/type/__postgres_database/explorer/state
@@ -1,6 +1,7 @@
 #!/bin/sh
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,25 +19,13 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$("${__explorer}/os")"
-in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
-esac
+postgres_user=$("${__type_explorer:?}/postgres_user")
 
+name=${__object_id:?}
 
-name="$__object_id"
-
-if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='$name'\"")"
+if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='${name}'\"")"
 then
-    echo 'present'
+	echo 'present'
 else
-    echo 'absent'
+	echo 'absent'
 fi
diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote
index 0f11cff4..d70fc436 100755
--- a/cdist/conf/type/__postgres_database/gencode-remote
+++ b/cdist/conf/type/__postgres_database/gencode-remote
@@ -18,23 +18,12 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
-in
-   netbsd)
-      postgres_user='pgsql'
-      ;;
-   openbsd)
-      postgres_user='_postgresql'
-      ;;
-   *)
-      postgres_user='postgres'
-      ;;
-esac
 
 
 name="$__object_id"
 state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
 if [ "$state_should" != "$state_is" ]; then
    case "$state_should" in

From 58b279a8d0121dcbef05a4999d1d88208656aeed Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:34:17 +0200
Subject: [PATCH 424/527] [type/__postgres_database] Improve quoting

---
 .../type/__postgres_database/gencode-remote   | 95 +++++++++++--------
 1 file changed, 55 insertions(+), 40 deletions(-)

diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote
index d70fc436..7d7d6fa2 100755
--- a/cdist/conf/type/__postgres_database/gencode-remote
+++ b/cdist/conf/type/__postgres_database/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,49 +19,63 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
 
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
 
-name="$__object_id"
-state_should="$(cat "$__object/parameter/state")"
-state_is="$(cat "$__object/explorer/state")"
 postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
-if [ "$state_should" != "$state_is" ]; then
-   case "$state_should" in
-      present)
-         owner=""
-         if [ -f "$__object/parameter/owner" ]; then
-            owner="-O \"$(cat "$__object/parameter/owner")\""
-         fi
+dbname=${__object_id:?}
+state_should=$(cat "${__object:?}/parameter/state")
+state_is=$(cat "${__object:?}/explorer/state")
 
-         template=""
-         if [ -f "$__object/parameter/template" ]; then
-            template="--template \"$(cat "$__object/parameter/template")\""
-         fi
-
-         encoding=""
-         if [ -f "$__object/parameter/encoding" ]; then
-            encoding="--encoding \"$(cat "$__object/parameter/encoding")\""
-         fi
-
-         lc_collate=""
-         if [ -f "$__object/parameter/lc-collate" ]; then
-            lc_collate="--lc-collate \"$(cat "$__object/parameter/lc-collate")\""
-         fi
-
-         lc_ctype=""
-         if [ -f "$__object/parameter/lc-ctype" ]; then
-            lc_ctype="--lc-ctype \"$(cat "$__object/parameter/lc-ctype")\""
-         fi
-
-         cat << EOF
-su - '$postgres_user' -c "createdb $owner \"$name\" $template $encoding $lc_collate $lc_ctype"
-EOF
-      ;;
-      absent)
-         cat << EOF
-su - '$postgres_user' -c "dropdb \"$name\""
-EOF
-      ;;
-   esac
+if test "${state_should}" = "$state_is"
+then
+	exit 0
 fi
+
+case ${state_should}
+in
+	(present)
+		set --
+
+		while read -r param_name opt
+		do
+			if test -f "${__object:?}/parameter/${param_name}"
+			then
+				set -- "$@" "${opt}" "$(cat "${__object:?}/parameter/${param_name}")"
+			fi
+		done <<-'EOF'
+		owner -O
+		template --template
+		encoding --encoding
+		lc_collate --lc-collate
+		lc_ctype --lc-ctype
+		EOF
+
+		set -- "$@" "${dbname}"
+
+		cat <<-EOF
+		su - $(quote "${postgres_user}") -c $(quote "$(quote createdb "$@")")
+		EOF
+		;;
+	(absent)
+		cat <<-EOF
+		su - $(quote "${postgres_user}") -c $(quote "$(quote dropdb "${dbname}")")
+		EOF
+		;;
+esac

From beb8da6d5fd141ac63eacfb526cfcc43cbf22d8f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:48:51 +0200
Subject: [PATCH 425/527] [type/__postgres_role] Include postgres_user explorer
 from __postgres_conf

---
 .../type/__postgres_role/explorer/postgres_user   |  1 +
 cdist/conf/type/__postgres_role/explorer/state    | 14 +-------------
 cdist/conf/type/__postgres_role/gencode-remote    | 15 +--------------
 3 files changed, 3 insertions(+), 27 deletions(-)
 create mode 120000 cdist/conf/type/__postgres_role/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_role/explorer/postgres_user b/cdist/conf/type/__postgres_role/explorer/postgres_user
new file mode 120000
index 00000000..714e7237
--- /dev/null
+++ b/cdist/conf/type/__postgres_role/explorer/postgres_user
@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user
\ No newline at end of file
diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 34069de9..0b264e6f 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -19,19 +19,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case $("${__explorer:?}/os")
-in
-	(netbsd)
-		postgres_user='pgsql'
-		;;
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(*)
-		postgres_user='postgres'
-		;;
-esac
-
+postgres_user=$("${__type_explorer:?}/postgres_user")
 rolename=${__object_id:?}
 
 
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index d7631fbd..7324b80c 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -28,20 +28,7 @@ quote() {
 	fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
 }
 
-case $(cat "${__global:?}/explorer/os")
-in
-	(netbsd)
-		postgres_user='pgsql'
-		;;
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(*)
-		postgres_user='postgres'
-		;;
-esac
-
-
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 rolename=${__object_id:?}
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")

From 3cf93249c38a6f27c097877add619b5d6499a049 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:55:11 +0200
Subject: [PATCH 426/527] [type/__postgres_extension] Include postgres_user
 explorer from __postgres_conf

---
 .../__postgres_extension/explorer/postgres_user     |  1 +
 cdist/conf/type/__postgres_extension/gencode-remote | 13 +------------
 2 files changed, 2 insertions(+), 12 deletions(-)
 create mode 120000 cdist/conf/type/__postgres_extension/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_extension/explorer/postgres_user b/cdist/conf/type/__postgres_extension/explorer/postgres_user
new file mode 120000
index 00000000..714e7237
--- /dev/null
+++ b/cdist/conf/type/__postgres_extension/explorer/postgres_user
@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user
\ No newline at end of file
diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote
index af9c97f1..a1c84828 100755
--- a/cdist/conf/type/__postgres_extension/gencode-remote
+++ b/cdist/conf/type/__postgres_extension/gencode-remote
@@ -22,18 +22,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
-in
-   netbsd)
-      postgres_user='pgsql'
-      ;;
-   openbsd)
-      postgres_user='_postgresql'
-      ;;
-   *)
-      postgres_user='postgres'
-      ;;
-esac
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
 
 dbname=$(    echo "$__object_id" | cut -d":" -f1 )

From 8296051653a8187374ad51a3a487315767b73a19 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 16 Apr 2021 19:22:58 +0200
Subject: [PATCH 427/527] [type/__postgres_extension] Add state explorer

---
 .../type/__postgres_extension/explorer/state  | 41 ++++++++++++++
 .../type/__postgres_extension/gencode-remote  | 44 ++++++++++-----
 cdist/conf/type/__postgres_extension/man.rst  | 55 +++++++++++--------
 3 files changed, 104 insertions(+), 36 deletions(-)
 create mode 100644 cdist/conf/type/__postgres_extension/explorer/state

diff --git a/cdist/conf/type/__postgres_extension/explorer/state b/cdist/conf/type/__postgres_extension/explorer/state
new file mode 100644
index 00000000..9d156be7
--- /dev/null
+++ b/cdist/conf/type/__postgres_extension/explorer/state
@@ -0,0 +1,41 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Prints "present" if the extension is currently installed.
+# "absent" otherwise.
+
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+
+postgres_user=$("${__type_explorer:?}/postgres_user")
+
+IFS=: read -r dbname extname <<EOF
+${__object_id:?}
+EOF
+
+psql_exec() {
+	su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")"
+}
+
+if psql_exec "${dbname}" 'SELECT extname FROM pg_extension' | grep -qFx "${extname}"
+then
+	echo present
+else
+	echo absent
+fi
diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote
index a1c84828..4ce72622 100755
--- a/cdist/conf/type/__postgres_extension/gencode-remote
+++ b/cdist/conf/type/__postgres_extension/gencode-remote
@@ -2,9 +2,10 @@
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2013 Tomas Pospisek (tpo_deb at sourcepole.ch)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This type was created by Tomas Pospisek based on the
-#__postgres_role type by Steven Armstrong
+# __postgres_role type by Steven Armstrong.
 #
 # This file is part of cdist.
 #
@@ -24,19 +25,36 @@
 
 postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_cmd() {
+	printf 'su - %s -c %s\n' \
+		"$(quote "${postgres_user}")" \
+		"$(quote psql "$(quote "$1")" -c "$(quote "$2")")"
+}
 
-dbname=$(    echo "$__object_id" | cut -d":" -f1 )
-extension=$( echo "$__object_id" | cut -d":" -f2 )
 
-state_should=$( cat "$__object/parameter/state" )
+IFS=: read -r dbname extname <<EOF
+${__object_id:?}
+EOF
 
-case "$state_should" in
-    present)
-         cmd="CREATE EXTENSION IF NOT EXISTS $extension"
-         echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'"
-    ;;
-    absent)
-         cmd="DROP EXTENSION IF EXISTS $extension"
-         echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'"
-    ;;
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
+
+case ${state_should}
+in
+	(present)
+		psql_cmd "${dbname}" "CREATE EXTENSION ${extname}"
+		;;
+	(absent)
+		psql_cmd "${dbname}" "DROP EXTENSION ${extname}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
 esac
diff --git a/cdist/conf/type/__postgres_extension/man.rst b/cdist/conf/type/__postgres_extension/man.rst
index 79645b2b..442239f6 100644
--- a/cdist/conf/type/__postgres_extension/man.rst
+++ b/cdist/conf/type/__postgres_extension/man.rst
@@ -3,32 +3,36 @@ cdist-type__postgres_extension(7)
 
 NAME
 ----
-cdist-type__postgres_extension - manage postgres extensions
+cdist-type__postgres_extension - Manage PostgreSQL extensions
 
 
 DESCRIPTION
 -----------
-This cdist type allows you to create or drop postgres extensions.
+This cdist type allows you to manage PostgreSQL extensions.
 
-The object you need to pass to __postgres_extension consists of
-the database name and the extension name joined by a colon in the
-following form:
-
-.. code-block:: sh
-
-    dbname:extension
-
-f.ex.
+The ``__object_id`` to pass to ``__postgres_extension`` is of the form
+``dbname:extension``, e.g.:
 
 .. code-block:: sh
 
     rails_test:unaccent
 
 
+**CAUTION!** Be careful when installing extensions from (untrusted) third-party
+sources:
+
+   | Installing an extension as superuser requires trusting that the extension's
+     author wrote the extension installation script in a secure fashion. It is
+     not terribly difficult for a malicious user to create trojan-horse objects
+     that will compromise later execution of a carelessly-written extension
+     script, allowing that user to acquire superuser privileges.
+   | – `<https://www.postgresql.org/docs/13/sql-createextension.html#id-1.9.3.64.7>`_
+
+
 OPTIONAL PARAMETERS
 -------------------
 state
-    either "present" or "absent", defaults to "present"
+    either ``present`` or ``absent``, defaults to ``present``.
 
 
 EXAMPLES
@@ -36,24 +40,29 @@ EXAMPLES
 
 .. code-block:: sh
 
-    __postgres_extension           rails_test:unaccent
-    __postgres_extension --present rails_test:unaccent
-    __postgres_extension --absent  rails_test:unaccent
+   # Install extension unaccent into database rails_test
+   __postgres_extension rails_test:unaccent
+
+   # Drop extension unaccent from database fails_test
+   __postgres_extension rails_test:unaccent --state absent
 
 
 SEE ALSO
 --------
-:strong:`cdist-type__postgre_database`\ (7)
+- :strong:`cdist-type__postgres_database`\ (7)
+- PostgreSQL "CREATE EXTENSION" documentation at:
+  `<http://www.postgresql.org/docs/current/static/sql-createextension.html>`_.
 
-Postgres "Create Extension" documentation at: <http://www.postgresql.org/docs/current/static/sql-createextension.html>.
 
-AUTHOR
+AUTHORS
 -------
-Tomas Pospisek <tpo_deb--@--sourcepole.ch>
+| Tomas Pospisek <tpo_deb--@--sourcepole.ch>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+
 
 COPYING
 -------
-Copyright \(C) 2014 Tomas Pospisek. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2014 Tomas Pospisek, 2021 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.

From 0d33407b182964672b1cfc3ffc49c7fa7cf462a0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 16 Apr 2021 19:35:26 +0200
Subject: [PATCH 428/527] [type/__postgres_database] Proper quoting in state
 explorer

---
 cdist/conf/type/__postgres_database/explorer/state | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state
index bb044e0e..6a25df86 100755
--- a/cdist/conf/type/__postgres_database/explorer/state
+++ b/cdist/conf/type/__postgres_database/explorer/state
@@ -21,9 +21,14 @@
 
 postgres_user=$("${__type_explorer:?}/postgres_user")
 
-name=${__object_id:?}
+dbname=${__object_id:?}
 
-if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='${name}'\"")"
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_exec() {
+	su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")"
+}
+
+if psql_exec postgres "SELECT datname FROM pg_database" | grep -qFx "${dbname}"
 then
 	echo 'present'
 else

From 0f05f38384c94b79232ca0917d1323eeefd6e1ed Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 19 Apr 2021 19:09:46 +0200
Subject: [PATCH 429/527] [type/__postgres_role] Treat --password '' like no
 --password

---
 cdist/conf/type/__postgres_role/explorer/state | 11 ++++-------
 cdist/conf/type/__postgres_role/gencode-remote |  2 +-
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 0b264e6f..822816c1 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -43,8 +43,7 @@ role_properties=$(
 	  BEGIN { RS = "\036"; FS = "\034" }
 	  /^\([0-9]+ rows?\)/ { exit }
 	  NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next }
-	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }
-	  '
+	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }'
 )
 
 if test -n "${role_properties}"
@@ -78,12 +77,10 @@ then
 	# Check password
 	passwd_stored=$(
 		psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \
-		| awk 'BEGIN { RS = "\036" } NR == 2'
-		printf .
-	)
-	passwd_stored=${passwd_stored%?.}
+		| awk 'BEGIN { RS = "\036" } NR == 2 { printf "%s.", $0 }')
+	passwd_stored=${passwd_stored%.}
 
-	if test -f "${__object:?}/parameter/password"
+	if test -s "${__object:?}/parameter/password"
 	then
 		passwd_should=$(cat "${__object:?}/parameter/password"; printf .)
 	fi
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 7324b80c..4cb78330 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -46,7 +46,7 @@ psql_query() {
 
 psql_set_password() {
 	# NOTE: Always make sure that the password does not end up in psql_history!
-	# NOTE: Never set an empty string as the password, because they can be
+	# NOTE: Never set an empty string as the password, because it can be
 	#       interpreted differently by different tooling.
 	if test -s "${__object:?}/parameter/password"
 	then

From 92fff7cb77271009f25d999b6451dc1e43e9bb6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 26 Apr 2021 12:09:20 +0200
Subject: [PATCH 430/527] [scanner] fix crash on --list with name mapper
 provided

---
 cdist/scan/commandline.py | 10 ++++++++--
 cdist/scan/scan.py        |  2 +-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 1a0ab0a7..3eb7eec4 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -65,11 +65,17 @@ def list(scan, args):
         'last configured'.ljust(date_max_size)))
     print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
     for host in hosts:
+        last_seen = host.last_seen()
+        last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-'
+
+        last_configured = host.last_configured()
+        last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-'
+
         print("{} | {} | {} | {}".format(
             host.name(default='-').ljust(name_max_size),
             host.address().ljust(ipv6_max_size),
-            host.last_seen().ljust(date_max_size),
-            host.last_configured().ljust(date_max_size)))
+            last_seen.ljust(date_max_size),
+            last_configured.ljust(date_max_size)))
 
 # CLI processing is defined outside of the main scan class to handle
 # non-available optional scapy dependency (instead of crashing mid-flight).
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 459138e2..69a4121d 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -97,7 +97,7 @@ class Host(object):
                  return default
              else:
                 value = out.stdout.decode()
-                return (None if len(value) == 0 else value)
+                return (default if len(value) == 0 else value)
         else:
             return default
 

From 3a9dd5b1669fa29e1713cdadec1596d859d377f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 26 Apr 2021 12:09:55 +0200
Subject: [PATCH 431/527] [scanner] add minimal (non-configurable) config mode

---
 cdist/scan/scan.py | 35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 69a4121d..152abb4e 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -122,8 +122,20 @@ class Host(object):
         now = datetime.datetime.now().strftime(datetime_format)
         self.__set('last_seen', now)
 
+    # XXX: There's no easy way to use the config module without feeding it with
+    # CLI args. Might as well call everything from scratch!
     def configure(self):
-        # TODO: configure.
+        target = self.name() or self.address()
+        cmd = ['cdist', 'config', '-v', target ]
+
+        fname = os.path.join(self.workdir, 'last_configuration_log')
+        with open(fname, "w") as fd:
+            log.debug("Executing: %s", cmd)
+            completed_process = subprocess.run(cmd, stdout=fd, stderr=fd)
+            if completed_process.returncode != 0:
+                log.error("%s return with non-zero code %i - see %s for details.",
+                        cmd, completed_process.returncode, fname)
+
         now = datetime.datetime.now().strftime(datetime_format)
         self.__set('last_configured', now)
 
@@ -194,7 +206,7 @@ class Scanner(object):
                 log.verbose("Host %s is alive", host.address())
             host.seen()
 
-            # TODO check last config.
+            # Configure if needed.
             if self.autoconfigure and \
                     host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now():
                 self.config(host)
@@ -206,27 +218,18 @@ class Scanner(object):
 
         return hosts
 
-
     def config(self, host):
-        """
-        Configure a host
-
-        - Assume we are only called if necessary
-        - However we need to ensure to not run in parallel
-        - Maybe keep dict storing per host processes
-        - Save the result
-        - Save the output -> probably aligned to config mode
-        """
-
         if host.name() == None:
             log.debug("config - could not resolve name for %s, aborting.", host.address())
             return
 
-        if self.running_configs.get(host.name()) != None:
+        previous_config_process = self.running_configs.get(host.name())
+        if previous_config_process != None and previous_config_process.is_alive():
             log.debug("config - is already running for %s, aborting.", host.name())
 
-        log.info("config - running against host %s.", host.name())
-        p = host.configure()
+        log.info("config - running against host %s (%s).", host.name(), host.address())
+        p = Process(target=host.configure())
+        p.start()
         self.running_configs[host.name()] = p
 
     def start(self):

From 2232435c2206b2b3a9a5f2b976df16540b3f1eeb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 26 Apr 2021 14:39:26 +0200
Subject: [PATCH 432/527] [scanner] initial documentation

Note: still needs to patch main cdist(1) manpage
---
 cdist/scan/scan.py      | 32 -------------
 docs/src/cdist-scan.rst | 99 +++++++++++++++++++++++++++++++++++++++++
 docs/src/index.rst      |  1 +
 3 files changed, 100 insertions(+), 32 deletions(-)
 create mode 100644 docs/src/cdist-scan.rst

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 152abb4e..2912dab3 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -19,38 +19,6 @@
 #
 #
 
-#
-# Interface to be implemented:
-# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated
-#   scan: scan / listen for icmp6 replies
-#   trigger: send trigger to multicast
-#   config: configure newly detected hosts
-#   install: install newly detected hosts
-#
-# Scanner logic
-#  - save results to configdir:
-#     basedir = ~/.cdist/scan/<ipv6-address>
-#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time
-#           or similar
-#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record
-#           unix time or similar
-#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record
-#           unix time or similar
-#
-#
-#
-#
-# cdist scan --list
-#       Show all known hosts including last seen flag
-#
-# Logic for reconfiguration:
-#
-#  - record when configured last time
-#  - introduce a parameter --reconfigure-after that takes time argument
-#  - reconfigure if a) host alive and b) reconfigure-after time passed
-#
-
-
 from multiprocessing import Process
 import os
 import logging
diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst
new file mode 100644
index 00000000..02193456
--- /dev/null
+++ b/docs/src/cdist-scan.rst
@@ -0,0 +1,99 @@
+Scan
+=====
+
+Description
+-----------
+Runs cdist as a daemon that discover/watch on hosts and reconfigure them
+periodically. It is especially useful in netboot-based environment where hosts
+boot unconfigured, and to ensure your infrastructure stays in sync with your
+configuration.
+
+This feature is still consider to be in **beta** stage.
+
+Usage (Examples)
+----------------
+
+Discover hosts on local network and configure those whose name is resolved by
+the name mapper script.
+
+.. code-block:: sh
+
+    $ cdist scan --beta --interface eth0 \
+      --mode scan --name-mapper path/to/script \
+      --mode trigger --mode config
+
+List known hosts and exit.
+
+.. code-block:: sh
+
+    $ cdist scan --beta --list --name-mapper path/to/script
+
+Please refer to `cdist(1)` for a detailed list of parameters.
+
+Modes
+-----
+
+The scanner has 3 modes that can be independently toggled. If the `--mode`
+parameter is not specified, only `tigger` and `scan` are enabled (= hosts are
+not configured).
+
+trigger
+  Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to
+  trigger detection by the `scan` module.
+
+scan
+  Watch for incoming ICMPv6 replies and optionally configure detected hosts.
+
+config
+  Enable configuration of hosts detected by `scan`.
+
+Name Mapper Script
+------------------
+
+The name mapper script takes an IPv6 address as first argument and writes the
+resolved name to stdout - if any. The script must be executable.
+
+Simplest script:
+
+.. code-block:: sh
+  #!/bin/sh
+
+  case "$1" in
+  	"fe80::20d:b9ff:fe57:3524")
+  		printf "my-host-01"
+  		;;
+  	"fe80::7603:bdff:fe05:89bb")
+  		printf "my-host-02"
+  		;;
+  esac
+
+Resolving name from `PTR` DNS record:
+
+.. code-block:: sh
+  #!/bin/sh
+
+  for cmd in dig sed; do
+  	if ! command -v $cmd > /dev/null; then
+  		exit 1
+  	fi
+  done
+
+  dig +short -x "$1" | sed -e 's/.$//'
+
+
+Trigger Source Script
+---------------------
+
+This script returns a list of addresses (separated by a newline) to be used by
+`trigger` mode. It is not used to map names. The script must be executable.
+
+Simplest script:
+
+.. code-block:: sh
+  #!/bin/sh
+
+  cat << EOF
+  server1.domain.tld
+  server2.domain.tld
+  server3.domain.tld
+  EOF
diff --git a/docs/src/index.rst b/docs/src/index.rst
index 31c044dc..369d5309 100644
--- a/docs/src/index.rst
+++ b/docs/src/index.rst
@@ -34,6 +34,7 @@ It natively supports IPv6 since the first release.
    cdist-parallelization
    cdist-inventory
    cdist-preos
+   cdist-scan
    cdist-integration
    cdist-reference
    cdist-best-practice

From a4122882f2a14034d1a219bd0314892176c5cb64 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Apr 2021 16:39:51 +0200
Subject: [PATCH 433/527] [type/__debconf_set_selections] Add state explorer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…and to make it work, replace --file with --line.

--file is deprecated because it does not work with the state explorer as the
contents of the file are not available on the target.
---
 .../__debconf_set_selections/explorer/state   | 124 ++++++++++++++++++
 .../__debconf_set_selections/gencode-remote   |  37 ++++--
 .../type/__debconf_set_selections/man.rst     |  49 ++++---
 .../type/__debconf_set_selections/manifest    |  21 +++
 .../parameter/{required => deprecated}        |   0
 .../parameter/optional_multiple               |   1 +
 6 files changed, 206 insertions(+), 26 deletions(-)
 create mode 100644 cdist/conf/type/__debconf_set_selections/explorer/state
 create mode 100755 cdist/conf/type/__debconf_set_selections/manifest
 rename cdist/conf/type/__debconf_set_selections/parameter/{required => deprecated} (100%)
 create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/optional_multiple

diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state
new file mode 100644
index 00000000..68d28941
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/explorer/state
@@ -0,0 +1,124 @@
+#!/bin/sh -e
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Determine current debconf selections' state.
+# Prints one of:
+#   present: all selections are already set as they should.
+#   different: one or more of the selections have a different value.
+#   absent: one or more of the selections are not (currently) defined.
+#
+
+test -x /usr/bin/perl || {
+	# cannot find perl (no perl ~ no debconf)
+	echo 'absent'
+	exit 0
+}
+
+linesfile="${__object:?}/parameter/line"
+test -s "${linesfile}" || {
+	if test -s "${__object:?}/parameter/file"
+	then
+		echo absent
+	else
+		echo present
+	fi
+	exit 0
+}
+
+/usr/bin/perl -- - "${linesfile}" <<'EOF'
+use strict;
+use warnings "all";
+
+use Debconf::Db;
+use Debconf::Question;
+
+# Extract @known... arrays from debconf-set-selections
+# These values are required to distinguish flags and values in the given lines.
+# DC: I couldn't think of a more ugly solution to the problem…
+my @knownflags;
+my @knowntypes;
+my $debconf_set_selections = '/usr/bin/debconf-set-selections';
+if (-e $debconf_set_selections) {
+	my $sed_known = 's/^my \(@known\(flags\|types\) = qw([a-z ]*);\).*$/\1/p';
+	eval `sed -n '$sed_known' '$debconf_set_selections'`;
+}
+
+sub mungeline ($) {
+	my $line = shift;
+	chomp $line;
+	$line =~ s/\r$//;
+	return $line;
+}
+
+sub fatal { printf STDERR @_; exit 1; }
+
+my $state = 'present';
+
+sub state {
+	my $new = shift;
+	if ($state eq 'present'
+	    or ($state eq 'different' and $new eq 'absent')) {
+		$state = $new;
+	}
+}
+
+Debconf::Db->load(readonly => 'true');
+
+while (<>) {
+	# Read and process lines (taken from debconf-set-selections)
+	$_ = mungeline($_);
+	while (/\\$/ && ! eof) {
+		s/\\$//;
+		$_ .= mungeline(<>);
+	}
+	next if /^\s*$/ || /^\s*\#/;
+
+	my ($owner, $label, $type, $content) = /^\s*(\S+)\s+(\S+)\s+(\S+)(?:\s(.*))?/
+		or fatal "invalid line: %s\n", $_;
+	$content = '' unless defined $content;
+
+
+	# Compare is and should state
+	my $q = Debconf::Question->get($label);
+
+	unless (defined $q) {
+		# probably a preseed
+		state 'absent';
+		next;
+	}
+
+	if (grep { $_ eq $q->type } @knownflags) {
+		# This line wants to set a flag, presumably.
+		if ($q->flag($q->type) ne $content) {
+			state 'different';
+		}
+	} else {
+		# Otherwise, it's probably a value…
+		if ($q->value ne $content) {
+			state 'different';
+		}
+
+		unless (grep { $_ eq $owner } (split /, /, $q->owners)) {
+			state 'different';
+		}
+	}
+}
+
+printf "%s\n", $state;
+EOF
diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote
index e99aef40..50d898c6 100755
--- a/cdist/conf/type/__debconf_set_selections/gencode-remote
+++ b/cdist/conf/type/__debconf_set_selections/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,16 +18,32 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# Setup selections
-#
 
-filename="$(cat "$__object/parameter/file")"
-
-if [ "$filename" = "-" ]; then
-    filename="$__object/stdin"
+if test -f "${__object:?}/parameter/line"
+then
+	filename="${__object:?}/parameter/line"
+elif test -s "${__object:?}/parameter/file"
+then
+	filename=$(cat "${__object:?}/parameter/file")
+	if test "${filename}" = '-'
+	then
+		filename="${__object:?}/stdin"
+	fi
+else
+	printf 'Neither --line nor --file set.\n' >&2
+	exit 1
 fi
 
-echo "debconf-set-selections << __file-eof"
-cat "$filename"
-echo "__file-eof"
+# setting no lines makes no sense
+test -s "${filename}" || exit 0
+
+state_is=$(cat "${__object:?}/explorer/state")
+
+if test "${state_is}" != 'present'
+then
+	cat <<-CODE
+	debconf-set-selections <<'EOF'
+	$(cat "${filename}")
+	EOF
+	CODE
+fi
diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst
index 58c25b81..690e3e49 100644
--- a/cdist/conf/type/__debconf_set_selections/man.rst
+++ b/cdist/conf/type/__debconf_set_selections/man.rst
@@ -8,15 +8,33 @@ cdist-type__debconf_set_selections - Setup debconf selections
 
 DESCRIPTION
 -----------
-On Debian and alike systems debconf-set-selections(1) can be used
+On Debian and alike systems :strong:`debconf-set-selections`\ (1) can be used
 to setup configuration parameters.
 
 
 REQUIRED PARAMETERS
 -------------------
+cf. ``--line``.
+
+
+OPTIONAL PARAMETERS
+-------------------
 file
-   Use the given filename as input for debconf-set-selections(1)
-   If filename is "-", read from stdin.
+   Use the given filename as input for :strong:`debconf-set-selections`\ (1)
+   If filename is ``-``, read from stdin.
+
+   **This parameter is deprecated, because it doesn't work with state detection.**
+line
+   A line in :strong:`debconf-set-selections`\ (1) compatible format.
+   This parameter can be used multiple times to set multiple options.
+
+   (This parameter is actually required, but marked optional because the
+   deprecated ``--file`` is still accepted.)
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
 
 
 EXAMPLES
@@ -24,30 +42,29 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # Setup configuration for nslcd
-    __debconf_set_selections nslcd --file /path/to/file
+    # Setup gitolite's gituser
+    __debconf_set_selections nslcd --line 'gitolite gitolite/gituser string git'
 
-    # Setup configuration for nslcd from another type
-    __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd"
-
-    __debconf_set_selections nslcd --file - << eof
-    gitolite gitolite/gituser string git
-    eof
+    # Setup configuration for nslcd from a file.
+    # NB: Multiple lines can be passed to --line, although this can be considered a hack.
+    __debconf_set_selections nslcd --line "$(cat "${__files:?}/preseed/nslcd.debconf")"
 
 
 SEE ALSO
 --------
-:strong:`debconf-set-selections`\ (1), :strong:`cdist-type__update_alternatives`\ (7)
+- :strong:`cdist-type__update_alternatives`\ (7)
+- :strong:`debconf-set-selections`\ (1)
 
 
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2014 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2011-2014 Nico Schottelius, 2021 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__debconf_set_selections/manifest b/cdist/conf/type/__debconf_set_selections/manifest
new file mode 100755
index 00000000..0f4fb2e2
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/manifest
@@ -0,0 +1,21 @@
+#!/bin/sh -e
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+__package_apt debconf
diff --git a/cdist/conf/type/__debconf_set_selections/parameter/required b/cdist/conf/type/__debconf_set_selections/parameter/deprecated
similarity index 100%
rename from cdist/conf/type/__debconf_set_selections/parameter/required
rename to cdist/conf/type/__debconf_set_selections/parameter/deprecated
diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple
new file mode 100644
index 00000000..a999a0c2
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple
@@ -0,0 +1 @@
+line

From 9cf19388abe95a5f8df9f5c94d2e54fa1060b2ef Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Apr 2021 16:47:44 +0200
Subject: [PATCH 434/527] [type/__debconf_set_selections] Send message about
 each debconf setting that is changed

---
 cdist/conf/type/__debconf_set_selections/gencode-remote | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote
index 50d898c6..9ba28f09 100755
--- a/cdist/conf/type/__debconf_set_selections/gencode-remote
+++ b/cdist/conf/type/__debconf_set_selections/gencode-remote
@@ -46,4 +46,9 @@ then
 	$(cat "${filename}")
 	EOF
 	CODE
+
+	awk '
+		{
+			printf "set %s %s %s %s\n", $1, $2, $3, $4
+		}' "${filename}" >>"${__messages_out:?}"
 fi

From 3a25b804664963e092d1f75c1fdded25d7016ef3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 26 Apr 2021 21:27:15 +0200
Subject: [PATCH 435/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index c41743cd..497ae91a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
+	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From a42ebc7a78e09e138954b1376376f7cb643420e0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 27 Apr 2021 19:41:10 +0200
Subject: [PATCH 436/527] [type/__debconf_set_selections] Synchronise objects

Works around locking error:

	debconf: DbDriver "config": /var/cache/debconf/config.dat is locked by another process: Resource temporarily unavailable
---
 .../__debconf_set_selections/explorer/state   | 20 ++++++++++++++++++-
 .../type/__debconf_set_selections/nonparallel |  0
 2 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 cdist/conf/type/__debconf_set_selections/nonparallel

diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state
index 68d28941..f8a3f6c8 100644
--- a/cdist/conf/type/__debconf_set_selections/explorer/state
+++ b/cdist/conf/type/__debconf_set_selections/explorer/state
@@ -41,10 +41,15 @@ test -s "${linesfile}" || {
 	exit 0
 }
 
+# assert __type_explorer is set (because it is used by the Perl script)
+: "${__type_explorer:?}"
+
 /usr/bin/perl -- - "${linesfile}" <<'EOF'
 use strict;
 use warnings "all";
 
+use Fcntl qw(:DEFAULT :flock);
+
 use Debconf::Db;
 use Debconf::Question;
 
@@ -78,7 +83,20 @@ sub state {
 	}
 }
 
-Debconf::Db->load(readonly => 'true');
+
+# Load Debconf DB but manually lock on the state explorer script,
+# because Debconf aborts immediately if executed concurrently.
+# This is not really an ideal solution because the Debconf DB could be locked by
+# another process (e.g. apt-get), but no way to achieve this could be found.
+# If you know how to, please provide a patch.
+my $lockfile = "%ENV{'__type_explorer'}/state";
+if (open my $lock_fh, '+<', $lockfile) {
+   flock $lock_fh, LOCK_EX or die "Cannot lock $lockfile";
+}
+{
+	Debconf::Db->load(readonly => 'true');
+}
+
 
 while (<>) {
 	# Read and process lines (taken from debconf-set-selections)
diff --git a/cdist/conf/type/__debconf_set_selections/nonparallel b/cdist/conf/type/__debconf_set_selections/nonparallel
new file mode 100644
index 00000000..e69de29b

From c00c8c20127af9881b4e295b7732524e7e1c5031 Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Mon, 10 May 2021 12:08:22 +0200
Subject: [PATCH 437/527] [__apt_key*] Deprecate __apt_key_uri and improve
 __apt_key

Previously this type was falling back to using the deprecated apt-key(8) by
checking for existence of files/directories on the controller host in
gencode-remote.

Adding `--use-deprecated-apt-key` as an explicit boolean serves two purposes:
1. It prevents fallbacks that might end up doing the wrong thing
   (as was the case)
2. It allows for a simple way to remove keys from the keyring that were
   previously added with apt-key(8) to /etc/apt/trusted.gpg

This parameter is added marked as deprecated as is only intended use is to
migrate to directory-based keyrings as recommended by Debian for a few releases.
It will be removed when Debian 11 stops being supported.

During the review process of this merge request, it was noted that the state of
PGP Key Servers is somewhat suboptimal, that the examples encouraged bad
practise (it is trivial to produce collisions for short key IDs), and that
this use does not require the Web of Trust, but instead only the public key
that is signing the repository.

That is why this also adds `--source` as an argument allowing for in-type or
in-manifest provision of such public keys by the type/manifest maintainer and
the use of Key Servers is still supported, but discouraged.
---
 cdist/conf/type/__apt_key/explorer/state      |  27 +++--
 cdist/conf/type/__apt_key/gencode-remote      |  53 ++++-----
 cdist/conf/type/__apt_key/man.rst             |  84 ++++++++++----
 cdist/conf/type/__apt_key/manifest            | 104 +++++++++++++++++-
 cdist/conf/type/__apt_key/parameter/boolean   |   1 +
 .../deprecated/use-deprecated-apt-key         |   3 +
 cdist/conf/type/__apt_key/parameter/optional  |   5 +-
 cdist/conf/type/__apt_key_uri/deprecated      |   1 +
 8 files changed, 209 insertions(+), 69 deletions(-)
 create mode 100644 cdist/conf/type/__apt_key/parameter/boolean
 create mode 100644 cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key
 create mode 100644 cdist/conf/type/__apt_key_uri/deprecated

diff --git a/cdist/conf/type/__apt_key/explorer/state b/cdist/conf/type/__apt_key/explorer/state
index 38f1bd3c..8ab268c1 100755
--- a/cdist/conf/type/__apt_key/explorer/state
+++ b/cdist/conf/type/__apt_key/explorer/state
@@ -27,18 +27,25 @@ else
    keyid="$__object_id"
 fi
 
+# From apt-key(8):
+#   Use of apt-key is deprecated, except for the use of apt-key del in
+#   maintainer scripts to remove existing keys from the main keyring.
+#   If such usage of apt-key is desired the additional installation of
+#   the GNU Privacy Guard suite (packaged in gnupg) is required.
+if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then
+   if apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK"
+      then echo present
+      else echo absent
+   fi
+   exit
+fi
+
 keydir="$(cat "$__object/parameter/keydir")"
 keyfile="$keydir/$__object_id.gpg"
 
-if [ -d "$keydir" ]
+if [ -f "$keyfile" ]
 then
-   if [ -f "$keyfile" ]
-   then echo present
-   else echo absent
-   fi
-else
-   # fallback to deprecated apt-key
-   apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \
-      && echo present \
-      || echo absent
+   echo present
+   exit
 fi
+echo absent
diff --git a/cdist/conf/type/__apt_key/gencode-remote b/cdist/conf/type/__apt_key/gencode-remote
index 0c96ff67..17dc9bfc 100755
--- a/cdist/conf/type/__apt_key/gencode-remote
+++ b/cdist/conf/type/__apt_key/gencode-remote
@@ -25,11 +25,7 @@ else
 fi
 state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
-
-if [ "$state_should" = "$state_is" ]; then
-   # nothing to do
-   exit 0
-fi
+method="$(cat "$__object/key_method")"
 
 keydir="$(cat "$__object/parameter/keydir")"
 keyfile="$keydir/$__object_id.gpg"
@@ -37,30 +33,18 @@ keyfile="$keydir/$__object_id.gpg"
 case "$state_should" in
    present)
       keyserver="$(cat "$__object/parameter/keyserver")"
-
-      if [ -f "$__object/parameter/uri" ]; then
-         uri="$(cat "$__object/parameter/uri")"
-
-         if [ -d "$keydir" ]; then
-            cat << EOF
-
-curl -s -L \\
-    -o "$keyfile" \\
-    "$uri"
-
-key="\$( cat "$keyfile" )"
-
-if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK'
-then
-    echo "\$key" | gpg --dearmor > "$keyfile"
-fi
-
-EOF
-         else
-            # fallback to deprecated apt-key
-            echo "curl -s -L '$uri' | apt-key add -"
+      # Using __download or __file as key source
+      # Propagate messages if needed
+      if [ "${method}" = "uri" ] || [ "${method}" = "source" ]; then
+         if grep -Eq "^__(file|download)$keyfile" "$__messages_in"; then
+            echo "added '$keyid'" >> "$__messages_out"
          fi
-      elif [ -d "$keydir" ]; then
+         exit 0
+      elif [ "${state_is}" = "present" ]; then
+         exit 0
+      fi
+      # Using key servers to fetch the key
+      if [ ! -f "$__object/parameter/use-deprecated-apt-key" ]; then
          # we need to kill gpg after 30 seconds, because gpg
          # can get stuck if keyserver is not responding.
          # exporting env var and not exit 1,
@@ -100,13 +84,16 @@ EOF
       echo "added '$keyid'" >> "$__messages_out"
    ;;
    absent)
-      if [ -f "$keyfile" ]; then
-         echo "rm '$keyfile'"
-      else
+      # Removal for keys added from a keyserver without this flag
+      # is done in the manifest
+      if [ "$state_is" != "absent" ] && \
+            [ -f "$__object/parameter/use-deprecated-apt-key" ]; then
          # fallback to deprecated apt-key
          echo "apt-key del \"$keyid\""
+         echo "removed '$keyid'" >> "$__messages_out"
+      # Propagate messages if needed
+      elif grep -Eq "^__file$keyfile" "$__messages_in"; then
+         echo "removed '$keyid'" >> "$__messages_out"
       fi
-
-      echo "removed '$keyid'" >> "$__messages_out"
    ;;
 esac
diff --git a/cdist/conf/type/__apt_key/man.rst b/cdist/conf/type/__apt_key/man.rst
index 234bc715..e35eaa0f 100644
--- a/cdist/conf/type/__apt_key/man.rst
+++ b/cdist/conf/type/__apt_key/man.rst
@@ -10,6 +10,14 @@ DESCRIPTION
 -----------
 Manages the list of keys used by apt to authenticate packages.
 
+This is done by placing the requested key in a file named
+``$__object_id.gpg`` in the ``keydir`` directory.
+
+This is supported by modern releases of Debian-based distributions.
+
+In order of preference, exactly one of: ``source``, ``uri`` or ``keyid``
+must be specified.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -18,21 +26,49 @@ None.
 
 OPTIONAL PARAMETERS
 -------------------
+keydir
+   keyring directory, defaults to ``/etc/apt/trusted.pgp.d``, which is
+   enabled system-wide by default.
+
+source
+   path to a file containing the GPG key of the repository.
+   Using this is recommended as it ensures that the manifest/type manintainer
+   has validated the key.
+   If ``-``, the GPG key is read from the type's stdin.
+
 state
    'present' or 'absent'. Defaults to 'present'
 
+uri
+   the URI from which to download the key.
+   It is highly recommended that you only use protocols with TLS like HTTPS.
+   This uses ``__download`` but does not use checksums, if you want to ensure
+   that the key doesn't change, you are better off downloading it and using
+   ``--source``.
+
+
+DEPRECATED OPTIONAL PARAMETERS
+------------------------------
 keyid
-   the id of the key to add. Defaults to __object_id
+   the id of the key to download from the ``keyserver``.
+   This is to be used in absence of ``--source`` and ``--uri`` or together
+   with ``--use-deprecated-apt-key`` for key removal.
+   Defaults to ``$__object_id``.
 
 keyserver
-   the keyserver from which to fetch the key. If omitted the default set
-   in ./parameter/default/keyserver is used.
+   the keyserver from which to fetch the key.
+   Defaults to ``pool.sks-keyservers.net``.
 
-keydir
-   key save location, defaults to ``/etc/apt/trusted.pgp.d``
 
-uri
-   the URI from which to download the key
+DEPRECATED BOOLEAN PARAMETERS
+-----------------------------
+use-deprecated-apt-key
+   ``apt-key(8)`` will last be available in Debian 11 and Ubuntu 22.04.
+   You can use this parameter to force usage of ``apt-key(8)``.
+   Please only use this parameter to *remove* keys from the keyring,
+   in order to prepare for removal of ``apt-key``.
+   Adding keys should be done without this parameter.
+   This parameter will be removed when Debian 11 stops being supported.
 
 
 EXAMPLES
@@ -40,33 +76,39 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # Add Ubuntu Archive Automatic Signing Key
-    __apt_key 437D05B5
-    # Same thing
-    __apt_key 437D05B5 --state present
-    # Get rid of it
-    __apt_key 437D05B5 --state absent
+    # add a key that has been verified by a type maintainer
+    __apt_key jitsi_meet_2021 \
+       --source cdist-contrib/type/__jitsi_meet/files/apt_2021.gpg
 
-    # same thing with human readable name and explicit keyid
-    __apt_key UbuntuArchiveKey --keyid 437D05B5
+    # remove an old, deprecated or expired key
+    __apt_key jitsi_meet_2016 --state absent
 
-    # same thing with other keyserver
-    __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com
+    # Get rid of a key that might have been added to
+    # /etc/apt/trusted.gpg with apt-key
+    __apt_key 0x40976EAF437D05B5 --use-deprecated-apt-key --state absent
 
-    # download key from the internet
-    __apt_key rabbitmq \
-       --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc
+    # add a key that we define in-line
+    __apt_key jitsi_meet_2021 --source '-' <<EOF
+    -----BEGIN PGP PUBLIC KEY BLOCK-----
+    [...]
+    -----END PGP PUBLIC KEY BLOCK-----
+    EOF
+
+    # download or update key from the internet
+    __apt_key rabbitmq_2007 \
+       --uri https://www.rabbitmq.com/rabbitmq-signing-key-public.asc
 
 
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
 Ander Punnar <ander-at-kvlt-dot-ee>
+Evilham <contact~~@~~evilham.com>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can
+Copyright \(C) 2011-2021 Steven Armstrong, Ander Punnar and Evilham. You can
 redistribute it and/or modify it under the terms of the GNU General Public
 License as published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__apt_key/manifest b/cdist/conf/type/__apt_key/manifest
index 010357cd..889a764a 100755
--- a/cdist/conf/type/__apt_key/manifest
+++ b/cdist/conf/type/__apt_key/manifest
@@ -2,7 +2,105 @@
 
 __package gnupg
 
-if [ -f "$__object/parameter/uri" ]
-then __package curl
-else __package dirmngr
+state_should="$(cat "${__object}/parameter/state")"
+
+incompatible_args()
+{
+	cat >> /dev/stderr <<-EOF
+	This type does not support --${1} and --${method} simultaneously.
+	EOF
+	exit 1
+}
+
+if [ -f "${__object}/parameter/source" ]; then
+	method="source"
+	src="$(cat "${__object}/parameter/source")"
+	if [ "${src}" = "-" ]; then
+		src="${__object}/stdin"
+	fi
+fi
+if [ -f "${__object}/parameter/uri" ]; then
+	if [ -n "${method}" ]; then
+		incompatible_args uri
+	fi
+	method="uri"
+	src="$(cat "${__object}/parameter/uri")"
+fi
+if [ -f "${__object}/parameter/keyid" ]; then
+	if [ -n "${method}" ]; then
+		incompatible_args keyid
+	fi
+	method="keyid"
+fi
+# Keep old default
+if [ -z "${method}" ]; then
+	method="keyid"
+fi
+# Save this for later in gencode-remote
+echo "${method}" > "${__object}/key_method"
+
+# Required remotely (most likely already installed)
+__package dirmngr
+# We need this in case a key has to be dearmor'd
+__package gnupg
+export require="__package/gnupg"
+
+if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then
+	# This is required if apt-key(8) is to be used
+	if [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then
+		incompatible_args use-deprecated-apt-key
+	fi
+else
+	if [ "${state_should}" = "absent" ] && \
+			[ -f "${__object}/parameter/keyid" ]; then
+		cat >> /dev/stderr <<EOF
+You can't reliably remove by keyid without --use-deprecated-apt-key.
+This would very likely do something you do not intend.
+EOF
+		exit 1
+	fi
+fi
+
+keydir="$(cat "${__object}/parameter/keydir")"
+keyfile="${keydir}/${__object_id}.gpg"
+keyfilecdist="${keyfile}.cdist"
+if [ "${state_should}" != "absent" ]; then
+	# Ensure keydir exists
+	__directory "${keydir}" --state exists --mode 0755
+fi
+
+if [ "${state_should}" = "absent" ]; then
+	__file "${keyfile}" --state "absent"
+	__file "${keyfilecdist}" --state "absent"
+elif [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then
+	dearmor="$(cat <<-EOF
+	if [ '${state_should}' = 'present' ]; then
+		# Dearmor if necessary
+		if grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK' '${keyfilecdist}'; then
+			gpg --dearmor < '${keyfilecdist}' > '${keyfile}'
+		else
+			cp '${keyfilecdist}' '${keyfile}'
+		fi
+		# Ensure permissions
+		chown root '${keyfile}'
+		chmod 0444 '${keyfile}'
+	fi
+	EOF
+	)"
+
+	if [ "${method}" = "uri" ]; then
+		__download "${keyfilecdist}" \
+			--url "${src}" \
+			--onchange "${dearmor}"
+		require="__download${keyfilecdist}" \
+			__file "${keyfile}" \
+				--owner root \
+				--mode 0444 \
+				--state pre-exists
+	else
+		__file "${keyfilecdist}" --state "${state_should}" \
+			--mode 0444 \
+			--source "${src}" \
+			--onchange "${dearmor}"
+	fi
 fi
diff --git a/cdist/conf/type/__apt_key/parameter/boolean b/cdist/conf/type/__apt_key/parameter/boolean
new file mode 100644
index 00000000..7263e1b2
--- /dev/null
+++ b/cdist/conf/type/__apt_key/parameter/boolean
@@ -0,0 +1 @@
+use-deprecated-apt-key
diff --git a/cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key b/cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key
new file mode 100644
index 00000000..d18ebe57
--- /dev/null
+++ b/cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key
@@ -0,0 +1,3 @@
+apt-key(8) will last be available in Debian 11 and Ubuntu 22.04.
+Use this flag *only* to migrate to placing a keyring directly in the
+/etc/apt/trusted.gpg.d/ directory with a descriptive name.
diff --git a/cdist/conf/type/__apt_key/parameter/optional b/cdist/conf/type/__apt_key/parameter/optional
index de647375..975d1fd8 100644
--- a/cdist/conf/type/__apt_key/parameter/optional
+++ b/cdist/conf/type/__apt_key/parameter/optional
@@ -1,5 +1,6 @@
-state
+keydir
 keyid
 keyserver
-keydir
+source
+state
 uri
diff --git a/cdist/conf/type/__apt_key_uri/deprecated b/cdist/conf/type/__apt_key_uri/deprecated
new file mode 100644
index 00000000..05550587
--- /dev/null
+++ b/cdist/conf/type/__apt_key_uri/deprecated
@@ -0,0 +1 @@
+Please migrate to using __apt_key key_id --uri URI.

From a696f3cf0026ec96e4cf7235ee8634391ccfecdb Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Mon, 10 May 2021 12:10:00 +0200
Subject: [PATCH 438/527] [__letsencrypt_cert] Revamp explorers, add locking.

This would fix #839

Certbot uses locking [1] even for read-only operations and does not properly
use exit codes, which means that sometimes it would print:
"Another instance of Certbot is already running" and exit with success.

However, the previous explorers would take that as the certificate being absent
and would trigger code generation.

The issue was made worse by having many explorers running certbot, so for N
certificates, we'd run certbot N*4 times, potentially "in parallel".

[1]: https://certbot.eff.org/docs/using.html#id5

This patch joins all explorers in one to avoid starting multiple remote python
processes and uses a cdist-specific lock in /tmp/certbot.cdist.lock with a
60 seconds timeout.

It has been tested with certbot 0.31.0 and 0.17 that the:

    from certbot.main import main

trick works. It is somewhat well documented so it can be somewhat relied upon.
---
 .../__letsencrypt_cert/explorer/certbot-path  |  3 -
 .../explorer/certificate-data                 | 78 +++++++++++++++++++
 .../explorer/certificate-domains              |  8 --
 .../explorer/certificate-exists               | 13 ----
 .../explorer/certificate-is-test              | 14 ----
 .../type/__letsencrypt_cert/gencode-remote    | 11 ++-
 cdist/conf/type/__letsencrypt_cert/manifest   |  2 +-
 7 files changed, 87 insertions(+), 42 deletions(-)
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certbot-path
 create mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-data
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test

diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path
deleted file mode 100755
index 3c6076df..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh -e
-
-command -v certbot 2>/dev/null || true
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data
new file mode 100755
index 00000000..ff62e742
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data
@@ -0,0 +1,78 @@
+#!/bin/sh -e
+certbot_path="$(command -v certbot 2>/dev/null || true)"
+# Defaults
+certificate_exists="no"
+certificate_is_test="no"
+
+if [ -n "${certbot_path}" ]; then
+	# Find python executable that has access to certbot's module
+	python_path=$(sed -n '1s/^#! *//p' "${certbot_path}")
+
+	# Use a lock for cdist due to certbot not exiting with failure
+	# or having any flags for concurrent use.
+	_certbot() {
+		${python_path} - 2>/dev/null <<EOF
+from certbot.main import main
+import fcntl
+lock_file = "/tmp/certbot.cdist.lock"
+timeout=60
+with open(lock_file, 'w') as fd:
+    for i in range(timeout):
+        try:
+            # Get exclusive lock
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            break
+        except:
+            # Wait if that fails
+            import time
+            time.sleep(1)
+    else:
+        # Timed out, exit with failure
+        import sys
+        sys.exit(1)
+    # Do list certificates
+    main(["certificates", "--cert-name", "${__object_id:?}"])
+EOF
+	}
+
+
+	_certificate_exists() {
+		if grep -q "  Certificate Name: ${__object_id:?}$"; then
+			echo yes
+		else
+			echo no
+		fi
+	}
+
+	_certificate_is_test() {
+		if grep -q 'INVALID: TEST_CERT'; then
+			echo yes
+		else
+			echo no
+		fi
+	}
+
+	_certificate_domains() {
+		grep '    Domains: ' | cut -d ' ' -f 6- | tr ' ' '\n'
+	}
+
+	# Get data about all available certificates
+	certificates="$(_certbot)"
+
+	# Check whether or not the certificate exists
+	certificate_exists="$(echo "${certificates}" | _certificate_exists)"
+
+	# Check whether or not the certificate is for testing
+	certificate_is_test="$(echo "${certificates}" | _certificate_is_test)"
+
+	# Get domains for certificate
+	certificate_domains="$(echo "${certificates}" | _certificate_domains)"
+fi
+
+# Return received data
+cat <<EOF
+certbot_path:${certbot_path}
+certificate_exists:${certificate_exists}
+certificate_is_test:${certificate_is_test}
+${certificate_domains}
+EOF
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains
deleted file mode 100755
index db605b63..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	certbot certificates --cert-name "${__object_id:?}" | grep '    Domains: ' | \
-		cut -d ' ' -f 6- | tr ' ' '\n'
-fi
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists
deleted file mode 100755
index 4e6f44db..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	if certbot certificates | grep -q "  Certificate Name: ${__object_id:?}$"; then
-		echo yes
-	else
-		echo no
-	fi
-else
-	echo no
-fi
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test
deleted file mode 100755
index 9b445059..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	if certbot certificates --cert-name "${__object_id:?}" | \
-		grep -q 'INVALID: TEST_CERT'; then
-		echo yes
-	else
-		echo no
-	fi
-else
-	echo no
-fi
diff --git a/cdist/conf/type/__letsencrypt_cert/gencode-remote b/cdist/conf/type/__letsencrypt_cert/gencode-remote
index 375570a4..60a77ec3 100755
--- a/cdist/conf/type/__letsencrypt_cert/gencode-remote
+++ b/cdist/conf/type/__letsencrypt_cert/gencode-remote
@@ -1,6 +1,10 @@
 #!/bin/sh -e
 
-certificate_exists=$(cat "${__object:?}/explorer/certificate-exists")
+_explorer_var() {
+	grep "^$1:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-
+}
+
+certificate_exists="$(_explorer_var certificate_exists)"
 name="${__object_id:?}"
 state=$(cat "${__object}/parameter/state")
 
@@ -29,8 +33,9 @@ case "${state}" in
 		fi
 
 		if [ "${certificate_exists}" = "yes" ]; then
-			existing_domains="${__object}/explorer/certificate-domains"
-			certificate_is_test=$(cat "${__object}/explorer/certificate-is-test")
+			existing_domains=$(mktemp "${TMPDIR:-/tmp}/existing_domains.cdist.XXXXXXXXXX")
+			tail -n +4 "${__object:?}/explorer/certificate-data" | grep -v '^$' > "${existing_domains}"
+			certificate_is_test="$(_explorer_var certificate_is_test)"
 
 			sort -uo "${requested_domains}" "${requested_domains}"
 			sort -uo "${existing_domains}" "${existing_domains}"
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 1df3574a..6394f629 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")"
+certbot_fullpath="$(grep "^certbot_path:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-)"
 state=$(cat "${__object}/parameter/state")
 os="$(cat "${__global:?}/explorer/os")"
 

From f14623e45f56aec4bdce53bc2b367e11ff157865 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 10 May 2021 12:17:08 +0200
Subject: [PATCH 439/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 497ae91a..5c3b547a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,8 @@ Changelog
 next:
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
+	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)
+	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 6210cccb28ace8ec2807ffe6ca9f9e4d9007990b Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 10 May 2021 12:34:04 +0200
Subject: [PATCH 440/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 5c3b547a..a7310788 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)
 	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
+	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking.
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 503a06ed28f743aad47797a0989be735c67b07a6 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 28 Apr 2021 13:32:10 +0300
Subject: [PATCH 441/527] [__git] fix group explorer

group name from numberic id wasn't resolved correctly.

try to use getent and fallback to reading /etc/group directly.
---
 cdist/conf/type/__git/explorer/group | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group
index 1365c60d..ab4396b1 100644
--- a/cdist/conf/type/__git/explorer/group
+++ b/cdist/conf/type/__git/explorer/group
@@ -14,6 +14,11 @@ then
 	then
 		printf '%u\n' "${group_gid}"
 	else
-		printf '%s\n' "$(id -u -n "${group_gid}")"
+		if command -v getent > /dev/null
+		then
+			getent group "${group_gid}" | cut -d : -f 1
+		else
+			awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group
+		fi
 	fi
 fi

From 75c71f69c1fed4371c5891ddcca0eaf28ad928e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 10:17:48 +0200
Subject: [PATCH 442/527] [scanner] pycodestyle compliance

---
 cdist/scan/commandline.py | 20 +++++++++++----
 cdist/scan/scan.py        | 54 ++++++++++++++++++++++++---------------
 2 files changed, 48 insertions(+), 26 deletions(-)

diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 3eb7eec4..b42bc7b2 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -25,13 +25,15 @@ from datetime import datetime
 
 log = logging.getLogger("scan")
 
+
 def run(scan, args):
     # We run each component in a separate process since they
     # must not block on each other.
     processes = []
 
     if 'trigger' in args.mode:
-        t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay)
+        t = scan.Trigger(interfaces=args.interfaces,
+                         sleeptime=args.trigger_delay)
         t.start()
         processes.append(t)
         log.debug("Trigger started")
@@ -48,6 +50,7 @@ def run(scan, args):
     for process in processes:
         process.join()
 
+
 def list(scan, args):
     s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper)
     hosts = s.list()
@@ -66,10 +69,16 @@ def list(scan, args):
     print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
     for host in hosts:
         last_seen = host.last_seen()
-        last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-'
+        if last_seen:
+            last_seen = last_seen.strftime(scan.datetime_format)
+        else:
+            last_seen = '-'
 
         last_configured = host.last_configured()
-        last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-'
+        if last_configured:
+            last_configured = last_configured.strftime(scan.datetime_format)
+        else:
+            '-'
 
         print("{} | {} | {} | {}".format(
             host.name(default='-').ljust(name_max_size),
@@ -77,6 +86,7 @@ def list(scan, args):
             last_seen.ljust(date_max_size),
             last_configured.ljust(date_max_size)))
 
+
 # CLI processing is defined outside of the main scan class to handle
 # non-available optional scapy dependency (instead of crashing mid-flight).
 def commandline(args):
@@ -94,9 +104,9 @@ def commandline(args):
         # By default scan and trigger, but do not call any action.
         args.mode = ['scan', 'trigger', ]
 
-    if 'config' in args.mode and args.name_mapper == None:
+    if 'config' in args.mode and args.name_mapper is None:
         print('--name-mapper must be specified for scanner config mode.',
-                file=sys.stderr)
+              file=sys.stderr)
         sys.exit(1)
 
     # Print known hosts and exit is --list is specified - do not start
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 2912dab3..4a20f511 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -33,6 +33,7 @@ logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("scan")
 datetime_format = '%Y-%m-%d %H:%M:%S'
 
+
 class Host(object):
     def __init__(self, addr, outdir, name_mapper=None):
         self.addr = addr
@@ -43,7 +44,7 @@ class Host(object):
 
     def __get(self, key, default=None):
         fname = os.path.join(self.workdir, key)
-        value=default
+        value = default
         if os.path.isfile(fname):
             with open(fname, "r") as fd:
                 value = fd.readline()
@@ -55,15 +56,15 @@ class Host(object):
             fd.write(f"{value}")
 
     def name(self, default=None):
-        if self.name_mapper == None:
+        if self.name_mapper is None:
             return default
 
         fpath = os.path.join(os.getcwd(), self.name_mapper)
         if os.path.isfile(fpath) and os.access(fpath, os.X_OK):
-             out = subprocess.run([fpath, self.addr], capture_output=True)
-             if out.returncode != 0:
-                 return default
-             else:
+            out = subprocess.run([fpath, self.addr], capture_output=True)
+            if out.returncode != 0:
+                return default
+            else:
                 value = out.stdout.decode()
                 return (default if len(value) == 0 else value)
         else:
@@ -94,19 +95,20 @@ class Host(object):
     # CLI args. Might as well call everything from scratch!
     def configure(self):
         target = self.name() or self.address()
-        cmd = ['cdist', 'config', '-v', target ]
+        cmd = ['cdist', 'config', '-v', target]
 
         fname = os.path.join(self.workdir, 'last_configuration_log')
         with open(fname, "w") as fd:
             log.debug("Executing: %s", cmd)
             completed_process = subprocess.run(cmd, stdout=fd, stderr=fd)
             if completed_process.returncode != 0:
-                log.error("%s return with non-zero code %i - see %s for details.",
-                        cmd, completed_process.returncode, fname)
+                log.error("%s return with non-zero code %i - see %s for \
+                        details.", cmd, completed_process.returncode, fname)
 
         now = datetime.datetime.now().strftime(datetime_format)
         self.__set('last_configured', now)
 
+
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
@@ -140,10 +142,12 @@ class Trigger(object):
     def trigger(self, interface):
         try:
             log.debug("Sending ICMPv6EchoRequest on %s", interface)
-            packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest()
+            packet = IPv6(
+                    dst="ff02::1%{}".format(interface)
+                    ) / ICMPv6EchoRequest()
             send(packet, verbose=self.verbose)
         except Exception as e:
-            log.error( "Could not send ICMPv6EchoRequest: %s", e)
+            log.error("Could not send ICMPv6EchoRequest: %s", e)
 
 
 class Scanner(object):
@@ -151,9 +155,10 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None):
+    def __init__(self, interfaces, autoconfigure=False, outdir=None,
+                 name_mapper=None):
         self.interfaces = interfaces
-        self.autoconfigure=autoconfigure
+        self.autoconfigure = autoconfigure
         self.name_mapper = name_mapper
         self.config_delay = datetime.timedelta(seconds=3600)
 
@@ -169,14 +174,17 @@ class Scanner(object):
         if ICMPv6EchoReply in pkg:
             host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper)
             if host.name():
-                log.verbose("Host %s (%s) is alive", host.name(), host.address())
+                log.verbose("Host %s (%s) is alive", host.name(),
+                            host.address())
             else:
                 log.verbose("Host %s is alive", host.address())
+
             host.seen()
 
             # Configure if needed.
             if self.autoconfigure and \
-                    host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now():
+                    host.last_configured(default=datetime.datetime.min) + \
+                    self.config_delay < datetime.datetime.now():
                 self.config(host)
 
     def list(self):
@@ -187,15 +195,19 @@ class Scanner(object):
         return hosts
 
     def config(self, host):
-        if host.name() == None:
-            log.debug("config - could not resolve name for %s, aborting.", host.address())
+        if host.name() is None:
+            log.debug("config - could not resolve name for %s, aborting.",
+                      host.address())
             return
 
         previous_config_process = self.running_configs.get(host.name())
-        if previous_config_process != None and previous_config_process.is_alive():
-            log.debug("config - is already running for %s, aborting.", host.name())
+        if previous_config_process is not None and \
+                previous_config_process.is_alive():
+            log.debug("config - is already running for %s, aborting.",
+                      host.name())
 
-        log.info("config - running against host %s (%s).", host.name(), host.address())
+        log.info("config - running against host %s (%s).", host.name(),
+                 host.address())
         p = Process(target=host.configure())
         p.start()
         self.running_configs[host.name()] = p
@@ -214,4 +226,4 @@ class Scanner(object):
                   filter="icmp6",
                   prn=self.handle_pkg)
         except Exception as e:
-            log.error( "Could not start listener: %s", e)
+            log.error("Could not start listener: %s", e)

From ab10b453f275d4289f3d37988b3caec204227194 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 11:15:41 +0200
Subject: [PATCH 443/527] [scanner] populate cdist(1)

---
 docs/src/man1/cdist.rst | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 0ecb4a61..599ec3b7 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -88,6 +88,9 @@ SYNOPSIS
     cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t]
                [pattern]
 
+    cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list]
+               [-d CONFIG_DELAY] [-t TRIGGER_DELAY]
+
 
 DESCRIPTION
 -----------
@@ -641,6 +644,31 @@ Display information for cdist (global explorers, types).
 **-t, --types**
     Display info for types.
 
+SCAN
+----
+
+Runs cdist as a daemon that discover/watch on hosts and reconfigure them
+periodically.
+
+**-I INTERFACE, --interfaces INTERFACE**
+    Interface to listen on. Can be specified multiple times.
+
+**-m MODE, --mode MODE**
+    Scanner components to enable. Can be specified multiple time to enable more
+    than one component. Supported modes are: scan, trigger and config. Defaults
+    to tiggger and scan.
+
+**--name-mapper PATH_TO_SCRIPT**
+    Path to script used to resolve a remote host name from an IPv6 address.
+
+**--list**
+    List known hosts and exit.
+
+**-d CONFIG_DELAY, --config-delay CONFIG_DELAY**
+    How long (seconds) to wait before reconfiguring after last try (config mode only).
+
+**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY**
+    How long (seconds) to wait between ICMPv6 echo requests (trigger mode only).
 
 CONFIGURATION
 -------------

From b8733c65f52776facce7a8de0265538a856ead64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 11:26:35 +0200
Subject: [PATCH 444/527] [scanner] fix minor CLI handling and --list bugs /
 typo

---
 cdist/argparse.py         |  4 ++--
 cdist/scan/commandline.py | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index bedb23ac..f17315e7 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -495,7 +495,7 @@ def get_parsers():
         action='store_true',
         help='Try to configure detected hosts')
     parser['scan'].add_argument(
-        '-I', '--interfaces',
+        '-I', '--interface',
         action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
     parser['scan'].add_argument(
@@ -503,7 +503,7 @@ def get_parsers():
         action='store',  default=None,
         help='Map addresses to names, required for config mode')
     parser['scan'].add_argument(
-        '-d', '--delay',
+        '-d', '--config-delay',
         action='store',  default=3600, type=int,
         help='How long (seconds) to wait before reconfiguring after last try')
     parser['scan'].add_argument(
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index b42bc7b2..ddbe4933 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -32,7 +32,7 @@ def run(scan, args):
     processes = []
 
     if 'trigger' in args.mode:
-        t = scan.Trigger(interfaces=args.interfaces,
+        t = scan.Trigger(interfaces=args.interface,
                          sleeptime=args.trigger_delay)
         t.start()
         processes.append(t)
@@ -41,7 +41,7 @@ def run(scan, args):
     if 'scan' in args.mode:
         s = scan.Scanner(
                 autoconfigure='config' in args.mode,
-                interfaces=args.interfaces,
+                interfaces=args.interface,
                 name_mapper=args.name_mapper)
         s.start()
         processes.append(s)
@@ -52,7 +52,7 @@ def run(scan, args):
 
 
 def list(scan, args):
-    s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper)
+    s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper)
     hosts = s.list()
 
     # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
@@ -75,10 +75,10 @@ def list(scan, args):
             last_seen = '-'
 
         last_configured = host.last_configured()
-        if last_configured:
+        if last_configured is not None:
             last_configured = last_configured.strftime(scan.datetime_format)
         else:
-            '-'
+            last_configured = '-'
 
         print("{} | {} | {} | {}".format(
             host.name(default='-').ljust(name_max_size),

From e0c52d0e1dfbaa3814a2ff26482177c2909a15ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 11:27:11 +0200
Subject: [PATCH 445/527] [scanner] remove mention of non-implemented trigger
 soruce script

---
 docs/src/cdist-scan.rst | 21 ++-------------------
 1 file changed, 2 insertions(+), 19 deletions(-)

diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst
index 02193456..064e65ff 100644
--- a/docs/src/cdist-scan.rst
+++ b/docs/src/cdist-scan.rst
@@ -8,7 +8,8 @@ periodically. It is especially useful in netboot-based environment where hosts
 boot unconfigured, and to ensure your infrastructure stays in sync with your
 configuration.
 
-This feature is still consider to be in **beta** stage.
+This feature is still consider to be in **beta** stage, and only operate on
+IPv6 (including link-local).
 
 Usage (Examples)
 ----------------
@@ -79,21 +80,3 @@ Resolving name from `PTR` DNS record:
   done
 
   dig +short -x "$1" | sed -e 's/.$//'
-
-
-Trigger Source Script
----------------------
-
-This script returns a list of addresses (separated by a newline) to be used by
-`trigger` mode. It is not used to map names. The script must be executable.
-
-Simplest script:
-
-.. code-block:: sh
-  #!/bin/sh
-
-  cat << EOF
-  server1.domain.tld
-  server2.domain.tld
-  server3.domain.tld
-  EOF

From defa3c22eaef88652a2bec6135d038a84710c41e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 29 May 2021 11:21:34 +0200
Subject: [PATCH 446/527] ++changelog

---
 docs/changelog | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index a7310788..403e7653 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,7 +6,8 @@ next:
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)
 	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
-	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking.
+	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham)
+	* Type __git: Fix group explorer (Ander Punnar)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From d596986af894749877fb3dcadf90628c9cfa9d13 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 31 May 2021 09:06:52 +0200
Subject: [PATCH 447/527] [type/__pyvenv] Fix group explorer

---
 cdist/conf/type/__pyvenv/explorer/group | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group
index f31a1cb7..922ce3df 100755
--- a/cdist/conf/type/__pyvenv/explorer/group
+++ b/cdist/conf/type/__pyvenv/explorer/group
@@ -14,6 +14,11 @@ then
 	then
 		printf '%u\n' "${group_gid}"
 	else
-		printf '%s\n' "$(id -u -n "${group_gid}")"
+		if command -v getent >/dev/null 2>&1
+		then
+			getent group "${group_gid}" | cut -d : -f 1
+		else
+			awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group
+		fi
 	fi
 fi

From 6ede76b08b96b7504c65ca2dc6737e31be7e7f24 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 8 Jun 2021 16:20:55 +0200
Subject: [PATCH 448/527] [type/__debconf_set_selections] man.rst: Fix line
 break in AUTHORS

---
 cdist/conf/type/__debconf_set_selections/man.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst
index 690e3e49..fd0040ae 100644
--- a/cdist/conf/type/__debconf_set_selections/man.rst
+++ b/cdist/conf/type/__debconf_set_selections/man.rst
@@ -58,8 +58,8 @@ SEE ALSO
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
-Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING

From c308a2896971ac4808be0152630ef41b7f96a2de Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 10 Jun 2021 06:39:55 +0200
Subject: [PATCH 449/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 403e7653..97ea204b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
 	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham)
 	* Type __git: Fix group explorer (Ander Punnar)
+	* Type __pyvenv: Fix group explorer (Dennis Camera)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From a3102022e18a23ce9b54eeaf7415b55361f80bd1 Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Fri, 11 Jun 2021 15:05:17 +0100
Subject: [PATCH 450/527] More sensible defaults; reword debian-only error
 message

---
 cdist/conf/type/__apt_pin/manifest                  | 13 +++++++++----
 cdist/conf/type/__apt_pin/nonparallel               |  0
 cdist/conf/type/__apt_pin/parameter/default/package |  1 -
 3 files changed, 9 insertions(+), 5 deletions(-)
 create mode 100644 cdist/conf/type/__apt_pin/nonparallel
 delete mode 100644 cdist/conf/type/__apt_pin/parameter/default/package

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index b1372ad0..909bc80d 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -19,9 +19,17 @@
 #
 
 
+name="$__object_id"
+
 os=$(cat "$__global/explorer/os")
 state="$(cat "$__object/parameter/state")"
-package="$(cat "$__object/parameter/package")"
+
+if [ -f "$__object/parameter/package" ]; then
+    package="$(cat "$__object/parameter/package")"
+else
+    package=$name
+fi
+
 distribution="$(cat "$__object/parameter/distribution")"
 priority="$(cat "$__object/parameter/priority")"
 
@@ -31,13 +39,10 @@ case "$os" in
    ;;
    *)
       printf "This type is specific to Debian and it's derivatives" >&2
-      printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2
       exit 1
    ;;
 esac
 
-name="$__object_id"
-
 case $distribution in
     stabletesting|unsatbel|experimental)
         pin="release a=$distribution"
diff --git a/cdist/conf/type/__apt_pin/nonparallel b/cdist/conf/type/__apt_pin/nonparallel
new file mode 100644
index 00000000..e69de29b
diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package
deleted file mode 100644
index 72e8ffc0..00000000
--- a/cdist/conf/type/__apt_pin/parameter/default/package
+++ /dev/null
@@ -1 +0,0 @@
-*

From b726697e070e5266eae38c7951ced14a2305acb2 Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Fri, 11 Jun 2021 15:05:33 +0100
Subject: [PATCH 451/527] Add documentation

---
 cdist/conf/type/__apt_pin/man.rst | 37 ++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst
index 7fcae6f8..0c91cdec 100644
--- a/cdist/conf/type/__apt_pin/man.rst
+++ b/cdist/conf/type/__apt_pin/man.rst
@@ -13,11 +13,21 @@ This space intentionally left blank.
 
 REQUIRED PARAMETERS
 -------------------
-None.
+distribution
+   Specifies what distribution the package should be pinned to. Accepts both codenames (buster/bullseye/sid) and suite names (stable/testing/...).
 
 
 OPTIONAL PARAMETERS
 -------------------
+package
+   Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used.
+
+priority
+   The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority)
+
+state
+   Will be passed to underlying `__file` type; see there for valid values and defaults.
+
 None.
 
 
@@ -31,14 +41,31 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # TODO
-    __apt_pin
+   # Add the bullseye repo to buster, but do not install any pacakges by default
+   # only if explicitely asked for
+    __apt_pin bullseye-default \
+       --package "*" \
+       --distribution bullseye \
+       --priority -1
+
+    require="__apt_pin/bullseye-default" __apt_source bullseye \
+       --uri http://deb.debian.org/debian/ \
+       --distribution bullseye \
+       --component main
+       # TODO
+       __apt_pin
+
+    __apt_pin foo --package "foo foo-*" --distribution bullseye
+
+    __foo # Installs the `foo` package internally
+
+    __package foo-plugin-extras
 
 
 SEE ALSO
 --------
-:strong:`TODO`\ (7)
-
+:strong:`apt_preferences`\ (7)
+:strong:`cdist-type__file`\ (7)
 
 AUTHORS
 -------

From 7b3f268df25922c515174862da95569aea547367 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 22 Jun 2021 16:36:30 +0300
Subject: [PATCH 452/527] [__download] improvements

1. post download checksum verification
2. detect hashes without prefix
3. add optional --destination
4. updated man
---
 .../conf/type/__download/explorer/remote_cmd  |  19 ---
 .../type/__download/explorer/remote_cmd_get   |  16 +++
 .../type/__download/explorer/remote_cmd_sum   |  82 +++++++++++
 cdist/conf/type/__download/explorer/state     |  60 ++------
 cdist/conf/type/__download/gencode-local      | 131 +++++++++++++++---
 cdist/conf/type/__download/gencode-remote     |  46 +++++-
 cdist/conf/type/__download/man.rst            |  31 ++++-
 cdist/conf/type/__download/manifest           |   2 +-
 cdist/conf/type/__download/parameter/optional |   3 +-
 9 files changed, 292 insertions(+), 98 deletions(-)
 delete mode 100755 cdist/conf/type/__download/explorer/remote_cmd
 create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_get
 create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_sum

diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd
deleted file mode 100755
index e3e35b45..00000000
--- a/cdist/conf/type/__download/explorer/remote_cmd
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh -e
-
-if [ -f "$__object/parameter/cmd-get" ]
-then
-    cmd="$( cat "$__object/parameter/cmd-get" )"
-
-elif command -v curl > /dev/null
-then
-    cmd="curl -L -o - '%s'"
-
-elif command -v fetch > /dev/null
-then
-    cmd="fetch -o - '%s'"
-
-else
-    cmd="wget -O - '%s'"
-fi
-
-echo "$cmd"
diff --git a/cdist/conf/type/__download/explorer/remote_cmd_get b/cdist/conf/type/__download/explorer/remote_cmd_get
new file mode 100755
index 00000000..9f1cd59c
--- /dev/null
+++ b/cdist/conf/type/__download/explorer/remote_cmd_get
@@ -0,0 +1,16 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cat "$__object/parameter/cmd-get"
+elif
+    command -v curl > /dev/null
+then
+    echo "curl -sSL -o - '%s'"
+elif
+    command -v fetch > /dev/null
+then
+    echo "fetch -o - '%s'"
+else
+    echo "wget -O - '%s'"
+fi
diff --git a/cdist/conf/type/__download/explorer/remote_cmd_sum b/cdist/conf/type/__download/explorer/remote_cmd_sum
new file mode 100755
index 00000000..84df663c
--- /dev/null
+++ b/cdist/conf/type/__download/explorer/remote_cmd_sum
@@ -0,0 +1,82 @@
+#!/bin/sh -e
+
+if [ ! -f "$__object/parameter/sum" ]
+then
+    exit 0
+fi
+
+if [ -f "$__object/parameter/cmd-sum" ]
+then
+    cat "$__object/parameter/cmd-sum"
+    exit 0
+fi
+
+sum_should="$( cat "$__object/parameter/sum" )"
+
+if echo "$sum_should" | grep -Fq ':'
+then
+    sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
+else
+    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+    then
+        sum_hash='cksum'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
+    then
+        sum_hash='md5'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
+    then
+        sum_hash='sha1'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
+    then
+        sum_hash='sha256'
+    else
+        echo 'hash format detection failed' >&2
+        exit 1
+    fi
+fi
+
+os="$( "$__explorer/os" )"
+
+case "$sum_hash" in
+    cksum)
+        echo "cksum %s | awk '{print \$1\" \"\$2}'"
+    ;;
+    md5)
+        case "$os" in
+            freebsd)
+                echo "md5 -q %s"
+            ;;
+            *)
+                echo "md5sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    sha1)
+        case "$os" in
+            freebsd)
+                echo "sha1 -q %s"
+            ;;
+            *)
+                echo "sha1sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    sha256)
+        case "$os" in
+            freebsd)
+                echo "sha256 -q %s"
+            ;;
+            *)
+                echo "sha256sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    *)
+        # we arrive here only if --sum is given with unknown format prefix
+        echo "unknown hash format: $sum_hash" >&2
+        exit 1
+    ;;
+esac
diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 68b517c5..881a1c09 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -1,6 +1,11 @@
 #!/bin/sh -e
 
-dst="/$__object_id"
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
+else
+    dst="/$__object_id"
+fi
 
 if [ ! -f "$dst" ]
 then
@@ -16,57 +21,18 @@ fi
 
 sum_should="$( cat "$__object/parameter/sum" )"
 
-if [ -f "$__object/parameter/cmd-sum" ]
+if echo "$sum_should" | grep -Fq ':'
 then
-    # shellcheck disable=SC2059
-    sum_is="$( eval "$( printf \
-        "$( cat "$__object/parameter/cmd-sum" )" \
-        "$dst" )" )"
-else
-    os="$( "$__explorer/os" )"
-
-    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
-    then
-        sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
-
-    elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="md5:$( md5 -q "$dst" )"
-            ;;
-            *)
-                sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-
-    elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="sha1:$( sha1 -q "$dst" )"
-            ;;
-            *)
-                sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-
-    elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="sha256:$( sha256 -q "$dst" )"
-            ;;
-            *)
-                sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-    fi
+    sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
 fi
 
+sum_cmd="$( "$__type_explorer/remote_cmd_sum" )"
+
+sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )"
+
 if [ -z "$sum_is" ]
 then
-    echo 'no checksum from target' >&2
+    echo 'existing destination checksum failed' >&2
     exit 1
 fi
 
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 571d2c3c..d1b0d0d5 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -11,34 +11,133 @@ fi
 
 url="$( cat "$__object/parameter/url" )"
 
-tmp="$( mktemp )"
-
-dst="/$__object_id"
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
+else
+    dst="/$__object_id"
+fi
 
 if [ -f "$__object/parameter/cmd-get" ]
 then
     cmd="$( cat "$__object/parameter/cmd-get" )"
 
-elif command -v wget > /dev/null
-then
-    cmd="wget -O - '%s'"
-
 elif command -v curl > /dev/null
 then
-    cmd="curl -L -o - '%s'"
+    cmd="curl -sSL -o - '%s'"
 
 elif command -v fetch > /dev/null
 then
     cmd="fetch -o - '%s'"
 
+elif command -v wget > /dev/null
+then
+    cmd="wget -O - '%s'"
+
 else
-    echo 'no usable locally installed utility for downloading' >&2
+    echo 'local download failed, no usable utility' >&2
     exit 1
 fi
 
-printf "$cmd > %s\n" \
-    "$url" \
-    "$tmp"
+echo "download_tmp=\"\$( mktemp )\""
+
+# shellcheck disable=SC2059
+printf "$cmd > \"\$download_tmp\"\n" "$url"
+
+if [ -f "$__object/parameter/sum" ]
+then
+    sum_should="$( cat "$__object/parameter/sum" )"
+
+    if [ -f "$__object/parameter/cmd-sum" ]
+    then
+        local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
+    else
+        if echo "$sum_should" | grep -Fq ':'
+        then
+            sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
+
+            sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
+        else
+            if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+            then
+                sum_hash='cksum'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
+            then
+                sum_hash='md5'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
+            then
+                sum_hash='sha1'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
+            then
+                sum_hash='sha256'
+            else
+                echo 'hash format detection failed' >&2
+                exit 1
+            fi
+        fi
+
+        case "$sum_hash" in
+            cksum)
+                local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'"
+            ;;
+            md5)
+                if command -v md5 > /dev/null
+                then
+                    local_cmd_sum="md5 -q %s"
+                elif
+                    command -v md5sum > /dev/null
+                then
+                    local_cmd_sum="md5sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            sha1)
+                if command -v sha1 > /dev/null
+                then
+                    local_cmd_sum="sha1 -q %s"
+                elif
+                    command -v sha1sum > /dev/null
+                then
+                    local_cmd_sum="sha1sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            sha256)
+                if command -v sha256 > /dev/null
+                then
+                    local_cmd_sum="sha256 -q %s"
+                elif
+                    command -v sha256sum > /dev/null
+                then
+                    local_cmd_sum="sha256sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            *)
+                # we arrive here only if --sum is given with unknown format prefix
+                echo "unknown hash format: $sum_hash" >&2
+                exit 1
+            ;;
+        esac
+
+        if [ -z "$local_cmd_sum" ]
+        then
+            echo 'local checksum verification failed, no usable utility' >&2
+            exit 1
+        fi
+    fi
+
+    # shellcheck disable=SC2059
+    echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\""
+
+    echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
+
+    echo "echo 'local download checksum mismatch' >&2"
+
+    echo "rm -f \"\$download_tmp\""
+
+    echo 'exit 1; fi'
+fi
 
 if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$'
 then
@@ -47,12 +146,10 @@ else
     target_host="$__target_host"
 fi
 
-printf '%s %s %s:%s\n' \
+# shellcheck disable=SC2016
+printf '%s "$download_tmp" %s:%s\n' \
     "$__remote_copy" \
-    "$tmp" \
     "$target_host" \
     "$dst"
 
-echo "rm -f '$tmp'"
-
-echo 'downloaded' > "$__messages_out"
+echo "rm -f \"\$download_tmp\""
diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
index 029a0801..e49bcec3 100755
--- a/cdist/conf/type/__download/gencode-remote
+++ b/cdist/conf/type/__download/gencode-remote
@@ -6,17 +6,51 @@ state_is="$( cat "$__object/explorer/state" )"
 
 if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
 then
-    cmd="$( cat "$__object/explorer/remote_cmd" )"
+    cmd_get="$( cat "$__object/explorer/remote_cmd_get" )"
 
     url="$( cat "$__object/parameter/url" )"
 
-    dst="/$__object_id"
+    if [ -f "$__object/parameter/destination" ]
+    then
+        dst="$( cat "$__object/parameter/destination" )"
+    else
+        dst="/$__object_id"
+    fi
 
-    printf "$cmd > %s\n" \
-        "$url" \
-        "$dst"
+    echo "download_tmp=\"\$( mktemp )\""
 
-    echo 'downloaded' > "$__messages_out"
+    # shellcheck disable=SC2059
+    printf "$cmd_get > \"\$download_tmp\"\n" "$url"
+
+    if [ -f "$__object/parameter/sum" ]
+    then
+        sum_should="$( cat "$__object/parameter/sum" )"
+
+        if [ -f "$__object/parameter/cmd-sum" ]
+        then
+            remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
+        else
+            remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )"
+
+            if echo "$sum_should" | grep -Fq ':'
+            then
+                sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
+            fi
+        fi
+
+        # shellcheck disable=SC2059
+        echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\""
+
+        echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
+
+        echo "echo 'remote download checksum mismatch' >&2"
+
+        echo "rm -f \"\$download_tmp\""
+
+        echo 'exit 1; fi'
+    fi
+
+    echo "mv \"\$download_tmp\" '$dst'"
 fi
 
 if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ]
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index a1278cfb..c16510a9 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -8,7 +8,7 @@ cdist-type__download - Download a file
 
 DESCRIPTION
 -----------
-By default type will try to use ``wget``, ``curl`` or ``fetch``.
+By default type will try to use ``curl``, ``fetch`` or ``wget``.
 If download happens in target (see ``--download``) then type will
 fallback to (and install) ``wget``.
 
@@ -16,6 +16,8 @@ If download happens in local machine, then environment variables like
 ``{http,https,ftp}_proxy`` etc can be used on cdist execution
 (``http_proxy=foo cdist config ...``).
 
+To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -25,14 +27,29 @@ url
 
 OPTIONAL PARAMETERS
 -------------------
+destination
+   Downloaded file's destination in target. If unset, ``$__object_id`` is used.
+
 sum
-   Checksum is used to decide if existing destination file must be redownloaded.
-   By default output of ``cksum`` without filename is expected.
-   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
+   Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256.
+
+   Type tries to detect hash format with regexes, but prefixes
+   ``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported.
+
+   Checksum have two purposes - state check and post-download verification.
+   In state check, if destination checksum mismatches, then content of URL
+   will be downloaded to temporary file. If downloaded temporary file's
+   checksum matches, then it will be moved to destination (overwritten).
+
+   For local downloads it is expected that usable utilities for checksum
+   calculation exist in the system.
 
 download
-   If ``local`` (default), then download file to local storage and copy
-   it to target host. If ``remote``, then download happens in target.
+   If ``local`` (default), then file is downloaded to local storage and copied
+   to target host. If ``remote``, then download happens in target.
+
+   For local downloads it is expected that usable utilities for downloading
+   exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``.
 
 cmd-get
    Command used for downloading.
@@ -62,7 +79,7 @@ EXAMPLES
     require='__directory/opt/cpma' \
         __download /opt/cpma/cnq3.zip \
             --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
-            --sum md5:46da3021ca9eace277115ec9106c5b46
+            --sum 46da3021ca9eace277115ec9106c5b46
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest
index 7ec8d86d..3d4c498b 100755
--- a/cdist/conf/type/__download/manifest
+++ b/cdist/conf/type/__download/manifest
@@ -1,6 +1,6 @@
 #!/bin/sh -e
 
-if grep -Eq '^wget' "$__object/explorer/remote_cmd"
+if grep -Eq '^wget' "$__object/explorer/remote_cmd_get"
 then
     __package wget
 fi
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index d69e083e..e809ef78 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,5 +1,6 @@
-sum
 cmd-get
 cmd-sum
+destination
 download
 onchange
+sum

From 2db40d8d704b427768307fbea29384bd3dc8dbd7 Mon Sep 17 00:00:00 2001
From: fancsali <fancsali@gmail.com>
Date: Mon, 28 Jun 2021 12:54:20 +0200
Subject: [PATCH 453/527] Use $__remote_exec and thus the ssh multiplexing

---
 cdist/conf/type/__rsync/gencode-local | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index e36ded2f..36addc36 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -36,4 +36,5 @@ fi
 
 echo rsync -a \
     --no-owner --no-group \
+    -e "$__remote_exec" \
     -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"

From d937d53f3dfd10830a07aee0450596eea62f2a1a Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Mon, 28 Jun 2021 18:09:35 +0100
Subject: [PATCH 454/527] Add quotes to rsync command

---
 cdist/conf/type/__rsync/gencode-local | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index 36addc36..f1bddc16 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -36,5 +36,5 @@ fi
 
 echo rsync -a \
     --no-owner --no-group \
-    -e "$__remote_exec" \
+    -e \"$__remote_exec\" \
     -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"

From 60753ddfcc3ac49303d572da7f6a68f398c02227 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 1 Jul 2021 14:42:10 +0300
Subject: [PATCH 455/527] fix shellcheck

---
 cdist/conf/type/__download/explorer/state | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 881a1c09..8c5d5ce1 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -28,6 +28,7 @@ fi
 
 sum_cmd="$( "$__type_explorer/remote_cmd_sum" )"
 
+# shellcheck disable=SC2059
 sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )"
 
 if [ -z "$sum_is" ]

From a90e642c1354cf01be7c9a1ba8a468ad624c4202 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 1 Jul 2021 14:50:40 +0300
Subject: [PATCH 456/527] update README

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index de6901c7..a468dd86 100644
--- a/README.md
+++ b/README.md
@@ -24,8 +24,8 @@ For community-maintained types there is
 
 ## Participating
 
-IRC: ``#cdist`` @ freenode
+IRC: ``#cdist`` @ [libera](https://libera.chat)
 
 Matrix: ``#cdist:ungleich.ch``
 
-Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist
+Matrix and IRC are bridged.

From 243a4b904a2de638d00bac57f6e762a076f9ae54 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 2 Jul 2021 06:50:02 +0200
Subject: [PATCH 457/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 97ea204b..d48c657e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,8 @@ next:
 	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham)
 	* Type __git: Fix group explorer (Ander Punnar)
 	* Type __pyvenv: Fix group explorer (Dennis Camera)
+	* Type __download: Improve checksum verification, add optional --destination (Ander Punnar)
+	* Type __debconf_set_selections: Add state explorer (Dennis Camera)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 30ba796d060fd3d0c4affd4167f14784156b5354 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 1 Jul 2021 11:49:07 +0300
Subject: [PATCH 458/527] new type: __snakeoil_cert

---
 .../__snakeoil_cert/explorer/ssl-cert-group   |  8 ++
 .../conf/type/__snakeoil_cert/explorer/state  | 24 ++++++
 .../conf/type/__snakeoil_cert/gencode-remote  | 73 +++++++++++++++++++
 cdist/conf/type/__snakeoil_cert/man.rst       | 60 +++++++++++++++
 .../parameter/default/cert-path               |  1 +
 .../parameter/default/key-path                |  1 +
 .../parameter/default/key-type                |  1 +
 .../type/__snakeoil_cert/parameter/optional   |  4 +
 8 files changed, 172 insertions(+)
 create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group
 create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/state
 create mode 100755 cdist/conf/type/__snakeoil_cert/gencode-remote
 create mode 100644 cdist/conf/type/__snakeoil_cert/man.rst
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/cert-path
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-path
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-type
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/optional

diff --git a/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group
new file mode 100755
index 00000000..a6cb3dfd
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+if grep -Eq '^ssl-cert:' /etc/group
+then
+    echo 'present'
+else
+    echo 'absent'
+fi
diff --git a/cdist/conf/type/__snakeoil_cert/explorer/state b/cdist/conf/type/__snakeoil_cert/explorer/state
new file mode 100755
index 00000000..cc5aae0b
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/explorer/state
@@ -0,0 +1,24 @@
+#!/bin/sh -e
+
+key_path="$( cat "$__object/parameter/key-path" )"
+
+if echo "$key_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    key_path="$( printf "$key_path" "$__object_id" )"
+fi
+
+cert_path="$( cat "$__object/parameter/cert-path" )"
+
+if echo "$cert_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    cert_path="$( printf "$cert_path" "$__object_id" )"
+fi
+
+if [ ! -f "$key_path" ] || [ ! -f "$cert_path" ]
+then
+    echo 'absent'
+else
+    echo 'present'
+fi
diff --git a/cdist/conf/type/__snakeoil_cert/gencode-remote b/cdist/conf/type/__snakeoil_cert/gencode-remote
new file mode 100755
index 00000000..8ffbfad1
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/gencode-remote
@@ -0,0 +1,73 @@
+#!/bin/sh -e
+
+state="$( cat "$__object/explorer/state" )"
+
+if [ "$state" = 'present' ]
+then
+    exit 0
+fi
+
+if [ -f "$__object/parameter/common-name" ]
+then
+    common_name="$( cat "$__object/parameter/common-name" )"
+else
+    common_name="$__object_id"
+fi
+
+key_path="$( cat "$__object/parameter/key-path" )"
+
+if echo "$key_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    key_path="$( printf "$key_path" "$__object_id" )"
+fi
+
+cert_path="$( cat "$__object/parameter/cert-path" )"
+
+if echo "$cert_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    cert_path="$( printf "$cert_path" "$__object_id" )"
+fi
+
+key_type="$( cat "$__object/parameter/key-type" )"
+
+key_type_arg="$( echo "$key_type" | cut -d : -f 2 )"
+
+case "$key_type" in
+    rsa:*)
+        echo "openssl genrsa -out '$key_path' $key_type_arg"
+    ;;
+    ec:*)
+        echo "openssl ecparam -name $key_type_arg -genkey -noout -out '$key_path'"
+    ;;
+esac
+
+# shellcheck disable=SC2016
+echo 'csr_path="$( mktemp )"'
+
+echo "openssl req -new -subj '/CN=$common_name' -key '$key_path' -out \"\$csr_path\""
+
+echo "openssl x509 -req -sha256 -days 3650 -in \"\$csr_path\" -signkey '$key_path' -out '$cert_path'"
+
+# shellcheck disable=SC2016
+echo 'rm -f "$csr_path"'
+
+if [ "$( cat "$__object/explorer/ssl-cert-group" )" = 'present' ]
+then
+    key_group='ssl-cert'
+else
+    key_group='root'
+fi
+
+echo "chmod 640 '$key_path'"
+
+echo "chown root '$key_path'"
+
+echo "chgrp $key_group '$key_path'"
+
+echo "chmod 644 '$cert_path'"
+
+echo "chown root '$cert_path'"
+
+echo "chgrp root '$cert_path'"
diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst
new file mode 100644
index 00000000..0b547804
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/man.rst
@@ -0,0 +1,60 @@
+cdist-type__snakeoil_cert(7)
+============================
+
+NAME
+----
+cdist-type__snakeoil_cert - Generate self-signed certificate
+
+
+DESCRIPTION
+-----------
+The purpose of this type is to generate **self-signed** certificate and private key
+for **testing purposes**. Certificate will expire in 3650 days.
+
+Certificate's and key's access bits will be ``644`` and ``640`` respectively.
+If target system has ``ssl-cert`` group, then it will be used as key's group.
+Use ``require='__snakeoil_cert/...' __file ...`` to override.
+
+
+OPTIONAL PARAMETERS
+-------------------
+common-name
+   Defaults to ``$__object_id``.
+
+key-path
+   ``%s`` in path will be replaced with ``$__object_id``.
+   Defaults to ``/etc/ssl/private/%s.pem``.
+
+key-type
+   Possible values are ``rsa:$bits`` and ``ec:$name``.
+   For possible EC names see ``openssl ecparam -list_curves``.
+   Defaults to ``rsa:2048``.
+
+cert-path
+   ``%s`` in path will be replaced with ``$__object_id``.
+   Defaults to ``/etc/ssl/certs/%s.pem``.
+
+
+EXAMPLES
+--------
+.. code-block:: sh
+	__snakeoil_cert localhost-rsa \
+	    --common-name localhost \
+	    --key-type rsa:4096
+
+	__snakeoil_cert localhost-ec \
+	    --common-name localhost \
+	    --key-type ec:prime256v1
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path
new file mode 100644
index 00000000..4bbae089
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path
@@ -0,0 +1 @@
+/etc/ssl/certs/%s.pem
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-path b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path
new file mode 100644
index 00000000..86eb9359
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path
@@ -0,0 +1 @@
+/etc/ssl/private/%s.pem
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-type b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type
new file mode 100644
index 00000000..f13f8ada
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type
@@ -0,0 +1 @@
+rsa:2048
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/optional b/cdist/conf/type/__snakeoil_cert/parameter/optional
new file mode 100644
index 00000000..76d08c0a
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/optional
@@ -0,0 +1,4 @@
+common-name
+key-path
+key-type
+cert-path

From 853e5cf7b4f615e1470c7ec2cfa8fe68cfa85ce7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 5 Jul 2021 09:07:06 +0200
Subject: [PATCH 459/527] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index d48c657e..0c3c64e1 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,8 @@ next:
 	* Type __pyvenv: Fix group explorer (Dennis Camera)
 	* Type __download: Improve checksum verification, add optional --destination (Ander Punnar)
 	* Type __debconf_set_selections: Add state explorer (Dennis Camera)
+	* Core: Implement usable cdist scan (Timothée Floure)
+	* New type: __snakeoil_cert (Ander Punnar)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From be92731c5c8a8543448f0d87fafae67e22ac76a1 Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Mon, 5 Jul 2021 12:38:26 +0100
Subject: [PATCH 460/527] Shell check quoting

We're actually echo-ing the command, hence the escape in front of the
quotes - the issue Shellcheck alludes too would actually occur, had the
escaping bakcslashes been omitted.
---
 cdist/conf/type/__rsync/gencode-local | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index f1bddc16..be4feabb 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -34,7 +34,8 @@ if [ -f "$__object/parameter/rsync-opts" ]; then
     done < "$__object/parameter/rsync-opts"
 fi
 
+# shellcheck disable=SC2086
 echo rsync -a \
     --no-owner --no-group \
-    -e \"$__remote_exec\" \
+    -e \"${__remote_exec}\" \
     -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"

From 521241d74102e37fae5f55552e1ef565d26ea9d2 Mon Sep 17 00:00:00 2001
From: fancsali <fancsali@gmail.com>
Date: Mon, 5 Jul 2021 15:28:05 +0200
Subject: [PATCH 461/527] Refine docs even more

---
 cdist/conf/type/__apt_pin/man.rst | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst
index 0c91cdec..4229c0cd 100644
--- a/cdist/conf/type/__apt_pin/man.rst
+++ b/cdist/conf/type/__apt_pin/man.rst
@@ -3,12 +3,12 @@ cdist-type__apt_pin(7)
 
 NAME
 ----
-cdist-type__apt_pin - TODO
+cdist-type__apt_pin - Manage apt pinning rules
 
 
 DESCRIPTION
 -----------
-This space intentionally left blank.
+Adds/removes/edits rules to pin some packages to a specific distribution. Useful if using multiple debian repositories at the same time. (Useful, if one wants to use a few specific packages from backports or perhaps Debain testing... or even sid.)
 
 
 REQUIRED PARAMETERS
@@ -20,7 +20,7 @@ distribution
 OPTIONAL PARAMETERS
 -------------------
 package
-   Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used.
+   Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used.
 
 priority
    The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority)
@@ -28,7 +28,6 @@ priority
 state
    Will be passed to underlying `__file` type; see there for valid values and defaults.
 
-None.
 
 
 BOOLEAN PARAMETERS
@@ -41,8 +40,8 @@ EXAMPLES
 
 .. code-block:: sh
 
-   # Add the bullseye repo to buster, but do not install any pacakges by default
-   # only if explicitely asked for
+   # Add the bullseye repo to buster, but do not install any packages by default,
+   # only if explicitely asked for (-1 means "never" for apt)
     __apt_pin bullseye-default \
        --package "*" \
        --distribution bullseye \
@@ -52,19 +51,19 @@ EXAMPLES
        --uri http://deb.debian.org/debian/ \
        --distribution bullseye \
        --component main
-       # TODO
-       __apt_pin
 
     __apt_pin foo --package "foo foo-*" --distribution bullseye
 
-    __foo # Installs the `foo` package internally
+    __foo # Assuming, this installs the `foo` package internally
 
-    __package foo-plugin-extras
+    __package foo-plugin-extras # Assuming we also need some extra stuff
 
 
 SEE ALSO
 --------
-:strong:`apt_preferences`\ (7)
+:strong:`apt_preferences`\ (5)
+:strong:`cdist-type__apt_source`\ (7)
+:strong:`cdist-type__apt_backports`\ (7)
 :strong:`cdist-type__file`\ (7)
 
 AUTHORS

From 166b58aeea09da41086525849291c70a3c3a571c Mon Sep 17 00:00:00 2001
From: fancsali <fancsali@gmail.com>
Date: Mon, 5 Jul 2021 15:32:27 +0200
Subject: [PATCH 462/527] Fix typo in distro names...

---
 cdist/conf/type/__apt_pin/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index 909bc80d..e72a8fdd 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -44,7 +44,7 @@ case "$os" in
 esac
 
 case $distribution in
-    stabletesting|unsatbel|experimental)
+    stable|testing|unstable|experimental)
         pin="release a=$distribution"
         ;;
     *)

From 485283f2e531f5fa23a876fa52411e0030f32589 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 7 Jul 2021 20:47:22 +0300
Subject: [PATCH 463/527] new type: __sed

---
 cdist/conf/type/__sed/explorer/file           | 22 +++++++++
 cdist/conf/type/__sed/gencode-remote          | 46 +++++++++++++++++
 cdist/conf/type/__sed/man.rst                 | 49 +++++++++++++++++++
 cdist/conf/type/__sed/parameter/boolean       |  1 +
 cdist/conf/type/__sed/parameter/optional      |  1 +
 .../type/__sed/parameter/required_multiple    |  1 +
 6 files changed, 120 insertions(+)
 create mode 100755 cdist/conf/type/__sed/explorer/file
 create mode 100755 cdist/conf/type/__sed/gencode-remote
 create mode 100644 cdist/conf/type/__sed/man.rst
 create mode 100644 cdist/conf/type/__sed/parameter/boolean
 create mode 100644 cdist/conf/type/__sed/parameter/optional
 create mode 100644 cdist/conf/type/__sed/parameter/required_multiple

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
new file mode 100755
index 00000000..a40f71b3
--- /dev/null
+++ b/cdist/conf/type/__sed/explorer/file
@@ -0,0 +1,22 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/file" ]
+then
+    file="$( cat "$__object/parameter/file" )"
+else
+    file="/$__object_id"
+fi
+
+if [ -f "$file" ]
+then
+    if [ -s "$file" ]
+    then
+        cat "$file"
+    else
+        echo "$file is empty" >&2
+        exit 1
+    fi
+else
+    echo "$file do not exist" >&2
+    exit 1
+fi
diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
new file mode 100755
index 00000000..391e3bc1
--- /dev/null
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -0,0 +1,46 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/file" ]
+then
+    file="$( cat "$__object/parameter/file" )"
+else
+    file="/$__object_id"
+fi
+
+script="$( cat "$__object/parameter/script" )"
+
+if [ "$script" = '-' ]
+then
+    script="$( cat "$__object/stdin" )"
+elif
+    [ -f "$script" ]
+then
+    script="$( cat "$script" )"
+fi
+
+file_from_target="$__object/explorer/file"
+
+sed_cmd='sed'
+
+if [ -f "$__object/parameter/regexp-extended" ]
+then
+    sed_cmd="$sed_cmd --regexp-extended"
+fi
+
+if ! echo "$script" \
+    | "$sed_cmd" -f - "$file_from_target" \
+    | diff "$file_from_target" - \
+    > /dev/null
+then
+    echo 'tmp="$( mktemp )"'
+
+    echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
+
+    echo "$script"
+
+    echo 'EOF'
+
+    echo "cp \"\$tmp\" '$file'"
+    
+    echo 'rm -f "$tmp"'
+fi
diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst
new file mode 100644
index 00000000..fbac212d
--- /dev/null
+++ b/cdist/conf/type/__sed/man.rst
@@ -0,0 +1,49 @@
+cdist-type__sed(7)
+==================
+
+NAME
+----
+cdist-type__sed - Transform text files with ``sed``
+
+
+DESCRIPTION
+-----------
+TODO
+
+
+REQUIRED MULTIPLE PARAMETERS
+----------------------------
+script
+   TODO
+
+
+OPTIONAL PARAMETERS
+-------------------
+file
+   TODO
+
+
+BOOLEAN PARAMETERS
+------------------
+regexp-extended
+   TODO
+
+
+EXAMPLES
+--------
+.. code-block:: sh
+
+   true
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
diff --git a/cdist/conf/type/__sed/parameter/boolean b/cdist/conf/type/__sed/parameter/boolean
new file mode 100644
index 00000000..1ad75c5d
--- /dev/null
+++ b/cdist/conf/type/__sed/parameter/boolean
@@ -0,0 +1 @@
+regexp-extended
diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional
new file mode 100644
index 00000000..f73f3093
--- /dev/null
+++ b/cdist/conf/type/__sed/parameter/optional
@@ -0,0 +1 @@
+file
diff --git a/cdist/conf/type/__sed/parameter/required_multiple b/cdist/conf/type/__sed/parameter/required_multiple
new file mode 100644
index 00000000..84f7e31d
--- /dev/null
+++ b/cdist/conf/type/__sed/parameter/required_multiple
@@ -0,0 +1 @@
+script

From 7a5896acfad0f992d6e7fe9cbe0bd08035ce19db Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 7 Jul 2021 21:23:25 +0300
Subject: [PATCH 464/527] add --onchange, fix shellcheck

---
 cdist/conf/type/__sed/gencode-remote     | 9 ++++++++-
 cdist/conf/type/__sed/man.rst            | 3 +++
 cdist/conf/type/__sed/parameter/optional | 1 +
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 391e3bc1..6b1bd216 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -32,6 +32,7 @@ if ! echo "$script" \
     | diff "$file_from_target" - \
     > /dev/null
 then
+    # shellcheck disable=SC2016
     echo 'tmp="$( mktemp )"'
 
     echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
@@ -41,6 +42,12 @@ then
     echo 'EOF'
 
     echo "cp \"\$tmp\" '$file'"
-    
+
+    # shellcheck disable=SC2016
     echo 'rm -f "$tmp"'
+
+    if [ -f "$__object/parameter/onchange" ]
+    then
+        cat "$__object/parameter/onchange"
+    fi
 fi
diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst
index fbac212d..4a728184 100644
--- a/cdist/conf/type/__sed/man.rst
+++ b/cdist/conf/type/__sed/man.rst
@@ -22,6 +22,9 @@ OPTIONAL PARAMETERS
 file
    TODO
 
+onchange
+   TODO
+
 
 BOOLEAN PARAMETERS
 ------------------
diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional
index f73f3093..fa86f917 100644
--- a/cdist/conf/type/__sed/parameter/optional
+++ b/cdist/conf/type/__sed/parameter/optional
@@ -1 +1,2 @@
 file
+onchange

From cf0032d667eb4bd8b7fe1bebacec341bbf71ed42 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 7 Jul 2021 21:28:00 +0300
Subject: [PATCH 465/527] add messaging and exit earlier

---
 cdist/conf/type/__sed/gencode-remote | 42 +++++++++++++++-------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 6b1bd216..836506f0 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -27,27 +27,31 @@ then
     sed_cmd="$sed_cmd --regexp-extended"
 fi
 
-if ! echo "$script" \
+if echo "$script" \
     | "$sed_cmd" -f - "$file_from_target" \
     | diff "$file_from_target" - \
     > /dev/null
 then
-    # shellcheck disable=SC2016
-    echo 'tmp="$( mktemp )"'
-
-    echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
-
-    echo "$script"
-
-    echo 'EOF'
-
-    echo "cp \"\$tmp\" '$file'"
-
-    # shellcheck disable=SC2016
-    echo 'rm -f "$tmp"'
-
-    if [ -f "$__object/parameter/onchange" ]
-    then
-        cat "$__object/parameter/onchange"
-    fi
+    exit 0
+fi
+
+# shellcheck disable=SC2016
+echo 'tmp="$( mktemp )"'
+
+echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
+
+echo "$script"
+
+echo 'EOF'
+
+echo "cp \"\$tmp\" '$file'"
+
+# shellcheck disable=SC2016
+echo 'rm -f "$tmp"'
+
+echo 'change' >> "$__messages_out"
+
+if [ -f "$__object/parameter/onchange" ]
+then
+    cat "$__object/parameter/onchange"
 fi

From 3e76d1cd3fbe53fa77c460cb2ce8416698b101bd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 8 Jul 2021 08:09:05 +0200
Subject: [PATCH 466/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 0c3c64e1..04f826f0 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -13,6 +13,7 @@ next:
 	* Type __debconf_set_selections: Add state explorer (Dennis Camera)
 	* Core: Implement usable cdist scan (Timothée Floure)
 	* New type: __snakeoil_cert (Ander Punnar)
+	* Type __rsync: Honour $__remote_exec env var (Daniel Fancsali)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 77dab4c5c63070aef962875af7fe8b1565f5ba78 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 10 Jul 2021 20:37:02 +0200
Subject: [PATCH 467/527] Release 6.9.7

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 04f826f0..284293c1 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)

From 65c43d3c1db938ac0063f54b1cb6b5090bb0a665 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 10 Jul 2021 21:02:27 +0200
Subject: [PATCH 468/527] Fix docs code block errors

---
 cdist/conf/type/__snakeoil_cert/man.rst | 1 +
 docs/src/cdist-scan.rst                 | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst
index 0b547804..b0b0a2e9 100644
--- a/cdist/conf/type/__snakeoil_cert/man.rst
+++ b/cdist/conf/type/__snakeoil_cert/man.rst
@@ -38,6 +38,7 @@ cert-path
 EXAMPLES
 --------
 .. code-block:: sh
+
 	__snakeoil_cert localhost-rsa \
 	    --common-name localhost \
 	    --key-type rsa:4096
diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst
index 064e65ff..86b7fab6 100644
--- a/docs/src/cdist-scan.rst
+++ b/docs/src/cdist-scan.rst
@@ -57,6 +57,7 @@ resolved name to stdout - if any. The script must be executable.
 Simplest script:
 
 .. code-block:: sh
+
   #!/bin/sh
 
   case "$1" in
@@ -71,6 +72,7 @@ Simplest script:
 Resolving name from `PTR` DNS record:
 
 .. code-block:: sh
+
   #!/bin/sh
 
   for cmd in dig sed; do

From 0e611af2a6388572eef5112c2ffaed082803965c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 13 Jul 2021 00:13:22 +0300
Subject: [PATCH 469/527] [__rsync] rewrite

---
 cdist/conf/type/__rsync/gencode-local         | 123 +++++++++++++-----
 cdist/conf/type/__rsync/gencode-remote        |  37 ------
 cdist/conf/type/__rsync/man.rst               | 107 +++++----------
 cdist/conf/type/__rsync/manifest              |  18 ---
 .../type/__rsync/parameter/default/options    |   1 +
 cdist/conf/type/__rsync/parameter/optional    |   4 +-
 .../type/__rsync/parameter/optional_multiple  |   2 +-
 7 files changed, 130 insertions(+), 162 deletions(-)
 delete mode 100755 cdist/conf/type/__rsync/gencode-remote
 create mode 100644 cdist/conf/type/__rsync/parameter/default/options

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index be4feabb..612d237e 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -1,41 +1,100 @@
 #!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
-source=$(cat "$__object/parameter/source")
-remote_user=$(cat "$__object/parameter/remote-user")
+if ! command -v rsync > /dev/null
+then
+    echo 'rsync is missing in local machine' >&2
+    exit 1
+fi
 
-if [ -f "$__object/parameter/destination" ]; then
-    destination=$(cat "$__object/parameter/destination")
+src="$( cat "$__object/parameter/source" )"
+
+if [ ! -e "$src" ]
+then
+    echo "$src not found" >&2
+    exit 1
+fi
+
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
 else
-    destination="/$__object_id"
+    dst="/$__object_id"
 fi
 
-set --
-if [ -f "$__object/parameter/rsync-opts" ]; then
-    while read -r opts; do
-        set -- "$@" "--$opts"
-    done < "$__object/parameter/rsync-opts"
+# if source is directory, then make sure that
+# source and destination are ending with slash,
+# because this is what you almost always want when
+# rsyncing two directories.
+
+if [ -d "$src" ]
+then
+    if ! echo "$src" | grep -Eq '/$'
+    then
+        src="$src/"
+    fi
+
+    if ! echo "$dst" | grep -Eq '/$'
+    then
+        dst="$dst/"
+    fi
 fi
 
+remote_user="$( cat "$__object/parameter/remote-user" )"
+
+options="$( cat "$__object/parameter/options" )"
+
+if [ -f "$__object/parameter/option" ]
+then
+    while read -r l
+    do
+        # there's a limitation in argparse: value can't begin with '-'.
+        # to workaround this, let's prefix opts with '\' in manifest and remove here.
+        # read more about argparse issue: https://bugs.python.org/issue9334
+
+        options="$options $( echo "$l" | sed 's/\\//g' )"
+    done \
+        < "$__object/parameter/option"
+fi
+
+if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ]
+then
+    options="$options --chown="
+
+    if [ -f "$__object/parameter/owner" ]
+    then
+        owner="$( cat "$__object/parameter/owner" )"
+        options="$options$owner"
+    fi
+
+    if [ -f "$__object/parameter/group" ]
+    then
+        group="$( cat "$__object/parameter/group" )"
+        options="$options:$group"
+    fi
+fi
+
+if [ -f "$__object/parameter/mode" ]
+then
+    mode="$( cat "$__object/parameter/mode" )"
+    options="$options --chmod=$mode"
+fi
+
+# IMPORTANT
+#
+# 1. we first dry-run rsync with change summary to find out
+#    if there are any changes and code generation is needed.
+# 2. normally, to get current state or target host, we run
+#    such operations in type explorers, but that's not
+#    possible due to how rsync works.
+# 3. redirecting output of dry-run to stderr to ease debugging.
+# 4. to understand how that cryptic regex works, please
+#    open rsync manpage and read about --itemize-changes.
+
 # shellcheck disable=SC2086
-echo rsync -a \
-    --no-owner --no-group \
-    -e \"${__remote_exec}\" \
-    -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"
+if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \
+    | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2
+then
+    exit 0
+fi
+
+echo "rsync $options $src $remote_user@$__target_host:$dst"
diff --git a/cdist/conf/type/__rsync/gencode-remote b/cdist/conf/type/__rsync/gencode-remote
deleted file mode 100755
index 074246af..00000000
--- a/cdist/conf/type/__rsync/gencode-remote
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-if [ -f "$__object/parameter/destination" ]; then
-    destination=$(cat "$__object/parameter/destination")
-else
-    destination="/$__object_id"
-fi
-
-ownergroup=""
-if [ -f "$__object/parameter/owner" ]; then
-    ownergroup=$(cat "$__object/parameter/owner")
-fi
-if [ -f "$__object/parameter/group" ]; then
-    ownergroup="${ownergroup}:$(cat "$__object/parameter/group")"
-fi
-
-if [ "$ownergroup" ]; then
-    echo chown -R "$ownergroup" "$destination"
-fi
diff --git a/cdist/conf/type/__rsync/man.rst b/cdist/conf/type/__rsync/man.rst
index 94b06d63..88019c92 100644
--- a/cdist/conf/type/__rsync/man.rst
+++ b/cdist/conf/type/__rsync/man.rst
@@ -3,112 +3,73 @@ cdist-type__rsync(7)
 
 NAME
 ----
-cdist-type__rsync - Mirror directories using rsync
+cdist-type__rsync - Mirror directories using ``rsync``
 
 
 DESCRIPTION
 -----------
-WARNING: This type is of BETA quality:
-
-- it has not been tested widely
-- interfaces *may* change
-- if there is a better approach to solve the problem -> the type may even vanish
-
-If you are fine with these constraints, please read on.
-
-
-This cdist type allows you to mirror local directories to the
-target host using rsync. Rsync will be installed in the manifest of the type.
-If group or owner are giveng, a recursive chown will be executed on the 
-target host.
-
-A slash will be appended to the source directory so that only the contents
-of the directory are taken and not the directory name itself.
+The purpose of this type is to bring power of ``rsync`` into ``cdist``.
 
 
 REQUIRED PARAMETERS
 -------------------
 source
-    Where to take files from
+   Source directory in local machine.
+   If source is directory, slash (``/``) will be added to source and destination paths.
 
 
 OPTIONAL PARAMETERS
 -------------------
-group
-   Group to chgrp to.
+destination
+   Destination directory. Defaults to ``$__object_id``.
 
 owner
-   User to chown to.
+   Will be passed to ``rsync`` as ``--chown=OWNER``.
+   Read ``rsync(1)`` for more details.
 
-destination
-    Use this as the base destination instead of the object id
+group
+   Will be passed to ``rsync`` as ``--chown=:GROUP``.
+   Read ``rsync(1)`` for more details.
+
+mode
+   Will be passed to ``rsync`` as ``--chmod=MODE``.
+   Read ``rsync(1)`` for more details.
+
+options
+   Defaults to ``--recursive --links --perms --times``.
+   Due to `bug in Python's argparse<https://bugs.python.org/issue9334>`_, value must be prefixed with ``\``.
 
 remote-user
-    Use this user instead of the default "root" for rsync operations.
+   Defaults to ``root``.
 
 
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
-rsync-opts
-    Use this option to give rsync options with.
-    See rsync(1) for available options.
-    Only "--" options are supported.
-    Write the options without the beginning "--"
-    Can be specified multiple times.
-
-
-MESSAGES
---------
-NONE
+option
+   Pass additional options to ``rsync``.
+   See ``rsync(1)`` for all possible options.
+   Due to `bug in Python's argparse<https://bugs.python.org/issue9334>`_, value must be prefixed with ``\``.
 
 
 EXAMPLES
 --------
-
 .. code-block:: sh
 
-    # You can use any source directory
-    __rsync /tmp/testdir \
-        --source /etc
-
-    # Use source from type
-    __rsync /etc \
-        --source "$__type/files/package"
-
-    # Allow multiple __rsync objects to write to the same dir
-    __rsync mystuff \
-        --destination /usr/local/bin \
-        --source "$__type/files/package"
-
-    __rsync otherstuff \
-        --destination /usr/local/bin \
-        --source "$__type/files/package2"
-
-    # Use rsync option --exclude
-    __rsync /tmp/testdir \
-        --source /etc \
-        --rsync-opts exclude=sshd_conf
-
-    # Use rsync with multiple options --exclude --dry-run
-    __rsync /tmp/testing \
-        --source /home/tester \
-        --rsync-opts exclude=id_rsa \
-        --rsync-opts dry-run
-
-
-SEE ALSO
---------
-:strong:`rsync`\ (1)
+    __rsync /var/www/example.com \
+        --owner root \
+        --group www-data \
+        --mode 'D750,F640' \
+        --source "$__files/example.com/www"
 
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+Ander Punnar <ander-at-kvlt-dot-ee>
 
 
 COPYING
 -------
-Copyright \(C) 2015 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
diff --git a/cdist/conf/type/__rsync/manifest b/cdist/conf/type/__rsync/manifest
index 9bd44c6d..64fa804e 100755
--- a/cdist/conf/type/__rsync/manifest
+++ b/cdist/conf/type/__rsync/manifest
@@ -1,21 +1,3 @@
 #!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
 __package rsync
diff --git a/cdist/conf/type/__rsync/parameter/default/options b/cdist/conf/type/__rsync/parameter/default/options
new file mode 100644
index 00000000..d967b110
--- /dev/null
+++ b/cdist/conf/type/__rsync/parameter/default/options
@@ -0,0 +1 @@
+--recursive --links --perms --times
diff --git a/cdist/conf/type/__rsync/parameter/optional b/cdist/conf/type/__rsync/parameter/optional
index ac2b2390..833e9bbe 100644
--- a/cdist/conf/type/__rsync/parameter/optional
+++ b/cdist/conf/type/__rsync/parameter/optional
@@ -1,4 +1,6 @@
 destination
-owner
 group
+mode
+options
+owner
 remote-user
diff --git a/cdist/conf/type/__rsync/parameter/optional_multiple b/cdist/conf/type/__rsync/parameter/optional_multiple
index fdb7cd88..01925a15 100644
--- a/cdist/conf/type/__rsync/parameter/optional_multiple
+++ b/cdist/conf/type/__rsync/parameter/optional_multiple
@@ -1 +1 @@
-rsync-opts
+option

From 46b5c24cd240cd9006d93cfcc12a4d81b46a5238 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 18 Jul 2021 16:25:00 +0300
Subject: [PATCH 470/527] use $__remote_exec for RSYNC_RSH

---
 cdist/conf/type/__rsync/gencode-local | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index 612d237e..e9f3c131 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -90,6 +90,8 @@ fi
 # 4. to understand how that cryptic regex works, please
 #    open rsync manpage and read about --itemize-changes.
 
+export RSYNC_RSH="$__remote_exec"
+
 # shellcheck disable=SC2086
 if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \
     | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2
@@ -97,4 +99,6 @@ then
     exit 0
 fi
 
+echo "export RSYNC_RSH='$__remote_exec'"
+
 echo "rsync $options $src $remote_user@$__target_host:$dst"

From 5229337611b7e7afb2597729a786c637a8bec1f6 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 18 Jul 2021 17:41:29 +0200
Subject: [PATCH 471/527] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 284293c1..f0746218 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __rsync: Rewrite (Ander Punnar)
+
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)

From de116661613002e91b54194ac8f0d5c3dd3eebbd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 18 Jul 2021 17:45:19 +0200
Subject: [PATCH 472/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index f0746218..ea51ac1b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __rsync: Rewrite (Ander Punnar)
+	* New type: __apt_pin (Daniel Fancsali)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 24c9406ea0f2c6c8edc87e5bbc1be25b5e8e1572 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 19 Jul 2021 12:13:23 +0200
Subject: [PATCH 473/527] [explorer/os_version] Convert Devuan ceres to version
 number

Conversion of Devuan ceres to version numbers is done based on Devuan codenames.
The version number is the version number of the final release - 0.01.

Analogous to Debian.
---
 cdist/conf/explorer/os_version | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 3b02dedd..6c94915c 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -1,6 +1,7 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,12 +18,11 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
 # All os variables are lower case
 #
-#
 
-case "$("$__explorer/os")" in
+case $("${__explorer:?}/os")
+in
    amazon)
       cat /etc/system-release
    ;;
@@ -59,7 +59,23 @@ case "$("$__explorer/os")" in
       esac
    ;;
    devuan)
-      cat /etc/devuan_version
+      devuan_version=$(cat /etc/devuan_version)
+      case ${devuan_version}
+      in
+         (*/ceres)
+            # ceres versions don't have a number, so we decode by codename:
+            case ${devuan_version}
+            in
+               (chimaera/ceres) echo 3.99 ;;
+               (beowulf/ceres) echo 2.99 ;;
+               (ascii/ceres) echo 1.99 ;;
+               (*) exit 1
+            esac
+            ;;
+         (*)
+            echo "${devuan_version}"
+            ;;
+      esac
    ;;
    fedora)
       cat /etc/fedora-release

From fbc9594729ece74b2f8612fe8046ddea966e8c05 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 20 Jul 2021 06:38:46 +0200
Subject: [PATCH 474/527] ++changelog

---
 docs/changelog | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/docs/changelog b/docs/changelog
index ea51ac1b..55d7cb73 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)
+	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
@@ -146,7 +147,7 @@ next:
 	* Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham)
 	* Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham)
 	* Configuration: Add notes to cdist.cfg.skeleton (Evil Ham)
-	* Explorers cpu_cores, memory: Improve *BSD support (Evil Ham)
+	* Explorers cpu_cores, memory: Improve BSD support (Evil Ham)
 	* Core: Remove debug logging noise (Evil Ham)
 
 6.5.4: 2020-04-11
@@ -211,7 +212,7 @@ next:
 	* Documentation: PreOS english nitpicking (Evil Ham)
 	* Documentation: Add installing from source with signature verification (Darko Poljak)
 	* Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak)
-	* Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham)
+	* Type __start_on_boot: Docs: remove unsupported BSD claim (Evil Ham)
 	* New type: __openldap_server (Evil Ham)
 
 6.2.0: 2019-11-30
@@ -1070,9 +1071,9 @@ next:
 	* Removed type __removeline (replaced by __line) (Nico Schottelius)
 	* Type __directory: Parameter --parents and --recursive are now boolean (Nico Schottelius)
 	* Type __package_apt, __package_luarocks, __package_opkg,
-		__package_pacman, __package_pkg_freebsd, __package_pkg_openbsd,
-		__package_rubygem, __package_yum, __process:
-			Parameter state accepts only "present" and "absent" (Nico Schottelius)
+	  __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd,
+	  __package_rubygem, __package_yum, __process:
+	  Parameter state accepts only "present" and "absent" (Nico Schottelius)
 	* Dist: Initial support for pypi packaging (Nico Schottelius)
 
 2.0.15: 2012-11-02

From c7daaabc6c28cbfaebaf6c620cc3cac130684d2c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 20 Jul 2021 09:03:16 +0200
Subject: [PATCH 475/527] [docs] Bump copyright year to 2021

---
 docs/src/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/conf.py b/docs/src/conf.py
index 47765413..a3dfafca 100644
--- a/docs/src/conf.py
+++ b/docs/src/conf.py
@@ -56,7 +56,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'cdist'
-copyright = 'ungleich GmbH 2020'
+copyright = 'ungleich GmbH 2021'
 # author = 'Darko Poljak'
 
 # The version info for the project you're documenting, acts as replacement for

From fed01ded83b217a3ad147f2403ca863be373eeaa Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 22 Jul 2021 11:17:41 +0200
Subject: [PATCH 476/527] [cdist.log] Define custom log functions on
 logging.Logger

Define out custom logger functions on logging.Logger so that they are passed on
to all other loggers.

Also, the logger functions need to take a self argument so that they can log on
the corrent Logger.
---
 cdist/log.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/cdist/log.py b/cdist/log.py
index 113f3b4c..62e457fe 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -36,25 +36,27 @@ import threading
 logging.OFF = logging.CRITICAL + 10  # disable logging
 logging.addLevelName(logging.OFF, 'OFF')
 
+
 logging.VERBOSE = logging.INFO - 5
 logging.addLevelName(logging.VERBOSE, 'VERBOSE')
 
 
-def _verbose(msg, *args, **kwargs):
-    logging.log(logging.VERBOSE, msg, *args, **kwargs)
+def _verbose(self, msg, *args, **kwargs):
+    self.log(logging.VERBOSE, msg, args, **kwargs)
 
 
-logging.verbose = _verbose
+logging.Logger.verbose = _verbose
+
 
 logging.TRACE = logging.DEBUG - 5
 logging.addLevelName(logging.TRACE, 'TRACE')
 
 
-def _trace(msg, *args, **kwargs):
-    logging.log(logging.TRACE, msg, *args, **kwargs)
+def _trace(self, msg, *args, **kwargs):
+    self.log(logging.TRACE, msg, *args, **kwargs)
 
 
-logging.trace = _trace
+logging.Logger.trace = _trace
 
 
 class CdistFormatter(logging.Formatter):

From 71fee1fd6b4874abffee5173a96c31842a9583bd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 23 Jul 2021 08:06:45 +0200
Subject: [PATCH 477/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 55d7cb73..1273e432 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
+	* Core: Fix logging bug (Dennis Camera)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 67bcc6cae38e2d480c3336b6a4d59d01196190e3 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 24 Jul 2021 02:37:58 +0200
Subject: [PATCH 478/527] Improve Makefile compatibility and build docs

We now use `$(MAKE)` for subsequent calls to `make`.
This means that systems that do not default to GNU make can run `gmake man` and
produce the man pages.

While there also document a dependency on the rtd theme for sphinx.
---
 Makefile                   | 6 +++---
 docs/src/cdist-install.rst | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 3712511c..89286310 100644
--- a/Makefile
+++ b/Makefile
@@ -35,9 +35,9 @@ DOCS_SRC_DIR=./docs/src
 SPEECHDIR=./docs/speeches
 TYPEDIR=./cdist/conf/type
 
-SPHINXM=make -C $(DOCS_SRC_DIR) man
-SPHINXH=make -C $(DOCS_SRC_DIR) html
-SPHINXC=make -C $(DOCS_SRC_DIR) clean
+SPHINXM=$(MAKE) -C $(DOCS_SRC_DIR) man
+SPHINXH=$(MAKE) -C $(DOCS_SRC_DIR) html
+SPHINXC=$(MAKE) -C $(DOCS_SRC_DIR) clean
 
 ################################################################################
 # Manpages
diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst
index 18863145..390ab9ec 100644
--- a/docs/src/cdist-install.rst
+++ b/docs/src/cdist-install.rst
@@ -12,7 +12,7 @@ This is the machine from which you will configure target hosts.
  * /bin/sh: A POSIX like shell (for instance bash, dash, zsh)
  * Python >= 3.5
  * SSH client
- * sphinx (for building html docs and/or the man pages)
+ * sphinx with the rtd theme (for building html docs and/or the man pages)
 
 Target Hosts
 ~~~~~~~~~~~~

From cb8695cc88478640d7e76992438d90d3d6a68a90 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 24 Jul 2021 12:53:39 +0200
Subject: [PATCH 479/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 1273e432..181c4139 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* New type: __apt_pin (Daniel Fancsali)
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
 	* Core: Fix logging bug (Dennis Camera)
+	* Build: Improve Makefile compatibility (Evilham)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 4156fea9001aa267c8b173247cada2f919511c1b Mon Sep 17 00:00:00 2001
From: Joachim Desroches <joachim.desroches@epfl.ch>
Date: Wed, 28 Jul 2021 12:56:39 +0200
Subject: [PATCH 480/527] [filesystem] Add ubuntu as supported distribution.

---
 cdist/conf/type/__filesystem/explorer/lsblk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk
index 9be3c575..d376c09f 100644
--- a/cdist/conf/type/__filesystem/explorer/lsblk
+++ b/cdist/conf/type/__filesystem/explorer/lsblk
@@ -27,7 +27,7 @@ else
 fi
 
 case "$os" in
-    alpine|centos|fedora|redhat|suse|gentoo)
+    alpine|centos|fedora|gentoo|redhat|suse|ubuntu)
         if [ ! -x "$(command -v lsblk)" ]; then
             echo "lsblk is required for __filesystem type" >&2
             exit 1

From 542674dae81ab03a4de2c4ad0b2eaf264ee2c442 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 30 Jul 2021 10:30:33 +0200
Subject: [PATCH 481/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 181c4139..9fc10b20 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
 	* Core: Fix logging bug (Dennis Camera)
 	* Build: Improve Makefile compatibility (Evilham)
+	* Type __filesystem: Support ubuntu (Joachim Desroches)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 53334fb4eb311550af7f5b73f279a2e86fa1c504 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 4 Aug 2021 19:50:10 +0200
Subject: [PATCH 482/527] [explorer/os_version] Fix for FreeBSD < 10.0 (again)

---
 cdist/conf/explorer/os_version | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 6c94915c..cc976608 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -89,7 +89,14 @@ in
    freebsd)
       # Apparently uname -r is not a reliable way to get the patch level.
       # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
-      freebsd-version
+      if command -v freebsd-version >/dev/null 2>&1
+      then
+         # get userland version
+         freebsd-version -u
+      else
+         # fallback to kernel release for FreeBSD < 10.0
+         uname -r
+      fi
    ;;
    *bsd|solaris)
       uname -r

From e108cbc205cb5e7ac0d2e07b82cef4c83eaa285f Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Tue, 3 Aug 2021 13:20:43 +0200
Subject: [PATCH 483/527] [explorer/os_version] Ubuntu: fall back to
 os-release/lsb-release files

---
 cdist/conf/explorer/os_version | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 6c94915c..e70fe7f4 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -21,6 +21,17 @@
 # All os variables are lower case
 #
 
+rc_getvar() {
+   awk -F= -v varname="$2" '
+      function unquote(s) {
+         if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/)
+            return substr(s, 2, length(s) - 2)
+         else
+            return s
+      }
+      $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1"
+}
+
 case $("${__explorer:?}/os")
 in
    amazon)
@@ -114,7 +125,20 @@ in
       fi
    ;;
    ubuntu)
-      lsb_release -sr
+      if command -v lsb_release >/dev/null 2>&1
+      then
+         lsb_release -sr
+      elif test -r /usr/lib/os-release
+      then
+         # fallback to /usr/lib/os-release if lsb_release is not present (like
+         # on minimized Ubuntu installations)
+         rc_getvar /usr/lib/os-release VERSION_ID
+      elif test -r /etc/lsb-release
+      then
+         # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old
+         # versions without /usr/lib/os-release.
+         rc_getvar /etc/lsb-release DISTRIB_RELEASE
+      fi
    ;;
    alpine)
        cat /etc/alpine-release

From 83fe6e9f5b2537db73d5c6a142b6d24eef75ac58 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Tue, 3 Aug 2021 19:26:55 +0200
Subject: [PATCH 484/527] [explorer/memory] Fix conversion of large numbers (>=
 2GiB)

At least mawk uses scientific notation when using print for
numbers >=2^31 (INT_MAX of a signed 32-bit int).

`printf "%.f\n"` works around this.
---
 cdist/conf/explorer/memory | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory
index 63aba9c6..c6d113cf 100755
--- a/cdist/conf/explorer/memory
+++ b/cdist/conf/explorer/memory
@@ -27,19 +27,18 @@
 str2bytes() {
 	awk -F' ' '
 	$2 ==   "B" || !$2 { print $1 }
-	$2 ==  "kB" { print $1 * 1000 }
-	$2 ==  "MB" { print $1 * 1000 * 1000 }
-	$2 ==  "GB" { print $1 * 1000 * 1000 * 1000 }
-	$2 ==  "TB" { print $1 * 1000 * 1000 * 1000 * 1000 }
-	$2 == "kiB" { print $1 * 1024 }
-	$2 == "MiB" { print $1 * 1024 * 1024 }
-	$2 == "GiB" { print $1 * 1024 * 1024 * 1024 }
-	$2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }'
+	$2 ==  "kB" { printf "%.f\n", ($1 * 1000) }
+	$2 ==  "MB" { printf "%.f\n", ($1 * 1000 * 1000) }
+	$2 ==  "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) }
+	$2 ==  "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) }
+	$2 == "kiB" { printf "%.f\n", ($1 * 1024) }
+	$2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) }
+	$2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) }
+	$2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }'
 }
 
 bytes2kib() {
-	set -- "$(cat)"
-	test "$1" -gt 0 && echo $(($1 / 1024))
+	awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }'
 }
 
 

From a7d6481a7ddc7cb72b1a55bfca7fdfed20514a62 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Mon, 2 Aug 2021 21:23:50 +0200
Subject: [PATCH 485/527] [type/__update_alternatives] Secure cdist-defined
 environment variables with :?

---
 .../__update_alternatives/explorer/alternatives     |  2 +-
 cdist/conf/type/__update_alternatives/explorer/link |  6 +++---
 .../type/__update_alternatives/explorer/path_is     |  4 ++--
 .../explorer/path_should_state                      |  2 +-
 .../conf/type/__update_alternatives/gencode-remote  | 13 ++++++-------
 5 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives
index 34aaca56..ecc62f4b 100755
--- a/cdist/conf/type/__update_alternatives/explorer/alternatives
+++ b/cdist/conf/type/__update_alternatives/explorer/alternatives
@@ -1,4 +1,4 @@
 #!/bin/sh -e
 
-update-alternatives --display "$__object_id" 2>/dev/null \
+update-alternatives --display "${__object_id:?}" 2>/dev/null \
     | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link
index 6519e7c2..c6fd1c98 100755
--- a/cdist/conf/type/__update_alternatives/explorer/link
+++ b/cdist/conf/type/__update_alternatives/explorer/link
@@ -18,12 +18,12 @@ for altdir in \
     /var/lib/dpkg/alternatives \
     /var/lib/alternatives
 do
-    if [ ! -f "$altdir/$__object_id" ]
+    if [ ! -f "$altdir/${__object_id:?}" ]
     then
         continue
     fi
 
-    link="$( awk 'NR==2' "$altdir/$__object_id" )"
+    link="$( awk 'NR==2' "$altdir/${__object_id:?}" )"
 
     if [ -n "$link" ]
     then
@@ -33,7 +33,7 @@ done
 
 if [ -z "$link" ]
 then
-    echo "unable to get link for $__object_id" >&2
+    echo "unable to get link for ${__object_id:?}" >&2
     exit 1
 fi
 
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
index fc304d5d..a24bd40e 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_is
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -1,11 +1,11 @@
 #!/bin/sh -e
 
-path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \
+path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \
     | awk '/link currently points to/ {print $5}' )"
 
 if [ -z "$path_is" ]
 then
-    echo "unable to get current path for $__object_id" >&2
+    echo "unable to get current path for ${__object_id:?}" >&2
     exit 1
 fi
 
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state
index 59e015c5..b74a7ee8 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_should_state
+++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state
@@ -1,6 +1,6 @@
 #!/bin/sh -e
 
-if [ -f "$( cat "$__object/parameter/path" )" ]
+if [ -f "$( cat "${__object:?}/parameter/path" )" ]
 then
     echo 'present'
 else
diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index e393cdef..13666805 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -18,26 +18,25 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 
-path_is="$( cat "$__object/explorer/path_is" )"
+path_is="$( cat "${__object:?}/explorer/path_is" )"
 
-path_should="$( cat "$__object/parameter/path" )"
+path_should="$( cat "${__object:?}/parameter/path" )"
 
 if [ "$path_is" = "$path_should" ]
 then
     exit 0
 fi
 
-if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ]
+if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \
+    && [ -z "${__cdist_dry_run+dry run}" ]
 then
     echo "$path_should does not exist in target" >&2
     exit 1
 fi
 
-name="$__object_id"
+name=${__object_id:?}
 
-alternatives="$( cat "$__object/explorer/alternatives" )"
-
-if ! echo "$alternatives" | grep -Fxq "$path_should"
+if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives"
 then
     if [ ! -f "$__object/parameter/install" ]
     then

From 0b3b47396f2aafa377e3d5d9a13f51ace2303d41 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Mon, 2 Aug 2021 21:25:08 +0200
Subject: [PATCH 486/527] [type/__update_alternatives] dry-run fixes

---
 cdist/conf/type/__update_alternatives/explorer/link |  5 ++++-
 .../type/__update_alternatives/explorer/path_is     |  5 ++++-
 .../conf/type/__update_alternatives/gencode-remote  | 13 ++++++++-----
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link
index c6fd1c98..d1087c75 100755
--- a/cdist/conf/type/__update_alternatives/explorer/link
+++ b/cdist/conf/type/__update_alternatives/explorer/link
@@ -31,8 +31,11 @@ do
     fi
 done
 
-if [ -z "$link" ]
+if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then
+    # NOTE: ignore error for dry-runs because a package providing the link
+    #       might be managed by another cdist object (which wasn't executed,
+    #       because dry run…).
     echo "unable to get link for ${__object_id:?}" >&2
     exit 1
 fi
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
index a24bd40e..9208df7b 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_is
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -3,8 +3,11 @@
 path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \
     | awk '/link currently points to/ {print $5}' )"
 
-if [ -z "$path_is" ]
+if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then
+    # NOTE: ignore error for dry-runs because a package providing the
+    #       alternative might be managed by another cdist object (which
+    #       wasn't executed, because dry run…).
     echo "unable to get current path for ${__object_id:?}" >&2
     exit 1
 fi
diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index 13666805..e91ea78f 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -38,16 +38,19 @@ name=${__object_id:?}
 
 if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives"
 then
-    if [ ! -f "$__object/parameter/install" ]
+    if [ -f "${__object:?}/parameter/install" ]
     then
+        link="$( cat "${__object:?}/explorer/link" )"
+        echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
+    elif [ -z "${__cdist_dry_run+dry run}" ]
+    then
+        # NOTE: ignore error for dry-runs because a package providing the link
+        #       to be installed might be managed by another cdist object (which
+        #       wasn't executed, because dry run…).
         echo "$path_should is not in $name alternatives." >&2
         echo 'Please install missing packages or use --install to add path to alternatives.' >&2
         exit 1
     fi
-
-    link="$( cat "$__object/explorer/link" )"
-
-    echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
 fi
 
 echo "update-alternatives --set '$name' '$path_should'"

From bbcc81a9841f2619e1b9e13b25a941337489a681 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 4 Aug 2021 21:44:04 +0200
Subject: [PATCH 487/527] [type/__update_alternatives] Fix for non-English
 locales

Since update-alternatives(1) is localized, screen scraping its output breaks
if the locale is set to non-English.
---
 cdist/conf/type/__update_alternatives/explorer/alternatives | 4 ++--
 cdist/conf/type/__update_alternatives/explorer/path_is      | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives
index ecc62f4b..bb1619a9 100755
--- a/cdist/conf/type/__update_alternatives/explorer/alternatives
+++ b/cdist/conf/type/__update_alternatives/explorer/alternatives
@@ -1,4 +1,4 @@
 #!/bin/sh -e
 
-update-alternatives --display "${__object_id:?}" 2>/dev/null \
-    | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
+LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \
+| awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
index 9208df7b..5cf4fa4b 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_is
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -1,7 +1,8 @@
 #!/bin/sh -e
 
-path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \
-    | awk '/link currently points to/ {print $5}' )"
+path_is=$(
+    LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \
+    | awk '/link currently points to/ { print $5 }')
 
 if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then

From 2a0c073d4021206c7459015cefaba218004235ce Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 4 Aug 2021 21:54:17 +0200
Subject: [PATCH 488/527] [explorer/os_version] Fix for legacy Mac OS X
 versions

---
 cdist/conf/explorer/os_version | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 3b02dedd..96eca1ee 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -68,7 +68,8 @@ case "$("$__explorer/os")" in
       cat /etc/gentoo-release
    ;;
    macosx)
-      sw_vers -productVersion
+      # NOTE: Legacy versions (< 10.3) do not support options
+      sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }'
    ;;
    freebsd)
       # Apparently uname -r is not a reliable way to get the patch level.

From 3ae5a606ca7182ec7fe13134670400506e198ec1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 5 Aug 2021 10:27:51 +0200
Subject: [PATCH 489/527] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 9fc10b20..f9409d7e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,10 @@ next:
 	* Core: Fix logging bug (Dennis Camera)
 	* Build: Improve Makefile compatibility (Evilham)
 	* Type __filesystem: Support ubuntu (Joachim Desroches)
+	* Explorer os_version: Fall back to os-release/lsb-release file on Ubuntu (Dennis Camera)
+	* Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera)
+	* Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera)
+	* Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From edcac70b2a99ce532dd5bca9a0b8fc5bf6dc5148 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 21 Jul 2021 13:22:34 +0200
Subject: [PATCH 490/527] [explorer/machine_type] Reimplement

---
 cdist/conf/explorer/machine_type | 986 ++++++++++++++++++++++++++++---
 1 file changed, 904 insertions(+), 82 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 1c84f4d7..29f98849 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -1,8 +1,6 @@
-#!/bin/sh
+#!/bin/sh -e
 #
-# 2014 Daniel Heule  (hda at sfs.biz)
-# 2014 Thomas Oettli (otho at sfs.biz)
-# 2020 Evilham (contact at evilham.com)
+# 2021 Dennis Camera (cdist at dtnr.ch)
 #
 # This file is part of cdist.
 #
@@ -19,91 +17,915 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+# This explorer tries to determine what type of machine the target to be
+# configured is (container, virtual machine, bare-metal).
+#
+# It will print one line for each layer it can detect.
+# The format of all lines is: TYPE[ VERB VENDOR]
+#
+# VERB does not have a special meaning, it is just for better readability.
+#
+# e.g.
+# container on lxc
+# virtual by kvm-spapr
+#
+# The third word of each line can be composed of different parts concatenated with a `-'
+# (minus) character, with each component being a specification of the previous,
+# e.g.:
+#  - lxc-libvirt             (LXC container, managed by libvirt)
+#  - lpar-s390 / lpar-power  (LPAR running on IBM S/390 or POWER, respectively)
+#  - xen-hvm / xen-pv        (Xen HVM vs para-virtualization)
+#
+# If this explorer cannot determine any information it will print nothing.
+#
 
-os=$("$__explorer/os")
+# Add /sbin and /usr/sbin to the path so we can find system
+# binaries like dmidecode.
+PATH=$(getconf PATH 2>/dev/null) || PATH='/usr/bin:/bin'
+PATH="/sbin:/usr/sbin:${PATH}"
+export PATH
 
-vendor_string_to_machine_type() {
-    for vendor in vmware bochs kvm qemu virtualbox bhyve; do
-        if echo "${1}" | grep -q -i "${vendor}"; then
-            if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then
-                vendor="kvm"
-            fi
-            echo "virtual_by_${vendor}"
-            exit
-        fi
-    done
+arch=$(uname -m | sed -e 's/i.86/i386/' -e 's/arm.*/arm/')
+uname_s=$(uname -s)
+
+
+is_command() { command -v "$1" >/dev/null 2>&1; }
+
+is_oneof() (
+	x=$1; shift
+	for y
+	do
+		test "${x}" = "${y}" || continue
+		return 0
+	done
+	return 1
+)
+
+tolower() { LC_ALL=C tr '[:upper:]' '[:lower:]'; }
+
+# shellcheck disable=SC2086
+glob_exists() { set -- $1; test -e "$1"; }
+
+get_dmi_field() {
+	if is_oneof "${uname_s}" NetBSD
+	then
+		case $1
+		in
+			(system-manufacturer) _mib=machdep.dmi.system-vendor ;;
+			(system-product-name) _mib=machdep.dmi.system-product ;;
+			(system-version|system-uuid) _mib=machdep.dmi.$1 ;;
+			(bios-vendor|bios-version) _mib=machdep.dmi.$1 ;;
+			(biod-release-date) _mib=machdep.dmi.bios-date ;;
+			(*) _mib= ;;
+		esac
+
+		test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return
+	fi
+
+	if is_command dmidecode
+	then
+		dmidecode -s "$1"
+	elif test -d "${dmi_sysfs-}"
+	then
+		case $1
+		in
+			(system-manufacturer) _filename=sys_vendor ;;
+			(system-product-name) _filename=product_name ;;
+			(*) _filename=$(echo "$1" | tr - _) ;;
+		esac
+		if test -r "${dmi_sysfs-}/${_filename}"
+		then
+			cat "${dmi_sysfs}/${_filename}"
+		fi
+		unset _filename
+	elif test "${uname_s}" = OpenBSD
+	then
+		# NOTE: something similar to system-manufacutrer and system-product-name
+		#       is available on OpenBSD in sysctl
+		case $1
+		in
+			(system-manufacturer) _mib=hw.vendor ;;
+			(system-product-name) _mib=hw.product ;;
+			(*) _mib= ;;
+		esac
+
+		test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return
+	fi
+
+	return 1
 }
 
-case "$os" in
-    "freebsd")
-        # FreeBSD does not have /proc/cpuinfo even when procfs is used.
-        # Instead there is a sysctl kern.vm_guest.
-        # Which is 'none' if physical, else the virtualisation.
-        vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)"
-        if [ -n "${vm_guest}" ]; then
-            if [ "${vm_guest}" = "none" ]; then
-                echo "physical"
-                exit
-            fi
-            echo "virtual_by_${vm_guest}"
-            exit
-        fi
-    ;;
+has_cpuinfo() { test -e /proc/cpuinfo; }
 
-    "openbsd")
-        # OpenBSD can also use the sysctl's: hw.vendor or hw.product.
-        # Note we can be reasonably sure about a machine being virtualised
-        # as long as we can identify the virtualisation technology.
-        # But not so much about it being physical...
-        # Patches are welcome / reach out if you have better ideas.
-        for sysctl in hw.vendor hw.product; do
-            # This exits if we can make a reasonable judgement
-            vendor_string_to_machine_type "$(sysctl -n "${sysctl}")"
-        done
-    ;;
+get_sysctl() {
+	is_command sysctl && sysctl -n "$1" 2>/dev/null
+}
 
-    *)
-        # Defaulting to linux for compatibility with previous cdist behaviour
 
-        if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then
-            echo openvz
-            exit
-        fi
+# Check for container
 
-        if [ -e "/proc/1/environ" ] &&
-            tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then
-            echo lxc
-            exit
-        fi
+has_ct_pid_1() {
+	test -r /run/systemd/container -o -r /proc/1/environ
+}
 
-        if [ -r /proc/cpuinfo ]; then
-            # this should only exist on virtual guest machines,
-            # tested on vmware, xen, kvm, bhyve
-            if grep -q "hypervisor" /proc/cpuinfo; then
-                # this file is aviable in xen guest systems
-                if [ -r /sys/hypervisor/type ]; then
-                    if grep -q -i "xen" /sys/hypervisor/type; then
-                        echo virtual_by_xen
-                        exit
-                    fi
-                else
-                    for vendor_file in /sys/class/dmi/id/product_name \
-                                       /sys/class/dmi/id/sys_vendor \
-                                       /sys/class/dmi/id/chasis_vendor; do
-                        if [ -r ${vendor_file} ]; then
-                            # This exits if we can make a reasonable judgement
-                            vendor_string_to_machine_type "$(cat "${vendor_file}")"
-                        fi
-                    done
-                fi
-                echo "virtual_by_unknown"
-                exit
-            else
-                echo "physical"
-                exit
-            fi
-        fi
-    ;;
-esac
+translate_container_name() {
+	case $1
+	in
+		('lxc')
+			echo lxc ;;
+		('lxc-libvirt')
+			echo lxc-libvirt ;;
+		('podman')
+			echo podman ;;
+		('systemd-nspawn')
+			echo systemd_nspawn ;;
+		(*)
+			return 1 ;;
+	esac
+	return 0
+}
 
-echo "unknown"
+check_ct_pid_1() {
+	if test -r /run/systemd/container
+	then
+		translate_container_name "$(head -n1 /run/systemd/container)" \
+		&& return 0
+	fi
+
+	if test -r /proc/1/environ
+	then
+		translate_container_name "$(
+			LC_ALL=C tr '\000' '\n' </proc/1/environ \
+			| sed -n -e 's/^container=//p')" \
+		&& return 0
+	fi
+
+	return 1
+}
+
+has_ct_cgroup() {
+	test -r /proc/self/cgroup
+}
+
+check_ct_cgroup() {
+	if grep -q -e ':/docker/' /proc/self/cgroup
+	then
+		echo docker
+	elif grep -q -e ':/lxc/.*\.libvirt-lxc$' /proc/self/cgroup
+	then
+		echo libvirt-lxc
+	elif grep -q -F '/libpod-' /proc/self/cgroup
+	then
+		echo podman
+	else
+		return 1
+	fi
+}
+
+check_ct_files() {
+	if test -e /.dockerenv -o -e /run/.dockerenv -o -e /.dockerinit
+	then
+		echo docker
+	elif test -f /run/.containerenv
+	then
+		# https://github.com/containers/podman/issues/3586#issuecomment-661918679
+		echo podman
+	elif test -d /proc/vz -a ! -d /proc/bc -a -f /proc/user_beancounters \
+		|| test -d /.cpt_hardlink_dir_*/
+	then
+		# Virtuozzo / OpenVZ
+		#
+		# /proc/vz: always exists if OpenVZ kernel is running
+		# /proc/bc: exists only on node (not inside container)
+		# /proc/user_beancounters - exists inside container
+		#
+		# /.cpt_hardlink_dir_*/ may be (?) created by vzctl:
+		# https://github.com/kolyshkin/vzctl/commit/09e974f
+
+		echo openvz
+	elif grep -qie 'Microsoft\|WSL' /proc/sys/kernel/osrelease /proc/version 2>/dev/null
+	then
+		# https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364
+		echo wsl
+	elif test -d /var/.cagefs
+	then
+		# https://docs.cloudlinux.com/cloudlinux_os_components/#cagefs
+		# CageFS is not "really" a container, but it isn't a chroot either.
+		echo cagefs
+	elif test -e /proc/self/status && grep -q -e '^VxID: [0-9]\{1,\}' /proc/self/status
+	then
+		# Linux-VServer
+		if grep -q -x -F 'VxID: 0' /proc/self/status
+		then
+			# host
+			return 1
+		else
+			# guest
+			echo linux_vserver
+		fi
+	else
+		return 1
+	fi
+}
+
+check_ct_os_specific() (
+	if jailed=$(get_sysctl security.jail.jailed) && test "${jailed}" = 1
+	then
+		# FreeBSD jail
+		echo jail
+		return 0
+	fi
+
+	if is_command zonename && test "$(zonename)" != global
+	then
+		# Solaris zone
+		echo zone
+		return 0
+	fi
+
+	return 1
+)
+
+
+# Check for hypervisor
+
+guess_hypervisor_from_cpu_model() {
+	case $1
+	in
+		(*\ KVM\ *)
+			echo kvm ;;
+		(*\ QEMU\ *|QEMU\ *)
+			echo qemu ;;
+		(*)
+			return 1 ;;
+	esac
+}
+
+has_vm_cpuinfo() { has_cpuinfo; }
+
+check_vm_cpuinfo() {
+	if grep -q -F 'User Mode Linux' /proc/cpuinfo \
+		|| grep -q -F 'UML' /proc/cpuinfo
+	then
+		# User Mode Linux
+		echo uml
+	elif grep -q -e '^vendor_id.*: PowerVM Lx86' /proc/cpuinfo
+	then
+		# IBM PowerVM Lx86 (Linux/x86 emulator)
+		echo powervm_lx86
+	elif grep -q -e '^vendor_id.*: IBM/S390' /proc/cpuinfo
+	then
+		# IBM SystemZ (S/390)
+		if test -f /proc/sysinfo
+		then
+			if grep -q -e '^VM[0-9]* Control Program: KVM/Linux' /proc/sysinfo
+			then
+				echo kvm-s390
+				return 0
+			elif grep -q -e '^VM[0-9]* Control Program: z/VM' /proc/sysinfo
+			then
+				echo zvm
+				return 0
+			elif grep -q -e '^LPAR ' /proc/sysinfo
+			then
+				echo zvm-lpar
+				return 0
+			fi
+		fi
+		return 1
+	else
+		if grep -q -e '^model name.*:' /proc/cpuinfo
+		then
+			sed -n -e 's/^model name[^:]*: *//p' /proc/cpuinfo \
+			| while read -r _cpu_model
+			  do
+				  guess_hypervisor_from_cpu_model "${_cpu_model}"
+			  done \
+			| sort \
+			| uniq -c \
+			| awk '
+			  { if ($1 > most_c) { most_c = $1; most_s = $2 } }
+			  END {
+				if (most_s) print most_s
+				exit !most_s
+			  }' \
+			&& return 0
+		fi
+		return 1
+	fi
+}
+
+check_vm_arch_specific() {
+	case ${arch}
+	in
+		(ppc64|ppc64le)
+			# Check PPC64 LPAR, KVM
+
+			# example /proc/cpuinfo line indicating 'not baremetal'
+			# platform	: pSeries
+			#
+			# example /proc/ppc64/lparcfg systemtype line
+			# system_type=IBM pSeries (emulated by qemu)
+
+			if has_cpuinfo && grep -q -e 'platform.**pSeries' /proc/cpuinfo
+			then
+				if test -e /proc/ppc64/lparcfg
+				then
+					# Assume LPAR, now detect shared or dedicated
+					if grep -q -x -F 'shared_processor_mode=1' /proc/ppc64/lparcfg
+					then
+						echo powervm-shared
+						return 0
+					else
+						echo powervm-dedicated
+						return 0
+					fi
+				fi
+			fi
+			;;
+		(sparc*)
+			# Check for SPARC LDoms
+
+			if test -e /dev/mdesc
+			then
+				if test -d /sys/class/vlds/ctrl -a -d /sys/class/vlds/sp
+				then
+					# control LDom
+					return 1
+				else
+					# guest LDom
+					echo ldom-sparc
+				fi
+
+				# MDPROP=/usr/lib/ldoms/mdprop.py
+				# if test -x "${MDPROP}"
+				# then
+				# 	if test -n "$("${MDPROP}" -v iodevice device-type=pciex)"
+				# 	then
+				# 		echo ldoms-root
+				# 		echo ldoms-io
+				# 	elif test -n "$("${MDPROP}" -v iov-device vf-id=0)"
+				# 	then
+				# 		echo ldoms-io
+				# 	fi
+				# fi
+				return 0
+			fi
+			;;
+		(i?86|x86*|amd64|i86pc)
+			# Check CPUID
+			#
+			# Many fullvirt hypervisors give an indication through CPUID.  Use
+			# the virt-what helper program to get this information if available.
+
+			for CPUID_HELPER in \
+				$(command -v virt-what-cpuid-helper 2>/dev/null) \
+				/usr/lib/x86_64-*/virt-what-cpuid-helper \
+				/usr/lib/i?86-*/virt-what-cpuid-helper \
+				/usr/lib/virt-what/virt-what-cpuid-helper
+			do
+				if test -x "${CPUID_HELPER:?}"; then break; fi
+			done
+
+			if test -x "${CPUID_HELPER-}"
+			then
+				case $(command "${CPUID_HELPER}")
+				in
+					('bhyve bhyve ')
+						echo bhyve
+						;;
+					('LKVMLKVMLKVM')
+						echo lkvm
+						;;
+					('KVMKVMKVM')
+						echo kvm
+						;;
+					('TCGTCGTCGTCG')
+						echo qemu-tcg
+						;;
+					('Microsoft Hv')
+						# http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx
+						echo hyperv
+						;;
+					('OpenBSDVMM58')
+						# OpenBSD/VMM
+						echo openbsd_vmm
+						;;
+					('VMwareVMware')
+						# check added by Chetan Loke.
+						echo vmware
+						;;
+					('XenVMMXenVMM')
+						if has dmi
+						then
+							# https://access.redhat.com/solutions/222903
+							echo xen-hvm
+						else
+							echo xen-paravirt
+						fi
+						;;
+					(*)
+						return 1 ;;
+				esac
+				return 0
+			fi
+
+			unset CPUID_HELPER
+
+			# VMM CPUID flag denotes that this system is running under a VMM
+			if is_oneof "${uname_s}" Darwin
+			then
+				get_sysctl machdep.cpu.features | tr ' ' '\n' | grep -qixF VMM \
+				&& return 0
+			fi
+			if has_cpuinfo \
+				&& grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo
+			then
+				return 0
+			fi
+			;;
+		(ia64)
+			if test -d /sys/bus/xen -a ! -d /sys/bus/xen-backend
+			then
+				# PV-on-HVM drivers installed in a Xen guest
+				echo xen-hvm
+				return 0
+			fi
+			;;
+	esac
+	return 1
+}
+
+has_vm_dmi() {
+	# Check for various products in SMBIOS/DMI.
+	# Note that DMI doesn't exist on all architectures (only x86 and some ARM).
+	# On other architectures the $dmi variable will be empty.
+
+	if test -d /sys/class/dmi/id/
+	then
+		dmi_sysfs=/sys/class/dmi/id
+	elif test -d /sys/devices/virtual/dmi/id/
+	then
+		dmi_sysfs=/sys/devices/virtual/dmi/id
+	fi
+
+	# shellcheck disable=SC2015
+	{
+		is_command dmidecode \
+		&& (
+			# dmidecode needs to exit 0 and not print the No SMBIOS/DMI line
+			dmi_out=$(dmidecode 2>&1) \
+			&& ! printf '%s\n' "${dmi_out}" \
+				 | grep -qF 'No SMBIOS nor DMI entry point found, sorry.'
+			) \
+		|| test -d "${dmi_sysfs}"
+	}
+}
+
+check_vm_dmi() {
+	case $(get_dmi_field system-product-name)
+	in
+		(*.metal)
+			if test "$(get_dmi_field system-manufacturer)" = 'Amazon EC2'
+			then
+				# AWS EC2 bare metal -> no virtualisation
+				return 1
+			fi
+			;;
+		('BHYVE')
+			echo bhyve
+			return 0
+			;;
+		('Google Compute Engine')
+			echo gce
+			return 0
+			;;
+		('RHEV Hypervisor')
+			# Red Hat Enterprise Virtualization
+			echo rhev
+			return 0
+			;;
+		('KVM'|'Bochs'|'KVM Virtual Machine')
+			echo kvm
+			return 0
+			;;
+		('Parallels Virtual Platform')
+			echo parallels
+			return 0
+			;;
+		('VirtualBox')
+			echo virtualbox
+			return 0
+			;;
+		('VMware Virtual Platform')
+			echo vmware
+			return 0
+			;;
+	esac
+
+	case $(get_dmi_field system-manufacturer)
+	in
+		('Alibaba'*)
+			case $(get_dmi_field system-product-name)
+			in
+				('Alibaba Cloud ECS')
+					echo alibaba-ecs
+					;;
+				(*)
+					echo alibaba
+					;;
+			esac
+			return 0
+			;;
+		('Amazon EC2')
+			# AWS on bare-metal or KVM
+			echo aws-ec2
+			return 0
+			;;
+		('innotek GmbH'|'Oracle Corporation')
+			echo virtualbox
+			return 0
+			;;
+		('Joyent')
+			if test "$(get_dmi_field system-product-name)" = 'SmartDC HVM'
+			then
+				# SmartOS KVM
+				echo kvm-smartdc_hvm
+				return 0
+			fi
+			;;
+		('Microsoft Corporation'*)
+			if test "$(get_dmi_field system-product-name)" = 'Virtual Machine'
+			then
+				if test -e /proc/irq/7/hyperv \
+				|| expr "$(get_dmi_field bios-version)" : 'VRTUAL.*' >/dev/null
+				then
+					echo hyperv
+					return 0
+				fi
+
+				case $(get_dmi_field system-version)
+				in
+					(VPC[0-9]*|VS2005*|*[Vv]irtual*[Pp][Cc]*)
+						echo virtualpc
+						return 0
+						;;
+					(*)
+						echo hyperv
+						return 0
+						;;
+				esac
+			fi
+			;;
+		('Nutanix')
+			# Nutanix AHV. Similar to KVM.
+			if test "$(get_dmi_field system-product-name)" = 'AHV'
+			then
+				echo nutanix_ahv
+				return 0
+			fi
+			;;
+		('oVirt')
+			echo ovirt
+			return 0
+			;;
+		('Parallels Software International Inc.')
+			echo parallels
+			return 0
+			;;
+		('QEMU')
+			echo qemu
+			return 0
+			;;
+		('VMware, Inc.')
+			echo vmware
+			return 0
+			;;
+	esac
+
+	case $(get_dmi_field bios-vendor)
+	in
+		('Amazon EC2')
+			# AWS on bare-metal or KVM
+			echo aws-ec2
+			return 0
+			;;
+		('BHYVE')
+			echo bhyve
+			return 0
+			;;
+		('innotek GmbH')
+			echo virtualbox
+			return 0
+			;;
+		('Parallels Software International Inc.')
+			echo parallels
+			return 0
+			;;
+		('Xen')
+			if get_dmi_field bios-version | grep -q -e '\([0-9]\{1,\}\.\)\{2\}amazon'
+			then
+				# AWS on Xen
+				echo aws-xen
+				return 0
+			fi
+			;;
+	esac
+
+	return 1
+}
+
+check_vm_hyp_specific() {
+	if is_command vmware-checkvm && vmware-checkvm >/dev/null
+	then
+		# vmware-checkvm is provided by VMware's open-vm-tools
+		echo vmware
+		return 0
+	elif test -d /proc/xen
+	then
+		test -r /proc/xen/capabilities &&
+		if grep -q -F 'control_d' /proc/xen/capabilities 2>/dev/null
+		then
+			# Xen dom0
+			return 1
+		else
+			# Xen domU
+			echo xen
+			return 0
+		fi
+	fi
+	return 1
+}
+
+has_vm_dt() {
+	# OpenFirmware/Das U-Boot device-tree
+	test -d /proc/device-tree
+}
+
+check_vm_dt() {
+	case ${arch}
+	in
+		(arm|aarch64)
+			if test -r /proc/device-tree/hypervisor/compatible
+			then
+				if grep -q -F 'xen' /proc/device-tree/hypervisor/compatible
+				then
+					echo xen
+					return 0
+				elif grep -q -F 'vmware' /proc/device-tree/hypervisor/compatible
+				then
+					# e.g. VMware ESXi on ARM
+					echo vmware
+					return 0
+				fi
+			fi
+			if glob_exists /proc/device-tree/fw-cfg@*/compatible
+			then
+				# qemu,fw-cfg-mmio
+				sed -e 's/,.*$//' /proc/device-tree/fw-cfg@*/compatible | head -n1
+				return 0
+			fi
+			if grep -q -F 'dummy-virt' /proc/device-tree/compatible
+			then
+				echo lkvm
+				return 0
+			fi
+			;;
+		(ppc64*)
+			if test -d /proc/device-tree/hypervisor \
+				&& grep -qF 'linux,kvm' /proc/device-tree/hypervisor/compatible
+			then
+				# We are running as a spapr KVM guest on ppc64
+				echo kvm-spapr
+				return 0
+			fi
+			if test -r /proc/device-tree/ibm,partition-name \
+				&& test -r /proc/device-tree/hmc-managed\? \
+				&& test -r /proc/device-tree/chosen/qemu,graphic-width
+			then
+				echo powervm
+			fi
+			;;
+	esac
+	return 1
+}
+
+has_vm_sys_hypervisor() {
+	test -d /sys/hypervisor/
+}
+
+check_vm_sys_hypervisor() {
+	test -r /sys/hypervisor/type &&
+	case $(head -n1 /sys/hypervisor/type)
+	in
+		(xen)
+			# Ordinary kernel with pv_ops.	There does not seem to be
+			# enough information at present to tell whether this is dom0
+			# or domU.
+			echo xen
+			return 0
+			;;
+	esac
+	return 1
+}
+
+check_vm_os_specific() {
+	_hyp_generic=false
+
+	case ${uname_s}
+	in
+		(Darwin)
+			if hv_vmm_present=$(get_sysctl kern.hv_vmm_present) \
+				&& test "${hv_vmm_present}" -ne 0
+			then
+				_hyp_generic=true
+			fi
+			;;
+		(FreeBSD)
+			# FreeBSD does not have /proc/cpuinfo even when procfs is used.
+			# Instead there is a sysctl kern.vm_guest.
+			# Which is 'none' if physical, else the virtualisation.
+			vm_guest=$(get_sysctl kern.vm_guest | tolower) &&
+			case ${vm_guest}
+			in
+				(none) ;;
+				(generic) _hyp_generic=true ;;
+				(*)
+					# kernel could detect hypervisor
+					case ${vm_guest}
+					in
+						(hv) echo hyperv ;;
+						(vbox) echo virtualbox ;;
+						(*) echo "${vm_guest}" ;;
+					esac
+					return 0
+					;;
+			esac
+			;;
+		(NetBSD)
+			machdep_hv=$(get_sysctl machdep.hypervisor | tolower) &&
+			case ${machdep_hv}
+			in
+				(none) ;;
+				(generic) _hyp_generic=true ;;
+				(*)
+					# kernel could detect hypervisor
+					case ${machdep_hv}
+					in
+						(hyper-v) echo hyperv ;;
+						(xenhvm*) echo xen-hvm ;;
+						(xenpv*) echo xen-pv ;;
+						(xen*) echo xen ;;
+						(*) echo "${machdep_hv}" ;;
+					esac
+					return 0
+					;;
+			esac
+			;;
+		(OpenBSD)
+			if is_command hostctl && glob_exists /dev/pvbus[0-9]*
+			then
+				for _pvbus in /dev/pvbus[0-9]*
+				do
+					_h_out=$(hostctl -f "${_pvbus}" -t 2>/dev/null) || continue
+					case $(expr "${_h_out}" : '[^:]*: *\(.*\)$')
+					in
+						(KVM) echo kvm ;;
+						(Hyper-V) echo hyperv ;;
+						(VMware) echo vmware ;;
+						(Xen) echo xen ;;
+						(bhyve) echo bhyve ;;
+						(OpenBSD) echo openbsd_vmm ;;
+					esac
+					return 0
+				done
+			fi
+			;;
+		(SunOS)
+			diag_conf=$(prtdiag | sed -n -e 's/.*Configuration: *//p' -e '/^$/q')
+			# NOTE: Don't use -e or -F in Solaris grep
+			if printf '%s\n' "${diag_conf}" | grep -q -i QEMU
+			then
+				echo qemu
+				return 0
+			elif printf '%s\n' "${diag_conf}" | grep -q -i VMware
+			then
+				echo vmware
+				return 0
+			fi
+			;;
+		(Linux)
+			if is_command dmesg
+			then
+				while read -r line
+				do
+					case ${line}
+					in
+						('Booting paravirtualized kernel on ')
+							case $(expr "${line}" : '.* kernel on \(.*\)')
+							in
+								('Xen')
+									echo xen-pv; return 0 ;;
+								('bare hardware')
+									return 1 ;;
+							esac
+							;;
+						('Hypervisor detected')
+							case $(expr "${line}" : '.*: *\(.*\)')
+							in
+								('ACRN')
+									echo acrn ;;
+								('Jailhouse')
+									echo jailhouse ;;
+								('KVM')
+									echo kvm ;;
+								('Microsoft Hyper-V')
+									echo hyperv ;;
+								('VMware')
+									echo vmware ;;
+								('Xen HVM')
+									echo xen-hvm ;;
+								('Xen PV')
+									echo xen-pv ;;
+							esac
+							return 0
+							;;
+						(lpar:*' under hypervisor')
+							return 0 ;;
+					esac
+				done <<-EOF
+				$(dmesg 2>/dev/null | awk '
+				/Booting paravirtualized kernel on /
+				/Hypervisor detected: /
+				/lpar: .* under hypervisor/
+				')
+				EOF
+			fi
+	esac
+
+	# Try to guess hypervisor based on CPU model (sysctl hw.model if available)
+	if cpu_model=$(get_sysctl hw.model)
+	then
+		guess_hypervisor_from_cpu_model "${cpu_model}" && return 0
+	fi
+
+	if ${_hyp_generic}
+	then
+		# cannot say which hypervisor, but one was detected
+		return 0
+	else
+		return 1
+	fi
+}
+
+run_stage() {
+	if type "has_$1_$2" >/dev/null 2>&1
+	then
+		"has_$1_$2"
+	else
+		true
+	fi \
+	&& "check_$1_$2"
+}
+
+
+# Execute container stages
+
+for stage in \
+	pid_1 cgroup files os_specific
+do
+	ctengine=$(run_stage ct ${stage}) || continue
+	is_contained=true
+	if test -n "${ctengine}"
+	then
+		echo container on "${ctengine}"
+		break
+	fi
+done
+if ${is_contained:-false} && test -z "${ctengine}"
+then
+	# none of the stages could determine the specific container engine, but
+	# we are running in some container.
+	echo container
+fi
+
+
+# Execute virtual machine / hypervisor stages
+
+for stage in \
+	os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
+do
+	hypervisor=$(run_stage vm ${stage}) || continue
+	is_virtual=true
+	if test -n "${hypervisor}"
+	then
+		echo virtual by "${hypervisor}"
+		break
+	fi
+done
+if ${is_virtual:-false} && test -z "${hypervisor}"
+then
+	# none of the stages could determine the specific hypervisor, but
+	# we are virtual.
+	echo virtual
+fi

From abc6d009b21b0d1ce3fc5107201e30740f127200 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 19:29:41 +0200
Subject: [PATCH 491/527] [explorer/machine_type] Print top most machine layer
 as first line (fallback to physical)

---
 cdist/conf/explorer/machine_type | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 29f98849..90c441da 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -26,17 +26,19 @@
 # VERB does not have a special meaning, it is just for better readability.
 #
 # e.g.
+# container
 # container on lxc
 # virtual by kvm-spapr
 #
-# The third word of each line can be composed of different parts concatenated with a `-'
-# (minus) character, with each component being a specification of the previous,
-# e.g.:
+# The third word of each line (except the first) can be composed of different
+# parts concatenated with a `-' (minus) character, with each component being
+# a specification of the previous, e.g.:
 #  - lxc-libvirt             (LXC container, managed by libvirt)
 #  - lpar-s390 / lpar-power  (LPAR running on IBM S/390 or POWER, respectively)
 #  - xen-hvm / xen-pv        (Xen HVM vs para-virtualization)
 #
-# If this explorer cannot determine any information it will print nothing.
+# If this explorer cannot collect enough information about virtualization it
+# will fall back to 'physical'.
 #
 
 # Add /sbin and /usr/sbin to the path so we can find system
@@ -121,6 +123,10 @@ get_sysctl() {
 	is_command sysctl && sysctl -n "$1" 2>/dev/null
 }
 
+detected_layer() {
+	test -n "${_toplayer:-}" || echo "${_toplayer:=${1:?}}"
+}
+
 
 # Check for container
 
@@ -895,6 +901,7 @@ for stage in \
 	pid_1 cgroup files os_specific
 do
 	ctengine=$(run_stage ct ${stage}) || continue
+	detected_layer 'container'
 	is_contained=true
 	if test -n "${ctengine}"
 	then
@@ -916,6 +923,7 @@ for stage in \
 	os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
 do
 	hypervisor=$(run_stage vm ${stage}) || continue
+	detected_layer 'virtual machine'
 	is_virtual=true
 	if test -n "${hypervisor}"
 	then
@@ -929,3 +937,8 @@ then
 	# we are virtual.
 	echo virtual
 fi
+
+
+# Fallback
+
+detected_layer physical

From 2ffa895f578f12d615ed65b9a93f3e2ae07f1d07 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 19:57:24 +0200
Subject: [PATCH 492/527] [explorer/machine_type] Remove CPUID check

it's a lot of code and depends on a binary helper unlikely to be installed.
---
 cdist/conf/explorer/machine_type | 59 --------------------------------
 1 file changed, 59 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 90c441da..7ce035e3 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -379,65 +379,6 @@ check_vm_arch_specific() {
 			fi
 			;;
 		(i?86|x86*|amd64|i86pc)
-			# Check CPUID
-			#
-			# Many fullvirt hypervisors give an indication through CPUID.  Use
-			# the virt-what helper program to get this information if available.
-
-			for CPUID_HELPER in \
-				$(command -v virt-what-cpuid-helper 2>/dev/null) \
-				/usr/lib/x86_64-*/virt-what-cpuid-helper \
-				/usr/lib/i?86-*/virt-what-cpuid-helper \
-				/usr/lib/virt-what/virt-what-cpuid-helper
-			do
-				if test -x "${CPUID_HELPER:?}"; then break; fi
-			done
-
-			if test -x "${CPUID_HELPER-}"
-			then
-				case $(command "${CPUID_HELPER}")
-				in
-					('bhyve bhyve ')
-						echo bhyve
-						;;
-					('LKVMLKVMLKVM')
-						echo lkvm
-						;;
-					('KVMKVMKVM')
-						echo kvm
-						;;
-					('TCGTCGTCGTCG')
-						echo qemu-tcg
-						;;
-					('Microsoft Hv')
-						# http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx
-						echo hyperv
-						;;
-					('OpenBSDVMM58')
-						# OpenBSD/VMM
-						echo openbsd_vmm
-						;;
-					('VMwareVMware')
-						# check added by Chetan Loke.
-						echo vmware
-						;;
-					('XenVMMXenVMM')
-						if has dmi
-						then
-							# https://access.redhat.com/solutions/222903
-							echo xen-hvm
-						else
-							echo xen-paravirt
-						fi
-						;;
-					(*)
-						return 1 ;;
-				esac
-				return 0
-			fi
-
-			unset CPUID_HELPER
-
 			# VMM CPUID flag denotes that this system is running under a VMM
 			if is_oneof "${uname_s}" Darwin
 			then

From 23fbfaf0352eadaabd43504d3ee074bb9f696fcd Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 21:29:24 +0200
Subject: [PATCH 493/527] [explorer/machine_type] Use systemd-detect-virt (if
 available) to detect containers and VMs

---
 cdist/conf/explorer/machine_type | 49 ++++++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 7ce035e3..1a38fda0 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -130,6 +130,25 @@ detected_layer() {
 
 # Check for container
 
+has_ct_systemd() {
+	is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^  -c'
+}
+
+check_ct_systemd() (
+	_ctengine=$(systemd-detect-virt -c 2>/dev/null) &&
+	case ${_ctengine}
+	in
+		(''|'none')
+			return 1 ;;
+		('container-other')
+			return 0 ;;
+		('systemd-nspawn')
+			echo systemd_nspawn ;;
+		(*)
+			echo "${_ctengine}" ;;
+	esac
+)
+
 has_ct_pid_1() {
 	test -r /run/systemd/container -o -r /proc/1/environ
 }
@@ -267,6 +286,32 @@ guess_hypervisor_from_cpu_model() {
 	esac
 }
 
+has_vm_systemd() {
+	is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^  -v'
+}
+
+check_vm_systemd() (
+	_hypervisor=$(systemd-detect-virt -v 2>/dev/null) &&
+	case ${_hypervisor}
+	in
+		(''|'none')
+			return 1 ;;
+		('amazon')
+			echo aws ;;
+		('bochs')
+			echo kvm ;;
+		('microsoft')
+			# assumption
+			echo hyperv ;;
+		('oracle')
+			echo virtualbox ;;
+		('vm-other')
+			return 0 ;;
+		(*)
+			echo "${_hypervisor}" ;;
+	esac
+)
+
 has_vm_cpuinfo() { has_cpuinfo; }
 
 check_vm_cpuinfo() {
@@ -839,7 +884,7 @@ run_stage() {
 # Execute container stages
 
 for stage in \
-	pid_1 cgroup files os_specific
+	systemd pid_1 cgroup files os_specific
 do
 	ctengine=$(run_stage ct ${stage}) || continue
 	detected_layer 'container'
@@ -861,7 +906,7 @@ fi
 # Execute virtual machine / hypervisor stages
 
 for stage in \
-	os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
+	systemd os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
 do
 	hypervisor=$(run_stage vm ${stage}) || continue
 	detected_layer 'virtual machine'

From 4a05669765c2c00c19af0ef1b607b0f2efa10f42 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 22:01:28 +0200
Subject: [PATCH 494/527] [explorer/machine_type] Implement chroot detection

---
 cdist/conf/explorer/machine_type | 53 ++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 1a38fda0..10c914ba 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -53,6 +53,21 @@ uname_s=$(uname -s)
 
 is_command() { command -v "$1" >/dev/null 2>&1; }
 
+files_same() {
+	# shellcheck disable=SC2012
+	LC_ALL=C df -P "$1" "$2" 2>/dev/null | {
+		read -r _  # skip header line
+		read -r fs1 _ _ _ _ mp1
+		read -r fs2 _ _ _ _ mp2
+		test "${fs1}" = "${fs2}" -a "${mp1}" = "${mp2}" || return 1
+	} &&
+	ls -1Ldi "$1" "$2" 2>/dev/null | {
+		read -r ino1 _
+		read -r ino2 _
+		test "${ino1}" = "${ino2}" || return 1
+	}
+}
+
 is_oneof() (
 	x=$1; shift
 	for y
@@ -128,6 +143,32 @@ detected_layer() {
 }
 
 
+# Check for chroot
+
+has_chroot_systemd() {
+	is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^  -r'
+}
+
+check_chroot_systemd() {
+	systemd-detect-virt -r
+}
+
+has_chroot_debian_ischroot() {
+	is_command ischroot
+}
+
+check_chroot_debian_ischroot() {
+	ischroot --default-false
+}
+
+has_chroot_procfs() {
+	test -d /proc/
+}
+
+check_chroot_procfs() {
+	test -e /proc/1/root && ! files_same /proc/1/root /
+}
+
 # Check for container
 
 has_ct_systemd() {
@@ -881,6 +922,18 @@ run_stage() {
 }
 
 
+# Execute chroot stages
+
+for stage in \
+	procfs debian_ischroot systemd
+do
+	run_stage chroot ${stage} || continue
+	detected_layer 'chroot'
+	echo chroot
+	break
+done
+
+
 # Execute container stages
 
 for stage in \

From 5af1317c2969995871da1d17b951de60dbc3fd09 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sun, 1 Aug 2021 16:40:20 +0200
Subject: [PATCH 495/527] [explorer/machine_type] Try to detect chroot path

---
 cdist/conf/explorer/machine_type | 38 ++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 10c914ba..fa68ec4d 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -166,7 +166,28 @@ has_chroot_procfs() {
 }
 
 check_chroot_procfs() {
-	test -e /proc/1/root && ! files_same /proc/1/root /
+	if test -e /proc/1/root && ! files_same /proc/1/root /
+	then
+		# try to determine where the chroot has been mounted
+		(
+			rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}')
+
+			if test -e "${rootdev}"
+			then
+				# escape chroot to determine where the device containing the
+				# chroot's / is mounted
+				rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}')
+
+				# shellcheck disable=SC2012
+				root_ino=$(ls -1di / | awk '{print $1}')
+
+				# Get mount point
+				chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}"
+			fi
+		)
+		return 0
+	fi
+	return 1
 }
 
 # Check for container
@@ -927,11 +948,20 @@ run_stage() {
 for stage in \
 	procfs debian_ischroot systemd
 do
-	run_stage chroot ${stage} || continue
+	chrootpnt=$(run_stage chroot ${stage}) || continue
+	is_chrooted=true
 	detected_layer 'chroot'
-	echo chroot
-	break
+	if test -n "${chrootpnt}"
+	then
+		echo chroot at "${chrootpnt}"
+		break
+	fi
 done
+if ${is_chrooted:-false} && test -z "${chrootpnt}"
+then
+	# could determine chroot, but not its mount point
+	echo chroot
+fi
 
 
 # Execute container stages

From 05c2a62191f533ceff1e074df571278a103d75ab Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sun, 1 Aug 2021 23:09:02 +0200
Subject: [PATCH 496/527] [explorer/machine_type] Implement chroot detection
 using /proc/.../mountinfo

---
 cdist/conf/explorer/machine_type | 64 +++++++++++++++++++++-----------
 1 file changed, 43 insertions(+), 21 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index fa68ec4d..00646c75 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -165,30 +165,52 @@ has_chroot_procfs() {
 	test -d /proc/
 }
 
-check_chroot_procfs() {
+check_chroot_procfs() (
+	is_chroot=false  # default
 	if test -e /proc/1/root && ! files_same /proc/1/root /
 	then
-		# try to determine where the chroot has been mounted
-		(
-			rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}')
-
-			if test -e "${rootdev}"
-			then
-				# escape chroot to determine where the device containing the
-				# chroot's / is mounted
-				rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}')
-
-				# shellcheck disable=SC2012
-				root_ino=$(ls -1di / | awk '{print $1}')
-
-				# Get mount point
-				chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}"
-			fi
-		)
-		return 0
+		is_chroot=true
 	fi
-	return 1
-}
+	if test -e /proc/1/mountinfo -a -e /proc/self/mountinfo
+	then
+		has_mountinfo=true
+		cmp -s /proc/1/mountinfo /proc/self/mountinfo || is_chroot=true
+	fi
+
+	if ${is_chroot}
+	then
+		# try to determine where the chroot has been mounted
+		rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}')
+
+		if test -e "${rootdev}"
+		then
+			# escape chroot to determine where the device containing the
+			# chroot's / is mounted
+			rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}')
+
+			# shellcheck disable=SC2012
+			root_ino=$(ls -1di / | awk '{print $1}')
+
+			# escape chroot and find mount point by inode
+			chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}"
+		elif ${has_mountinfo}
+		then
+			while read -r mntid _ _ _ cmntpnt _
+			do
+				read -r _ _ _ _ hmntpnt _ <<-EOF
+				$(grep -e "^$((mntid)) " /proc/1/mountinfo)
+				EOF
+				printf '%s\n' "${hmntpnt%${cmntpnt}}"
+			done </proc/self/mountinfo \
+			| sort -u \
+			| head -n 1  # just take the first...
+		fi
+
+		return 0
+	else
+		return 1
+	fi
+)
 
 # Check for container
 

From 67f85546ec3bf865d368f268fdbbc121ccfd5742 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 23 Aug 2021 09:57:20 +0300
Subject: [PATCH 497/527] [explorer/os_version] add new debian code names:
 bookworm and trixie

---
 cdist/conf/explorer/os_version | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 7bc6dd6b..aea3c43f 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -54,6 +54,8 @@ in
               # sid versions don't have a number, so we decode by codename:
               case $(expr "$debian_version" : '\([a-z]\{1,\}\)/')
               in
+                  trixie) echo 12.99 ;;
+                  bookworm) echo 11.99 ;;
                   bullseye) echo 10.99 ;;
                   buster) echo 9.99 ;;
                   stretch) echo 8.99 ;;

From e1e134899811e51e4694e89e421790e41cb894bf Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 23 Aug 2021 10:44:48 +0300
Subject: [PATCH 498/527] [explorer/os_version] use 99.99 as fallback for
 unknown code names in */sid

---
 cdist/conf/explorer/os_version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index aea3c43f..bbc9e4f0 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -63,7 +63,7 @@ in
                   wheezy) echo 6.99 ;;
                   squeeze) echo 5.99 ;;
                   lenny) echo 4.99 ;;
-                  *) exit 1
+                  *) echo 99.99 ;;
               esac
               ;;
           *)

From 46ed48d546af6ff663b131c803a9027b19798ba1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 08:09:47 +0200
Subject: [PATCH 499/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index f9409d7e..8507c663 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,7 @@ next:
 	* Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera)
 	* Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera)
 	* Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera)
+	* Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 0546283d0ebc0058dbff68cec9d18b9281c26edd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 20:32:44 +0200
Subject: [PATCH 500/527] Update shellcheck disable

---
 cdist/conf/type/__apt_backports/manifest             | 1 +
 cdist/conf/type/__package_pkg_freebsd/gencode-remote | 1 +
 cdist/conf/type/__ssh_authorized_keys/explorer/keys  | 1 +
 3 files changed, 3 insertions(+)

diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index bc47d8de..6fcd9212 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -28,6 +28,7 @@
 #  lsb_release may not be given in all installations
 codename_os_release() {
     # shellcheck disable=SC1090
+    # shellcheck disable=SC1091
     . "$__global/explorer/os_release"
     printf "%s" "$VERSION_CODENAME"
 }
diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote
index 3f88f6bc..ca9aa45a 100755
--- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote
+++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote
@@ -37,6 +37,7 @@ assert ()                 #  If condition false,
 	then
 		echo "Assertion failed:  \"$1\""
 		# shellcheck disable=SC2039
+		# shellcheck disable=SC3044
 		echo "File \"$0\", line $lineno, called by $(caller 0)"
 		exit $E_ASSERT_FAILED
 	fi
diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
index cec25746..9694a64b 100755
--- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys
+++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 
 # shellcheck disable=SC1090
+# shellcheck disable=SC1091
 file="$( . "$__type_explorer/file" )"
 
 if [ -f "$file" ]

From 44741e714b16f7a00bf84bd54ae33eacc7593192 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 20:25:49 +0200
Subject: [PATCH 501/527] Release 6.9.8

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 8507c663..dcdc4b3d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)

From b8eb6e984c1638e8e167394c6b3aa482cb8aad49 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 20:47:50 +0200
Subject: [PATCH 502/527] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index dcdc4b3d..693d028f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Explorer machine_type: Rewrite (Dennis Camera)
+
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)

From d7fdc8006f747be86b64a2790266b50219bab507 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 21:54:45 +0300
Subject: [PATCH 503/527] allow empty file

---
 cdist/conf/type/__sed/explorer/file | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
index a40f71b3..5f0f4edd 100755
--- a/cdist/conf/type/__sed/explorer/file
+++ b/cdist/conf/type/__sed/explorer/file
@@ -9,13 +9,7 @@ fi
 
 if [ -f "$file" ]
 then
-    if [ -s "$file" ]
-    then
-        cat "$file"
-    else
-        echo "$file is empty" >&2
-        exit 1
-    fi
+    cat "$file"
 else
     echo "$file do not exist" >&2
     exit 1

From 0f6e48dbc657a9d9145b965eacf79f021a1419f5 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:24:26 +0300
Subject: [PATCH 504/527] use $__object/tempfile in target instead of mktemp,
 add comments

---
 cdist/conf/type/__sed/gencode-remote | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 836506f0..df929b42 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -35,8 +35,11 @@ then
     exit 0
 fi
 
+# we can't use -i, because it's not posix, so we fly with tempfile and cp
+# and we use cp because we want to preserve destination file's attributes
+
 # shellcheck disable=SC2016
-echo 'tmp="$( mktemp )"'
+echo 'tmp="$__object/tempfile"'
 
 echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
 

From 90488fcebcc00f3b4ea260867f8b31625df61e46 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:27:42 +0300
Subject: [PATCH 505/527] use -e

---
 cdist/conf/type/__sed/explorer/file | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
index 5f0f4edd..9262b613 100755
--- a/cdist/conf/type/__sed/explorer/file
+++ b/cdist/conf/type/__sed/explorer/file
@@ -7,10 +7,10 @@ else
     file="/$__object_id"
 fi
 
-if [ -f "$file" ]
+if [ ! -e "$file" ]
 then
-    cat "$file"
-else
     echo "$file do not exist" >&2
     exit 1
 fi
+
+cat "$file"

From b7f392fa371ff78591abb910f7e224e7587c81be Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:38:55 +0300
Subject: [PATCH 506/527] use -E for better compat (not really sure if it is
 posix at all)

---
 cdist/conf/type/__sed/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index df929b42..625448e6 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -24,7 +24,7 @@ sed_cmd='sed'
 
 if [ -f "$__object/parameter/regexp-extended" ]
 then
-    sed_cmd="$sed_cmd --regexp-extended"
+    sed_cmd="$sed_cmd -E"
 fi
 
 if echo "$script" \

From aabef7f44a98ad144a7e1fe68bfb562079916ef8 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:40:06 +0300
Subject: [PATCH 507/527] remove reading script from file

---
 cdist/conf/type/__sed/gencode-remote | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 625448e6..be7690e1 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -12,10 +12,6 @@ script="$( cat "$__object/parameter/script" )"
 if [ "$script" = '-' ]
 then
     script="$( cat "$__object/stdin" )"
-elif
-    [ -f "$script" ]
-then
-    script="$( cat "$script" )"
 fi
 
 file_from_target="$__object/explorer/file"

From 5bf0c71e7af49ff6e7c1cccec192f763a019503e Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:45:36 +0300
Subject: [PATCH 508/527] update man

---
 cdist/conf/type/__sed/man.rst | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst
index 4a728184..86789363 100644
--- a/cdist/conf/type/__sed/man.rst
+++ b/cdist/conf/type/__sed/man.rst
@@ -8,35 +8,40 @@ cdist-type__sed - Transform text files with ``sed``
 
 DESCRIPTION
 -----------
-TODO
+Transform text files with ``sed``.
 
 
 REQUIRED MULTIPLE PARAMETERS
 ----------------------------
 script
-   TODO
+   ``sed`` script.
+   If ``-`` then the script is read from ``stdin``.
 
 
 OPTIONAL PARAMETERS
 -------------------
 file
-   TODO
+   Path to the file. Defaults to ``$__object_id``.
 
 onchange
-   TODO
+   Execute this command if ``sed`` changes file.
 
 
 BOOLEAN PARAMETERS
 ------------------
 regexp-extended
-   TODO
+   Use extended regular expressions in the script.
+   Might not be supported with every ``sed`` version.
 
 
 EXAMPLES
 --------
+
 .. code-block:: sh
 
-   true
+   __sed /tmp/foobar --script 's/foo/bar/'
+
+   echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script -
 
 
 AUTHORS

From cd4acde67ec9c3967e6e08450734cf65040e787f Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 15 Sep 2021 09:22:27 +0300
Subject: [PATCH 509/527] grammar

---
 cdist/conf/type/__sed/explorer/file | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
index 9262b613..ec3d0fe8 100755
--- a/cdist/conf/type/__sed/explorer/file
+++ b/cdist/conf/type/__sed/explorer/file
@@ -9,7 +9,7 @@ fi
 
 if [ ! -e "$file" ]
 then
-    echo "$file do not exist" >&2
+    echo "$file does not exist" >&2
     exit 1
 fi
 

From 7b6789ddeb74450f70991efe957e3eb0f11043f6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 15:04:12 +0200
Subject: [PATCH 510/527] __package_update_index: fix complain about suite
 change

2 of 4th fix for ticket #861
---
 .../type/__package_update_index/gencode-remote     | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote
index 803468b5..a10c16d3 100755
--- a/cdist/conf/type/__package_update_index/gencode-remote
+++ b/cdist/conf/type/__package_update_index/gencode-remote
@@ -41,7 +41,19 @@ fi
 case "$type" in
     yum) ;;
     apt)
-        echo "apt-get --quiet update"
+        # There are special arguments to apt(8) to prevent aborts if apt woudn't been
+        # updated after the 19th April 2021 till the bullseye release. The additional
+        # arguments acknoledge the happend suite change (the apt(8) update does the
+        # same by itself).
+        #
+        # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+        # allows backward compatablility to pre-buster Debian versions.
+        #
+        # See more: ticket #861
+        # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+        apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
+        echo "apt-get --quiet $apt_opts update"
         echo "apt-cache updated (age was: $currage)" >> "$__messages_out"
         ;;
     pacman)

From 12787ffe2c48f5c6d534d10f8faadde30db477ac Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 15:12:20 +0200
Subject: [PATCH 511/527] __apt_source: fix complain about suite change

3 of 4th fix for ticket #861
---
 cdist/conf/type/__apt_source/gencode-remote | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_source/gencode-remote b/cdist/conf/type/__apt_source/gencode-remote
index 1e8592c6..973b0f6c 100755
--- a/cdist/conf/type/__apt_source/gencode-remote
+++ b/cdist/conf/type/__apt_source/gencode-remote
@@ -22,7 +22,21 @@
 name="$__object_id"
 destination="/etc/apt/sources.list.d/${name}.list"
 
+# There are special arguments to apt(8) to prevent aborts if apt woudn't been
+# updated after the 19th April 2021 till the bullseye release. The additional
+# arguments acknoledge the happend suite change (the apt(8) update does the
+# same by itself).
+#
+# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+# allows backward compatablility to pre-buster Debian versions.
+#
+# See more: ticket #861
+# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
+# run 'apt-get update' only if something changed with our sources.list file
+#   it will be run a second time on error as a redundancy messure to success
 if grep -q "^__file${destination}" "$__messages_in"; then
-   printf 'apt-get update || apt-get update\n'
+   printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts"
 fi
 

From d246e067109dc8b85ea89668f742affbb221d1a6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 14:49:23 +0200
Subject: [PATCH 512/527] __apt_update_index: fix complain about suite change

1 of 4th fix for ticket #861
---
 .../conf/type/__apt_update_index/gencode-remote  | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_update_index/gencode-remote b/cdist/conf/type/__apt_update_index/gencode-remote
index 70b59710..2d7f9030 100755
--- a/cdist/conf/type/__apt_update_index/gencode-remote
+++ b/cdist/conf/type/__apt_update_index/gencode-remote
@@ -18,9 +18,23 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+
+# There are special arguments to apt(8) to prevent aborts if apt woudn't been
+# updated after the 19th April 2021 till the bullseye release. The additional
+# arguments acknoledge the happend suite change (the apt(8) update does the
+# same by itself).
+#
+# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+# allows backward compatablility to pre-buster Debian versions.
+#
+# See more: ticket #861
+# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
 # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists
+#   it will be run a second time on error as a redundancy messure to success
 cat << DONE
 if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then
-   apt-get update || apt-get update
+   apt-get $apt_opts update || apt-get $apt_opts update
 fi
 DONE

From 3d7b31cbb43067271c2c510281f90f19dc5ec5fb Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 15:22:16 +0200
Subject: [PATCH 513/527] __package_apt: fix complain about suite change

the last fix for ticket #861 :-)
---
 cdist/conf/type/__package_apt/gencode-remote | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote
index fbfca330..79c0d9d3 100755
--- a/cdist/conf/type/__package_apt/gencode-remote
+++ b/cdist/conf/type/__package_apt/gencode-remote
@@ -81,12 +81,24 @@ aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::=
 
 case "$state_should" in
     present)
+        # There are special arguments to apt(8) to prevent aborts if apt woudn't been
+        # updated after the 19th April 2021 till the bullseye release. The additional
+        # arguments acknoledge the happend suite change (the apt(8) update does the
+        # same by itself).
+        #
+        # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+        # allows backward compatablility to pre-buster Debian versions.
+        #
+        # See more: ticket #861
+        # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+        apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
         # following is bit ugly, but important hack.
         # due to how cdist config run works, there isn't
         # currently better way to do it :(
         cat << EOF
 if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ]
-then echo apt-get update > /dev/null 2>&1 || true
+then echo apt-get $apt_opts update > /dev/null 2>&1 || true
 fi
 EOF
         if [ -n "$version" ]; then

From 72ff48154c958ebdedc8b6d919b6f372efa63be2 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 16 Sep 2021 21:36:39 +0300
Subject: [PATCH 514/527] add comments, add -u to diff

---
 cdist/conf/type/__sed/gencode-remote | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index be7690e1..f99c5a88 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -14,6 +14,8 @@ then
     script="$( cat "$__object/stdin" )"
 fi
 
+# since stdin is not available in explorer, we pull file from target with explorer
+
 file_from_target="$__object/explorer/file"
 
 sed_cmd='sed'
@@ -23,10 +25,10 @@ then
     sed_cmd="$sed_cmd -E"
 fi
 
-if echo "$script" \
-    | "$sed_cmd" -f - "$file_from_target" \
-    | diff "$file_from_target" - \
-    > /dev/null
+# do sed dry run, diff result and if no change, then there's nothing to do
+# also redirect diff's output to stderr for debugging purposes
+
+if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2
 then
     exit 0
 fi

From bf222d05435000063930ff199bd0dec94d7d8cd5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 21 Sep 2021 08:55:54 +0200
Subject: [PATCH 515/527] ++changelog

---
 docs/changelog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 693d028f..ab637f5f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,11 @@ Changelog
 
 next:
 	* Explorer machine_type: Rewrite (Dennis Camera)
+	* New type: __sed (Ander Punnar)
+	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From 15c642a9b7af42c368eb5a8e22cab5f07b39b95e Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 12:06:45 +0200
Subject: [PATCH 516/527] [__debconf_set_selections] Fix --file not being
 supported

Even if deprecated, the parameter *must* be supported, which isn't the case
right now.

This was due to a misunderstanding of how deprecating parameters work, see:
https://www.cdi.st/manual/latest/cdist-type.html#deprecated-parameters
---
 .../conf/type/__debconf_set_selections/parameter/deprecated/file | 1 +
 .../__debconf_set_selections/parameter/{deprecated => optional}  | 0
 2 files changed, 1 insertion(+)
 create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/deprecated/file
 rename cdist/conf/type/__debconf_set_selections/parameter/{deprecated => optional} (100%)

diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file
new file mode 100644
index 00000000..09db545a
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file
@@ -0,0 +1 @@
+'file' has been deprecated in favour of 'line' in order to provide idempotency.
diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated b/cdist/conf/type/__debconf_set_selections/parameter/optional
similarity index 100%
rename from cdist/conf/type/__debconf_set_selections/parameter/deprecated
rename to cdist/conf/type/__debconf_set_selections/parameter/optional

From 5b7cca99f73e4ec3dd5cdb90e4dbfd40ec8fe349 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 12:09:42 +0200
Subject: [PATCH 517/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index ab637f5f..142602c8 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
+	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From fc9bd40c9a34cf54e2d6cfd2fec089bd25e1b172 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 13:14:57 +0200
Subject: [PATCH 518/527] Improve bullseye support, perticularly
 __letsencrypt_cert

---
 cdist/conf/type/__grafana_dashboard/manifest | 2 +-
 cdist/conf/type/__letsencrypt_cert/manifest  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest
index d145c4c3..0d944482 100755
--- a/cdist/conf/type/__grafana_dashboard/manifest
+++ b/cdist/conf/type/__grafana_dashboard/manifest
@@ -15,7 +15,7 @@ case $os in
                 # Differntation not needed anymore
                 apt_source_distribution=stable
                 ;;
-            10*)
+            10*|11*)
                 # Differntation not needed anymore
                 apt_source_distribution=stable
                 ;;
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 6394f629..638a99e0 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -41,7 +41,7 @@ if [ -z "${certbot_fullpath}" ]; then
 					require="__apt_source/stretch-backports" __package_apt certbot \
 						--target-release stretch-backports
 				;;
-				10*)
+				10*|11*)
 					__package_apt certbot
 				;;
 

From 560374a6861281695b61c8c2298123be41d92326 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 13:16:11 +0200
Subject: [PATCH 519/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 142602c8..6f717cf4 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,7 @@ next:
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
+	* Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From c33d99ee120ad180daba144be80a25d77b473f56 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sun, 31 Oct 2021 17:38:10 +0100
Subject: [PATCH 520/527] [__haproxy_dualstack] New type with PROXY protocol
 support

This is backwards compatible with what is already used internally @ungleich, but
adds on top of that the ability to customise ports and, most importantly, it
adds PROXY protocol support.
---
 .../conf/type/__haproxy_dualstack/files/http  |   8 +
 .../conf/type/__haproxy_dualstack/files/https |  10 ++
 .../conf/type/__haproxy_dualstack/files/imaps |  12 ++
 .../conf/type/__haproxy_dualstack/files/smtps |  12 ++
 cdist/conf/type/__haproxy_dualstack/man.rst   | 121 ++++++++++++++
 cdist/conf/type/__haproxy_dualstack/manifest  | 155 ++++++++++++++++++
 .../parameter/default/protocol                |   1 +
 .../parameter/optional_multiple               |   3 +
 cdist/conf/type/__haproxy_dualstack/singleton |   0
 9 files changed, 322 insertions(+)
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/http
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/https
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/imaps
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/smtps
 create mode 100644 cdist/conf/type/__haproxy_dualstack/man.rst
 create mode 100644 cdist/conf/type/__haproxy_dualstack/manifest
 create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/default/protocol
 create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple
 create mode 100644 cdist/conf/type/__haproxy_dualstack/singleton

diff --git a/cdist/conf/type/__haproxy_dualstack/files/http b/cdist/conf/type/__haproxy_dualstack/files/http
new file mode 100644
index 00000000..0508a465
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/http
@@ -0,0 +1,8 @@
+frontend http
+	bind	BIND@:80
+	mode	http
+	option	httplog
+	default_backend	http
+
+backend http
+	mode	http
diff --git a/cdist/conf/type/__haproxy_dualstack/files/https b/cdist/conf/type/__haproxy_dualstack/files/https
new file mode 100644
index 00000000..73deac46
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/https
@@ -0,0 +1,10 @@
+frontend https
+	bind	BIND@:443
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	https
+
+backend https
+	mode	tcp
diff --git a/cdist/conf/type/__haproxy_dualstack/files/imaps b/cdist/conf/type/__haproxy_dualstack/files/imaps
new file mode 100644
index 00000000..b1ec3793
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/imaps
@@ -0,0 +1,12 @@
+frontend imaps
+	bind	BIND@:143
+	bind	BIND@:993
+
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	imaps
+
+backend imaps
+	mode	tcp
diff --git a/cdist/conf/type/__haproxy_dualstack/files/smtps b/cdist/conf/type/__haproxy_dualstack/files/smtps
new file mode 100644
index 00000000..dce6ed4a
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/smtps
@@ -0,0 +1,12 @@
+frontend smtps
+	bind	BIND@:25
+	bind	BIND@:465
+
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	smtps
+
+backend smtps
+	mode	tcp
diff --git a/cdist/conf/type/__haproxy_dualstack/man.rst b/cdist/conf/type/__haproxy_dualstack/man.rst
new file mode 100644
index 00000000..6c131cbe
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/man.rst
@@ -0,0 +1,121 @@
+cdist-type__haproxy_dualstack(7)
+================================
+
+
+NAME
+----
+cdist-type__haproxy_dualstack - Proxy services from a dual-stack server
+
+
+DESCRIPTION
+-----------
+This (singleton) type installs and configures haproxy to act as a dual-stack
+proxy for single-stack services.
+
+This can be useful to add IPv4 support to IPv6-only services while only using
+one IPv4 for many such services.
+
+By default this type uses the plain TCP proxy mode, which means that there is no
+need for TLS termination on this host when SNI is supported.
+This also means that proxied services will not receive the client's IP address,
+but will see the proxy's IP address instead (that of `$__target_host`).
+
+This can be solved by using the PROXY protocol, but do take into account that,
+e.g. nginx cannot serve both regular HTTP(S) and PROXY protocols on the same
+port, so you will need to use other ports for that.
+
+As a recommendation in this type: use TCP ports 8080 and 591 respectively to
+serve HTTP and HTTPS using the PROXY protocol.
+
+See the EXAMPLES for more details.
+
+
+OPTIONAL PARAMETERS
+-------------------
+v4proxy
+    Proxy incoming IPv4 connections to the equivalent IPv6 endpoint.
+    In its simplest use, it must be a NAME with an `AAAA` DNS entry, which is
+    the IP address actually providing the proxied services.
+    The full format of this argument is:
+    `[proxy:]NAME[[:PROTOCOL_1=PORT_1]...[:PROTOCOL_N=PORT_N]]`
+    Where starting with `proxy:` determines that the PROXY protocol must be
+    used and each `:PROTOCOL=PORT` (e.g. `:http=8080` or `:https=591`) is a PORT
+    override for the given PROTOCOL (see `--protocol`), if not present the
+    PROTOCOL's default port will be used.
+
+
+v6proxy
+    Proxy incoming IPv6 connections to the equivalent IPv4 endpoint.
+    In its simplest use, it must be a NAME with an `A` DNS entry, which is
+    the IP address actually providing the proxied services.
+    See `--v4proxy` for more options and details.
+
+protocol
+    Can be passed multiple times or as a space-separated list of protocols.
+    Currently supported protocols are: `http`, `https`, `imaps`, `smtps`.
+    This defaults to: `http https imaps smtps`.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Proxy the IPv6-only services so IPv4-only clients can access them
+    # This uses HAProxy's TCP mode for http, https, imaps and smtps
+    __haproxy_dualstack \
+        --v4proxy ipv6.chat \
+        --v4proxy matrix.ungleich.ch
+
+    # Proxy the IPv6-only HTTP(S) services so IPv4-only clients can access them
+    # Note this means that the backend IPv6-only server will only see
+    # the IPv6 address of the haproxy host managed by cdist, which can be
+    # troublesome if this information is relevant for analytics/security/...
+    # See the PROXY example below
+    __haproxy_dualstack \
+        --protocol http --protocol https \
+        --v4proxy ipv6.chat \
+        --v4proxy matrix.ungleich.ch
+
+    # Use the PROXY protocol to proxy the IPv6-only HTTP(S) services enabling
+    # IPv4-only clients to access them while maintaining the client's IP address
+    __haproxy_dualstack \
+        --protocol http --protocol https \
+        --v4proxy proxy:ipv6.chat:http=8080:https=591 \
+        --v4proxy proxy:matrix.ungleich.ch:http=8080:https=591
+    # Note however that the PROXY protocol is not compatible with regular
+    # HTTP(S) protocols, so your nginx will have to listen on different ports
+    # with the PROXY settings.
+    # Note that you will need to restrict access to the 8080 port to prevent
+    # Client IP spoofing.
+    # This can be something like:
+    # server {
+    #     # listen for regular HTTP connections
+    #     listen [::]:80 default_server;
+    #     listen 80 default_server;
+    #     # listen for PROXY HTTP connections
+    #     listen [::]:8080 proxy_protocol;
+    #     # Accept the Client's IP from the PROXY protocol
+    #     real_ip_header proxy_protocol;
+    # }
+
+
+SEE ALSO
+--------
+- https://www.haproxy.com/blog/enhanced-ssl-load-balancing-with-server-name-indication-sni-tls-extension/
+- https://www.haproxy.com/blog/haproxy/proxy-protocol/
+- https://docs.nginx.com/nginx/admin-guide/load-balancer/using-proxy-protocol/
+
+
+AUTHORS
+-------
+ungleich <foss--@--ungleich.ch>
+Evilham <cvs--@--evilham.com>
+
+
+COPYING
+-------
+Copyright \(C) 2021 ungleich glarus ag. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__haproxy_dualstack/manifest b/cdist/conf/type/__haproxy_dualstack/manifest
new file mode 100644
index 00000000..d110eea6
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/manifest
@@ -0,0 +1,155 @@
+#!/bin/sh -eu
+
+__package haproxy
+require="__package/haproxy" __start_on_boot haproxy
+
+tmpdir="$__object/files"
+mkdir "$tmpdir"
+configtmp="$__object/files/haproxy.cfg"
+
+os=$(cat "$__global/explorer/os")
+case $os in
+	freebsd)
+		CONFIG_FILE="/usr/local/etc/haproxy.conf"
+		cat <<EOF > "$configtmp"
+global
+	maxconn	4000
+	user	nobody
+	group	nogroup
+	daemon
+
+EOF
+
+		;;
+	*)
+		CONFIG_FILE="/etc/haproxy/haproxy.cfg"
+		cat <<EOF > "$configtmp"
+global
+	log	[::1] local2
+	chroot	/var/lib/haproxy
+	pidfile	/var/run/haproxy.pid
+	maxconn	4000
+	user	haproxy
+	group	haproxy
+	daemon
+
+	# turn on stats unix socket
+	stats socket	/var/lib/haproxy/stats
+
+EOF
+		;;
+esac
+
+cat <<EOF >> "$configtmp"
+defaults
+	retries	3
+	log	global
+	timeout http-request	10s
+	timeout queue		1m
+	timeout connect		10s
+	timeout client		1m
+	timeout server		1m
+	timeout http-keep-alive	10s
+	timeout check		10s
+EOF
+
+dig_cmd="$(command -v dig || true)"
+get_ip() {
+	# Usage: get_ip (ipv4|ipv6) NAME
+	# uses "dig" if available, else fallback to "host"
+	case $1 in
+		ipv4)
+			if [ -n "${dig_cmd}" ]; then
+				${dig_cmd} +short A "$2"
+			else
+				host -t A "$2" | cut -d ' ' -f 4 | grep -v 'found:'
+			fi
+			;;
+		ipv6)
+			if [ -n "${dig_cmd}" ]; then
+				${dig_cmd} +short AAAA "$2"
+			else
+				host -t AAAA "$2" | cut -d ' ' -f 5 | grep -v 'NXDOMAIN'
+			fi
+			;;
+	esac
+}
+
+PROTOCOLS="$(cat "$__object/parameter/protocol")"
+
+for proxy in v4proxy v6proxy; do
+	param=$__object/parameter/$proxy
+	# no backend? skip generating code
+	if [ ! -f "$param" ]; then
+		continue
+	fi
+
+	# turn backend name into bind parameter: v4backend -> ipv4@
+	bind=$(echo $proxy | sed -e 's/^/ip/' -e 's/proxy//')
+
+	case $bind in
+		ipv4)
+			backendproto=ipv6
+			;;
+		ipv6)
+			backendproto=ipv4
+			;;
+	esac
+
+	for proto in ${PROTOCOLS}; do
+		# Add protocol "header"
+		printf "\n# %s %s \n" "${bind}" "${proto}" >> "$configtmp"
+
+		sed -e "s/BIND/$bind/" \
+			-e "s/\(frontend[[:space:]].*\)/\1$bind/" \
+			-e "s/\(backend[[:space:]].*\)/\\1$bind/" \
+			"$__type/files/$proto" >> "$configtmp"
+
+		while read -r hostdefinition; do
+			if echo "$hostdefinition" | grep -qE '^proxy:'; then
+				# Proxy protocol was requested
+				host="$(echo "$hostdefinition" | sed -E 's/^proxy:([^:]+).*$/\1/')"
+				send_proxy=" send-proxy"
+			else
+				# Just use tcp proxy mode
+				host="$hostdefinition"
+				send_proxy=""
+			fi
+			if echo "$hostdefinition" | grep -qE ":${proto}="; then
+				# Use custom port definition if requested
+				port="$(echo "$hostdefinition" | sed -E "s/^(.*:)?${proto}=([0-9]+).*$/:\2/")"
+			else
+				# Else use the default
+				port=""
+			fi
+			servername=$host
+
+			res=$(get_ip "$bind" "$servername")
+
+			if [ -z "$res" ]; then
+				echo "$servername does not resolve - aborting config" >&2
+				exit 1
+			fi
+
+			# Treat protocols without TLS+SNI specially
+			if [ "$proto" = http ]; then
+				echo "	use-server $servername if { hdr(host) -i $host }" >> "$configtmp"
+			else
+				echo "	use-server $servername if { req_ssl_sni -i $host }" >> "$configtmp"
+			fi
+
+			# Create the "server" itself.
+			# Note that port and send_proxy will be empty unless
+			# they were requested by the type user
+			echo "	server $servername ${backendproto}@${host}${port}${send_proxy}" >> "$configtmp"
+
+		done < "$param"
+	done
+done
+
+# Create config file
+require="__package/haproxy" __file ${CONFIG_FILE} --source "$configtmp" --mode 0644
+
+require="__file${CONFIG_FILE}" __check_messages "haproxy_reload" \
+	--pattern "^__file${CONFIG_FILE}" \
+	--execute "service haproxy reload || service haproxy restart"
diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol
new file mode 100644
index 00000000..dc8bb7bf
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol
@@ -0,0 +1 @@
+http https imaps smtps
diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple
new file mode 100644
index 00000000..8c482bd4
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple
@@ -0,0 +1,3 @@
+protocol
+v4proxy
+v6proxy
diff --git a/cdist/conf/type/__haproxy_dualstack/singleton b/cdist/conf/type/__haproxy_dualstack/singleton
new file mode 100644
index 00000000..e69de29b

From e2500248f2ddc83129e77f2e6b8dffb64904dbae Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Wed, 3 Nov 2021 11:03:33 +0100
Subject: [PATCH 521/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 6f717cf4..99a8c08b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Explorer machine_type: Rewrite (Dennis Camera)
 	* New type: __sed (Ander Punnar)
+	* New type: __haproxy_dualstack (Evilham and ungleich)
 	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)

From 3a321469a8ba5aea55220bd70bd4900de732e917 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 16 Nov 2021 11:11:45 +0100
Subject: [PATCH 522/527] Python 3.10: collections.X -> collections.abc.X

---
 cdist/integration.py     | 2 +-
 cdist/util/fsproperty.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/integration.py b/cdist/integration.py
index 17b65f09..04470ea7 100644
--- a/cdist/integration.py
+++ b/cdist/integration.py
@@ -84,7 +84,7 @@ def _process_hosts_simple(action, host, manifest, verbose,
     """
     if isinstance(host, str):
         hosts = [host, ]
-    elif isinstance(host, collections.Iterable):
+    elif isinstance(host, collections.abc.Iterable):
         hosts = host
     else:
         raise cdist.Error('Invalid host argument: {}'.format(host))
diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py
index 09e9cc19..6bf935e8 100644
--- a/cdist/util/fsproperty.py
+++ b/cdist/util/fsproperty.py
@@ -33,7 +33,7 @@ class AbsolutePathRequiredError(cdist.Error):
         return 'Absolute path required, got: {}'.format(self.path)
 
 
-class FileList(collections.MutableSequence):
+class FileList(collections.abc.MutableSequence):
     """A list that stores it's state in a file.
 
     """
@@ -102,7 +102,7 @@ class FileList(collections.MutableSequence):
         self.__write(lines)
 
 
-class DirectoryDict(collections.MutableMapping):
+class DirectoryDict(collections.abc.MutableMapping):
     """A dict that stores it's items as files in a directory.
 
     """

From 6e3ad11ea0177865e7e0288b1765267df8d5d020 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Thu, 23 Dec 2021 20:07:28 +0100
Subject: [PATCH 523/527] [__package_upgrade_all] Add new --apt-with-new-pkgs
 argument

---
 cdist/conf/type/__package_upgrade_all/gencode-remote    | 6 +++++-
 cdist/conf/type/__package_upgrade_all/man.rst           | 8 ++++++++
 cdist/conf/type/__package_upgrade_all/parameter/boolean | 1 +
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote
index 38aa001e..d332e851 100755
--- a/cdist/conf/type/__package_upgrade_all/gencode-remote
+++ b/cdist/conf/type/__package_upgrade_all/gencode-remote
@@ -28,6 +28,10 @@ apt_clean="$__object/parameter/apt-clean"
 
 apt_dist_upgrade="$__object/parameter/apt-dist-upgrade"
 
+if [ -f "$__object/parameter/apt-with-new-pkgs" ]; then
+	apt_with_new_pkgs="--with-new-pkgs"
+fi
+
 if [ -f "$type" ]; then
     type="$(cat "$type")"
 else
@@ -54,7 +58,7 @@ case "$type" in
     apt)
         if [ -f "$apt_dist_upgrade" ]
         then echo "$aptget dist-upgrade"
-        else echo "$aptget upgrade"
+        else echo "$aptget $apt_with_new_pkgs upgrade"
         fi
 
         if [ -f "$apt_clean" ]
diff --git a/cdist/conf/type/__package_upgrade_all/man.rst b/cdist/conf/type/__package_upgrade_all/man.rst
index e9e2b8ce..0c116bac 100644
--- a/cdist/conf/type/__package_upgrade_all/man.rst
+++ b/cdist/conf/type/__package_upgrade_all/man.rst
@@ -33,6 +33,14 @@ BOOLEAN PARAMETERS
 apt-dist-upgrade
     Do dist-upgrade instead of upgrade.
 
+apt-with-new-pkg
+    Allow installing new packages when used in conjunction with
+    upgrade. This is useful if the update of an installed package
+    requires new dependencies to be installed. Instead of holding the
+    package back upgrade will upgrade the package and install the new
+    dependencies. Note that upgrade with this option will never remove
+    packages, only allow adding new ones.
+
 apt-clean
     Clean out the local repository of retrieved package files.
 
diff --git a/cdist/conf/type/__package_upgrade_all/parameter/boolean b/cdist/conf/type/__package_upgrade_all/parameter/boolean
index 7a56a34b..cd22eb90 100644
--- a/cdist/conf/type/__package_upgrade_all/parameter/boolean
+++ b/cdist/conf/type/__package_upgrade_all/parameter/boolean
@@ -1,2 +1,3 @@
 apt-clean
 apt-dist-upgrade
+apt-with-new-pkgs

From c2c5668b704e1648ff6c8fb88219badddd028346 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Thu, 23 Dec 2021 20:08:49 +0100
Subject: [PATCH 524/527] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 99a8c08b..26d89057 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* New type: __haproxy_dualstack (Evilham and ungleich)
 	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_upgrade_all: Add new --apt-with-new-pkgs argument (Evilham)
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)

From 08ff41efded6e3112fc462ba13d9166e620b4082 Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Tue, 8 Mar 2022 12:04:58 +0100
Subject: [PATCH 525/527] Added rm of tmpfile.

---
 cdist/conf/type/__ssh_authorized_key/gencode-remote | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote
index 61c77fb9..cbffde94 100755
--- a/cdist/conf/type/__ssh_authorized_key/gencode-remote
+++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote
@@ -40,6 +40,7 @@ if [ -f "$file" ]; then
    grep -v -F -x '$line' '$file' >\$tmpfile
 fi
 cat "\$tmpfile" >"$file"
+rm -f "\$tmpfile"
 DONE
 }
 

From e0150e779681e232f95bdbefd957c666f05daa89 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Wed, 9 Mar 2022 16:16:44 +0100
Subject: [PATCH 526/527] ++changes

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 26d89057..81be51f6 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,8 @@ next:
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
 	* Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham)
+	* Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom)
+	* Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From bd44c023d33eb51a09afad12b18c082f6a3ae36d Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Fri, 11 Jun 2021 11:22:31 +0100
Subject: [PATCH 527/527] Fix typos; add default priority; comments in
 generated files

---
 cdist/conf/type/__apt_pin/manifest                   | 5 +++++
 cdist/conf/type/__apt_pin/parameter/default/priority | 1 +
 cdist/conf/type/__apt_pin/parameter/optional         | 1 +
 cdist/conf/type/__apt_pin/parameter/required         | 1 -
 4 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 cdist/conf/type/__apt_pin/parameter/default/priority

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index e72a8fdd..983b2b42 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -57,6 +57,11 @@ __file "/etc/apt/preferences.d/$name" \
     --owner root --group root --mode 0644 \
     --state "$state" \
     --source - << EOF
+# Created by cdist ${__type##*/}
+# Do not change. Changes will be overwritten.
+#
+
+# $name
 Package: $package
 Pin: $pin
 Pin-Priority: $priority
diff --git a/cdist/conf/type/__apt_pin/parameter/default/priority b/cdist/conf/type/__apt_pin/parameter/default/priority
new file mode 100644
index 00000000..1b79f38e
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/default/priority
@@ -0,0 +1 @@
+500
diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional
index 52f01fd2..847e703d 100644
--- a/cdist/conf/type/__apt_pin/parameter/optional
+++ b/cdist/conf/type/__apt_pin/parameter/optional
@@ -1,2 +1,3 @@
 state
 package
+priority
diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required
index 4b4e9741..c8572d92 100644
--- a/cdist/conf/type/__apt_pin/parameter/required
+++ b/cdist/conf/type/__apt_pin/parameter/required
@@ -1,2 +1 @@
 distribution
-priority