From 26dfdf37c21ce3c469ae0d6a717d390f422f9fbf Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 21 Jun 2020 01:13:30 +0300
Subject: [PATCH 001/411] [__download] support multiple checksum formats and
 download utilities, add --onchange and other minor changes

---
 cdist/conf/type/__download/explorer/state     | 78 +++++++++++++++----
 cdist/conf/type/__download/gencode-local      | 25 +++++-
 cdist/conf/type/__download/gencode-remote     |  7 ++
 cdist/conf/type/__download/man.rst            | 25 ++++--
 .../type/__download/parameter/default/cmd-get |  1 -
 .../type/__download/parameter/default/cmd-sum |  1 -
 cdist/conf/type/__download/parameter/optional |  1 +
 7 files changed, 116 insertions(+), 22 deletions(-)
 create mode 100755 cdist/conf/type/__download/gencode-remote
 delete mode 100644 cdist/conf/type/__download/parameter/default/cmd-get
 delete mode 100644 cdist/conf/type/__download/parameter/default/cmd-sum

diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 6a50f5a5..00362545 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -2,19 +2,71 @@
 
 dst="/$__object_id"
 
-# shellcheck disable=SC2059
-cmd="$( printf "$( cat "$__object/parameter/cmd-sum" )" "$dst" )"
-
-sum="$( cat "$__object/parameter/sum" )"
-
-if [ -f "$dst" ]
+if [ ! -f "$dst" ]
 then
-    if [ "$( eval "$cmd" )" = "$sum" ]
-    then
-        echo 'present'
-    else
-        echo 'mismatch'
-    fi
-else
     echo 'absent'
+    exit 0
+fi
+
+sum_should="$( cat "$__object/parameter/sum" )"
+
+if [ -f "$__object/parameter/cmd-sum" ]
+then
+    # shellcheck disable=SC2059
+    sum_is="$( eval "$( printf \
+        "$( cat "$__object/parameter/cmd-sum" )" \
+        "$dst" )" )"
+else
+    os="$( "$__explorer/os" )"
+
+    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+    then
+        sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
+
+    elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="md5:$( md5 -q "$dst" )"
+            ;;
+            *)
+                sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+
+    elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="sha1:$( sha1 -q "$dst" )"
+            ;;
+            *)
+                sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+
+    elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="sha256:$( sha256 -q "$dst" )"
+            ;;
+            *)
+                sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+    fi
+fi
+
+if [ -z "$sum_is" ]
+then
+    echo 'no checksum from target' >&2
+    exit 1
+fi
+
+if [ "$sum_is" = "$sum_should" ]
+then
+    echo 'present'
+else
+    echo 'mismatch'
 fi
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 49e9c699..85ef3a60 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -9,12 +9,31 @@ fi
 
 url="$( cat "$__object/parameter/url" )"
 
-cmd="$( cat "$__object/parameter/cmd-get" )"
-
 tmp="$( mktemp )"
 
 dst="/$__object_id"
 
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cmd="$( cat "$__object/parameter/cmd-get" )"
+
+elif command -v wget > /dev/null
+then
+    cmd="wget -O - '%s'"
+
+elif command -v curl > /dev/null
+then
+    cmd="curl -o - '%s'"
+
+elif command -v fetch > /dev/null
+then
+    cmd="fetch -o - '%s'"
+
+else
+    echo 'no usable locally installed utility for downloading' >&2
+    exit 1
+fi
+
 printf "$cmd > %s\n" \
     "$url" \
     "$tmp"
@@ -33,3 +52,5 @@ printf '%s %s %s:%s\n' \
     "$dst"
 
 echo "rm -f '$tmp'"
+
+echo 'downloaded' > "$__messages_out"
diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
new file mode 100755
index 00000000..b08d0050
--- /dev/null
+++ b/cdist/conf/type/__download/gencode-remote
@@ -0,0 +1,7 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/onchange" ] \
+    && grep -Fq "$__object_id:downloaded" "$__messages_in"
+then
+    cat "$__object/parameter/onchange"
+fi
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index c973448f..63a41bc4 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -10,7 +10,13 @@ DESCRIPTION
 -----------
 You must use persistent storage in target host for destination file
 (``$__object_id``) because it will be used for checksum calculation
-in order to decide if file must be downloaded.
+in order to decide if file must be (re-)downloaded.
+
+By default type will try to use following locally installed utilities
+for downloading (in order): ``wget``, ``curl`` or ``fetch``.
+
+Environment variables like ``{http,https,ftp}_proxy`` etc can be used on
+cdist execution (``http_proxy=foo cdist config ...``).
 
 
 REQUIRED PARAMETERS
@@ -19,20 +25,29 @@ url
    URL from which to download the file.
 
 sum
-   Checksum of downloaded file.
+   Checksum of file going to be downloaded.
+   By default output of ``cksum`` without filename is expected.
+   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
+
+onchange
+   Execute this command after download.
 
 
 OPTIONAL PARAMETERS
 -------------------
 cmd-get
    Command used for downloading.
-   Default is ``wget -O- '%s'``.
    Command must output to ``stdout``.
+   Parameter will be used for ``printf`` and must include only one
+   variable ``%s`` which will become URL.
+   For example: ``wget -O - '%s'``.
 
 cmd-sum
    Command used for checksum calculation.
-   Default is ``md5sum '%s' | awk '{print $1}'``.
    Command output and ``--sum`` parameter must match.
+   Parameter will be used for ``printf`` and must include only one
+   variable ``%s`` which will become destination.
+   For example: ``md5sum '%s' | awk '{print $1}'``.
 
 
 EXAMPLES
@@ -45,7 +60,7 @@ EXAMPLES
     require='__directory/opt/cpma' \
         __download /opt/cpma/cnq3.zip \
             --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
-            --sum 46da3021ca9eace277115ec9106c5b46
+            --sum md5:46da3021ca9eace277115ec9106c5b46
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
diff --git a/cdist/conf/type/__download/parameter/default/cmd-get b/cdist/conf/type/__download/parameter/default/cmd-get
deleted file mode 100644
index 2daa38a1..00000000
--- a/cdist/conf/type/__download/parameter/default/cmd-get
+++ /dev/null
@@ -1 +0,0 @@
-wget -O- '%s'
diff --git a/cdist/conf/type/__download/parameter/default/cmd-sum b/cdist/conf/type/__download/parameter/default/cmd-sum
deleted file mode 100644
index 3e8a9295..00000000
--- a/cdist/conf/type/__download/parameter/default/cmd-sum
+++ /dev/null
@@ -1 +0,0 @@
-md5sum '%s' | awk '{print $1}'
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index 22783e02..38c0ce4d 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,2 +1,3 @@
 cmd-get
 cmd-sum
+onchange

From 49dde11def71cec121e5f698d0fb6ed3a539f97a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 24 Jun 2020 07:04:06 +0200
Subject: [PATCH 002/411] Remove deprecated __pf_apply

---
 cdist/conf/type/__pf_apply/deprecated     |  1 -
 cdist/conf/type/__pf_apply/explorer/rcvar | 36 ---------------
 cdist/conf/type/__pf_apply/gencode-remote | 51 ---------------------
 cdist/conf/type/__pf_apply/man.rst        | 55 -----------------------
 cdist/conf/type/__pf_apply/singleton      |  0
 5 files changed, 143 deletions(-)
 delete mode 100644 cdist/conf/type/__pf_apply/deprecated
 delete mode 100755 cdist/conf/type/__pf_apply/explorer/rcvar
 delete mode 100755 cdist/conf/type/__pf_apply/gencode-remote
 delete mode 100644 cdist/conf/type/__pf_apply/man.rst
 delete mode 100644 cdist/conf/type/__pf_apply/singleton

diff --git a/cdist/conf/type/__pf_apply/deprecated b/cdist/conf/type/__pf_apply/deprecated
deleted file mode 100644
index 36cfed90..00000000
--- a/cdist/conf/type/__pf_apply/deprecated
+++ /dev/null
@@ -1 +0,0 @@
-Consider moving to __pf_apply_anchor. Get in touch if you need __pf_apply.
diff --git a/cdist/conf/type/__pf_apply/explorer/rcvar b/cdist/conf/type/__pf_apply/explorer/rcvar
deleted file mode 100755
index 7c8d535f..00000000
--- a/cdist/conf/type/__pf_apply/explorer/rcvar
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/sh
-#
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Get the location of the pf ruleset on the target host.
-#
-
-# Debug
-#exec >&2
-#set -x
-
-# Check /etc/rc.conf for pf's configuration file name. Default to /etc/pf.conf
-
-RC="/etc/rc.conf"
-PFCONF="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')"
-echo "${PFCONF:-"/etc/pf.conf"}"
-
-# Debug
-#set +x
-
diff --git a/cdist/conf/type/__pf_apply/gencode-remote b/cdist/conf/type/__pf_apply/gencode-remote
deleted file mode 100755
index c8f7a25a..00000000
--- a/cdist/conf/type/__pf_apply/gencode-remote
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/sh -e
-#
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Apply pf(4) ruleset on *BSD
-#
-
-# Debug
-#exec >&2
-#set -x
-
-rcvar=$(cat "$__object/explorer/rcvar")
-
-cat <<EOF
-if [ -f "${rcvar}.old" ]; then # rcvar.old exists, we must need to disable pf
-   # Disable pf
-   # If it already is disabled, pfctl -d returns 1, go on with life
-   pfctl -d || true
-   # Cleanup
-   rm -f "${rcvar}.old"
-elif [ -f "${rcvar}.new" ]; then # rcvar.new exists, we must need to apply it
-   # Ensure that pf is enabled in the first place
-   # If it already is enabled, pfctl -e returns 1, go on with life
-   mv "${rcvar}.new" "${rcvar}"
-   pfctl -e || true
-   pfctl -f "${rcvar}"
-   if [ "\$?" -ne "0" ]; then # failed to configure new ruleset
-      echo "Failed to configure the new ruleset on ${__target_host}!" >&2
-   fi
-fi
-EOF
-
-# Debug
-#set +x
-
diff --git a/cdist/conf/type/__pf_apply/man.rst b/cdist/conf/type/__pf_apply/man.rst
deleted file mode 100644
index eee345e7..00000000
--- a/cdist/conf/type/__pf_apply/man.rst
+++ /dev/null
@@ -1,55 +0,0 @@
-cdist-type__pf_apply(7)
-=======================
-
-NAME
-----
-cdist-type__pf_apply - Apply pf(4) ruleset on \*BSD
-
-
-DESCRIPTION
------------
-This type is used on \*BSD systems to manage the pf firewall's active ruleset.
-
-
-REQUIRED PARAMETERS
--------------------
-NONE
-
-
-OPTIONAL PARAMETERS
--------------------
-NONE
-
-
-EXAMPLES
---------
-
-.. code-block:: sh
-
-    # Modify the ruleset on $__target_host:
-    __pf_ruleset --state present --source /my/pf/ruleset.conf
-    require="__pf_ruleset" \
-       __pf_apply
-
-    # Remove the ruleset on $__target_host (implies disabling pf(4):
-    __pf_ruleset --state absent
-    require="__pf_ruleset" \
-       __pf_apply
-
-
-SEE ALSO
---------
-:strong:`pf`\ (4), :strong:`cdist-type__pf_ruleset`\ (7)
-
-
-AUTHORS
--------
-Jake Guffey <jake.guffey--@--eprotex.com>
-
-
-COPYING
--------
-Copyright \(C) 2012 Jake Guffey. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
diff --git a/cdist/conf/type/__pf_apply/singleton b/cdist/conf/type/__pf_apply/singleton
deleted file mode 100644
index e69de29b..00000000

From 85614aabd6f3abef5af6d362203db956c9d72aa9 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 28 Jun 2020 16:38:15 +0300
Subject: [PATCH 003/411] [__download] add --download (local|remote), update
 manual

---
 .../conf/type/__download/explorer/remote_cmd  | 19 +++++++++++++++
 cdist/conf/type/__download/gencode-local      |  4 +++-
 cdist/conf/type/__download/gencode-remote     | 22 +++++++++++++++--
 cdist/conf/type/__download/man.rst            | 24 ++++++++++++-------
 cdist/conf/type/__download/manifest           |  6 +++++
 .../__download/parameter/default/download     |  1 +
 cdist/conf/type/__download/parameter/optional |  1 +
 7 files changed, 65 insertions(+), 12 deletions(-)
 create mode 100755 cdist/conf/type/__download/explorer/remote_cmd
 create mode 100755 cdist/conf/type/__download/manifest
 create mode 100644 cdist/conf/type/__download/parameter/default/download

diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd
new file mode 100755
index 00000000..fbd4d84c
--- /dev/null
+++ b/cdist/conf/type/__download/explorer/remote_cmd
@@ -0,0 +1,19 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cmd="$( cat "$__object/parameter/cmd-get" )"
+
+elif command -v curl > /dev/null
+then
+    cmd="curl -o - '%s'"
+
+elif command -v fetch > /dev/null
+then
+    cmd="fetch -o - '%s'"
+
+else
+    cmd="wget -O - '%s'"
+fi
+
+echo "$cmd"
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 85ef3a60..339827c2 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -1,8 +1,10 @@
 #!/bin/sh -e
 
+download="$( cat "$__object/parameter/download" )"
+
 state_is="$( cat "$__object/explorer/state" )"
 
-if [ "$state_is" = 'present' ]
+if [ "$download" != 'local' ] || [ "$state_is" = 'present' ]
 then
     exit 0
 fi
diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
index b08d0050..89ba72af 100755
--- a/cdist/conf/type/__download/gencode-remote
+++ b/cdist/conf/type/__download/gencode-remote
@@ -1,7 +1,25 @@
 #!/bin/sh -e
 
-if [ -f "$__object/parameter/onchange" ] \
-    && grep -Fq "$__object_id:downloaded" "$__messages_in"
+download="$( cat "$__object/parameter/download" )"
+
+state_is="$( cat "$__object/explorer/state" )"
+
+if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
+then
+    cmd="$( cat "$__object/explorer/remote_cmd" )"
+
+    url="$( cat "$__object/parameter/url" )"
+
+    dst="/$__object_id"
+
+    printf "$cmd > %s\n" \
+        "$url" \
+        "$dst"
+
+    echo 'downloaded' > "$__messages_out"
+fi
+
+if [ -f "$__object/parameter/onchange" ] && [ "$state" != "present" ]
 then
     cat "$__object/parameter/onchange"
 fi
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index 63a41bc4..c161f4e4 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -3,26 +3,28 @@ cdist-type__download(7)
 
 NAME
 ----
-cdist-type__download - Download file to local storage and copy it to target host
+cdist-type__download - Download a file
 
 
 DESCRIPTION
 -----------
-You must use persistent storage in target host for destination file
-(``$__object_id``) because it will be used for checksum calculation
-in order to decide if file must be (re-)downloaded.
+Persistent storage for destination file in target host must be used
+(``$__object_id``) because it will be used for checksum calculation in
+order to decide if file must be (re-)downloaded.
 
-By default type will try to use following locally installed utilities
-for downloading (in order): ``wget``, ``curl`` or ``fetch``.
+By default type will try to use ``wget``, ``curl`` or ``fetch`` for
+downloading.  If ``--download remote`` type will fallback to (and
+install) ``wget``.
 
-Environment variables like ``{http,https,ftp}_proxy`` etc can be used on
-cdist execution (``http_proxy=foo cdist config ...``).
+If ``--download local`` (default), then environment variables like
+``{http,https,ftp}_proxy`` etc can be used on cdist execution
+(``http_proxy=foo cdist config ...``).
 
 
 REQUIRED PARAMETERS
 -------------------
 url
-   URL from which to download the file.
+   File's URL.
 
 sum
    Checksum of file going to be downloaded.
@@ -35,6 +37,10 @@ onchange
 
 OPTIONAL PARAMETERS
 -------------------
+download
+   If ``local`` (default), then download file to local storage and copy
+   it to target host. If ``remote``, then download happens in target.
+
 cmd-get
    Command used for downloading.
    Command must output to ``stdout``.
diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest
new file mode 100755
index 00000000..7ec8d86d
--- /dev/null
+++ b/cdist/conf/type/__download/manifest
@@ -0,0 +1,6 @@
+#!/bin/sh -e
+
+if grep -Eq '^wget' "$__object/explorer/remote_cmd"
+then
+    __package wget
+fi
diff --git a/cdist/conf/type/__download/parameter/default/download b/cdist/conf/type/__download/parameter/default/download
new file mode 100644
index 00000000..40830374
--- /dev/null
+++ b/cdist/conf/type/__download/parameter/default/download
@@ -0,0 +1 @@
+local
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index 38c0ce4d..838e2fbf 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,3 +1,4 @@
 cmd-get
 cmd-sum
+download
 onchange

From b6bf90e3f1a775ec1bd27a99015107d24a397787 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 28 Jun 2020 16:43:45 +0300
Subject: [PATCH 004/411] [__download] update manual

---
 cdist/conf/type/__download/man.rst | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index c161f4e4..eafa9dfc 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -8,15 +8,14 @@ cdist-type__download - Download a file
 
 DESCRIPTION
 -----------
-Persistent storage for destination file in target host must be used
-(``$__object_id``) because it will be used for checksum calculation in
-order to decide if file must be (re-)downloaded.
+Destination (``$__object_id``) in target host must be persistent storage
+in order to calculate checksum and decide if file must be (re-)downloaded.
 
-By default type will try to use ``wget``, ``curl`` or ``fetch`` for
-downloading.  If ``--download remote`` type will fallback to (and
-install) ``wget``.
+By default type will try to use ``wget``, ``curl`` or ``fetch``.
+If download happens in target (see ``--download``) then type will
+fallback to (and install) ``wget``.
 
-If ``--download local`` (default), then environment variables like
+If download happens in local machine, then environment variables like
 ``{http,https,ftp}_proxy`` etc can be used on cdist execution
 (``http_proxy=foo cdist config ...``).
 

From 3860f1feeaa2844191eeea84444963818bb42c4b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 26 Jun 2020 14:33:16 +0200
Subject: [PATCH 005/411] [type/{__file/__directory}] Support
 setuid,setguid,sticky bits

---
 cdist/conf/type/__directory/explorer/stat  | 17 +++++++++--------
 cdist/conf/type/__directory/gencode-remote |  4 ++--
 cdist/conf/type/__file/explorer/stat       | 14 +++++++-------
 cdist/conf/type/__file/gencode-remote      |  4 ++--
 4 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat
index 105d894f..a7dc8431 100755
--- a/cdist/conf/type/__directory/explorer/stat
+++ b/cdist/conf/type/__directory/explorer/stat
@@ -33,7 +33,7 @@ fallback() {
    group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
-   mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }')
+   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
 
    printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\n' \
       "$("$__type_explorer/type")" \
@@ -51,13 +51,14 @@ then
    exit
 fi
 
-case $("$__explorer/os") in
-   "freebsd"|"netbsd"|"openbsd"|"macosx")
-      stat -f "type: %HT
+case $("$__explorer/os")
+in
+   freebsd|netbsd|openbsd|macosx)
+      stat -f 'type: %HT
 owner: %Du %Su
 group: %Dg %Sg
-mode: %Lp %Sp
-" "$destination" | awk '/^type/ { print tolower($0); next } { print }'
+mode: %Mp%03Lp %Sp
+' "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
     solaris)
         ls1="$( ls -ld "$destination" )"
@@ -92,9 +93,9 @@ mode: %Lp %Sp
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
       #       we fall through to the shell fallback.
-       stat -c "type: %F
+       stat -c 'type: %F
 owner: %u %U
 group: %g %G
-mode: %a %A" "$destination" 2>/dev/null || fallback
+mode: %04a %A' "$destination" 2>/dev/null || fallback
    ;;
 esac
diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote
index a1a32ea2..2c2c56fd 100755
--- a/cdist/conf/type/__directory/gencode-remote
+++ b/cdist/conf/type/__directory/gencode-remote
@@ -97,9 +97,9 @@ case "$state_should" in
             value_should="$(cat "$__object/parameter/$attribute")"
             value_is="$(get_current_value "$attribute" "$value_should")"
 
-            # change 0xxx format to xxx format => same as stat returns
+            # format mode in four digits => same as stat returns
             if [ "$attribute" = mode ]; then
-                value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')"
+                value_should=$(printf '%04u' "${value_should}")
             fi
 
             if [ "$set_attributes" = 1 ] || [ "$value_should" != "$value_is" ]; then
diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat
index 91c8cc84..231768f6 100755
--- a/cdist/conf/type/__file/explorer/stat
+++ b/cdist/conf/type/__file/explorer/stat
@@ -34,7 +34,7 @@ fallback() {
    group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
-   mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }')
+   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
 
    size=$(echo "$ls_line" | awk '{ print $5 }')
    links=$(echo "$ls_line" | awk '{ print $2 }')
@@ -63,13 +63,13 @@ fi
 case $("$__explorer/os")
 in
    freebsd|netbsd|openbsd|macosx)
-      stat -f "type: %HT
+      stat -f 'type: %HT
 owner: %Du %Su
 group: %Dg %Sg
-mode: %Lp %Sp
+mode: %Mp%03Lp %Sp
 size: %Dz
 links: %Dl
-" "$destination" | awk '/^type/ { print tolower($0); next } { print }'
+' "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
     solaris)
         ls1="$( ls -ld "$destination" )"
@@ -106,11 +106,11 @@ links: %Dl
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
       #       we fall through to the shell fallback.
-      stat -c "type: %F
+      stat -c 'type: %F
 owner: %u %U
 group: %g %G
-mode: %a %A
+mode: %04a %A
 size: %s
-links: %h" "$destination" 2>/dev/null || fallback
+links: %h' "$destination" 2>/dev/null || fallback
          ;;
 esac
diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index 815593bd..a69154df 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -68,9 +68,9 @@ case "$state_should" in
             if [ -f "$__object/parameter/$attribute" ]; then
                 value_should="$(cat "$__object/parameter/$attribute")"
 
-                # change 0xxx format to xxx format => same as stat returns
+                # format mode in four digits => same as stat returns
                 if [ "$attribute" = mode ]; then
-                    value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')"
+                    value_should=$(printf '%04u' "${value_should}")
                 fi
 
                 value_is="$(get_current_value "$attribute" "$value_should")"

From fe193ecab8addf9b1c5b6d3c06c11ae9f9803378 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 1 Jul 2020 14:05:45 +0200
Subject: [PATCH 006/411] Make code consistent

* Remove supreflous checking and warning message.
* Fix cache recording.
---
 cdist/emulator.py | 65 +++++++++++++++--------------------------------
 1 file changed, 21 insertions(+), 44 deletions(-)

diff --git a/cdist/emulator.py b/cdist/emulator.py
index adbcbd9d..9fe84056 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -91,10 +91,6 @@ class Emulator:
         self.type_name = os.path.basename(argv[0])
         self.cdist_type = core.CdistType(self.type_base_path, self.type_name)
 
-        # If set then object alreay exists and this var holds existing
-        # requirements.
-        self._existing_reqs = None
-
         self.__init_log()
 
     def run(self):
@@ -230,9 +226,6 @@ class Emulator:
                 self.parameters[key] = value
 
         if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env:
-            # Make existing requirements a set so that we can compare it
-            # later with new requirements.
-            self._existing_reqs = set(self.cdist_object.requirements)
             obj_params = self._object_params_in_context()
             if obj_params != self.parameters:
                 errmsg = ("Object %s already exists with conflicting "
@@ -251,23 +244,26 @@ class Emulator:
             else:
                 self.cdist_object.create()
             self.cdist_object.parameters = self.parameters
-            # record the created object in typeorder file
-            with open(self.typeorder_path, 'a') as typeorderfile:
-                print(self.cdist_object.name, file=typeorderfile)
-            # record the created object in parent object typeorder file
-            __object_name = self.env.get('__object_name', None)
-            depname = self.cdist_object.name
-            if __object_name:
-                parent = self.cdist_object.object_from_name(__object_name)
-                parent.typeorder.append(self.cdist_object.name)
-                if self._order_dep_on():
-                    self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
-                                    ' for %s'), depname, parent.name)
-                    parent.typeorder_dep.append(depname)
-            elif self._order_dep_on():
-                self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
-                               depname)
-                self._add_typeorder_dep(depname)
+        # Do the following recording even if object exists, but with
+        # different requirements.
+
+        # record the created object in typeorder file
+        with open(self.typeorder_path, 'a') as typeorderfile:
+            print(self.cdist_object.name, file=typeorderfile)
+        # record the created object in parent object typeorder file
+        __object_name = self.env.get('__object_name', None)
+        depname = self.cdist_object.name
+        if __object_name:
+            parent = self.cdist_object.object_from_name(__object_name)
+            parent.typeorder.append(self.cdist_object.name)
+            if self._order_dep_on():
+                self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
+                                ' for %s'), depname, parent.name)
+                parent.typeorder_dep.append(depname)
+        elif self._order_dep_on():
+            self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
+                           depname)
+            self._add_typeorder_dep(depname)
 
         # Record / Append source
         self.cdist_object.source.append(self.object_source)
@@ -322,8 +318,6 @@ class Emulator:
         # This ensures pattern matching is done against sanitised list
         self.cdist_object.requirements.append(cdist_object.name)
 
-        return cdist_object.name
-
     def _order_dep_on(self):
         return os.path.exists(self.order_dep_state_path)
 
@@ -392,7 +386,6 @@ class Emulator:
                 # so do not set a requirement
                 pass
 
-        reqs = set()
         if "require" in self.env:
             requirements = self.env['require']
             self.log.debug("reqs = " + requirements)
@@ -400,23 +393,7 @@ class Emulator:
                 # Ignore empty fields - probably the only field anyway
                 if len(requirement) == 0:
                     continue
-                object_name = self.record_requirement(requirement)
-                reqs.add(object_name)
-        if self._existing_reqs is not None:
-            # If object exists then compare existing and new requirements.
-            if self._existing_reqs != reqs:
-                warnmsg = ("Object {} already exists with requirements:\n"
-                           "{}: {}\n"
-                           "{}: {}\n"
-                           "Dependency resolver could not handle dependencies "
-                           "as expected.".format(
-                               self.cdist_object.name,
-                               " ".join(self.cdist_object.source),
-                               self._existing_reqs,
-                               self.object_source,
-                               reqs
-                           ))
-                self.log.warning(warnmsg)
+                self.record_requirement(requirement)
 
     def record_auto_requirements(self):
         """An object shall automatically depend on all objects that it

From 8ee71e67f49ff673db3844498521dd289e40cd27 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Tue, 2 Jun 2020 22:33:30 +0200
Subject: [PATCH 007/411] [__lxc_container] initial work for lxc contaienr type

This should theoretically work, but is untested. It is able to create
and delete, change basic configuration and start/stop/freeze the
container.
---
 .../conf/type/__lxc_container/explorer/config |  37 ++++
 .../type/__lxc_container/explorer/lxcpath     |  16 ++
 .../conf/type/__lxc_container/explorer/state  |  18 ++
 cdist/conf/type/__lxc_container/gencode-local |  54 +++++
 .../conf/type/__lxc_container/gencode-remote  | 203 ++++++++++++++++++
 cdist/conf/type/__lxc_container/helper-exec   |  68 ++++++
 cdist/conf/type/__lxc_container/helper-param  |  27 +++
 cdist/conf/type/__lxc_container/issue.txt     |   1 +
 cdist/conf/type/__lxc_container/man.rst       | 125 +++++++++++
 cdist/conf/type/__lxc_container/manifest      | 106 +++++++++
 .../type/__lxc_container/parameter/boolean    |   1 +
 .../__lxc_container/parameter/default/state   |   1 +
 .../__lxc_container/parameter/default/user    |   1 +
 .../type/__lxc_container/parameter/optional   |  10 +
 .../parameter/optional_multiple               |   3 +
 cdist/conf/type/__lxc_container/todo.txt      |  28 +++
 16 files changed, 699 insertions(+)
 create mode 100755 cdist/conf/type/__lxc_container/explorer/config
 create mode 100755 cdist/conf/type/__lxc_container/explorer/lxcpath
 create mode 100755 cdist/conf/type/__lxc_container/explorer/state
 create mode 100755 cdist/conf/type/__lxc_container/gencode-local
 create mode 100755 cdist/conf/type/__lxc_container/gencode-remote
 create mode 100644 cdist/conf/type/__lxc_container/helper-exec
 create mode 100644 cdist/conf/type/__lxc_container/helper-param
 create mode 100644 cdist/conf/type/__lxc_container/issue.txt
 create mode 100644 cdist/conf/type/__lxc_container/man.rst
 create mode 100755 cdist/conf/type/__lxc_container/manifest
 create mode 100644 cdist/conf/type/__lxc_container/parameter/boolean
 create mode 100644 cdist/conf/type/__lxc_container/parameter/default/state
 create mode 100644 cdist/conf/type/__lxc_container/parameter/default/user
 create mode 100644 cdist/conf/type/__lxc_container/parameter/optional
 create mode 100644 cdist/conf/type/__lxc_container/parameter/optional_multiple
 create mode 100644 cdist/conf/type/__lxc_container/todo.txt

diff --git a/cdist/conf/type/__lxc_container/explorer/config b/cdist/conf/type/__lxc_container/explorer/config
new file mode 100755
index 00000000..d82dbb25
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/explorer/config
@@ -0,0 +1,37 @@
+#!/bin/sh -e
+# explorer/config
+
+# Prints out the current container configuration (if required). This makes
+# it easy to differ changes.
+
+# abort if container will be absent - no config required
+if [ "$(cat "$__object/parameter/state")" = "absent" ]; then
+	exit 0
+fi
+
+
+# read the lxcpath (reusing explorer)
+lxcpath="$( "$__type_explorer/lxcpath" )"
+
+# get name
+name="$__object/parameter/name"
+if [ -f "$name" ]; then
+    name="$(cat "$name")"
+else
+    name="$__object_id"
+fi
+
+
+# assemble the container configuration file
+config="$lxcpath/$name/config"
+
+# check if the file exist
+if [ -r "$config" ]; then
+	# print configuration and strip files to just values
+	#  grep will hide all comments and empty lines - all others must be key-values
+	#  sed will uniform patterns, that no spaces will be problematic (before, in the middle and at end)
+	grep -E -v "^([[:blank:]]*)(#|$)" "$config" | sed 's/^[[:blank:]]*//; s/[[:blank:]]*=[[:blank:]]*/ = /; s/[[:blank:]]*$//'
+else
+	>&2 printf "config file \"%s\" does not exist or is not readable\n" "$config"
+	exit 1
+fi
diff --git a/cdist/conf/type/__lxc_container/explorer/lxcpath b/cdist/conf/type/__lxc_container/explorer/lxcpath
new file mode 100755
index 00000000..a3505f94
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/explorer/lxcpath
@@ -0,0 +1,16 @@
+#!/bin/sh -e
+# explorer/lxcpath
+
+# Echos the lxcpath variable for all container instances
+
+# Load important functions and parameter handling
+. "$__type/helper-param"
+. "$__type/helper-exec"
+
+
+# lxcpath parameter
+if [ -z "$lxcpath" ]; then
+	lxc_g config "lxc.lxcpath"
+else
+	echo "$lxcpath"
+fi
diff --git a/cdist/conf/type/__lxc_container/explorer/state b/cdist/conf/type/__lxc_container/explorer/state
new file mode 100755
index 00000000..23900ba6
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/explorer/state
@@ -0,0 +1,18 @@
+#!/bin/sh -e
+# explorer/state
+
+# Outputs if the container exist
+#  possible values: present || absent || running || stopped || frozen
+
+# source param lib
+. "$__type/helper-param"
+. "$__type/helper-exec"
+
+# the lxc-info command will exit non-zero if container not exist
+if ! lxc_c info -H -S >/dev/null 2>&1; then
+	# not exist
+	echo "absent"
+else
+	# print state (command output matches to type states)
+	lxc_c info -H -s | tr '[[:upper:]]' '[[:lower:]]'
+fi
diff --git a/cdist/conf/type/__lxc_container/gencode-local b/cdist/conf/type/__lxc_container/gencode-local
new file mode 100755
index 00000000..1c8a18c8
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/gencode-local
@@ -0,0 +1,54 @@
+#!/bin/sh -e
+# gencode-local
+
+# This uploads the diff of the configuration file, which is applied by gencode-remote
+
+
+# check both states
+state_is="$(cat "$__object/explorer/state")"
+state_should="$(cat "$__object/parameter/state")"
+
+# Check the configurations only if the container exists
+if [ "$state_should" != "absent" ]; then
+	# do changes
+
+	# predict remote lxc config file
+	container_config="$(cat "$__object/explorer/lxcpath")/$name/config"
+
+	# IPv6 fix
+	if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$'
+	then
+		my_target_host="[${__target_host}]"
+	else
+		my_target_host="${__target_host}"
+	fi
+
+
+	# config-present
+	# remove all config lines from config-present that are already set or absent anyway
+	if grep -v -f "$__object/explorer/config" -f "$__object/files/config-absent" \
+		"$__object/files/config-present" > "$__object/files/config.add"; then
+
+		# get temp file name
+		add_dest="$($__remote_exec $__target_host "mktemp $container_config-add.XXXXXXXXXX")"
+		printf "%s" "$add_dest" > "$__object/files/remote-tmpfile_add"
+		# upload diff
+		cat <<OUT
+$__remote_copy "$__object/files/config.add" "$my_target_host:$add_dest"
+OUT
+	fi
+
+	# config-absent
+	# only find those lines which should be absent in the config
+	if grep -f "$__object/explorer/config" "$__object/files/config-absent" \
+		> "$__object/files/config.del"; then
+
+		# get temp file name
+		del_dest="$($__remote_exec $__target_host "mktemp $container_config-del.XXXXXXXXXX")"
+		printf "%s" "$del_dest" > "$__object/files/remote-tmpfile_del"
+		# upload diff
+		cat <<OUT
+$__remote_copy "$__object/files/config.add" "$my_target_host:$del_dest"
+OUT
+	fi
+fi
diff --git a/cdist/conf/type/__lxc_container/gencode-remote b/cdist/conf/type/__lxc_container/gencode-remote
new file mode 100755
index 00000000..d6cf4e74
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/gencode-remote
@@ -0,0 +1,203 @@
+#!/bin/sh -e
+# gencode-remote
+
+# Does all common stuff for the container on the host
+
+
+# Source to the common parameters
+. "$__type/helper-param"
+
+# check both states
+state_is="$(cat "$__object/explorer/state")"
+state_should="$(cat "$__object/parameter/state")"
+
+
+# Check if there is a difference within the states
+if [ "$state_is" != "$state_should" ]; then
+	# change privileges (too complex to make exceptions)
+	cat <<USER
+su "$user" - <<SU
+USER
+
+	# handle if the container should exist or not
+	case "$state_should" in
+		present|running|stopped|frozen)
+			# check, if the container should be created or cloned
+			if [ -f "$__object/parameter/clone" ]; then
+				copy_from="$(cat "$__object/parameter/clone")"
+				copypath="$(cat "$__object/parameter/clonepath" || true)"
+
+				# assemble own optional lxc options
+				lxc_opts=""
+				if [ -n "$copypath" ]; then
+					# this is set as the default $lxcpath, because it is the origin
+					lxc_opts="$lxc_opts -P \"$copypath\""
+				fi
+				if [ -n "$lxcpath" ]; then
+					# this is set as the newpath, because it is the destination
+					lxc_opts="$lxc_opts -p \"$lxcpath\""
+				fi
+
+				# print lxc-copy code (because of the lxcpath thing, $LXC_PARAM conflicts)
+				cat <<LXC
+lxc-copy $lxc_opts -n "$copy_from" --newname "$name"
+LXC
+			else
+				template="$(cat "$__object/parameter/template")"
+
+				# assemble general creation options
+				create_opts=""
+				if [ -f "$__object/parameter/no-default-config" ]; then
+					# generate a random empty file and append
+					empty_config="$($__remote_exec $__target_host "mktemp \${TMPDIR:-/tmp}/cdist_lxc-empty.XXXXXXXXXX")"
+					create_opts="$create_opts -f \"$empty_config\""
+
+				elif [ -f "$__object/parameter/default-config" ]; then
+					# append the configuration
+					create_opts="$create_opts -f \"$(cat "$__object/parameter/default-config")\""
+				fi
+
+				# assemble template options
+				template_opts=""
+				# add common options
+				if [ -f "$__object/parameter/release" ]; then
+					template_opts="$template_opts -r \"$(cat "$__object/parameter/release")\""
+				fi
+				if [ -f "$__object/parameter/arch" ]; then
+					template_opts="$template_opts -a \"$(cat "$__object/parameter/arch")\""
+				fi
+
+				# at last, apply custom options
+				if [ -f "$__object/parameter/template-opts" ]; then
+					while read line; do
+						template_opts="$template_opts $line"
+					done < "$__object/parameter/template-opts"
+				fi
+
+				# print lxc-create code
+				cat <<LXC
+lxc-create $LXC_PARAM -n "$name" -t "$template" $create_opts -- $template_opts
+LXC
+			fi
+
+			# write to the message system
+			echo "create" >> "$__messages_out"
+			;;
+
+		absent)
+			# shutdown and delete the container
+			#  we could force-delete, but better be polite
+			#  snapshots are deleted, too - to keep everything clean. May someone does not want it?
+			cat <<LXC
+lxc-stop $LXC_PARAM -n "$name"
+lxc-destroy $LXC_PARAM --snapshots -n "$name"
+LXC
+
+			# write to the message system
+			echo "destroy" >> "$__messages_out"
+			;;
+
+		# error handling: invalid state
+		*)
+			printf "Container '%s' in unknown state %s\n" "$name" "$state_should" >&2
+			exit 2
+			;;
+	esac
+
+	# end of su here-document
+	echo "SU"
+fi
+
+
+# Check the configurations only if the container exists and there are changes
+if [ "$state_should" != "absent" ] && \
+	( [ -f "${tmppath}_add" ] || [ -f "${tmppath}_del" ] ); then
+	# shortcut
+	tmppath="$__object/files/remote-tmpfile"
+
+	# generate config path
+	container_config="$(cat "$__object/explorer/lxcpath")/$name/config"
+	# create remote tmpfile for config
+	cat <<DONE
+tmpconfig="\$(mktemp "$container_config.XXXXXXXXXX")"
+cp -p "$container_config" "\$tmpconfig"
+DONE
+
+	# check if smth. to be added
+	if [ -f "${tmppath}_add" ]; then
+		cat <<DONE
+cat "$(cat "${tmppath}_add")" >> "\$tmpconfig"
+rm -rf "$(cat "${tmppath}_add")"
+DONE
+	fi
+
+	# check if smth. to be deleted
+	if [ -f "${tmppath}_del" ]; then
+		cat <<DONE
+grep -v -f "$(cat "${tmppath}_del")" "\$tmpconfig" > "\$tmpconfig"
+rm -rf "$(cat "${tmppath}_del")"
+DONE
+	fi
+
+
+	# apply all changes
+	cat <<DONE
+rm -rf "$container_config"
+mv "\$tmpconfig" "$container_config"
+DONE
+	# write message
+	echo "config" >> "$__messages_out"
+fi
+
+
+# TODO user context again
+# Check if there is a difference within the states
+if [ "$state_is" != "$state_should" ] && [ "$state_should" != "absent" ]; then
+	# change privileges (too complex to make exceptions)
+	cat <<USER
+su "$user" - <<SU
+USER
+
+	# handle if the container should exist or not
+	case "$state_should" in
+		running)
+			if [ "$state_is" = "frozen" ]; then
+				cat <<LXC
+lxc-unfreeze $LXC_PARAM -n "$name"
+LXC
+			fi
+			cat <<LXC
+lxc-start $LXC_PARAM -n "$name"
+LXC
+			;;
+
+		stopped)
+			if [ "$state_is" = "frozen" ]; then
+				cat <<LXC
+lxc-unfreeze $LXC_PARAM -n "$name"
+LXC
+			fi
+			cat <<LXC
+lxc-stop $LXC_PARAM -n "$name"
+LXC
+			;;
+
+		frozen)
+			# stopped containers can't be frozen; they must be started first
+			if [ "$state_is" = "stopped" ]; then
+				cat <<LXC
+lxc-start $LXC_PARAM -n "$name"
+LXC
+			fi
+			cat <<LXC
+lxc-freeze $LXC_PARAM -n "$name"
+LXC
+			;;
+
+		*)
+			;; # must be checked by previous case
+	esac
+
+	# end of su here-document
+	echo "SU"
+fi
diff --git a/cdist/conf/type/__lxc_container/helper-exec b/cdist/conf/type/__lxc_container/helper-exec
new file mode 100644
index 00000000..e8290ab7
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/helper-exec
@@ -0,0 +1,68 @@
+# vi: syntax=sh
+# helper-exec
+
+# Adds functions to easily handle the execution of lxc commands
+#  requires to source the helper-param before .. ;)
+
+# Following global variables required (from helper-param):
+#  $user: the unix user to execute lxc commands with
+#  $name: the lxc container name
+
+
+# Execute a command with a special user.
+#
+# Parameters:
+#  1: the execution user
+#  2: the execution command
+#  3: all further parameters for the command
+exec_user() {
+    # save important arguments
+    _user="$1"
+    _cmd="$(which -- "$2")"
+    shift 2
+
+    # check if the user must be switched
+    if [ "$_user" != "$USER" ]; then
+        # execute it wit su
+        #  here, compatibility to non-sudo systems is provided and the "shell parameter thing" is misused
+        #  to provide more parameter safety.
+        su -s "$_cmd" -- "$_user" "$@"
+
+    # no switch of the user - keep the same
+    else
+        # execute the command
+        "$_cmd" "$@"
+    fi
+}
+
+# Function to execute lxc commands in the correct execution. Only works for container-specific commands.
+# Default things such as default parameters (see $LXC_PARAM), execution user and the container name are
+# applied within the function.
+#
+# Parameters:
+#  1: the lxc command without the preceding "lxc"
+#  2-n: all other lxc arguments
+lxc_c() {
+    # get exec name
+    lxccmd="lxc-$1"
+    shift
+
+    # execute the command
+    exec_user "$user" "$lxccmd" $LXC_PARAM -n "$name" "$@"
+}
+
+# Function to execute global lxc commands within the correct execution environment. Container-specific
+# commands require to pass the -n flag with the container name, rather lxc_c is recommended.
+# Default things such as default parameters (see $LXC_PARAM) and execution are applied to the command.
+#
+# Parameters:
+#  1: the lxc command without the preceding "lxc"
+#  2-n: all other lxc arguments
+lxc_g() {
+    # get exec name
+    lxccmd="lxc-$1"
+    shift
+
+    # execute the command
+    exec_user "$user" "$lxccmd" $LXC_PARAM "$@"
+}
diff --git a/cdist/conf/type/__lxc_container/helper-param b/cdist/conf/type/__lxc_container/helper-param
new file mode 100644
index 00000000..eae7f462
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/helper-param
@@ -0,0 +1,27 @@
+# vi: syntax=sh
+# helper-param
+
+# Shortcut to optain all basic required parameters for working.
+
+
+# Parameter gathering
+name="$__object/parameter/name"
+if [ -f "$name" ]; then
+    name="$(cat "$name")"
+else
+    name="$__object_id"
+fi
+user="$(cat "$__object/parameter/user")"
+
+# general lxc things
+lxcpath="$(cat "$__object/parameter/lxcpath" || true)"
+
+
+# Summerize common parameter arguments, listed in manual as "COMMON OPTIONS"
+#  will passed raw; must qouted manually
+LXC_PARAM="-q"
+
+# if lxcpath given
+if [ -n "$lxcpath" ]; then
+    LXC_PARAM="$LXC_PARAM -P \"$lxcpath\""
+fi
diff --git a/cdist/conf/type/__lxc_container/issue.txt b/cdist/conf/type/__lxc_container/issue.txt
new file mode 100644
index 00000000..f65f9780
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/issue.txt
@@ -0,0 +1 @@
+$__object_id == name ? sperate parameter required?? multiple same names possible due to --lxcpath!
diff --git a/cdist/conf/type/__lxc_container/man.rst b/cdist/conf/type/__lxc_container/man.rst
new file mode 100644
index 00000000..f84b0619
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/man.rst
@@ -0,0 +1,125 @@
+cdist-type__lxc_container(7)
+============================
+
+NAME
+----
+cdist-type__lxc_container - Controls the configuration of a lxc container
+
+DESCRIPTION
+-----------
+
+TBA.
+
+REQUIRED PARAMETERS
+-------------------
+
+None.
+
+OPTIONAL PARAMETERS
+-------------------
+
+name
+    String which will used as container name instead of the object id. This should only required
+    if multiple lxc instances used from different users/directories (in junction with `--lxcpath`).
+
+user
+    The unix user name in which lxc commands are executed. Each user have a different lxc instance.
+
+state
+    The state of the container, if it should exist or not.
+
+    running
+        The container exist and is running (default)
+
+    stopped
+        The container exist, but does not run
+
+    frozen
+        The container exist and is frozen
+
+    present
+        The container exist, but it is ignored if the container will run or not
+
+    absent
+        The container does not exist
+
+lxcpath
+    Set an other directory as lxc container location instead of the program default. This will passed
+    to all lxc commands who do smth. with containers.
+
+config
+    Contains configuration file paths and/or single configuration lines to assemble the lxc container
+    configuration file. There is no deep dive into files referenced with `lxc.include`.  Can be used
+    multiple times.
+
+config-absent
+    Contains configuration file paths and/ro single configuration lines which makes the same as the
+    `--config` parameter execpt all configuration lines will be removed instead of being added, Can
+    be used multiple times.
+
+BOOLEAN PARAMETERS
+------------------
+
+None.
+
+TEMPLATE PARAMETERS
+-------------------
+This or the *CLONE PARAMETERS* are required to create an container and must be present if the container
+should be created. If the template parameters are choosen, `--template` must be present. Then, none of
+the *CLONE PARAMETERS* must be present.
+
+Because the templates vary in the possiblities of arguments and unkown arguments causing the abort of the
+container creation, the type will throw an error if the compatibility is not garanteed to the given
+template.
+
+The template options are only applied if the container will be created, there are not applied when
+the container is already present.
+
+template
+    The template which is used to create the the container. The name of the template must passed with
+    this argument and should exist on the target host.
+
+default-config
+    Alternative path to the user-defined default config. For the root user, this is commonly at
+    `/etc/lxc/default.conf`. It will be included into the container configuration file at creation
+    time.
+
+no-default-config
+    **(Boolean value)** This parameter avoids using a default config file by using an empty file instead.
+
+template-opts
+    Raw options which get passed to the template directly. Can be used multiple times. The argument must
+    contain a single string (*at best, everything is inside one quote*), else it will interpreted as an
+    further argument for the type. Some values should be quoted inside the quotes, too.
+
+release
+    TBA.
+
+arch
+    TBA.
+
+CLONE PARAMETERS
+----------------
+This or the *TEMPLATE PARAMETERS* are required to create an container and must be present if the container
+should be created. If the clone parameters are choosen, `--clone` must be present. Then, none of the
+*TEMPLATE PARAMETERS* must be present.
+
+clone
+    Instead of creating a new container with a given template, clone an other container and use him. The
+    argument takes the container name, which will be cloned. He should exist.
+
+clonepath
+    The container path for the container to clone. It is like the `--lxcpath` parameter, but for the container
+    which will be cloned, not the target container.
+
+MESSAGES
+--------
+
+create
+    The container was created by a template or clone.
+
+destroy
+    The container was destroyed.
+
+config
+    The container configuration changed.
diff --git a/cdist/conf/type/__lxc_container/manifest b/cdist/conf/type/__lxc_container/manifest
new file mode 100755
index 00000000..7c82e59f
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/manifest
@@ -0,0 +1,106 @@
+#!/bin/sh -e
+# manifest
+
+# Manifest; check if everything is valid
+
+# Exit codes:
+#  0: successful or warnings
+#  2: parameters in wrong context given
+
+# basepath of the parameter folder (shortcut)
+param="$__object/parameter/"
+
+# ====================== #
+# ==  WARNING CHECKS  == #
+# ====================== #
+
+
+# ==================== #
+# ==  ERROR CHECKS  == #
+# ==================== #
+
+# valid check if only --template or --clone given
+if [ -f "$param/template" ] && [ -f "$param/clone" ]; then
+	# error and exit
+	>&2 echo "error: can not create and clone a container at once!"
+	>&2 echo "error: --template and --clone can not work together"
+	exit 2
+fi
+
+# no clone option if --template given
+if [ -f "$param/template" ]; then
+	if [ -f "$param/clonepath" ]
+	then
+		>&2 echo "error: container will created by template, no clone options required!"
+		exit 2
+	fi
+fi
+
+# no template options if --clone given
+if [ -f "$__object/parameter/clone" ]; then
+	if [ -f "$param/template-opts" ] \
+		|| [ -f "$param/default-config" ] \
+		|| [ -f "$param/no-default-config" ] \
+		|| [ -f "$param/release" ] \
+		|| [ -f "$param/arch" ]
+	then
+		>&2 echo "error: container will created by clone, no template options required!"
+		exit 2
+	fi
+fi
+
+
+
+
+# ============================== #
+# ==  CONFIGURATION HANDLING  == #
+# ============================== #
+
+# Function to read multiple parameter input for configuration and print it to a single file.
+# It will handle file paths as well as the stdin. Instead of the file, the content will be printed.
+#
+# Parameters:
+#  1: parameter name
+#  2: file to print
+write_multi_param() {
+	touch "$2"
+	while read line; do
+		if [ "$line" = "-" ]; then
+			# append stdin
+			cat "$__object/stdin" >> "$2"
+		elif [ -f "$line" ]; then
+			# append file content
+			cat "$line" >> "$2"
+		else
+			# print out content
+			printf "%s\n" "$line" >> "$2"
+		fi
+	done < "$__object/parameter/$1"
+}
+
+# Function to get rid of whitespaces inside of key-value pairs. It will clear all of these.
+#
+# Parameters:
+#  1: the file which should be handled
+trimm_whitespaces() {
+	cp "$1" "$1"~  # backup file to see original content
+	grep -E -v "^([[:blank:]]*)(#|$)" "$1" \
+		| sed 's/^[[:blank:]]*//; s/[[:blank:]]*=[[:blank:]]*/ = /; s/[[:blank:]]*$//'
+}
+
+# Only required if container will not be absent
+# get the wanted configuration lines
+if [ "$(cat "$__object/parameter/state")" != "absent" ]; then
+	# Create the files directory
+	mkdir "$__object/files/"
+
+	# Create the file of the locally configuration
+	conffile="$__object/files/config-present"
+	write_multi_param config "$conffile"
+	trimm_whitespaces "$conffile" > "$conffile"
+
+	# Create the file of the absent configuration
+	absentfile="$__object/files/config-absent"
+	write_multi_param config-absent "$absentfile"
+	trimm_whitespaces "$absentfile" > "$absentfile"
+fi
diff --git a/cdist/conf/type/__lxc_container/parameter/boolean b/cdist/conf/type/__lxc_container/parameter/boolean
new file mode 100644
index 00000000..caf89a8e
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/parameter/boolean
@@ -0,0 +1 @@
+no-default-config
diff --git a/cdist/conf/type/__lxc_container/parameter/default/state b/cdist/conf/type/__lxc_container/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__lxc_container/parameter/default/user b/cdist/conf/type/__lxc_container/parameter/default/user
new file mode 100644
index 00000000..d8649da3
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/parameter/default/user
@@ -0,0 +1 @@
+root
diff --git a/cdist/conf/type/__lxc_container/parameter/optional b/cdist/conf/type/__lxc_container/parameter/optional
new file mode 100644
index 00000000..fc4bb71b
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/parameter/optional
@@ -0,0 +1,10 @@
+name
+user
+state
+lxcpath
+template
+default-config
+release
+arch
+clone
+clonepath
diff --git a/cdist/conf/type/__lxc_container/parameter/optional_multiple b/cdist/conf/type/__lxc_container/parameter/optional_multiple
new file mode 100644
index 00000000..069c02b1
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/parameter/optional_multiple
@@ -0,0 +1,3 @@
+config
+config-absent
+template-opts
diff --git a/cdist/conf/type/__lxc_container/todo.txt b/cdist/conf/type/__lxc_container/todo.txt
new file mode 100644
index 00000000..17c4edaf
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/todo.txt
@@ -0,0 +1,28 @@
+# List of all featues that should be implemented
+storage:
+  - backing_storage
+  - other settings prior to storage
+template:
+  - type: a.E. debian, ubuntu -> what on change?
+  - release
+  - arch
+  - possible ssh auth key?
+  - possible root_password?
+  - possible packages?
+  - possible mirrors?
+  - other options??
+auto..:
+  - group
+  - start
+lxc config ..
+how to find lxc containers path?
+lxc-copy: ephemeral containers? -> seperate type if it makes sense
+
+If config already created, it's considered the container is already created
+  -> first create; then setup
+  detection if template changed??
+
+
+lxc configuration at startup:
+  flag -f can used to set a default config (instead the system default)
+  there is no problem to change config before the first startup

From 1b735fb1508edbe144263f892b5b1a1fdc03c95e Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 28 Jun 2020 14:12:49 +0200
Subject: [PATCH 008/411] [__lxc_container] fix SC2021

Problems with backets and tr.
---
 cdist/conf/type/__lxc_container/explorer/state | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__lxc_container/explorer/state b/cdist/conf/type/__lxc_container/explorer/state
index 23900ba6..0c79d3c8 100755
--- a/cdist/conf/type/__lxc_container/explorer/state
+++ b/cdist/conf/type/__lxc_container/explorer/state
@@ -14,5 +14,5 @@ if ! lxc_c info -H -S >/dev/null 2>&1; then
 	echo "absent"
 else
 	# print state (command output matches to type states)
-	lxc_c info -H -s | tr '[[:upper:]]' '[[:lower:]]'
+	lxc_c info -H -s | tr '[:upper:]' '[:lower:]'
 fi

From 939abf6d45eca4ba6fb6133acbb8fc936c9a2d3b Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Tue, 30 Jun 2020 18:05:19 +0200
Subject: [PATCH 009/411] [__lxc_container] got type to fly with heavier
 changes

Helper scripts do not work with explorers, so there are useless; and
some other problems detected at runtime.

`gencode-remote` *will be refactored - be aware*
---
 .../conf/type/__lxc_container/explorer/config |   5 +-
 .../type/__lxc_container/explorer/lxcpath     |  15 ++-
 .../conf/type/__lxc_container/explorer/state  |  35 +++++-
 cdist/conf/type/__lxc_container/gencode-local |   2 +-
 .../conf/type/__lxc_container/gencode-remote  | 108 +++++++++++-------
 cdist/conf/type/__lxc_container/helper-exec   |  68 -----------
 cdist/conf/type/__lxc_container/helper-param  |  27 -----
 cdist/conf/type/__lxc_container/manifest      |  28 +++--
 8 files changed, 128 insertions(+), 160 deletions(-)
 delete mode 100644 cdist/conf/type/__lxc_container/helper-exec
 delete mode 100644 cdist/conf/type/__lxc_container/helper-param

diff --git a/cdist/conf/type/__lxc_container/explorer/config b/cdist/conf/type/__lxc_container/explorer/config
index d82dbb25..287a5063 100755
--- a/cdist/conf/type/__lxc_container/explorer/config
+++ b/cdist/conf/type/__lxc_container/explorer/config
@@ -25,13 +25,10 @@ fi
 # assemble the container configuration file
 config="$lxcpath/$name/config"
 
-# check if the file exist
+# check if the file exist, else the container is absent
 if [ -r "$config" ]; then
 	# print configuration and strip files to just values
 	#  grep will hide all comments and empty lines - all others must be key-values
 	#  sed will uniform patterns, that no spaces will be problematic (before, in the middle and at end)
 	grep -E -v "^([[:blank:]]*)(#|$)" "$config" | sed 's/^[[:blank:]]*//; s/[[:blank:]]*=[[:blank:]]*/ = /; s/[[:blank:]]*$//'
-else
-	>&2 printf "config file \"%s\" does not exist or is not readable\n" "$config"
-	exit 1
 fi
diff --git a/cdist/conf/type/__lxc_container/explorer/lxcpath b/cdist/conf/type/__lxc_container/explorer/lxcpath
index a3505f94..a4116b9b 100755
--- a/cdist/conf/type/__lxc_container/explorer/lxcpath
+++ b/cdist/conf/type/__lxc_container/explorer/lxcpath
@@ -3,14 +3,21 @@
 
 # Echos the lxcpath variable for all container instances
 
-# Load important functions and parameter handling
-. "$__type/helper-param"
-. "$__type/helper-exec"
+# get parameter if exist
+lxcpath="$(cat "$__object/parameter/lxcpath" 2>/dev/null || true)"
 
 
 # lxcpath parameter
 if [ -z "$lxcpath" ]; then
-	lxc_g config "lxc.lxcpath"
+	user="$(cat "$__object/parameter/user")"
+
+	# gather default value from config
+	if [ "$user" != "$USER" ]; then
+		su -s "$(which -- lxc-config)" -- "$user" lxc.lxcpath
+	else
+		lxc-config lxc.lxcpath
+	fi
 else
+	# output the parameter
 	echo "$lxcpath"
 fi
diff --git a/cdist/conf/type/__lxc_container/explorer/state b/cdist/conf/type/__lxc_container/explorer/state
index 0c79d3c8..fc37edf7 100755
--- a/cdist/conf/type/__lxc_container/explorer/state
+++ b/cdist/conf/type/__lxc_container/explorer/state
@@ -4,15 +4,40 @@
 # Outputs if the container exist
 #  possible values: present || absent || running || stopped || frozen
 
-# source param lib
-. "$__type/helper-param"
-. "$__type/helper-exec"
+# general parameters
+name="$__object/parameter/name"
+if [ -f "$name" ]; then
+    name="$(cat "$name")"
+else
+    name="$__object_id"
+fi
+user="$(cat "$__object/parameter/user")"
+
+# lxcpath
+lxcpath="$(cat "$__object/parameter/lxcpath" 2>/dev/null || true)"
+
+# assemble optional parameters
+LXC_PARAM=""
+if [ "$lxcpath" ]; then
+	LXC_PARAM="$LXC_PARAM -P \"$lxcpath\""
+fi
+
+
+# function to get information
+lxc_info() {
+	if [ "$user" != "$USER" ]; then
+		su -s "$(which -- lxc-info)" -- "$user" $LXC_PARAM -n "$name" -H "$@"
+	else
+		lxc-info $LXC_PARAM -n "$name" -H "$@"
+	fi
+}
+
 
 # the lxc-info command will exit non-zero if container not exist
-if ! lxc_c info -H -S >/dev/null 2>&1; then
+if ! lxc_info -S >/dev/null 2>&1; then
 	# not exist
 	echo "absent"
 else
 	# print state (command output matches to type states)
-	lxc_c info -H -s | tr '[:upper:]' '[:lower:]'
+	lxc_info -s | tr '[:upper:]' '[:lower:]'
 fi
diff --git a/cdist/conf/type/__lxc_container/gencode-local b/cdist/conf/type/__lxc_container/gencode-local
index 1c8a18c8..dbfca7e8 100755
--- a/cdist/conf/type/__lxc_container/gencode-local
+++ b/cdist/conf/type/__lxc_container/gencode-local
@@ -48,7 +48,7 @@ OUT
 		printf "%s" "$del_dest" > "$__object/files/remote-tmpfile_del"
 		# upload diff
 		cat <<OUT
-$__remote_copy "$__object/files/config.add" "$my_target_host:$del_dest"
+$__remote_copy "$__object/files/config.del" "$my_target_host:$del_dest"
 OUT
 	fi
 fi
diff --git a/cdist/conf/type/__lxc_container/gencode-remote b/cdist/conf/type/__lxc_container/gencode-remote
index d6cf4e74..b03f13d2 100755
--- a/cdist/conf/type/__lxc_container/gencode-remote
+++ b/cdist/conf/type/__lxc_container/gencode-remote
@@ -4,24 +4,44 @@
 # Does all common stuff for the container on the host
 
 
-# Source to the common parameters
-. "$__type/helper-param"
+# Parameter gathering
+name="$__object/parameter/name"
+if [ -f "$name" ]; then
+    name="$(cat "$name")"
+else
+    name="$__object_id"
+fi
+user="$(cat "$__object/parameter/user")"
 
 # check both states
 state_is="$(cat "$__object/explorer/state")"
 state_should="$(cat "$__object/parameter/state")"
 
+# general lxc things
+lxcpath="$(cat "$__object/parameter/lxcpath" || true)"
 
-# Check if there is a difference within the states
-if [ "$state_is" != "$state_should" ]; then
-	# change privileges (too complex to make exceptions)
-	cat <<USER
+
+# Summerize common parameter arguments, listed in manual as "COMMON OPTIONS"
+#  will passed raw; must qouted manually
+LXC_PARAM="-q"
+
+# if lxcpath given
+if [ -n "$lxcpath" ]; then
+    LXC_PARAM="$LXC_PARAM -P \"$lxcpath\""
+fi
+
+
+
+# handle if the container should exist or not
+case "$state_should" in
+	present|running|stopped|frozen)
+		# only relevant if the container does not exist before
+		if [ "$state_is" = "absent" ]; then
+			# change privileges (too complex to make exceptions)
+			cat <<USER
 su "$user" - <<SU
 USER
 
-	# handle if the container should exist or not
-	case "$state_should" in
-		present|running|stopped|frozen)
 			# check, if the container should be created or cloned
 			if [ -f "$__object/parameter/clone" ]; then
 				copy_from="$(cat "$__object/parameter/clone")"
@@ -79,41 +99,50 @@ LXC
 lxc-create $LXC_PARAM -n "$name" -t "$template" $create_opts -- $template_opts
 LXC
 			fi
+		fi
 
-			# write to the message system
-			echo "create" >> "$__messages_out"
-			;;
+		# end of su here-document
+		echo "SU"
 
-		absent)
-			# shutdown and delete the container
-			#  we could force-delete, but better be polite
-			#  snapshots are deleted, too - to keep everything clean. May someone does not want it?
-			cat <<LXC
+		# write to the message system
+		echo "create" >> "$__messages_out"
+		;;
+
+	absent)
+		# change privileges (too complex to make exceptions)
+		cat <<USER
+su "$user" - <<SU
+USER
+
+		# shutdown and delete the container
+		#  we could force-delete, but better be polite
+		#  snapshots are deleted, too - to keep everything clean. May someone does not want it?
+		cat <<LXC
 lxc-stop $LXC_PARAM -n "$name"
 lxc-destroy $LXC_PARAM --snapshots -n "$name"
 LXC
 
-			# write to the message system
-			echo "destroy" >> "$__messages_out"
-			;;
+		# end of su here-document
+		echo "SU"
 
-		# error handling: invalid state
-		*)
-			printf "Container '%s' in unknown state %s\n" "$name" "$state_should" >&2
-			exit 2
-			;;
-	esac
+		# write to the message system
+		echo "destroy" >> "$__messages_out"
+		;;
 
-	# end of su here-document
-	echo "SU"
-fi
+	# error handling: invalid state
+	*)
+		printf "Container '%s' in unknown state %s\n" "$name" "$state_should" >&2
+		exit 2
+		;;
+esac
 
 
+# shortcut
+tmppath="$__object/files/remote-tmpfile"
+
 # Check the configurations only if the container exists and there are changes
 if [ "$state_should" != "absent" ] && \
 	( [ -f "${tmppath}_add" ] || [ -f "${tmppath}_del" ] ); then
-	# shortcut
-	tmppath="$__object/files/remote-tmpfile"
 
 	# generate config path
 	container_config="$(cat "$__object/explorer/lxcpath")/$name/config"
@@ -123,6 +152,15 @@ tmpconfig="\$(mktemp "$container_config.XXXXXXXXXX")"
 cp -p "$container_config" "\$tmpconfig"
 DONE
 
+	# check if smth. to be deleted
+	# must be before adding, because it takes the content of the original file
+	if [ -f "${tmppath}_del" ]; then
+		cat <<DONE
+grep -v -f "$(cat "${tmppath}_del")" "$container_config" > "\$tmpconfig"
+rm -rf "$(cat "${tmppath}_del")"
+DONE
+	fi
+
 	# check if smth. to be added
 	if [ -f "${tmppath}_add" ]; then
 		cat <<DONE
@@ -131,14 +169,6 @@ rm -rf "$(cat "${tmppath}_add")"
 DONE
 	fi
 
-	# check if smth. to be deleted
-	if [ -f "${tmppath}_del" ]; then
-		cat <<DONE
-grep -v -f "$(cat "${tmppath}_del")" "\$tmpconfig" > "\$tmpconfig"
-rm -rf "$(cat "${tmppath}_del")"
-DONE
-	fi
-
 
 	# apply all changes
 	cat <<DONE
diff --git a/cdist/conf/type/__lxc_container/helper-exec b/cdist/conf/type/__lxc_container/helper-exec
deleted file mode 100644
index e8290ab7..00000000
--- a/cdist/conf/type/__lxc_container/helper-exec
+++ /dev/null
@@ -1,68 +0,0 @@
-# vi: syntax=sh
-# helper-exec
-
-# Adds functions to easily handle the execution of lxc commands
-#  requires to source the helper-param before .. ;)
-
-# Following global variables required (from helper-param):
-#  $user: the unix user to execute lxc commands with
-#  $name: the lxc container name
-
-
-# Execute a command with a special user.
-#
-# Parameters:
-#  1: the execution user
-#  2: the execution command
-#  3: all further parameters for the command
-exec_user() {
-    # save important arguments
-    _user="$1"
-    _cmd="$(which -- "$2")"
-    shift 2
-
-    # check if the user must be switched
-    if [ "$_user" != "$USER" ]; then
-        # execute it wit su
-        #  here, compatibility to non-sudo systems is provided and the "shell parameter thing" is misused
-        #  to provide more parameter safety.
-        su -s "$_cmd" -- "$_user" "$@"
-
-    # no switch of the user - keep the same
-    else
-        # execute the command
-        "$_cmd" "$@"
-    fi
-}
-
-# Function to execute lxc commands in the correct execution. Only works for container-specific commands.
-# Default things such as default parameters (see $LXC_PARAM), execution user and the container name are
-# applied within the function.
-#
-# Parameters:
-#  1: the lxc command without the preceding "lxc"
-#  2-n: all other lxc arguments
-lxc_c() {
-    # get exec name
-    lxccmd="lxc-$1"
-    shift
-
-    # execute the command
-    exec_user "$user" "$lxccmd" $LXC_PARAM -n "$name" "$@"
-}
-
-# Function to execute global lxc commands within the correct execution environment. Container-specific
-# commands require to pass the -n flag with the container name, rather lxc_c is recommended.
-# Default things such as default parameters (see $LXC_PARAM) and execution are applied to the command.
-#
-# Parameters:
-#  1: the lxc command without the preceding "lxc"
-#  2-n: all other lxc arguments
-lxc_g() {
-    # get exec name
-    lxccmd="lxc-$1"
-    shift
-
-    # execute the command
-    exec_user "$user" "$lxccmd" $LXC_PARAM "$@"
-}
diff --git a/cdist/conf/type/__lxc_container/helper-param b/cdist/conf/type/__lxc_container/helper-param
deleted file mode 100644
index eae7f462..00000000
--- a/cdist/conf/type/__lxc_container/helper-param
+++ /dev/null
@@ -1,27 +0,0 @@
-# vi: syntax=sh
-# helper-param
-
-# Shortcut to optain all basic required parameters for working.
-
-
-# Parameter gathering
-name="$__object/parameter/name"
-if [ -f "$name" ]; then
-    name="$(cat "$name")"
-else
-    name="$__object_id"
-fi
-user="$(cat "$__object/parameter/user")"
-
-# general lxc things
-lxcpath="$(cat "$__object/parameter/lxcpath" || true)"
-
-
-# Summerize common parameter arguments, listed in manual as "COMMON OPTIONS"
-#  will passed raw; must qouted manually
-LXC_PARAM="-q"
-
-# if lxcpath given
-if [ -n "$lxcpath" ]; then
-    LXC_PARAM="$LXC_PARAM -P \"$lxcpath\""
-fi
diff --git a/cdist/conf/type/__lxc_container/manifest b/cdist/conf/type/__lxc_container/manifest
index 7c82e59f..b1f05b43 100755
--- a/cdist/conf/type/__lxc_container/manifest
+++ b/cdist/conf/type/__lxc_container/manifest
@@ -10,6 +10,7 @@
 # basepath of the parameter folder (shortcut)
 param="$__object/parameter/"
 
+
 # ====================== #
 # ==  WARNING CHECKS  == #
 # ====================== #
@@ -63,17 +64,18 @@ fi
 #  1: parameter name
 #  2: file to print
 write_multi_param() {
-	touch "$2"
+	if ! [ -f "$__object/parameter/$1" ]; then return 0; fi
+
 	while read line; do
 		if [ "$line" = "-" ]; then
 			# append stdin
-			cat "$__object/stdin" >> "$2"
+			cat "$__object/stdin"
 		elif [ -f "$line" ]; then
 			# append file content
-			cat "$line" >> "$2"
+			cat "$line"
 		else
 			# print out content
-			printf "%s\n" "$line" >> "$2"
+			printf "%s\n" "$line"
 		fi
 	done < "$__object/parameter/$1"
 }
@@ -83,24 +85,26 @@ write_multi_param() {
 # Parameters:
 #  1: the file which should be handled
 trimm_whitespaces() {
-	cp "$1" "$1"~  # backup file to see original content
-	grep -E -v "^([[:blank:]]*)(#|$)" "$1" \
-		| sed 's/^[[:blank:]]*//; s/[[:blank:]]*=[[:blank:]]*/ = /; s/[[:blank:]]*$//'
+	mv "$1" "$1"~  # backup file to see original content
+	grep -E -v "^([[:blank:]]*)(#|$)" "$1"~ \
+		| sed 's/^[[:blank:]]*//; s/[[:blank:]]*=[[:blank:]]*/ = /; s/[[:blank:]]*$//' \
+		> "$1"
 }
 
+
 # Only required if container will not be absent
-# get the wanted configuration lines
+# prepare the wanted configuration lines
 if [ "$(cat "$__object/parameter/state")" != "absent" ]; then
 	# Create the files directory
 	mkdir "$__object/files/"
 
 	# Create the file of the locally configuration
 	conffile="$__object/files/config-present"
-	write_multi_param config "$conffile"
-	trimm_whitespaces "$conffile" > "$conffile"
+	write_multi_param config > "$conffile"
+	trimm_whitespaces "$conffile"
 
 	# Create the file of the absent configuration
 	absentfile="$__object/files/config-absent"
-	write_multi_param config-absent "$absentfile"
-	trimm_whitespaces "$absentfile" > "$absentfile"
+	write_multi_param config-absent > "$absentfile"
+	trimm_whitespaces "$absentfile"
 fi

From c92d56293488eebf8df96f55153a0ca1fb16edc7 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 4 Jul 2020 14:25:13 +0200
Subject: [PATCH 010/411] [__lxc_container] refactored gencode-remote
 processing

litte changes to `gencode-local` and more messages, too.
---
 cdist/conf/type/__lxc_container/gencode-local |   3 +-
 .../conf/type/__lxc_container/gencode-remote  | 133 ++++++++++++------
 cdist/conf/type/__lxc_container/man.rst       |  12 ++
 3 files changed, 101 insertions(+), 47 deletions(-)

diff --git a/cdist/conf/type/__lxc_container/gencode-local b/cdist/conf/type/__lxc_container/gencode-local
index dbfca7e8..04935070 100755
--- a/cdist/conf/type/__lxc_container/gencode-local
+++ b/cdist/conf/type/__lxc_container/gencode-local
@@ -4,8 +4,7 @@
 # This uploads the diff of the configuration file, which is applied by gencode-remote
 
 
-# check both states
-state_is="$(cat "$__object/explorer/state")"
+# Check desired state
 state_should="$(cat "$__object/parameter/state")"
 
 # Check the configurations only if the container exists
diff --git a/cdist/conf/type/__lxc_container/gencode-remote b/cdist/conf/type/__lxc_container/gencode-remote
index b03f13d2..d1e06b6e 100755
--- a/cdist/conf/type/__lxc_container/gencode-remote
+++ b/cdist/conf/type/__lxc_container/gencode-remote
@@ -2,6 +2,16 @@
 # gencode-remote
 
 # Does all common stuff for the container on the host
+#
+# Following stages need to be observed when executing this type
+#  1. Container creation IF $state_is = absent
+#  2. patching container config IF NOT $state_should = absent
+#  3. look after runstates (running,frozen,stopped)
+#  4. Container destruction IF $state_should = absent
+#
+# It is only required if:
+#  - $state_is != $state_should
+#  - config changed (detects if changes are uploaded)
 
 
 # Parameter gathering
@@ -18,7 +28,7 @@ state_is="$(cat "$__object/explorer/state")"
 state_should="$(cat "$__object/parameter/state")"
 
 # general lxc things
-lxcpath="$(cat "$__object/parameter/lxcpath" || true)"
+lxcpath="$(cat "$__object/parameter/lxcpath" 2>/dev/null || true)"
 
 
 # Summerize common parameter arguments, listed in manual as "COMMON OPTIONS"
@@ -31,9 +41,25 @@ if [ -n "$lxcpath" ]; then
 fi
 
 
+# shortcut
+tmppath="$__object/files/remote-tmpfile"
+# shortcut function
+config_changes() {
+	if ( [ -f "${tmppath}_add" ] || [ -f "${tmppath}_del" ] ); then
+		return 0
+	else
+		return 1
+	fi
+}
+
+
+# Short curcit if nothing changed
+if [ "$state_is" = "$state_should" ] && ( ! config_changes ); then exit; fi
+
 
 # handle if the container should exist or not
 case "$state_should" in
+	# all states where the container should exist
 	present|running|stopped|frozen)
 		# only relevant if the container does not exist before
 		if [ "$state_is" = "absent" ]; then
@@ -98,35 +124,25 @@ LXC
 				cat <<LXC
 lxc-create $LXC_PARAM -n "$name" -t "$template" $create_opts -- $template_opts
 LXC
+				# remove empty tempfile
+				if [ "$empty_config" ]; then
+					cat <<RM
+rm -rf "$empty_config"
+RM
+				fi
 			fi
+
+			# end of su here-document
+			echo "SU"
+
+			# write to the message system
+			echo "create" >> "$__messages_out"
 		fi
-
-		# end of su here-document
-		echo "SU"
-
-		# write to the message system
-		echo "create" >> "$__messages_out"
 		;;
 
+
+	# list other states to get a correct error message in the wildcard case
 	absent)
-		# change privileges (too complex to make exceptions)
-		cat <<USER
-su "$user" - <<SU
-USER
-
-		# shutdown and delete the container
-		#  we could force-delete, but better be polite
-		#  snapshots are deleted, too - to keep everything clean. May someone does not want it?
-		cat <<LXC
-lxc-stop $LXC_PARAM -n "$name"
-lxc-destroy $LXC_PARAM --snapshots -n "$name"
-LXC
-
-		# end of su here-document
-		echo "SU"
-
-		# write to the message system
-		echo "destroy" >> "$__messages_out"
 		;;
 
 	# error handling: invalid state
@@ -137,13 +153,8 @@ LXC
 esac
 
 
-# shortcut
-tmppath="$__object/files/remote-tmpfile"
-
 # Check the configurations only if the container exists and there are changes
-if [ "$state_should" != "absent" ] && \
-	( [ -f "${tmppath}_add" ] || [ -f "${tmppath}_del" ] ); then
-
+if config_changes; then
 	# generate config path
 	container_config="$(cat "$__object/explorer/lxcpath")/$name/config"
 	# create remote tmpfile for config
@@ -180,54 +191,86 @@ DONE
 fi
 
 
-# TODO user context again
 # Check if there is a difference within the states
-if [ "$state_is" != "$state_should" ] && [ "$state_should" != "absent" ]; then
+if [ "$state_is" != "$state_should" ] && [ "$state_should" != "present" ]; then
 	# change privileges (too complex to make exceptions)
 	cat <<USER
 su "$user" - <<SU
 USER
 
+	# unfreeze if nessecary to change state
+	if [ "$state_is" = "frozen" ]; then
+		cat <<LXC
+lxc-unfreeze $LXC_PARAM -n "$name"
+LXC
+		echo "melt" >> "$__messages_out"
+	fi
+
 	# handle if the container should exist or not
 	case "$state_should" in
 		running)
-			if [ "$state_is" = "frozen" ]; then
+			# already running if unfreezed
+			if [ "$state_is" != "frozen" ]; then
 				cat <<LXC
-lxc-unfreeze $LXC_PARAM -n "$name"
-LXC
-			fi
-			cat <<LXC
 lxc-start $LXC_PARAM -n "$name"
 LXC
+				echo "start" >> "$__messages_out"
+			fi
 			;;
 
+		# stop the container if it should be removed
 		stopped)
-			if [ "$state_is" = "frozen" ]; then
-				cat <<LXC
-lxc-unfreeze $LXC_PARAM -n "$name"
-LXC
-			fi
 			cat <<LXC
 lxc-stop $LXC_PARAM -n "$name"
 LXC
+			echo "stop" >> "$__messages_out"
+			;;
+
+		# container should be stopped if not already done
+		absent)
+			if [ "$state_is" != "stopped" ]; then
+				cat <<LXC
+lxc-stop $LXC_PARAM -n "$name"
+LXC
+			fi
 			;;
 
 		frozen)
 			# stopped containers can't be frozen; they must be started first
-			if [ "$state_is" = "stopped" ]; then
+			if [ "$state_is" = "stopped" ] || [ "$state_is" = "absent" ]; then
 				cat <<LXC
 lxc-start $LXC_PARAM -n "$name"
 LXC
+				echo "start" >> "$__messages_out"
 			fi
 			cat <<LXC
 lxc-freeze $LXC_PARAM -n "$name"
 LXC
+			echo "freeze" >> "$__messages_out"
 			;;
 
 		*)
-			;; # must be checked by previous case
+			# must be checked by previous case
+			;;
 	esac
 
 	# end of su here-document
 	echo "SU"
 fi
+
+
+# Check if the container needs to be removed
+if [ "$state_should" = "absent" ]; then
+	# change privileges (too complex to make exceptions)
+	# then, shutdown and delete the container
+	#  we could force-delete, but better be polite
+	#  snapshots are deleted, too - to keep everything clean. May someone does not want it?
+	cat <<USER
+su "$user" - <<SU
+lxc-destroy $LXC_PARAM --snapshots -n "$name"
+SU
+USER
+
+	# write to the message system
+	echo "destroy" >> "$__messages_out"
+fi
diff --git a/cdist/conf/type/__lxc_container/man.rst b/cdist/conf/type/__lxc_container/man.rst
index f84b0619..ddf08daa 100644
--- a/cdist/conf/type/__lxc_container/man.rst
+++ b/cdist/conf/type/__lxc_container/man.rst
@@ -123,3 +123,15 @@ destroy
 
 config
     The container configuration changed.
+
+start
+    Started the container.
+
+stop
+    Stopped the container.
+
+freeze
+    Freezed all container processes. The container will be started if not yet done to be able to freeze.
+
+melt
+    Unfreezed all container processes. Will be done if any other state (except `present`) should be reached.

From e9256b6e8e3403e1435e4f1b65726dbb5dd991eb Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 4 Jul 2020 18:19:46 +0200
Subject: [PATCH 011/411] [__lxc_container] create: added backingstore and
 template options

With some minor changes, the backingstore parameters and some more
template options where added (incl. `man.rst`).
---
 cdist/conf/type/__lxc_container/gencode-local | 22 ++++++-
 .../conf/type/__lxc_container/gencode-remote  | 56 ++++++++++++++----
 cdist/conf/type/__lxc_container/man.rst       | 59 +++++++++++++++++--
 cdist/conf/type/__lxc_container/manifest      | 39 ++++++++++--
 .../type/__lxc_container/parameter/optional   |  9 +++
 cdist/conf/type/__lxc_container/todo.txt      | 17 +-----
 6 files changed, 161 insertions(+), 41 deletions(-)

diff --git a/cdist/conf/type/__lxc_container/gencode-local b/cdist/conf/type/__lxc_container/gencode-local
index 04935070..e0572bb4 100755
--- a/cdist/conf/type/__lxc_container/gencode-local
+++ b/cdist/conf/type/__lxc_container/gencode-local
@@ -4,9 +4,18 @@
 # This uploads the diff of the configuration file, which is applied by gencode-remote
 
 
-# Check desired state
+# Get required parameters
+name="$__object/parameter/name"
+if [ -f "$name" ]; then
+    name="$(cat "$name")"
+else
+    name="$__object_id"
+fi
+
+state_is="$(cat "$__object/explorer/state")"
 state_should="$(cat "$__object/parameter/state")"
 
+
 # Check the configurations only if the container exists
 if [ "$state_should" != "absent" ]; then
 	# do changes
@@ -22,6 +31,13 @@ if [ "$state_should" != "absent" ]; then
 		my_target_host="${__target_host}"
 	fi
 
+	# create remote container directory if container is not yet available
+	if [ "$state_is" = "absent" ]; then
+		# this must be done because the container will be created after the configuration is uploaded
+		# would also be possible to do everything in gencode-remote to keep the order
+		$__remote_exec $__target_host "mkdir $(dirname "$container_config")"
+	fi
+
 
 	# config-present
 	# remove all config lines from config-present that are already set or absent anyway
@@ -31,7 +47,7 @@ if [ "$state_should" != "absent" ]; then
 		# get temp file name
 		add_dest="$($__remote_exec $__target_host "mktemp $container_config-add.XXXXXXXXXX")"
 		printf "%s" "$add_dest" > "$__object/files/remote-tmpfile_add"
-		# upload diff
+		# upload delta
 		cat <<OUT
 $__remote_copy "$__object/files/config.add" "$my_target_host:$add_dest"
 OUT
@@ -45,7 +61,7 @@ OUT
 		# get temp file name
 		del_dest="$($__remote_exec $__target_host "mktemp $container_config-del.XXXXXXXXXX")"
 		printf "%s" "$del_dest" > "$__object/files/remote-tmpfile_del"
-		# upload diff
+		# upload delta
 		cat <<OUT
 $__remote_copy "$__object/files/config.del" "$my_target_host:$del_dest"
 OUT
diff --git a/cdist/conf/type/__lxc_container/gencode-remote b/cdist/conf/type/__lxc_container/gencode-remote
index d1e06b6e..8ee5623b 100755
--- a/cdist/conf/type/__lxc_container/gencode-remote
+++ b/cdist/conf/type/__lxc_container/gencode-remote
@@ -41,9 +41,8 @@ if [ -n "$lxcpath" ]; then
 fi
 
 
-# shortcut
+# shortcut for checking config changes
 tmppath="$__object/files/remote-tmpfile"
-# shortcut function
 config_changes() {
 	if ( [ -f "${tmppath}_add" ] || [ -f "${tmppath}_del" ] ); then
 		return 0
@@ -52,6 +51,26 @@ config_changes() {
 	fi
 }
 
+# shortcut to add bdev parameters if available
+#  $1: $__object parameter name without "bdev-"
+#  $2: target parameter flag (like --foo), else "--$1" will be used
+bdev_add_if_available() {
+	_param_path="$__object/parameter/bdev-$1"
+	if [ -f "$_param_path" ]; then
+		backingstore_opts="$backingstore_opts ${2:---$1} '$(cat "$_param_path")'"
+	fi
+}
+
+# shortcut to add template parameters if available
+#  $1: $__object parameter
+#  $2: target parameter flag (like --foo), else "--$1" will be used
+template_add_if_available() {
+	_param_path="$__object/parameter/$1"
+	if [ -f "$_param_path" ]; then
+		template_opts="$template_opts ${2:---$1} '$(cat "$_param_path")'"
+	fi
+}
+
 
 # Short curcit if nothing changed
 if [ "$state_is" = "$state_should" ] && ( ! config_changes ); then exit; fi
@@ -68,6 +87,21 @@ case "$state_should" in
 su "$user" - <<SU
 USER
 
+			# backingstore options
+			backingstore_opts=""
+			if [ -f "$__object/parameter/backingstore" ]; then
+				# -B works for both types, they have different long opts
+				backingstore_opts="-B '$(cat "$__object/parameter/backingstore")'"
+
+				# add parameters if available
+				bdev_add_if_available dir
+				bdev_add_if_available fstype
+				bdev_add_if_available fssize
+				bdev_add_if_available lvname
+				bdev_add_if_available vgname
+				bdev_add_if_available thinpool
+			fi
+
 			# check, if the container should be created or cloned
 			if [ -f "$__object/parameter/clone" ]; then
 				copy_from="$(cat "$__object/parameter/clone")"
@@ -86,7 +120,7 @@ USER
 
 				# print lxc-copy code (because of the lxcpath thing, $LXC_PARAM conflicts)
 				cat <<LXC
-lxc-copy $lxc_opts -n "$copy_from" --newname "$name"
+lxc-copy $lxc_opts $backingstore_opts -n "$copy_from" --newname "$name"
 LXC
 			else
 				template="$(cat "$__object/parameter/template")"
@@ -95,7 +129,7 @@ LXC
 				create_opts=""
 				if [ -f "$__object/parameter/no-default-config" ]; then
 					# generate a random empty file and append
-					empty_config="$($__remote_exec $__target_host "mktemp \${TMPDIR:-/tmp}/cdist_lxc-empty.XXXXXXXXXX")"
+					empty_config="$($__remote_exec $__target_host "mktemp \${TMPDIR:-/tmp}/cdist_lxc-empty-conf.XXXXXXXXXX")"
 					create_opts="$create_opts -f \"$empty_config\""
 
 				elif [ -f "$__object/parameter/default-config" ]; then
@@ -105,13 +139,13 @@ LXC
 
 				# assemble template options
 				template_opts=""
-				# add common options
-				if [ -f "$__object/parameter/release" ]; then
-					template_opts="$template_opts -r \"$(cat "$__object/parameter/release")\""
-				fi
-				if [ -f "$__object/parameter/arch" ]; then
-					template_opts="$template_opts -a \"$(cat "$__object/parameter/arch")\""
+				# add common options (may require different options for different templates)
+				template_add_if_available release -r
+				template_add_if_available arch -a
+				if [ -f "$__object/parameter/mirror" ]; then
+					template_opts="$template_opts --mirror='$(cat "$__object/parameter/mirror")'"
 				fi
+				template_add_if_available ssh-key -S
 
 				# at last, apply custom options
 				if [ -f "$__object/parameter/template-opts" ]; then
@@ -122,7 +156,7 @@ LXC
 
 				# print lxc-create code
 				cat <<LXC
-lxc-create $LXC_PARAM -n "$name" -t "$template" $create_opts -- $template_opts
+lxc-create $LXC_PARAM $backingstore_opts -n "$name" -t "$template" $create_opts -- $template_opts
 LXC
 				# remove empty tempfile
 				if [ "$empty_config" ]; then
diff --git a/cdist/conf/type/__lxc_container/man.rst b/cdist/conf/type/__lxc_container/man.rst
index ddf08daa..547178c5 100644
--- a/cdist/conf/type/__lxc_container/man.rst
+++ b/cdist/conf/type/__lxc_container/man.rst
@@ -62,6 +62,45 @@ BOOLEAN PARAMETERS
 
 None.
 
+CREATE PARAMETERS
+-----------------
+
+This parameters will be used with *TEMPLATE PARAMETERS* or *CLONE PARAMETERS* to create the container
+if the container is absent. All of these parameters will only used if the container will be created,
+in all other cases, these parameters will be ignored.
+
+All backingstore parameters (incl. all bdev-* ones) can be read from the manual lxc-create(1) at option
+`--bdev` or lxc-copy(1) at option `--backingstore`. The possibilities may differ from creating and cloning.
+The bdev-* parameters are only available for some backingstores. If used for other backingstores, the
+type aborts.
+
+backingstore
+    Set the storage for the container root partition. Commonly, there should be at minimum `dir` (or `none`),
+    `lvm`, `loop`, `btrfs`, `zfs` or `best` be possible.
+
+bdev-dir
+    Specifies an other path for the rootfs directory instead in the lxc configuration directoy. Only
+    available for the backingstore `dir` or `none`.
+
+bdev-fstype
+    Specifies the filesystem to be created instead of the default value. Only available for the
+    backingstore `lvm` and `loop`.
+
+bdev-fssize
+    Specifies the filesystem size to be created instead of the default value. Only available for the
+    backingstore `lvm` and `loop`.
+
+bdev-lvname
+    The custom LVM logical volume that will be created for the new container. Only availabe for the
+    backingstore `lvm`.
+
+bdev-vgname
+    The custon LVM volume group in there the logical volume will be created. Only available for the
+    backingstore `lvm`.
+
+bdev-thinpool
+    The custom thinpool the logical volume will created in. Only available for the backingstore `lvm`.
+
 TEMPLATE PARAMETERS
 -------------------
 This or the *CLONE PARAMETERS* are required to create an container and must be present if the container
@@ -88,15 +127,25 @@ no-default-config
     **(Boolean value)** This parameter avoids using a default config file by using an empty file instead.
 
 template-opts
-    Raw options which get passed to the template directly. Can be used multiple times. The argument must
-    contain a single string (*at best, everything is inside one quote*), else it will interpreted as an
-    further argument for the type. Some values should be quoted inside the quotes, too.
+    Raw options which get passed to the template directly. Can be used multiple times. If the argument
+    contain spaces, it will be interpreted as multiple arguments and must be escaped for an normal posix
+    shell if this is not wanted.
+
+The following parameters are shortcuts for some template options. As templates may support it or not, you
+should be careful when using them. You can check all available template options with
+`lxc-create -t $template -h` (incl. lxc-create help, too) or `/usr/share/lxc/templates/lxc-$template -h`.
 
 release
-    TBA.
+    Sets the release for the wanted distribution. Uses the `-r` option.
 
 arch
-    TBA.
+    Sets the architecture used for the creating container. Uses the `-a` option.
+
+mirror
+    Sets the mirror to download from. Uses the `--mirror=` option.
+
+ssh-key
+    Sets the ssh key for the root user- Uses the `-S` option.
 
 CLONE PARAMETERS
 ----------------
diff --git a/cdist/conf/type/__lxc_container/manifest b/cdist/conf/type/__lxc_container/manifest
index b1f05b43..d903fa1c 100755
--- a/cdist/conf/type/__lxc_container/manifest
+++ b/cdist/conf/type/__lxc_container/manifest
@@ -11,11 +11,6 @@
 param="$__object/parameter/"
 
 
-# ====================== #
-# ==  WARNING CHECKS  == #
-# ====================== #
-
-
 # ==================== #
 # ==  ERROR CHECKS  == #
 # ==================== #
@@ -38,7 +33,7 @@ if [ -f "$param/template" ]; then
 fi
 
 # no template options if --clone given
-if [ -f "$__object/parameter/clone" ]; then
+if [ -f "$param/clone" ]; then
 	if [ -f "$param/template-opts" ] \
 		|| [ -f "$param/default-config" ] \
 		|| [ -f "$param/no-default-config" ] \
@@ -50,6 +45,38 @@ if [ -f "$__object/parameter/clone" ]; then
 	fi
 fi
 
+# check backingstore values
+if [ -f "$param/backingstore" ]; then
+	backingstore="$(cat "$param/backingstore")"
+
+	if [ "$backingstore" != "dir" ] && [ "$backingstore" != "none" ]; then
+		if [ -f "$param/bdev-dir" ]
+		then
+			>&2 echo "error: --bdev-dir is only possible for backingstore 'dir' or 'none'"
+			exit 2
+		fi
+	fi
+
+	if [ "$backingstore" != "lvm" ] && [ "$backingstore" != "loop" ]; then
+		if [ -f "$param/bdev-fstype" ] \
+			|| [ -f "$param/bdev-fssize" ]
+		then
+			>&2 echo "error: --bdev-fstype and --bdev-fssize only available for backingstore 'lvm' or 'loop'"
+			exit 2
+		fi
+	fi
+
+	if [ "$backingstore" != "lvm" ]; then
+		if [ -f "$param/bdev-lvname" ] \
+			|| [ -f "$param/bdev-vgname" ] \
+			|| [ -f "$param/bdev-thinpool" ]
+		then
+			>&2 echo "error: --bdev-{lv,vg}name and --bdev-thinpool only available for backingstore 'lvm'"
+			exit 2
+		fi
+	fi
+fi
+
 
 
 
diff --git a/cdist/conf/type/__lxc_container/parameter/optional b/cdist/conf/type/__lxc_container/parameter/optional
index fc4bb71b..257283d8 100644
--- a/cdist/conf/type/__lxc_container/parameter/optional
+++ b/cdist/conf/type/__lxc_container/parameter/optional
@@ -6,5 +6,14 @@ template
 default-config
 release
 arch
+mirror
+ssh-key
 clone
 clonepath
+backingstore
+bdev-dir
+bdev-fstype
+bdev-fssize
+bdev-lvname
+bdev-vgname
+bdev-thinpool
diff --git a/cdist/conf/type/__lxc_container/todo.txt b/cdist/conf/type/__lxc_container/todo.txt
index 17c4edaf..e5f8f471 100644
--- a/cdist/conf/type/__lxc_container/todo.txt
+++ b/cdist/conf/type/__lxc_container/todo.txt
@@ -1,28 +1,13 @@
 # List of all featues that should be implemented
-storage:
-  - backing_storage
-  - other settings prior to storage
 template:
   - type: a.E. debian, ubuntu -> what on change?
-  - release
-  - arch
-  - possible ssh auth key?
   - possible root_password?
   - possible packages?
-  - possible mirrors?
   - other options??
-auto..:
+auto.. (lxc autostart):
   - group
   - start
-lxc config ..
-how to find lxc containers path?
-lxc-copy: ephemeral containers? -> seperate type if it makes sense
 
 If config already created, it's considered the container is already created
   -> first create; then setup
   detection if template changed??
-
-
-lxc configuration at startup:
-  flag -f can used to set a default config (instead the system default)
-  there is no problem to change config before the first startup

From 1c8eee17499012a1f8b318b502603d37ca01dc13 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 5 Jul 2020 11:52:34 +0200
Subject: [PATCH 012/411] [__lxc_container] Updated manpage (incl. examples,
 copyright, ..)

Should be fine now.
---
 .../conf/type/__lxc_container/gencode-remote  |  2 +-
 cdist/conf/type/__lxc_container/man.rst       | 99 ++++++++++++++++---
 2 files changed, 86 insertions(+), 15 deletions(-)

diff --git a/cdist/conf/type/__lxc_container/gencode-remote b/cdist/conf/type/__lxc_container/gencode-remote
index 8ee5623b..ef61c66d 100755
--- a/cdist/conf/type/__lxc_container/gencode-remote
+++ b/cdist/conf/type/__lxc_container/gencode-remote
@@ -105,7 +105,7 @@ USER
 			# check, if the container should be created or cloned
 			if [ -f "$__object/parameter/clone" ]; then
 				copy_from="$(cat "$__object/parameter/clone")"
-				copypath="$(cat "$__object/parameter/clonepath" || true)"
+				copypath="$(cat "$__object/parameter/clonepath" 2>/dev/null || true)"
 
 				# assemble own optional lxc options
 				lxc_opts=""
diff --git a/cdist/conf/type/__lxc_container/man.rst b/cdist/conf/type/__lxc_container/man.rst
index 547178c5..991c86fb 100644
--- a/cdist/conf/type/__lxc_container/man.rst
+++ b/cdist/conf/type/__lxc_container/man.rst
@@ -3,16 +3,19 @@ cdist-type__lxc_container(7)
 
 NAME
 ----
-cdist-type__lxc_container - Controls the configuration of a lxc container
+cdist-type__lxc_container - Controls the state and configuration of a lxc container
 
 DESCRIPTION
 -----------
 
-TBA.
+This type handles lxc containers. It supports containers from different users and paths.
+The state describes if the container exists and is running. The container will be created
+with the template or clone parameters if required, whatever is set. These options will
+be ignored if the container is already created, as the template script only runs at
+creation time.
 
 REQUIRED PARAMETERS
 -------------------
-
 None.
 
 OPTIONAL PARAMETERS
@@ -28,17 +31,17 @@ user
 state
     The state of the container, if it should exist or not.
 
+    present
+        The container exist, but it is ignored if the container will run or not **(default)**
+
     running
-        The container exist and is running (default)
+        The container exist and is running
 
     stopped
         The container exist, but does not run
 
     frozen
-        The container exist and is frozen
-
-    present
-        The container exist, but it is ignored if the container will run or not
+        The container exist and is frozen. If it is stopped before, it will be started and then freezed.
 
     absent
         The container does not exist
@@ -59,7 +62,6 @@ config-absent
 
 BOOLEAN PARAMETERS
 ------------------
-
 None.
 
 CREATE PARAMETERS
@@ -103,6 +105,7 @@ bdev-thinpool
 
 TEMPLATE PARAMETERS
 -------------------
+
 This or the *CLONE PARAMETERS* are required to create an container and must be present if the container
 should be created. If the template parameters are choosen, `--template` must be present. Then, none of
 the *CLONE PARAMETERS* must be present.
@@ -119,9 +122,9 @@ template
     this argument and should exist on the target host.
 
 default-config
-    Alternative path to the user-defined default config. For the root user, this is commonly at
-    `/etc/lxc/default.conf`. It will be included into the container configuration file at creation
-    time.
+    Alternative path to the user-defined default config. This file must exist on the target machine.
+    For the root user, this is commonly at `/etc/lxc/default.conf`. It will be included into the
+    container configuration file at creation time.
 
 no-default-config
     **(Boolean value)** This parameter avoids using a default config file by using an empty file instead.
@@ -149,13 +152,15 @@ ssh-key
 
 CLONE PARAMETERS
 ----------------
+
 This or the *TEMPLATE PARAMETERS* are required to create an container and must be present if the container
 should be created. If the clone parameters are choosen, `--clone` must be present. Then, none of the
 *TEMPLATE PARAMETERS* must be present.
 
 clone
     Instead of creating a new container with a given template, clone an other container and use him. The
-    argument takes the container name, which will be cloned. He should exist.
+    argument takes the container name, which will be cloned. He should exist and must be stopped. Else,
+    the clone will not work.
 
 clonepath
     The container path for the container to clone. It is like the `--lxcpath` parameter, but for the container
@@ -180,7 +185,73 @@ stop
     Stopped the container.
 
 freeze
-    Freezed all container processes. The container will be started if not yet done to be able to freeze.
+    Freezed all container processes. The container will be started too if the container is stopped before.
 
 melt
     Unfreezed all container processes. Will be done if any other state (except `present`) should be reached.
+
+ABORTS
+------
+Aborts in the following cases:
+
+The type aborts if there are incompatible arguments found. This may be *CLONE* and *TEMPLATE PARAMETERS*,
+backingstorage parameters for the wrong storage or incompatible template option shortcuts.
+
+When cloning, it aborts if the container to clone from does not exist or is not stopped with no warning. It
+may be possible to clone while the container is running, which requires a backing storage supporting it, but
+there is nothing in lxc-copy(1) which indicades something about the container state while cloning.
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # create a container that "just should exist" (does not start the container)
+    #  --template is required if the container needs to be created
+    __lxc_container foo --template debian
+    __lxc_container foo --state present --template debian
+
+    # remove the container
+    __lxc_container bar --state absent
+
+    # container that should definitely running
+    __lxc_container foobar --state running
+    # freeze this one
+    __lxc_container water --state frozen
+
+    # create with some template options
+    __lxc_container special --state stopped --template debian \
+        --release stretch --arch amd64 \
+        --template-opts "--packages=foo,bar --flush-cache"
+
+    # clone a container instead of creating it
+    #  foo must be stopped that this will work
+    __lxc_container foofighters --clone foo --state running
+
+    # handle configuration
+    __lxc_container peter --state running \
+        --config "$__files/lxc/default.conf" \  # content of files are possible
+        --config "lxc.group = onboot" \         # one lines
+        --config "lxc.start.auto   =  1" \      # spaces around the equal sign are ignored
+        --config-absent "lxc.group = testing" \ # configuration options can be removed
+        --config - <<ABSENT                     # stdin can be read once
+lxc.group = dev
+lxc.group = foobar
+ABSENT
+
+    # some more configuration
+    # configuration lookup does not go deeper to included files
+    __lxc_container dieter --state stopped \
+        --config "lxc.include = /etc/lxc/some-config.conf"
+        --config-absent "lxc.start.auto = 1"
+
+AUTHORS
+-------
+Matthias Stecher <matthiasstecher at gmx.de>
+
+COPYRIGHT
+---------
+Copyright \(C) 2020 Matthias Stecher. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

From 211393814a8c997d9f1ca27f86367ea8aace7cec Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 5 Jul 2020 20:15:41 +0200
Subject: [PATCH 013/411] [__lxc_container] Updated manpage (completly
 proof-read it)

I think it should be fine now .. :-)
---
 cdist/conf/type/__lxc_container/man.rst | 116 +++++++++++++-----------
 1 file changed, 62 insertions(+), 54 deletions(-)

diff --git a/cdist/conf/type/__lxc_container/man.rst b/cdist/conf/type/__lxc_container/man.rst
index 991c86fb..d0abf2e2 100644
--- a/cdist/conf/type/__lxc_container/man.rst
+++ b/cdist/conf/type/__lxc_container/man.rst
@@ -14,6 +14,13 @@ with the template or clone parameters if required, whatever is set. These option
 be ignored if the container is already created, as the template script only runs at
 creation time.
 
+The container can be `absent`, which means nothing exists. `present` is the opposite and
+completely does not care if the container runs or not. Further states imply that the
+container exists. There is `running` and `stopped`, which should be self-explainable.
+The state `frozen` means all container processes are frozen. If it moves from `stopped`
+to `frozen`, the container will be first started to be frozen. From `frozen` to any other
+state, the container will be first unfrozen to continue.
+
 REQUIRED PARAMETERS
 -------------------
 None.
@@ -23,13 +30,14 @@ OPTIONAL PARAMETERS
 
 name
     String which will used as container name instead of the object id. This should only required
-    if multiple lxc instances used from different users/directories (in junction with `--lxcpath`).
+    if multiple lxc instances used from different users/directories (in junction with ``--user``
+    and ``--lxcpath``).
 
 user
-    The unix user name in which lxc commands are executed. Each user have a different lxc instance.
+    The unix user name with all lxc commands are executed. Each user have a different lxc instance.
 
 state
-    The state of the container, if it should exist or not.
+    The state of the container, if it should exist or not, and running.
 
     present
         The container exist, but it is ignored if the container will run or not **(default)**
@@ -41,23 +49,23 @@ state
         The container exist, but does not run
 
     frozen
-        The container exist and is frozen. If it is stopped before, it will be started and then freezed.
+        The container exist and all processes are frozen
 
     absent
         The container does not exist
 
 lxcpath
     Set an other directory as lxc container location instead of the program default. This will passed
-    to all lxc commands who do smth. with containers.
+    to all lxc commands.
 
 config
     Contains configuration file paths and/or single configuration lines to assemble the lxc container
-    configuration file. There is no deep dive into files referenced with `lxc.include`.  Can be used
+    configuration file. There is no deep dive into files referenced with ``lxc.include``.  Can be used
     multiple times.
 
 config-absent
-    Contains configuration file paths and/ro single configuration lines which makes the same as the
-    `--config` parameter execpt all configuration lines will be removed instead of being added, Can
+    Contains configuration file paths and/or single configuration lines which makes the same as the
+    ``--config`` parameter execpt all configuration lines will be removed instead of being added. Can
     be used multiple times.
 
 BOOLEAN PARAMETERS
@@ -67,63 +75,63 @@ None.
 CREATE PARAMETERS
 -----------------
 
-This parameters will be used with *TEMPLATE PARAMETERS* or *CLONE PARAMETERS* to create the container
+This parameters will be used with **TEMPLATE PARAMETERS** or **CLONE PARAMETERS** to create the container
 if the container is absent. All of these parameters will only used if the container will be created,
 in all other cases, these parameters will be ignored.
 
-All backingstore parameters (incl. all bdev-* ones) can be read from the manual lxc-create(1) at option
-`--bdev` or lxc-copy(1) at option `--backingstore`. The possibilities may differ from creating and cloning.
-The bdev-* parameters are only available for some backingstores. If used for other backingstores, the
-type aborts.
+All backingstore parameters (incl. all `bdev-*` ones) can be read from the manual lxc-create(1) at option
+``--bdev`` or lxc-copy(1) at option ``--backingstore``. The possibilities may differ from creating and
+cloning. The `bdev-*` parameters are only available for some backingstores. If used for other backingstores,
+the type aborts.
 
 backingstore
-    Set the storage for the container root partition. Commonly, there should be at minimum `dir` (or `none`),
-    `lvm`, `loop`, `btrfs`, `zfs` or `best` be possible.
+    Set the storage for the container root filesystem. Commonly, there should be at minimum ``dir`` (or ``none``),
+    ``lvm``, ``loop``, ``btrfs``, ``zfs`` or ``best`` be possible.
 
 bdev-dir
     Specifies an other path for the rootfs directory instead in the lxc configuration directoy. Only
-    available for the backingstore `dir` or `none`.
+    available for the backingstore ``dir`` or ``none``.
 
 bdev-fstype
     Specifies the filesystem to be created instead of the default value. Only available for the
-    backingstore `lvm` and `loop`.
+    backingstore ``lvm`` and ``loop``.
 
 bdev-fssize
     Specifies the filesystem size to be created instead of the default value. Only available for the
-    backingstore `lvm` and `loop`.
+    backingstore ``lvm`` and ``loop``.
 
 bdev-lvname
     The custom LVM logical volume that will be created for the new container. Only availabe for the
-    backingstore `lvm`.
+    backingstore ``lvm``.
 
 bdev-vgname
-    The custon LVM volume group in there the logical volume will be created. Only available for the
-    backingstore `lvm`.
+    The custon LVM volume group in where the logical volume will be created. Only available for the
+    backingstore ``lvm``.
 
 bdev-thinpool
-    The custom thinpool the logical volume will created in. Only available for the backingstore `lvm`.
+    The custom thinpool the logical volume will created in. Only available for the backingstore ``lvm``.
 
 TEMPLATE PARAMETERS
 -------------------
 
-This or the *CLONE PARAMETERS* are required to create an container and must be present if the container
-should be created. If the template parameters are choosen, `--template` must be present. Then, none of
-the *CLONE PARAMETERS* must be present.
+This or the **CLONE PARAMETERS** are required to create an container and must be present if the container
+should be created. If the template parameters are choosen, ``--template`` must be present. Then, none of
+the **CLONE PARAMETERS** must be present.
 
 Because the templates vary in the possiblities of arguments and unkown arguments causing the abort of the
 container creation, the type will throw an error if the compatibility is not garanteed to the given
 template.
 
-The template options are only applied if the container will be created, there are not applied when
-the container is already present.
+The template options are only applied if the container will be created. There are not applied if the
+container is already present.
 
 template
-    The template which is used to create the the container. The name of the template must passed with
+    The template which is used to create the the container. The name of the template must passed by
     this argument and should exist on the target host.
 
 default-config
-    Alternative path to the user-defined default config. This file must exist on the target machine.
-    For the root user, this is commonly at `/etc/lxc/default.conf`. It will be included into the
+    Alternative path for the user-defined default config. This file must exist on the target machine.
+    For the root user, this is commonly at ``/etc/lxc/default.conf``. It will be included into the
     container configuration file at creation time.
 
 no-default-config
@@ -131,31 +139,31 @@ no-default-config
 
 template-opts
     Raw options which get passed to the template directly. Can be used multiple times. If the argument
-    contain spaces, it will be interpreted as multiple arguments and must be escaped for an normal posix
-    shell if this is not wanted.
+    contain spaces, it will be interpreted as multiple arguments by the target host and must be escaped
+    for a normal posix shell if this is not wanted.
 
 The following parameters are shortcuts for some template options. As templates may support it or not, you
 should be careful when using them. You can check all available template options with
-`lxc-create -t $template -h` (incl. lxc-create help, too) or `/usr/share/lxc/templates/lxc-$template -h`.
+``lxc-create -t $template -h`` (incl. lxc-create help, too) or ``/usr/share/lxc/templates/lxc-$template -h``.
 
 release
-    Sets the release for the wanted distribution. Uses the `-r` option.
+    Sets the release for the wanted distribution. Uses the ``-r`` option.
 
 arch
-    Sets the architecture used for the creating container. Uses the `-a` option.
+    Sets the architecture used for the creating container. Uses the ``-a`` option.
 
 mirror
-    Sets the mirror to download from. Uses the `--mirror=` option.
+    Sets the mirror to download from. Uses the ``--mirror=`` option.
 
 ssh-key
-    Sets the ssh key for the root user- Uses the `-S` option.
+    Sets the ssh key for the root user. Uses the ``-S`` option.
 
 CLONE PARAMETERS
 ----------------
 
-This or the *TEMPLATE PARAMETERS* are required to create an container and must be present if the container
-should be created. If the clone parameters are choosen, `--clone` must be present. Then, none of the
-*TEMPLATE PARAMETERS* must be present.
+This or the **TEMPLATE PARAMETERS** are required to create an container and must be present if the container
+should be created. If the clone parameters are choosen, ``--clone`` must be present. Then, none of the
+**TEMPLATE PARAMETERS** must be present.
 
 clone
     Instead of creating a new container with a given template, clone an other container and use him. The
@@ -163,8 +171,8 @@ clone
     the clone will not work.
 
 clonepath
-    The container path for the container to clone. It is like the `--lxcpath` parameter, but for the container
-    which will be cloned, not the target container.
+    The container path for the container to clone from. It is like the ``--lxcpath`` parameter, but for the
+    container from where will be cloned, not the target container (the container given by ``--clone``).
 
 MESSAGES
 --------
@@ -179,27 +187,27 @@ config
     The container configuration changed.
 
 start
-    Started the container.
+    Container was started.
 
 stop
-    Stopped the container.
+    Container was stopped.
 
 freeze
-    Freezed all container processes. The container will be started too if the container is stopped before.
+    Freezed all container processes.
 
 melt
-    Unfreezed all container processes. Will be done if any other state (except `present`) should be reached.
+    Unfreezed all container processes.
 
 ABORTS
 ------
 Aborts in the following cases:
 
-The type aborts if there are incompatible arguments found. This may be *CLONE* and *TEMPLATE PARAMETERS*,
+The type aborts if there are incompatible arguments found. This may be **CLONE** and **TEMPLATE PARAMETERS**,
 backingstorage parameters for the wrong storage or incompatible template option shortcuts.
 
-When cloning, it aborts if the container to clone from does not exist or is not stopped with no warning. It
-may be possible to clone while the container is running, which requires a backing storage supporting it, but
-there is nothing in lxc-copy(1) which indicades something about the container state while cloning.
+When cloning, it aborts if the container to clone from does not exist or is not stopped with no warning.
+It may be possible to clone while the container is running, which requires a backing storage supporting
+it, but there is nothing in lxc-copy(1) which indicades something about the container state while cloning.
 
 EXAMPLES
 --------
@@ -221,7 +229,7 @@ EXAMPLES
 
     # create with some template options
     __lxc_container special --state stopped --template debian \
-        --release stretch --arch amd64 \
+        --release buster --arch amd64 \
         --template-opts "--packages=foo,bar --flush-cache"
 
     # clone a container instead of creating it
@@ -235,9 +243,9 @@ EXAMPLES
         --config "lxc.start.auto   =  1" \      # spaces around the equal sign are ignored
         --config-absent "lxc.group = testing" \ # configuration options can be removed
         --config - <<ABSENT                     # stdin can be read once
-lxc.group = dev
-lxc.group = foobar
-ABSENT
+    lxc.group = dev
+    lxc.group = foobar
+    ABSENT
 
     # some more configuration
     # configuration lookup does not go deeper to included files

From 1b69d037acab1f44ae0d10938d907c0231c154c5 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 5 Jul 2020 20:50:40 +0200
Subject: [PATCH 014/411] [__lxc_container] fix some shellchecks (incl.
 warnings)

To the error in the `explorer/state`: It's intended, as it passes
possible parameters to the command. Varibale will set unquoted, so
word splitting take action, but the quotes inside the variable work.
---
 cdist/conf/type/__lxc_container/explorer/state | 2 ++
 cdist/conf/type/__lxc_container/gencode-remote | 6 +++---
 cdist/conf/type/__lxc_container/manifest       | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__lxc_container/explorer/state b/cdist/conf/type/__lxc_container/explorer/state
index fc37edf7..c3c7f30f 100755
--- a/cdist/conf/type/__lxc_container/explorer/state
+++ b/cdist/conf/type/__lxc_container/explorer/state
@@ -19,12 +19,14 @@ lxcpath="$(cat "$__object/parameter/lxcpath" 2>/dev/null || true)"
 # assemble optional parameters
 LXC_PARAM=""
 if [ "$lxcpath" ]; then
+	# shellcheck disable=SC2089
 	LXC_PARAM="$LXC_PARAM -P \"$lxcpath\""
 fi
 
 
 # function to get information
 lxc_info() {
+	# shellcheck disable=SC2090 disable=SC2086
 	if [ "$user" != "$USER" ]; then
 		su -s "$(which -- lxc-info)" -- "$user" $LXC_PARAM -n "$name" -H "$@"
 	else
diff --git a/cdist/conf/type/__lxc_container/gencode-remote b/cdist/conf/type/__lxc_container/gencode-remote
index ef61c66d..2177e377 100755
--- a/cdist/conf/type/__lxc_container/gencode-remote
+++ b/cdist/conf/type/__lxc_container/gencode-remote
@@ -44,7 +44,7 @@ fi
 # shortcut for checking config changes
 tmppath="$__object/files/remote-tmpfile"
 config_changes() {
-	if ( [ -f "${tmppath}_add" ] || [ -f "${tmppath}_del" ] ); then
+	if [ -f "${tmppath}_add" ] || [ -f "${tmppath}_del" ]; then
 		return 0
 	else
 		return 1
@@ -91,7 +91,7 @@ USER
 			backingstore_opts=""
 			if [ -f "$__object/parameter/backingstore" ]; then
 				# -B works for both types, they have different long opts
-				backingstore_opts="-B '$(cat "$__object/parameter/backingstore")'"
+				backingstore_opts="$backingstore_opts -B '$(cat "$__object/parameter/backingstore")'"
 
 				# add parameters if available
 				bdev_add_if_available dir
@@ -149,7 +149,7 @@ LXC
 
 				# at last, apply custom options
 				if [ -f "$__object/parameter/template-opts" ]; then
-					while read line; do
+					while read -r line; do
 						template_opts="$template_opts $line"
 					done < "$__object/parameter/template-opts"
 				fi
diff --git a/cdist/conf/type/__lxc_container/manifest b/cdist/conf/type/__lxc_container/manifest
index d903fa1c..4a74ea18 100755
--- a/cdist/conf/type/__lxc_container/manifest
+++ b/cdist/conf/type/__lxc_container/manifest
@@ -93,7 +93,7 @@ fi
 write_multi_param() {
 	if ! [ -f "$__object/parameter/$1" ]; then return 0; fi
 
-	while read line; do
+	while read -r line; do
 		if [ "$line" = "-" ]; then
 			# append stdin
 			cat "$__object/stdin"

From 93506d2113e1d05202527639efe883a5b44b220d Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 8 Jul 2020 00:17:12 +0300
Subject: [PATCH 015/411] [__download] curl follow redirects

---
 cdist/conf/type/__download/explorer/remote_cmd | 2 +-
 cdist/conf/type/__download/gencode-local       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd
index fbd4d84c..e3e35b45 100755
--- a/cdist/conf/type/__download/explorer/remote_cmd
+++ b/cdist/conf/type/__download/explorer/remote_cmd
@@ -6,7 +6,7 @@ then
 
 elif command -v curl > /dev/null
 then
-    cmd="curl -o - '%s'"
+    cmd="curl -L -o - '%s'"
 
 elif command -v fetch > /dev/null
 then
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 339827c2..571d2c3c 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -25,7 +25,7 @@ then
 
 elif command -v curl > /dev/null
 then
-    cmd="curl -o - '%s'"
+    cmd="curl -L -o - '%s'"
 
 elif command -v fetch > /dev/null
 then

From e9062662868d7677a235889ba39a6091085388f0 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 8 Jul 2020 00:20:55 +0300
Subject: [PATCH 016/411] [__download] s/variable/format specification/

---
 cdist/conf/type/__download/man.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index eafa9dfc..6ec0b19a 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -44,14 +44,14 @@ cmd-get
    Command used for downloading.
    Command must output to ``stdout``.
    Parameter will be used for ``printf`` and must include only one
-   variable ``%s`` which will become URL.
+   format specification ``%s`` which will become URL.
    For example: ``wget -O - '%s'``.
 
 cmd-sum
    Command used for checksum calculation.
    Command output and ``--sum`` parameter must match.
    Parameter will be used for ``printf`` and must include only one
-   variable ``%s`` which will become destination.
+   format specification ``%s`` which will become destination.
    For example: ``md5sum '%s' | awk '{print $1}'``.
 
 

From cb9933b4a0d343a059899101810b6ee86fc87dbc Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 8 Jul 2020 12:43:55 +0200
Subject: [PATCH 017/411] Fix state -> state_is

---
 cdist/conf/type/__download/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
index 89ba72af..029a0801 100755
--- a/cdist/conf/type/__download/gencode-remote
+++ b/cdist/conf/type/__download/gencode-remote
@@ -19,7 +19,7 @@ then
     echo 'downloaded' > "$__messages_out"
 fi
 
-if [ -f "$__object/parameter/onchange" ] && [ "$state" != "present" ]
+if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ]
 then
     cat "$__object/parameter/onchange"
 fi

From b8752e9ee390e983c1ce6467761aca8101b6548d Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 10 Jul 2020 21:03:35 +0200
Subject: [PATCH 018/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index b823421b..54c42bde 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Type __package_opkg: Add locking (Dennis Camera)
 	* Type __hosts: Add --alias parameter (Dennis Camera)
 	* Type __user: Fix shadow explorer for OpenBSD (Dennis Camera)
+	* Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 304d974f7bbb488ba80a522cb0f01fc3b79aa1b2 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 11 Jul 2020 18:18:55 +0200
Subject: [PATCH 019/411] [__lxc_container] moved config diff incl.
 gencode-local to gencode-remote

Rather than the config handling is split up over gencode-{local,remote}
and the manifest, it is now completly moved to the gencode-remote script.

This avoids complexity, file uploading and direct ssh interactions as it
was used before. The diff creation was extracted into an extra script, and
the gencode-local was removed.

Now, the gencode-remote script executes the config diff script before
the short curcuit and uses here-documents and pipes if the diff is
required on the remote host. With this, the script size grows, but
should be no difference if the file is downloaded temporarily.

With this, the remote handling of an empty configuration file was
enhanced by handling it completely into the code-remote script (hope
escaping works correctly).
---
 .../type/__lxc_container/diff-conf-changes.sh | 72 +++++++++++++++++++
 cdist/conf/type/__lxc_container/gencode-local | 69 ------------------
 .../conf/type/__lxc_container/gencode-remote  | 30 +++++---
 cdist/conf/type/__lxc_container/manifest      | 59 ---------------
 4 files changed, 91 insertions(+), 139 deletions(-)
 create mode 100755 cdist/conf/type/__lxc_container/diff-conf-changes.sh
 delete mode 100755 cdist/conf/type/__lxc_container/gencode-local

diff --git a/cdist/conf/type/__lxc_container/diff-conf-changes.sh b/cdist/conf/type/__lxc_container/diff-conf-changes.sh
new file mode 100755
index 00000000..37d9f195
--- /dev/null
+++ b/cdist/conf/type/__lxc_container/diff-conf-changes.sh
@@ -0,0 +1,72 @@
+#!/bin/sh -e
+# diff-conf-changes.sh
+
+# This script handles the configuration given to the type. It will be diffed against the existing
+# container options and write the "to be done"-options to a separate file.
+#
+# Output files in "$__object/files/" :
+#  - config-{present,absent}
+#  - config.add
+#  - config.del
+
+
+# Function to read multiple parameter input for configuration and print it to a single file.
+# It will handle file paths as well as the stdin. Instead of the file, the content will be printed.
+#
+# Parameters:
+#  1: parameter name
+#  2: file to print
+write_multi_param() {
+	if ! [ -f "$__object/parameter/$1" ]; then return 0; fi
+
+	while read -r line; do
+		if [ "$line" = "-" ]; then
+			# append stdin
+			cat "$__object/stdin"
+		elif [ -f "$line" ]; then
+			# append file content
+			cat "$line"
+		else
+			# print out content
+			printf "%s\n" "$line"
+		fi
+	done < "$__object/parameter/$1"
+}
+
+# Function to get rid of whitespaces inside of key-value pairs. It will clear all of these.
+#
+# Parameters:
+#  1: the file which should be handled
+trimm_whitespaces() {
+	mv "$1" "$1"~  # backup file to read write back original file
+	grep -E -v "^([[:blank:]]*)(#|$)" "$1"~ \
+		| sed 's/^[[:blank:]]*//; s/[[:blank:]]*=[[:blank:]]*/ = /; s/[[:blank:]]*$//' \
+		> "$1"
+	rm -f "$1"~
+}
+
+
+# Only required if container will not be absent
+# prepare the wanted configuration lines
+if [ "$(cat "$__object/parameter/state")" != "absent" ]; then
+	# create the files directory
+	mkdir "$__object/files/"
+
+	# Summary of the locally given configuration options
+	# config-present
+	conffile="$__object/files/config-present"
+	write_multi_param config > "$conffile"
+	trimm_whitespaces "$conffile"
+	# config-absent
+	absentfile="$__object/files/config-absent"
+	write_multi_param config-absent > "$absentfile"
+	trimm_whitespaces "$absentfile"
+
+	# Diff the configuration options by removing duplicate config options
+	# config.add
+	grep -v -f "$__object/explorer/config" -f "$__object/files/config-absent" \
+		"$__object/files/config-present" > "$__object/files/config.add" || true
+	# config.del
+	grep -f "$__object/explorer/config" \
+		"$__object/files/config-absent" > "$__object/files/config.del" || true
+fi
diff --git a/cdist/conf/type/__lxc_container/gencode-local b/cdist/conf/type/__lxc_container/gencode-local
deleted file mode 100755
index e0572bb4..00000000
--- a/cdist/conf/type/__lxc_container/gencode-local
+++ /dev/null
@@ -1,69 +0,0 @@
-#!/bin/sh -e
-# gencode-local
-
-# This uploads the diff of the configuration file, which is applied by gencode-remote
-
-
-# Get required parameters
-name="$__object/parameter/name"
-if [ -f "$name" ]; then
-    name="$(cat "$name")"
-else
-    name="$__object_id"
-fi
-
-state_is="$(cat "$__object/explorer/state")"
-state_should="$(cat "$__object/parameter/state")"
-
-
-# Check the configurations only if the container exists
-if [ "$state_should" != "absent" ]; then
-	# do changes
-
-	# predict remote lxc config file
-	container_config="$(cat "$__object/explorer/lxcpath")/$name/config"
-
-	# IPv6 fix
-	if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$'
-	then
-		my_target_host="[${__target_host}]"
-	else
-		my_target_host="${__target_host}"
-	fi
-
-	# create remote container directory if container is not yet available
-	if [ "$state_is" = "absent" ]; then
-		# this must be done because the container will be created after the configuration is uploaded
-		# would also be possible to do everything in gencode-remote to keep the order
-		$__remote_exec $__target_host "mkdir $(dirname "$container_config")"
-	fi
-
-
-	# config-present
-	# remove all config lines from config-present that are already set or absent anyway
-	if grep -v -f "$__object/explorer/config" -f "$__object/files/config-absent" \
-		"$__object/files/config-present" > "$__object/files/config.add"; then
-
-		# get temp file name
-		add_dest="$($__remote_exec $__target_host "mktemp $container_config-add.XXXXXXXXXX")"
-		printf "%s" "$add_dest" > "$__object/files/remote-tmpfile_add"
-		# upload delta
-		cat <<OUT
-$__remote_copy "$__object/files/config.add" "$my_target_host:$add_dest"
-OUT
-	fi
-
-	# config-absent
-	# only find those lines which should be absent in the config
-	if grep -f "$__object/explorer/config" "$__object/files/config-absent" \
-		> "$__object/files/config.del"; then
-
-		# get temp file name
-		del_dest="$($__remote_exec $__target_host "mktemp $container_config-del.XXXXXXXXXX")"
-		printf "%s" "$del_dest" > "$__object/files/remote-tmpfile_del"
-		# upload delta
-		cat <<OUT
-$__remote_copy "$__object/files/config.del" "$my_target_host:$del_dest"
-OUT
-	fi
-fi
diff --git a/cdist/conf/type/__lxc_container/gencode-remote b/cdist/conf/type/__lxc_container/gencode-remote
index 2177e377..ca2f2827 100755
--- a/cdist/conf/type/__lxc_container/gencode-remote
+++ b/cdist/conf/type/__lxc_container/gencode-remote
@@ -42,9 +42,9 @@ fi
 
 
 # shortcut for checking config changes
-tmppath="$__object/files/remote-tmpfile"
+# returns success (0) if there are changes
 config_changes() {
-	if [ -f "${tmppath}_add" ] || [ -f "${tmppath}_del" ]; then
+	if [ -s "$__object/files/config.add" ] || [ -s "$__object/files/config.del" ]; then
 		return 0
 	else
 		return 1
@@ -72,6 +72,9 @@ template_add_if_available() {
 }
 
 
+# Generate configuration diff files
+"$__type"/diff-conf-changes.sh
+
 # Short curcit if nothing changed
 if [ "$state_is" = "$state_should" ] && ( ! config_changes ); then exit; fi
 
@@ -129,8 +132,10 @@ LXC
 				create_opts=""
 				if [ -f "$__object/parameter/no-default-config" ]; then
 					# generate a random empty file and append
-					empty_config="$($__remote_exec $__target_host "mktemp \${TMPDIR:-/tmp}/cdist_lxc-empty-conf.XXXXXXXXXX")"
-					create_opts="$create_opts -f \"$empty_config\""
+					cat <<TMPFILE
+empty_conf="\$(mktemp "\${TMPDIR:-/tmp}/cdist__lxc_container-empty-conf.XXXXXXXXXX")"
+TMPFILE
+					create_opts="$create_opts -f \"\$empty_config\""
 
 				elif [ -f "$__object/parameter/default-config" ]; then
 					# append the configuration
@@ -198,19 +203,22 @@ cp -p "$container_config" "\$tmpconfig"
 DONE
 
 	# check if smth. to be deleted
-	# must be before adding, because it takes the content of the original file
-	if [ -f "${tmppath}_del" ]; then
+	#  must be before adding, because it takes the content of the original file
+	#  because 'cat $file > $file' will not work (> opens before command reads it)
+	if [ -s "$__object/files/config.del" ]; then
 		cat <<DONE
-grep -v -f "$(cat "${tmppath}_del")" "$container_config" > "\$tmpconfig"
-rm -rf "$(cat "${tmppath}_del")"
+grep -v -f - <<'ABSENT' "$container_config" > "\$tmpconfig"
+$(cat "$__object/files/config.del")
+ABSENT
 DONE
 	fi
 
 	# check if smth. to be added
-	if [ -f "${tmppath}_add" ]; then
+	if [ -s "$__object/files/config.add" ]; then
 		cat <<DONE
-cat "$(cat "${tmppath}_add")" >> "\$tmpconfig"
-rm -rf "$(cat "${tmppath}_add")"
+cat <<'PRESENT' >> "\$tmpconfig"
+$(cat "$__object/files/config.add")
+PRESENT
 DONE
 	fi
 
diff --git a/cdist/conf/type/__lxc_container/manifest b/cdist/conf/type/__lxc_container/manifest
index 4a74ea18..99cfa1c0 100755
--- a/cdist/conf/type/__lxc_container/manifest
+++ b/cdist/conf/type/__lxc_container/manifest
@@ -76,62 +76,3 @@ if [ -f "$param/backingstore" ]; then
 		fi
 	fi
 fi
-
-
-
-
-# ============================== #
-# ==  CONFIGURATION HANDLING  == #
-# ============================== #
-
-# Function to read multiple parameter input for configuration and print it to a single file.
-# It will handle file paths as well as the stdin. Instead of the file, the content will be printed.
-#
-# Parameters:
-#  1: parameter name
-#  2: file to print
-write_multi_param() {
-	if ! [ -f "$__object/parameter/$1" ]; then return 0; fi
-
-	while read -r line; do
-		if [ "$line" = "-" ]; then
-			# append stdin
-			cat "$__object/stdin"
-		elif [ -f "$line" ]; then
-			# append file content
-			cat "$line"
-		else
-			# print out content
-			printf "%s\n" "$line"
-		fi
-	done < "$__object/parameter/$1"
-}
-
-# Function to get rid of whitespaces inside of key-value pairs. It will clear all of these.
-#
-# Parameters:
-#  1: the file which should be handled
-trimm_whitespaces() {
-	mv "$1" "$1"~  # backup file to see original content
-	grep -E -v "^([[:blank:]]*)(#|$)" "$1"~ \
-		| sed 's/^[[:blank:]]*//; s/[[:blank:]]*=[[:blank:]]*/ = /; s/[[:blank:]]*$//' \
-		> "$1"
-}
-
-
-# Only required if container will not be absent
-# prepare the wanted configuration lines
-if [ "$(cat "$__object/parameter/state")" != "absent" ]; then
-	# Create the files directory
-	mkdir "$__object/files/"
-
-	# Create the file of the locally configuration
-	conffile="$__object/files/config-present"
-	write_multi_param config > "$conffile"
-	trimm_whitespaces "$conffile"
-
-	# Create the file of the absent configuration
-	absentfile="$__object/files/config-absent"
-	write_multi_param config-absent > "$absentfile"
-	trimm_whitespaces "$absentfile"
-fi

From a5ae26116bd611fcab76c0383998fb4b05f1ae1d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 11 Jul 2020 18:57:47 +0200
Subject: [PATCH 020/411] [type/__hosts] Fix when used without --alias

---
 cdist/conf/type/__hosts/manifest | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/type/__hosts/manifest b/cdist/conf/type/__hosts/manifest
index 0d9e61f8..8103ebd5 100755
--- a/cdist/conf/type/__hosts/manifest
+++ b/cdist/conf/type/__hosts/manifest
@@ -19,21 +19,24 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-set -e -u
+set -e
 
 hostname=$__object_id
-state=$(cat "$__object/parameter/state")
-marker="# __hosts/$hostname"
+state=$(cat "${__object}/parameter/state")
+marker="# __hosts/${hostname}"
 
-if [ "$state" = 'absent' ]
+if test "${state}" != 'absent'
 then
-	set -- --regex "$marker"
-else
-	ip=$(cat "$__object/parameter/ip")
-	aliases=$(while read -r a; do printf '\t%s' "$a"; done <"$__object/parameter/alias")
+	ip=$(cat "${__object}/parameter/ip")
+	if test -s "${__object}/parameter/alias"
+	then
+		aliases=$(while read -r a; do printf '\t%s' "$a"; done <"$__object/parameter/alias")
+	fi
 
 	set -- --line "$(printf '%s\t%s%s  %s' \
-		"$ip" "$hostname" "$aliases" "$marker")"
+		"${ip}" "${hostname}" "${aliases}" "${marker}")"
+else
+	set -- --regex "$(echo "${marker}" | sed -e 's/\./\\./')$"
 fi
 
-__line "__hosts/$hostname" --file /etc/hosts --state "$state" "$@"
+__line "/etc/hosts:${hostname}" --file /etc/hosts --state "${state}" "$@"

From 19514662b022c79c4e6a56573c32dad45055310c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 12 Jul 2020 12:24:00 +0200
Subject: [PATCH 021/411] [type/{__file/__directory}] Fix typo

---
 cdist/conf/type/__directory/explorer/stat | 2 +-
 cdist/conf/type/__file/explorer/stat      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat
index a7dc8431..422c5819 100755
--- a/cdist/conf/type/__directory/explorer/stat
+++ b/cdist/conf/type/__directory/explorer/stat
@@ -30,7 +30,7 @@ fallback() {
    gid=$(echo "$ls_line" | awk '{ print $4 }')
 
    owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd)
-   group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
+   group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
    mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat
index 231768f6..3f971488 100755
--- a/cdist/conf/type/__file/explorer/stat
+++ b/cdist/conf/type/__file/explorer/stat
@@ -31,7 +31,7 @@ fallback() {
    gid=$(echo "$ls_line" | awk '{ print $4 }')
 
    owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd)
-   group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
+   group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
    mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')

From 9fb7e151b889214576cb891595a68ca8c6196d45 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 12 Jul 2020 12:31:55 +0200
Subject: [PATCH 022/411] [type/{__file/__directory}] Remove special Solaris
 blocks

Solaris 11 has GNU stat (handled by *)
Solaris 10 (and older?) does not have stat (handled by failing command -v stat)

On Solaris 10 (at least on UFS), setgid cannot be set on directories.
Unlike on other systems `chmod 2400` is not `-r----S---`, but `-r----l---`.
---
 cdist/conf/type/__directory/explorer/stat | 40 +++--------------------
 cdist/conf/type/__file/explorer/stat      | 40 +++--------------------
 2 files changed, 9 insertions(+), 71 deletions(-)

diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat
index 422c5819..f817cb02 100755
--- a/cdist/conf/type/__directory/explorer/stat
+++ b/cdist/conf/type/__directory/explorer/stat
@@ -33,7 +33,7 @@ fallback() {
    group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
-   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
+   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[lst]/)*2^(9+i/3)}printf("%04o",k)}')
 
    printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\n' \
       "$("$__type_explorer/type")" \
@@ -45,11 +45,10 @@ fallback() {
 # nothing to work with, nothing we could do
 [ -e "$destination" ] || exit 0
 
-if ! command -v stat >/dev/null
-then
+command -v stat >/dev/null 2>&1 || {
    fallback
    exit
-fi
+}
 
 case $("$__explorer/os")
 in
@@ -60,42 +59,13 @@ group: %Dg %Sg
 mode: %Mp%03Lp %Sp
 ' "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
-    solaris)
-        ls1="$( ls -ld "$destination" )"
-        ls2="$( ls -ldn "$destination" )"
-
-        if [ -f "$__object/parameter/mode" ]
-        then mode_should="$( cat "$__object/parameter/mode" )"
-        fi
-
-        # yes, it is ugly hack, but if you know better way...
-        if [ -z "$( find "$destination" -perm "$mode_should" )" ]
-        then octets=888
-        else octets="$( echo "$mode_should" | sed 's/^0//' )"
-        fi
-
-        case "$( echo "$ls1" | cut -c1-1 )" in
-            -) echo 'type: regular file' ;;
-            d) echo 'type: directory' ;;
-        esac
-
-        echo "owner: $( echo "$ls2" \
-            | awk '{print $3}' ) $( echo "$ls1" \
-                | awk '{print $3}' )"
-
-        echo "group: $( echo "$ls2" \
-            | awk '{print $4}' ) $( echo "$ls1" \
-                | awk '{print $4}' )"
-
-        echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )"
-    ;;
    *)
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
       #       we fall through to the shell fallback.
-       stat -c 'type: %F
+      stat -c 'type: %F
 owner: %u %U
 group: %g %G
 mode: %04a %A' "$destination" 2>/dev/null || fallback
-   ;;
+      ;;
 esac
diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat
index 3f971488..29b3c8a3 100755
--- a/cdist/conf/type/__file/explorer/stat
+++ b/cdist/conf/type/__file/explorer/stat
@@ -34,7 +34,7 @@ fallback() {
    group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
-   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
+   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[lst]/)*2^(9+i/3)}printf("%04o",k)}')
 
    size=$(echo "$ls_line" | awk '{ print $5 }')
    links=$(echo "$ls_line" | awk '{ print $2 }')
@@ -53,11 +53,10 @@ fallback() {
 [ -e "$destination" ] || exit 0
 
 
-if ! command -v stat >/dev/null
-then
+command -v stat >/dev/null 2>&1 || {
    fallback
    exit
-fi
+}
 
 
 case $("$__explorer/os")
@@ -71,37 +70,6 @@ size: %Dz
 links: %Dl
 ' "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
-    solaris)
-        ls1="$( ls -ld "$destination" )"
-        ls2="$( ls -ldn "$destination" )"
-
-        if [ -f "$__object/parameter/mode" ]
-        then mode_should="$( cat "$__object/parameter/mode" )"
-        fi
-
-        # yes, it is ugly hack, but if you know better way...
-        if [ -z "$( find "$destination" -perm "$mode_should" )" ]
-        then octets=888
-        else octets="$( echo "$mode_should" | sed 's/^0//' )"
-        fi
-
-        case "$( echo "$ls1" | cut -c1-1 )" in
-            -) echo 'type: regular file' ;;
-            d) echo 'type: directory' ;;
-        esac
-
-        echo "owner: $( echo "$ls2" \
-            | awk '{print $3}' ) $( echo "$ls1" \
-                | awk '{print $3}' )"
-
-        echo "group: $( echo "$ls2" \
-            | awk '{print $4}' ) $( echo "$ls1" \
-                | awk '{print $4}' )"
-
-        echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )"
-        echo "size: $( echo "$ls1" | awk '{print $5}' )"
-        echo "links: $( echo "$ls1" | awk '{print $2}' )"
-    ;;
    *)
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
@@ -112,5 +80,5 @@ group: %g %G
 mode: %04a %A
 size: %s
 links: %h' "$destination" 2>/dev/null || fallback
-         ;;
+      ;;
 esac

From 6d29b0542a82f6cf12429291aec488c649bf1323 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 12 Jul 2020 20:37:59 +0200
Subject: [PATCH 023/411] [__lxc_container] better support config modifications
 at creation time

When a container is created, the configuration is unknown to the
explorer. To do the job without the knowleage, following is added: If
the container will be created, it adds all absent configuration options
to be deleted. Because it do not know which configuration options exist,
it tries to remove all if there exists.

This behaviour is extended for containers that will be cloned: Because
the container who will be cloned is known and it will copy the config
nearly one to one, so it will read this configuration.

The removal of configuration is generally improved by generating a
pattern to ignore spaces at the beginning, near the equal sign and at
the end. This deletes the correct configuration lines even they have
malformed whitespaces.
---
 .../type/__lxc_container/diff-conf-changes.sh | 15 +++++++++++++-
 .../conf/type/__lxc_container/explorer/config | 20 +++++++++++++++++--
 .../conf/type/__lxc_container/gencode-remote  |  8 ++++++--
 3 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__lxc_container/diff-conf-changes.sh b/cdist/conf/type/__lxc_container/diff-conf-changes.sh
index 37d9f195..d860a8b8 100755
--- a/cdist/conf/type/__lxc_container/diff-conf-changes.sh
+++ b/cdist/conf/type/__lxc_container/diff-conf-changes.sh
@@ -46,9 +46,13 @@ trimm_whitespaces() {
 }
 
 
+# save states
+state_should="$(cat "$__object/parameter/state")"
+state_is="$(cat "$__object/explorer/state")"
+
 # Only required if container will not be absent
 # prepare the wanted configuration lines
-if [ "$(cat "$__object/parameter/state")" != "absent" ]; then
+if [ "$state_should" != "absent" ]; then
 	# create the files directory
 	mkdir "$__object/files/"
 
@@ -69,4 +73,13 @@ if [ "$(cat "$__object/parameter/state")" != "absent" ]; then
 	# config.del
 	grep -f "$__object/explorer/config" \
 		"$__object/files/config-absent" > "$__object/files/config.del" || true
+
+
+	# fallback: if template parameter exists and container currently absent; try to remove all configs
+	#  If the container is currently absent, nothing will be removed, even it's in the default config
+	#  In this case, it will try to delete every configuration option to be sure
+	# only the cloning configuration is known, but not if the container will be created in an other way
+	if [ "$state_is" = "absent" ] && ! [ -f "$__object/parameter/clone" ]; then
+		cp "$__object/files/config-absent" "$__object/files/config.del"
+	fi
 fi
diff --git a/cdist/conf/type/__lxc_container/explorer/config b/cdist/conf/type/__lxc_container/explorer/config
index 287a5063..16fcf14a 100755
--- a/cdist/conf/type/__lxc_container/explorer/config
+++ b/cdist/conf/type/__lxc_container/explorer/config
@@ -22,8 +22,24 @@ else
 fi
 
 
-# assemble the container configuration file
-config="$lxcpath/$name/config"
+# if the container will be cloned, use the configuration of the container to clone from
+# else, just use the normal container configuration
+clone="$__object/parameter/clone"
+if [ -f "$clone" ] && [ "$("$__type_explorer/state")" = "absent" ]; then
+	clone="$(cat "$clone")"
+	clonepath="$__object/parameter/clonepath"
+	if [ -f "$clonepath" ]; then
+		clonepath="$(cat "$clonepath")"
+	else
+		clonepath="$lxcpath"
+	fi
+
+	# set the config path of the container to clone from
+	config="$clonepath/$clone/config"
+else
+	# assemble the configuration file of the container
+	config="$lxcpath/$name/config"
+fi
 
 # check if the file exist, else the container is absent
 if [ -r "$config" ]; then
diff --git a/cdist/conf/type/__lxc_container/gencode-remote b/cdist/conf/type/__lxc_container/gencode-remote
index ca2f2827..bd3f49b6 100755
--- a/cdist/conf/type/__lxc_container/gencode-remote
+++ b/cdist/conf/type/__lxc_container/gencode-remote
@@ -205,10 +205,14 @@ DONE
 	# check if smth. to be deleted
 	#  must be before adding, because it takes the content of the original file
 	#  because 'cat $file > $file' will not work (> opens before command reads it)
+	# will create extended regex pattern for grep with malformed spaces
 	if [ -s "$__object/files/config.del" ]; then
 		cat <<DONE
-grep -v -f - <<'ABSENT' "$container_config" > "\$tmpconfig"
-$(cat "$__object/files/config.del")
+grep -v -E -f - <<'ABSENT' "$container_config" > "\$tmpconfig"
+$(awk -v FS=' = ' -v OFS=' = ' -v blank='[[:blank:]]*' '
+function ntostring(n) { ret=""; for(i=n; i<=NF; i++) ret=ret $i (i<NF ? OFS : ""); return ret }
+{ printf "^%s%s%s=%s%s%s$%s", blank, $1, blank, blank, ntostring(2), blank, ORS }
+	' "$__object/files/config.del" )
 ABSENT
 DONE
 	fi

From 12a6d52a464f2854c5d9c031f8e5b7898ffc4951 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 12 Jul 2020 21:30:44 +0200
Subject: [PATCH 024/411] [__lxc_container] handle $empty_conf for
 --no-default-config correctly

Fixed wrong variable spelling and correct check to remove the tempfile
again (forgotten from 304d974f).
---
 cdist/conf/type/__lxc_container/gencode-remote | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__lxc_container/gencode-remote b/cdist/conf/type/__lxc_container/gencode-remote
index bd3f49b6..ddba593a 100755
--- a/cdist/conf/type/__lxc_container/gencode-remote
+++ b/cdist/conf/type/__lxc_container/gencode-remote
@@ -132,10 +132,11 @@ LXC
 				create_opts=""
 				if [ -f "$__object/parameter/no-default-config" ]; then
 					# generate a random empty file and append
+					# maybe work with an nonexistant file - but just be correct
 					cat <<TMPFILE
 empty_conf="\$(mktemp "\${TMPDIR:-/tmp}/cdist__lxc_container-empty-conf.XXXXXXXXXX")"
 TMPFILE
-					create_opts="$create_opts -f \"\$empty_config\""
+					create_opts="$create_opts -f \"\$empty_conf\""
 
 				elif [ -f "$__object/parameter/default-config" ]; then
 					# append the configuration
@@ -163,10 +164,11 @@ TMPFILE
 				cat <<LXC
 lxc-create $LXC_PARAM $backingstore_opts -n "$name" -t "$template" $create_opts -- $template_opts
 LXC
-				# remove empty tempfile
-				if [ "$empty_config" ]; then
+
+				# remove empty tempfile if it was created
+				if [ -f "$__object/parameter/no-default-config" ]; then
 					cat <<RM
-rm -rf "$empty_config"
+rm -rf "\$empty_conf"
 RM
 				fi
 			fi

From bc970731316a59c21f8aa784fb27f1fa1b13ab89 Mon Sep 17 00:00:00 2001
From: Darko Poljak <darko.poljak@ungleich.ch>
Date: Mon, 13 Jul 2020 05:44:26 +0000
Subject: [PATCH 025/411] Merge branch 'bugfix/postfix-master-option' into
 '6.6'

Fix broken --option parameter in __postfix_master type

See merge request ungleich-public/cdist!905

(cherry picked from commit 2f433a1458f3a1f7f8859e9ae165178a0ec5b7a0)

9496b234 The option parameter is actually multi-valued
4009bbd7 Protect postfix variables in options
---
 cdist/conf/type/__postfix_master/gencode-remote              | 2 +-
 cdist/conf/type/__postfix_master/parameter/optional          | 1 -
 cdist/conf/type/__postfix_master/parameter/optional_multiple | 1 +
 3 files changed, 2 insertions(+), 2 deletions(-)
 create mode 100644 cdist/conf/type/__postfix_master/parameter/optional_multiple

diff --git a/cdist/conf/type/__postfix_master/gencode-remote b/cdist/conf/type/__postfix_master/gencode-remote
index 7c109a69..73de1088 100755
--- a/cdist/conf/type/__postfix_master/gencode-remote
+++ b/cdist/conf/type/__postfix_master/gencode-remote
@@ -67,7 +67,7 @@ case "$state_should" in
          remove_entry
       fi
       cat << DONE
-cat >> "$config" << ${__type##*/}_DONE
+cat >> "$config" << "${__type##*/}_DONE"
 $(cat "$entry")
 ${__type##*/}_DONE
 DONE
diff --git a/cdist/conf/type/__postfix_master/parameter/optional b/cdist/conf/type/__postfix_master/parameter/optional
index 792b42c5..410482b8 100644
--- a/cdist/conf/type/__postfix_master/parameter/optional
+++ b/cdist/conf/type/__postfix_master/parameter/optional
@@ -4,6 +4,5 @@ unpriv
 chroot
 wakeup
 maxproc
-option
 comment
 state
diff --git a/cdist/conf/type/__postfix_master/parameter/optional_multiple b/cdist/conf/type/__postfix_master/parameter/optional_multiple
new file mode 100644
index 00000000..01925a15
--- /dev/null
+++ b/cdist/conf/type/__postfix_master/parameter/optional_multiple
@@ -0,0 +1 @@
+option

From 8903540e9106db93e490515c07929a207326be77 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 13 Jul 2020 07:54:12 +0200
Subject: [PATCH 026/411] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 54c42bde..dac9cbce 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,8 @@ next:
 	* Type __hosts: Add --alias parameter (Dennis Camera)
 	* Type __user: Fix shadow explorer for OpenBSD (Dennis Camera)
 	* Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak)
+	* Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera)
+	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 9fd892a22413104b3a34554588468cd94d1a599e Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 13 Jul 2020 20:31:18 +0200
Subject: [PATCH 027/411] [__lxc_container] add --restart-if-changed parameter
 to apply changes

This parameter restarts the container if configuration changes and it is
already running and should keept running. It handles the freeze state,
because it would be expected to apply changes in this state, too.
---
 .../conf/type/__lxc_container/gencode-remote  | 49 ++++++++++++++++++-
 cdist/conf/type/__lxc_container/man.rst       |  6 ++-
 .../type/__lxc_container/parameter/boolean    |  1 +
 3 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__lxc_container/gencode-remote b/cdist/conf/type/__lxc_container/gencode-remote
index ddba593a..480d7d41 100755
--- a/cdist/conf/type/__lxc_container/gencode-remote
+++ b/cdist/conf/type/__lxc_container/gencode-remote
@@ -234,8 +234,55 @@ DONE
 rm -rf "$container_config"
 mv "\$tmpconfig" "$container_config"
 DONE
-	# write message
 	echo "config" >> "$__messages_out"
+
+
+	# restart container only if it is running
+	#  do not echo messages start/stop or melt/start/stop/freeze; only restart to avoid junk
+	case "$state_is" in
+		running|frozen)
+			# only do it if the container will run again
+			case "$state_should" in
+				present|running|frozen)
+					# su stuff
+					cat <<USER
+su "$user" - <<SU
+USER
+
+					# unfreeze if required
+					if [ "$state_is" = "frozen" ]; then
+						cat <<LXC
+lxc-unfreeze $LXC_PARAM -n "$name"
+LXC
+					fi
+
+					# stop and start
+					cat <<LXC
+lxc-stop $LXC_PARAM -n "$name"
+lxc-start $LXC_PARAM -n "$name"
+LXC
+
+					# freeze it again if it was there before, but be clever
+					# if $state_is = running but should be frozen, later code will do that
+					if [ "$state_is" = "frozen" ]; then
+						if [ "$state_should" = "frozen" ] || [ "$state_should" = "present" ]; then
+							cat <<LXC
+lxc-freeze $LXC_PARAM -n "$name"
+LXC
+						else
+							# correct current state
+							state_is="running"
+							echo "melt" >> "$__messages_out"
+						fi
+					fi
+
+					# ending
+					echo "restart" >> "$__messages_out"
+					echo "SU"
+				;;
+			esac
+			;;
+	esac
 fi
 
 
diff --git a/cdist/conf/type/__lxc_container/man.rst b/cdist/conf/type/__lxc_container/man.rst
index d0abf2e2..7d1f6522 100644
--- a/cdist/conf/type/__lxc_container/man.rst
+++ b/cdist/conf/type/__lxc_container/man.rst
@@ -70,7 +70,11 @@ config-absent
 
 BOOLEAN PARAMETERS
 ------------------
-None.
+
+restart-if-changed
+    Restarts the container if configuration changes. It stop and start the container that new configuration
+    can be applied. If it whould be restarted with ``lxc-stop -n $name --reboot``, configuration changes
+    would be ignored.
 
 CREATE PARAMETERS
 -----------------
diff --git a/cdist/conf/type/__lxc_container/parameter/boolean b/cdist/conf/type/__lxc_container/parameter/boolean
index caf89a8e..724fa14b 100644
--- a/cdist/conf/type/__lxc_container/parameter/boolean
+++ b/cdist/conf/type/__lxc_container/parameter/boolean
@@ -1 +1,2 @@
 no-default-config
+restart-if-changed

From 5d6f90555bdf60354dc298c87e06cf8a551a2b1c Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 13 Jul 2020 21:16:18 +0200
Subject: [PATCH 028/411] [__lxc_container] s/\t/    /g; renewed todo.txt file

Replaced all tabs with 4 spaces (~/.vimrc is modified, too).

The issue.txt is not required anymore and the todo.txt was updated.
---
 .../type/__lxc_container/diff-conf-changes.sh |  86 ++--
 .../conf/type/__lxc_container/explorer/config |  32 +-
 .../type/__lxc_container/explorer/lxcpath     |  18 +-
 .../conf/type/__lxc_container/explorer/state  |  24 +-
 .../conf/type/__lxc_container/gencode-remote  | 396 +++++++++---------
 cdist/conf/type/__lxc_container/issue.txt     |   1 -
 cdist/conf/type/__lxc_container/manifest      |  86 ++--
 cdist/conf/type/__lxc_container/todo.txt      |  10 +-
 8 files changed, 328 insertions(+), 325 deletions(-)
 delete mode 100644 cdist/conf/type/__lxc_container/issue.txt

diff --git a/cdist/conf/type/__lxc_container/diff-conf-changes.sh b/cdist/conf/type/__lxc_container/diff-conf-changes.sh
index d860a8b8..b18b9fe4 100755
--- a/cdist/conf/type/__lxc_container/diff-conf-changes.sh
+++ b/cdist/conf/type/__lxc_container/diff-conf-changes.sh
@@ -17,20 +17,20 @@
 #  1: parameter name
 #  2: file to print
 write_multi_param() {
-	if ! [ -f "$__object/parameter/$1" ]; then return 0; fi
+    if ! [ -f "$__object/parameter/$1" ]; then return 0; fi
 
-	while read -r line; do
-		if [ "$line" = "-" ]; then
-			# append stdin
-			cat "$__object/stdin"
-		elif [ -f "$line" ]; then
-			# append file content
-			cat "$line"
-		else
-			# print out content
-			printf "%s\n" "$line"
-		fi
-	done < "$__object/parameter/$1"
+    while read -r line; do
+        if [ "$line" = "-" ]; then
+            # append stdin
+            cat "$__object/stdin"
+        elif [ -f "$line" ]; then
+            # append file content
+            cat "$line"
+        else
+            # print out content
+            printf "%s\n" "$line"
+        fi
+    done < "$__object/parameter/$1"
 }
 
 # Function to get rid of whitespaces inside of key-value pairs. It will clear all of these.
@@ -38,11 +38,11 @@ write_multi_param() {
 # Parameters:
 #  1: the file which should be handled
 trimm_whitespaces() {
-	mv "$1" "$1"~  # backup file to read write back original file
-	grep -E -v "^([[:blank:]]*)(#|$)" "$1"~ \
-		| sed 's/^[[:blank:]]*//; s/[[:blank:]]*=[[:blank:]]*/ = /; s/[[:blank:]]*$//' \
-		> "$1"
-	rm -f "$1"~
+    mv "$1" "$1"~  # backup file to read write back original file
+    grep -E -v "^([[:blank:]]*)(#|$)" "$1"~ \
+        | sed 's/^[[:blank:]]*//; s/[[:blank:]]*=[[:blank:]]*/ = /; s/[[:blank:]]*$//' \
+        > "$1"
+    rm -f "$1"~
 }
 
 
@@ -53,33 +53,33 @@ state_is="$(cat "$__object/explorer/state")"
 # Only required if container will not be absent
 # prepare the wanted configuration lines
 if [ "$state_should" != "absent" ]; then
-	# create the files directory
-	mkdir "$__object/files/"
+    # create the files directory
+    mkdir "$__object/files/"
 
-	# Summary of the locally given configuration options
-	# config-present
-	conffile="$__object/files/config-present"
-	write_multi_param config > "$conffile"
-	trimm_whitespaces "$conffile"
-	# config-absent
-	absentfile="$__object/files/config-absent"
-	write_multi_param config-absent > "$absentfile"
-	trimm_whitespaces "$absentfile"
+    # Summary of the locally given configuration options
+    # config-present
+    conffile="$__object/files/config-present"
+    write_multi_param config > "$conffile"
+    trimm_whitespaces "$conffile"
+    # config-absent
+    absentfile="$__object/files/config-absent"
+    write_multi_param config-absent > "$absentfile"
+    trimm_whitespaces "$absentfile"
 
-	# Diff the configuration options by removing duplicate config options
-	# config.add
-	grep -v -f "$__object/explorer/config" -f "$__object/files/config-absent" \
-		"$__object/files/config-present" > "$__object/files/config.add" || true
-	# config.del
-	grep -f "$__object/explorer/config" \
-		"$__object/files/config-absent" > "$__object/files/config.del" || true
+    # Diff the configuration options by removing duplicate config options
+    # config.add
+    grep -v -f "$__object/explorer/config" -f "$__object/files/config-absent" \
+        "$__object/files/config-present" > "$__object/files/config.add" || true
+    # config.del
+    grep -f "$__object/explorer/config" \
+        "$__object/files/config-absent" > "$__object/files/config.del" || true
 
 
-	# fallback: if template parameter exists and container currently absent; try to remove all configs
-	#  If the container is currently absent, nothing will be removed, even it's in the default config
-	#  In this case, it will try to delete every configuration option to be sure
-	# only the cloning configuration is known, but not if the container will be created in an other way
-	if [ "$state_is" = "absent" ] && ! [ -f "$__object/parameter/clone" ]; then
-		cp "$__object/files/config-absent" "$__object/files/config.del"
-	fi
+    # fallback: if template parameter exists and container currently absent; try to remove all configs
+    #  If the container is currently absent, nothing will be removed, even it's in the default config
+    #  In this case, it will try to delete every configuration option to be sure
+    # only the cloning configuration is known, but not if the container will be created in an other way
+    if [ "$state_is" = "absent" ] && ! [ -f "$__object/parameter/clone" ]; then
+        cp "$__object/files/config-absent" "$__object/files/config.del"
+    fi
 fi
diff --git a/cdist/conf/type/__lxc_container/explorer/config b/cdist/conf/type/__lxc_container/explorer/config
index 16fcf14a..5aa020e4 100755
--- a/cdist/conf/type/__lxc_container/explorer/config
+++ b/cdist/conf/type/__lxc_container/explorer/config
@@ -6,7 +6,7 @@
 
 # abort if container will be absent - no config required
 if [ "$(cat "$__object/parameter/state")" = "absent" ]; then
-	exit 0
+    exit 0
 fi
 
 
@@ -26,25 +26,25 @@ fi
 # else, just use the normal container configuration
 clone="$__object/parameter/clone"
 if [ -f "$clone" ] && [ "$("$__type_explorer/state")" = "absent" ]; then
-	clone="$(cat "$clone")"
-	clonepath="$__object/parameter/clonepath"
-	if [ -f "$clonepath" ]; then
-		clonepath="$(cat "$clonepath")"
-	else
-		clonepath="$lxcpath"
-	fi
+    clone="$(cat "$clone")"
+    clonepath="$__object/parameter/clonepath"
+    if [ -f "$clonepath" ]; then
+        clonepath="$(cat "$clonepath")"
+    else
+        clonepath="$lxcpath"
+    fi
 
-	# set the config path of the container to clone from
-	config="$clonepath/$clone/config"
+    # set the config path of the container to clone from
+    config="$clonepath/$clone/config"
 else
-	# assemble the configuration file of the container
-	config="$lxcpath/$name/config"
+    # assemble the configuration file of the container
+    config="$lxcpath/$name/config"
 fi
 
 # check if the file exist, else the container is absent
 if [ -r "$config" ]; then
-	# print configuration and strip files to just values
-	#  grep will hide all comments and empty lines - all others must be key-values
-	#  sed will uniform patterns, that no spaces will be problematic (before, in the middle and at end)
-	grep -E -v "^([[:blank:]]*)(#|$)" "$config" | sed 's/^[[:blank:]]*//; s/[[:blank:]]*=[[:blank:]]*/ = /; s/[[:blank:]]*$//'
+    # print configuration and strip files to just values
+    #  grep will hide all comments and empty lines - all others must be key-values
+    #  sed will uniform patterns, that no spaces will be problematic (before, in the middle and at end)
+    grep -E -v "^([[:blank:]]*)(#|$)" "$config" | sed 's/^[[:blank:]]*//; s/[[:blank:]]*=[[:blank:]]*/ = /; s/[[:blank:]]*$//'
 fi
diff --git a/cdist/conf/type/__lxc_container/explorer/lxcpath b/cdist/conf/type/__lxc_container/explorer/lxcpath
index a4116b9b..e633cff6 100755
--- a/cdist/conf/type/__lxc_container/explorer/lxcpath
+++ b/cdist/conf/type/__lxc_container/explorer/lxcpath
@@ -9,15 +9,15 @@ lxcpath="$(cat "$__object/parameter/lxcpath" 2>/dev/null || true)"
 
 # lxcpath parameter
 if [ -z "$lxcpath" ]; then
-	user="$(cat "$__object/parameter/user")"
+    user="$(cat "$__object/parameter/user")"
 
-	# gather default value from config
-	if [ "$user" != "$USER" ]; then
-		su -s "$(which -- lxc-config)" -- "$user" lxc.lxcpath
-	else
-		lxc-config lxc.lxcpath
-	fi
+    # gather default value from config
+    if [ "$user" != "$USER" ]; then
+        su -s "$(which -- lxc-config)" -- "$user" lxc.lxcpath
+    else
+        lxc-config lxc.lxcpath
+    fi
 else
-	# output the parameter
-	echo "$lxcpath"
+    # output the parameter
+    echo "$lxcpath"
 fi
diff --git a/cdist/conf/type/__lxc_container/explorer/state b/cdist/conf/type/__lxc_container/explorer/state
index c3c7f30f..86831ce2 100755
--- a/cdist/conf/type/__lxc_container/explorer/state
+++ b/cdist/conf/type/__lxc_container/explorer/state
@@ -19,27 +19,27 @@ lxcpath="$(cat "$__object/parameter/lxcpath" 2>/dev/null || true)"
 # assemble optional parameters
 LXC_PARAM=""
 if [ "$lxcpath" ]; then
-	# shellcheck disable=SC2089
-	LXC_PARAM="$LXC_PARAM -P \"$lxcpath\""
+    # shellcheck disable=SC2089
+    LXC_PARAM="$LXC_PARAM -P \"$lxcpath\""
 fi
 
 
 # function to get information
 lxc_info() {
-	# shellcheck disable=SC2090 disable=SC2086
-	if [ "$user" != "$USER" ]; then
-		su -s "$(which -- lxc-info)" -- "$user" $LXC_PARAM -n "$name" -H "$@"
-	else
-		lxc-info $LXC_PARAM -n "$name" -H "$@"
-	fi
+    # shellcheck disable=SC2090 disable=SC2086
+    if [ "$user" != "$USER" ]; then
+        su -s "$(which -- lxc-info)" -- "$user" $LXC_PARAM -n "$name" -H "$@"
+    else
+        lxc-info $LXC_PARAM -n "$name" -H "$@"
+    fi
 }
 
 
 # the lxc-info command will exit non-zero if container not exist
 if ! lxc_info -S >/dev/null 2>&1; then
-	# not exist
-	echo "absent"
+    # not exist
+    echo "absent"
 else
-	# print state (command output matches to type states)
-	lxc_info -s | tr '[:upper:]' '[:lower:]'
+    # print state (command output matches to type states)
+    lxc_info -s | tr '[:upper:]' '[:lower:]'
 fi
diff --git a/cdist/conf/type/__lxc_container/gencode-remote b/cdist/conf/type/__lxc_container/gencode-remote
index 480d7d41..f9e4dceb 100755
--- a/cdist/conf/type/__lxc_container/gencode-remote
+++ b/cdist/conf/type/__lxc_container/gencode-remote
@@ -44,31 +44,31 @@ fi
 # shortcut for checking config changes
 # returns success (0) if there are changes
 config_changes() {
-	if [ -s "$__object/files/config.add" ] || [ -s "$__object/files/config.del" ]; then
-		return 0
-	else
-		return 1
-	fi
+    if [ -s "$__object/files/config.add" ] || [ -s "$__object/files/config.del" ]; then
+        return 0
+    else
+        return 1
+    fi
 }
 
 # shortcut to add bdev parameters if available
 #  $1: $__object parameter name without "bdev-"
 #  $2: target parameter flag (like --foo), else "--$1" will be used
 bdev_add_if_available() {
-	_param_path="$__object/parameter/bdev-$1"
-	if [ -f "$_param_path" ]; then
-		backingstore_opts="$backingstore_opts ${2:---$1} '$(cat "$_param_path")'"
-	fi
+    _param_path="$__object/parameter/bdev-$1"
+    if [ -f "$_param_path" ]; then
+        backingstore_opts="$backingstore_opts ${2:---$1} '$(cat "$_param_path")'"
+    fi
 }
 
 # shortcut to add template parameters if available
 #  $1: $__object parameter
 #  $2: target parameter flag (like --foo), else "--$1" will be used
 template_add_if_available() {
-	_param_path="$__object/parameter/$1"
-	if [ -f "$_param_path" ]; then
-		template_opts="$template_opts ${2:---$1} '$(cat "$_param_path")'"
-	fi
+    _param_path="$__object/parameter/$1"
+    if [ -f "$_param_path" ]; then
+        template_opts="$template_opts ${2:---$1} '$(cat "$_param_path")'"
+    fi
 }
 
 
@@ -81,291 +81,291 @@ if [ "$state_is" = "$state_should" ] && ( ! config_changes ); then exit; fi
 
 # handle if the container should exist or not
 case "$state_should" in
-	# all states where the container should exist
-	present|running|stopped|frozen)
-		# only relevant if the container does not exist before
-		if [ "$state_is" = "absent" ]; then
-			# change privileges (too complex to make exceptions)
-			cat <<USER
+    # all states where the container should exist
+    present|running|stopped|frozen)
+        # only relevant if the container does not exist before
+        if [ "$state_is" = "absent" ]; then
+            # change privileges (too complex to make exceptions)
+            cat <<USER
 su "$user" - <<SU
 USER
 
-			# backingstore options
-			backingstore_opts=""
-			if [ -f "$__object/parameter/backingstore" ]; then
-				# -B works for both types, they have different long opts
-				backingstore_opts="$backingstore_opts -B '$(cat "$__object/parameter/backingstore")'"
+            # backingstore options
+            backingstore_opts=""
+            if [ -f "$__object/parameter/backingstore" ]; then
+                # -B works for both types, they have different long opts
+                backingstore_opts="$backingstore_opts -B '$(cat "$__object/parameter/backingstore")'"
 
-				# add parameters if available
-				bdev_add_if_available dir
-				bdev_add_if_available fstype
-				bdev_add_if_available fssize
-				bdev_add_if_available lvname
-				bdev_add_if_available vgname
-				bdev_add_if_available thinpool
-			fi
+                # add parameters if available
+                bdev_add_if_available dir
+                bdev_add_if_available fstype
+                bdev_add_if_available fssize
+                bdev_add_if_available lvname
+                bdev_add_if_available vgname
+                bdev_add_if_available thinpool
+            fi
 
-			# check, if the container should be created or cloned
-			if [ -f "$__object/parameter/clone" ]; then
-				copy_from="$(cat "$__object/parameter/clone")"
-				copypath="$(cat "$__object/parameter/clonepath" 2>/dev/null || true)"
+            # check, if the container should be created or cloned
+            if [ -f "$__object/parameter/clone" ]; then
+                copy_from="$(cat "$__object/parameter/clone")"
+                copypath="$(cat "$__object/parameter/clonepath" 2>/dev/null || true)"
 
-				# assemble own optional lxc options
-				lxc_opts=""
-				if [ -n "$copypath" ]; then
-					# this is set as the default $lxcpath, because it is the origin
-					lxc_opts="$lxc_opts -P \"$copypath\""
-				fi
-				if [ -n "$lxcpath" ]; then
-					# this is set as the newpath, because it is the destination
-					lxc_opts="$lxc_opts -p \"$lxcpath\""
-				fi
+                # assemble own optional lxc options
+                lxc_opts=""
+                if [ -n "$copypath" ]; then
+                    # this is set as the default $lxcpath, because it is the origin
+                    lxc_opts="$lxc_opts -P \"$copypath\""
+                fi
+                if [ -n "$lxcpath" ]; then
+                    # this is set as the newpath, because it is the destination
+                    lxc_opts="$lxc_opts -p \"$lxcpath\""
+                fi
 
-				# print lxc-copy code (because of the lxcpath thing, $LXC_PARAM conflicts)
-				cat <<LXC
+                # print lxc-copy code (because of the lxcpath thing, $LXC_PARAM conflicts)
+                cat <<LXC
 lxc-copy $lxc_opts $backingstore_opts -n "$copy_from" --newname "$name"
 LXC
-			else
-				template="$(cat "$__object/parameter/template")"
+            else
+                template="$(cat "$__object/parameter/template")"
 
-				# assemble general creation options
-				create_opts=""
-				if [ -f "$__object/parameter/no-default-config" ]; then
-					# generate a random empty file and append
-					# maybe work with an nonexistant file - but just be correct
-					cat <<TMPFILE
+                # assemble general creation options
+                create_opts=""
+                if [ -f "$__object/parameter/no-default-config" ]; then
+                    # generate a random empty file and append
+                    # maybe work with an nonexistant file - but just be correct
+                    cat <<TMPFILE
 empty_conf="\$(mktemp "\${TMPDIR:-/tmp}/cdist__lxc_container-empty-conf.XXXXXXXXXX")"
 TMPFILE
-					create_opts="$create_opts -f \"\$empty_conf\""
+                    create_opts="$create_opts -f \"\$empty_conf\""
 
-				elif [ -f "$__object/parameter/default-config" ]; then
-					# append the configuration
-					create_opts="$create_opts -f \"$(cat "$__object/parameter/default-config")\""
-				fi
+                elif [ -f "$__object/parameter/default-config" ]; then
+                    # append the configuration
+                    create_opts="$create_opts -f \"$(cat "$__object/parameter/default-config")\""
+                fi
 
-				# assemble template options
-				template_opts=""
-				# add common options (may require different options for different templates)
-				template_add_if_available release -r
-				template_add_if_available arch -a
-				if [ -f "$__object/parameter/mirror" ]; then
-					template_opts="$template_opts --mirror='$(cat "$__object/parameter/mirror")'"
-				fi
-				template_add_if_available ssh-key -S
+                # assemble template options
+                template_opts=""
+                # add common options (may require different options for different templates)
+                template_add_if_available release -r
+                template_add_if_available arch -a
+                if [ -f "$__object/parameter/mirror" ]; then
+                    template_opts="$template_opts --mirror='$(cat "$__object/parameter/mirror")'"
+                fi
+                template_add_if_available ssh-key -S
 
-				# at last, apply custom options
-				if [ -f "$__object/parameter/template-opts" ]; then
-					while read -r line; do
-						template_opts="$template_opts $line"
-					done < "$__object/parameter/template-opts"
-				fi
+                # at last, apply custom options
+                if [ -f "$__object/parameter/template-opts" ]; then
+                    while read -r line; do
+                        template_opts="$template_opts $line"
+                    done < "$__object/parameter/template-opts"
+                fi
 
-				# print lxc-create code
-				cat <<LXC
+                # print lxc-create code
+                cat <<LXC
 lxc-create $LXC_PARAM $backingstore_opts -n "$name" -t "$template" $create_opts -- $template_opts
 LXC
 
-				# remove empty tempfile if it was created
-				if [ -f "$__object/parameter/no-default-config" ]; then
-					cat <<RM
+                # remove empty tempfile if it was created
+                if [ -f "$__object/parameter/no-default-config" ]; then
+                    cat <<RM
 rm -rf "\$empty_conf"
 RM
-				fi
-			fi
+                fi
+            fi
 
-			# end of su here-document
-			echo "SU"
+            # end of su here-document
+            echo "SU"
 
-			# write to the message system
-			echo "create" >> "$__messages_out"
-		fi
-		;;
+            # write to the message system
+            echo "create" >> "$__messages_out"
+        fi
+        ;;
 
 
-	# list other states to get a correct error message in the wildcard case
-	absent)
-		;;
+    # list other states to get a correct error message in the wildcard case
+    absent)
+        ;;
 
-	# error handling: invalid state
-	*)
-		printf "Container '%s' in unknown state %s\n" "$name" "$state_should" >&2
-		exit 2
-		;;
+    # error handling: invalid state
+    *)
+        printf "Container '%s' in unknown state %s\n" "$name" "$state_should" >&2
+        exit 2
+        ;;
 esac
 
 
 # Check the configurations only if the container exists and there are changes
 if config_changes; then
-	# generate config path
-	container_config="$(cat "$__object/explorer/lxcpath")/$name/config"
-	# create remote tmpfile for config
-	cat <<DONE
+    # generate config path
+    container_config="$(cat "$__object/explorer/lxcpath")/$name/config"
+    # create remote tmpfile for config
+    cat <<DONE
 tmpconfig="\$(mktemp "$container_config.XXXXXXXXXX")"
 cp -p "$container_config" "\$tmpconfig"
 DONE
 
-	# check if smth. to be deleted
-	#  must be before adding, because it takes the content of the original file
-	#  because 'cat $file > $file' will not work (> opens before command reads it)
-	# will create extended regex pattern for grep with malformed spaces
-	if [ -s "$__object/files/config.del" ]; then
-		cat <<DONE
+    # check if smth. to be deleted
+    #  must be before adding, because it takes the content of the original file
+    #  because 'cat $file > $file' will not work (> opens before command reads it)
+    # will create extended regex pattern for grep with malformed spaces
+    if [ -s "$__object/files/config.del" ]; then
+        cat <<DONE
 grep -v -E -f - <<'ABSENT' "$container_config" > "\$tmpconfig"
 $(awk -v FS=' = ' -v OFS=' = ' -v blank='[[:blank:]]*' '
 function ntostring(n) { ret=""; for(i=n; i<=NF; i++) ret=ret $i (i<NF ? OFS : ""); return ret }
 { printf "^%s%s%s=%s%s%s$%s", blank, $1, blank, blank, ntostring(2), blank, ORS }
-	' "$__object/files/config.del" )
+    ' "$__object/files/config.del" )
 ABSENT
 DONE
-	fi
+    fi
 
-	# check if smth. to be added
-	if [ -s "$__object/files/config.add" ]; then
-		cat <<DONE
+    # check if smth. to be added
+    if [ -s "$__object/files/config.add" ]; then
+        cat <<DONE
 cat <<'PRESENT' >> "\$tmpconfig"
 $(cat "$__object/files/config.add")
 PRESENT
 DONE
-	fi
+    fi
 
 
-	# apply all changes
-	cat <<DONE
+    # apply all changes
+    cat <<DONE
 rm -rf "$container_config"
 mv "\$tmpconfig" "$container_config"
 DONE
-	echo "config" >> "$__messages_out"
+    echo "config" >> "$__messages_out"
 
 
-	# restart container only if it is running
-	#  do not echo messages start/stop or melt/start/stop/freeze; only restart to avoid junk
-	case "$state_is" in
-		running|frozen)
-			# only do it if the container will run again
-			case "$state_should" in
-				present|running|frozen)
-					# su stuff
-					cat <<USER
+    # restart container only if it is running
+    #  do not echo messages start/stop or melt/start/stop/freeze; only restart to avoid junk
+    case "$state_is" in
+        running|frozen)
+            # only do it if the container will run again
+            case "$state_should" in
+                present|running|frozen)
+                    # su stuff
+                    cat <<USER
 su "$user" - <<SU
 USER
 
-					# unfreeze if required
-					if [ "$state_is" = "frozen" ]; then
-						cat <<LXC
+                    # unfreeze if required
+                    if [ "$state_is" = "frozen" ]; then
+                        cat <<LXC
 lxc-unfreeze $LXC_PARAM -n "$name"
 LXC
-					fi
+                    fi
 
-					# stop and start
-					cat <<LXC
+                    # stop and start
+                    cat <<LXC
 lxc-stop $LXC_PARAM -n "$name"
 lxc-start $LXC_PARAM -n "$name"
 LXC
 
-					# freeze it again if it was there before, but be clever
-					# if $state_is = running but should be frozen, later code will do that
-					if [ "$state_is" = "frozen" ]; then
-						if [ "$state_should" = "frozen" ] || [ "$state_should" = "present" ]; then
-							cat <<LXC
+                    # freeze it again if it was there before, but be clever
+                    # if $state_is = running but should be frozen, later code will do that
+                    if [ "$state_is" = "frozen" ]; then
+                        if [ "$state_should" = "frozen" ] || [ "$state_should" = "present" ]; then
+                            cat <<LXC
 lxc-freeze $LXC_PARAM -n "$name"
 LXC
-						else
-							# correct current state
-							state_is="running"
-							echo "melt" >> "$__messages_out"
-						fi
-					fi
+                        else
+                            # correct current state
+                            state_is="running"
+                            echo "melt" >> "$__messages_out"
+                        fi
+                    fi
 
-					# ending
-					echo "restart" >> "$__messages_out"
-					echo "SU"
-				;;
-			esac
-			;;
-	esac
+                    # ending
+                    echo "restart" >> "$__messages_out"
+                    echo "SU"
+                ;;
+            esac
+            ;;
+    esac
 fi
 
 
 # Check if there is a difference within the states
 if [ "$state_is" != "$state_should" ] && [ "$state_should" != "present" ]; then
-	# change privileges (too complex to make exceptions)
-	cat <<USER
+    # change privileges (too complex to make exceptions)
+    cat <<USER
 su "$user" - <<SU
 USER
 
-	# unfreeze if nessecary to change state
-	if [ "$state_is" = "frozen" ]; then
-		cat <<LXC
+    # unfreeze if nessecary to change state
+    if [ "$state_is" = "frozen" ]; then
+        cat <<LXC
 lxc-unfreeze $LXC_PARAM -n "$name"
 LXC
-		echo "melt" >> "$__messages_out"
-	fi
+        echo "melt" >> "$__messages_out"
+    fi
 
-	# handle if the container should exist or not
-	case "$state_should" in
-		running)
-			# already running if unfreezed
-			if [ "$state_is" != "frozen" ]; then
-				cat <<LXC
+    # handle if the container should exist or not
+    case "$state_should" in
+        running)
+            # already running if unfreezed
+            if [ "$state_is" != "frozen" ]; then
+                cat <<LXC
 lxc-start $LXC_PARAM -n "$name"
 LXC
-				echo "start" >> "$__messages_out"
-			fi
-			;;
+                echo "start" >> "$__messages_out"
+            fi
+            ;;
 
-		# stop the container if it should be removed
-		stopped)
-			cat <<LXC
+        # stop the container if it should be removed
+        stopped)
+            cat <<LXC
 lxc-stop $LXC_PARAM -n "$name"
 LXC
-			echo "stop" >> "$__messages_out"
-			;;
+            echo "stop" >> "$__messages_out"
+            ;;
 
-		# container should be stopped if not already done
-		absent)
-			if [ "$state_is" != "stopped" ]; then
-				cat <<LXC
+        # container should be stopped if not already done
+        absent)
+            if [ "$state_is" != "stopped" ]; then
+                cat <<LXC
 lxc-stop $LXC_PARAM -n "$name"
 LXC
-			fi
-			;;
+            fi
+            ;;
 
-		frozen)
-			# stopped containers can't be frozen; they must be started first
-			if [ "$state_is" = "stopped" ] || [ "$state_is" = "absent" ]; then
-				cat <<LXC
+        frozen)
+            # stopped containers can't be frozen; they must be started first
+            if [ "$state_is" = "stopped" ] || [ "$state_is" = "absent" ]; then
+                cat <<LXC
 lxc-start $LXC_PARAM -n "$name"
 LXC
-				echo "start" >> "$__messages_out"
-			fi
-			cat <<LXC
+                echo "start" >> "$__messages_out"
+            fi
+            cat <<LXC
 lxc-freeze $LXC_PARAM -n "$name"
 LXC
-			echo "freeze" >> "$__messages_out"
-			;;
+            echo "freeze" >> "$__messages_out"
+            ;;
 
-		*)
-			# must be checked by previous case
-			;;
-	esac
+        *)
+            # must be checked by previous case
+            ;;
+    esac
 
-	# end of su here-document
-	echo "SU"
+    # end of su here-document
+    echo "SU"
 fi
 
 
 # Check if the container needs to be removed
 if [ "$state_should" = "absent" ]; then
-	# change privileges (too complex to make exceptions)
-	# then, shutdown and delete the container
-	#  we could force-delete, but better be polite
-	#  snapshots are deleted, too - to keep everything clean. May someone does not want it?
-	cat <<USER
+    # change privileges (too complex to make exceptions)
+    # then, shutdown and delete the container
+    #  we could force-delete, but better be polite
+    #  snapshots are deleted, too - to keep everything clean. May someone does not want it?
+    cat <<USER
 su "$user" - <<SU
 lxc-destroy $LXC_PARAM --snapshots -n "$name"
 SU
 USER
 
-	# write to the message system
-	echo "destroy" >> "$__messages_out"
+    # write to the message system
+    echo "destroy" >> "$__messages_out"
 fi
diff --git a/cdist/conf/type/__lxc_container/issue.txt b/cdist/conf/type/__lxc_container/issue.txt
deleted file mode 100644
index f65f9780..00000000
--- a/cdist/conf/type/__lxc_container/issue.txt
+++ /dev/null
@@ -1 +0,0 @@
-$__object_id == name ? sperate parameter required?? multiple same names possible due to --lxcpath!
diff --git a/cdist/conf/type/__lxc_container/manifest b/cdist/conf/type/__lxc_container/manifest
index 99cfa1c0..0071c4fb 100755
--- a/cdist/conf/type/__lxc_container/manifest
+++ b/cdist/conf/type/__lxc_container/manifest
@@ -17,62 +17,62 @@ param="$__object/parameter/"
 
 # valid check if only --template or --clone given
 if [ -f "$param/template" ] && [ -f "$param/clone" ]; then
-	# error and exit
-	>&2 echo "error: can not create and clone a container at once!"
-	>&2 echo "error: --template and --clone can not work together"
-	exit 2
+    # error and exit
+    >&2 echo "error: can not create and clone a container at once!"
+    >&2 echo "error: --template and --clone can not work together"
+    exit 2
 fi
 
 # no clone option if --template given
 if [ -f "$param/template" ]; then
-	if [ -f "$param/clonepath" ]
-	then
-		>&2 echo "error: container will created by template, no clone options required!"
-		exit 2
-	fi
+    if [ -f "$param/clonepath" ]
+    then
+        >&2 echo "error: container will created by template, no clone options required!"
+        exit 2
+    fi
 fi
 
 # no template options if --clone given
 if [ -f "$param/clone" ]; then
-	if [ -f "$param/template-opts" ] \
-		|| [ -f "$param/default-config" ] \
-		|| [ -f "$param/no-default-config" ] \
-		|| [ -f "$param/release" ] \
-		|| [ -f "$param/arch" ]
-	then
-		>&2 echo "error: container will created by clone, no template options required!"
-		exit 2
-	fi
+    if [ -f "$param/template-opts" ] \
+        || [ -f "$param/default-config" ] \
+        || [ -f "$param/no-default-config" ] \
+        || [ -f "$param/release" ] \
+        || [ -f "$param/arch" ]
+    then
+        >&2 echo "error: container will created by clone, no template options required!"
+        exit 2
+    fi
 fi
 
 # check backingstore values
 if [ -f "$param/backingstore" ]; then
-	backingstore="$(cat "$param/backingstore")"
+    backingstore="$(cat "$param/backingstore")"
 
-	if [ "$backingstore" != "dir" ] && [ "$backingstore" != "none" ]; then
-		if [ -f "$param/bdev-dir" ]
-		then
-			>&2 echo "error: --bdev-dir is only possible for backingstore 'dir' or 'none'"
-			exit 2
-		fi
-	fi
+    if [ "$backingstore" != "dir" ] && [ "$backingstore" != "none" ]; then
+        if [ -f "$param/bdev-dir" ]
+        then
+            >&2 echo "error: --bdev-dir is only possible for backingstore 'dir' or 'none'"
+            exit 2
+        fi
+    fi
 
-	if [ "$backingstore" != "lvm" ] && [ "$backingstore" != "loop" ]; then
-		if [ -f "$param/bdev-fstype" ] \
-			|| [ -f "$param/bdev-fssize" ]
-		then
-			>&2 echo "error: --bdev-fstype and --bdev-fssize only available for backingstore 'lvm' or 'loop'"
-			exit 2
-		fi
-	fi
+    if [ "$backingstore" != "lvm" ] && [ "$backingstore" != "loop" ]; then
+        if [ -f "$param/bdev-fstype" ] \
+            || [ -f "$param/bdev-fssize" ]
+        then
+            >&2 echo "error: --bdev-fstype and --bdev-fssize only available for backingstore 'lvm' or 'loop'"
+            exit 2
+        fi
+    fi
 
-	if [ "$backingstore" != "lvm" ]; then
-		if [ -f "$param/bdev-lvname" ] \
-			|| [ -f "$param/bdev-vgname" ] \
-			|| [ -f "$param/bdev-thinpool" ]
-		then
-			>&2 echo "error: --bdev-{lv,vg}name and --bdev-thinpool only available for backingstore 'lvm'"
-			exit 2
-		fi
-	fi
+    if [ "$backingstore" != "lvm" ]; then
+        if [ -f "$param/bdev-lvname" ] \
+            || [ -f "$param/bdev-vgname" ] \
+            || [ -f "$param/bdev-thinpool" ]
+        then
+            >&2 echo "error: --bdev-{lv,vg}name and --bdev-thinpool only available for backingstore 'lvm'"
+            exit 2
+        fi
+    fi
 fi
diff --git a/cdist/conf/type/__lxc_container/todo.txt b/cdist/conf/type/__lxc_container/todo.txt
index e5f8f471..41822842 100644
--- a/cdist/conf/type/__lxc_container/todo.txt
+++ b/cdist/conf/type/__lxc_container/todo.txt
@@ -8,6 +8,10 @@ auto.. (lxc autostart):
   - group
   - start
 
-If config already created, it's considered the container is already created
-  -> first create; then setup
-  detection if template changed??
+config:
+  - remove whole sections
+  - keep prettier config files?
+
+
+if multi-valued object ids come to be a thing, following pattern is also used internal at lxc:
+  $lxcpath:$name

From 3965c7f73844873aabb1d839b2298eed70a6b35c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 21 Jul 2020 19:42:40 +0200
Subject: [PATCH 029/411] [type/__user] Install user{add,mod,del} packages on
 OpenWrt

---
 cdist/conf/type/__user/manifest | 42 +++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/type/__user/manifest b/cdist/conf/type/__user/manifest
index 8f10b38c..b9fad65b 100644
--- a/cdist/conf/type/__user/manifest
+++ b/cdist/conf/type/__user/manifest
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,16 +18,37 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
 # Manage users.
+#
 
-os=$(cat "$__global/explorer/os")
-
-case "$os" in
-    alpine)
-        __package shadow
-        ;;
-    *)
-        :
-        ;;
+case $(cat "${__global}/explorer/os")
+in
+	(alpine)
+		__package shadow
+		;;
+	(openwrt)
+		case $(cat "${__object}/parameter/state")
+		in
+			(present)
+				if test -s "${__object}/explorer/passwd"
+				then
+					# NOTE: The package might not be required if no changes
+					# are required, but determining if changes are required is
+					# out of scope here, and 40k should be okay, I hope.
+					__package shadow-usermod
+				else
+					__package shadow-useradd
+				fi
+				;;
+			(absent)
+				if test -s "${__object}/explorer/passwd"
+				then
+					__package shadow-userdel
+				fi
+				;;
+		esac
+		;;
+	(*)
+		:
+		;;
 esac

From d8b5c733f6cec871ceb1933807f4877a4ee77ffa Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 22 Jul 2020 06:36:27 +0200
Subject: [PATCH 030/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index dac9cbce..040de9d8 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,6 +10,7 @@ next:
 	* Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak)
 	* Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera)
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
+	* Type __user: Install user packages on OpenWRT (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From fdef468f1ab6f2163da22a477df154a2082b57b4 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 22 Jul 2020 18:28:41 +0200
Subject: [PATCH 031/411] Fix OpenWrt spelling

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 040de9d8..4c17ee58 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,7 +10,7 @@ next:
 	* Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak)
 	* Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera)
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
-	* Type __user: Install user packages on OpenWRT (Dennis Camera)
+	* Type __user: Install user packages on OpenWrt (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 595e43b8d5f06641179c13de8ac04be46c1b2820 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 23 Jul 2020 09:41:53 +0200
Subject: [PATCH 032/411] [type/{__file,__directory}] Fix incorrect
 interpretation of strings with leading 0s as octal

---
 cdist/conf/type/__directory/gencode-remote | 4 +++-
 cdist/conf/type/__file/gencode-remote      | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote
index 2c2c56fd..d9c00b56 100755
--- a/cdist/conf/type/__directory/gencode-remote
+++ b/cdist/conf/type/__directory/gencode-remote
@@ -99,7 +99,9 @@ case "$state_should" in
 
             # format mode in four digits => same as stat returns
             if [ "$attribute" = mode ]; then
-                value_should=$(printf '%04u' "${value_should}")
+                # Convert to four-digit octal number (printf interprets
+                # strings with leading 0s as octal!)
+                value_should=$(printf '%04o' "0${value_should}")
             fi
 
             if [ "$set_attributes" = 1 ] || [ "$value_should" != "$value_is" ]; then
diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index a69154df..35356b13 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -70,7 +70,9 @@ case "$state_should" in
 
                 # format mode in four digits => same as stat returns
                 if [ "$attribute" = mode ]; then
-                    value_should=$(printf '%04u' "${value_should}")
+                    # Convert to four-digit octal number (printf interprets
+                    # strings with leading 0s as octal!)
+                    value_should=$(printf '%04o' "0${value_should}")
                 fi
 
                 value_is="$(get_current_value "$attribute" "$value_should")"

From ae5f0bba0b103aabcfbad0f105c69c0b403697ac Mon Sep 17 00:00:00 2001
From: fnux <timothee.floure@ungleich.ch>
Date: Fri, 24 Jul 2020 12:26:35 +0200
Subject: [PATCH 033/411] Add Alpine support to __openldap_server

---
 cdist/conf/type/__openldap_server/man.rst  |  4 +--
 cdist/conf/type/__openldap_server/manifest | 42 ++++++++++++++++++----
 2 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
index fbad21d8..a96c7dad 100644
--- a/cdist/conf/type/__openldap_server/man.rst
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -103,8 +103,8 @@ syncrepl-host
     Set once per host that will replicate the directory.
 
 module
-    LDAP module to load. See `slapd.conf(5)`.
-    Default value is OS-dependent, see manifest.
+    LDAP module to load. See `slapd.conf(5)`. Some dependencies might have to
+    be installed beforehand. Default value is OS-dependent, see manifest.
 
 schema
     Name of LDAP schema to load. Must be the name without extension of a
diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest
index 84ba176f..2aeece26 100644
--- a/cdist/conf/type/__openldap_server/manifest
+++ b/cdist/conf/type/__openldap_server/manifest
@@ -25,6 +25,7 @@ case "${os}" in
         SLAPD_DATA_DIR="/var/db/openldap-data"
         SLAPD_RUN_DIR="/var/run/openldap"
         SLAPD_MODULE_PATH="/usr/local/libexec/openldap"
+        SLAPD_MODULE_TYPE="la"
         if [ -z "${slapd_modules}" ]; then
             # It looks like ppolicy and syncprov must be compiled
             slapd_modules="back_mdb back_monitor"
@@ -43,13 +44,34 @@ case "${os}" in
         SLAPD_DATA_DIR="/var/lib/ldap"
         SLAPD_RUN_DIR="/var/run/slapd"
         SLAPD_MODULE_PATH="/usr/lib/ldap"
+        SLAPD_MODULE_TYPE="la"
         if [ -z "${slapd_modules}" ]; then
             slapd_modules="back_mdb ppolicy syncprov back_monitor"
         fi
+        CONF_OWNER="openldap"
+        CONF_GROUP="openldap"
         if [ -z "${tls_cipher_suite}" ]; then
             tls_cipher_suite="NORMAL"
         fi
         ;;
+    alpine)
+        PKGS="openldap openldap-clients"
+        ETC="/etc"
+        SLAPD_DIR="/etc/openldap"
+        SLAPD_DATA_DIR="/var/lib/openldap"
+        SLAPD_RUN_DIR="/var/run/openldap"
+        SLAPD_MODULE_PATH="/usr/lib/openldap"
+        SLAPD_MODULE_TYPE="so"
+        if [ -z "${slapd_modules}" ]; then
+            slapd_modules="back_mdb ppolicy syncprov back_monitor"
+            PKGS="$PKGS openldap-back-mdb openldap-back-monitor openldap-overlay-all"
+        fi
+        CONF_OWNER="ldap"
+        CONF_GROUP="$SLAPD_USER"
+        if [ -z "${tls_cipher_suite}" ]; then
+            tls_cipher_suite="DEFAULT"
+        fi
+        ;;
     *)
         echo "Don't know the openldap defaults for: $os" >&2
         exit 1
@@ -156,6 +178,12 @@ case "${os}" in
                --line "SLAPD_SERVICES=\"${slapd_urls}\"" \
                --state present
         ;;
+    alpine)
+        require="__package/${PKG_MAIN}" __line add_slapd_services \
+               --file ${ETC}/conf.d/slapd \
+               --line "command_args=\"-h '${slapd_urls}'\"" \
+               --state present
+        ;;
     *)
         # Nothing to do here, move on.
         ;;
@@ -170,20 +198,22 @@ if [ -z "${_skip_letsencrypt_cert}" ]; then
     fi
 
     # shellcheck disable=SC2086
-    __letsencrypt_cert "${name}" --admin-email "${admin_email}" \
-        --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R openldap:openldap ${SLAPD_DIR}/sasl2 && service slapd restart" \
-        --automatic-renewal ${staging}
+    __directory ${SLAPD_DIR}/sasl2
+    require="__directory/${SLAPD_DIR}/sasl2" __letsencrypt_cert "${name}" \
+        --admin-email "${admin_email}" \
+        --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R ${CONF_OWNER}:${CONF_GROUP} ${SLAPD_DIR}/sasl2 && service slapd restart" \
+        --automatic-renewal "${staging}"
 fi
 
 require="__package/${PKG_MAIN}" __directory ${SLAPD_DIR}/slapd.d --state absent
 
 if [ -z "${_skip_letsencrypt_cert}" ]; then
     require="__package/${PKG_MAIN} __letsencrypt_cert/${name}" \
-           __file ${SLAPD_DIR}/slapd.conf --owner ${CONF_OWNER} --group ${CONF_GROUP} --mode 644 \
+           __file "${SLAPD_DIR}/slapd.conf" --owner "${CONF_OWNER}" --group "${CONF_GROUP}" --mode 644 \
            --source "${ldapconf}"
 else
     require="__package/${PKG_MAIN}" \
-           __file ${SLAPD_DIR}/slapd.conf --owner ${CONF_OWNER} --group ${CONF_GROUP} --mode 644 \
+           __file "${SLAPD_DIR}/slapd.conf" --owner "${CONF_OWNER}" --group "${CONF_GROUP}" --mode 644 \
            --source "${ldapconf}"
 fi
 
@@ -210,7 +240,7 @@ done
 # Add specified modules
 echo "modulepath ${SLAPD_MODULE_PATH}" >> "${ldapconf}"
 for module in ${slapd_modules}; do
-    echo "moduleload ${module}.la" >> "${ldapconf}"
+    echo "moduleload ${module}.${SLAPD_MODULE_TYPE}" >> "${ldapconf}"
 done
 
 # Rest of the config

From 8654cbe4661762f25cd75c484f28344ce91ec55b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 24 Jul 2020 12:29:02 +0200
Subject: [PATCH 034/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 4c17ee58..0b9b8308 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,7 @@ next:
 	* Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera)
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
 	* Type __user: Install user packages on OpenWrt (Dennis Camera)
+	* Type __openldap_server: Add Alpine support (Timothée Floure)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 8b53f35ffab4e7b74d3fd019e7602c8a05338173 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 24 Jul 2020 12:33:16 +0200
Subject: [PATCH 035/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 0b9b8308..2457e90b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,7 @@ next:
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
 	* Type __user: Install user packages on OpenWrt (Dennis Camera)
 	* Type __openldap_server: Add Alpine support (Timothée Floure)
+	* Type __pf_apply: Remove deprecated type (Darko Poljak)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From ee71cad047ba0c86343fe8ee9ea2bc79bf950256 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 25 Jul 2020 19:19:38 +0200
Subject: [PATCH 036/411] [type/__package_apt] Fix type for legacy APT versions

--no-install-recommends was introduced with Debian 5.
The APT::Install-Recommends option gets ignored by old versions and
produces no error.
---
 cdist/conf/type/__package_apt/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote
index e02564a2..f3d91566 100755
--- a/cdist/conf/type/__package_apt/gencode-remote
+++ b/cdist/conf/type/__package_apt/gencode-remote
@@ -64,7 +64,7 @@ esac
 
 # Hint if we need to avoid questions at some point:
 # DEBIAN_PRIORITY=critical can reduce the number of questions
-aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes --no-install-recommends -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
+aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o APT::Install-Recommends=0 -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
 
 if [ "$state_is" = "$state_should" ]; then
     if [ -z "$version" ] || [ "$version" = "$version_is" ]; then

From 46d09392f08b0659557048c02e109f189ffd85f3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 19:36:34 +0200
Subject: [PATCH 037/411] [type/__key_value] Get AWK from POSIX PATH

This is required here, because Solaris /usr/bin/awk does not support the
sub() function.
So xpg4 AWK needs to be used.
---
 cdist/conf/type/__key_value/explorer/state         | 4 +++-
 cdist/conf/type/__key_value/files/remote_script.sh | 5 ++++-
 cdist/conf/type/__key_value/gencode-remote         | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__key_value/explorer/state b/cdist/conf/type/__key_value/explorer/state
index 7b2de1df..d24600af 100755
--- a/cdist/conf/type/__key_value/explorer/state
+++ b/cdist/conf/type/__key_value/explorer/state
@@ -40,7 +40,9 @@ else
 fi
 export key state delimiter value exact_delimiter
 
-awk -f - "$file" <<"AWK_EOF"
+awk_bin=$(PATH=$(getconf PATH 2>/dev/null) && command -v awk || echo awk)
+
+"${awk_bin}" -f - "$file" <<"AWK_EOF"
 BEGIN {
     state=ENVIRON["state"]
     key=ENVIRON["key"]
diff --git a/cdist/conf/type/__key_value/files/remote_script.sh b/cdist/conf/type/__key_value/files/remote_script.sh
index f7a1add5..faf080cb 100644
--- a/cdist/conf/type/__key_value/files/remote_script.sh
+++ b/cdist/conf/type/__key_value/files/remote_script.sh
@@ -24,7 +24,10 @@ if [ -f "$file" ]; then
 else
     touch "$file"
 fi
-awk -f - "$file" >"$tmpfile" <<"AWK_EOF"
+
+awk_bin=$(PATH=$(getconf PATH 2>/dev/null) && command -v awk || echo awk)
+
+"${awk_bin}" -f - "$file" >"$tmpfile" <<"AWK_EOF"
 BEGIN {
     # import variables in a secure way ..
     state=ENVIRON["state"]
diff --git a/cdist/conf/type/__key_value/gencode-remote b/cdist/conf/type/__key_value/gencode-remote
index 13cc27c7..1174400e 100755
--- a/cdist/conf/type/__key_value/gencode-remote
+++ b/cdist/conf/type/__key_value/gencode-remote
@@ -25,7 +25,7 @@ state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
 fire_onchange=''
 
-if [  "$state_is" = "$state_should" ]; then
+if [ "$state_is" = "$state_should" ]; then
     exit 0
 fi
 

From a5905044365e0f28e793f01e0be0cac8e6b2b379 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 24 Jul 2020 10:19:10 +0200
Subject: [PATCH 038/411] [type/__locale_system] RedHat systems on systemd use
 /etc/locale.conf

---
 cdist/conf/type/__locale_system/manifest | 26 ++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 4a1fdeed..22531a40 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -3,6 +3,7 @@
 # 2012-2016 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2016 Carlos Ortigoza (carlos.ortigoza at ungleich.ch)
 # 2016 Nico Schottelius (nico.schottelius at ungleich.ch)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -32,8 +33,25 @@ case "$os" in
     archlinux)
         locale_conf="/etc/locale.conf"
         ;;
-    redhat|centos)
-        locale_conf="/etc/sysconfig/i18n"
+    centos|redhat|scientific)
+        # shellcheck source=/dev/null
+        version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
+        if expr "${version_id}" '>=' 7 >/dev/null
+        then
+            locale_conf="/etc/locale.conf"
+        else
+            locale_conf="/etc/sysconfig/i18n"
+        fi
+        ;;
+    fedora)
+        # shellcheck source=/dev/null
+        version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
+        if expr "${version_id}" '>=' 18 >/dev/null
+        then
+            locale_conf="/etc/locale.conf"
+        else
+            locale_conf="/etc/sysconfig/i18n"
+        fi
         ;;
     *)
         echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
@@ -47,9 +65,9 @@ __file "$locale_conf" \
        --state exists
 
 require="__file/$locale_conf" \
-       __key_value "$locale_conf:$__object_id" \
+__key_value "$locale_conf:$__object_id" \
        --file "$locale_conf" \
        --key "$__object_id" \
-       --delimiter = \
+       --delimiter '=' --exact_delimiter \
        --state "$(cat "$__object/parameter/state")" \
        --value "$(cat "$__object/parameter/value")"

From 47e28fc441187b5deb869f53396eac1ee0f8cbe7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 12:07:38 +0200
Subject: [PATCH 039/411] [type/__locale_system] Support old Debian derivatives

---
 cdist/conf/type/__locale_system/manifest | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 22531a40..b9991fa3 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -27,9 +27,29 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-    debian|devuan|ubuntu)
+    debian)
+        os_version=$(cat "${__global}/explorer/os_version")
+        if expr "${os_version}" '>=' 4 >/dev/null
+        then
+            # Debian 4 (etch) and later
+            locale_conf="/etc/default/locale"
+        else
+            locale_conf="/etc/environment"
+        fi
+        ;;
+    devuan)
         locale_conf="/etc/default/locale"
         ;;
+    ubuntu)
+        os_version=$(cat "${__global}/explorer/os_version")
+        if expr "${os_version}" '>=' 6.10 >/dev/null
+        then
+            # Ubuntu 6.10 (edgy) and later
+            locale_conf="/etc/default/locale"
+        else
+            locale_conf="/etc/environment"
+        fi
+        ;;
     archlinux)
         locale_conf="/etc/locale.conf"
         ;;
@@ -61,7 +81,7 @@ case "$os" in
 esac
 
 __file "$locale_conf" \
-       --owner root --group root --mode 644 \
+       --owner root --group root --mode 0644 \
        --state exists
 
 require="__file/$locale_conf" \

From 0ef54a721d6f540f70959df2b7f1321a5688a594 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 12:17:30 +0200
Subject: [PATCH 040/411] [type/__locale_system] Add support for Gentoo Linux

---
 cdist/conf/type/__locale_system/manifest | 45 ++++++++++++++++++------
 1 file changed, 34 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index b9991fa3..0049ed5f 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -24,9 +24,19 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
+onchange_cmd=  # none, by default
+quote_value=false
+
+catval() {
+    # shellcheck disable=SC2059
+    printf "$($quote_value && echo '"%s"' || echo '%s')" "$(cat "$1")"
+}
+
+
 os=$(cat "$__global/explorer/os")
 
-case "$os" in
+case $os
+in
     debian)
         os_version=$(cat "${__global}/explorer/os_version")
         if expr "${os_version}" '>=' 4 >/dev/null
@@ -69,10 +79,24 @@ case "$os" in
         if expr "${version_id}" '>=' 18 >/dev/null
         then
             locale_conf="/etc/locale.conf"
+            quote_value=false
         else
             locale_conf="/etc/sysconfig/i18n"
         fi
         ;;
+    gentoo)
+        case $(cat "${__global}/explorer/init")
+        in
+            (*openrc*)
+                locale_conf="/etc/env.d/02locale"
+                onchange_cmd="env-update --no-ldconfig"
+                quote_value=true
+                ;;
+            (systemd)
+                locale_conf="/etc/locale.conf"
+                ;;
+        esac
+        ;;
     *)
         echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
         echo "Please contribute an implementation for it if you can." >&2
@@ -80,14 +104,13 @@ case "$os" in
         ;;
 esac
 
-__file "$locale_conf" \
-       --owner root --group root --mode 0644 \
-       --state exists
+__file "${locale_conf}" --state exists --owner root --group root --mode 0644
 
-require="__file/$locale_conf" \
-__key_value "$locale_conf:$__object_id" \
-       --file "$locale_conf" \
-       --key "$__object_id" \
-       --delimiter '=' --exact_delimiter \
-       --state "$(cat "$__object/parameter/state")" \
-       --value "$(cat "$__object/parameter/value")"
+require="__file/${locale_conf}" \
+__key_value "${locale_conf}:${__object_id}" \
+    --file "${locale_conf}" \
+    --key "${__object_id}" \
+    --delimiter '=' --exact_delimiter \
+    --state "$(cat "${__object}/parameter/state")" \
+    --value "$(catval "${__object}/parameter/value")" \
+    --onchange "${onchange_cmd}"

From 630d987d5f5a40ae783fd9d4ff0a10653fcab04c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 12:28:20 +0200
Subject: [PATCH 041/411] [type/__locale_system] Add support for Void Linux

---
 cdist/conf/type/__locale_system/manifest | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 0049ed5f..ff0a8c23 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -97,6 +97,9 @@ in
                 ;;
         esac
         ;;
+    voidlinux)
+        locale_conf="/etc/locale.conf"
+        ;;
     *)
         echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
         echo "Please contribute an implementation for it if you can." >&2

From 0ae0935afafd6b10516e9689c19000ef729d1b9b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 14:05:36 +0200
Subject: [PATCH 042/411] [type/__locale_system] Add support for SuSE

---
 cdist/conf/type/__locale_system/manifest | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index ff0a8c23..26d4f7b5 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -24,6 +24,7 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
+key=$__object_id
 onchange_cmd=  # none, by default
 quote_value=false
 
@@ -97,6 +98,25 @@ in
                 ;;
         esac
         ;;
+    suse)
+        os_version=$(cat "${__global}/explorer/os_version")
+        os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
+
+        # https://documentation.suse.com/sles/15-SP2/html/SLES-all/cha-suse.html#sec-suse-l10n
+        if expr "${os_major}" '>=' 15 \& "${os_major}" != 42
+        then
+            # It seems that starting with SuSE 15 the systemd /etc/locale.conf
+            # is the preferred way to set locales, although
+            # /etc/sysconfig/language is still available.
+            # Older documentation doesn't mention /etc/locale.conf, even though
+            # is it created when localectl is used.
+            locale_conf="/etc/locale.conf"
+        else
+            locale_conf="/etc/sysconfig/language"
+            quote_value=true
+            key="RC_${__object_id}"
+        fi
+        ;;
     voidlinux)
         locale_conf="/etc/locale.conf"
         ;;
@@ -110,9 +130,9 @@ esac
 __file "${locale_conf}" --state exists --owner root --group root --mode 0644
 
 require="__file/${locale_conf}" \
-__key_value "${locale_conf}:${__object_id}" \
+__key_value "${locale_conf}:${key}" \
     --file "${locale_conf}" \
-    --key "${__object_id}" \
+    --key "${key}" \
     --delimiter '=' --exact_delimiter \
     --state "$(cat "${__object}/parameter/state")" \
     --value "$(catval "${__object}/parameter/value")" \

From cbf22f3b2c8b9d0fef8f6b7b0bf16508f96675b3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 15:23:55 +0200
Subject: [PATCH 043/411] [type/__locale_system] Add support for Solaris

---
 cdist/conf/type/__locale_system/manifest | 41 ++++++++++++++++++++++--
 1 file changed, 39 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 26d4f7b5..71491fe5 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -33,6 +33,7 @@ catval() {
     printf "$($quote_value && echo '"%s"' || echo '%s')" "$(cat "$1")"
 }
 
+state_should=$(cat "${__object}/parameter/state")
 
 os=$(cat "$__global/explorer/os")
 
@@ -98,6 +99,39 @@ in
                 ;;
         esac
         ;;
+    solaris)
+        locale_conf="/etc/default/init"
+        locale_conf_group="sys"
+
+        if expr "$(cat "${__global}/explorer/os_version")" '>=' 5.11 >/dev/null
+        then
+            # mode on Oracle Solaris 11 is actually 0444,
+            # but the write bit makes sense, IMO
+            locale_conf_mode=0644
+
+            # Oracle Solaris 11.2 and later uses SMF to store environment info.
+            # This is a hack, but I didn't feel like modifying the whole type
+            # just for some Oracle nonsense.
+            # 11.3 apparently added nlsadm(1m), but it is missing from 11.2.
+            # Illumos continues to use /etc/default/init
+            # NOTE: Remember not to use "cool" POSIX features like -q or -e with
+            # Solaris grep.
+            release_regex='Oracle Solaris 11.[2-9][0-9]*'
+            case $state_should
+            in
+                (present)
+                    svccfg_cmd="svccfg -s svc:/system/environment:init setprop environment/${key} = astring: '$(cat "${__object}/parameter/value")'"
+                    ;;
+                (absent)
+                    svccfg_cmd="svccfg -s svc:/system/environment:init delprop environment/${key}"
+                    ;;
+            esac
+            refresh_cmd='svcadm refresh svc:/system/environment'
+            onchange_cmd="grep '${release_regex}' /etc/release >&- || exit 0; ${svccfg_cmd:-:} && ${refresh_cmd}"
+        else
+            locale_conf_mode=0555
+        fi
+        ;;
     suse)
         os_version=$(cat "${__global}/explorer/os_version")
         os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
@@ -127,13 +161,16 @@ in
         ;;
 esac
 
-__file "${locale_conf}" --state exists --owner root --group root --mode 0644
+__file "${locale_conf}" --state exists \
+    --owner "${locale_conf_owner:-0}" \
+    --group "${locale_conf_group:-0}" \
+    --mode "${locale_conf_mode:-0644}"
 
 require="__file/${locale_conf}" \
 __key_value "${locale_conf}:${key}" \
     --file "${locale_conf}" \
     --key "${key}" \
     --delimiter '=' --exact_delimiter \
-    --state "$(cat "${__object}/parameter/state")" \
+    --state "${state_should}" \
     --value "$(catval "${__object}/parameter/value")" \
     --onchange "${onchange_cmd}"

From a923e75d9b0054212031507986d3b935d74d52bf Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 18:29:14 +0200
Subject: [PATCH 044/411] [type/__locale_system] Add support for NetBSD

---
 cdist/conf/type/__locale_system/manifest | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 71491fe5..92af852f 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -99,6 +99,15 @@ in
                 ;;
         esac
         ;;
+    netbsd)
+        # NetBSD doesn't  have a separate configuration file to set locales.
+        # So the shell login file will have to do.
+        # "Non-POSIX" shells like csh will not be updated here.
+
+        locale_conf="/etc/profile"
+        quote_value=true
+        value="$(catval "${__object}/parameter/value"); export ${key}"
+        ;;
     solaris)
         locale_conf="/etc/default/init"
         locale_conf_group="sys"
@@ -172,5 +181,5 @@ __key_value "${locale_conf}:${key}" \
     --key "${key}" \
     --delimiter '=' --exact_delimiter \
     --state "${state_should}" \
-    --value "$(catval "${__object}/parameter/value")" \
+    --value "${value:-$(catval "${__object}/parameter/value")}" \
     --onchange "${onchange_cmd}"

From 511d8c96aa9201049eaafc459a2de9694fe2a4e4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 19:23:17 +0200
Subject: [PATCH 045/411] [type/__locale_system] Add support for Slackware

---
 cdist/conf/type/__locale_system/manifest | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 92af852f..180788e6 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -141,6 +141,12 @@ in
             locale_conf_mode=0555
         fi
         ;;
+    slackware)
+        # NOTE: lang.csh (csh config) is ignored here.
+        locale_conf="/etc/profile.d/lang.sh"
+        locale_conf_mode=0755
+        key="export ${__object_id}"
+        ;;
     suse)
         os_version=$(cat "${__global}/explorer/os_version")
         os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
@@ -176,7 +182,7 @@ __file "${locale_conf}" --state exists \
     --mode "${locale_conf_mode:-0644}"
 
 require="__file/${locale_conf}" \
-__key_value "${locale_conf}:${key}" \
+__key_value "${locale_conf}:${key#export }" \
     --file "${locale_conf}" \
     --key "${key}" \
     --delimiter '=' --exact_delimiter \

From 70d1228dc0c6a3af105f0864b46d7dea806aedc2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 19:26:45 +0200
Subject: [PATCH 046/411] [type/__locale_system] Add support for FreeBSD

---
 cdist/conf/type/__locale_system/manifest | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 180788e6..e4286ef6 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -99,9 +99,10 @@ in
                 ;;
         esac
         ;;
-    netbsd)
-        # NetBSD doesn't  have a separate configuration file to set locales.
-        # So the shell login file will have to do.
+    freebsd|netbsd)
+        # NetBSD doesn't have a separate configuration file to set locales.
+        # In FreeBSD locales could be configured via /etc/login.conf but parsing
+        # that would be annoying, so the shell login file will have to do.
         # "Non-POSIX" shells like csh will not be updated here.
 
         locale_conf="/etc/profile"

From 73f1937636bfa59eea0432e3625c0a27cc8171b5 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 27 Jul 2020 06:20:21 +0200
Subject: [PATCH 047/411] [__unpack] no mkdir by default, because destination
 can be file, but tar needs mkdir andrar needs slash at the end

---
 cdist/conf/type/__unpack/explorer/state       | 37 +++++++++
 cdist/conf/type/__unpack/gencode-remote       | 75 ++++++++++++++++++
 cdist/conf/type/__unpack/man.rst              | 79 +++++++++++++++++++
 cdist/conf/type/__unpack/manifest             | 41 ++++++++++
 cdist/conf/type/__unpack/parameter/boolean    |  2 +
 cdist/conf/type/__unpack/parameter/optional   |  2 +
 cdist/conf/type/__unpack/parameter/required   |  1 +
 cdist/conf/type/__unpack/test/README          |  3 +
 .../type/__unpack/test/make-init-manifest.sh  | 22 ++++++
 .../type/__unpack/test/make-test-files.sh     | 44 +++++++++++
 10 files changed, 306 insertions(+)
 create mode 100755 cdist/conf/type/__unpack/explorer/state
 create mode 100755 cdist/conf/type/__unpack/gencode-remote
 create mode 100644 cdist/conf/type/__unpack/man.rst
 create mode 100755 cdist/conf/type/__unpack/manifest
 create mode 100644 cdist/conf/type/__unpack/parameter/boolean
 create mode 100644 cdist/conf/type/__unpack/parameter/optional
 create mode 100644 cdist/conf/type/__unpack/parameter/required
 create mode 100644 cdist/conf/type/__unpack/test/README
 create mode 100755 cdist/conf/type/__unpack/test/make-init-manifest.sh
 create mode 100755 cdist/conf/type/__unpack/test/make-test-files.sh

diff --git a/cdist/conf/type/__unpack/explorer/state b/cdist/conf/type/__unpack/explorer/state
new file mode 100755
index 00000000..38bc0978
--- /dev/null
+++ b/cdist/conf/type/__unpack/explorer/state
@@ -0,0 +1,37 @@
+#!/bin/sh -e
+
+src="/$__object_id"
+
+if [ -f "$__object/parameter/sum-file" ]
+then
+    src_sum_was_file="$( cat "$__object/parameter/sum-file" )"
+else
+    src_sum_was_file="$src.cdist__unpack_sum"
+fi
+
+if [ ! -f "$src" ]
+then
+    if [ -n "$__cdist_dry_run" ]
+    then
+        echo 'mismatch'
+    else
+        echo 'missing'
+    fi
+else
+    if [ ! -f "$src_sum_was_file" ]
+    then
+        echo 'mismatch'
+        exit 0
+    fi
+
+    src_sum_was="$( cat "$src_sum_was_file" )"
+
+    src_sum_is="$( cksum "$src" | awk '{ print $1$2 }' )"
+
+    if [ "$src_sum_was" = "$src_sum_is" ]
+    then
+        echo 'match'
+    else
+        echo 'mismatch'
+    fi
+fi
diff --git a/cdist/conf/type/__unpack/gencode-remote b/cdist/conf/type/__unpack/gencode-remote
new file mode 100755
index 00000000..45c7173a
--- /dev/null
+++ b/cdist/conf/type/__unpack/gencode-remote
@@ -0,0 +1,75 @@
+#!/bin/sh -e
+
+if grep -Eq '^(missing|match)$' "$__object/explorer/state"
+then
+    exit 0
+fi
+
+os="$( cat "$__global/explorer/os" )"
+
+src="/$__object_id"
+
+dst="$( sed 's/\/$//' "$__object/parameter/destination" )"
+
+cmd=''
+
+case "$src" in
+    *.tar|*.tgz|*.tar.*)
+        cmd="mkdir -p '$dst' && tar --directory='$dst' --extract --file='$src'"
+
+        if [ -f "$__object/parameter/tar-strip" ]
+        then
+            tar_strip="$( cat "$__object/parameter/tar-strip" )"
+
+            cmd="$cmd --strip-components=$tar_strip"
+        fi
+    ;;
+    *.7z)
+        case "$os" in
+            centos|fedora|redhat)
+                cmd='7za'
+            ;;
+            *)
+                cmd='7zr'
+            ;;
+        esac
+
+        cmd="$cmd e -aoa -o'$dst' '$src'"
+    ;;
+    *.bz2)
+        cmd="bunzip2 --stdout '$src' > '$dst'"
+    ;;
+    *.gz)
+        cmd="gunzip --stdout '$src' > '$dst'"
+    ;;
+    *.lzma|*.xz)
+        cmd="xz --uncompress --stdout '$src' > '$dst'"
+    ;;
+    *.rar)
+        cmd="unrar x -o+ '$src' '$dst/'"
+    ;;
+    *.zip)
+        cmd="unzip -o '$src' -d '$dst'"
+    ;;
+esac
+
+if [ -f "$__object/parameter/backup-destination" ]
+then
+    echo "if [ -e '$dst' ]; then mv '$dst' '$dst.cdist__unpack_backup_$( date +%s )'; fi"
+fi
+
+echo "$cmd"
+
+if [ -f "$__object/parameter/sum-file" ]
+then
+    sum_file="$( cat "$__object/parameter/sum-file" )"
+else
+    sum_file="$src.cdist__unpack_sum"
+fi
+
+echo "cksum '$src' | awk '{ print \$1\$2 }' > '$sum_file'"
+
+if [ ! -f "$__object/parameter/preserve-archive" ]
+then
+    echo "rm -f '$src'"
+fi
diff --git a/cdist/conf/type/__unpack/man.rst b/cdist/conf/type/__unpack/man.rst
new file mode 100644
index 00000000..8fe96e43
--- /dev/null
+++ b/cdist/conf/type/__unpack/man.rst
@@ -0,0 +1,79 @@
+cdist-type__unpack(7)
+=====================
+
+NAME
+----
+cdist-type__unpack - Unpack archives
+
+
+DESCRIPTION
+-----------
+Unpack ``.tar``, ``.tgz``, ``.tar.*``, ``.7z``, ``.bz2``, ``.gz``,
+``.lzma``, ``.xz``, ``.rar`` and ``.zip`` archives. Archive type is
+detected by extension.
+
+To achieve idempotency, checksum file will be created in target. See
+``--sum-file`` parameter for details.
+
+
+REQUIRED PARAMETERS
+-------------------
+destination
+   Depending on archive format file or directory to where archive
+   contents will be written.
+
+
+OPTIONAL PARAMETERS
+-------------------
+sum-file
+    Override archive's checksum file in target. By default
+    ``XXX.cdist__unpack_sum`` will be used, where ``XXX`` is source
+    archive path. This file must be kept in target's persistent storage.
+
+tar-strip
+    Tarball specific. See ``man tar`` for ``--strip-components``.
+
+
+OPTIONAL BOOLEAN PARAMETERS
+---------------------------
+backup-destination
+    By default destination file will be overwritten. In case destination
+    is directory, files from archive will be added to or overwritten in
+    directory. This parameter moves existing destination to
+    ``XXX.cdist__unpack_backup_YYY``, where ``XXX`` is destination and
+    ``YYY`` current UNIX timestamp.
+
+preserve-archive
+    Don't delete archive after unpacking.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __directory /opt/cpma
+
+    require='__directory/opt/cpma' \
+        __download /opt/cpma/cnq3.zip \
+            --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
+            --sum md5:46da3021ca9eace277115ec9106c5b46
+
+    require='__download/opt/cpma/cnq3.zip' \
+        __unpack /opt/cpma/cnq3.zip \
+            --backup-destination \
+            --preserve-archive \
+            --destination /opt/cpma/server
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__unpack/manifest b/cdist/conf/type/__unpack/manifest
new file mode 100755
index 00000000..6bdf5a10
--- /dev/null
+++ b/cdist/conf/type/__unpack/manifest
@@ -0,0 +1,41 @@
+#!/bin/sh -e
+
+os="$( cat "$__global/explorer/os" )"
+
+src="/$__object_id"
+
+case "$src" in
+    *.7z)
+        __package p7zip
+    ;;
+    *.bz2)
+        case "$os" in
+            freebsd)
+                # bzip2 is part of freebsd base system
+            ;;
+            *)
+                __package bzip2
+            ;;
+        esac
+    ;;
+    *.lzma|*.xz|*.txz)
+        case "$os" in
+            debian|ubuntu|devuan)
+                __package xz-utils
+            ;;
+            alpine|centos)
+                __package xz
+            ;;
+        esac
+    ;;
+    *.rar)
+        case "$os" in
+            debian|ubuntu|devuan|alpine|freebsd)
+                __package unrar
+            ;;
+        esac
+    ;;
+    *.zip)
+        __package unzip
+    ;;
+esac
diff --git a/cdist/conf/type/__unpack/parameter/boolean b/cdist/conf/type/__unpack/parameter/boolean
new file mode 100644
index 00000000..99dca934
--- /dev/null
+++ b/cdist/conf/type/__unpack/parameter/boolean
@@ -0,0 +1,2 @@
+backup-destination
+preserve-archive
diff --git a/cdist/conf/type/__unpack/parameter/optional b/cdist/conf/type/__unpack/parameter/optional
new file mode 100644
index 00000000..d136dd0c
--- /dev/null
+++ b/cdist/conf/type/__unpack/parameter/optional
@@ -0,0 +1,2 @@
+sum-file
+tar-strip
diff --git a/cdist/conf/type/__unpack/parameter/required b/cdist/conf/type/__unpack/parameter/required
new file mode 100644
index 00000000..ac459b09
--- /dev/null
+++ b/cdist/conf/type/__unpack/parameter/required
@@ -0,0 +1 @@
+destination
diff --git a/cdist/conf/type/__unpack/test/README b/cdist/conf/type/__unpack/test/README
new file mode 100644
index 00000000..54f3972a
--- /dev/null
+++ b/cdist/conf/type/__unpack/test/README
@@ -0,0 +1,3 @@
+./make-test-files.sh
+./make-init-manifest.sh | cdist config -i - localhost
+sudo find /tmp/cdist__unpack_test/ -type f -exec cat {} \; | sort
diff --git a/cdist/conf/type/__unpack/test/make-init-manifest.sh b/cdist/conf/type/__unpack/test/make-init-manifest.sh
new file mode 100755
index 00000000..404bc106
--- /dev/null
+++ b/cdist/conf/type/__unpack/test/make-init-manifest.sh
@@ -0,0 +1,22 @@
+#!/bin/sh -e
+
+p="$( pwd )"
+d=/tmp/cdist__unpack_test
+
+echo 'export CDIST_ORDER_DEPENDENCY=1'
+
+echo "__directory $d"
+
+find "$p" -name 'test.*' -and -not -name '*.cdist__unpack_sum' \
+    | sort \
+    | while read -r l
+do
+    n="$( basename "$l" )"
+
+    printf '__unpack %s --destination %s/%s\n' \
+        "$l" \
+        "$d" \
+        "$n"
+done
+
+echo "__clean_path $p --pattern '.+/test\..+'"
diff --git a/cdist/conf/type/__unpack/test/make-test-files.sh b/cdist/conf/type/__unpack/test/make-test-files.sh
new file mode 100755
index 00000000..d18e9e9f
--- /dev/null
+++ b/cdist/conf/type/__unpack/test/make-test-files.sh
@@ -0,0 +1,44 @@
+#!/bin/sh -ex
+
+echo test.7z > test
+7z a test.7z test > /dev/null
+
+echo test.bz2 > test
+bzip2 test
+
+echo test.gz > test
+gzip test
+
+echo test.lzma > test
+lzma test
+
+echo test.rar > test
+rar a test.rar test > /dev/null
+
+echo test.tar.bz2 > test
+tar cf test.tar test
+bzip2 test.tar
+
+echo test.tar.xz > test
+tar cf test.tar test
+xz test.tar
+
+echo test.tgz > test
+tar cf test.tar test
+gzip test.tar
+mv test.tar.gz test.tgz
+
+echo test.tar.gz > test
+tar cf test.tar test
+gzip test.tar
+
+echo test.tar > test
+tar cf test.tar test
+
+echo test.xz > test
+xz test
+
+echo test.zip > test
+zip test.zip test > /dev/null
+
+rm test

From 463b6cd6b52da8918d2803721164748c2a632696 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 27 Jul 2020 06:22:25 +0200
Subject: [PATCH 048/411] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 2457e90b..8d26e767 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -13,6 +13,9 @@ next:
 	* Type __user: Install user packages on OpenWrt (Dennis Camera)
 	* Type __openldap_server: Add Alpine support (Timothée Floure)
 	* Type __pf_apply: Remove deprecated type (Darko Poljak)
+	* Type __package_apt: Fix for legacy APT versions that do not support --no-install-recommends (Dennis Camera)
+	* Type __key_value: Get awk from POSIX PATH (Dennis Camera)
+	* New type: __unpack (Ander Punnar)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From d26c36914a4fa9889af57f549cb2854968663a0b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 27 Jul 2020 10:55:28 +0200
Subject: [PATCH 049/411] [__timezone] Make type singleton

---
 cdist/conf/type/__timezone/gencode-remote     |  2 +-
 cdist/conf/type/__timezone/man.rst            | 22 ++++++++++++-------
 cdist/conf/type/__timezone/manifest           |  2 +-
 cdist/conf/type/__timezone/parameter/required |  1 +
 cdist/conf/type/__timezone/singleton          |  0
 5 files changed, 17 insertions(+), 10 deletions(-)
 create mode 100644 cdist/conf/type/__timezone/parameter/required
 create mode 100644 cdist/conf/type/__timezone/singleton

diff --git a/cdist/conf/type/__timezone/gencode-remote b/cdist/conf/type/__timezone/gencode-remote
index 5299f548..b685c990 100755
--- a/cdist/conf/type/__timezone/gencode-remote
+++ b/cdist/conf/type/__timezone/gencode-remote
@@ -22,7 +22,7 @@
 # This type allows to configure the desired localtime timezone.
 
 timezone_is=$(cat "$__object/explorer/timezone_is")
-timezone_should="$__object_id"
+timezone_should=$(cat "$__object/parameter/tz")
 os=$(cat "$__global/explorer/os")
 
 if [ "$timezone_is" = "$timezone_should" ]; then
diff --git a/cdist/conf/type/__timezone/man.rst b/cdist/conf/type/__timezone/man.rst
index 8a945c16..6012c552 100644
--- a/cdist/conf/type/__timezone/man.rst
+++ b/cdist/conf/type/__timezone/man.rst
@@ -14,7 +14,8 @@ This type creates a symlink (/etc/localtime) to the selected timezone
 
 REQUIRED PARAMETERS
 -------------------
-None.
+tz
+    The name of timezone to set.
 
 
 OPTIONAL PARAMETERS
@@ -27,19 +28,24 @@ EXAMPLES
 
 .. code-block:: sh
 
-    #Set up Europe/Andorra as our timezone.
-    __timezone Europe/Andorra
+    # Set up Europe/Andorra as our timezone.
+    __timezone --tz Europe/Andorra
 
-    #Set up US/Central as our timezone.
-    __timezone US/Central
+    # Set up US/Central as our timezone.
+    __timezone --tz US/Central
 
 
 AUTHORS
 -------
-Ramon Salvadó <rsalvado--@--gnuine--dot--com>
+| Steven Armstrong <steven-cdist--@--armstrong.cc>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Ramon Salvadó <rsalvado--@--gnuine--dot--com>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Free use of this software is
-granted under the terms of the GNU General Public License version 3 (GPLv3).
+Copyright \(C) 2012-2020 the `AUTHORS`_. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__timezone/manifest b/cdist/conf/type/__timezone/manifest
index 3d28ccba..0eb7fb9c 100755
--- a/cdist/conf/type/__timezone/manifest
+++ b/cdist/conf/type/__timezone/manifest
@@ -22,7 +22,7 @@
 #
 # This type allows to configure the desired localtime timezone.
 
-timezone="$__object_id"
+timezone=$(cat "$__object/parameter/tz")
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
diff --git a/cdist/conf/type/__timezone/parameter/required b/cdist/conf/type/__timezone/parameter/required
new file mode 100644
index 00000000..975445e4
--- /dev/null
+++ b/cdist/conf/type/__timezone/parameter/required
@@ -0,0 +1 @@
+tz
diff --git a/cdist/conf/type/__timezone/singleton b/cdist/conf/type/__timezone/singleton
new file mode 100644
index 00000000..e69de29b

From 627d215b637f1893e48a4a04c7e60686b960a698 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 27 Jul 2020 13:09:53 +0200
Subject: [PATCH 050/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8d26e767..e3711be2 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -16,6 +16,7 @@ next:
 	* Type __package_apt: Fix for legacy APT versions that do not support --no-install-recommends (Dennis Camera)
 	* Type __key_value: Get awk from POSIX PATH (Dennis Camera)
 	* New type: __unpack (Ander Punnar)
+	* Type __locale_system: Support more OSes (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 5dfc996febd085e4592fe590c4837bd4802ef4c3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 22:53:34 +0200
Subject: [PATCH 051/411] Fix global explorers for NetBSD

On NetBSD sysctl is at /sbin/sysctl, but the default PATH does not
contain /sbin.
---
 cdist/conf/explorer/cpu_cores | 1 +
 cdist/conf/explorer/disks     | 5 ++---
 cdist/conf/explorer/memory    | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/explorer/cpu_cores b/cdist/conf/explorer/cpu_cores
index c6744142..81e5294e 100755
--- a/cdist/conf/explorer/cpu_cores
+++ b/cdist/conf/explorer/cpu_cores
@@ -33,6 +33,7 @@ case "$os" in
     ;;
 
     "freebsd"|"netbsd")
+        PATH=$(getconf PATH)
         sysctl -n hw.ncpu
     ;;
 
diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks
index 24540601..56d62d10 100755
--- a/cdist/conf/explorer/disks
+++ b/cdist/conf/explorer/disks
@@ -30,9 +30,8 @@ case $uname_s in
         sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+'
     ;;
     NetBSD)
-        PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin"
-        sysctl -n hw.disknames \
-        | awk 'BEGIN { RS = " " } /^[lsw]d[0-9]+/'
+        PATH=$(getconf PATH)
+        sysctl -n hw.disknames | awk -v RS=' ' '/^[lsw]d[0-9]+/'
     ;;
     Linux)
         # list of major device numbers toexclude:
diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory
index 302b4cda..5ea15ada 100755
--- a/cdist/conf/explorer/memory
+++ b/cdist/conf/explorer/memory
@@ -30,6 +30,7 @@ case "$os" in
     ;;
 
     *"bsd")
+        PATH=$(getconf PATH)
         echo "$(sysctl -n hw.physmem) / 1048576" | bc
     ;;
 

From 3a87a447d00ef12eb6ace7c240926568aec2a042 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 23:37:48 +0200
Subject: [PATCH 052/411] [type/__sysctl] Fix on NetBSD

---
 cdist/conf/type/__sysctl/explorer/value | 9 +++++++--
 cdist/conf/type/__sysctl/gencode-remote | 2 ++
 cdist/conf/type/__sysctl/man.rst        | 7 +++++++
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__sysctl/explorer/value b/cdist/conf/type/__sysctl/explorer/value
index fc85b3d8..3e93c151 100755
--- a/cdist/conf/type/__sysctl/explorer/value
+++ b/cdist/conf/type/__sysctl/explorer/value
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -18,5 +18,10 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+if test "$(uname -s)" = NetBSD
+then
+	PATH=$(getconf PATH)
+fi
+
 # get the current runtime value
-sysctl -n "$__object_id" || true
+sysctl -n "${__object_id}" || true
diff --git a/cdist/conf/type/__sysctl/gencode-remote b/cdist/conf/type/__sysctl/gencode-remote
index 711d54e5..f0f6deef 100755
--- a/cdist/conf/type/__sysctl/gencode-remote
+++ b/cdist/conf/type/__sysctl/gencode-remote
@@ -44,6 +44,8 @@ case "$os" in
       flag='-w'
       ;;
    netbsd)
+      # shellcheck disable=SC2016
+      echo 'PATH=$(getconf PATH)'
       flag='-w'
       ;;
    freebsd|openbsd)
diff --git a/cdist/conf/type/__sysctl/man.rst b/cdist/conf/type/__sysctl/man.rst
index 6873003e..dbb9a1ac 100644
--- a/cdist/conf/type/__sysctl/man.rst
+++ b/cdist/conf/type/__sysctl/man.rst
@@ -26,6 +26,13 @@ EXAMPLES
 
     __sysctl net.ipv4.ip_forward --value 1
 
+    # On some operating systems, e.g. NetBSD, to prevent an error if the
+    # MIB style name does not exist (e.g. optional kernel components),
+    # name and value can be separated by `?=`. The same effect can be achieved
+    # in cdist by appending a `?` to the key:
+
+    __sysctl ddb.onpanic? --value -1
+
 
 AUTHORS
 -------

From 76bb214b5300707426030649cf03f35808e43256 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 27 Jul 2020 15:31:38 +0200
Subject: [PATCH 053/411] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index e3711be2..8fef4653 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -17,6 +17,8 @@ next:
 	* Type __key_value: Get awk from POSIX PATH (Dennis Camera)
 	* New type: __unpack (Ander Punnar)
 	* Type __locale_system: Support more OSes (Dennis Camera)
+	* Explorers cpu_cores, disks, memory: Fix for NetBSD (Dennis Camera)
+	* Type __sysctl: Fix for NetBSD (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From f5b367dfdbf60e75cfb2be038a05b6f226a1c010 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 28 Jul 2020 07:14:26 +0200
Subject: [PATCH 054/411] Release 6.7.0

---
 docs/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/changelog b/docs/changelog
index 8fef4653..269a2049 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,8 @@
 Changelog
 ---------
 
-next:
+6.7.0: 2020-07-28
+	* Delete deprecated type: __pf_apply (Darko Poljak)
 	* New type: __download (Ander Punnar)
 	* Type __locale_system: Add devuan support (Dennis Camera)
 	* Type __package_opkg: Add locking (Dennis Camera)
@@ -12,7 +13,6 @@ next:
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
 	* Type __user: Install user packages on OpenWrt (Dennis Camera)
 	* Type __openldap_server: Add Alpine support (Timothée Floure)
-	* Type __pf_apply: Remove deprecated type (Darko Poljak)
 	* Type __package_apt: Fix for legacy APT versions that do not support --no-install-recommends (Dennis Camera)
 	* Type __key_value: Get awk from POSIX PATH (Dennis Camera)
 	* New type: __unpack (Ander Punnar)

From c053a2c4a000b0c15df186ef2e067ea68ec9499c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 29 Jul 2020 11:31:12 +0200
Subject: [PATCH 055/411] Fix building man pages

Resolves #830.
---
 Makefile                                  | 4 ++--
 cdist/conf/type/__openldap_server/man.rst | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index f89ac1e7..3712511c 100644
--- a/Makefile
+++ b/Makefile
@@ -81,7 +81,7 @@ version:
 	}
 
 # Manpages #3: generic part
-man: version $(MANTYPES) $(DOCSREF)
+man: version configskel $(MANTYPES) $(DOCSREF) $(DOCSTYPESREF)
 	$(SPHINXM)
 
 html: version configskel $(MANTYPES) $(DOCSREF) $(DOCSTYPESREF)
@@ -104,7 +104,7 @@ DOTMANTYPES=$(subst /man.rst,.rst,$(DOTMANTYPEPREFIX))
 $(DOTMAN7DSTDIR)/cdist-type%.rst: $(DOTTYPEDIR)/%/man.rst
 	ln -sf "$^" $@
 
-dotman: version $(DOTMANTYPES)
+dotman: version configskel $(DOTMANTYPES) $(DOCSREF) $(DOCSTYPESREF)
 	$(SPHINXM)
 
 ################################################################################
diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
index a96c7dad..fa714ec0 100644
--- a/cdist/conf/type/__openldap_server/man.rst
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -31,8 +31,8 @@ manager-password-hash
     Generate e.g. with: `slappasswd -s weneedgoodsecurity`.
     See `slappasswd(8C)`, `slapd.conf(5)`.
     TODO: implement this: http://blog.adamsbros.org/2015/06/09/openldap-ssha-salted-hashes-by-hand/
-      to derive from the manager-password parameter and ensure idempotency (care with salts).
-      At that point, manager-password-hash should be deprecated and ignored.
+    to derive from the manager-password parameter and ensure idempotency (care with salts).
+    At that point, manager-password-hash should be deprecated and ignored.
 
 serverid
     The server for the directory.

From d37d2dc307d066f4d5f1bbbb93e0480965dfe3b8 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 2 Aug 2020 13:53:38 +0300
Subject: [PATCH 056/411] [__unpack] add parameter --tar-extra-args

---
 cdist/conf/type/__unpack/gencode-remote     |  7 +++++++
 cdist/conf/type/__unpack/man.rst            | 11 +++++++++++
 cdist/conf/type/__unpack/parameter/optional |  1 +
 3 files changed, 19 insertions(+)

diff --git a/cdist/conf/type/__unpack/gencode-remote b/cdist/conf/type/__unpack/gencode-remote
index 45c7173a..3b7a19a7 100755
--- a/cdist/conf/type/__unpack/gencode-remote
+++ b/cdist/conf/type/__unpack/gencode-remote
@@ -23,6 +23,13 @@ case "$src" in
 
             cmd="$cmd --strip-components=$tar_strip"
         fi
+
+        if [ -f "$__object/parameter/tar-extra-args" ]
+        then
+            tar_extra_args="$( cat "$__object/parameter/tar-extra-args" )"
+
+            cmd="$cmd $tar_extra_args"
+        fi
     ;;
     *.7z)
         case "$os" in
diff --git a/cdist/conf/type/__unpack/man.rst b/cdist/conf/type/__unpack/man.rst
index 8fe96e43..bd0603cf 100644
--- a/cdist/conf/type/__unpack/man.rst
+++ b/cdist/conf/type/__unpack/man.rst
@@ -33,6 +33,10 @@ sum-file
 tar-strip
     Tarball specific. See ``man tar`` for ``--strip-components``.
 
+tar-extra-args
+    Tarball sepcific. Append additional arguments to ``tar`` command.
+    See ``man tar`` for possible arguments.
+
 
 OPTIONAL BOOLEAN PARAMETERS
 ---------------------------
@@ -65,6 +69,13 @@ EXAMPLES
             --preserve-archive \
             --destination /opt/cpma/server
 
+    # example usecase for --tar-* args
+    __unpack /root/strelaysrv.tar.gz \
+        --preserve-archive \
+        --destination /usr/local/bin \
+        --tar-strip 1 \
+        --tar-extra-args '--wildcards "*/strelaysrv"'
+
 
 AUTHORS
 -------
diff --git a/cdist/conf/type/__unpack/parameter/optional b/cdist/conf/type/__unpack/parameter/optional
index d136dd0c..da26ca63 100644
--- a/cdist/conf/type/__unpack/parameter/optional
+++ b/cdist/conf/type/__unpack/parameter/optional
@@ -1,2 +1,3 @@
 sum-file
 tar-strip
+tar-extra-args

From 935f2395bc8f02d373f06d1e22acad813403ead2 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 2 Aug 2020 13:54:30 +0300
Subject: [PATCH 057/411] [__locale_system] fix for debian and ubuntu

ubuntu 6.10 and debian etch are 10+ years old and EOL. rather than
preserving compatibility I'll just remove it. while /etc/environment
works too, correct place is /etc/default/locale (as it was before
breaking change). also /etc/debian_version (os_version explorer) may
contain minor version with dot (10.5) or string (bullseye/sid).
---
 cdist/conf/type/__locale_system/manifest | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index e4286ef6..f7d75387 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -40,27 +40,13 @@ os=$(cat "$__global/explorer/os")
 case $os
 in
     debian)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 4 >/dev/null
-        then
-            # Debian 4 (etch) and later
-            locale_conf="/etc/default/locale"
-        else
-            locale_conf="/etc/environment"
-        fi
+        locale_conf="/etc/default/locale"
         ;;
     devuan)
         locale_conf="/etc/default/locale"
         ;;
     ubuntu)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 6.10 >/dev/null
-        then
-            # Ubuntu 6.10 (edgy) and later
-            locale_conf="/etc/default/locale"
-        else
-            locale_conf="/etc/environment"
-        fi
+        locale_conf="/etc/default/locale"
         ;;
     archlinux)
         locale_conf="/etc/locale.conf"

From 885d5a58f40e5adfd6aa616712e4c7246eca7880 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 16:49:01 +0200
Subject: [PATCH 058/411] [type/__locale_system] Fix floating point version
 comparison

---
 cdist/conf/type/__locale_system/manifest | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index e4286ef6..521f1007 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -24,6 +24,9 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
+bccmp() { test "$(bc)" -gt 0; }
+
+
 key=$__object_id
 onchange_cmd=  # none, by default
 quote_value=false
@@ -41,7 +44,7 @@ case $os
 in
     debian)
         os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 4 >/dev/null
+        if printf '%f >= 4\n' "${os_version}" | bccmp
         then
             # Debian 4 (etch) and later
             locale_conf="/etc/default/locale"
@@ -54,7 +57,7 @@ in
         ;;
     ubuntu)
         os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 6.10 >/dev/null
+        if printf '%f >= 6.10\n' "${os_version}" | bccmp
         then
             # Ubuntu 6.10 (edgy) and later
             locale_conf="/etc/default/locale"
@@ -68,7 +71,7 @@ in
     centos|redhat|scientific)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if expr "${version_id}" '>=' 7 >/dev/null
+        if printf '%f >= 7\n' "${version_id}" | bccmp
         then
             locale_conf="/etc/locale.conf"
         else
@@ -78,7 +81,7 @@ in
     fedora)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if expr "${version_id}" '>=' 18 >/dev/null
+        if printf '%f >= 18\n' "${version_id}" | bccmp
         then
             locale_conf="/etc/locale.conf"
             quote_value=false
@@ -113,7 +116,8 @@ in
         locale_conf="/etc/default/init"
         locale_conf_group="sys"
 
-        if expr "$(cat "${__global}/explorer/os_version")" '>=' 5.11 >/dev/null
+        os_version=$(cat "${__global}/explorer/os_version")
+        if printf '%f >= 5.11\n' "${os_version}" | bccmp
         then
             # mode on Oracle Solaris 11 is actually 0444,
             # but the write bit makes sense, IMO

From 71710fa00a86e3dcda6be82b959956cb88ba9902 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 20:17:00 +0200
Subject: [PATCH 059/411] [type/__locale_system] Implement "proper" version
 comparison

Proper in the sense that it can handle all numeric version numbers even if they
are not floating point (e.g. 16.04.6).
---
 cdist/conf/type/__locale_system/manifest | 28 ++++++++++++++++--------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 521f1007..88eadb31 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -24,7 +24,20 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
-bccmp() { test "$(bc)" -gt 0; }
+version_ge() {
+    awk -F '[^0-9.]' -v target="${1:?}" '
+    function max(x, y) { return x > y ? x : y }
+    BEGIN {
+        getline
+        nx = split($1, x, ".")
+        ny = split(target, y, ".")
+        for (i = 1; i <= max(nx, ny); ++i) {
+            diff = int(x[i]) - int(y[i])
+            if (diff == 0) continue
+            exit (diff < 0)
+        }
+    }'
+}
 
 
 key=$__object_id
@@ -43,8 +56,7 @@ os=$(cat "$__global/explorer/os")
 case $os
 in
     debian)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if printf '%f >= 4\n' "${os_version}" | bccmp
+        if version_ge 4 <"${__global}/explorer/os_version"
         then
             # Debian 4 (etch) and later
             locale_conf="/etc/default/locale"
@@ -56,8 +68,7 @@ in
         locale_conf="/etc/default/locale"
         ;;
     ubuntu)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if printf '%f >= 6.10\n' "${os_version}" | bccmp
+        if version_ge 6.10 <"${__global}/explorer/os_version"
         then
             # Ubuntu 6.10 (edgy) and later
             locale_conf="/etc/default/locale"
@@ -71,7 +82,7 @@ in
     centos|redhat|scientific)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if printf '%f >= 7\n' "${version_id}" | bccmp
+        if echo "${version_id}" | version_ge 7
         then
             locale_conf="/etc/locale.conf"
         else
@@ -81,7 +92,7 @@ in
     fedora)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if printf '%f >= 18\n' "${version_id}" | bccmp
+        if echo "${version_id}" | version_ge 18
         then
             locale_conf="/etc/locale.conf"
             quote_value=false
@@ -116,8 +127,7 @@ in
         locale_conf="/etc/default/init"
         locale_conf_group="sys"
 
-        os_version=$(cat "${__global}/explorer/os_version")
-        if printf '%f >= 5.11\n' "${os_version}" | bccmp
+        if version_ge 5.11 <"${__global}/explorer/os_version"
         then
             # mode on Oracle Solaris 11 is actually 0444,
             # but the write bit makes sense, IMO

From 7b480f4293b84a97850430a9436e28c66ccf5736 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 22:08:14 +0200
Subject: [PATCH 060/411] [type/__locale_system] Fix version extraction for
 SuSE

---
 cdist/conf/type/__locale_system/manifest | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 88eadb31..4b996ebc 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -163,7 +163,13 @@ in
         key="export ${__object_id}"
         ;;
     suse)
-        os_version=$(cat "${__global}/explorer/os_version")
+        if test -s "${__global}/explorer/os_release"
+        then
+            # shellcheck source=/dev/null
+            os_version=$(. "${__global}/explorer/os_release" && echo "${VERSION}")
+        else
+            os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global}/explorer/os_version")
+        fi
         os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
 
         # https://documentation.suse.com/sles/15-SP2/html/SLES-all/cha-suse.html#sec-suse-l10n

From b370b70ff4ccc1c6efca97f61f220a5ce0b191da Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 22:13:41 +0200
Subject: [PATCH 061/411] [explorer/os] Fix OS detection for openSUSE

All distros with ID_LIKE suse should be treated as "suse".
My openSUSE Leap 15.1 installation has:
ID_LIKE="suse opensuse"

This patch doesn't require a strict "suse" value but only the word suse to be in
the list.
---
 cdist/conf/explorer/os | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os
index 2d2aede6..8f54d9c8 100755
--- a/cdist/conf/explorer/os
+++ b/cdist/conf/explorer/os
@@ -144,7 +144,8 @@ esac
 
 if [ -f /etc/os-release ]; then
    # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse
-   if grep -q ^ID_LIKE=\"suse\" /etc/os-release 2>/dev/null; then
+   if (. /etc/os-release && echo "${ID_LIKE}" | grep -q '\(^\|\ \)suse\($\|\ \)')
+   then
       echo suse
       exit 0
    fi

From 17ab4bd80c9c9dda2fa4f8ccde5ee454f7b5ba2d Mon Sep 17 00:00:00 2001
From: Joachim Desroches <joachim.desroches@epfl.ch>
Date: Thu, 6 Aug 2020 11:45:05 +0200
Subject: [PATCH 062/411] Add Alpine Linux as supported for __filesystem.

---
 cdist/conf/type/__filesystem/explorer/lsblk | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk
index 9ae544ac..9be3c575 100644
--- a/cdist/conf/type/__filesystem/explorer/lsblk
+++ b/cdist/conf/type/__filesystem/explorer/lsblk
@@ -18,16 +18,16 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$("$__explorer/os")
+os=$("${__explorer:?}/os")
 
-if [ -f "$__object/parameter/device" ]; then
+if [ -f "${__object:?}/parameter/device" ]; then
     blkdev="$(cat "$__object/parameter/device")"
 else
-    blkdev="$__object_id"
+    blkdev="${__object_id:?}"
 fi
 
 case "$os" in
-    centos|fedora|redhat|suse|gentoo)
+    alpine|centos|fedora|redhat|suse|gentoo)
         if [ ! -x "$(command -v lsblk)" ]; then
             echo "lsblk is required for __filesystem type" >&2
             exit 1

From 74dd47c8c352cc3de3b5684bbe90520f61d95f37 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 15 Aug 2020 21:11:43 +0200
Subject: [PATCH 063/411] ++changelog

---
 docs/changelog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 269a2049..2c09bfea 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,12 @@
 Changelog
 ---------
 
+next:
+	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)
+	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)
+	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)
+	* Type __filesystem: Support Alpine Linux (Joachim Desroches)
+
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)
 	* New type: __download (Ander Punnar)

From 8f94a226c7f0b875987e25e57682414204380fa2 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 15 Aug 2020 21:54:07 +0200
Subject: [PATCH 064/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 2c09bfea..9e44cc4d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)
 	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)
 	* Type __filesystem: Support Alpine Linux (Joachim Desroches)
+	* Type __locale_system: Fix version comparison (Dennis Camera)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From 6fed1785298eb506e7ea6cd177efe7d96658769f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 17 Aug 2020 09:27:48 +0200
Subject: [PATCH 065/411] [explorer/os_version] Convert Debian sid to version
 number.

Conversion of Debian sid to versions is done based on Debian codenames.
The version number is the version number of the final release - 0.01.

It is unknown if Debian < 4.0 has any sort of version information
available (apart from maybe checking base-files package version).
But I don't think any of these systems are still alive,
so I think going with 3.99 is fine for those.
---
 cdist/conf/explorer/os_version | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 1d54ea60..a7b1d3bc 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -31,7 +31,32 @@ case "$("$__explorer/os")" in
       cat /etc/arch-release
    ;;
    debian)
-      cat /etc/debian_version
+      debian_version=$(cat /etc/debian_version)
+      case $debian_version
+      in
+          testing/unstable)
+              # previous to Debian 4.0 testing/unstable was used
+              # cf. https://metadata.ftp-master.debian.org/changelogs/main/b/base-files/base-files_11_changelog
+              echo 3.99
+              ;;
+          */sid)
+              # sid versions don't have a number, so we decode by codename:
+              case $(expr "$debian_version" : '\([a-z]\{1,\}\)/')
+              in
+                  bullseye) echo 10.99 ;;
+                  buster) echo 9.99 ;;
+                  stretch) echo 8.99 ;;
+                  jessie) echo 7.99 ;;
+                  wheezy) echo 6.99 ;;
+                  squeeze) echo 5.99 ;;
+                  lenny) echo 4.99 ;;
+                  *) exit 1
+              esac
+              ;;
+          *)
+              echo "$debian_version"
+              ;;
+      esac
    ;;
    devuan)
       cat /etc/devuan_version
@@ -73,4 +98,4 @@ case "$("$__explorer/os")" in
    alpine)
        cat /etc/alpine-release
    ;;
-esac
\ No newline at end of file
+esac

From 502d75304707704d3ba73374701394b98d259506 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 18 Aug 2020 00:46:07 +0300
Subject: [PATCH 066/411] [__unpack] add --onchange

---
 cdist/conf/type/__unpack/gencode-remote     | 5 +++++
 cdist/conf/type/__unpack/man.rst            | 3 +++
 cdist/conf/type/__unpack/parameter/optional | 1 +
 3 files changed, 9 insertions(+)

diff --git a/cdist/conf/type/__unpack/gencode-remote b/cdist/conf/type/__unpack/gencode-remote
index 3b7a19a7..c4451f73 100755
--- a/cdist/conf/type/__unpack/gencode-remote
+++ b/cdist/conf/type/__unpack/gencode-remote
@@ -80,3 +80,8 @@ if [ ! -f "$__object/parameter/preserve-archive" ]
 then
     echo "rm -f '$src'"
 fi
+
+if [ -f "$__object/parameter/onchange" ]
+then
+    cat "$__object/parameter/onchange"
+fi
diff --git a/cdist/conf/type/__unpack/man.rst b/cdist/conf/type/__unpack/man.rst
index bd0603cf..daa03814 100644
--- a/cdist/conf/type/__unpack/man.rst
+++ b/cdist/conf/type/__unpack/man.rst
@@ -50,6 +50,9 @@ backup-destination
 preserve-archive
     Don't delete archive after unpacking.
 
+onchange
+    Execute this command after unpack.
+
 
 EXAMPLES
 --------
diff --git a/cdist/conf/type/__unpack/parameter/optional b/cdist/conf/type/__unpack/parameter/optional
index da26ca63..d846ac75 100644
--- a/cdist/conf/type/__unpack/parameter/optional
+++ b/cdist/conf/type/__unpack/parameter/optional
@@ -1,3 +1,4 @@
 sum-file
 tar-strip
 tar-extra-args
+onchange

From d239169c4f91a4ce33be170ed29b08299c51234f Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 18 Aug 2020 00:48:58 +0300
Subject: [PATCH 067/411] [__download] fix manual: onchange parameter in wrong
 section

---
 cdist/conf/type/__download/man.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index 6ec0b19a..eb3ac971 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -30,9 +30,6 @@ sum
    By default output of ``cksum`` without filename is expected.
    Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
 
-onchange
-   Execute this command after download.
-
 
 OPTIONAL PARAMETERS
 -------------------
@@ -54,6 +51,9 @@ cmd-sum
    format specification ``%s`` which will become destination.
    For example: ``md5sum '%s' | awk '{print $1}'``.
 
+onchange
+   Execute this command after download.
+
 
 EXAMPLES
 --------

From ba26a437be4d8034037e7a6eb42382aee01f829e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 18 Aug 2020 11:06:19 +0200
Subject: [PATCH 068/411] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 9e44cc4d..a3e4167a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,9 @@ next:
 	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)
 	* Type __filesystem: Support Alpine Linux (Joachim Desroches)
 	* Type __locale_system: Fix version comparison (Dennis Camera)
+	* Type __unpack: Add --onchange parameter (Ander Punnar)
+	* Type __download: Fix manual (Ander Punnar)
+	* Explorer os_version: Convert Debian sid to version number (Dennis Camera)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From c17541f24cf118b0989a5899cab3a439ed534a46 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 22 Aug 2020 19:53:42 +0200
Subject: [PATCH 069/411] Expand and split by consecutive require delimiters

Resolves #832.
---
 cdist/emulator.py               |  6 +++++-
 cdist/test/emulator/__init__.py | 10 ++++++++++
 docs/src/cdist-manifest.rst     |  3 ++-
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/cdist/emulator.py b/cdist/emulator.py
index 9fe84056..60b94880 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -25,6 +25,7 @@ import argparse
 import logging
 import os
 import sys
+import re
 
 import cdist
 from cdist import core
@@ -389,12 +390,15 @@ class Emulator:
         if "require" in self.env:
             requirements = self.env['require']
             self.log.debug("reqs = " + requirements)
-            for requirement in requirements.split(" "):
+            for requirement in self._parse_require(requirements):
                 # Ignore empty fields - probably the only field anyway
                 if len(requirement) == 0:
                     continue
                 self.record_requirement(requirement)
 
+    def _parse_require(self, require):
+        return re.split(r'[ \t\n]+', require)
+
     def record_auto_requirements(self):
         """An object shall automatically depend on all objects that it
            defined in it's type manifest.
diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py
index e375676c..befd7b57 100644
--- a/cdist/test/emulator/__init__.py
+++ b/cdist/test/emulator/__init__.py
@@ -685,6 +685,16 @@ class EmulatorAlreadyExistingRequirementsWarnTestCase(test.CdistTestCase):
         self.env['require'] = '__directory/spam'
         emu = emulator.Emulator(argv, env=self.env)
 
+    def test_parse_require(self):
+        require = " \t \n  \t\t\n\t\na\tb\nc d \te\t\nf\ng\t "
+        expected = ['', 'a', 'b', 'c', 'd', 'e', 'f', 'g', '', ]
+
+        argv = ['__directory', 'spam']
+        emu = emulator.Emulator(argv, env=self.env)
+        requirements = emu._parse_require(require)
+
+        self.assertEqual(expected, requirements)
+
 
 if __name__ == '__main__':
     import unittest
diff --git a/docs/src/cdist-manifest.rst b/docs/src/cdist-manifest.rst
index 5dbca479..2e49a721 100644
--- a/docs/src/cdist-manifest.rst
+++ b/docs/src/cdist-manifest.rst
@@ -95,7 +95,8 @@ Dependencies
 ------------
 If you want to describe that something requires something else, just
 setup the variable "require" to contain the requirements. Multiple
-requirements can be added white space separated.
+requirements can be added separated with (optionally consecutive)
+delimiters including space, tab and newline.
 
 ::
 

From b5a40eb0d1c8990f0feb8e918a95e54e4fd35d98 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 27 Aug 2020 12:25:11 +0200
Subject: [PATCH 070/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index a3e4167a..eda77a12 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,6 +10,7 @@ next:
 	* Type __unpack: Add --onchange parameter (Ander Punnar)
 	* Type __download: Fix manual (Ander Punnar)
 	* Explorer os_version: Convert Debian sid to version number (Dennis Camera)
+	* Core: Expand require delimiter characters, split by consecutive delimiters (Darko Poljak)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From b1375464cc7507b38dd02f2df52c75489837d129 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 6 Sep 2020 13:24:58 +0200
Subject: [PATCH 071/411] __systemd_service: fix manpage typos

---
 cdist/conf/type/__systemd_service/man.rst | 25 +++++++++++++++--------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__systemd_service/man.rst b/cdist/conf/type/__systemd_service/man.rst
index 7eca398b..cd14c985 100644
--- a/cdist/conf/type/__systemd_service/man.rst
+++ b/cdist/conf/type/__systemd_service/man.rst
@@ -1,9 +1,10 @@
-cdist-type__systemd-service(7)
+cdist-type__systemd_service(7)
 ==============================
 
 NAME
 ----
-cdist-type__systemd-service - Controls a systemd service state
+cdist-type__systemd_service - Controls a systemd service state
+
 
 DESCRIPTION
 -----------
@@ -14,11 +15,12 @@ service after configuration applied or shutdown one service.
 The activation or deactivation is out of scope. Look for the
 :strong:`cdist-type__systemd_util`\ (7) type instead.
 
+
 REQUIRED PARAMETERS
 -------------------
-
 None.
 
+
 OPTIONAL PARAMETERS
 -------------------
 
@@ -31,12 +33,12 @@ state
     running
         Service should run (default)
 
-    stoppend
-        Service should stopped
+    stopped
+        Service should be stopped
 
 action
     Executes an action on on the service. It will only execute it if the
-    service keeps the state **running**. There are following actions, where:
+    service keeps the state ``running``. There are following actions, where:
 
     reload
         Reloads the service
@@ -48,11 +50,12 @@ BOOLEAN PARAMETERS
 ------------------
 
 if-required
-    Only execute the action if minimum one required type outputs a message to
-    **$__messages_out**. Through this, the action should only executed if a
+    Only execute the action if at minimum one required type outputs a message
+    to ``$__messages_out``. Through this, the action should only executed if a
     dependency did something. The action will not executed if no dependencies
     given.
 
+
 MESSAGES
 --------
 
@@ -68,12 +71,14 @@ restart
 reload
     Reloaded the service
 
+
 ABORTS
 ------
 Aborts in following cases:
 
 systemd or the service does not exist
 
+
 EXAMPLES
 --------
 .. code-block:: sh
@@ -95,13 +100,15 @@ EXAMPLES
 
     # reload the service for a modified configuration file
     # only reloads the service if the file really changed
-    require="__config_file/etc/foo.conf" __systemd_service foo \
+    require="__file/etc/foo.conf" __systemd_service foo \
         --action reload --if-required
 
+
 AUTHORS
 -------
 Matthias Stecher <matthiasstecher at gmx.de>
 
+
 COPYRIGHT
 ---------
 Copyright \(C) 2020 Matthias Stecher. You can redistribute it

From 6b262a61c19234f6450cef11f715020dcd5f40d5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 10 Sep 2020 13:24:58 +0200
Subject: [PATCH 072/411] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index eda77a12..e4ec3fdd 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,8 @@ next:
 	* Type __download: Fix manual (Ander Punnar)
 	* Explorer os_version: Convert Debian sid to version number (Dennis Camera)
 	* Core: Expand require delimiter characters, split by consecutive delimiters (Darko Poljak)
+	* Type __timezone: Make singleton (Dennis Camera)
+	* Type __systemd_service: Fix manpage typos (Matthias Stecher)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From 53b91adbd8be4d858f1f4da3389b63babc85db0c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 11 Sep 2020 14:19:45 +0200
Subject: [PATCH 073/411] Fix shellcheck

---
 cdist/conf/explorer/os | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os
index 8f54d9c8..46d87f3e 100755
--- a/cdist/conf/explorer/os
+++ b/cdist/conf/explorer/os
@@ -144,6 +144,7 @@ esac
 
 if [ -f /etc/os-release ]; then
    # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse
+   # shellcheck disable=SC1091
    if (. /etc/os-release && echo "${ID_LIKE}" | grep -q '\(^\|\ \)suse\($\|\ \)')
    then
       echo suse

From 2885c6a24838956167ae6c33ebf87067e8790083 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 11 Sep 2020 14:15:26 +0200
Subject: [PATCH 074/411] Release 6.8.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index e4ec3fdd..7a0d050d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)
 	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)
 	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)

From decc0ad54dffbafcd7d3644c28ae4d149926e612 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 18 Sep 2020 19:33:36 +0300
Subject: [PATCH 075/411] [__package_pip] detect pip binary

---
 cdist/conf/type/__package_pip/explorer/pip   | 10 ++++++++++
 cdist/conf/type/__package_pip/explorer/state |  2 +-
 cdist/conf/type/__package_pip/gencode-remote |  7 ++++++-
 3 files changed, 17 insertions(+), 2 deletions(-)
 create mode 100755 cdist/conf/type/__package_pip/explorer/pip

diff --git a/cdist/conf/type/__package_pip/explorer/pip b/cdist/conf/type/__package_pip/explorer/pip
new file mode 100755
index 00000000..cf9fae89
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/pip
@@ -0,0 +1,10 @@
+#!/bin/sh -e
+
+for bin in pip3 pip
+do
+    if check="$( command -v "$bin" )"
+    then
+        echo "$check"
+        break
+    fi
+done
diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state
index 5be07280..3cc98ab9 100644
--- a/cdist/conf/type/__package_pip/explorer/state
+++ b/cdist/conf/type/__package_pip/explorer/state
@@ -32,7 +32,7 @@ pipparam="$__object/parameter/pip"
 if [ -f "$pipparam" ]; then
     pip=$(cat "$pipparam")
 else
-    pip="pip"
+    pip="$( "$__type_explorer/pip" )"
 fi
 
 # If there is no pip, it may get created from somebody else.
diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index dcc4fdf9..9c5eb0d1 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -38,7 +38,12 @@ pipparam="$__object/parameter/pip"
 if [ -f "$pipparam" ]; then
     pip=$(cat "$pipparam")
 else
-    pip="pip"
+    pip="$( cat "$__object/explorer/pip" )"
+    if [ -z "$pip" ]
+    then
+        echo 'pip not found in path' >&2
+        exit 1
+    fi
 fi
 
 runasparam="$__object/parameter/runas"

From 89b621511561105cfcfcf12bbf820b74ab03396d Mon Sep 17 00:00:00 2001
From: Darko Poljak <darko.poljak@ungleich.ch>
Date: Mon, 21 Sep 2020 09:04:05 +0200
Subject: [PATCH 076/411] Clarify stdin input

Resolve #836.
---
 cdist/argparse.py                | 36 ++++++--------------------
 cdist/config.py                  |  6 +++--
 cdist/inventory.py               |  4 +--
 cdist/test/inventory/__init__.py |  3 +--
 docs/src/man1/cdist.rst          | 44 ++++++++++----------------------
 5 files changed, 29 insertions(+), 64 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 77303591..1d16bb25 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -273,8 +273,7 @@ def get_parsers():
             '-f', '--file',
             help=('Read specified file for a list of additional hosts to '
                   'operate on or if \'-\' is given, read stdin (one host per '
-                  'line). If no host or host file is specified then, by '
-                  'default, read hosts from stdin.'),
+                  'line).'),
             dest='hostfile', required=False)
     parser['config_args'].add_argument(
            '-p', '--parallel', nargs='?', metavar='HOST_MAX',
@@ -326,9 +325,7 @@ def get_parsers():
     parser['add-host'].add_argument(
            '-f', '--file',
            help=('Read additional hosts to add from specified file '
-                 'or from stdin if \'-\' (each host on separate line). '
-                 'If no host or host file is specified then, by default, '
-                 'read from stdin.'),
+                 'or from stdin if \'-\' (each host on separate line). '),
            dest='hostfile', required=False)
 
     parser['add-tag'] = parser['invsub'].add_parser(
@@ -342,20 +339,12 @@ def get_parsers():
     parser['add-tag'].add_argument(
            '-f', '--file',
            help=('Read additional hosts to add tags from specified file '
-                 'or from stdin if \'-\' (each host on separate line). '
-                 'If no host or host file is specified then, by default, '
-                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                 ' are specified then tags are read from stdin and are'
-                 ' added to all hosts.'),
+                 'or from stdin if \'-\' (each host on separate line). '),
            dest='hostfile', required=False)
     parser['add-tag'].add_argument(
            '-T', '--tag-file',
            help=('Read additional tags to add from specified file '
-                 'or from stdin if \'-\' (each tag on separate line). '
-                 'If no tag or tag file is specified then, by default, '
-                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                 ' are specified then tags are read from stdin and are'
-                 ' added to all hosts.'),
+                 'or from stdin if \'-\' (each tag on separate line). '),
            dest='tagfile', required=False)
     parser['add-tag'].add_argument(
            '-t', '--taglist',
@@ -376,9 +365,7 @@ def get_parsers():
     parser['del-host'].add_argument(
             '-f', '--file',
             help=('Read additional hosts to delete from specified file '
-                  'or from stdin if \'-\' (each host on separate line). '
-                  'If no host or host file is specified then, by default, '
-                  'read from stdin.'),
+                  'or from stdin if \'-\' (each host on separate line). '),
             dest='hostfile', required=False)
 
     parser['del-tag'] = parser['invsub'].add_parser(
@@ -396,20 +383,13 @@ def get_parsers():
     parser['del-tag'].add_argument(
             '-f', '--file',
             help=('Read additional hosts to delete tags for from specified '
-                  'file or from stdin if \'-\' (each host on separate line). '
-                  'If no host or host file is specified then, by default, '
-                  'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                  ' are specified then tags are read from stdin and are'
-                  ' deleted from all hosts.'),
+                  'file or from stdin if \'-\' (each host on separate '
+                  'line). '),
             dest='hostfile', required=False)
     parser['del-tag'].add_argument(
             '-T', '--tag-file',
             help=('Read additional tags from specified file '
-                  'or from stdin if \'-\' (each tag on separate line). '
-                  'If no tag or tag file is specified then, by default, '
-                  'read from stdin. If no tags/tagfile nor'
-                  ' hosts/hostfile are specified then tags are read from'
-                  ' stdin and are added to all hosts.'),
+                  'or from stdin if \'-\' (each tag on separate line). '),
             dest='tagfile', required=False)
     parser['del-tag'].add_argument(
             '-t', '--taglist',
diff --git a/cdist/config.py b/cdist/config.py
index 30416008..e84f6f84 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -175,9 +175,11 @@ class Config:
             raise cdist.Error(("Cannot read both, manifest and host file, "
                                "from stdin"))
 
-        # if no host source is specified then read hosts from stdin
         if not (args.hostfile or args.host):
-            args.hostfile = '-'
+            if args.tag or args.all_tagged_hosts:
+                raise cdist.Error(("Target host tag(s) missing"))
+            else:
+                raise cdist.Error(("Target host(s) missing"))
 
         if args.manifest == '-':
             # read initial manifest from stdin
diff --git a/cdist/inventory.py b/cdist/inventory.py
index 6ab20fa7..0387f326 100644
--- a/cdist/inventory.py
+++ b/cdist/inventory.py
@@ -299,7 +299,7 @@ class InventoryHost(Inventory):
         self.all = all
 
         if not self.hosts and not self.hostfile:
-            self.hostfile = "-"
+            raise cdist.Error("Host(s) missing")
 
     def _new_hostpath(self, hostpath):
         # create empty file
@@ -355,7 +355,7 @@ class InventoryTag(Inventory):
         else:
             self.allhosts = False
         if not self.tags and not self.tagfile:
-            self.tagfile = "-"
+            raise cdist.Error("Tag(s) missing")
 
         if self.hostfile == "-" and self.tagfile == "-":
             raise cdist.Error("Cannot read both, hosts and tags, from stdin")
diff --git a/cdist/test/inventory/__init__.py b/cdist/test/inventory/__init__.py
index 287c855c..a8cd8bf8 100644
--- a/cdist/test/inventory/__init__.py
+++ b/cdist/test/inventory/__init__.py
@@ -307,11 +307,10 @@ class InventoryTestCase(test.CdistTestCase):
             raise e
 
     # InventoryTag
+    @unittest.expectedFailure
     def test_inventory_tag_init(self):
         invTag = inventory.InventoryTag(db_basedir=inventory_dir,
                                         action="add")
-        self.assertTrue(invTag.allhosts)
-        self.assertEqual(invTag.tagfile, "-")
 
     def test_inventory_tag_stdin_multiple_hosts(self):
         try:
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index aa2607f8..0ecb4a61 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -177,10 +177,8 @@ Install command is currently in beta.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read specified file for a list of additional hosts to operate on
-    or if '-' is given, read stdin (one host per line).
-    If no host or host file is specified then, by default,
-    read hosts from stdin. For the file format see
-    :strong:`HOSTFILE FORMAT` below.
+    or if '-' is given, read stdin (one host per line). For the file
+    format see :strong:`HOSTFILE FORMAT` below.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -299,9 +297,8 @@ Add host(s) to inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to add from specified file or
-    from stdin if '-' (each host on separate line). If no
-    host or host file is specified then, by default, read
-    from stdin. Hostfile format is the same as config hostfile format.
+    from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -327,11 +324,8 @@ Add tag(s) to inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to add tags from specified file
-    or from stdin if '-' (each host on separate line). If
-    no host or host file is specified then, by default,
-    read from stdin. If no tags/tagfile nor hosts/hostfile
-    are specified then tags are read from stdin and are
-    added to all hosts. Hostfile format is the same as config hostfile format.
+    or from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -346,11 +340,8 @@ Add tag(s) to inventory database.
 
 **-T TAGFILE, --tag-file TAGFILE**
     Read additional tags to add from specified file or
-    from stdin if '-' (each tag on separate line). If no
-    tag or tag file is specified then, by default, read
-    from stdin. If no tags/tagfile nor hosts/hostfile are
-    specified then tags are read from stdin and are added
-    to all hosts. Tagfile format is the same as config hostfile format.
+    from stdin if '-' (each tag on separate line).
+    Tagfile format is the same as config hostfile format.
 
 **-t TAGLIST, --taglist TAGLIST**
     Tag list to be added for specified host(s), comma
@@ -372,9 +363,8 @@ Delete host(s) from inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to delete from specified file or
-    from stdin if '-' (each host on separate line). If no
-    host or host file is specified then, by default, read
-    from stdin. Hostfile format is the same as config hostfile format.
+    from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -404,11 +394,8 @@ Delete tag(s) from inventory database.
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to delete tags for from
     specified file or from stdin if '-' (each host on
-    separate line). If no host or host file is specified
-    then, by default, read from stdin. If no tags/tagfile
-    nor hosts/hostfile are specified then tags are read
-    from stdin and are deleted from all hosts. Hostfile
-    format is the same as config hostfile format.
+    separate line). Hostfile format is the same as
+    config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -423,11 +410,8 @@ Delete tag(s) from inventory database.
 
 **-T TAGFILE, --tag-file TAGFILE**
     Read additional tags from specified file or from stdin
-    if '-' (each tag on separate line). If no tag or tag
-    file is specified then, by default, read from stdin.
-    If no tags/tagfile nor hosts/hostfile are specified
-    then tags are read from stdin and are added to all
-    hosts. Tagfile format is the same as config hostfile format.
+    if '-' (each tag on separate line).
+    Tagfile format is the same as config hostfile format.
 
 **-t TAGLIST, --taglist TAGLIST**
     Tag list to be deleted for specified host(s), comma

From 89a0080e133052b82ce19e7fdbaf045dcc833da7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:09:26 +0200
Subject: [PATCH 077/411] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 7a0d050d..260b4fe4 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* Core: Clarify stdin input (Darko Poljak)
+	* Type __package_pip: Detect pip binary (Ander Punnar)
+
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)
 	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)

From 0fc10749edc698d1f70f826603d009fba943d252 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:11:35 +0200
Subject: [PATCH 078/411] Fix shellcheck

---
 cdist/conf/type/__package_pip/gencode-remote | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 9c5eb0d1..a1375c2d 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -60,7 +60,7 @@ case "$state_should" in
         then
             echo "su -c '$pip install -q $name' $runas"
         else
-            echo $pip install -q "$name"
+            echo "$pip" install -q "$name"
         fi
         echo "installed" >> "$__messages_out"
     ;;
@@ -69,7 +69,7 @@ case "$state_should" in
         then
             echo "su -c '$pip uninstall -q -y $name' $runas"
         else
-            echo $pip uninstall -q -y "$name"
+            echo "$pip" uninstall -q -y "$name"
         fi
         echo "removed" >> "$__messages_out"
     ;;

From b6922508b9e07834009a6bfae2d3933b5ea62fbf Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:17:34 +0200
Subject: [PATCH 079/411] Update helper script

---
 bin/build-helper | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/build-helper b/bin/build-helper
index ed41e438..d4d603ed 100755
--- a/bin/build-helper
+++ b/bin/build-helper
@@ -371,7 +371,6 @@ eof
 Manual steps post release:
     - cdist-web
     - send generated mailinglist.tmp mail
-    - twitter
 eof
     ;;
 

From 84a7818121047cf6bb23d8bd3c055d09bb033d32 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 23 Sep 2020 20:29:47 +0200
Subject: [PATCH 080/411] docs: make varaibles environment-aware

There are all overwriting the environment, even the comment states
otherwise. Fixes it.
---
 docs/src/Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/src/Makefile b/docs/src/Makefile
index 2ecf7a32..ba87d170 100644
--- a/docs/src/Makefile
+++ b/docs/src/Makefile
@@ -2,10 +2,10 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-PAPER         =
-BUILDDIR      = ../dist
+SPHINXOPTS   ?=
+SPHINXBUILD  ?= sphinx-build
+PAPER        ?=
+BUILDDIR     ?= ../dist
 # for cache, etc.
 _BUILDDIR     = _build
 

From 73d6c9d469d1429999ea2e980b3ac4c63a3e0dd4 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 27 Sep 2020 10:17:35 +0200
Subject: [PATCH 081/411] Add custom remote copy/exec examples

---
 docs/src/cdist-remote-exec-copy.rst | 389 +++++++++++++++++++++++++++-
 1 file changed, 388 insertions(+), 1 deletion(-)

diff --git a/docs/src/cdist-remote-exec-copy.rst b/docs/src/cdist-remote-exec-copy.rst
index bb818310..e7b7b226 100644
--- a/docs/src/cdist-remote-exec-copy.rst
+++ b/docs/src/cdist-remote-exec-copy.rst
@@ -10,7 +10,7 @@ By default this is accomplished with ssh and scp respectively.
 The default implementations used by cdist are::
 
     __remote_exec: ssh -o User=root
-    __remote_copy: scp -o User=root
+    __remote_copy: scp -o User=root -q
 
 The user can override these defaults by providing custom implementations and
 passing them to cdist with the --remote-exec and/or --remote-copy arguments.
@@ -26,3 +26,390 @@ specified by enclosed in square brackets (see :strong:`ssh`\ (1) and
 With this simple interface the user can take total control of how cdist
 interacts with the target when required, while the default implementation 
 remains as simple as possible.
+
+
+Examples
+--------
+
+Here are examples of using alternative __remote_copy and __remote_exec scripts.
+
+All scripts from below are present in cdist sources in `other/examples/remote`
+directory.
+
+ssh
+~~~
+
+Same as cdist default.
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    #echo "$@" | logger -t "cdist-ssh-copy"
+    scp -o User=root -q $@
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    #echo "$@" | logger -t "cdist-ssh-exec"
+    ssh -o User=root $@
+
+local
+~~~~~
+
+This effectively turns remote calling into local calling. Probably most useful
+for the unit testing.
+
+**copy**
+
+.. code-block:: sh
+
+    code="$(echo "$@" | sed "s|\([[:space:]]\)$__target_host:|\1|g")"
+    cp -L $code
+
+**exec**
+
+.. code-block:: sh
+
+    target_host=$1; shift
+    echo "$@" | /bin/sh 
+
+chroot
+~~~~~~
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script /path/to/your/chroot" target-id
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-chroot-copy"
+       :
+    }
+
+    chroot="$1"; shift
+    target_host="$__target_host"
+
+    # replace target_host with chroot location
+    code="$(echo "$@" | sed "s|$target_host:|$chroot|g")"
+
+    log "target_host: $target_host"
+    log "chroot: $chroot"
+    log "$@"
+    log "$code"
+
+    # copy files into chroot
+    cp $code
+
+    log "-----"
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script /path/to/your/chroot" target-id
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-chroot-exec"
+       :
+    }
+
+    chroot="$1"; shift
+    target_host="$1"; shift
+
+    script=$(mktemp "${chroot}/tmp/chroot-${0##*/}.XXXXXXXXXX")
+    trap cleanup INT TERM EXIT
+    cleanup() {
+       [ $__cdist_debug ] || rm "$script"
+    }
+
+    log "target_host: $target_host"
+    log "script: $script"
+    log "@: $@"
+    echo "#!/bin/sh -l" > "$script"
+    echo "$@" >> "$script"
+    chmod +x "$script"
+
+    relative_script="${script#$chroot}"
+    log "relative_script: $relative_script"
+
+    # run in chroot
+    chroot "$chroot" "$relative_script"
+
+    log "-----"
+
+rsync
+~~~~~
+
+**copy**
+
+Usage: cdist config --remote-copy /path/to/this/script target_host
+
+.. code-block:: sh
+
+    # For rsync to do the right thing, the source has to end with "/" if it is
+    # a directory. The below preprocessor loop takes care of that.
+
+    # second last argument is the source
+    source_index=$(($#-1))
+    index=0
+    for arg in $@; do
+       if [ $index -eq 0 ]; then
+          # reset $@
+          set --
+       fi
+       index=$((index+=1))
+       if [ $index -eq $source_index -a -d "$arg" ]; then
+          arg="${arg%/}/"
+       fi
+       set -- "$@" "$arg"
+    done
+
+    rsync --backup --suffix=~cdist -e 'ssh -o User=root' $@
+
+schroot
+~~~~~~~
+
+__remote_copy and __remote_exec scripts to run cdist against a chroot on the
+target host over ssh.
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script schroot-chroot-name" target_host
+
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-schroot-copy"
+       :
+    }
+
+    chroot_name="$1"; shift
+    target_host="$__target_host"
+
+    # get directory for given chroot_name
+    chroot="$(ssh -o User=root -q $target_host schroot -c $chroot_name --config | awk -F = '/directory=/ {print $2}')"
+
+    # prefix destination with chroot
+    code="$(echo "$@" | sed "s|$target_host:|$target_host:$chroot|g")"
+
+    log "target_host: $target_host"
+    log "chroot_name: $chroot_name"
+    log "chroot: $chroot"
+    log "@: $@"
+    log "code: $code"
+
+    # copy files into remote chroot
+    scp -o User=root -q $code
+
+    log "-----"
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script schroot-chroot-name" target_host
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-schroot-exec"
+       :
+    }
+
+    chroot_name="$1"; shift
+    target_host="$1"; shift
+
+    code="ssh -o User=root -q $target_host schroot -c $chroot_name -- $@"
+
+    log "target_host: $target_host"
+    log "chroot_name: $chroot_name"
+    log "@: $@"
+    log "code: $code"
+
+    # run in remote chroot
+    $code
+
+    log "-----"
+
+schroot-uri
+~~~~~~~~~~~
+
+__remote_exec/__remote_copy script to run cdist against a schroot target URI.
+
+Usage::
+
+    cdist config \
+        --remote-exec "/path/to/this/script exec" \
+        --remote-copy "/path/to/this/script copy" \
+        target_uri
+
+    # target_uri examples:
+    schroot:///chroot-name
+    schroot://foo.ethz.ch/chroot-name
+    schroot://user-name@foo.ethz.ch/chroot-name
+
+    # and how to match them in .../manifest/init
+    case "$target_host" in
+    schroot://*)
+        # any schroot
+    ;;
+    schroot://foo.ethz.ch/*)
+        # any schroot on specific host
+    ;;
+    schroot://foo.ethz.ch/chroot-name)
+        # specific schroot on specific host
+    ;;
+    schroot:///chroot-name)
+        # specific schroot on localhost
+    ;;
+    esac
+
+**copy/exec**
+
+.. code-block:: sh
+
+    my_name="${0##*/}"
+    mode="$1"; shift
+
+    log() {
+       # uncomment me for debugging
+       #echo "$@" | logger -t "cdist-$my_name-$mode"
+       :
+    }
+
+    die() {
+       echo "$@" >&2
+       exit 1
+    }
+
+
+    uri="$__target_host"
+
+    scheme="${uri%%:*}"; rest="${uri#$scheme:}"; rest="${rest#//}"
+    authority="${rest%%/*}"; rest="${rest#$authority}"
+    path="${rest%\?*}"; rest="${rest#$path}"
+    schroot_name="${path#/}"
+
+    [ "$scheme" = "schroot" ] || die "Failed to parse scheme from __target_host ($__target_host). Expected 'schroot', got '$scheme'"
+    [ -n "$schroot_name" ] || die "Failed to parse schroot name from __target_host: $__target_host"
+
+    case "$authority" in
+       '')
+          # authority is empty, neither user nor host given
+          user=""
+          host=""
+       ;; 
+       *@*)
+          # authority contains @, take user from authority
+          user="${authority%@*}"
+          host="${authority#*@}"
+       ;; 
+       *) 
+          # no user in authority, default to root
+          user="root"
+          host="$authority"
+       ;;
+    esac
+
+    log "mode: $mode"
+    log "@: $@"
+    log "uri: $uri"
+    log "scheme: $scheme"
+    log "authority: $authority"
+    log "user: $user"
+    log "host: $host"
+    log "path: $path"
+    log "schroot_name: $schroot_name"
+
+    exec_prefix=""
+    copy_prefix=""
+    if [ -n "$host" ]; then
+       # we are working on a remote host
+       exec_prefix="ssh -o User=$user -q $host"
+       copy_prefix="scp -o User=$user -q"
+       copy_destination_prefix="$host:"
+    else
+       # working on local machine
+       copy_prefix="cp"
+       copy_destination_prefix=""
+    fi
+    log "exec_prefix: $exec_prefix"
+    log "copy_prefix: $copy_prefix"
+    log "copy_destination_prefix: $copy_destination_prefix"
+
+    case "$mode" in
+       exec)
+          # In exec mode the first argument is the __target_host which we already got from env. Get rid of it.
+          shift
+          code="$exec_prefix schroot -c $schroot_name -- sh -c '$@'"
+       ;;
+       copy)
+          # get directory for given chroot_name
+          schroot_directory="$($exec_prefix schroot -c $schroot_name --config | awk -F = '/directory=/ {print $2}')"
+          [ -n "$schroot_directory" ] || die "Failed to retreive schroot directory for schroot: $schroot_name"
+          log "schroot_directory: $schroot_directory"
+          # prefix destination with chroot
+          code="$copy_prefix $(echo "$@" | sed "s|$uri:|${copy_destination_prefix}${schroot_directory}|g")"
+       ;;
+       *) die "Unknown mode: $mode";;
+    esac
+
+    log "code: $code"
+
+    # Run the code
+    $code
+
+    log "-----"
+
+sudo
+~~~~
+
+**copy**
+
+Use rsync over ssh to copy files. Uses the "--rsync-path" option
+to run the remote rsync instance with sudo.
+
+This command assumes your ssh configuration is already set up in ~/.ssh/config.
+
+Usage: cdist config --remote-copy /path/to/this/script target_host
+
+.. code-block:: sh
+
+    # For rsync to do the right thing, the source has to end with "/" if it is
+    # a directory. The below preprocessor loop takes care of that.
+
+    # second last argument is the source
+    source_index=$(($#-1))
+    index=0
+    for arg in $@; do
+       if [ $index -eq 0 ]; then
+          # reset $@
+          set --
+       fi
+       index=$((index+=1))
+       if [ $index -eq $source_index -a -d "$arg" ]; then
+          arg="${arg%/}/"
+       fi
+       set -- "$@" "$arg"
+    done
+
+    rsync --copy-links --rsync-path="sudo rsync" -e 'ssh' "$@"
+
+**exec**
+
+Prefixes all remote commands with sudo.
+
+This command assumes your ssh configuration is already set up in ~/.ssh/config.
+
+Usage: cdist config --remote-exec "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    host="$1"; shift
+    ssh -q "$host" sudo sh -c \""$@"\"

From 652c89185816526e5165ff9d5686a0d245ec3e5f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 29 Sep 2020 05:57:54 +0200
Subject: [PATCH 082/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 260b4fe4..8d380524 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
+	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From f994226d0e23b667309fd75116c0c8a14d15f34d Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 29 Sep 2020 19:44:47 +0200
Subject: [PATCH 083/411] [__package_pkgng_freebsd] Bootstrap pkg if necessary

In a pristine FreeBSD base installation, pkg is really a bootstrapper utility,
in such cases the type used to fail instead of automatically bootstrapping pkg.
---
 .../type/__package_pkgng_freebsd/explorer/pkg_bootstrapped   | 4 ++++
 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version | 5 +++++
 cdist/conf/type/__package_pkgng_freebsd/gencode-remote       | 5 +++++
 3 files changed, 14 insertions(+)
 create mode 100755 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped

diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped
new file mode 100755
index 00000000..429f15d3
--- /dev/null
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+if pkg -N >/dev/null 2>&1; then
+	echo "YES"
+fi
diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
index 92ce0623..f0fb9127 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
@@ -21,6 +21,11 @@
 # Retrieve the status of a package - parsed dpkg output
 #
 
+if ! pkg -N >/dev/null 2>&1; then
+   # Nothing to do if pkg is not bootstrapped
+   exit
+fi
+
 if [ -f "$__object/parameter/name" ]; then
    name="$(cat "$__object/parameter/name")"
 else
diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
index dd36efda..b5944177 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
+++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
@@ -43,6 +43,7 @@ fi
 repo="$(cat "$__object/parameter/repo")"
 state="$(cat "$__object/parameter/state")"
 curr_version="$(cat "$__object/explorer/pkg_version")"
+pkg_bootstrapped="$(cat "$__object/explorer/pkg_bootstrapped")"
 add_cmd="pkg install -y"
 rm_cmd="pkg delete -y"
 upg_cmd="pkg upgrade -y"
@@ -73,6 +74,10 @@ execcmd(){
          ;;
    esac
 
+   if [ -z "${pkg_bootstrapped}" ]; then
+       echo "pkg bootstrap -y >/dev/null 2>&1"
+   fi
+
    echo "$_cmd >/dev/null 2>&1"   # Silence the output of the command
    echo "status=\$?"
    echo "if [ \"\$status\" -ne \"0\" ]; then"

From 52b5f05163416196403634cb9a52f746cdb96391 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 30 Sep 2020 08:56:31 +0200
Subject: [PATCH 084/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8d380524..34c97d67 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
+	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 3fa74b454a9512319e3559a53dd477f38724fd9b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 30 Sep 2020 15:43:32 +0200
Subject: [PATCH 085/411] Fix typo

---
 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
index f0fb9127..1c6ba5e5 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Retrieve the status of a package - parsed dpkg output
+# Retrieve the status of a package - parsed pkgng output
 #
 
 if ! pkg -N >/dev/null 2>&1; then

From 5aeed14b1b8b022f952680cceb12a5099a099558 Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Thu, 8 Oct 2020 16:15:20 +0200
Subject: [PATCH 086/411] Fixed calling of __systemd_service type with correct
 arguments.

---
 cdist/conf/type/__service/manifest | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__service/manifest b/cdist/conf/type/__service/manifest
index cb5af234..beb0713c 100644
--- a/cdist/conf/type/__service/manifest
+++ b/cdist/conf/type/__service/manifest
@@ -7,7 +7,9 @@ action="$(cat "$__object/parameter/action")"
 
 case "$manager" in
 	systemd)
-		__systemd_service "$name" --action "$action"
+		test "$action" = "start" && action="running"
+		test "$action" = "stop"  && action="stopped"
+		__systemd_service "$name" --state "$action"
 	;;
 	*)
 		# Unknown: handled by `service $NAME $action` in gencode-remote.

From c030deea3dbccfe73637b730f0d545db7f8d5f35 Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Fri, 9 Oct 2020 06:51:44 +0200
Subject: [PATCH 087/411] [__line] Add support for '--state replace'

It is currently counter-intuitive that something like:

    # File '/thing' contents
    #SomeSetting WrongValue

    # Manifest
    __line '/thing' \
           --line 'SomeSeting GoodValue' \
           --regex '^(#[[:space:]]*)?SomeSetting[[:space:]]'

Produces:

    # Resulting '/thing' contents
    #SomeSetting WrongValue

This makes sense given the implementation, but it masks a very common use-case.

Changing the default behaviour for such a base type is not really an option, so
instead we add a `replace` as a valid value for `--state`, which would result
in:

    # Resulting '/thing' contents with: --state replace
    SomeSetting GoodValue

For compatibility, if the regex is missing, `--state replace` behaves just as
`--state present`.
---
 cdist/conf/type/__line/explorer/state | 12 +++++++++++-
 cdist/conf/type/__line/gencode-remote | 10 +++++++---
 cdist/conf/type/__line/man.rst        | 13 +++++++++++--
 3 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state
index e8fc3630..9d480b19 100755
--- a/cdist/conf/type/__line/explorer/state
+++ b/cdist/conf/type/__line/explorer/state
@@ -53,8 +53,10 @@ function _find(_text, _pattern) {
 BEGIN {
    getline anchor < (ENVIRON["__object"] "/parameter/" position)
    getline pattern < (ENVIRON["__object"] "/parameter/" needle)
+   getline line < (ENVIRON["__object"] "/parameter/line")
 
    found_line = 0
+   correct_line = 0
    correct_pos = (position != "after" && position != "before")
 }
 {
@@ -63,15 +65,18 @@ BEGIN {
          getline
          if (_find($0, pattern)) {
             found_line++
+	    if (index($0, line) == 1) { correct_line++ }
             correct_pos = 1
             exit 0
          }
       } else if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
       }
    } else if (position == "before") {
       if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
          getline
          if (match($0, anchor)) {
             correct_pos = 1
@@ -81,13 +86,18 @@ BEGIN {
    } else {
       if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
          exit 0
       }
    }
 }
 END {
    if (found_line && correct_pos) {
-      print "present"
+      if (correct_line) {
+        print "present"
+      } else {
+        print "matching"
+      }
    } else if (found_line) {
       print "wrongposition"
    } else {
diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote
index 88cae68b..a89886da 100755
--- a/cdist/conf/type/__line/gencode-remote
+++ b/cdist/conf/type/__line/gencode-remote
@@ -38,7 +38,11 @@ if [ -z "$state_is" ]; then
    exit 1
 fi
 
-if [ "$state_should" = "$state_is" ]; then
+if [ "$state_should" = "$state_is" ] || \
+	{ [ "$state_should" = "present" ] && [ "$state_is" = "matching" ] ;} || \
+	{ [ "$state_should" = "replace" ] && [ "$state_is" = "present" ] ;} ; then
+   # If state matches already, or 'present' is used and regex matches
+   # or 'replace' is used and the exact line is present, then there is
    # nothing to do
    exit 0
 fi
@@ -61,8 +65,8 @@ fi
 add=0
 remove=0
 case "$state_should" in
-   present)
-      if [ "$state_is" = "wrongposition" ]; then
+   present|replace)
+      if [ "$state_is" = "wrongposition" ] || [ "$state_is" = "matching" ]; then
          echo updated >> "$__messages_out"
          remove=1
       else
diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst
index f76cab64..70490f68 100644
--- a/cdist/conf/type/__line/man.rst
+++ b/cdist/conf/type/__line/man.rst
@@ -31,7 +31,7 @@ file
 line
     Specifies the line which should be absent or present.
 
-    Must be present, if state is 'present'.
+    Must be present, if state is 'present' or 'replace'.
     Ignored if regex is given and state is 'absent'.
 
 regex
@@ -41,10 +41,13 @@ regex
     If state is 'absent', ensure all lines matching the regular expression
     are absent.
 
+    If state is 'replace', ensure all lines matching the regular expression
+    are exactly 'line'.
+
     The regular expression is interpreted by awk's match function.
 
 state
-    'present' or 'absent', defaults to 'present'
+    'present', 'absent' or 'replace', defaults to 'present'.
 
 onchange
     The code to run if line is added, removed or updated.
@@ -99,6 +102,12 @@ EXAMPLES
         --line '-session required pam_exec.so debug log=/tmp/classify.log /usr/local/libexec/classify' \
         --after '^session[[:space:]]+include[[:space:]]+password-auth-ac$'
 
+    # Uncomment as needed and set a value in a configuration file.
+    __line /etc/example.conf \
+        --line 'SomeSetting SomeValue' \
+        --regex '^(#[[:space:]]*)?SomeSetting[[:space:]]' \
+        --state replace
+
 
 SEE ALSO
 --------

From 4df5c91912edf9f31a6716d7cbe0adaedbc2b0e3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 9 Oct 2020 06:52:52 +0200
Subject: [PATCH 088/411] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 34c97d67..6d237449 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,8 @@ next:
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
 	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
+	* Type __service: Fix calling __systemd_service (Mark Verboom)
+	* Type __line: Add 'replace' state (Evil Ham)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 8ecae42199f27693eec771513174ce8143c8ca83 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:00:11 +0300
Subject: [PATCH 089/411] remove bin/cdist script

---
 bin/cdist | 33 ---------------------------------
 1 file changed, 33 deletions(-)
 delete mode 100755 bin/cdist

diff --git a/bin/cdist b/bin/cdist
deleted file mode 100755
index 645020a1..00000000
--- a/bin/cdist
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/sh
-# -*- coding: utf-8 -*-
-#
-# 2012 Nico Schottelius (nico-cdist at schottelius.org)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-# Wrapper for real script to allow execution from checkout
-dir=${0%/*}
-
-# Ensure version is present - the bundled/shipped version contains a static version,
-# the git version contains a dynamic version
-"$dir/build-helper" version
-
-libdir=$(cd "${dir}/../" && pwd -P)
-export PYTHONPATH="${libdir}"
-
-"$dir/../scripts/cdist" "$@"

From 45d51c0e1553ce12fcea09b1b9e0d2fb9c0304c5 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:00:44 +0300
Subject: [PATCH 090/411] rename build-helper -> cdist-build-helper

---
 bin/{build-helper => cdist-build-helper} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename bin/{build-helper => cdist-build-helper} (100%)

diff --git a/bin/build-helper b/bin/cdist-build-helper
similarity index 100%
rename from bin/build-helper
rename to bin/cdist-build-helper

From 3f1939716f4c4c2cdf538a22170a67fe181c675c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:02:45 +0300
Subject: [PATCH 091/411] enable running scripts/cdist directly and symlinked

---
 scripts/cdist | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scripts/cdist b/scripts/cdist
index b1d782ab..2ce40ae0 100755
--- a/scripts/cdist
+++ b/scripts/cdist
@@ -22,7 +22,15 @@
 #
 
 import logging
+import os
 import sys
+
+cdist_bin = os.path.abspath(__file__)
+if os.path.islink(cdist_bin):
+    cdist_bin = os.readlink(cdist_bin)
+cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
+sys.path.insert(0, cdist_dir)
+
 import cdist
 import cdist.argparse
 import cdist.banner

From fdc1ab93e968575b1d1f6f3b2a6cefa6ed6bb850 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:03:11 +0300
Subject: [PATCH 092/411] move scripts/* to bin/

---
 {scripts => bin}/cdist          | 0
 {scripts => bin}/cdist-dump     | 0
 {scripts => bin}/cdist-new-type | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 rename {scripts => bin}/cdist (100%)
 rename {scripts => bin}/cdist-dump (100%)
 rename {scripts => bin}/cdist-new-type (100%)

diff --git a/scripts/cdist b/bin/cdist
similarity index 100%
rename from scripts/cdist
rename to bin/cdist
diff --git a/scripts/cdist-dump b/bin/cdist-dump
similarity index 100%
rename from scripts/cdist-dump
rename to bin/cdist-dump
diff --git a/scripts/cdist-new-type b/bin/cdist-new-type
similarity index 100%
rename from scripts/cdist-new-type
rename to bin/cdist-new-type

From 86057cef19b10859cd66ceb4e3ebade5986aff89 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:05:17 +0300
Subject: [PATCH 093/411] don't die if there is no version.py

---
 cdist/__init__.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index be573170..26e7d071 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -24,10 +24,13 @@ import os
 import hashlib
 
 import cdist.log
-import cdist.version
 
 
-VERSION = cdist.version.VERSION
+try:
+    import cdist.version
+    VERSION = cdist.version.VERSION
+except ModuleNotFoundError:
+    VERSION = 'from git'
 
 BANNER = """
              ..          .       .x+=:.        s

From fd04c036139f150be79801e5fd3b49086e5d7263 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 13:42:16 +0300
Subject: [PATCH 094/411] add parent dir to module search path only when
 importing fails

---
 bin/cdist | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index 2ce40ae0..66ccbb5a 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -25,13 +25,19 @@ import logging
 import os
 import sys
 
-cdist_bin = os.path.abspath(__file__)
-if os.path.islink(cdist_bin):
-    cdist_bin = os.readlink(cdist_bin)
-cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
-sys.path.insert(0, cdist_dir)
+# try to import cdist and if that fails, then add this file's parent dir to
+# module search path and try again. additionally check if this file is
+# symlinked, so user can symlink this file to, for example, ~/.local/bin.
+try:
+    import cdist
+except ModuleNotFoundError:
+    cdist_bin = os.path.abspath(__file__)
+    if os.path.islink(cdist_bin):
+        cdist_bin = os.readlink(cdist_bin)
+    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
+    sys.path.insert(0, cdist_dir)
+    import cdist
 
-import cdist
 import cdist.argparse
 import cdist.banner
 import cdist.config

From 1614b62f702a82731b3b9c636894268b4310821b Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 13:48:28 +0300
Subject: [PATCH 095/411] fallback VERSION to "unknown version"

---
 cdist/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index 26e7d071..350a2765 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -30,7 +30,7 @@ try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
-    VERSION = 'from git'
+    VERSION = 'unknown version'
 
 BANNER = """
              ..          .       .x+=:.        s

From 174aa7728065da611dc8887d9f33ead7e27c473a Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:11:00 +0300
Subject: [PATCH 096/411] __file__ already is absolute

---
 bin/cdist | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/cdist b/bin/cdist
index 66ccbb5a..d7eae312 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -31,7 +31,7 @@ import sys
 try:
     import cdist
 except ModuleNotFoundError:
-    cdist_bin = os.path.abspath(__file__)
+    cdist_bin = __file__
     if os.path.islink(cdist_bin):
         cdist_bin = os.readlink(cdist_bin)
     cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))

From 65c8af4ba3751dddbfe07fd5077e5a99b5d240b8 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:11:12 +0300
Subject: [PATCH 097/411] overengineered version discovery

---
 cdist/__init__.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index 350a2765..f659506f 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -22,6 +22,7 @@
 
 import os
 import hashlib
+import subprocess
 
 import cdist.log
 
@@ -30,7 +31,19 @@ try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
-    VERSION = 'unknown version'
+    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+    if os.path.isdir(os.path.join(cdist_dir, '.git')):
+        run_git = subprocess.run(
+            ['git', 'describe', '--always'],
+            cwd=cdist_dir,
+            capture_output=True,
+            text=True)
+        if run_git.returncode == 0:
+            VERSION = str(run_git.stdout)
+        else:
+            VERSION = 'from git'
+    else:
+        VERSION = 'unknown version'
 
 BANNER = """
              ..          .       .x+=:.        s

From 42d5d6c3e29f6500c3f6321b01c3b268e6886ec6 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:12:39 +0300
Subject: [PATCH 098/411] redundant str()

---
 cdist/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index f659506f..c16562e7 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -39,7 +39,7 @@ except ModuleNotFoundError:
             capture_output=True,
             text=True)
         if run_git.returncode == 0:
-            VERSION = str(run_git.stdout)
+            VERSION = run_git.stdout
         else:
             VERSION = 'from git'
     else:

From b41d80075a616a1c7ac3a361079c748424409995 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:16:04 +0300
Subject: [PATCH 099/411] update paths in setup.py

---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 7b000041..002be19c 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ import subprocess
 
 
 # We have it only if it is a git cloned repo.
-build_helper = os.path.join('bin', 'build-helper')
+build_helper = os.path.join('bin', 'cdist-build-helper')
 # Version file path.
 version_file = os.path.join('cdist', 'version.py')
 # If we have build-helper we could be a git repo.
@@ -56,7 +56,7 @@ setup(
     name="cdist",
     packages=["cdist", "cdist.core", "cdist.exec", "cdist.util", ],
     package_data={'cdist': package_data},
-    scripts=["scripts/cdist", "scripts/cdist-dump", "scripts/cdist-new-type"],
+    scripts=["bin/cdist", "bin/cdist-dump", "bin/cdist-new-type"],
     version=cdist.version.VERSION,
     description="A Usable Configuration Management System",
     author="Nico Schottelius",

From e55db1b427fcc7f750adfcb3646a3eeb446113de Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 15:41:38 +0300
Subject: [PATCH 100/411] use check_output for git describe execution and
 define fallback VERSION earlier

---
 cdist/__init__.py | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index c16562e7..1c60ae0f 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -27,23 +27,21 @@ import subprocess
 import cdist.log
 
 
+VERSION = 'unknown version'
+
 try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
     cdist_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
     if os.path.isdir(os.path.join(cdist_dir, '.git')):
-        run_git = subprocess.run(
-            ['git', 'describe', '--always'],
-            cwd=cdist_dir,
-            capture_output=True,
-            text=True)
-        if run_git.returncode == 0:
-            VERSION = run_git.stdout
-        else:
-            VERSION = 'from git'
-    else:
-        VERSION = 'unknown version'
+        try:
+            VERSION = subprocess.check_output(
+                ['git', 'describe', '--always'],
+                cwd=cdist_dir,
+                universal_newlines=True)
+        except:
+            pass
 
 BANNER = """
              ..          .       .x+=:.        s

From 54d83a62118ed3f0c6328c4b25a3b9ee56adf27a Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 15:50:50 +0300
Subject: [PATCH 101/411] there is no single author anymore, also remove www.

---
 setup.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 002be19c..858c2c17 100644
--- a/setup.py
+++ b/setup.py
@@ -59,9 +59,8 @@ setup(
     scripts=["bin/cdist", "bin/cdist-dump", "bin/cdist-new-type"],
     version=cdist.version.VERSION,
     description="A Usable Configuration Management System",
-    author="Nico Schottelius",
-    author_email="nico-cdist-pypi@schottelius.org",
-    url="https://www.cdi.st/",
+    author="cdist contributors",
+    url="https://cdi.st",
     classifiers=[
         "Development Status :: 6 - Mature",
         "Environment :: Console",

From 507fa6fa93210959b52bb639a9394c8ac52b40e5 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 17 Oct 2020 17:05:09 +0200
Subject: [PATCH 102/411] __download: fix non-existent parameter of __unpack

Probably happened due to renaming .. guess it's correct now.
---
 cdist/conf/type/__download/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index eb3ac971..d8814683 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -69,7 +69,7 @@ EXAMPLES
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
-            --move-existing-destination \
+            --backup-destination \
             --destination /opt/cpma/server
 
 

From d20fb743243f0341820a5b41a5725fb705eb5aae Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sat, 17 Oct 2020 23:16:42 +0300
Subject: [PATCH 103/411] use os.path.realpath instead, because it eliminates
 any symbolic links encountered in the path

---
 bin/cdist | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index d7eae312..1f92f157 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -25,16 +25,16 @@ import logging
 import os
 import sys
 
-# try to import cdist and if that fails, then add this file's parent dir to
-# module search path and try again. additionally check if this file is
-# symlinked, so user can symlink this file to, for example, ~/.local/bin.
+# try to import cdist and if that fails,
+# then add this file's parent dir to
+# module search path and try again.
 try:
     import cdist
 except ModuleNotFoundError:
-    cdist_bin = __file__
-    if os.path.islink(cdist_bin):
-        cdist_bin = os.readlink(cdist_bin)
-    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
+    cdist_dir = os.path.realpath(
+        os.path.join(
+            os.path.dirname(os.path.realpath(__file__)),
+            os.pardir))
     sys.path.insert(0, cdist_dir)
     import cdist
 

From b2e6afb57e6799b934a2eaa387cea337d65d8aac Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 17 Oct 2020 23:01:36 +0200
Subject: [PATCH 104/411] __download: adapt download+unpack example in manpage

---
 cdist/conf/type/__download/man.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index d8814683..54503470 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -70,6 +70,7 @@ EXAMPLES
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
             --backup-destination \
+            --preserve-archive \
             --destination /opt/cpma/server
 
 

From 955b84727611caa5aa05e4521af326c94c649e89 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 18 Oct 2020 15:55:14 +0200
Subject: [PATCH 105/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 6d237449..8ad2c624 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
 	* Type __service: Fix calling __systemd_service (Mark Verboom)
 	* Type __line: Add 'replace' state (Evil Ham)
+	* Type __download: Fix man page (Matthias Stecher)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 6964070282a6c5d09a458017623012fc17e3008d Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 18 Oct 2020 17:13:22 +0300
Subject: [PATCH 106/411] s/build-helper/cdist-build-helper/

---
 .gitlab-ci.yml             | 8 ++++----
 README-maintainers         | 2 +-
 bin/cdist-build-helper     | 2 +-
 docs/src/cdist-install.rst | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e215652c..e48355ea 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,15 +6,15 @@ image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
 unit_tests:
     stage: test
     script:
-        - ./bin/build-helper version
-        - ./bin/build-helper test
+        - ./bin/cdist-build-helper version
+        - ./bin/cdist-build-helper test
 
 pycodestyle:
     stage: test
     script:
-        - ./bin/build-helper pycodestyle
+        - ./bin/cdist-build-helper pycodestyle
 
 shellcheck:
     stage: test
     script:
-        - ./bin/build-helper shellcheck
+        - ./bin/cdist-build-helper shellcheck
diff --git a/README-maintainers b/README-maintainers
index af57f475..5766dd7d 100644
--- a/README-maintainers
+++ b/README-maintainers
@@ -1,4 +1,4 @@
-Maintainers should use ./bin/build-helper script.
+Maintainers should use ./bin/cdist-build-helper script.
 
 Makefile is intended for end users. It can be used for non-maintaining
 targets that can be run from pure source (without git repository).
diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper
index d4d603ed..bdef0dbb 100755
--- a/bin/cdist-build-helper
+++ b/bin/cdist-build-helper
@@ -495,7 +495,7 @@ eof
     ;;
 
     shellcheck-build-helper)
-        ${SHELLCHECKCMD} ./bin/build-helper
+        ${SHELLCHECKCMD} ./bin/cdist-build-helper
     ;;
 
     check-shellcheck)
diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst
index 6f4f14d7..18863145 100644
--- a/docs/src/cdist-install.rst
+++ b/docs/src/cdist-install.rst
@@ -49,7 +49,7 @@ create version.py:
 
 .. code-block:: sh
 
-    ./bin/build-helper version
+    ./bin/cdist-build-helper version
 
 Then you install it with:
 
@@ -70,7 +70,7 @@ Or directly with distutils:
 
     python setup.py install
 
-Note that `bin/build-helper` script is intended for cdist maintainers.
+Note that `bin/cdist-build-helper` script is intended for cdist maintainers.
 
 
 Available versions in git

From e3d906a85fe5e0c9f254d3620e8568437c93d97c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 15 Sep 2020 00:55:26 +0300
Subject: [PATCH 107/411] [__acl] remove deprecated parameters, fix some bugs
 and improve manual

---
 cdist/conf/type/__acl/explorer/checks         | 39 -------------------
 cdist/conf/type/__acl/explorer/getent         |  4 ++
 cdist/conf/type/__acl/gencode-remote          | 36 ++++++++---------
 cdist/conf/type/__acl/man.rst                 | 12 +++---
 .../conf/type/__acl/parameter/deprecated/acl  |  1 -
 .../type/__acl/parameter/deprecated/group     |  1 -
 .../conf/type/__acl/parameter/deprecated/mask |  1 -
 .../type/__acl/parameter/deprecated/other     |  1 -
 .../conf/type/__acl/parameter/deprecated/user |  1 -
 cdist/conf/type/__acl/parameter/optional      |  2 -
 .../type/__acl/parameter/optional_multiple    |  3 --
 11 files changed, 26 insertions(+), 75 deletions(-)
 delete mode 100755 cdist/conf/type/__acl/explorer/checks
 create mode 100755 cdist/conf/type/__acl/explorer/getent
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/acl
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/group
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/mask
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/other
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/user

diff --git a/cdist/conf/type/__acl/explorer/checks b/cdist/conf/type/__acl/explorer/checks
deleted file mode 100755
index 70bb0412..00000000
--- a/cdist/conf/type/__acl/explorer/checks
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/sh -e
-#
-# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-# TODO check if filesystem has ACL turned on etc
-
-if [ -f "$__object/parameter/acl" ]
-then
-    grep -E '^(default:)?(user|group):' "$__object/parameter/acl" \
-    | while read -r acl
-    do
-        param="$( echo "$acl" | awk -F: '{print $(NF-2)}' )"
-        check="$( echo "$acl" | awk -F: '{print $(NF-1)}' )"
-
-        [ "$param" = 'user' ] && db=passwd || db="$param"
-
-        if ! getent "$db" "$check" > /dev/null
-        then
-            echo "missing $param '$check'" >&2
-            exit 1
-        fi
-    done
-fi
diff --git a/cdist/conf/type/__acl/explorer/getent b/cdist/conf/type/__acl/explorer/getent
new file mode 100755
index 00000000..7e6c2c30
--- /dev/null
+++ b/cdist/conf/type/__acl/explorer/getent
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+getent passwd | awk -F: '{print "user:"$1}'
+getent group | awk -F: '{print "group:"$1}'
diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote
index e5404a9d..32318e91 100755
--- a/cdist/conf/type/__acl/gencode-remote
+++ b/cdist/conf/type/__acl/gencode-remote
@@ -22,8 +22,8 @@ file_is="$( cat "$__object/explorer/file_is" )"
 
 if [ "$file_is" = 'missing' ] \
     && [ -z "$__cdist_dry_run" ] \
-    && \( [ ! -f "$__object/parameter/file" ] \
-        || [ ! -f "$__object/parameter/directory" ] \)
+    && [ ! -f "$__object/parameter/file" ] \
+    && [ ! -f "$__object/parameter/directory" ]
 then
     exit 0
 fi
@@ -47,28 +47,26 @@ then
 elif [ -f "$__object/parameter/entry" ]
 then
     acl_should="$( cat "$__object/parameter/entry" )"
-elif [ -f "$__object/parameter/acl" ]
-then
-    acl_should="$( cat "$__object/parameter/acl" )"
-elif
-    [ -f "$__object/parameter/user" ] \
-        || [ -f "$__object/parameter/group" ] \
-        || [ -f "$__object/parameter/mask" ] \
-        || [ -f "$__object/parameter/other" ]
-then
-    acl_should="$( for param in user group mask other
-    do
-        [ ! -f "$__object/parameter/$param" ] && continue
-
-        echo "$param" | grep -Eq 'mask|other' && sep=:: || sep=:
-
-        echo "$param$sep$( cat "$__object/parameter/$param" )"
-    done )"
 else
     echo 'no parameters set' >&2
     exit 1
 fi
 
+# instead of setfacl's non-helpful message "Option -m: Invalid argument near character X"
+# let's check if target has necessary users and groups, since mistyped or missing
+# users/groups in target is most common reason.
+echo "$acl_should" \
+    | grep -Po '(user|group):[^:]+' \
+    | sort -u \
+    | while read -r l
+    do
+        if ! grep "$l" -Fxq "$__object/explorer/getent"
+        then
+            echo "no $l' in target" | sed "s/:/ '/" >&2
+            exit 1
+        fi
+    done
+
 if [ -f "$__object/parameter/default" ]
 then
     acl_should="$( echo "$acl_should" \
diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst
index 28412871..307be72b 100644
--- a/cdist/conf/type/__acl/man.rst
+++ b/cdist/conf/type/__acl/man.rst
@@ -12,11 +12,14 @@ Fully supported and tested on Linux (ext4 filesystem), partial support for FreeB
 
 See ``setfacl`` and ``acl`` manpages for more details.
 
+One of ``--entry`` or ``--source`` must be used.
 
-REQUIRED MULTIPLE PARAMETERS
+
+OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
 entry
    Set ACL entry following ``getfacl`` output syntax.
+   Must be used if ``--source`` is not used.
 
 
 OPTIONAL PARAMETERS
@@ -25,6 +28,7 @@ source
    Read ACL entries from stdin or file.
    Ordering of entries is not important.
    When reading from file, comments and empty lines are ignored.
+   Must be used if ``--entry`` is not used.
 
 file
    Create/change file with ``__file`` using ``user:group:mode`` pattern.
@@ -48,12 +52,6 @@ remove
    ``mask`` and ``other`` entries can't be removed, but only changed.
 
 
-DEPRECATED PARAMETERS
----------------------
-Parameters ``acl``, ``user``, ``group``, ``mask`` and ``other`` are deprecated and they
-will be removed in future versions. Please use ``entry`` parameter instead.
-
-
 EXAMPLES
 --------
 
diff --git a/cdist/conf/type/__acl/parameter/deprecated/acl b/cdist/conf/type/__acl/parameter/deprecated/acl
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/acl
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/group b/cdist/conf/type/__acl/parameter/deprecated/group
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/group
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/mask b/cdist/conf/type/__acl/parameter/deprecated/mask
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/mask
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/other b/cdist/conf/type/__acl/parameter/deprecated/other
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/other
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/user b/cdist/conf/type/__acl/parameter/deprecated/user
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/user
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/optional b/cdist/conf/type/__acl/parameter/optional
index cdcbc0b8..5a0c29a3 100644
--- a/cdist/conf/type/__acl/parameter/optional
+++ b/cdist/conf/type/__acl/parameter/optional
@@ -1,5 +1,3 @@
-mask
-other
 source
 file
 directory
diff --git a/cdist/conf/type/__acl/parameter/optional_multiple b/cdist/conf/type/__acl/parameter/optional_multiple
index c615d507..4c884f03 100644
--- a/cdist/conf/type/__acl/parameter/optional_multiple
+++ b/cdist/conf/type/__acl/parameter/optional_multiple
@@ -1,4 +1 @@
 entry
-acl
-user
-group

From 716cd37281903aafd7d65df539265a6722e809b1 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 21 Sep 2020 11:18:39 +0300
Subject: [PATCH 108/411] [__update_alternatives] rewrite and support --install

---
 .../explorer/alternatives                     |  4 ++
 .../type/__update_alternatives/explorer/link  | 40 +++++++++++++++++++
 .../__update_alternatives/explorer/path_is    | 12 ++++++
 .../explorer/path_should_state                |  8 ++++
 .../type/__update_alternatives/explorer/state |  8 ----
 .../type/__update_alternatives/gencode-remote | 37 ++++++++++++++---
 cdist/conf/type/__update_alternatives/man.rst | 13 ++++--
 .../__update_alternatives/parameter/boolean   |  1 +
 8 files changed, 107 insertions(+), 16 deletions(-)
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/alternatives
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/link
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/path_is
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/path_should_state
 delete mode 100755 cdist/conf/type/__update_alternatives/explorer/state
 create mode 100644 cdist/conf/type/__update_alternatives/parameter/boolean

diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives
new file mode 100755
index 00000000..34aaca56
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/alternatives
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+update-alternatives --display "$__object_id" 2>/dev/null \
+    | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link
new file mode 100755
index 00000000..6519e7c2
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/link
@@ -0,0 +1,40 @@
+#!/bin/sh -e
+
+# fedora's (update-)alternatives --display output doesn't have
+# "link <name> is <path>" line, but debian does. so, let's find
+# out how they store this information.
+#
+# debian and friends:
+#   https://salsa.debian.org/dpkg-team/dpkg/-/blob/master/utils/update-alternatives.c
+#   see calls to altdb_print_line function
+#
+# fedora and friends:
+#   https://github.com/fedora-sysv/chkconfig/blob/master/alternatives.c
+#   see calls to parseLine function
+#
+# conclusion: it is safe to assume that (master) link is on second line
+
+for altdir in \
+    /var/lib/dpkg/alternatives \
+    /var/lib/alternatives
+do
+    if [ ! -f "$altdir/$__object_id" ]
+    then
+        continue
+    fi
+
+    link="$( awk 'NR==2' "$altdir/$__object_id" )"
+
+    if [ -n "$link" ]
+    then
+        break
+    fi
+done
+
+if [ -z "$link" ]
+then
+    echo "unable to get link for $__object_id" >&2
+    exit 1
+fi
+
+echo "$link"
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
new file mode 100755
index 00000000..fc304d5d
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -0,0 +1,12 @@
+#!/bin/sh -e
+
+path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \
+    | awk '/link currently points to/ {print $5}' )"
+
+if [ -z "$path_is" ]
+then
+    echo "unable to get current path for $__object_id" >&2
+    exit 1
+fi
+
+echo "$path_is"
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state
new file mode 100755
index 00000000..59e015c5
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+if [ -f "$( cat "$__object/parameter/path" )" ]
+then
+    echo 'present'
+else
+    echo 'absent'
+fi
diff --git a/cdist/conf/type/__update_alternatives/explorer/state b/cdist/conf/type/__update_alternatives/explorer/state
deleted file mode 100755
index 04a78aaa..00000000
--- a/cdist/conf/type/__update_alternatives/explorer/state
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh -e
-path="$(cat "$__object/parameter/path")"
-name="$__object_id"
-link="$(readlink "/etc/alternatives/$name")"
-if [ "$path" = "$link" ]
-then echo present
-else echo absent
-fi
diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index c0b49814..e393cdef 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Ander Punnar (ander@kvlt.ee)
 #
 # This file is part of cdist.
 #
@@ -16,12 +17,38 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
-if [ "$(cat "$__object/explorer/state")" = 'present' ]
-then exit 0
+path_is="$( cat "$__object/explorer/path_is" )"
+
+path_should="$( cat "$__object/parameter/path" )"
+
+if [ "$path_is" = "$path_should" ]
+then
+    exit 0
+fi
+
+if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ]
+then
+    echo "$path_should does not exist in target" >&2
+    exit 1
 fi
 
-path="$(cat "$__object/parameter/path")"
 name="$__object_id"
-echo "update-alternatives --quiet --set '$name' '$path'"
+
+alternatives="$( cat "$__object/explorer/alternatives" )"
+
+if ! echo "$alternatives" | grep -Fxq "$path_should"
+then
+    if [ ! -f "$__object/parameter/install" ]
+    then
+        echo "$path_should is not in $name alternatives." >&2
+        echo 'Please install missing packages or use --install to add path to alternatives.' >&2
+        exit 1
+    fi
+
+    link="$( cat "$__object/explorer/link" )"
+
+    echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
+fi
+
+echo "update-alternatives --set '$name' '$path_should'"
diff --git a/cdist/conf/type/__update_alternatives/man.rst b/cdist/conf/type/__update_alternatives/man.rst
index 73d82d11..0dc973f2 100644
--- a/cdist/conf/type/__update_alternatives/man.rst
+++ b/cdist/conf/type/__update_alternatives/man.rst
@@ -19,6 +19,12 @@ path
    Use this path for the given alternative
 
 
+BOOLEAN PARAMETERS
+------------------
+install
+   Add (``update-alternatives --install``) missing path to alternatives.
+
+
 EXAMPLES
 --------
 
@@ -36,11 +42,12 @@ SEE ALSO
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Ander Punnar <ander@kvlt.ee>
 
 
 COPYING
 -------
-Copyright \(C) 2013 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
+Copyright \(C) 2013 Nico Schottelius and 2020 Ander Punnar. You can
+redistribute it and/or modify it under the terms of the GNU General Public
+License as published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__update_alternatives/parameter/boolean b/cdist/conf/type/__update_alternatives/parameter/boolean
new file mode 100644
index 00000000..7c32f559
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/parameter/boolean
@@ -0,0 +1 @@
+install

From 687c1d2dd9bed4130a65885d19ce31c87c9d79de Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 19 Oct 2020 06:57:00 +0200
Subject: [PATCH 109/411] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 8ad2c624..a0f1ead2 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,8 @@ next:
 	* Type __service: Fix calling __systemd_service (Mark Verboom)
 	* Type __line: Add 'replace' state (Evil Ham)
 	* Type __download: Fix man page (Matthias Stecher)
+	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
+	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From e8aede1e912f24ec7fde1d087dcd06f249d7701a Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 19 Oct 2020 18:00:05 +0200
Subject: [PATCH 110/411] __lxc_container: do not call rm recusivly for a file

---
 cdist/conf/type/__lxc_container/gencode-remote | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__lxc_container/gencode-remote b/cdist/conf/type/__lxc_container/gencode-remote
index f9e4dceb..42e1e43e 100755
--- a/cdist/conf/type/__lxc_container/gencode-remote
+++ b/cdist/conf/type/__lxc_container/gencode-remote
@@ -168,7 +168,7 @@ LXC
                 # remove empty tempfile if it was created
                 if [ -f "$__object/parameter/no-default-config" ]; then
                     cat <<RM
-rm -rf "\$empty_conf"
+rm -f "\$empty_conf"
 RM
                 fi
             fi
@@ -231,7 +231,7 @@ DONE
 
     # apply all changes
     cat <<DONE
-rm -rf "$container_config"
+rm -f "$container_config"
 mv "\$tmpconfig" "$container_config"
 DONE
     echo "config" >> "$__messages_out"

From 4e06ea78166c181cad37a6f9e3b8880719d6d386 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Tue, 20 Oct 2020 17:03:14 +0200
Subject: [PATCH 111/411] __lxc_container: no grep ERE used, switching to BRE

When checking for lxc configuration that should be removed, it says to
grep to use extended regular expressions (ERE). But there are no
characters not covered by basic regular expressions (BRE). So switching
to to BRE, as it may not match user input as regex.
---
 cdist/conf/type/__lxc_container/gencode-remote | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__lxc_container/gencode-remote b/cdist/conf/type/__lxc_container/gencode-remote
index 42e1e43e..6bef504e 100755
--- a/cdist/conf/type/__lxc_container/gencode-remote
+++ b/cdist/conf/type/__lxc_container/gencode-remote
@@ -207,10 +207,10 @@ DONE
     # check if smth. to be deleted
     #  must be before adding, because it takes the content of the original file
     #  because 'cat $file > $file' will not work (> opens before command reads it)
-    # will create extended regex pattern for grep with malformed spaces
+    # will create a basic regex pattern for grep to ignore malformed spaces
     if [ -s "$__object/files/config.del" ]; then
         cat <<DONE
-grep -v -E -f - <<'ABSENT' "$container_config" > "\$tmpconfig"
+grep -v -f - <<'ABSENT' "$container_config" > "\$tmpconfig"
 $(awk -v FS=' = ' -v OFS=' = ' -v blank='[[:blank:]]*' '
 function ntostring(n) { ret=""; for(i=n; i<=NF; i++) ret=ret $i (i<NF ? OFS : ""); return ret }
 { printf "^%s%s%s=%s%s%s$%s", blank, $1, blank, blank, ntostring(2), blank, ORS }

From 367da4b77e79b82fe84c2ce1a7f75cfad92db81e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 28 Oct 2020 18:18:24 +0100
Subject: [PATCH 112/411] [type/__file] Fix --state pre-exists

---
 cdist/conf/type/__file/gencode-remote | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index 35356b13..2675b03a 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -87,11 +87,6 @@ case "$state_should" in
         fi
     ;;
 
-    pre-exists)
-        # pre-exists should never reach gencode-remote…
-        exit 1
-   ;;
-
     absent)
         if [ "$type" = "file" ]; then
             echo "rm -f '$destination'"

From 9277e0ba19c346189658712acf17a772967dfa32 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 29 Oct 2020 09:30:58 +0100
Subject: [PATCH 113/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index a0f1ead2..5977d80e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,7 @@ next:
 	* Type __download: Fix man page (Matthias Stecher)
 	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
+	* Type __file: Fix state pre-exists (Dennis Camera)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 82a9aa79020ff2369b1f4d530a3a6d5ca41915f8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 29 Oct 2020 10:39:42 +0100
Subject: [PATCH 114/411] [type/__apt_norecommends] Use 00InstallRecommends
 file as debian-installer does

debian-installer can be preseeded with `base-installer/install-recommends` to
disable installation of recommended packages already during OS installation.
d-i will then create the file `/etc/apt/apt.conf.d/00InstallRecommends`
(cf. https://salsa.debian.org/installer-team/base-installer/-/blob/master/library.sh).

__apt_norecommends should use the same file to avoid having two config files
effectively doing the same thing.
---
 cdist/conf/type/__apt_norecommends/man.rst  |  9 +++--
 cdist/conf/type/__apt_norecommends/manifest | 41 +++++++++++----------
 2 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/cdist/conf/type/__apt_norecommends/man.rst b/cdist/conf/type/__apt_norecommends/man.rst
index 001fffe4..9297b518 100644
--- a/cdist/conf/type/__apt_norecommends/man.rst
+++ b/cdist/conf/type/__apt_norecommends/man.rst
@@ -32,11 +32,12 @@ EXAMPLES
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
+Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Copyright \(C) 2014 Steven Armstrong. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2014 Steven Armstrong, 2020 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__apt_norecommends/manifest b/cdist/conf/type/__apt_norecommends/manifest
index e737df89..fc187784 100755
--- a/cdist/conf/type/__apt_norecommends/manifest
+++ b/cdist/conf/type/__apt_norecommends/manifest
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,26 +20,28 @@
 #
 
 
-os=$(cat "$__global/explorer/os")
+os=$(cat "${__global:?}/explorer/os")
 
-case "$os" in
-   ubuntu|debian|devuan)
-      # No stinking recommends thank you very much.
-      # If I want something installed I will do so myself.
-      __file /etc/apt/apt.conf.d/99-no-recommends \
-         --owner root --group root --mode 644 \
-         --source - << DONE
-APT::Install-Recommends "0";
-APT::Install-Suggests "0";
-APT::AutoRemove::RecommendsImportant "0";
-APT::AutoRemove::SuggestsImportant "0";
-DONE
-   ;;
-   *)
-      cat >&2 << DONE
+case ${os}
+in
+	(ubuntu|debian|devuan)
+		__file /etc/apt/apt.conf.d/00InstallRecommends --state present \
+			--owner root --group root --mode 0644 --source - <<-'EOF'
+		APT::Install-Recommends "false";
+		APT::Install-Suggests "false";
+		APT::AutoRemove::RecommendsImportant "false";
+		APT::AutoRemove::SuggestsImportant "false";
+		EOF
+
+		# TODO: Remove the following object after some time
+		require=__file/etc/apt/apt.conf.d/00InstallRecommends \
+		__file /etc/apt/apt.conf.d/99-no-recommends --state absent
+		;;
+	(*)
+		cat >&2 <<EOF
 The developer of this type (${__type##*/}) did not think your operating system
 ($os) would have any use for it. If you think otherwise please submit a patch.
-DONE
-      exit 1
-   ;;
+EOF
+		exit 1
+		;;
 esac

From b9ad22595fdb75c17e88e33dc028c33dbd5a7de9 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 18:03:27 +0100
Subject: [PATCH 115/411] [scanner] begin scanner implementation - non invasive

---
 cdist/scan.py | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 cdist/scan.py

diff --git a/cdist/scan.py b/cdist/scan.py
new file mode 100644
index 00000000..fa1bf5de
--- /dev/null
+++ b/cdist/scan.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+from scapy.all import *
+from scapy.data import ETHER_TYPES
+
+
+class Scanner(object):
+    def recv_msg_cpu(self, pkg):
+#        print(pkg.__repr__())
+        if ICMPv6EchoReply in pkg:
+            host = pkg['IPv6'].src
+            print(f"Host {host} is alive")
+
+
+    def scan(self):
+        sniff(iface="wlan0",
+              filter="icmp6",
+              prn=self.recv_msg_cpu)
+
+
+if __name__ == '__main__':
+    s = Scanner()
+    s.scan()

From 87b46a622411362f441b62199972e71d85d7d2d6 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 18:49:20 +0100
Subject: [PATCH 116/411] [scanner] finish prototype

ping @poljakowski - it's your turn now
---
 cdist/scan.py | 138 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 131 insertions(+), 7 deletions(-)

diff --git a/cdist/scan.py b/cdist/scan.py
index fa1bf5de..e2100499 100644
--- a/cdist/scan.py
+++ b/cdist/scan.py
@@ -19,24 +19,148 @@
 #
 #
 
-from scapy.all import *
-from scapy.data import ETHER_TYPES
+#
+# Interface to be implemented:
+# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated
+#   scan: scan / listen for icmp6 replies
+#   trigger: send trigger to multicast
+#   config: configure newly detected hosts
+#   install: install newly detected hosts
+#
+# Scanner logic
+#  - save results to configdir:
+#     basedir = ~/.cdist/scan/<ipv6-address>
+#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time or similar
+#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#
+#
+#
+#
+# cdist scan --list
+#       Show all known hosts including last seen flag
+#
+# Logic for reconfiguration:
+#
+#  - record when configured last time
+#  - introduce a parameter --reconfigure-after that takes time argument
+#  - reconfigure if a) host alive and b) reconfigure-after time passed
+#
 
 
+from multiprocessing import Process
+import os
+
+# FIXME: fail gracefully if non existent - i.e. "scapy required for scanner - please install python3-scapy"
+from scapy.all import *
+
+# Datetime overwrites scapy.all.datetime - needs to be imported AFTER
+import datetime
+
+class Trigger(object):
+    """
+    Trigger an ICMPv6EchoReply from all hosts that are alive
+    """
+
+    def __init__(self, interfaces=None, verbose=False):
+        self.interfaces = interfaces
+        self.verbose = verbose
+
+        # Wait 5 seconds before triggering again - FIXME: add parameter
+        self.sleeptime = 5
+
+    def start(self):
+        self.processes = []
+        for interface in self.interfaces:
+            p = Process(target=self.run_interface, args=(interface,))
+            self.processes.append(p)
+            p.start()
+
+    def join(self):
+        for process in self.processes:
+            process.join()
+
+    def run_interface(self, interface):
+        while True:
+            self.trigger(interface)
+            time.sleep(self.sleeptime)
+
+    def trigger(self, interface):
+        packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
+        send(packet, verbose=self.verbose)
+
 class Scanner(object):
-    def recv_msg_cpu(self, pkg):
-#        print(pkg.__repr__())
+    """
+    Scan for replies of hosts, maintain the up-to-date database
+    """
+
+    def __init__(self, interfaces=None, outdir=None):
+        self.interfaces = interfaces
+
+        if outdir:
+            self.outdir = outdir
+        else:
+            self.outdir = "."
+
+    def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
             host = pkg['IPv6'].src
             print(f"Host {host} is alive")
 
+            dir = os.path.join(self.outdir, host)
+            fname = os.path.join(dir, "last_seen")
+
+            now = datetime.datetime.now()
+
+            os.makedirs(dir, exist_ok=True)
+
+            # FIXME: maybe adjust the format so we can easily parse again
+            with open(fname, "w") as fd:
+                fd.write(f"{now}\n")
+
 
     def scan(self):
-        sniff(iface="wlan0",
+        sniff(iface=self.interfaces,
               filter="icmp6",
-              prn=self.recv_msg_cpu)
+              prn=self.handle_pkg)
 
 
 if __name__ == '__main__':
-    s = Scanner()
+    t = Trigger(interfaces=["wlan0"])
+    t.start()
+
+    # Scanner can listen on many interfaces at the same time
+    s = Scanner(interfaces=["wlan0"])
     s.scan()
+
+    # Join back the trigger processes
+    t.join()
+
+    # Test in my lan shows:
+    # [18:48] bridge:cdist% ls -1d fe80::*
+    # fe80::142d:f0a5:725b:1103
+    # fe80::20d:b9ff:fe49:ac11
+    # fe80::20d:b9ff:fe4c:547d
+    # fe80::219:d2ff:feb2:2e12
+    # fe80::21b:fcff:feee:f446
+    # fe80::21b:fcff:feee:f45c
+    # fe80::21b:fcff:feee:f4b1
+    # fe80::21b:fcff:feee:f4ba
+    # fe80::21b:fcff:feee:f4bc
+    # fe80::21b:fcff:feee:f4c1
+    # fe80::21d:72ff:fe86:46b
+    # fe80::42b0:34ff:fe6f:f6f0
+    # fe80::42b0:34ff:fe6f:f863
+    # fe80::42b0:34ff:fe6f:f9b2
+    # fe80::4a5d:60ff:fea1:e55f
+    # fe80::77a3:5e3f:82cc:f2e5
+    # fe80::9e93:4eff:fe6c:c1f4
+    # fe80::ba69:f4ff:fec5:6041
+    # fe80::ba69:f4ff:fec5:8db7
+    # fe80::bad8:12ff:fe65:313d
+    # fe80::bad8:12ff:fe65:d9b1
+    # fe80::ce2d:e0ff:fed4:2611
+    # fe80::ce32:e5ff:fe79:7ea7
+    # fe80::d66d:6dff:fe33:e00
+    # fe80::e2ff:f7ff:fe00:20e6
+    # fe80::f29f:c2ff:fe7c:275e

From 91d99bf08accb3bde0da420294fc9fb68d3d4068 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 21:22:36 +0100
Subject: [PATCH 117/411] [RFC] scanner documentation

---
 docs/dev/logs/2020-10-29.org | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 docs/dev/logs/2020-10-29.org

diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org
new file mode 100644
index 00000000..718fd68c
--- /dev/null
+++ b/docs/dev/logs/2020-10-29.org
@@ -0,0 +1,34 @@
+* The scanner, 2020-10-29, Hacking Villa Diesbach
+** Motivation
+   - The purpose of cdist is to ensure systems are in a configured state
+   - If systems reboot into a clean (think: netboot) state they are
+     stuck in an unconfigured mode
+   - We can either trigger *from* those machines
+     - this is what cdist trigger is for
+   - Or we can regulary *scan* for machines
+     - This method does not need any modification to standard OS
+** How it works
+   - cdist scan uses the all nodes multicast group ff02::1
+   - It sends a ping packet there in regular intervals
+   - This even works in non-IPv6 networks, as all operating systems
+     are IPv6 capable and usually IPv6 enabled by default
+     - Link local is always accessible!
+   - cdist scan receives an answer from all alive hosts
+     - These results are stored in ~/.cdist/scan/${hostip}
+     - We record the last_seen date  ~/.cdist/scan/${hostip}/last_seen
+   - After a host is detected, cdist *can* try to configure it
+     - It saves the result (+/- logging needs to be defined) in
+       ~/.cdist/scan/${hostip}/{config, install}_result
+     - If logging is saved: maybe in ~/.cdist/scan/${hostip}/{config, install}_log
+     - Final naming TBD
+** Benefits from the scanning approach
+   - We know when a host is alive/dead
+   - We can use standard OS w/o trigger customisation
+     - Only requirement: we can ssh into it
+     - Can make use f.i. of Alpine Linux w/ ssh keys feeding in
+   - We can trigger regular reconfiguration
+     - If alive && last_config_time > 1d -> reconfigure
+   - Data can be exported to f.i. prometheus
+     - Record when configured (successfully)
+     - Record when seen
+   - Enables configurations in stateless environments

From 09dfcfe81e0d9e6520d9bf98598037f86c84641e Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 23:16:08 +0100
Subject: [PATCH 118/411] [scanner] add to beta commands

---
 cdist/argparse.py            | 32 ++++++++++++++++++++-
 cdist/scan/commandline.py    | 55 ++++++++++++++++++++++++++++++++++++
 cdist/{ => scan}/scan.py     | 20 +++++++++----
 docs/dev/logs/2020-10-29.org | 23 +++++++++++++++
 4 files changed, 124 insertions(+), 6 deletions(-)
 create mode 100644 cdist/scan/commandline.py
 rename cdist/{ => scan}/scan.py (90%)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 1d16bb25..ff195e8c 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -8,10 +8,11 @@ import cdist.configuration
 import cdist.log
 import cdist.preos
 import cdist.info
+import cdist.scan.commandline
 
 
 # set of beta sub-commands
-BETA_COMMANDS = set(('install', 'inventory', ))
+BETA_COMMANDS = set(('install', 'inventory', 'scan', ))
 # set of beta arguments for sub-commands
 BETA_ARGS = {
     'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
@@ -470,6 +471,35 @@ def get_parsers():
             'pattern', nargs='?', help='Glob pattern.')
     parser['info'].set_defaults(func=cdist.info.Info.commandline)
 
+    # Scan = config + further
+    parser['scan'] = parser['sub'].add_parser('scan', add_help=False,
+                                                 parents=[parser['config']])
+
+    parser['scan'] = parser['sub'].add_parser(
+            'scan', parents=[parser['loglevel'],
+                             parser['beta'],
+                             parser['colored_output'],
+                             parser['common'],
+                             parser['config_main']])
+
+    parser['scan'].add_argument(
+        '-m', '--mode', help='Which modes should run',
+        action='append', default=[],
+        choices=['scan', 'trigger'])
+    parser['scan'].add_argument(
+        '--config',
+        action='store_true',
+        help='Try to configure detected hosts')
+    parser['scan'].add_argument(
+        '-I', '--interfaces',
+        action='append',  default=[],
+        help='On which interfaces to scan/trigger')
+    parser['scan'].add_argument(
+        '-d', '--delay',
+        action='store',  default=3600,
+        help='How long to wait before reconfiguring after last try')
+    parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
+
     for p in parser:
         parser[p].epilog = EPILOG
 
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
new file mode 100644
index 00000000..0fb718e5
--- /dev/null
+++ b/cdist/scan/commandline.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import logging
+
+log = logging.getLogger("scan")
+
+
+# define this outside of the class to not handle scapy import errors by default
+def commandline(args):
+    log.debug(args)
+
+    try:
+        import cdist.scan.scan as scan
+    except ModuleNotFoundError:
+        print('cdist scan requires scapy to be installed')
+
+    processes = []
+
+    if not args.mode:
+        # By default scan and trigger, but do not call any action
+        args.mode = ['scan', 'trigger' ]
+
+    if 'trigger' in args.mode:
+        t = scan.Trigger(interfaces=args.interfaces)
+        t.start()
+        processes.append(t)
+        log.debug("Trigger started")
+
+    if 'scan' in args.mode:
+        s = scan.Scanner(interfaces=args.interfaces, args=args)
+        s.start()
+        processes.append(s)
+        log.debug("Scanner started")
+
+    for process in processes:
+        process.join()
diff --git a/cdist/scan.py b/cdist/scan/scan.py
similarity index 90%
rename from cdist/scan.py
rename to cdist/scan/scan.py
index e2100499..fcbf1899 100644
--- a/cdist/scan.py
+++ b/cdist/scan/scan.py
@@ -50,13 +50,14 @@
 
 from multiprocessing import Process
 import os
-
-# FIXME: fail gracefully if non existent - i.e. "scapy required for scanner - please install python3-scapy"
+import logging
 from scapy.all import *
 
 # Datetime overwrites scapy.all.datetime - needs to be imported AFTER
 import datetime
 
+log = logging.getLogger("scan")
+
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
@@ -87,6 +88,7 @@ class Trigger(object):
 
     def trigger(self, interface):
         packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
+        log.debug(f"Sending request on {interface}")
         send(packet, verbose=self.verbose)
 
 class Scanner(object):
@@ -94,18 +96,18 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces=None, outdir=None):
+    def __init__(self, interfaces=None, args=None, outdir=None):
         self.interfaces = interfaces
 
         if outdir:
             self.outdir = outdir
         else:
-            self.outdir = "."
+            self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan')
 
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
             host = pkg['IPv6'].src
-            print(f"Host {host} is alive")
+            log.verbose(f"Host {host} is alive")
 
             dir = os.path.join(self.outdir, host)
             fname = os.path.join(dir, "last_seen")
@@ -118,13 +120,21 @@ class Scanner(object):
             with open(fname, "w") as fd:
                 fd.write(f"{now}\n")
 
+    def start(self):
+        self.process = Process(target=self.scan)
+        self.process.start()
+
+    def join(self):
+        self.process.join()
 
     def scan(self):
+        log.debug("Scanning - zzzzz")
         sniff(iface=self.interfaces,
               filter="icmp6",
               prn=self.handle_pkg)
 
 
+
 if __name__ == '__main__':
     t = Trigger(interfaces=["wlan0"])
     t.start()
diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org
index 718fd68c..4461be8c 100644
--- a/docs/dev/logs/2020-10-29.org
+++ b/docs/dev/logs/2020-10-29.org
@@ -32,3 +32,26 @@
      - Record when configured (successfully)
      - Record when seen
    - Enables configurations in stateless environments
+** Sample output v2020-10-29
+23:14] bridge:~% sudo  cdist scan -b -I wlan0 -vv
+VERBOSE: cdist: version 6.8.0-36-g91d99bf0
+VERBOSE: scan: Host fe80::21d:72ff:fe86:46b is alive
+VERBOSE: scan: Host fe80::ce2d:e0ff:fed4:2611 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4c1 is alive
+VERBOSE: scan: Host fe80::e2ff:f7ff:fe00:20e6 is alive
+VERBOSE: scan: Host fe80::20d:b9ff:fe49:ac11 is alive
+VERBOSE: scan: Host fe80::9e93:4eff:fe6c:c1f4 is alive
+VERBOSE: scan: Host fe80::ce32:e5ff:fe79:7ea7 is alive
+VERBOSE: scan: Host fe80::219:d2ff:feb2:2e12 is alive
+VERBOSE: scan: Host fe80::d66d:6dff:fe33:e00 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f446 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4b1 is alive
+VERBOSE: scan: Host fe80::20d:b9ff:fe4c:547d is alive
+VERBOSE: scan: Host fe80::bad8:12ff:fe65:313d is alive
+VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f6f0 is alive
+VERBOSE: scan: Host fe80::ba69:f4ff:fec5:6041 is alive
+VERBOSE: scan: Host fe80::f29f:c2ff:fe7c:275e is alive
+VERBOSE: scan: Host fe80::ba69:f4ff:fec5:8db7 is alive
+VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f863 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4bc is alive
+...

From e30ecdda535200ac75a6e774590d2e1b5e1b5b28 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 13:05:40 +0200
Subject: [PATCH 119/411] Add __uci and __uci_commit types

---
 cdist/conf/type/__uci/explorer/state          | 89 +++++++++++++++++++
 cdist/conf/type/__uci/gencode-remote          | 67 ++++++++++++++
 cdist/conf/type/__uci/man.rst                 | 77 ++++++++++++++++
 cdist/conf/type/__uci/manifest                | 40 +++++++++
 cdist/conf/type/__uci/parameter/default/state |  1 +
 .../type/__uci/parameter/default/transaction  |  1 +
 cdist/conf/type/__uci/parameter/optional      |  2 +
 .../type/__uci/parameter/required_multiple    |  1 +
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++
 cdist/conf/type/__uci_commit/man.rst          | 58 ++++++++++++
 cdist/conf/type/__uci_commit/nonparallel      |  0
 11 files changed, 357 insertions(+)
 create mode 100644 cdist/conf/type/__uci/explorer/state
 create mode 100755 cdist/conf/type/__uci/gencode-remote
 create mode 100644 cdist/conf/type/__uci/man.rst
 create mode 100755 cdist/conf/type/__uci/manifest
 create mode 100644 cdist/conf/type/__uci/parameter/default/state
 create mode 100644 cdist/conf/type/__uci/parameter/default/transaction
 create mode 100644 cdist/conf/type/__uci/parameter/optional
 create mode 100644 cdist/conf/type/__uci/parameter/required_multiple
 create mode 100755 cdist/conf/type/__uci_commit/gencode-remote
 create mode 100644 cdist/conf/type/__uci_commit/man.rst
 create mode 100644 cdist/conf/type/__uci_commit/nonparallel

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
new file mode 100644
index 00000000..2e98b606
--- /dev/null
+++ b/cdist/conf/type/__uci/explorer/state
@@ -0,0 +1,89 @@
+#!/bin/sh
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current state of the configuration option
+# The output of this explorer is one of these values:
+#   present
+#     The configuration option is present and has the value of the
+#     parameter --value.
+#   absent
+#     The configuration option is not defined.
+#   different
+#     The configuration option is present but has a different value than the
+#     parameter --value.
+#   rearranged
+#     The configuration option is present (a list) and has the same values as
+#     the parameter --value, but in a different order.
+
+RS=$(printf '\036')
+
+option=${__object_id:?}
+
+values_is=$(uci -s -N -d "${RS}" get "${option}" 2>/dev/null) || {
+	echo absent
+	exit 0
+}
+
+# strip off trailing newline
+printf '%s' "${values_is}" \
+| awk '
+BEGIN {
+	state = "present"  # assume all is fine
+}
+NR == FNR {
+	# memoize "should" state
+	should[FNR] = $0
+
+	# go to next line (important!)
+	next
+}
+
+# compare "is" state
+$0 == should[FNR] { next }
+
+FNR > length(should) {
+	# there are more "is" records than "should" -> definitely different
+	state = "different"
+	exit
+}
+
+{
+	# see if we can find the value somewhere in should
+	for (i in should) {
+		if ($0 == should[i]) {
+			# ... value found -> rearranged
+			# FIXME: Duplicate values are not properly handled here. Do they matter?
+			state = "rearranged"
+			next
+		}
+	}
+
+	state = "different"
+	exit
+}
+
+END {
+	if (FNR < length(should)) {
+		# "is" was shorter than "should" -> different
+		state = "different"
+	}
+
+	print state
+}
+' "${__object:?}/parameter/value" RS="${RS}" -
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
new file mode 100755
index 00000000..48f114fe
--- /dev/null
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -0,0 +1,67 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
+
+config=${__object_id:?}
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+case ${state_should}
+in
+	(present)
+		if in_list "${state_is}" 'present' 'rearranged'
+		then
+			# NOTE: order is ignored so rearranged is also fine.
+			exit 0
+		fi
+
+		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
+		then
+			# "should" is a list
+			if test "${state_is}" != 'absent'
+			then
+				printf "uci delete '%s'\n" "${config}"
+			fi
+
+			while read -r value
+			do
+				printf "uci add_list '%s'='%s'\n" "${config}" "${value}"
+			done <"${__object:?}/parameter/value"
+		else
+			# "should" is a scalar
+			value=$(cat "${__object:?}/parameter/value")
+			printf "uci set '%s'='%s'\n" "${config}" "${value}"
+		fi
+		;;
+	(absent)
+		if in_list "${state_is}" 'absent'
+		then
+			exit 0
+		fi
+
+		printf "uci delete '%s'\n" "${config}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
new file mode 100644
index 00000000..d23d8b2b
--- /dev/null
+++ b/cdist/conf/type/__uci/man.rst
@@ -0,0 +1,77 @@
+cdist-type__uci(7)
+==================
+
+NAME
+----
+cdist-type__uci - Manage configuration values in OpenWrt's
+Unified Configuration Interface (UCI)
+
+
+DESCRIPTION
+-----------
+This cdist type can be used to alter configuration options in OpenWrt's UCI
+system.
+
+Options can be applied in batches if the `--transaction` parameter is used.
+It is important to ensure that the `__uci_commit` object is executed before a
+new transaction is started.
+
+REQUIRED PARAMETERS
+-------------------
+value
+    The value to be set. Can be used multiple times.
+
+    Due to the way cdist handles arguments, values **must not** contain newline
+    characters.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    `present` or `absent`, defaults to `present`.
+transaction
+    The name of the transaction this option belongs to.
+    If none is given: "default" is used.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Set the system hostname
+    __uci system.@system[0].hostname --value 'OpenWrt'
+
+    # Enable NTP and NTPd (in one transaction)
+    __uci system.ntp.enabled --value 1 --transaction ntp
+    __uci system.ntp.enable_server --value 1 --transaction ntp
+    __uci system.ntp.server --transaction ntp \
+        --value '0.openwrt.pool.ntp.org' \
+        --value '1.openwrt.pool.ntp.org' \
+        --value '2.openwrt.pool.ntp.org' \
+        --value '3.openwrt.pool.ntp.org'
+    export require=__uci_commit/ntp
+
+
+SEE ALSO
+--------
+- https://openwrt.org/docs/guide-user/base-system/uci
+- :strong:`cdist-type__uci_commit`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
new file mode 100755
index 00000000..e5b0fb30
--- /dev/null
+++ b/cdist/conf/type/__uci/manifest
@@ -0,0 +1,40 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+os=$(cat "${__global:?}/explorer/os")
+
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+case ${os}
+in
+	(openwrt)
+		# okay
+		;;
+	(*)
+		printf "Your operating system (%s) is currently not supported by this type (%s)\n" "${os}" "${__type##*/}" >&2
+		printf "Please contribute an implementation for it if you can.\n" >&2
+		exit 1
+		;;
+esac
+
+
+# Make sure the changes are being commited
+require=${__object_name:?} __uci_commit "${transaction_name}"
diff --git a/cdist/conf/type/__uci/parameter/default/state b/cdist/conf/type/__uci/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__uci/parameter/default/transaction b/cdist/conf/type/__uci/parameter/default/transaction
new file mode 100644
index 00000000..4ad96d51
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/default/transaction
@@ -0,0 +1 @@
+default
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
new file mode 100644
index 00000000..ddbbba16
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -0,0 +1,2 @@
+state
+transaction
diff --git a/cdist/conf/type/__uci/parameter/required_multiple b/cdist/conf/type/__uci/parameter/required_multiple
new file mode 100644
index 00000000..6d4e1507
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/required_multiple
@@ -0,0 +1 @@
+value
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
new file mode 100755
index 00000000..bed0eefb
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -0,0 +1,21 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+echo 'uci commit'
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
new file mode 100644
index 00000000..c55d06e5
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/man.rst
@@ -0,0 +1,58 @@
+cdist-type__uci_commit(7)
+=========================
+
+NAME
+----
+cdist-type__uci_commit - Commit a UCI transaction.
+
+
+DESCRIPTION
+-----------
+This type executes the "uci commit" command on the target.
+It is usually not required to use this type. Use the `--transaction` parameter
+of `cdist-type__uci`\ (7) instead.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Commit the default transaction
+    __uci_commit default
+
+    # Commit another transaction
+    __uci_commit my_transaction
+
+
+SEE ALSO
+--------
+:strong:`cdist-type__uci`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_commit/nonparallel b/cdist/conf/type/__uci_commit/nonparallel
new file mode 100644
index 00000000..e69de29b

From 55e7b32449c9a86e34054c321b20a49795c6652f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 14:00:29 +0200
Subject: [PATCH 120/411] [type/__uci] Only generate __uci_commit if changes
 are required

---
 cdist/conf/type/__uci/manifest | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index e5b0fb30..74524513 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,9 +18,12 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; } }
 
 os=$(cat "${__global:?}/explorer/os")
 
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case ${os}
@@ -35,6 +38,25 @@ in
 		;;
 esac
 
+changes_required=false
 
-# Make sure the changes are being commited
-require=${__object_name:?} __uci_commit "${transaction_name}"
+case ${state_should}
+in
+	(present)
+		# NOTE: order is ignored so rearranged is also fine.
+		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
+		;;
+	(absent)
+		in_list "${state_is}" 'absent' || changes_required=true
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
+
+if ${changes_required}
+then
+	# Make sure the changes are being committed
+	require=${__object_name:?} __uci_commit "${transaction_name}"
+fi

From a09120977f231fe3dda1c7c07073fc7e03fc5ea0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 14:07:10 +0200
Subject: [PATCH 121/411] [type/__uci] Allow omission of --value parameter if
 --state absent

---
 cdist/conf/type/__uci/explorer/state                   | 10 +++++++++-
 cdist/conf/type/__uci/man.rst                          |  1 +
 cdist/conf/type/__uci/manifest                         |  5 +++++
 .../parameter/{required_multiple => optional_multiple} |  0
 4 files changed, 15 insertions(+), 1 deletion(-)
 rename cdist/conf/type/__uci/parameter/{required_multiple => optional_multiple} (100%)

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
index 2e98b606..6fb4b173 100644
--- a/cdist/conf/type/__uci/explorer/state
+++ b/cdist/conf/type/__uci/explorer/state
@@ -40,6 +40,14 @@ values_is=$(uci -s -N -d "${RS}" get "${option}" 2>/dev/null) || {
 	exit 0
 }
 
+if test -f "${__object:?}/parameter/value"
+then
+	should_file="${__object:?}/parameter/value"
+else
+	should_file='/dev/null'
+fi
+
+
 # strip off trailing newline
 printf '%s' "${values_is}" \
 | awk '
@@ -86,4 +94,4 @@ END {
 
 	print state
 }
-' "${__object:?}/parameter/value" RS="${RS}" -
+' "${should_file}" RS="${RS}" -
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index d23d8b2b..c6bb81ab 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -20,6 +20,7 @@ REQUIRED PARAMETERS
 -------------------
 value
     The value to be set. Can be used multiple times.
+    This parameter is allowed to be omitted if `--state` is `absent`.
 
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index 74524513..e56462e5 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -43,6 +43,11 @@ changes_required=false
 case ${state_should}
 in
 	(present)
+		test -s "${__object:?}/parameter/value" || {
+			echo 'The parameter --value is required.' >&2
+			exit 1
+		}
+
 		# NOTE: order is ignored so rearranged is also fine.
 		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
 		;;
diff --git a/cdist/conf/type/__uci/parameter/required_multiple b/cdist/conf/type/__uci/parameter/optional_multiple
similarity index 100%
rename from cdist/conf/type/__uci/parameter/required_multiple
rename to cdist/conf/type/__uci/parameter/optional_multiple

From d8f20a6a204142360e7fb7afd6603c69ae613e2d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 12 Jun 2020 19:39:19 +0200
Subject: [PATCH 122/411] [type/__uci] Implement "real" transactions using
 batch files

---
 .../__uci/{gencode-remote => gencode-local}   | 22 ++++++++++++++++---
 cdist/conf/type/__uci/man.rst                 |  3 +--
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++++++++++++++-
 3 files changed, 40 insertions(+), 6 deletions(-)
 rename cdist/conf/type/__uci/{gencode-remote => gencode-local} (79%)

diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-local
similarity index 79%
rename from cdist/conf/type/__uci/gencode-remote
rename to cdist/conf/type/__uci/gencode-local
index 48f114fe..bba5944d 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-local
@@ -20,11 +20,27 @@
 
 in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
 
+uci_cmd() {
+	printf 'printf "%s\n"' "$1"
+	shift
+	printf " '%s'" "$@"
+	printf " >>'%s'\n" "${tmpfile}"
+}
+
 config=${__object_id:?}
 
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+tmpdir="${__global:?}/tmp/__uci"
+# HACK
+mkdir -p "${tmpdir}"
+
+tmpfile="${tmpdir}/${transaction_name}.txt"
+
+
 case ${state_should}
 in
 	(present)
@@ -44,12 +60,12 @@ in
 
 			while read -r value
 			do
-				printf "uci add_list '%s'='%s'\n" "${config}" "${value}"
+				uci_cmd "add_list '%s'='%s'" "${config}" "${value}"
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
 			value=$(cat "${__object:?}/parameter/value")
-			printf "uci set '%s'='%s'\n" "${config}" "${value}"
+			uci_cmd "set '%s'='%s'" "${config}" "${value}"
 		fi
 		;;
 	(absent)
@@ -58,7 +74,7 @@ in
 			exit 0
 		fi
 
-		printf "uci delete '%s'\n" "${config}"
+		uci_cmd "delete '%s'" "${config}"
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index c6bb81ab..2f64355b 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -13,8 +13,7 @@ This cdist type can be used to alter configuration options in OpenWrt's UCI
 system.
 
 Options can be applied in batches if the `--transaction` parameter is used.
-It is important to ensure that the `__uci_commit` object is executed before a
-new transaction is started.
+
 
 REQUIRED PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index bed0eefb..ac74e5e4 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -18,4 +18,23 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-echo 'uci commit'
+transaction_name=${__object_id:?}
+batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
+
+test -s "${batchfile}" || exit 0
+
+cat <<'EOF'
+rollback() {
+	uci changes \
+	| sed -e 's/\..*$//' -e 's/^-//' \
+	| while read -r package
+	  do
+		  uci revert "${package}"
+	  done
+}
+
+EOF
+
+echo "uci batch <<'EOF' && uci commit || rollback"
+cat "${batchfile}"
+echo 'EOF'

From d3574b2d3e807ae436afd90aa1ed9d01b78ca6a3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 16:43:16 +0200
Subject: [PATCH 123/411] [type/__uci] Send messages when options are set to be
 altered

---
 cdist/conf/type/__uci/gencode-local | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index bba5944d..ba27dc5e 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -53,6 +53,8 @@ in
 		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
 		then
 			# "should" is a list
+			printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+
 			if test "${state_is}" != 'absent'
 			then
 				printf "uci delete '%s'\n" "${config}"
@@ -64,6 +66,8 @@ in
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
+			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
 			value=$(cat "${__object:?}/parameter/value")
 			uci_cmd "set '%s'='%s'" "${config}" "${value}"
 		fi
@@ -74,6 +78,7 @@ in
 			exit 0
 		fi
 
+		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
 		uci_cmd "delete '%s'" "${config}"
 		;;
 	(*)

From 3a3be3631010b1bf17d084c775e306f11429f5d9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 12 Jun 2020 19:53:26 +0200
Subject: [PATCH 124/411] [type/__uci_commit] Send message on commit of a
 transaction

---
 cdist/conf/type/__uci_commit/gencode-remote | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index ac74e5e4..0332d7e0 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,6 +23,8 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
+printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
+
 cat <<'EOF'
 rollback() {
 	uci changes \

From e7369a1f9957ac31830a69de8101c434666da5da Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 17 Jun 2020 13:00:24 +0200
Subject: [PATCH 125/411] [type/__uci_commit] Abort if uncommited changes are
 present on the target

---
 cdist/conf/type/__uci_commit/explorer/changes | 22 ++++++++++++++++
 cdist/conf/type/__uci_commit/gencode-remote   | 25 +++++++++++++------
 2 files changed, 39 insertions(+), 8 deletions(-)
 create mode 100644 cdist/conf/type/__uci_commit/explorer/changes

diff --git a/cdist/conf/type/__uci_commit/explorer/changes b/cdist/conf/type/__uci_commit/explorer/changes
new file mode 100644
index 00000000..2690def7
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/explorer/changes
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer outputs the uncommited UCI changes on the target.
+
+uci changes
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index 0332d7e0..dd1d839c 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,20 +23,29 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
+if test -s "${__object:?}/explorer/changes"
+then
+	echo 'Uncommited UCI changes were found on the target:'
+	cat "${__object:?}/explorer/changes"
+	echo
+	echo 'This can be caused by manual changes or due to a previous failed run.'
+	echo 'Please investigate the situation, revert or commit the changes, and try again.'
+	exit 1
+fi >&2
+
 printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
 
-cat <<'EOF'
+cat <<CODE
 rollback() {
-	uci changes \
-	| sed -e 's/\..*$//' -e 's/^-//' \
+	uci changes \\
+	| sed -e 's/\..*\$//' -e 's/^-//' \\
 	| while read -r package
 	  do
-		  uci revert "${package}"
+		  uci revert "\${package}"
 	  done
 }
 
+uci batch <<'EOF' && uci commit || rollback
+$(cat "${batchfile}")
 EOF
-
-echo "uci batch <<'EOF' && uci commit || rollback"
-cat "${batchfile}"
-echo 'EOF'
+CODE

From cc599dab15ed5c8761b3bcdfb113403fa6d4bc77 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 5 Jul 2020 10:25:36 +0200
Subject: [PATCH 126/411] [type/__uci_commit] Move uncommited changes check
 from explorer to code-remote

This is done to prevent false positives/negatives (see NOTE in code)
---
 cdist/conf/type/__uci_commit/explorer/changes | 22 -------------------
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++++++++++-----
 2 files changed, 15 insertions(+), 28 deletions(-)
 delete mode 100644 cdist/conf/type/__uci_commit/explorer/changes

diff --git a/cdist/conf/type/__uci_commit/explorer/changes b/cdist/conf/type/__uci_commit/explorer/changes
deleted file mode 100644
index 2690def7..00000000
--- a/cdist/conf/type/__uci_commit/explorer/changes
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/sh
-#
-# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-# This explorer outputs the uncommited UCI changes on the target.
-
-uci changes
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index dd1d839c..504f7c3b 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,19 +23,28 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
-if test -s "${__object:?}/explorer/changes"
+printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
+
+# NOTE: Uncommited changes are checked in code-remote instead of in an explorer
+#       because in cdist there is no interlocking between explorers and code
+#       execution.
+#       Checking for uncommited changes in an explorer leaves a time slot in
+#       which changes made are not detected by the code.
+#       Furthermore an explorer running concurrently with another transactions
+#       code-remote could lead to a false positive.
+
+cat <<CODE
+changes=\$(uci changes)
+
+if test -n "\${changes}"
 then
 	echo 'Uncommited UCI changes were found on the target:'
-	cat "${__object:?}/explorer/changes"
-	echo
+	printf '%s\n\n' "\${changes}"
 	echo 'This can be caused by manual changes or due to a previous failed run.'
 	echo 'Please investigate the situation, revert or commit the changes, and try again.'
 	exit 1
 fi >&2
 
-printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
-
-cat <<CODE
 rollback() {
 	uci changes \\
 	| sed -e 's/\..*\$//' -e 's/^-//' \\

From 3ef638a61123069953a9e24044b6c94eba0b3b6b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 20 Jul 2020 12:12:15 +0200
Subject: [PATCH 127/411] [type/__uci_commit] Fail when uci(1) reports errors

---
 cdist/conf/type/__uci_commit/gencode-remote | 37 ++++++++++++++++-----
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index 504f7c3b..e5c93966 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -45,16 +45,37 @@ then
 	exit 1
 fi >&2
 
-rollback() {
-	uci changes \\
-	| sed -e 's/\..*\$//' -e 's/^-//' \\
-	| while read -r package
-	  do
-		  uci revert "\${package}"
-	  done
+check_errors() {
+	# reads stdin and forwards non-empty lines to stderr.
+	# returns 0 if stdin is empty, else 1.
+	! grep -e . >&2
 }
 
-uci batch <<'EOF' && uci commit || rollback
+commit() {
+	uci commit
+}
+
+rollback() {
+	echo >&2
+	echo 'An error occurred when trying to commit transaction ${transaction_name}!' >&2
+
+	uci changes \\
+	| sed -e 's/^-//' -e 's/\..*\$//' \\
+	| sort -u \\
+	| while read -r _package
+	  do
+		  uci revert "\${_package}"
+		  echo "\${_package}"  # for logging
+	  done \\
+	| awk '
+	  BEGIN { printf "Reverted changes in: " }
+	  { printf "%s%s", (FNR > 1 ? ", " : ""), \$0 }
+	  END { printf "\\n" }' >&2
+
+	return 1
+}
+
+uci batch <<'EOF' 2>&1 | check_errors && commit || rollback
 $(cat "${batchfile}")
 EOF
 CODE

From 4da39681188493981ec3c55ea918787195655972 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 20:04:04 +0200
Subject: [PATCH 128/411] [type/__uci_section] Add type

---
 cdist/conf/type/__uci_section/explorer/match  |  68 ++++++++++
 .../conf/type/__uci_section/explorer/options  |  28 ++++
 cdist/conf/type/__uci_section/explorer/type   |  25 ++++
 cdist/conf/type/__uci_section/man.rst         |  76 +++++++++++
 cdist/conf/type/__uci_section/manifest        | 122 ++++++++++++++++++
 .../__uci_section/parameter/default/state     |   1 +
 .../parameter/default/transaction             |   1 +
 .../type/__uci_section/parameter/optional     |   4 +
 .../__uci_section/parameter/optional_multiple |   1 +
 9 files changed, 326 insertions(+)
 create mode 100644 cdist/conf/type/__uci_section/explorer/match
 create mode 100644 cdist/conf/type/__uci_section/explorer/options
 create mode 100644 cdist/conf/type/__uci_section/explorer/type
 create mode 100644 cdist/conf/type/__uci_section/man.rst
 create mode 100755 cdist/conf/type/__uci_section/manifest
 create mode 100644 cdist/conf/type/__uci_section/parameter/default/state
 create mode 100644 cdist/conf/type/__uci_section/parameter/default/transaction
 create mode 100644 cdist/conf/type/__uci_section/parameter/optional
 create mode 100644 cdist/conf/type/__uci_section/parameter/optional_multiple

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
new file mode 100644
index 00000000..9cf1ea3e
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -0,0 +1,68 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer the "prefix" of the section matching --match.
+
+squote_values() {
+	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
+	    -e "/='.*'$/"'!{s/=/='\''/;s/$/'\''/}'
+}
+
+RS=$(printf '\036')
+
+if ! test -e "${__object:?}/parameter/match"
+then
+	if echo "${__object_id:?}" | grep -qvE '^[^.]+\.[^.]+$'
+	then
+		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
+		exit 1
+	fi
+
+	# If no --match is given, we take the __object_id as the section identifier.
+	echo "${__object_id:?}"
+	exit 0
+fi
+
+test -s "${__object:?}/parameter/match" \
+&& test -s "${__object:?}/parameter/type" \
+|| {
+	echo 'Parameters --match and --type must be used together.' >&2
+	exit 1
+}
+
+# Find by match
+match=$(cat "${__object:?}/parameter/match")
+sect_type_filter=$(cat "${__object:?}/parameter/type")
+
+package_filter=${sect_type_filter%%.*}
+section_filter=${sect_type_filter##*.}
+regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
+
+matched_sections=$(
+	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
+	| grep -e "${regex}" \
+	| sed -e 's/\.[^.]*=.*$//')
+
+if test "$(echo "${matched_sections}" | wc -l)" -gt 1
+then
+	printf 'Found multiple matching sections:\n%s\n' "${matched_sections}" >&2
+	exit 1
+fi
+
+echo "${matched_sections}"
diff --git a/cdist/conf/type/__uci_section/explorer/options b/cdist/conf/type/__uci_section/explorer/options
new file mode 100644
index 00000000..67b4f83f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/options
@@ -0,0 +1,28 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current options of the configuration section.
+
+RS=$(printf '\036')
+
+section=$("${__type_explorer:?}/match")
+test -n "${section}" || exit 0
+
+uci -s -N -d "${RS}" show "${section}" 2>/dev/null \
+| grep -v -e "^${section}=" || true
diff --git a/cdist/conf/type/__uci_section/explorer/type b/cdist/conf/type/__uci_section/explorer/type
new file mode 100644
index 00000000..1675c2e0
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/type
@@ -0,0 +1,25 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current section type.
+
+section=$("${__type_explorer:?}/match")
+test -n "${section}" || exit 0
+
+uci -s -N get "${section}" 2>/dev/null || true
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
new file mode 100644
index 00000000..3468bfc3
--- /dev/null
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -0,0 +1,76 @@
+cdist-type__uci_section(7)
+==========================
+
+NAME
+----
+cdist-type__uci_section - Manage configuration sections in OpenWrt's
+Unified Configuration Interface (UCI)
+
+
+DESCRIPTION
+-----------
+This cdist type can be used to replace whole configuration sections in OpenWrt's
+UCI system.
+It can be thought of as syntactic sugar for `cdist-type__uci`\ (7), as this type
+will generate the required `__uci` objects to make the section contain exactly
+the options specified via ``--option``.
+
+Since many default UCI sections are unnamed, this type allows to find the
+matching section by one of its options using the ``--match`` parameter.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+match
+    Allows to find a section to "replace" through one of its parameters.
+    The value to this parameter is a ``<option>=<string>`` string.
+option
+    An option that should be present in the section.
+    This parameter can be used multiple times to specify multiple options.
+    The value to this parameter is a ``<option>=<string>`` string.
+
+    Lists can be expressed by repeatedly using the same key.
+state
+    `present` or `absent`, defaults to `present`.
+transaction
+    The name of the transaction this option belongs to.
+    The value will be forwarded to `cdist-type__uci`\ (7).
+type
+    The type of the section in the format: ``<config>.<section-type>``
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # TODO
+    __uci_section ...
+
+
+SEE ALSO
+--------
+:strong:`cdist-type__uci`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
new file mode 100755
index 00000000..e16277ba
--- /dev/null
+++ b/cdist/conf/type/__uci_section/manifest
@@ -0,0 +1,122 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+grep_line() { echo "$2" | grep -qxF "$1"; }
+unquote_lines() {
+	sed -e '/^".*"$/{s/^"//;s/"$//}' \
+	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
+}
+
+append_values() {
+	while read -r _value
+	do
+		set -- "$@" --value "${_value}"
+	done
+	unset _value
+	"$@" </dev/null
+}
+
+section=$(cat "${__object:?}/explorer/match")
+
+state_should=$(cat "${__object:?}/parameter/state")
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+case $state_should
+in
+	(present)
+		test -f "${__object:?}/parameter/type" || {
+			echo 'Parameter --type is required.' >&2
+			exit 1
+		}
+		type_is=$(cat "${__object:?}/explorer/type")
+		type_should=$(cat "${__object:?}/parameter/type")
+
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
+		if test -n "${type_is}"
+		then
+			if test "${type_is}" != "${type_should##*.}"
+			then
+				# Check if section type matches (section exists and --type provided)
+				printf 'Section type "%s" does not match --type "%s".\n' \
+					"${type_is}" "${type_should}" >&2
+				exit 1
+			fi
+			sect_type=${type_is}
+		else
+			sect_type=${type_should##*.}
+		fi
+
+		if test -z "${section}"
+		then
+			# No section exists and --match was used.
+			# So we generate a new section identifier from $__object_id.
+			case ${__object_id:?}
+			in
+				(*.*) section=${__object_id:?} ;;
+				(*) section="${type_should%%.*}.${__object_id:?}" ;;
+			esac
+		fi
+
+		# Make sure the section itself is present
+		__uci "${section}" --state present --transaction "${transaction_name}" \
+			--value "${sect_type}"
+		export require=__uci/"${section}"
+
+		# Delete options not in "should"
+		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		| while read -r optname
+		  do
+			  if ! grep_line "${optname}" "${optnames_should}"
+			  then
+				  __uci "${section}.${optname}" --state absent \
+					  --transaction "${transaction_name}" </dev/null
+			  fi
+		  done
+
+		# Set "should" options
+		echo "${optnames_should}" \
+		| while read -r optname
+		  do
+			  test -n "${optname}" || continue  # ignore empty lines
+
+			  grep "^${optname}=" <"${__object:?}/parameter/option" \
+			  | sed -e 's/^.*=//' \
+			  | unquote_lines \
+			  | append_values \
+				    __uci "${section}.${optname}" --state present \
+					    --transaction "${transaction_name}"
+		  done
+		;;
+	(absent)
+		# if explorer found no section there is nothing to delete
+		test -n "${section}" || exit 0
+
+		__uci "${section}" --state absent --transaction "${transaction_name}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__uci_section/parameter/default/state b/cdist/conf/type/__uci_section/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__uci_section/parameter/default/transaction b/cdist/conf/type/__uci_section/parameter/default/transaction
new file mode 100644
index 00000000..4ad96d51
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/default/transaction
@@ -0,0 +1 @@
+default
diff --git a/cdist/conf/type/__uci_section/parameter/optional b/cdist/conf/type/__uci_section/parameter/optional
new file mode 100644
index 00000000..8b956bae
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/optional
@@ -0,0 +1,4 @@
+match
+state
+transaction
+type
diff --git a/cdist/conf/type/__uci_section/parameter/optional_multiple b/cdist/conf/type/__uci_section/parameter/optional_multiple
new file mode 100644
index 00000000..01925a15
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/optional_multiple
@@ -0,0 +1 @@
+option

From 179815b5e93bd51b6757d92d14470808c5bee746 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 21 Jun 2020 18:22:54 +0200
Subject: [PATCH 129/411] [type/__uci_section] Ignore SC2015 error (notabug)

---
 cdist/conf/type/__uci_section/explorer/match | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index 9cf1ea3e..a3832bff 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -39,8 +39,8 @@ then
 	exit 0
 fi
 
-test -s "${__object:?}/parameter/match" \
-&& test -s "${__object:?}/parameter/type" \
+# shellcheck disable=SC2015
+test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type" \
 || {
 	echo 'Parameters --match and --type must be used together.' >&2
 	exit 1

From d453d964e1e0827dad217e1dd424dbaf82e434fc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 20 Jul 2020 11:21:06 +0200
Subject: [PATCH 130/411] [type/__uci_section] Fix in section matching

---
 cdist/conf/type/__uci_section/explorer/match | 23 +++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index a3832bff..a5619d4e 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -17,7 +17,8 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-# This explorer the "prefix" of the section matching --match.
+# This explorer determines the "prefix" of the --type section matching --match
+# if set, or __object_id otherwise.
 
 squote_values() {
 	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
@@ -28,7 +29,7 @@ RS=$(printf '\036')
 
 if ! test -e "${__object:?}/parameter/match"
 then
-	if echo "${__object_id:?}" | grep -qvE '^[^.]+\.[^.]+$'
+	if ! echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$'
 	then
 		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
 		exit 1
@@ -47,16 +48,28 @@ test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type
 }
 
 # Find by match
-match=$(cat "${__object:?}/parameter/match")
-sect_type_filter=$(cat "${__object:?}/parameter/type")
+# NOTE: Apart from section types all values are printed in single quotes by UCI.
+match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
 
+if ! grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' "${__object:?}/parameter/type"
+then
+	echo 'Section types are a package name and section type separated by a "." (period).' >&2
+	exit 1
+fi
+
+sect_type_filter=$(cat "${__object:?}/parameter/type")
 package_filter=${sect_type_filter%%.*}
-section_filter=${sect_type_filter##*.}
+section_filter=${sect_type_filter#*.}
+# FIXME: Does not work with named sections
 regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
 
 matched_sections=$(
 	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
 	| grep -e "${regex}" \
+	| while read -r _line
+	  do
+		  if test "${_line#*=}" = "${match#*=}"; then echo "${_line}"; fi
+	  done \
 	| sed -e 's/\.[^.]*=.*$//')
 
 if test "$(echo "${matched_sections}" | wc -l)" -gt 1

From f782a5a370a956995f444a677ecffe7f1564764d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 23 Oct 2020 11:45:37 +0200
Subject: [PATCH 131/411] [type/__uci] Refactor to do proper quoting of UCI
 commands

---
 cdist/conf/type/__uci/gencode-local | 40 ++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index ba27dc5e..44bf9c4f 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -18,15 +18,37 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
+in_list() {
+	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
+}
+
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
 
 uci_cmd() {
-	printf 'printf "%s\n"' "$1"
-	shift
-	printf " '%s'" "$@"
-	printf " >>'%s'\n" "${tmpfile}"
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	# Will output a shell command to be executed locally to add the given
+	# command to $tmpfile.
+	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
+	printf ' >>%s\n' "$(quote "${tmpfile}")"
 }
 
+
 config=${__object_id:?}
 
 state_is=$(cat "${__object:?}/explorer/state")
@@ -57,19 +79,19 @@ in
 
 			if test "${state_is}" != 'absent'
 			then
-				printf "uci delete '%s'\n" "${config}"
+				uci_cmd delete "${config}"
 			fi
 
 			while read -r value
 			do
-				uci_cmd "add_list '%s'='%s'" "${config}" "${value}"
+				uci_cmd add_list "${config}"="${value}"
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
 			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
 
 			value=$(cat "${__object:?}/parameter/value")
-			uci_cmd "set '%s'='%s'" "${config}" "${value}"
+			uci_cmd set "${config}"="${value}"
 		fi
 		;;
 	(absent)
@@ -79,7 +101,7 @@ in
 		fi
 
 		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
-		uci_cmd "delete '%s'" "${config}"
+		uci_cmd delete "${config}"
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2

From 3a6b0851459f08ad56f1c6a138b168fedfba993e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 23 Oct 2020 22:03:25 +0200
Subject: [PATCH 132/411] [type/__uci] Check __object_id for syntax errors

---
 cdist/conf/type/__uci/gencode-local | 45 +++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index 44bf9c4f..af91a698 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -48,8 +48,53 @@ uci_cmd() {
 	printf ' >>%s\n' "$(quote "${tmpfile}")"
 }
 
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+uci_validate_tuple() (
+	tok=${1:?}
+	case $tok
+	in
+		(*.*.*)
+			# check option
+			option=${tok##*.}
+			uci_validate_name "${option}" || {
+				printf 'Invalid option: %s\n' "${option}" >&2
+				return 1
+			}
+			tok=${tok%.*}
+			;;
+		(*.*)
+			# no option (section definition)
+			;;
+		(*)
+			printf 'Invalid tuple: %s\n' "$1" >&2
+			return 1
+			;;
+	esac
+
+	case ${tok#*.}
+	in
+		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
+		(*)  section=${tok#*.} ;;
+	esac
+	uci_validate_name "${section}" || {
+		printf 'Invalid section: %s\n' "${1#*.}" >&2
+		return 1
+	}
+
+	config=${tok%%.*}
+	uci_validate_name "${config}" || {
+		printf 'Invalid config: %s\n' "${config}" >&2
+		return 1
+	}
+)
+
 
 config=${__object_id:?}
+uci_validate_tuple "${config}"
 
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")

From c37253b852477a0a22568643bad461d20b58706f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 22 Oct 2020 20:32:48 +0200
Subject: [PATCH 133/411] [type/__uci_section] Check __object_id for syntax
 errors

---
 cdist/conf/type/__uci_section/manifest | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index e16277ba..1a59b970 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -33,6 +33,32 @@ append_values() {
 	"$@" </dev/null
 }
 
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+## Check section name and error if invalid!
+case ${__object_id:?}
+in
+	(*.*)
+		uci_validate_name "${__object_id%%.*}" || {
+			printf 'Invalid package name: %s\n' "${__object_id%%.*}" >&2
+			exit 1
+		}
+		uci_validate_name "${__object_id#*.}" || {
+			printf 'Invalid section name: %s\n' "${__object_id#*.}" >&2
+			exit 1
+		}
+		;;
+	(*)
+		uci_validate_name "${__object_id:?}" || {
+			printf 'Invalid section name: %s\n' "${__object_id:?}" >&2
+			exit 1
+		}
+		;;
+esac
+
 section=$(cat "${__object:?}/explorer/match")
 
 state_should=$(cat "${__object:?}/parameter/state")

From fe26c119b52c3822f009fe9aeb1b52b60139e33a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 10:01:49 +0200
Subject: [PATCH 134/411] [type/__uci*] Update man pages

---
 cdist/conf/type/__uci/man.rst         | 15 ++++----
 cdist/conf/type/__uci_commit/man.rst  | 12 ++++---
 cdist/conf/type/__uci_section/man.rst | 51 ++++++++++++++++++++-------
 3 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 2f64355b..cf184f9d 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -3,23 +3,22 @@ cdist-type__uci(7)
 
 NAME
 ----
-cdist-type__uci - Manage configuration values in OpenWrt's
-Unified Configuration Interface (UCI)
+cdist-type__uci - Manage configuration values in UCI
 
 
 DESCRIPTION
 -----------
-This cdist type can be used to alter configuration options in OpenWrt's UCI
-system.
+This cdist type can be used to alter configuration options in OpenWrt's
+Unified Configuration Interface (UCI) system.
 
-Options can be applied in batches if the `--transaction` parameter is used.
+Options can be applied in batches if the ``--transaction`` parameter is used.
 
 
 REQUIRED PARAMETERS
 -------------------
 value
     The value to be set. Can be used multiple times.
-    This parameter is allowed to be omitted if `--state` is `absent`.
+    This parameter is ignored if ``--state`` is ``absent``.
 
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
@@ -28,10 +27,10 @@ value
 OPTIONAL PARAMETERS
 -------------------
 state
-    `present` or `absent`, defaults to `present`.
+    ``present`` or ``absent``, defaults to ``present``.
 transaction
     The name of the transaction this option belongs to.
-    If none is given: "default" is used.
+    If none is given: ``default`` is used.
 
 
 BOOLEAN PARAMETERS
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
index c55d06e5..ea6611eb 100644
--- a/cdist/conf/type/__uci_commit/man.rst
+++ b/cdist/conf/type/__uci_commit/man.rst
@@ -3,14 +3,16 @@ cdist-type__uci_commit(7)
 
 NAME
 ----
-cdist-type__uci_commit - Commit a UCI transaction.
+cdist-type__uci_commit - Commit UCI transactions
 
 
 DESCRIPTION
 -----------
-This type executes the "uci commit" command on the target.
-It is usually not required to use this type. Use the `--transaction` parameter
-of `cdist-type__uci`\ (7) instead.
+This type executes the ``uci commit`` command on the target with the commands
+queued in a transaction.
+It is usually not required to use this type. Use the ``--transaction`` parameter
+of :strong:`cdist-type__uci`\ (7) and :strong:`cdist-type__uci_section`\ (7)
+instead.
 
 
 REQUIRED PARAMETERS
@@ -42,7 +44,7 @@ EXAMPLES
 
 SEE ALSO
 --------
-:strong:`cdist-type__uci`\ (7)
+:strong:`cdist-type__uci`\ (7), :strong:`cdist-type__uci_section`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 3468bfc3..86729316 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -3,21 +3,23 @@ cdist-type__uci_section(7)
 
 NAME
 ----
-cdist-type__uci_section - Manage configuration sections in OpenWrt's
-Unified Configuration Interface (UCI)
+cdist-type__uci_section - Manage configuration sections in UCI
 
 
 DESCRIPTION
 -----------
 This cdist type can be used to replace whole configuration sections in OpenWrt's
-UCI system.
-It can be thought of as syntactic sugar for `cdist-type__uci`\ (7), as this type
-will generate the required `__uci` objects to make the section contain exactly
-the options specified via ``--option``.
+Unified Configuration Interface (UCI) system.
+It can be thought of as syntactic sugar for :strong:`cdist-type__uci`\ (7),
+as this type will generate the required `__uci` objects to make the section
+contain exactly the options specified using ``--option``.
 
 Since many default UCI sections are unnamed, this type allows to find the
 matching section by one of its options using the ``--match`` parameter.
 
+**NOTE:** Options already present on the target and not listed in ``--option``
+or ``--list`` will be deleted.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -28,18 +30,18 @@ OPTIONAL PARAMETERS
 -------------------
 match
     Allows to find a section to "replace" through one of its parameters.
-    The value to this parameter is a ``<option>=<string>`` string.
+    The value to this parameter is a ``<option>=<value>`` string.
 option
     An option that should be present in the section.
     This parameter can be used multiple times to specify multiple options.
-    The value to this parameter is a ``<option>=<string>`` string.
+    The value to this parameter is a ``<option>=<value>`` string.
 
     Lists can be expressed by repeatedly using the same key.
 state
-    `present` or `absent`, defaults to `present`.
+    ``present`` or ``absent``, defaults to ``present``.
 transaction
     The name of the transaction this option belongs to.
-    The value will be forwarded to `cdist-type__uci`\ (7).
+    The value will be forwarded to :strong:`cdist-type__uci`\ (7).
 type
     The type of the section in the format: ``<config>.<section-type>``
 
@@ -54,13 +56,36 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # TODO
-    __uci_section ...
+    # Configure the dropbear daemon
+    __uci_section dropbear --type dropbear.dropbear --transaction sshd \
+        --match Port=22 --option Port=22 \
+        --option PasswordAuth=off \
+        --option RootPasswordAuth=off
+
+    # Block SSH access from the guest network
+    __uci_section firewall.block_ssh_from_guest --type firewall.rule \
+        --transaction fwrules \
+        --option name='Block-SSH-Access-from-Guest' \
+        --option src='guest' \
+        --option proto='tcp' \
+        --option dest_port='22' \
+        --option target='REJECT'
+
+    # Configure a Wi-Fi access point
+    __uci_section wireless.default_radio0 --type wireless.wifi-iface \
+        --transaction wifi \
+        --option device='radio0' \
+        --option mode='ap' \
+        --option network='wlan' \
+        --option ssid='mywifi' \
+        --option encryption="psk2' \
+        --option key='hunter2'
 
 
 SEE ALSO
 --------
-:strong:`cdist-type__uci`\ (7)
+- https://openwrt.org/docs/guide-user/base-system/uci
+- :strong:`cdist-type__uci`\ (7)
 
 
 AUTHORS

From 0840afce03c50824278d64cd204b86e79077b48e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 12:57:40 +0200
Subject: [PATCH 135/411] [type/__uci] Add --type parameter

---
 cdist/conf/type/__uci/gencode-local      |  54 ++++++++----
 cdist/conf/type/__uci/gencode-remote     | 101 +++++++++++++++++++++++
 cdist/conf/type/__uci/man.rst            |   9 +-
 cdist/conf/type/__uci/parameter/optional |   1 +
 4 files changed, 146 insertions(+), 19 deletions(-)
 create mode 100644 cdist/conf/type/__uci/gencode-remote

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index af91a698..34482a01 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -117,27 +117,45 @@ in
 			exit 0
 		fi
 
-		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
-		then
-			# "should" is a list
-			printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+		# Determine type
+		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
+		case ${type}
+		in
+			(option|list) ;;
+			('')
+				# Guess type by the number of values
+				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
+					 && type=list \
+					 || type=option
+				;;
+			(*)
+				printf 'Invalid --type: %s\n' "${type}" >&2
+				exit 1
+				;;
+		esac
 
-			if test "${state_is}" != 'absent'
-			then
-				uci_cmd delete "${config}"
-			fi
+		case ${type}
+		in
+			(list)
+				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
 
-			while read -r value
-			do
-				uci_cmd add_list "${config}"="${value}"
-			done <"${__object:?}/parameter/value"
-		else
-			# "should" is a scalar
-			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+				if test "${state_is}" != 'absent'
+				then
+					uci_cmd delete "${config}"
+				fi
 
-			value=$(cat "${__object:?}/parameter/value")
-			uci_cmd set "${config}"="${value}"
-		fi
+				while read -r value
+				do
+					uci_cmd add_list "${config}"="${value}"
+				done <"${__object:?}/parameter/value"
+				;;
+			(option)
+				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
+				value=$(cat "${__object:?}/parameter/value")
+				uci_cmd set "${config}"="${value}"
+				;;
+		esac
 		;;
 	(absent)
 		if in_list "${state_is}" 'absent'
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
new file mode 100644
index 00000000..348d15a1
--- /dev/null
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -0,0 +1,101 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+config=${__object_id:?}
+uci_validate_tuple "${config}"
+
+
+case ${state_should}
+in
+	(present)
+		if in_list "${state_is}" 'present' 'rearranged'
+		then
+			# NOTE: order is ignored so rearranged is also fine.
+			exit 0
+		fi
+
+		# Determine type
+		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
+		case ${type}
+		in
+			(option|list) ;;
+			('')
+				# Guess type by the number of values
+				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
+					 && type=list \
+					 || type=option
+				;;
+			(*)
+				printf 'Invalid --type: %s\n' "${type}" >&2
+				exit 1
+				;;
+		esac
+
+		case ${type}
+		in
+			(list)
+				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+
+				if test "${state_is}" != 'absent'
+				then
+					uci_cmd delete "${config}"
+				fi
+
+				while read -r value
+				do
+					uci_cmd add_list "${config}"="${value}"
+				done <"${__object:?}/parameter/value"
+				;;
+			(option)
+				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
+				value=$(cat "${__object:?}/parameter/value")
+				uci_cmd set "${config}"="${value}"
+				;;
+		esac
+		;;
+	(absent)
+		if in_list "${state_is}" 'absent'
+		then
+			exit 0
+		fi
+
+		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
+		uci_cmd delete "${config}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index cf184f9d..9d33d618 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -31,6 +31,10 @@ state
 transaction
     The name of the transaction this option belongs to.
     If none is given: ``default`` is used.
+type
+    If the type should generate an option or a list.
+    One of: ``option`` or ``list``.
+    Defaults to auto-detect based on the number of ``--value`` parameters.
 
 
 BOOLEAN PARAMETERS
@@ -46,10 +50,13 @@ EXAMPLES
     # Set the system hostname
     __uci system.@system[0].hostname --value 'OpenWrt'
 
+    # Set DHCP option 252: tell DHCP clients to not ask for proxy information.
+    __uci dhcp.lan.dhcp_option --type list --value '252,"\n"'
+
     # Enable NTP and NTPd (in one transaction)
     __uci system.ntp.enabled --value 1 --transaction ntp
     __uci system.ntp.enable_server --value 1 --transaction ntp
-    __uci system.ntp.server --transaction ntp \
+    __uci system.ntp.server --type list --transaction ntp \
         --value '0.openwrt.pool.ntp.org' \
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
index ddbbba16..2703b3df 100644
--- a/cdist/conf/type/__uci/parameter/optional
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -1,2 +1,3 @@
 state
 transaction
+type

From 49e867fab4c1bcb6549079e735abec2149e42cf6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 15:59:25 +0200
Subject: [PATCH 136/411] [type/__uci_section] Add more parameter checks

---
 cdist/conf/type/__uci_section/manifest | 42 +++++++++++++++++++-------
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 1a59b970..b5fa4a3b 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -19,11 +19,15 @@
 #
 
 grep_line() { echo "$2" | grep -qxF "$1"; }
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
+}
+validate_options() { grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"; }
 unquote_lines() {
 	sed -e '/^".*"$/{s/^"//;s/"$//}' \
 	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
 }
-
 append_values() {
 	while read -r _value
 	do
@@ -32,12 +36,23 @@ append_values() {
 	unset _value
 	"$@" </dev/null
 }
-
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+print_errors() {
+	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
+		BEGIN {
+			if (getline) {
+				print prefix
+				print
+				rc = 1
+			}
+		}
+		{ print }
+		END {
+			if (rc && suffix) print suffix
+			exit rc
+		}' >&2
 }
 
+
 ## Check section name and error if invalid!
 case ${__object_id:?}
 in
@@ -74,12 +89,6 @@ in
 		type_is=$(cat "${__object:?}/explorer/type")
 		type_should=$(cat "${__object:?}/parameter/type")
 
-		if test -f "${__object:?}/parameter/option"
-		then
-			optnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
-		fi
-
 		if test -n "${type_is}"
 		then
 			if test "${type_is}" != "${type_should##*.}"
@@ -105,6 +114,17 @@ in
 			esac
 		fi
 
+		# Check options for syntax errors
+		validate_options "${__object:?}/parameter/object" \
+		| print_errors 'Found erroneous options in arguments:'
+
+		# Collect option names
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
 		# Make sure the section itself is present
 		__uci "${section}" --state present --transaction "${transaction_name}" \
 			--value "${sect_type}"

From b99ca3cbdf7c761db05598e31e9788e5e4e81cc6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 16:01:02 +0200
Subject: [PATCH 137/411] [type/__uci_section] Split up --option and --list

---
 cdist/conf/type/__uci_section/man.rst         | 19 +++++++-
 cdist/conf/type/__uci_section/manifest        | 48 ++++++++++++-------
 .../__uci_section/parameter/optional_multiple |  1 +
 3 files changed, 50 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 86729316..b8829433 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -28,15 +28,21 @@ None.
 
 OPTIONAL PARAMETERS
 -------------------
+list
+    An option that is part of a list and should be present in the section (as
+    part of a list).  Lists with multiple options can be expressed by using the
+    same ``<option>`` repeatedly.
+
+    The value to this parameter is a ``<option>=<value>`` string.
 match
     Allows to find a section to "replace" through one of its parameters.
+
     The value to this parameter is a ``<option>=<value>`` string.
 option
     An option that should be present in the section.
     This parameter can be used multiple times to specify multiple options.
-    The value to this parameter is a ``<option>=<value>`` string.
 
-    Lists can be expressed by repeatedly using the same key.
+    The value to this parameter is a ``<option>=<value>`` string.
 state
     ``present`` or ``absent``, defaults to ``present``.
 transaction
@@ -62,6 +68,15 @@ EXAMPLES
         --option PasswordAuth=off \
         --option RootPasswordAuth=off
 
+    # Define a firewall zone comprised of lan and wlan networks
+    __uci_section firewall.internal --type firewall.zone --transaction fw \
+        --option name='internal' \
+        --list network='lan' \
+        --list network='wlan' \
+        --option input='ACCEPT' \
+        --option output='ACCEPT' \
+        --option forward='ACCEPT'
+
     # Block SSH access from the guest network
     __uci_section firewall.block_ssh_from_guest --type firewall.rule \
         --transaction fwrules \
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index b5fa4a3b..a7865315 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,7 +18,14 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-grep_line() { echo "$2" | grep -qxF "$1"; }
+grep_line() { { shift; printf '%s\n' "$@"; } | grep -qxF "$1"; }
+prefix_lines() {
+	while test $# -gt 0
+	do
+		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
+		shift; shift
+	done
+}
 uci_validate_name() {
 	# like util.c uci_validate_name()
 	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
@@ -115,14 +122,25 @@ in
 		fi
 
 		# Check options for syntax errors
-		validate_options "${__object:?}/parameter/object" \
+		validate_options "${__object:?}/parameter/list" "${__object:?}/parameter/object" \
 		| print_errors 'Found erroneous options in arguments:'
 
 		# Collect option names
+		if test -f "${__object:?}/parameter/list"
+		then
+			listnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
+		fi
+
 		if test -f "${__object:?}/parameter/option"
 		then
-			optnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+			optnames_should=$(sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort)
+
+			echo "${optnames_should}" \
+			| uniq -d \
+			| print_errors \
+				  'Found duplicate --options:' \
+				  "$(printf '\nUse --list for lists, instead.')"
 		fi
 
 		# Make sure the section itself is present
@@ -130,29 +148,27 @@ in
 			--value "${sect_type}"
 		export require=__uci/"${section}"
 
-		# Delete options not in "should"
+		# Delete options/lists not in "should"
 		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
 		| while read -r optname
 		  do
-			  if ! grep_line "${optname}" "${optnames_should}"
-			  then
-				  __uci "${section}.${optname}" --state absent \
-					  --transaction "${transaction_name}" </dev/null
-			  fi
+			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
+				  __uci "${section}.${optname}" --state absent --transaction "${transaction_name}"
+			  } </dev/null
 		  done
 
 		# Set "should" options
-		echo "${optnames_should}" \
-		| while read -r optname
+		prefix_lines option "${optnames_should}" list "${listnames_should}" \
+		| while read -r _type _optname
 		  do
-			  test -n "${optname}" || continue  # ignore empty lines
+			  test -n "${_type}${_optname}" || continue  # ignore empty lines
 
-			  grep "^${optname}=" <"${__object:?}/parameter/option" \
+			  grep "^${_optname}=" <"${__object:?}/parameter/${_type}" \
 			  | sed -e 's/^.*=//' \
 			  | unquote_lines \
 			  | append_values \
-				    __uci "${section}.${optname}" --state present \
-					    --transaction "${transaction_name}"
+				    __uci "${section}.${_optname}" --state present \
+					    --type "${_type}" --transaction "${transaction_name}"
 		  done
 		;;
 	(absent)
diff --git a/cdist/conf/type/__uci_section/parameter/optional_multiple b/cdist/conf/type/__uci_section/parameter/optional_multiple
index 01925a15..98af35e6 100644
--- a/cdist/conf/type/__uci_section/parameter/optional_multiple
+++ b/cdist/conf/type/__uci_section/parameter/optional_multiple
@@ -1 +1,2 @@
+list
 option

From 8728817af6ae1fa801a39456df3f84d420f87744 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 21:11:50 +0200
Subject: [PATCH 138/411] [type/__uci] Unquote UCI reported values

Without unquoting values printed in single quotes by UCI would always lead to
the state explorer reporting "different".
---
 cdist/conf/type/__uci/explorer/state | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
index 6fb4b173..d7363dbf 100644
--- a/cdist/conf/type/__uci/explorer/state
+++ b/cdist/conf/type/__uci/explorer/state
@@ -51,21 +51,34 @@ fi
 # strip off trailing newline
 printf '%s' "${values_is}" \
 | awk '
+function unquote(s) {
+	# simplified dequoting of single quoted strings
+	if (s ~ /^'\''.*'\''$/) {
+		s = substr(s, 2, length(s) - 2)
+		sub(/'"'\\\\''"'/, "'\''", s)
+	}
+	return s
+}
+
 BEGIN {
 	state = "present"  # assume all is fine
 }
 NR == FNR {
 	# memoize "should" state
 	should[FNR] = $0
+	should_count++
 
 	# go to next line (important!)
 	next
 }
 
 # compare "is" state
+
+{ $0 = unquote($0) }
+
 $0 == should[FNR] { next }
 
-FNR > length(should) {
+FNR > should_count {
 	# there are more "is" records than "should" -> definitely different
 	state = "different"
 	exit
@@ -87,7 +100,7 @@ FNR > length(should) {
 }
 
 END {
-	if (FNR < length(should)) {
+	if (FNR < should_count) {
 		# "is" was shorter than "should" -> different
 		state = "different"
 	}

From 4aebb1f12756593408d6f012adb923e4c9b81e1e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 21:18:22 +0200
Subject: [PATCH 139/411] [type/__uci*] Update man.rst regarding quoting
 requirements

---
 cdist/conf/type/__uci/man.rst         | 2 ++
 cdist/conf/type/__uci_section/man.rst | 8 ++++++++
 2 files changed, 10 insertions(+)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 9d33d618..c7f5f4f0 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -23,6 +23,8 @@ value
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
 
+    Values do not need special quoting for UCI. The only requirement is that the
+    value is passed to the type as a single shell argument.
 
 OPTIONAL PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index b8829433..753d00b3 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -34,6 +34,10 @@ list
     same ``<option>`` repeatedly.
 
     The value to this parameter is a ``<option>=<value>`` string.
+
+    ``<value>`` does not need special quoting for UCI.
+    The only requirement is that the value is passed to the type as a single
+    shell argument.
 match
     Allows to find a section to "replace" through one of its parameters.
 
@@ -43,6 +47,10 @@ option
     This parameter can be used multiple times to specify multiple options.
 
     The value to this parameter is a ``<option>=<value>`` string.
+
+    ``<value>`` does not need special quoting for UCI.
+    The only requirement is that the value is passed to the type as a single
+    shell argument.
 state
     ``present`` or ``absent``, defaults to ``present``.
 transaction

From 63d41a1053bd8b03ddda130d7129d233f97ac671 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 21:10:27 +0100
Subject: [PATCH 140/411] [type/__uci_section] Improve --match support with
 existing named sections

Use section if named section exists without --match option (e.g. empty section).
---
 cdist/conf/type/__uci_section/explorer/match | 80 +++++++++++++-------
 1 file changed, 51 insertions(+), 29 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index a5619d4e..0768e404 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -20,62 +20,84 @@
 # This explorer determines the "prefix" of the --type section matching --match
 # if set, or __object_id otherwise.
 
+RS=$(printf '\036')
+NL=$(printf '\n '); NL=${NL% }
+
 squote_values() {
 	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
 	    -e "/='.*'$/"'!{s/=/='\''/;s/$/'\''/}'
 }
+count_lines() (
+	IFS=${NL?}
+	# shellcheck disable=SC2048,SC2086
+	set -f -- $*; echo $#
+)
 
-RS=$(printf '\036')
-
-if ! test -e "${__object:?}/parameter/match"
-then
-	if ! echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$'
-	then
-		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
-		exit 1
-	fi
+echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' || {
+	echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
+	exit 1
+}
 
+test -s "${__object:?}/parameter/match" || {
 	# If no --match is given, we take the __object_id as the section identifier.
 	echo "${__object_id:?}"
 	exit 0
-fi
-
-# shellcheck disable=SC2015
-test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type" \
-|| {
+}
+test -s "${__object:?}/parameter/type" || {
 	echo 'Parameters --match and --type must be used together.' >&2
 	exit 1
 }
 
-# Find by match
-# NOTE: Apart from section types all values are printed in single quotes by UCI.
-match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
-
-if ! grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' "${__object:?}/parameter/type"
-then
+sect_type_param=$(cat "${__object:?}/parameter/type")
+expr "${sect_type_param}" : '[^.]\{1,\}\.[^.]\{1,\}$' >/dev/null 2>&1 || {
 	echo 'Section types are a package name and section type separated by a "." (period).' >&2
 	exit 1
+}
+package_filter=${sect_type_param%%.*}
+section_filter=${sect_type_param#*.}
+
+# Find by --match
+# NOTE: Apart from section types all values are printed in single quotes by uci show.
+match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
+
+if uci -s -N get "${__object_id:?}" >/dev/null 2>&1
+then
+	# Named section exists: ensure if --match applies to it
+	# if the "matched" option does not exist (e.g. empty section) we use the
+	# section unconditionally.
+	if match_value_is=$(uci -s -N get "${__object_id:?}.${match%%=*}" 2>/dev/null)
+	then
+		match_value_should=$(expr "${match}" : ".*='\\(.*\\)'$")
+
+		test "${match_value_is}" = "${match_value_should}" || {
+			printf 'Named section "%s" does not match --match "%s"\n' \
+				   "${__object_id:?}" "${match}" >&2
+			exit 1
+		}
+	fi
+
+	echo "${__object_id:?}"
+	exit 0
 fi
 
-sect_type_filter=$(cat "${__object:?}/parameter/type")
-package_filter=${sect_type_filter%%.*}
-section_filter=${sect_type_filter#*.}
-# FIXME: Does not work with named sections
-regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
+# No correctly named section exists already: find one to which --match applies
+regex="^${package_filter}\\.@${section_filter}\\[[0-9]\\{1,\\}\\]\\.${match%%=*}="
 
 matched_sections=$(
 	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
 	| grep -e "${regex}" \
 	| while read -r _line
 	  do
-		  if test "${_line#*=}" = "${match#*=}"; then echo "${_line}"; fi
+		  if test "${_line#*=}" = "${match#*=}"
+		  then
+			  echo "${_line}"
+		  fi
 	  done \
 	| sed -e 's/\.[^.]*=.*$//')
 
-if test "$(echo "${matched_sections}" | wc -l)" -gt 1
-then
+test "$(count_lines "${matched_sections}")" -le 1 || {
 	printf 'Found multiple matching sections:\n%s\n' "${matched_sections}" >&2
 	exit 1
-fi
+}
 
 echo "${matched_sections}"

From 7b301195040c4861dbd15c5f2514c4e9eeadbe61 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Oct 2020 18:52:43 +0100
Subject: [PATCH 141/411] [type/__uci] Externalise functions to separate file

---
 cdist/conf/type/__uci/files/functions.sh | 75 +++++++++++++++++++++
 cdist/conf/type/__uci/gencode-local      | 85 ++----------------------
 cdist/conf/type/__uci/manifest           |  4 +-
 3 files changed, 85 insertions(+), 79 deletions(-)
 create mode 100644 cdist/conf/type/__uci/files/functions.sh

diff --git a/cdist/conf/type/__uci/files/functions.sh b/cdist/conf/type/__uci/files/functions.sh
new file mode 100644
index 00000000..4ddb82e0
--- /dev/null
+++ b/cdist/conf/type/__uci/files/functions.sh
@@ -0,0 +1,75 @@
+# -*- mode: sh; indent-tabs-mode: t -*-
+
+in_list() {
+	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
+}
+
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
+
+uci_cmd() {
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	# Will output a shell command to be executed locally to add the given
+	# command to $tmpfile.
+	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
+	printf ' >>%s\n' "$(quote "${tmpfile}")"
+}
+
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+uci_validate_tuple() (
+	tok=${1:?}
+	case $tok
+	in
+		(*.*.*)
+			# check option
+			option=${tok##*.}
+			uci_validate_name "${option}" || {
+				printf 'Invalid option: %s\n' "${option}" >&2
+				return 1
+			}
+			tok=${tok%.*}
+			;;
+		(*.*)
+			# no option (section definition)
+			;;
+		(*)
+			printf 'Invalid tuple: %s\n' "$1" >&2
+			return 1
+			;;
+	esac
+
+	case ${tok#*.}
+	in
+		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
+		(*)  section=${tok#*.} ;;
+	esac
+	uci_validate_name "${section}" || {
+		printf 'Invalid section: %s\n' "${1#*.}" >&2
+		return 1
+	}
+
+	config=${tok%%.*}
+	uci_validate_name "${config}" || {
+		printf 'Invalid config: %s\n' "${config}" >&2
+		return 1
+	}
+)
diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index 34482a01..b7664e85 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -18,84 +18,6 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() {
-	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
-}
-
-quote() {
-	for _arg
-	do
-		shift
-		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
-		then
-			# needs quoting
-			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
-		else
-			set -- "$@" "${_arg}"
-		fi
-	done
-	unset _arg
-
-	# NOTE: Use printf because POSIX echo interprets escape sequences
-	printf '%s' "$*"
-}
-
-uci_cmd() {
-	# Usage: uci_cmd [UCI ARGUMENTS]...
-	# Will output a shell command to be executed locally to add the given
-	# command to $tmpfile.
-	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
-	printf ' >>%s\n' "$(quote "${tmpfile}")"
-}
-
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
-}
-
-uci_validate_tuple() (
-	tok=${1:?}
-	case $tok
-	in
-		(*.*.*)
-			# check option
-			option=${tok##*.}
-			uci_validate_name "${option}" || {
-				printf 'Invalid option: %s\n' "${option}" >&2
-				return 1
-			}
-			tok=${tok%.*}
-			;;
-		(*.*)
-			# no option (section definition)
-			;;
-		(*)
-			printf 'Invalid tuple: %s\n' "$1" >&2
-			return 1
-			;;
-	esac
-
-	case ${tok#*.}
-	in
-		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
-		(*)  section=${tok#*.} ;;
-	esac
-	uci_validate_name "${section}" || {
-		printf 'Invalid section: %s\n' "${1#*.}" >&2
-		return 1
-	}
-
-	config=${tok%%.*}
-	uci_validate_name "${config}" || {
-		printf 'Invalid config: %s\n' "${config}" >&2
-		return 1
-	}
-)
-
-
-config=${__object_id:?}
-uci_validate_tuple "${config}"
-
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
@@ -107,6 +29,13 @@ mkdir -p "${tmpdir}"
 
 tmpfile="${tmpdir}/${transaction_name}.txt"
 
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+
+config=${__object_id:?}
+uci_validate_tuple "${config}"
+
 
 case ${state_should}
 in
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index e56462e5..d0c8239e 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,7 +18,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; } }
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
 
 os=$(cat "${__global:?}/explorer/os")
 

From a6c37095f1eea72acbebebfa88bdf84f65f61f51 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Oct 2020 18:50:34 +0100
Subject: [PATCH 142/411] [type/__uci_section] Externalise functions to
 separate file

---
 .../type/__uci_section/files/functions.sh     | 52 +++++++++++++++++++
 cdist/conf/type/__uci_section/manifest        | 42 +--------------
 2 files changed, 54 insertions(+), 40 deletions(-)
 create mode 100644 cdist/conf/type/__uci_section/files/functions.sh

diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
new file mode 100644
index 00000000..9f510958
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -0,0 +1,52 @@
+# -*- mode: sh; indent-tabs-mode: t -*-
+
+append_values() {
+	while read -r _value
+	do
+		set -- "$@" --value "${_value}"
+	done
+	unset _value
+	"$@" </dev/null
+}
+
+grep_line() {
+	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
+}
+
+prefix_lines() {
+	while test $# -gt 0
+	do
+		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
+		shift; shift
+	done
+}
+
+print_errors() {
+	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
+		BEGIN {
+			if (getline) {
+				print prefix
+				print
+				rc = 1
+			}
+		}
+		{ print }
+		END {
+			if (rc && suffix) print suffix
+			exit rc
+		}' >&2
+}
+
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
+}
+
+unquote_lines() {
+	sed -e '/^".*"$/{s/^"//;s/"$//}' \
+	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
+}
+
+validate_options() {
+	grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"
+}
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index a7865315..208a2c4e 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,46 +18,8 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-grep_line() { { shift; printf '%s\n' "$@"; } | grep -qxF "$1"; }
-prefix_lines() {
-	while test $# -gt 0
-	do
-		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
-		shift; shift
-	done
-}
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
-}
-validate_options() { grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"; }
-unquote_lines() {
-	sed -e '/^".*"$/{s/^"//;s/"$//}' \
-	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
-}
-append_values() {
-	while read -r _value
-	do
-		set -- "$@" --value "${_value}"
-	done
-	unset _value
-	"$@" </dev/null
-}
-print_errors() {
-	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
-		BEGIN {
-			if (getline) {
-				print prefix
-				print
-				rc = 1
-			}
-		}
-		{ print }
-		END {
-			if (rc && suffix) print suffix
-			exit rc
-		}' >&2
-}
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
 
 
 ## Check section name and error if invalid!

From c1ae3ccb2f725d0d94eaec9eeae7c8f5f98a5e45 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:13:06 +0100
Subject: [PATCH 143/411] [type/__uci*] Remove public-facing transaction
 "interface"

---
 cdist/conf/type/__uci/man.rst          | 13 ++++---------
 cdist/conf/type/__uci_section/man.rst  |  9 ++-------
 cdist/conf/type/__uci_section/manifest | 10 ++++------
 3 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index c7f5f4f0..8c3f79f1 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -11,8 +11,6 @@ DESCRIPTION
 This cdist type can be used to alter configuration options in OpenWrt's
 Unified Configuration Interface (UCI) system.
 
-Options can be applied in batches if the ``--transaction`` parameter is used.
-
 
 REQUIRED PARAMETERS
 -------------------
@@ -30,9 +28,6 @@ OPTIONAL PARAMETERS
 -------------------
 state
     ``present`` or ``absent``, defaults to ``present``.
-transaction
-    The name of the transaction this option belongs to.
-    If none is given: ``default`` is used.
 type
     If the type should generate an option or a list.
     One of: ``option`` or ``list``.
@@ -55,10 +50,10 @@ EXAMPLES
     # Set DHCP option 252: tell DHCP clients to not ask for proxy information.
     __uci dhcp.lan.dhcp_option --type list --value '252,"\n"'
 
-    # Enable NTP and NTPd (in one transaction)
-    __uci system.ntp.enabled --value 1 --transaction ntp
-    __uci system.ntp.enable_server --value 1 --transaction ntp
-    __uci system.ntp.server --type list --transaction ntp \
+    # Enable NTP and NTPd (each is applied individually)
+    __uci system.ntp.enabled --value 1
+    __uci system.ntp.enable_server --value 1
+    __uci system.ntp.server --type list \
         --value '0.openwrt.pool.ntp.org' \
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 753d00b3..a0ab78e8 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -53,9 +53,6 @@ option
     shell argument.
 state
     ``present`` or ``absent``, defaults to ``present``.
-transaction
-    The name of the transaction this option belongs to.
-    The value will be forwarded to :strong:`cdist-type__uci`\ (7).
 type
     The type of the section in the format: ``<config>.<section-type>``
 
@@ -71,13 +68,13 @@ EXAMPLES
 .. code-block:: sh
 
     # Configure the dropbear daemon
-    __uci_section dropbear --type dropbear.dropbear --transaction sshd \
+    __uci_section dropbear --type dropbear.dropbear \
         --match Port=22 --option Port=22 \
         --option PasswordAuth=off \
         --option RootPasswordAuth=off
 
     # Define a firewall zone comprised of lan and wlan networks
-    __uci_section firewall.internal --type firewall.zone --transaction fw \
+    __uci_section firewall.internal --type firewall.zone \
         --option name='internal' \
         --list network='lan' \
         --list network='wlan' \
@@ -87,7 +84,6 @@ EXAMPLES
 
     # Block SSH access from the guest network
     __uci_section firewall.block_ssh_from_guest --type firewall.rule \
-        --transaction fwrules \
         --option name='Block-SSH-Access-from-Guest' \
         --option src='guest' \
         --option proto='tcp' \
@@ -96,7 +92,6 @@ EXAMPLES
 
     # Configure a Wi-Fi access point
     __uci_section wireless.default_radio0 --type wireless.wifi-iface \
-        --transaction wifi \
         --option device='radio0' \
         --option mode='ap' \
         --option network='wlan' \
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 208a2c4e..41e30a07 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -46,7 +46,6 @@ esac
 section=$(cat "${__object:?}/explorer/match")
 
 state_should=$(cat "${__object:?}/parameter/state")
-transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case $state_should
 in
@@ -106,8 +105,7 @@ in
 		fi
 
 		# Make sure the section itself is present
-		__uci "${section}" --state present --transaction "${transaction_name}" \
-			--value "${sect_type}"
+		__uci "${section}" --state present --value "${sect_type}"
 		export require=__uci/"${section}"
 
 		# Delete options/lists not in "should"
@@ -115,7 +113,7 @@ in
 		| while read -r optname
 		  do
 			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
-				  __uci "${section}.${optname}" --state absent --transaction "${transaction_name}"
+				  __uci "${section}.${optname}" --state absent
 			  } </dev/null
 		  done
 
@@ -130,14 +128,14 @@ in
 			  | unquote_lines \
 			  | append_values \
 				    __uci "${section}.${_optname}" --state present \
-					    --type "${_type}" --transaction "${transaction_name}"
+					    --type "${_type}"
 		  done
 		;;
 	(absent)
 		# if explorer found no section there is nothing to delete
 		test -n "${section}" || exit 0
 
-		__uci "${section}" --state absent --transaction "${transaction_name}"
+		__uci "${section}" --state absent
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2

From e264fb004f797efc540f74191a0e9630c66dd819 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:27:07 +0100
Subject: [PATCH 144/411] [type/__uci] Convert to immediate remote execution

---
 cdist/conf/type/__uci/files/functions.sh |   6 +-
 cdist/conf/type/__uci/gencode-local      | 102 -----------------------
 cdist/conf/type/__uci/gencode-remote     |   8 --
 3 files changed, 2 insertions(+), 114 deletions(-)
 delete mode 100755 cdist/conf/type/__uci/gencode-local
 mode change 100644 => 100755 cdist/conf/type/__uci/gencode-remote

diff --git a/cdist/conf/type/__uci/files/functions.sh b/cdist/conf/type/__uci/files/functions.sh
index 4ddb82e0..277f648c 100644
--- a/cdist/conf/type/__uci/files/functions.sh
+++ b/cdist/conf/type/__uci/files/functions.sh
@@ -24,10 +24,8 @@ quote() {
 
 uci_cmd() {
 	# Usage: uci_cmd [UCI ARGUMENTS]...
-	# Will output a shell command to be executed locally to add the given
-	# command to $tmpfile.
-	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
-	printf ' >>%s\n' "$(quote "${tmpfile}")"
+	mkdir -p "${__object:?}/files"
+	printf '%s\n' "$(quote "$@")" >>"${__object:?}/files/uci_batch.txt"
 }
 
 uci_validate_name() {
diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
deleted file mode 100755
index b7664e85..00000000
--- a/cdist/conf/type/__uci/gencode-local
+++ /dev/null
@@ -1,102 +0,0 @@
-#!/bin/sh -e
-#
-# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-state_is=$(cat "${__object:?}/explorer/state")
-state_should=$(cat "${__object:?}/parameter/state")
-
-transaction_name=$(cat "${__object:?}/parameter/transaction")
-
-tmpdir="${__global:?}/tmp/__uci"
-# HACK
-mkdir -p "${tmpdir}"
-
-tmpfile="${tmpdir}/${transaction_name}.txt"
-
-# shellcheck source=files/functions.sh
-. "${__type:?}/files/functions.sh"
-
-
-config=${__object_id:?}
-uci_validate_tuple "${config}"
-
-
-case ${state_should}
-in
-	(present)
-		if in_list "${state_is}" 'present' 'rearranged'
-		then
-			# NOTE: order is ignored so rearranged is also fine.
-			exit 0
-		fi
-
-		# Determine type
-		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
-		case ${type}
-		in
-			(option|list) ;;
-			('')
-				# Guess type by the number of values
-				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
-					 && type=list \
-					 || type=option
-				;;
-			(*)
-				printf 'Invalid --type: %s\n' "${type}" >&2
-				exit 1
-				;;
-		esac
-
-		case ${type}
-		in
-			(list)
-				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
-
-				if test "${state_is}" != 'absent'
-				then
-					uci_cmd delete "${config}"
-				fi
-
-				while read -r value
-				do
-					uci_cmd add_list "${config}"="${value}"
-				done <"${__object:?}/parameter/value"
-				;;
-			(option)
-				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
-
-				value=$(cat "${__object:?}/parameter/value")
-				uci_cmd set "${config}"="${value}"
-				;;
-		esac
-		;;
-	(absent)
-		if in_list "${state_is}" 'absent'
-		then
-			exit 0
-		fi
-
-		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
-		uci_cmd delete "${config}"
-		;;
-	(*)
-		printf 'Invalid --state: %s\n' "${state_should}" >&2
-		exit 1
-		;;
-esac
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
old mode 100644
new mode 100755
index 348d15a1..a15f50a6
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -91,11 +91,3 @@ in
 		exit 1
 		;;
 esac
-
-if test -s "${__object:?}/files/uci_batch.txt"
-then
-	cat "${__type:?}/files/uci_apply.sh"
-	printf "uci_apply <<'EOF'\n"
-	cat "${__object:?}/files/uci_batch.txt"
-	printf '\nEOF\n'
-fi

From dfe9e08c280a2a3bc0ded15069b7d60df5a757f8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:06:05 +0100
Subject: [PATCH 145/411] [type/__uci_commit] Delete type

---
 cdist/conf/type/__uci/man.rst               |  2 -
 cdist/conf/type/__uci/manifest              | 18 -----
 cdist/conf/type/__uci_commit/gencode-remote | 81 ---------------------
 cdist/conf/type/__uci_commit/man.rst        | 60 ---------------
 cdist/conf/type/__uci_commit/nonparallel    |  0
 5 files changed, 161 deletions(-)
 delete mode 100755 cdist/conf/type/__uci_commit/gencode-remote
 delete mode 100644 cdist/conf/type/__uci_commit/man.rst
 delete mode 100644 cdist/conf/type/__uci_commit/nonparallel

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 8c3f79f1..81a53473 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -58,13 +58,11 @@ EXAMPLES
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
         --value '3.openwrt.pool.ntp.org'
-    export require=__uci_commit/ntp
 
 
 SEE ALSO
 --------
 - https://openwrt.org/docs/guide-user/base-system/uci
-- :strong:`cdist-type__uci_commit`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index d0c8239e..26920011 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,15 +18,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
-. "${__type:?}/files/functions.sh"
-
-
 os=$(cat "${__global:?}/explorer/os")
 
-state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
-transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case ${os}
 in
@@ -40,8 +34,6 @@ in
 		;;
 esac
 
-changes_required=false
-
 case ${state_should}
 in
 	(present)
@@ -49,21 +41,11 @@ in
 			echo 'The parameter --value is required.' >&2
 			exit 1
 		}
-
-		# NOTE: order is ignored so rearranged is also fine.
-		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
 		;;
 	(absent)
-		in_list "${state_is}" 'absent' || changes_required=true
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
 		exit 1
 		;;
 esac
-
-if ${changes_required}
-then
-	# Make sure the changes are being committed
-	require=${__object_name:?} __uci_commit "${transaction_name}"
-fi
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
deleted file mode 100755
index e5c93966..00000000
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/bin/sh -e
-#
-# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-transaction_name=${__object_id:?}
-batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
-
-test -s "${batchfile}" || exit 0
-
-printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
-
-# NOTE: Uncommited changes are checked in code-remote instead of in an explorer
-#       because in cdist there is no interlocking between explorers and code
-#       execution.
-#       Checking for uncommited changes in an explorer leaves a time slot in
-#       which changes made are not detected by the code.
-#       Furthermore an explorer running concurrently with another transactions
-#       code-remote could lead to a false positive.
-
-cat <<CODE
-changes=\$(uci changes)
-
-if test -n "\${changes}"
-then
-	echo 'Uncommited UCI changes were found on the target:'
-	printf '%s\n\n' "\${changes}"
-	echo 'This can be caused by manual changes or due to a previous failed run.'
-	echo 'Please investigate the situation, revert or commit the changes, and try again.'
-	exit 1
-fi >&2
-
-check_errors() {
-	# reads stdin and forwards non-empty lines to stderr.
-	# returns 0 if stdin is empty, else 1.
-	! grep -e . >&2
-}
-
-commit() {
-	uci commit
-}
-
-rollback() {
-	echo >&2
-	echo 'An error occurred when trying to commit transaction ${transaction_name}!' >&2
-
-	uci changes \\
-	| sed -e 's/^-//' -e 's/\..*\$//' \\
-	| sort -u \\
-	| while read -r _package
-	  do
-		  uci revert "\${_package}"
-		  echo "\${_package}"  # for logging
-	  done \\
-	| awk '
-	  BEGIN { printf "Reverted changes in: " }
-	  { printf "%s%s", (FNR > 1 ? ", " : ""), \$0 }
-	  END { printf "\\n" }' >&2
-
-	return 1
-}
-
-uci batch <<'EOF' 2>&1 | check_errors && commit || rollback
-$(cat "${batchfile}")
-EOF
-CODE
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
deleted file mode 100644
index ea6611eb..00000000
--- a/cdist/conf/type/__uci_commit/man.rst
+++ /dev/null
@@ -1,60 +0,0 @@
-cdist-type__uci_commit(7)
-=========================
-
-NAME
-----
-cdist-type__uci_commit - Commit UCI transactions
-
-
-DESCRIPTION
------------
-This type executes the ``uci commit`` command on the target with the commands
-queued in a transaction.
-It is usually not required to use this type. Use the ``--transaction`` parameter
-of :strong:`cdist-type__uci`\ (7) and :strong:`cdist-type__uci_section`\ (7)
-instead.
-
-
-REQUIRED PARAMETERS
--------------------
-None.
-
-
-OPTIONAL PARAMETERS
--------------------
-None.
-
-
-BOOLEAN PARAMETERS
-------------------
-None.
-
-
-EXAMPLES
---------
-
-.. code-block:: sh
-
-    # Commit the default transaction
-    __uci_commit default
-
-    # Commit another transaction
-    __uci_commit my_transaction
-
-
-SEE ALSO
---------
-:strong:`cdist-type__uci`\ (7), :strong:`cdist-type__uci_section`\ (7)
-
-
-AUTHORS
--------
-Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
-
-
-COPYING
--------
-Copyright \(C) 2020 Dennis Camera. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_commit/nonparallel b/cdist/conf/type/__uci_commit/nonparallel
deleted file mode 100644
index e69de29b..00000000

From ec984f81b5a1db6b7f3e54255b570e07cf5282f9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:36:10 +0100
Subject: [PATCH 146/411] [type/__uci] Delete --transaction parameter

---
 cdist/conf/type/__uci/parameter/default/transaction | 1 -
 cdist/conf/type/__uci/parameter/optional            | 1 -
 2 files changed, 2 deletions(-)
 delete mode 100644 cdist/conf/type/__uci/parameter/default/transaction

diff --git a/cdist/conf/type/__uci/parameter/default/transaction b/cdist/conf/type/__uci/parameter/default/transaction
deleted file mode 100644
index 4ad96d51..00000000
--- a/cdist/conf/type/__uci/parameter/default/transaction
+++ /dev/null
@@ -1 +0,0 @@
-default
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
index 2703b3df..d9080e3a 100644
--- a/cdist/conf/type/__uci/parameter/optional
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -1,3 +1,2 @@
 state
-transaction
 type

From 3e5f18d409e448f69e4f259fc451d4a0e5a7a548 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 18:42:50 +0100
Subject: [PATCH 147/411] [type/__uci] Apply all commands in a single batch

---
 cdist/conf/type/__uci/files/uci_apply.sh | 43 ++++++++++++++++++++++++
 cdist/conf/type/__uci/gencode-remote     |  8 +++++
 cdist/conf/type/__uci/nonparallel        |  0
 3 files changed, 51 insertions(+)
 create mode 100644 cdist/conf/type/__uci/files/uci_apply.sh
 create mode 100644 cdist/conf/type/__uci/nonparallel

diff --git a/cdist/conf/type/__uci/files/uci_apply.sh b/cdist/conf/type/__uci/files/uci_apply.sh
new file mode 100644
index 00000000..63f94290
--- /dev/null
+++ b/cdist/conf/type/__uci/files/uci_apply.sh
@@ -0,0 +1,43 @@
+changes=$(uci changes)
+
+if test -n "${changes}"
+then
+	echo 'Uncommited UCI changes were found on the target:'
+	printf '%s\n\n' "${changes}"
+	echo 'This can be caused by manual changes or due to a previous failed run.'
+	echo 'Please investigate the situation, revert or commit the changes, and try again.'
+	exit 1
+fi >&2
+
+check_errors() {
+	# reads stdin and forwards non-empty lines to stderr.
+	# returns 0 if stdin is empty, else 1.
+	! grep -e . >&2
+}
+
+commit() {
+	uci commit
+}
+
+rollback() {
+	printf '\nAn error occurred when trying to commit UCI transaction!\n' >&2
+
+	uci changes \
+	| sed -e 's/^-//' -e 's/\..*\$//' \
+	| sort -u \
+	| while read -r _package
+	  do
+		  uci revert "${_package}"
+		  echo "${_package}"  # for logging
+	  done \
+	| awk '
+	  BEGIN { printf "Reverted changes in: " }
+	  { printf "%s%s", (FNR > 1 ? ", " : ""), $0 }
+	  END { printf "\n" }' >&2
+
+	return 1
+}
+
+uci_apply() {
+	uci batch 2>&1 | check_errors && commit || rollback
+}
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
index a15f50a6..348d15a1 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -91,3 +91,11 @@ in
 		exit 1
 		;;
 esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci/nonparallel b/cdist/conf/type/__uci/nonparallel
new file mode 100644
index 00000000..e69de29b

From 9d40500570875f29aa48784c6637683aa31afe4e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 20:40:29 +0100
Subject: [PATCH 148/411] [type/__uci_section] Apply all commands in a single
 batch

---
 .../type/__uci_section/files/functions.sh     |  31 +++-
 .../type/__uci_section/files/uci_apply.sh     |   1 +
 cdist/conf/type/__uci_section/gencode-remote  | 146 ++++++++++++++++++
 cdist/conf/type/__uci_section/manifest        |  76 ++-------
 cdist/conf/type/__uci_section/nonparallel     |   0
 5 files changed, 180 insertions(+), 74 deletions(-)
 create mode 120000 cdist/conf/type/__uci_section/files/uci_apply.sh
 create mode 100755 cdist/conf/type/__uci_section/gencode-remote
 create mode 100644 cdist/conf/type/__uci_section/nonparallel

diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
index 9f510958..17c64bbc 100644
--- a/cdist/conf/type/__uci_section/files/functions.sh
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -1,13 +1,6 @@
 # -*- mode: sh; indent-tabs-mode: t -*-
 
-append_values() {
-	while read -r _value
-	do
-		set -- "$@" --value "${_value}"
-	done
-	unset _value
-	"$@" </dev/null
-}
+NL=$(printf '\n '); NL=${NL% }
 
 grep_line() {
 	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
@@ -37,6 +30,28 @@ print_errors() {
 		}' >&2
 }
 
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+	printf '%s' "$*"
+}
+
+uci_cmd() {
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	mkdir -p "${__object:?}/files"
+	printf '%s\n' "$(quote "$@")" >>"${__object:?}/files/uci_batch.txt"
+}
+
 uci_validate_name() {
 	# like util.c uci_validate_name()
 	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
diff --git a/cdist/conf/type/__uci_section/files/uci_apply.sh b/cdist/conf/type/__uci_section/files/uci_apply.sh
new file mode 120000
index 00000000..4209151f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/uci_apply.sh
@@ -0,0 +1 @@
+../../__uci/files/uci_apply.sh
\ No newline at end of file
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
new file mode 100755
index 00000000..05bca0a7
--- /dev/null
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -0,0 +1,146 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+
+section=$(cat "${__object:?}/explorer/match")
+
+state_is=$(test -s "${__object:?}/explorer/type" && echo present || echo absent)
+state_should=$(cat "${__object:?}/parameter/state")
+
+case $state_should
+in
+	(present)
+		test -f "${__object:?}/parameter/type" || {
+			echo 'Parameter --type is required.' >&2
+			exit 1
+		}
+
+		type_is=$(cat "${__object:?}/explorer/type")
+		type_should=$(cat "${__object:?}/parameter/type")
+
+		if test -n "${type_is}"
+		then
+			sect_type=${type_is}
+		else
+			sect_type=${type_should##*.}
+		fi
+
+		if test -z "${section}"
+		then
+			# No section exists and --match was used.
+			# So we generate a new section identifier from $__object_id.
+			case ${__object_id:?}
+			in
+				(*.*) section=${__object_id:?} ;;
+				(*) section="${type_should%%.*}.${__object_id:?}" ;;
+			esac
+		fi
+
+		# Collect option names
+		if test -f "${__object:?}/parameter/list"
+		then
+			listnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
+		fi
+
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
+		# Make sure the section itself is present
+		if test "${state_is}" = absent
+		then
+			printf 'set %s\n' "${section}" >>"${__messages_out:?}"
+			# shellcheck disable=SC2140
+			uci_cmd set "${section}"="${sect_type}"
+		fi
+
+		# Delete options/lists not in "should"
+		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		| while read -r _optname
+		  do
+			  grep_line "${_optname}" "${listnames_should}" "${optnames_should}" || {
+				  printf 'delete %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+				  uci_cmd delete "${section}.${_optname}"
+			  } </dev/null
+		  done
+
+		# Set "should" options
+		prefix_lines option "${optnames_should}" list "${listnames_should}" \
+		| while read -r _type _optname
+		  do
+			  test -n "${_type}${_optname}" || continue  # ignore empty lines
+
+			  case $_type
+			  in
+				  (list)
+					  printf 'set_list %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  if grep -q -e "^${_optname}=" "${__object:?}/explorer/options"
+					  then
+						  uci_cmd delete "${section}.${_optname}"
+					  fi
+
+					  grep "^${_optname}=" "${__object:?}/parameter/list" \
+					  | sed -e 's/^.*=//' \
+					  | unquote_lines \
+					  | while read -r _value
+					    do
+						    # shellcheck disable=SC2140
+						    uci_cmd add_list "${section}.${_optname}"="${_value}"
+					    done
+					  ;;
+				  (option)
+					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  # shellcheck disable=SC2140
+					  uci_cmd set "${section}.${_optname}"="$(
+						  grep "^${_optname}=" "${__object:?}/parameter/option" \
+						  | sed -e 's/^.*=//' \
+						  | unquote_lines \
+						  | head -n 1)"
+					  ;;
+			  esac
+		  done
+		;;
+	(absent)
+		if test "${state_is}" = absent
+		then
+			# if explorer found no section there is nothing to delete
+			exit 0
+		fi
+
+		printf 'delete %s\n' "${section}" >>"${__messages_out:?}"
+		uci_cmd delete "${section}"
+		;;
+esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 41e30a07..7e23e443 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -43,8 +43,6 @@ in
 		;;
 esac
 
-section=$(cat "${__object:?}/explorer/match")
-
 state_should=$(cat "${__object:?}/parameter/state")
 
 case $state_should
@@ -57,85 +55,31 @@ in
 		type_is=$(cat "${__object:?}/explorer/type")
 		type_should=$(cat "${__object:?}/parameter/type")
 
-		if test -n "${type_is}"
+		if test -n "${type_is}" && test "${type_is}" != "${type_should##*.}"
 		then
-			if test "${type_is}" != "${type_should##*.}"
-			then
-				# Check if section type matches (section exists and --type provided)
-				printf 'Section type "%s" does not match --type "%s".\n' \
-					"${type_is}" "${type_should}" >&2
-				exit 1
-			fi
-			sect_type=${type_is}
-		else
-			sect_type=${type_should##*.}
-		fi
-
-		if test -z "${section}"
-		then
-			# No section exists and --match was used.
-			# So we generate a new section identifier from $__object_id.
-			case ${__object_id:?}
-			in
-				(*.*) section=${__object_id:?} ;;
-				(*) section="${type_should%%.*}.${__object_id:?}" ;;
-			esac
+			# Check if section type matches (section exists and --type provided)
+			printf 'Section type "%s" does not match --type "%s".\n' \
+				"${type_is}" "${type_should}" >&2
+			exit 1
 		fi
 
 		# Check options for syntax errors
 		validate_options "${__object:?}/parameter/list" "${__object:?}/parameter/object" \
 		| print_errors 'Found erroneous options in arguments:'
 
-		# Collect option names
-		if test -f "${__object:?}/parameter/list"
+		# Check for duplicate option names
+		if test -s "${__object:?}/parameter/option"
 		then
-			listnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
-		fi
-
-		if test -f "${__object:?}/parameter/option"
-		then
-			optnames_should=$(sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort)
-
-			echo "${optnames_should}" \
+			sed -e 's/=.*$//' "${__object:?}/parameter/option" \
+			| sort \
 			| uniq -d \
 			| print_errors \
 				  'Found duplicate --options:' \
 				  "$(printf '\nUse --list for lists, instead.')"
 		fi
-
-		# Make sure the section itself is present
-		__uci "${section}" --state present --value "${sect_type}"
-		export require=__uci/"${section}"
-
-		# Delete options/lists not in "should"
-		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
-		| while read -r optname
-		  do
-			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
-				  __uci "${section}.${optname}" --state absent
-			  } </dev/null
-		  done
-
-		# Set "should" options
-		prefix_lines option "${optnames_should}" list "${listnames_should}" \
-		| while read -r _type _optname
-		  do
-			  test -n "${_type}${_optname}" || continue  # ignore empty lines
-
-			  grep "^${_optname}=" <"${__object:?}/parameter/${_type}" \
-			  | sed -e 's/^.*=//' \
-			  | unquote_lines \
-			  | append_values \
-				    __uci "${section}.${_optname}" --state present \
-					    --type "${_type}"
-		  done
 		;;
 	(absent)
-		# if explorer found no section there is nothing to delete
-		test -n "${section}" || exit 0
-
-		__uci "${section}" --state absent
+		:
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
diff --git a/cdist/conf/type/__uci_section/nonparallel b/cdist/conf/type/__uci_section/nonparallel
new file mode 100644
index 00000000..e69de29b

From ade69729ddd64938ef69c08a522a85ec825628e9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 31 Oct 2020 18:18:15 +0100
Subject: [PATCH 149/411] [type/__uci_section] Only generate UCI commands if
 state differs

---
 .../conf/type/__uci_section/explorer/options  | 22 ++++-
 .../type/__uci_section/files/functions.sh     |  8 --
 .../type/__uci_section/files/option_state.awk | 91 +++++++++++++++++++
 cdist/conf/type/__uci_section/gencode-remote  | 68 ++++++++++----
 4 files changed, 160 insertions(+), 29 deletions(-)
 create mode 100644 cdist/conf/type/__uci_section/files/option_state.awk

diff --git a/cdist/conf/type/__uci_section/explorer/options b/cdist/conf/type/__uci_section/explorer/options
index 67b4f83f..e1e60668 100644
--- a/cdist/conf/type/__uci_section/explorer/options
+++ b/cdist/conf/type/__uci_section/explorer/options
@@ -25,4 +25,24 @@ section=$("${__type_explorer:?}/match")
 test -n "${section}" || exit 0
 
 uci -s -N -d "${RS}" show "${section}" 2>/dev/null \
-| grep -v -e "^${section}=" || true
+| awk -v VSEP="${RS}" '
+  {
+	  # Strip off the config and section parts
+	  is_opt = sub(/^([^.]*\.){2}/, "")
+
+	  if (!is_opt) {
+		  # this line represents the section -> skip
+		  next
+	  }
+
+	  if (index($0, VSEP)) {
+		  # Put values each on a line, like --option and --list parameters
+		  opt = substr($0, 1, index($0, "=") - 1)
+		  split(substr($0, length(opt) + 2), values, VSEP)
+		  for (i in values) {
+			  printf "%s=%s\n", opt, values[i]
+		  }
+	  } else {
+		  print
+	  }
+  }'
diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
index 17c64bbc..60cb9148 100644
--- a/cdist/conf/type/__uci_section/files/functions.sh
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -6,14 +6,6 @@ grep_line() {
 	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
 }
 
-prefix_lines() {
-	while test $# -gt 0
-	do
-		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
-		shift; shift
-	done
-}
-
 print_errors() {
 	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
 		BEGIN {
diff --git a/cdist/conf/type/__uci_section/files/option_state.awk b/cdist/conf/type/__uci_section/files/option_state.awk
new file mode 100644
index 00000000..97cd94fb
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/option_state.awk
@@ -0,0 +1,91 @@
+# -*- mode: awk; indent-tabs-mode:t -*-
+# Usage: awk -f option_state.awk option_type option_name
+# e.g. awk -f option_state.awk option title
+#      awk -f option_state.awk list entry
+
+function unquote(s) {
+	# simplified dequoting of single quoted strings
+	if (s ~ /^'.*'$/) {
+		s = substr(s, 2, length(s) - 2)
+		sub(/'\\''/, "'", s)
+	}
+	return s
+}
+
+function valueof(line) {
+	if (line !~ /^[[:alpha:]_]+=/) return 0
+	return unquote(substr(line, index(line, "=") + 1))
+}
+
+BEGIN {
+	__object = ENVIRON["__object"]
+	if (!__object) exit 1
+
+	opttype = ARGV[1]
+	optname = ARGV[2]
+
+	if (opttype !~ /^(option|list)/ || !optname) {
+		print "invalid"
+		exit (e=1)
+	}
+
+	ARGV[1] = __object "/parameter/" opttype
+	ARGV[2] = __object "/explorer/options"
+
+	state = "present"
+}
+
+NR == FNR {
+	# memoize "should" state
+	if (index($0, optname "=") == 1) {
+		should[++should_count] = valueof($0)
+	}
+
+	# go to next line (important!)
+	next
+}
+
+{
+	# compare "is" state
+	if (index($0, optname "=") != 1)
+		next
+	++is_count
+
+	v = valueof($0)
+
+	if (v == should[is_count]) {
+		# looks good, but can't say definitely just from this line
+	} else if (is_count > should_count) {
+		# there are more "is" records than "should" -> definitely different
+		state = "different"
+		exit
+	} else {
+		# see if we can find the "is" value somewhere in "should"
+		for (i in should) {
+			if (v == should[i]) {
+				# value found -> could be rearranged
+				# FIXME: Duplicate values are not properly handled here. Do they matter?
+				state = "rearranged"
+				next
+			}
+		}
+
+		# "is" value could not be found in "should" -> definitely different
+		state = "different"
+		exit
+	}
+}
+
+END {
+	if (e) exit
+
+	if (!is_count) {
+		# no "is" values -> absent
+		state = "absent"
+	} else if (is_count < should_count) {
+		# "is" was shorter than "should" -> different
+		state = "different"
+	}
+
+	print state
+}
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
index 05bca0a7..285659b0 100755
--- a/cdist/conf/type/__uci_section/gencode-remote
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -70,7 +70,8 @@ in
 		fi
 
 		# Make sure the section itself is present
-		if test "${state_is}" = absent
+		if test "${state_is}" = absent \
+			|| test "${type_is}" != "${type_should#*.}"
 		then
 			printf 'set %s\n' "${section}" >>"${__messages_out:?}"
 			# shellcheck disable=SC2140
@@ -78,7 +79,7 @@ in
 		fi
 
 		# Delete options/lists not in "should"
-		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		sed -e 's/=.*$//' "${__object:?}/explorer/options" \
 		| while read -r _optname
 		  do
 			  grep_line "${_optname}" "${listnames_should}" "${optnames_should}" || {
@@ -87,18 +88,55 @@ in
 			  } </dev/null
 		  done
 
-		# Set "should" options
-		prefix_lines option "${optnames_should}" list "${listnames_should}" \
-		| while read -r _type _optname
-		  do
-			  test -n "${_type}${_optname}" || continue  # ignore empty lines
+		opt_proc_error() {
+			printf 'An error occurred during processing of option %s\n' "${1:?}" >&2
+			exit 1
+		}
 
-			  case $_type
+		# Set "should" options
+		echo "${optnames_should}" \
+		| grep -e . \
+		| while read -r _optname
+		  do
+			  _opt_state=$(awk -f "${__type:?}/files/option_state.awk" option "${_optname}") \
+				  || opt_proc_error "${_optname}"
+			  case ${_opt_state}
 			  in
-				  (list)
+				  (invalid)
+					  opt_proc_error "${_optname}"
+					  ;;
+				  (present)
+					  ;;
+				  (*)
+					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  # shellcheck disable=SC2140
+					  uci_cmd set "${section}.${_optname}"="$(
+						  grep -e "^${_optname}=" "${__object:?}/parameter/option" \
+						  | sed -e 's/^.*=//' \
+						  | unquote_lines \
+						  | head -n 1)"
+					  ;;
+			  esac
+		  done
+
+		echo "${listnames_should}" \
+		| grep -e . \
+		| while read -r _optname
+		  do
+			  _list_state=$(awk -f "${__type:?}/files/option_state.awk" list "${_optname}") \
+				  || opt_proc_error "${_optname}"
+			  case ${_list_state}
+			  in
+				  (invalid)
+					  opt_proc_error "${_optname}"
+					  ;;
+				  (present)
+					  ;;
+				  (*)
 					  printf 'set_list %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
 
-					  if grep -q -e "^${_optname}=" "${__object:?}/explorer/options"
+					  if test "${_list_state}" != absent
 					  then
 						  uci_cmd delete "${section}.${_optname}"
 					  fi
@@ -112,16 +150,6 @@ in
 						    uci_cmd add_list "${section}.${_optname}"="${_value}"
 					    done
 					  ;;
-				  (option)
-					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
-
-					  # shellcheck disable=SC2140
-					  uci_cmd set "${section}.${_optname}"="$(
-						  grep "^${_optname}=" "${__object:?}/parameter/option" \
-						  | sed -e 's/^.*=//' \
-						  | unquote_lines \
-						  | head -n 1)"
-					  ;;
 			  esac
 		  done
 		;;

From 2be8c634584561158960fc5cf2f363f3f5c4cc4c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 3 Nov 2020 06:43:57 +0100
Subject: [PATCH 150/411] pycodestyle fixes

---
 cdist/argparse.py         |  2 +-
 cdist/scan/commandline.py |  2 +-
 cdist/scan/scan.py        | 12 ++++++++----
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index ff195e8c..88759d7b 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -473,7 +473,7 @@ def get_parsers():
 
     # Scan = config + further
     parser['scan'] = parser['sub'].add_parser('scan', add_help=False,
-                                                 parents=[parser['config']])
+                                              parents=[parser['config']])
 
     parser['scan'] = parser['sub'].add_parser(
             'scan', parents=[parser['loglevel'],
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 0fb718e5..eca4cf13 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -37,7 +37,7 @@ def commandline(args):
 
     if not args.mode:
         # By default scan and trigger, but do not call any action
-        args.mode = ['scan', 'trigger' ]
+        args.mode = ['scan', 'trigger', ]
 
     if 'trigger' in args.mode:
         t = scan.Trigger(interfaces=args.interfaces)
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index fcbf1899..0ce4dff3 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -30,9 +30,12 @@
 # Scanner logic
 #  - save results to configdir:
 #     basedir = ~/.cdist/scan/<ipv6-address>
-#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time or similar
-#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
-#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time
+#           or similar
+#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record
+#           unix time or similar
+#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record
+#           unix time or similar
 #
 #
 #
@@ -58,6 +61,7 @@ import datetime
 
 log = logging.getLogger("scan")
 
+
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
@@ -91,6 +95,7 @@ class Trigger(object):
         log.debug(f"Sending request on {interface}")
         send(packet, verbose=self.verbose)
 
+
 class Scanner(object):
     """
     Scan for replies of hosts, maintain the up-to-date database
@@ -134,7 +139,6 @@ class Scanner(object):
               prn=self.handle_pkg)
 
 
-
 if __name__ == '__main__':
     t = Trigger(interfaces=["wlan0"])
     t.start()

From df881c0f9859759c10339e9d4a5ac24ff77fbe1a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 4 Nov 2020 08:34:17 +0100
Subject: [PATCH 151/411] [type/__file] Fix --state pre-exists also for
 non-dry-runs

---
 cdist/conf/type/__file/gencode-remote | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index 2675b03a..f7a528fd 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -95,6 +95,10 @@ case "$state_should" in
         fi
     ;;
 
+    pre-exists)
+        :
+        ;;
+
     *)
         echo "Unknown state: $state_should" >&2
         exit 1

From d28a70a73c2affcf3268623a51df2b7a6cdd534a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 6 Nov 2020 08:32:40 +0100
Subject: [PATCH 152/411] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 5977d80e..335a0f9f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,8 @@ next:
 	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 	* Type __file: Fix state pre-exists (Dennis Camera)
+	* Type __apt_norecommends: Use 00InstallRecommends file as debian-installer does (Dennis Camera)
+	* Core: Introduce scanner (noninvasive, beta) (Nico Schottelius)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 0f1df5ef68a60c5b8077f437bf0d07f1f79788e7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:07:58 +0100
Subject: [PATCH 153/411] Fix shellcheck source directives

---
 cdist/conf/type/__uci/gencode-remote         | 2 +-
 cdist/conf/type/__uci_section/gencode-remote | 2 +-
 cdist/conf/type/__uci_section/manifest       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
index 348d15a1..70a3d3e0 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 state_is=$(cat "${__object:?}/explorer/state")
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
index 285659b0..50fdfa4e 100755
--- a/cdist/conf/type/__uci_section/gencode-remote
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci_section/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 7e23e443..0325b92c 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci_section/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 

From c7c3075f629191ed7e05a76395b77dce01e2928f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:10:14 +0100
Subject: [PATCH 154/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 335a0f9f..40b816a6 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -13,6 +13,7 @@ next:
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 	* Type __file: Fix state pre-exists (Dennis Camera)
 	* Type __apt_norecommends: Use 00InstallRecommends file as debian-installer does (Dennis Camera)
+	* New types: __uci, __uci_section (Dennis Camera)
 	* Core: Introduce scanner (noninvasive, beta) (Nico Schottelius)
 
 6.8.0: 2020-09-11

From 348c6eedc903ecd28b02c5dda3821e71ef255a4f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:12:20 +0100
Subject: [PATCH 155/411] Release 6.9.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 40b816a6..c47123d9 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.0: 2020-11-07
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)

From 10abe514b8fb7e6d3ce3587a55773a21e78ce9e6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 11 Jun 2020 21:27:53 +0200
Subject: [PATCH 156/411] [type/__hostname] Add support for OpenWrt

---
 cdist/conf/type/__hostname/gencode-remote | 3 +++
 cdist/conf/type/__hostname/manifest       | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index ae224611..02afcbfb 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -69,6 +69,9 @@ in
     centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void)
         echo "hostname '$name_should'"
     ;;
+    openwrt)
+        echo "echo '$name_should' >/proc/sys/kernel/hostname"
+    ;;
     macosx)
         echo "scutil --set HostName '$name_should'"
     ;;
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index e1e356a0..bf8a331c 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -149,6 +149,10 @@ in
     openbsd)
         echo "$name_should" | __file /etc/myname --source -
     ;;
+    openwrt)
+        __uci system.@system[0].hostname --value "$name_should"
+            # --transaction hostname
+    ;;
     slackware)
         # We write the FQDN into /etc/HOSTNAME.  But /etc/rc.d/rc.M will only
         # read the first component from this file and set it as the running

From b0f3bb3350be9575c1632c6e3f669e7ede431020 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 7 Nov 2020 16:49:09 +0100
Subject: [PATCH 157/411] New type __dpkg_architecture

This type handles foreign architectures added to dpkg.
---
 .../__dpkg_architecture/explorer/architecture |   8 ++
 .../explorer/foreign-architectures            |   8 ++
 .../type/__dpkg_architecture/gencode-remote   |  64 +++++++++++
 cdist/conf/type/__dpkg_architecture/man.rst   | 103 ++++++++++++++++++
 .../parameter/default/state                   |   1 +
 .../__dpkg_architecture/parameter/optional    |   1 +
 6 files changed, 185 insertions(+)
 create mode 100755 cdist/conf/type/__dpkg_architecture/explorer/architecture
 create mode 100755 cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
 create mode 100755 cdist/conf/type/__dpkg_architecture/gencode-remote
 create mode 100644 cdist/conf/type/__dpkg_architecture/man.rst
 create mode 100644 cdist/conf/type/__dpkg_architecture/parameter/default/state
 create mode 100644 cdist/conf/type/__dpkg_architecture/parameter/optional

diff --git a/cdist/conf/type/__dpkg_architecture/explorer/architecture b/cdist/conf/type/__dpkg_architecture/explorer/architecture
new file mode 100755
index 00000000..4478bc6e
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/explorer/architecture
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+# __dpkg_architecture/explorer/architecture
+
+# Get the main architecture of this machine
+
+
+# print or die in the gencode-remote
+dpkg --print-architecture || true
diff --git a/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
new file mode 100755
index 00000000..ff01672a
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+# __dpkg_architecture/explorer/foreign-architectures
+
+# Print all additional architectures
+
+
+# print or die in the gencode-remote
+dpkg --print-foreign-architectures || true
diff --git a/cdist/conf/type/__dpkg_architecture/gencode-remote b/cdist/conf/type/__dpkg_architecture/gencode-remote
new file mode 100755
index 00000000..adf9d655
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/gencode-remote
@@ -0,0 +1,64 @@
+#!/bin/sh -e
+# __dpkg_architecture/gencode-remote
+
+
+# Get parameter and explorer
+state_should="$(cat "$__object/parameter/state")"
+arch_wanted="$__object_id"
+main_arch="$(cat "$__object/explorer/architecture")"
+
+# Exit here if dpkg do not work (empty explorer)
+if [ -z "$main_arch" ]; then
+    echo "dpkg is not available or unable to detect a architecture!" >&2
+    exit 1
+fi
+
+
+# Check if requested architecture is the main one
+if [ "$arch_wanted" = "$main_arch" ]; then
+    # higher than present; we can not remove it
+    state_is="present"
+    caution="yes"
+
+# Check if the architecture not already used
+elif grep -qFx "$arch_wanted" "$__object/explorer/foreign-architectures"; then
+    state_is="present"
+
+# arch does not exist
+else
+    state_is="absent"
+fi
+
+
+# Check what to do
+if [ "$state_is" != "$state_should" ]; then
+    case "$state_should" in
+        present)
+            # print add code
+            printf "dpkg --add-architecture '%s'\n" "$arch_wanted"
+            # updating the index to make the new architecture available
+            echo "apt update"
+
+            echo added >> "$__messages_out"
+            ;;
+
+        absent)
+            if [ "$caution" ]; then
+                printf "can not remove the main arch '%s' of the system!\n" "$main_arch" >&2
+                exit 1
+            fi
+
+            # removing all existing packages for the architecture
+            printf "apt purge '.*:%s'\n" "$arch_wanted"
+            # print remove code
+            printf "dpkg --remove-architecture '%s'\n" "$arch_wanted"
+
+            echo removed >> "$__messages_out"
+            ;;
+
+        *)
+            printf "state '%s' is unknown!\n" "$state_should" >&2
+            exit 1
+            ;;
+    esac
+fi
diff --git a/cdist/conf/type/__dpkg_architecture/man.rst b/cdist/conf/type/__dpkg_architecture/man.rst
new file mode 100644
index 00000000..fa196229
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/man.rst
@@ -0,0 +1,103 @@
+cdist-type__dpkg_architecture(7)
+================================
+
+NAME
+----
+cdist-type__dpkg_architecture - Handles foreign architectures on debian-like
+systems managed by `dpkg`
+
+
+DESCRIPTION
+-----------
+This type handles foreign architectures on systems managed by
+:strong:`dpkg`\ (1). The object id is the name of the architecture accepted by
+`dpkg`, which should be added or removed.
+
+If the architecture is not setup on the system, it adds a new architecture as a
+new foreign architecture in `dpkg`. Then, it updates the apt package index to
+make packages from the new architecture available.
+
+If the architecture should be removed, it will remove it if it is not the base
+architecture on where the system was installed on. Before it, it will purge
+every package based on the "to be removed" architecture via `apt` to be able to
+remove the selected architecture.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    ``present`` or ``absent``. Defaults to ``present``.
+
+
+MESSAGES
+--------
+added
+   Added the specified architecture
+
+removed
+   Removed the specified architecture
+
+
+ABORTS
+------
+Aborts in the following cases:
+
+If :strong:`dpkg`\ (1) is not available. It will abort with a proper error
+message.
+
+If the architecture is the same as the base architecture the system is build
+upon it (returned by ``dpkg --print-architecture``) and it should be removed.
+
+It will fail if it can not execute :strong:`apt`\ (8). It is assumed that it is
+already installed.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+  # add i386 (32 bit) architecture
+  __dpkg_architecture i386
+
+  # remove it again :)
+  __dpkg_architecture i386 --state absent
+
+
+SEE ALSO
+--------
+`Multiarch on Debian systems <https://wiki.debian.org/Multiarch>`_
+
+`How to setup multiarch on Debian <https://wiki.debian.org/Multiarch/HOWTO>`_
+
+:strong:`dpkg`\ (1)
+:strong:`cdist-type__package_dpkg`\ (7)
+:strong:`cdist-type__package_apt`\ (7)
+
+Useful commands:
+
+.. code-block:: sh
+
+   # base architecture installed on this system
+   dpkg --print-architecture
+
+   # extra architectures added
+   dpkg --print-foreign-architectures
+
+
+AUTHORS
+-------
+Matthias Stecher <matthiasstecher at gmx.de>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Matthias Stecher. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+ublished by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__dpkg_architecture/parameter/default/state b/cdist/conf/type/__dpkg_architecture/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__dpkg_architecture/parameter/optional b/cdist/conf/type/__dpkg_architecture/parameter/optional
new file mode 100644
index 00000000..ff72b5c7
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/parameter/optional
@@ -0,0 +1 @@
+state

From 7777580d8ff94f2b0725dbbcef84d8957998acb6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 7 Nov 2020 20:56:17 +0100
Subject: [PATCH 158/411] __dpkg_architecture: add copyright headers

---
 .../__dpkg_architecture/explorer/architecture  | 18 ++++++++++++++++++
 .../explorer/foreign-architectures             | 18 ++++++++++++++++++
 .../type/__dpkg_architecture/gencode-remote    | 18 ++++++++++++++++++
 3 files changed, 54 insertions(+)

diff --git a/cdist/conf/type/__dpkg_architecture/explorer/architecture b/cdist/conf/type/__dpkg_architecture/explorer/architecture
index 4478bc6e..03e7e386 100755
--- a/cdist/conf/type/__dpkg_architecture/explorer/architecture
+++ b/cdist/conf/type/__dpkg_architecture/explorer/architecture
@@ -1,5 +1,23 @@
 #!/bin/sh -e
 # __dpkg_architecture/explorer/architecture
+#
+# 2020 Matthias Stecher <matthiasstecher at gmx.de>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 # Get the main architecture of this machine
 
diff --git a/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
index ff01672a..a150d307 100755
--- a/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
+++ b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
@@ -1,5 +1,23 @@
 #!/bin/sh -e
 # __dpkg_architecture/explorer/foreign-architectures
+#
+# 2020 Matthias Stecher <matthiasstecher at gmx.de>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 # Print all additional architectures
 
diff --git a/cdist/conf/type/__dpkg_architecture/gencode-remote b/cdist/conf/type/__dpkg_architecture/gencode-remote
index adf9d655..47fb24e7 100755
--- a/cdist/conf/type/__dpkg_architecture/gencode-remote
+++ b/cdist/conf/type/__dpkg_architecture/gencode-remote
@@ -1,5 +1,23 @@
 #!/bin/sh -e
 # __dpkg_architecture/gencode-remote
+#
+# 2020 Matthias Stecher <matthiasstecher at gmx.de>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 
 # Get parameter and explorer

From 91bcc2a2939508b8722c2a148c89c92aaa4f830f Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 7 Nov 2020 21:03:38 +0100
Subject: [PATCH 159/411] __dpkg_architecture: make type nonparallel

I think it's not good that dpkg or apt is running in parallel.
---
 cdist/conf/type/__dpkg_architecture/nonparallel | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 cdist/conf/type/__dpkg_architecture/nonparallel

diff --git a/cdist/conf/type/__dpkg_architecture/nonparallel b/cdist/conf/type/__dpkg_architecture/nonparallel
new file mode 100644
index 00000000..e69de29b

From 9fc6ee0948347a0de20e633ce4a0ee358dc92c00 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 8 Nov 2020 13:14:13 +0100
Subject: [PATCH 160/411] __package_apt: add --install-recommends parameter

For a good reason, __package_apt doesn't install recommended packages as
default. But the option --install-recommends comes handy if you want to
install a package where you want to install all recommended packages
(and not to install all of them separately).

Also, the manpage now explains that the type won't install recommended
packages by default.
---
 cdist/conf/type/__package_apt/gencode-remote    | 17 ++++++++++++-----
 cdist/conf/type/__package_apt/man.rst           | 15 +++++++++++++--
 cdist/conf/type/__package_apt/parameter/boolean |  1 +
 3 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote
index f3d91566..fbfca330 100755
--- a/cdist/conf/type/__package_apt/gencode-remote
+++ b/cdist/conf/type/__package_apt/gencode-remote
@@ -42,6 +42,13 @@ else
    target_release=""
 fi
 
+if [ -f "$__object/parameter/install-recommends" ]; then
+    # required if __apt_norecommends is used
+    recommendsparam="-o APT::Install-Recommends=1"
+else
+    recommendsparam="-o APT::Install-Recommends=0"
+fi
+
 if [ -f "$__object/parameter/purge-if-absent" ]; then
 	purgeparam="--purge"
 else
@@ -62,16 +69,16 @@ case "$state_is" in
     ;;
 esac
 
-# Hint if we need to avoid questions at some point:
-# DEBIAN_PRIORITY=critical can reduce the number of questions
-aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o APT::Install-Recommends=0 -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
-
 if [ "$state_is" = "$state_should" ]; then
     if [ -z "$version" ] || [ "$version" = "$version_is" ]; then
         exit 0;
     fi
 fi
 
+# Hint if we need to avoid questions at some point:
+# DEBIAN_PRIORITY=critical can reduce the number of questions
+aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
+
 case "$state_should" in
     present)
         # following is bit ugly, but important hack.
@@ -85,7 +92,7 @@ EOF
         if [ -n "$version" ]; then
             name="${name}=${version}"
         fi
-        echo "$aptget install $target_release '$name'"
+        echo "$aptget $recommendsparam install $target_release '$name'"
         echo "installed" >> "$__messages_out"
     ;;
     absent)
diff --git a/cdist/conf/type/__package_apt/man.rst b/cdist/conf/type/__package_apt/man.rst
index a1691eac..4e6101a5 100644
--- a/cdist/conf/type/__package_apt/man.rst
+++ b/cdist/conf/type/__package_apt/man.rst
@@ -9,7 +9,9 @@ cdist-type__package_apt - Manage packages with apt-get
 DESCRIPTION
 -----------
 apt-get is usually used on Debian and variants (like Ubuntu) to
-manage packages.
+manage packages. The package will be installed without recommended
+or suggested packages. If such packages are required, install them
+separatly or use the parameter ``--install-recommends``.
 
 This type will also update package index, if it is older
 than one day, to avoid missing package error messages.
@@ -23,7 +25,7 @@ None
 OPTIONAL PARAMETERS
 -------------------
 name
-   If supplied, use the name and not the object id as the package name.
+    If supplied, use the name and not the object id as the package name.
 
 state
     Either "present" or "absent", defaults to "present"
@@ -39,6 +41,15 @@ version
 
 BOOLEAN PARAMETERS
 ------------------
+install-recommends
+    If the package will be installed, it also installs recommended packages
+    with it. It will not install recommended packages if the original package
+    is already installed.
+
+    In most cases, it is recommended to install recommended packages separatly
+    to control which additional packages will be installed to avoid useless
+    installed packages.
+
 purge-if-absent
     If this parameter is given when state is `absent`, the package is
     purged from the system (using `--purge`).
diff --git a/cdist/conf/type/__package_apt/parameter/boolean b/cdist/conf/type/__package_apt/parameter/boolean
index f9a0f6b0..a2e433f3 100644
--- a/cdist/conf/type/__package_apt/parameter/boolean
+++ b/cdist/conf/type/__package_apt/parameter/boolean
@@ -1 +1,2 @@
+install-recommends
 purge-if-absent

From fded60bd0f4ca7b6b9047c9f3b1470fdbd20c263 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 8 Nov 2020 13:27:01 +0100
Subject: [PATCH 161/411] ++changelog

---
 docs/changelog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index c47123d9..f271142a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,12 @@
 Changelog
 ---------
 
+next:
+	* Type __file: Fix state pre-exists (Dennis Camera)
+	* Type __hostname: Add support for OpenWrt (Dennis Camera)
+	* New type: __dpkg_architecture (Matthias Stecher)
+	* Type __package_apt: Add --install-recommends parameter (Matthias Stecher)
+
 6.9.0: 2020-11-07
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)

From d2506ac04eac987caf0fe0c33f48866ee6e43c0d Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 8 Nov 2020 13:31:57 +0100
Subject: [PATCH 162/411] Release 6.9.1

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index f271142a..b6b8c22b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)
 	* Type __hostname: Add support for OpenWrt (Dennis Camera)
 	* New type: __dpkg_architecture (Matthias Stecher)

From a95eab77a5606f2552ac28c0b817945f72538c97 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 8 Nov 2020 15:18:04 +0100
Subject: [PATCH 163/411] __locale: add state explorer

.. so it doesn't execute code all the time.
---
 cdist/conf/type/__locale/explorer/state | 36 +++++++++++++++++++++++++
 cdist/conf/type/__locale/gencode-remote | 15 ++++++++---
 2 files changed, 47 insertions(+), 4 deletions(-)
 create mode 100755 cdist/conf/type/__locale/explorer/state

diff --git a/cdist/conf/type/__locale/explorer/state b/cdist/conf/type/__locale/explorer/state
new file mode 100755
index 00000000..4494fcbc
--- /dev/null
+++ b/cdist/conf/type/__locale/explorer/state
@@ -0,0 +1,36 @@
+#!/bin/sh -e
+# __locale/explorer/state
+#
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Check if the locale is already installed on the system.
+# Outputs 'present' or 'absent' depending if the locale exists.
+#
+
+
+# Get user-defined locale
+# locale name is echoed differently than the user propably set it (for UTF-8)
+locale="$(echo "$__object_id" | sed 's/UTF-8/utf8/')"
+
+# Check if the given locale exists on the system
+if localedef --list-archive | grep -qFx "$locale"; then
+    echo present
+else
+    echo absent
+fi
diff --git a/cdist/conf/type/__locale/gencode-remote b/cdist/conf/type/__locale/gencode-remote
index 1feb9884..4639cef8 100755
--- a/cdist/conf/type/__locale/gencode-remote
+++ b/cdist/conf/type/__locale/gencode-remote
@@ -23,6 +23,15 @@
 
 locale="$__object_id"
 
+state_is=$(cat "$__object/explorer/state")
+state_should=$(cat "$__object/parameter/state")
+
+# short circuit if there is nothing to do
+if [ "$state_is" = "$state_should" ]; then
+    exit 0
+fi
+
+
 # Hardcoded, create a pull request with
 # branching on $os in case it is at another location
 alias=/usr/share/locale/locale.alias
@@ -35,8 +44,6 @@ charmap=$(echo "$locale" | cut -d . -f 2)
 # W-T-F!
 locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
 
-state=$(cat "$__object/parameter/state")
-
 os=$(cat "$__global/explorer/os")
 
 # Nothing to be done on alpine
@@ -46,7 +53,7 @@ case "$os" in
         ;;
 esac
 
-case "$state" in
+case "$state_should" in
     present)
         echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
     ;;
@@ -54,7 +61,7 @@ case "$state" in
         echo localedef --delete-from-archive "$locale_remove"
     ;;
     *)
-        echo "Unsupported state: $state" >&2
+        echo "Unsupported state: $state_should" >&2
         exit 1
     ;;
 esac

From 792b4b10762c094d861282e78953dfd29b203a41 Mon Sep 17 00:00:00 2001
From: ssrq <ungleich@ssrq-sds-fds.ch>
Date: Mon, 9 Nov 2020 12:08:54 +0100
Subject: [PATCH 164/411] Add missing 'config' command

---
 docs/src/cdist-best-practice.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/src/cdist-best-practice.rst b/docs/src/cdist-best-practice.rst
index 39ec453e..f7677276 100644
--- a/docs/src/cdist-best-practice.rst
+++ b/docs/src/cdist-best-practice.rst
@@ -200,15 +200,15 @@ of cdist:
 .. code-block:: sh
 
     # Singleton type without parameter
-    echo __ungleich_munin_server | cdist --initial-manifest - munin.panter.ch
+    echo __ungleich_munin_server | cdist config --initial-manifest - munin.panter.ch
 
     # Singleton type with parameter
     echo __ungleich_munin_node --allow 1.2.3.4 | \
-        cdist --initial-manifest - rails-19.panter.ch
+        cdist config --initial-manifest - rails-19.panter.ch
 
     # Normal type
     echo __file /tmp/stdintest --mode 0644 | \
-        cdist --initial-manifest - cdist-dev-01.ungleich.ch
+        cdist config --initial-manifest - cdist-dev-01.ungleich.ch
 
 
 Other content in cdist repository

From ba906510529549446952bb3abfd8b49b05da7ca0 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 11 Nov 2020 07:49:32 +0100
Subject: [PATCH 165/411] ++changelog

---
 docs/changelog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index b6b8c22b..9fa0926c 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,11 @@
 Changelog
 ---------
 
+next:
+	* Documentation: Fix examples in best practice (Dennis Camera)
+	* Type __locale: Add state explorer (Matthias Stecher)
+	* Core: Reorganize scripts, version generation (Ander Punnar)
+
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)
 	* Type __hostname: Add support for OpenWrt (Dennis Camera)

From 3e48ef9e11795bf1c8a59ae202ed28f58cff1753 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:09:26 +0100
Subject: [PATCH 166/411] [type/__hostname] Lint

- Error if expected environment variables are unset
- Always wrap variable expansions in {}
---
 cdist/conf/type/__hostname/gencode-remote |  82 ++++++------
 cdist/conf/type/__hostname/manifest       | 147 +++++++++++-----------
 2 files changed, 114 insertions(+), 115 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 02afcbfb..9447daa9 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -20,26 +20,27 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$(cat "$__global/explorer/os")
-name_running=$(cat "$__global/explorer/hostname")
-has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
+os=$(cat "${__global:?}/explorer/os")
+name_running=$(cat "${__global:?}/explorer/hostname")
+has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl")
 
 
-if test -s "$__object/parameter/name"
+if test -s "${__object:?}/parameter/name"
 then
-    name_should=$(cat "$__object/parameter/name")
+    name_should=$(cat "${__object:?}/parameter/name")
 else
-    case $os
+    case ${os}
     in
         # RedHat-derivatives and BSDs
-        centos|fedora|redhat|scientific|freebsd|macosx|netbsd|openbsd)
+        (centos|fedora|redhat|scientific|freebsd|macosx|netbsd|openbsd)
             # Hostname is FQDN
-            name_should="${__target_host}"
-        ;;
-        *)
+            name_should=${__target_host:?}
+            ;;
+        (*)
             # Hostname is only first component of FQDN
-            name_should="${__target_host%%.*}"
-        ;;
+            name_should=${__target_host:?}
+            name_should=${name_should%%.*}
+            ;;
     esac
 fi
 
@@ -47,46 +48,46 @@ fi
 ################################################################################
 # Check if the (running) hostname is already correct
 #
-test "$name_running" != "$name_should" || exit 0
+test "${name_running}" != "${name_should}" || exit 0
 
 
 ################################################################################
 # Setup hostname
 #
-echo 'changed' >>"$__messages_out"
+echo 'changed' >>"${__messages_out:?}"
 
 # Use the good old way to set the hostname.
-case $os
+case ${os}
 in
-    alpine|debian|devuan|ubuntu)
+    (alpine|debian|devuan|ubuntu)
         echo 'hostname -F /etc/hostname'
-    ;;
-    archlinux)
+        ;;
+    (archlinux)
         echo 'command -v hostnamectl >/dev/null 2>&1' \
-            "&& hostnamectl set-hostname '$name_should'" \
-            "|| hostname '$name_should'"
-    ;;
-    centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void)
-        echo "hostname '$name_should'"
-    ;;
-    openwrt)
-        echo "echo '$name_should' >/proc/sys/kernel/hostname"
-    ;;
-    macosx)
-        echo "scutil --set HostName '$name_should'"
-    ;;
-    solaris)
-        echo "uname -S '$name_should'"
-    ;;
-    slackware|suse|opensuse-leap)
+            "&& hostnamectl set-hostname '${name_should}'" \
+            "|| hostname '${name_should}'"
+        ;;
+    (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void)
+        echo "hostname '${name_should}'"
+        ;;
+    (openwrt)
+        echo "echo '${name_should}' >/proc/sys/kernel/hostname"
+        ;;
+    (macosx)
+        echo "scutil --set HostName '${name_should}'"
+        ;;
+    (solaris)
+        echo "uname -S '${name_should}'"
+        ;;
+    (slackware|suse|opensuse-leap)
         # We do not read from /etc/HOSTNAME, because the running
         # hostname is the first component only while the file contains
         # the FQDN.
-        echo "hostname '$name_should'"
-    ;;
-    *)
+        echo "hostname '${name_should}'"
+        ;;
+    (*)
         # Fall back to set the hostname using hostnamectl, if available.
-        if test -n "$has_hostnamectl"
+        if test -n "${has_hostnamectl}"
         then
             # Don't use hostnamectl as the primary means to set the hostname for
             # systemd systems, because it cannot be trusted to work reliably and
@@ -97,7 +98,8 @@ in
             echo "test \"\$(hostname)\" = \"\$(cat /etc/hostname)\"" \
                  " || hostname -F /etc/hostname"
         else
-            printf "echo 'Unsupported OS: %s' >&2\nexit 1\n" "$os"
+            printf "echo 'Unsupported OS: %s' >&2\n" "${os}"
+            printf 'exit 1\n'
         fi
-    ;;
+        ;;
 esac
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index bf8a331c..125bb2fb 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -20,69 +20,65 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-not_supported() {
-    echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
-    echo "Please contribute an implementation for it if you can." >&2
-    exit 1
-}
-
 set_hostname_systemd() {
     echo "$1" | __file /etc/hostname --source -
 }
 
-os=$(cat "$__global/explorer/os")
-os_version=$(cat "$__global/explorer/os_version")
-os_major=$(echo "$os_version" | grep -o '^[0-9][0-9]*' || true)
+os=$(cat "${__global:?}/explorer/os")
+os_version=$(cat "${__global:?}/explorer/os_version")
+os_major=$(echo "${os_version}" | grep -o '^[0-9][0-9]*' || true)
 
-max_len=$(cat "$__object/explorer/max_len")
-has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
+max_len=$(cat "${__object:?}/explorer/max_len")
+has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl")
 
-if test -s "$__object/parameter/name"
+if test -s "${__object:?}/parameter/name"
 then
-    name_should=$(cat "$__object/parameter/name")
+    name_should=$(cat "${__object:?}/parameter/name")
 else
-    case $os
+    case ${os}
     in
         # RedHat-derivatives and BSDs
-        centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware)
+        (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware)
             # Hostname is FQDN
-            name_should="${__target_host}"
-        ;;
-        suse|opensuse-leap)
+            name_should=${__target_host:?}
+            ;;
+        (suse|opensuse-leap)
+            name_should=${__target_host:?}
+
             # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
             # systemd does not. The running hostname is the first
             # component in both cases.
             # In versions before 15.x, the FQDN is stored in /etc/hostname.
-            if test -n "$has_hostnamectl" && test "$os_major" -ge 15 \
-                    && test "$os_major" -ne 42
+            if test -n "${has_hostnamectl}" \
+                    && test "${os_major}" -ge 15 \
+                    && test "${os_major}" -ne 42
             then
-                name_should="${__target_host%%.*}"
-            else
-                name_should="${__target_host}"
+                name_should=${name_should%%.*}
             fi
-        ;;
+            ;;
         *)
             # Hostname is only first component of FQDN on all other systems.
-            name_should="${__target_host%%.*}"
-        ;;
+            name_should=${__target_host:?}
+            name_should=${name_should%%.*}
+            ;;
     esac
 fi
 
-if test -n "$max_len" && test "$(printf '%s' "$name_should" | wc -c)" -gt "$max_len"
+if test -n "${max_len}" && test "$(printf '%s' "${name_should}" | wc -c)" -gt "${max_len}"
 then
     printf "Host name too long. Up to %u characters allowed.\n" "${max_len}" >&2
     exit 1
 fi
 
-case $os
+case ${os}
 in
-    alpine|debian|devuan|ubuntu|void)
-        echo "$name_should" | __file /etc/hostname --source -
-    ;;
-    archlinux)
-        if test -n "$has_hostnamectl"
+    (alpine|debian|devuan|ubuntu|void)
+        echo "${name_should}" | __file /etc/hostname --source -
+        ;;
+    (archlinux)
+        if test -n "${has_hostnamectl}"
         then
-            set_hostname_systemd "$name_should"
+            set_hostname_systemd "${name_should}"
         else
             echo 'Ancient ArchLinux variants without hostnamectl are not supported.' >&2
             exit 1
@@ -97,8 +93,8 @@ in
             #     --value "\"$name_should\""
         fi
         ;;
-    centos|fedora|redhat|scientific)
-        if test -z "$has_hostnamectl"
+    (centos|fedora|redhat|scientific)
+        if test -z "${has_hostnamectl}"
         then
             # Only write to /etc/sysconfig/network on non-systemd versions.
             # On systemd-based versions this entry is ignored.
@@ -106,63 +102,62 @@ in
                 --file /etc/sysconfig/network \
                 --delimiter '=' --exact_delimiter \
                 --key HOSTNAME \
-                --value "\"$name_should\""
+                --value "\"${name_should}\""
         else
-            set_hostname_systemd "$name_should"
+            set_hostname_systemd "${name_should}"
         fi
-    ;;
-    gentoo)
+        ;;
+    (gentoo)
         # Only write to /etc/conf.d/hostname on OpenRC-based installations.
         # On systemd use hostnamectl(1) in gencode-remote.
-        if test -z "$has_hostnamectl"
+        if test -z "${has_hostnamectl}"
         then
             __key_value '/etc/conf.d/hostname:hostname' \
                 --file /etc/conf.d/hostname \
                 --delimiter '=' --exact_delimiter \
                 --key 'hostname' \
-                --value "\"$name_should\""
+                --value "\"${name_should}\""
         else
             set_hostname_systemd "$name_should"
         fi
-    ;;
-    freebsd)
+        ;;
+    (freebsd)
         __key_value '/etc/rc.conf:hostname' \
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
-            --value "\"$name_should\""
-    ;;
-    macosx)
+            --value "\"${name_should}\""
+        ;;
+    (macosx)
         # handled in gencode-remote
-        :
-    ;;
-    netbsd)
+        ;;
+    (netbsd)
         __key_value '/etc/rc.conf:hostname' \
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
-            --value "\"$name_should\""
+            --value "\"${name_should}\""
 
         # To avoid confusion, ensure that the hostname is only stored once.
         __file /etc/myname --state absent
-    ;;
-    openbsd)
-        echo "$name_should" | __file /etc/myname --source -
-    ;;
-    openwrt)
-        __uci system.@system[0].hostname --value "$name_should"
+        ;;
+    (openbsd)
+        echo "${name_should}" | __file /etc/myname --source -
+        ;;
+    (openwrt)
+        __uci system.@system[0].hostname --value "${name_should}"
             # --transaction hostname
-    ;;
-    slackware)
+        ;;
+    (slackware)
         # We write the FQDN into /etc/HOSTNAME.  But /etc/rc.d/rc.M will only
         # read the first component from this file and set it as the running
         # hostname on boot.
-        echo "$name_should" | __file /etc/HOSTNAME --source -
-    ;;
-    solaris)
-        echo "$name_should" | __file /etc/nodename --source -
-    ;;
-    suse|opensuse-leap)
+        echo "${name_should}" | __file /etc/HOSTNAME --source -
+        ;;
+    (solaris)
+        echo "${name_should}" | __file /etc/nodename --source -
+        ;;
+    (suse|opensuse-leap)
         # Modern SuSE provides /etc/HOSTNAME as a symlink for
         # backwards-compatibility. Unfortunately it cannot be used
         # here as __file does not follow the symlink.
@@ -171,23 +166,25 @@ in
         # not work correctly on openSUSE 12.x which provides
         # hostnamectl but not /etc/hostname.
 
-        if test -n "$has_hostnamectl" -a "$os_major" -gt 12
+        if test -n "${has_hostnamectl}" -a "${os_major}" -gt 12
         then
-            hostname_file='/etc/hostname'
+            hostname_file=/etc/hostname
         else
-            hostname_file='/etc/HOSTNAME'
+            hostname_file=/etc/HOSTNAME
         fi
 
-        echo "$name_should" | __file "$hostname_file" --source -
-    ;;
-    *)
+        echo "${name_should}" | __file "${hostname_file}" --source -
+        ;;
+    (*)
         # On other operating systems we fall back to systemd's
         # hostnamectl if available…
-        if test -n "$has_hostnamectl"
+        if test -n "${has_hostnamectl}"
         then
-            set_hostname_systemd "$name_should"
+            set_hostname_systemd "${name_should}"
         else
-            not_supported
+            echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
+            echo "Please contribute an implementation for it if you can." >&2
+            exit 1
         fi
-    ;;
+        ;;
 esac

From 702f3eba4f8f88b634a010b377d077d501a6f0e9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:15:02 +0100
Subject: [PATCH 167/411] [type/__hostname] Remove opensuse-leap OS string
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

everything should be suse now…
---
 cdist/conf/type/__hostname/gencode-remote | 2 +-
 cdist/conf/type/__hostname/manifest       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 9447daa9..c1a97ac8 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -79,7 +79,7 @@ in
     (solaris)
         echo "uname -S '${name_should}'"
         ;;
-    (slackware|suse|opensuse-leap)
+    (slackware|suse)
         # We do not read from /etc/HOSTNAME, because the running
         # hostname is the first component only while the file contains
         # the FQDN.
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 125bb2fb..d044072b 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -42,7 +42,7 @@ else
             # Hostname is FQDN
             name_should=${__target_host:?}
             ;;
-        (suse|opensuse-leap)
+        (suse)
             name_should=${__target_host:?}
 
             # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
@@ -157,7 +157,7 @@ in
     (solaris)
         echo "${name_should}" | __file /etc/nodename --source -
         ;;
-    (suse|opensuse-leap)
+    (suse)
         # Modern SuSE provides /etc/HOSTNAME as a symlink for
         # backwards-compatibility. Unfortunately it cannot be used
         # here as __file does not follow the symlink.

From 87a0d91587568594a616c7f3588bc74106312761 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:20:18 +0100
Subject: [PATCH 168/411] [type/__hostname] Fix OS version detection for SuSE
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

everything should be suse now…
---
 cdist/conf/type/__hostname/manifest | 39 ++++++++++++++++-------------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index d044072b..b80aa2ef 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -25,8 +25,6 @@ set_hostname_systemd() {
 }
 
 os=$(cat "${__global:?}/explorer/os")
-os_version=$(cat "${__global:?}/explorer/os_version")
-os_major=$(echo "${os_version}" | grep -o '^[0-9][0-9]*' || true)
 
 max_len=$(cat "${__object:?}/explorer/max_len")
 has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl")
@@ -38,24 +36,10 @@ else
     case ${os}
     in
         # RedHat-derivatives and BSDs
-        (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware)
+        (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware|suse)
             # Hostname is FQDN
             name_should=${__target_host:?}
             ;;
-        (suse)
-            name_should=${__target_host:?}
-
-            # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
-            # systemd does not. The running hostname is the first
-            # component in both cases.
-            # In versions before 15.x, the FQDN is stored in /etc/hostname.
-            if test -n "${has_hostnamectl}" \
-                    && test "${os_major}" -ge 15 \
-                    && test "${os_major}" -ne 42
-            then
-                name_should=${name_should%%.*}
-            fi
-            ;;
         *)
             # Hostname is only first component of FQDN on all other systems.
             name_should=${__target_host:?}
@@ -158,6 +142,27 @@ in
         echo "${name_should}" | __file /etc/nodename --source -
         ;;
     (suse)
+        if test -s "${__global:?}/explorer/os_release"
+        then
+            # shellcheck source=/dev/null
+            os_version=$(. "${__global:?}/explorer/os_release" && echo "${VERSION}")
+        else
+            os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global:?}/explorer/os_version")
+        fi
+        os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
+
+        # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
+        # systemd does not. The running hostname is the first
+        # component in both cases.
+        # In versions before 15.x, the FQDN is stored in /etc/hostname.
+        if test -n "${has_hostnamectl}" \
+                && test "${os_major}" -ge 15 \
+                && test "${os_major}" -ne 42
+        then
+            # strip away everything but the first part from $name_should
+            name_should=${name_should%%.*}
+        fi
+
         # Modern SuSE provides /etc/HOSTNAME as a symlink for
         # backwards-compatibility. Unfortunately it cannot be used
         # here as __file does not follow the symlink.

From 21dd500c05c91c4927b2a4166690da65b1c98f97 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:44:44 +0100
Subject: [PATCH 169/411] Make pycodestyle pipeline happy

---
 cdist/__init__.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index 1c60ae0f..44366cd0 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -33,14 +33,15 @@ try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
-    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+    cdist_dir = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), os.pardir))
     if os.path.isdir(os.path.join(cdist_dir, '.git')):
         try:
             VERSION = subprocess.check_output(
                 ['git', 'describe', '--always'],
                 cwd=cdist_dir,
                 universal_newlines=True)
-        except:
+        except Exception:
             pass
 
 BANNER = """

From e2d4f8037ab7d30187e4d153aa25ef36e8afc91c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:45:05 +0100
Subject: [PATCH 170/411] [bin/cdist-build-helper] Fix paths to ex scripts/
 scripts

---
 bin/cdist-build-helper | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper
index bdef0dbb..fb8a67a4 100755
--- a/bin/cdist-build-helper
+++ b/bin/cdist-build-helper
@@ -45,7 +45,7 @@ usage() {
         shellcheck-manifests
         shellcheck-local-gencodes
         shellcheck-remote-gencodes
-        shellcheck-scripts
+        shellcheck-bin
         shellcheck-gencodes
         shellcheck-types
         shellcheck
@@ -100,7 +100,7 @@ case "$option" in
                     if (\$0 ~ /^$end/) {
                         exit
                     } else {
-                        print \$0 
+                        print \$0
                     }
                 }
             }" "$basedir/docs/changelog"
@@ -135,7 +135,7 @@ case "$option" in
 
         version=$1; shift
 
-        ( 
+        (
         cat << eof
 Subject: cdist $version has been released
 
@@ -336,7 +336,7 @@ eof
         make docs-clean
         make docs
 
-        ############################################################# 
+        #############################################################
         # Everything green, let's do the release
 
         # Tag the current commit
@@ -405,7 +405,7 @@ eof
     ;;
 
     pycodestyle|pep8)
-        pycodestyle "${basedir}" "${basedir}/scripts/cdist"
+        pycodestyle "${basedir}" "${basedir}/bin/cdist"
     ;;
 
     check-pycodestyle)
@@ -460,9 +460,10 @@ eof
         test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
-    shellcheck-scripts)
+    # NOTE: shellcheck-scripts is kept for compatibility
+    shellcheck-bin|shellcheck-scripts)
         # shellcheck disable=SC2086
-        ${SHELLCHECKCMD} scripts/cdist-dump scripts/cdist-new-type > "${SHELLCHECKTMP}"
+        ${SHELLCHECKCMD} bin/cdist-dump bin/cdist-new-type > "${SHELLCHECKTMP}"
         test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
@@ -480,7 +481,7 @@ eof
     shellcheck)
         "$0" shellcheck-global-explorers || exit 1
         "$0" shellcheck-types || exit 1
-        "$0" shellcheck-scripts || exit 1
+        "$0" shellcheck-bin || exit 1
     ;;
 
     shellcheck-type-files)

From f82e0167aaf295b8451a07fa931506a1290efcdc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:49:04 +0100
Subject: [PATCH 171/411] [.gitlab-ci.yml] Make version before other targets

---
 .gitlab-ci.yml | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e48355ea..a4bc67aa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,20 +1,23 @@
+---
+image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
+
 stages:
     - test
 
-image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
+before_script:
+    - ./bin/cdist-build-helper version
 
-unit_tests:
+shellcheck:
     stage: test
     script:
-        - ./bin/cdist-build-helper version
-        - ./bin/cdist-build-helper test
+        - ./bin/cdist-build-helper shellcheck
 
 pycodestyle:
     stage: test
     script:
         - ./bin/cdist-build-helper pycodestyle
 
-shellcheck:
+unit_tests:
     stage: test
     script:
-        - ./bin/cdist-build-helper shellcheck
+        - ./bin/cdist-build-helper test

From 0ee3fda94deff1461ac488d218a2af98057675c5 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 15:04:51 +0100
Subject: [PATCH 172/411] Fix paths to cdist executable

---
 cdist/integration.py          | 5 ++---
 cdist/test/__init__.py        | 2 +-
 cdist/test/config/__init__.py | 4 ++--
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/cdist/integration.py b/cdist/integration.py
index 03e4167d..17b65f09 100644
--- a/cdist/integration.py
+++ b/cdist/integration.py
@@ -54,13 +54,12 @@ _mydir = os.path.dirname(__file__)
 def find_cdist_exec():
     """Search cdist executable starting from local lib directory.
 
-    Detect if ../scripts/cdist (from local lib direcotry) exists and
+    Detect if ../bin/cdist (from local lib directory) exists and
     if it is executable. If not then try to find cdist exec path in
     os.get_exec_path() entries. If no cdist path is found rasie
     cdist.Error.
     """
-    cdist_path = os.path.abspath(os.path.join(_mydir, '..', 'scripts',
-                                              'cdist'))
+    cdist_path = os.path.abspath(os.path.join(_mydir, '..', 'bin', 'cdist'))
     if os.access(cdist_path, os.X_OK):
         return cdist_path
     cdist_path = find_cdist_exec_in_path()
diff --git a/cdist/test/__init__.py b/cdist/test/__init__.py
index faa3686a..16a311e4 100644
--- a/cdist/test/__init__.py
+++ b/cdist/test/__init__.py
@@ -26,7 +26,7 @@ import tempfile
 cdist_base_path = os.path.abspath(
     os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../"))
 
-cdist_exec_path = os.path.join(cdist_base_path, "scripts/cdist")
+cdist_exec_path = os.path.join(cdist_base_path, "bin/cdist")
 
 global_fixtures_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                    "fixtures"))
diff --git a/cdist/test/config/__init__.py b/cdist/test/config/__init__.py
index 0ed614b1..c405207c 100644
--- a/cdist/test/config/__init__.py
+++ b/cdist/test/config/__init__.py
@@ -202,7 +202,7 @@ class ConfigRunTestCase(test.CdistTestCase):
             host_dir_name=self.hostdir,
             # exec_path can not derivated from sys.argv in case of unittest
             exec_path=os.path.abspath(os.path.join(
-                my_dir, '../../../scripts/cdist')),
+                my_dir, '../../../bin/cdist')),
             initial_manifest=os.path.join(fixtures,
                                           'manifest/dryrun_manifest'),
             add_conf_dirs=[fixtures])
@@ -219,7 +219,7 @@ class ConfigRunTestCase(test.CdistTestCase):
             base_root_path=self.host_base_path,
             host_dir_name=self.hostdir,
             exec_path=os.path.abspath(os.path.join(
-                my_dir, '../../../scripts/cdist')),
+                my_dir, '../../../bin/cdist')),
             initial_manifest=os.path.join(
                 fixtures, 'manifest/init-deps-resolver'),
             add_conf_dirs=[fixtures])

From c39eb1dbce281ed6f64f804a16fb28e5c8dceccd Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 15:16:33 +0100
Subject: [PATCH 173/411] [cdist.emulator] Fix setting of log level (tests OK)

---
 cdist/emulator.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cdist/emulator.py b/cdist/emulator.py
index 60b94880..a2bdc3d4 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -120,17 +120,18 @@ class Emulator:
                 level = logging.WARNING
         else:
             level = logging.WARNING
+        self.log = logging.getLogger(self.target_host[0])
         try:
             logging.root.setLevel(level)
+            self.log.setLevel(level)
         except (ValueError, TypeError):
             # if invalid __cdist_log_level value
             logging.root.setLevel(logging.WARNING)
+            self.log.setLevel(logging.WARNING)
 
         colored_log = self.env.get('__cdist_colored_log', 'false')
         cdist.log.CdistFormatter.USE_COLORS = colored_log == 'true'
 
-        self.log = logging.getLogger(self.target_host[0])
-
     def commandline(self):
         """Parse command line"""
 

From 2f70a8b540f8d4c283029207e98f7883557338f0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 15:25:46 +0100
Subject: [PATCH 174/411] [bin/cdist-build-helper] Keep going in shellcheck
 targets

---
 bin/cdist-build-helper | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper
index fb8a67a4..0380b3f8 100755
--- a/bin/cdist-build-helper
+++ b/bin/cdist-build-helper
@@ -468,20 +468,26 @@ eof
     ;;
 
     shellcheck-gencodes)
-        "$0" shellcheck-local-gencodes || exit 1
-        "$0" shellcheck-remote-gencodes || exit 1
+        errors=false
+        "$0" shellcheck-local-gencodes || errors=true
+        "$0" shellcheck-remote-gencodes || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck-types)
-        "$0" shellcheck-type-explorers || exit 1
-        "$0" shellcheck-manifests || exit 1
-        "$0" shellcheck-gencodes || exit 1
+        errors=false
+        "$0" shellcheck-type-explorers || errors=true
+        "$0" shellcheck-manifests || errors=true
+        "$0" shellcheck-gencodes || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck)
-        "$0" shellcheck-global-explorers || exit 1
-        "$0" shellcheck-types || exit 1
-        "$0" shellcheck-bin || exit 1
+        errors=false
+        "$0" shellcheck-global-explorers || errors=true
+        "$0" shellcheck-types || errors=true
+        "$0" shellcheck-bin || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck-type-files)
@@ -491,8 +497,10 @@ eof
     ;;
 
     shellcheck-with-files)
-        "$0" shellcheck || exit 1
-        "$0" shellcheck-type-files || exit 1
+        errors=false
+        "$0" shellcheck || errors=true
+        "$0" shellcheck-type-files || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck-build-helper)

From ebf471e8d0216cc81928fcdff1fe159584187808 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 13 Nov 2020 02:32:45 +0100
Subject: [PATCH 175/411] [type/__hwclock] Add new type

---
 .../conf/type/__hwclock/explorer/adjtime_mode |  28 +++
 .../__hwclock/explorer/timedatectl_localrtc   |  27 +++
 cdist/conf/type/__hwclock/gencode-remote      |  62 +++++
 cdist/conf/type/__hwclock/man.rst             |  63 +++++
 cdist/conf/type/__hwclock/manifest            | 222 ++++++++++++++++++
 cdist/conf/type/__hwclock/parameter/required  |   1 +
 cdist/conf/type/__hwclock/singleton           |   0
 7 files changed, 403 insertions(+)
 create mode 100755 cdist/conf/type/__hwclock/explorer/adjtime_mode
 create mode 100755 cdist/conf/type/__hwclock/explorer/timedatectl_localrtc
 create mode 100755 cdist/conf/type/__hwclock/gencode-remote
 create mode 100644 cdist/conf/type/__hwclock/man.rst
 create mode 100755 cdist/conf/type/__hwclock/manifest
 create mode 100644 cdist/conf/type/__hwclock/parameter/required
 create mode 100644 cdist/conf/type/__hwclock/singleton

diff --git a/cdist/conf/type/__hwclock/explorer/adjtime_mode b/cdist/conf/type/__hwclock/explorer/adjtime_mode
new file mode 100755
index 00000000..2b27bedc
--- /dev/null
+++ b/cdist/conf/type/__hwclock/explorer/adjtime_mode
@@ -0,0 +1,28 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Prints the clock mode read from the /etc/adjtime file, if present.
+#
+
+# not all operating systems use an adjfile
+test -f /etc/adjtime || exit 0
+
+# 3rd line is clock mode
+# adjtime(5) https://man7.org/linux/man-pages/man5/adjtime.5.html
+sed -n 3p /etc/adjtime
diff --git a/cdist/conf/type/__hwclock/explorer/timedatectl_localrtc b/cdist/conf/type/__hwclock/explorer/timedatectl_localrtc
new file mode 100755
index 00000000..8239122e
--- /dev/null
+++ b/cdist/conf/type/__hwclock/explorer/timedatectl_localrtc
@@ -0,0 +1,27 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Prints the LocalRTC property using timedatectl on systemd-based systems.
+#
+
+command -v timedatectl >/dev/null 2>&1 || exit 0
+
+# NOTE: Older versions of timedatectl do not support `timedatectl show'
+timedatectl --no-pager status \
+| awk -F': ' '$1 ~ "RTC in local TZ$" { sub(/[ \t]*$/, "", $2); print $2 }'
diff --git a/cdist/conf/type/__hwclock/gencode-remote b/cdist/conf/type/__hwclock/gencode-remote
new file mode 100755
index 00000000..5995fb23
--- /dev/null
+++ b/cdist/conf/type/__hwclock/gencode-remote
@@ -0,0 +1,62 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+mode=$(cat "${__object:?}/parameter/mode")
+
+timedatectl_localrtc=$(cat "${__object:?}/explorer/timedatectl_localrtc")
+adjtime_mode=$(cat "${__object:?}/explorer/adjtime_mode")
+
+
+case ${mode}
+in
+	(localtime)
+		adjtime_str=LOCAL
+		local_rtc_str=yes
+		;;
+	(UTC|utc)
+		adjtime_str=UTC
+		local_rtc_str=no
+		;;
+	(*)
+		printf 'Invalid value for --mode: %s\n' "${mode}" >&2
+		printf 'Acceptable values are: localtime, utc.\n' >&2
+		exit 1
+esac
+
+
+if test -n "${timedatectl_localrtc}"
+then
+	# systemd
+	timedatectl_should=${local_rtc_str}
+	if test "${timedatectl_localrtc}" != "${timedatectl_should}"
+	then
+		printf 'timedatectl set-local-rtc %s\n' "${timedatectl_should}"
+	fi
+elif test -n "${adjtime_mode}"
+then
+	# others (update /etc/adjtime if present)
+	if test "${adjtime_mode}" != "${adjtime_str}"
+	then
+		# Update /etc/adjtime (3rd line is clock mode)
+		# adjtime(5) https://man7.org/linux/man-pages/man5/adjtime.5.html
+		# FIXME: Should maybe add third line if adjfile only contains two lines
+		printf "sed -i '3c\\\\\\n%s\\n' /etc/adjtime\\n" "${adjtime_str}"
+	fi
+fi
diff --git a/cdist/conf/type/__hwclock/man.rst b/cdist/conf/type/__hwclock/man.rst
new file mode 100644
index 00000000..65eb648f
--- /dev/null
+++ b/cdist/conf/type/__hwclock/man.rst
@@ -0,0 +1,63 @@
+cdist-type__hwclock(7)
+======================
+
+NAME
+----
+cdist-type__hwclock - Manage the hardware real time clock.
+
+
+DESCRIPTION
+-----------
+This type can be used to control how the hardware clock is used by the operating
+system.
+
+
+REQUIRED PARAMETERS
+-------------------
+mode
+    What mode the hardware clock is in.
+
+    Acceptable values:
+
+    localtime
+        The hardware clock is set to local time (common for systems also running
+        Windows.)
+    UTC
+        The hardware clock is set to UTC (common on UNIX systems.)
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Make the operating system treat the time read from the hwclock as UTC.
+    __hwclock --mode UTC
+
+
+SEE ALSO
+--------
+:strong:`hwclock`\ (8)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__hwclock/manifest b/cdist/conf/type/__hwclock/manifest
new file mode 100755
index 00000000..7d9ab88f
--- /dev/null
+++ b/cdist/conf/type/__hwclock/manifest
@@ -0,0 +1,222 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# TODO: Consider supporting BADYEAR
+
+os=$(cat "${__global:?}/explorer/os")
+mode=$(cat "${__object:?}/parameter/mode")
+
+has_systemd_timedatectl=$(test -s "${__object:?}/explorer/timedatectl_localrtc" && echo true || echo false)
+
+
+case ${mode}
+in
+	(localtime)
+		local_clock=true
+		;;
+	(UTC|utc)
+		local_clock=false
+		;;
+	(*)
+		printf 'Invalid value for --mode: %s\n' "${mode}" >&2
+		printf 'Acceptable values are: UTC, localtime.\n' >&2
+		exit 1
+esac
+
+
+case ${os}
+in
+	(alpine|gentoo)
+		if ! $has_systemd_timedatectl
+		then
+			# NOTE: Gentoo also supports systemd, in which case /etc/conf.d is
+			#       not used. So we check for systemd presence here and only
+			#       update /etc/conf.d if systemd is not installed.
+			# https://wiki.gentoo.org/wiki/System_time#Hardware_clock
+
+			export CDIST_ORDER_DEPENDENCY=true
+			__file /etc/conf.d/hwclock --state present \
+				--owner root --group root --mode 0644
+			__key_value /etc/conf.d/hwclock:clock \
+				--file /etc/conf.d/hwclock \
+				--key clock \
+				--delimiter '=' --exact_delimiter \
+				--value "\"$($local_clock && echo local || echo UTC)\""
+			unset CDIST_ORDER_DEPENDENCY
+		fi
+		;;
+	(centos|fedora|redhat|scientific)
+		os_version=$(cat "${__global:?}/explorer/os_version")
+		os_major=$(expr "${os_version}" : '.* release \([0-9]*\)')
+		case ${os}
+		in
+			(centos|scientific)
+				update_sysconfig=$(test "${os_major}" -lt 6 && echo true || echo false)
+				;;
+			(fedora)
+				update_sysconfig=$(test "${os_major}" -lt 10 && echo true || echo false)
+				;;
+			(redhat|*)
+				case ${os_version}
+				in
+					('Red Hat Enterprise Linux'*)
+						update_sysconfig=$(test "${os_major}" -lt 6 && echo true || echo false)
+						;;
+					('Red Hat Linux'*)
+						update_sysconfig=true
+						;;
+					(*)
+						printf 'Could not determine Red Hat distribution.\n' >&2
+						printf "Please contribute an implementation for it if you can.\n" >&2
+						exit 1
+						;;
+				esac
+				;;
+		esac
+
+		if ${update_sysconfig:?}
+		then
+			export CDIST_ORDER_DEPENDENCY=true
+			__file /etc/sysconfig/clock --state present \
+				--owner root --group root --mode 0644
+			__key_value /etc/sysconfig/clock:UTC \
+				--file /etc/sysconfig/clock \
+				--key UTC \
+				--delimiter '=' --exact_delimiter \
+				--value "$($local_clock && echo false || echo true)"
+			unset CDIST_ORDER_DEPENDENCY
+		fi
+		;;
+	(debian|devuan|ubuntu)
+		os_major=$(sed 's/[^0-9].*$//' "${__global:?}/explorer/os_version")
+
+		case ${os}
+		in
+			(debian)
+				if test "${os_major}" -ge 7
+				then
+					update_rcS=false
+				elif test "${os_major}" -ge 3
+				then
+					update_rcS=true
+				else
+					# Debian 2.2 should be supportable using rcS.
+					# Debian 2.1 uses the ancient GMT key.
+					# Debian 1.3 does not have rcS.
+					printf "Your operating system (Debian %s) is currently not supported by this type (%s)\n" \
+						"$(cat "${__global:?}/explorer/os_version")" "${__type##*/}" >&2
+					printf "Please contribute an implementation for it if you can.\n" >&2
+					exit 1
+				fi
+				;;
+			(devuan)
+				update_rcS=false
+				;;
+			(ubuntu)
+				update_rcS=$(test "${os_major}" -lt 16 && echo true || echo false)
+				;;
+		esac
+
+		if ${update_rcS}
+		then
+			export CDIST_ORDER_DEPENDENCY=true
+			__file /etc/default/rcS --state present \
+				--owner root --group root --mode 0644
+			__key_value /etc/default/rcS:UTC \
+				--file /etc/default/rcS \
+				--key UTC \
+				--delimiter '=' --exact_delimiter \
+				--value "$($local_clock && echo no || echo yes)"
+			unset CDIST_ORDER_DEPENDENCY
+		fi		  
+		;;
+	(freebsd)
+		# cf. adjkerntz(8)
+		__file /etc/wall_cmos_clock \
+			--state "$($local_clock && echo present || echo absent)" \
+			--owner root --group wheel --mode 0444
+		;;
+	(netbsd)
+		# https://wiki.netbsd.org/guide/boot/#index9h2
+		__key_value /etc/rc.conf:rtclocaltime \
+			--file /etc/rc.conf \
+			--key rtclocaltime \
+			--delimiter '=' --exact_delimiter \
+			--value "$($local_clock && echo YES || echo NO)"
+		;;
+	(slackware)
+		__file /etc/hardwareclock --owner root --group root --mode 0644 \
+			--source - <<-EOF
+		# /etc/hardwareclock
+		#
+		# Tells how the hardware clock time is stored.
+		# This file is managed by cdist.
+
+		$($local_clock && echo localtime || echo UTC)
+		EOF
+		;;
+	(suse)
+		if test -s "${__global:?}/explorer/os_release"
+		then
+			# shellcheck source=/dev/null
+			os_version=$(. "${__global:?}/explorer/os_release" && echo "${VERSION}")
+		else
+			os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global:?}/explorer/os_version")
+		fi
+		os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
+
+		# TODO: Consider using `yast2 timezone set hwclock' instead
+		if expr "${os_major}" \< 12
+		then
+			# Starting with SuSE 12 (first systemd-based version)
+			# /etc/sysconfig/clock does not contain the HWCLOCK line
+			# anymore.
+			# With SuSE 13, it has been reduced to TIMEZONE configuration.
+			__key_value /etc/sysconfig/clock:HWCLOCK \
+				--file /etc/sysconfig/clock \
+				--delimiter '=' --exact_delimiter \
+				--key HWCLOCK \
+				--value "$($local_clock && echo '"--localtime"' || echo '"-u"')"
+		fi
+		;;
+	(void)
+		export CDIST_ORDER_DEPENDENCY=true
+		__file /etc/rc.conf \
+			--owner root --group root --mode 0644 \
+			--state present
+		__key_value /etc/rc.conf:HARDWARECLOCK \
+			--file /etc/rc.conf \
+			--delimiter '=' --exact_delimiter \
+			--key HARDWARECLOCK \
+			--value "\"$($local_clock && echo localtime || echo UTC)\""
+		unset CDIST_ORDER_DEPENDENCY
+		;;
+	(*)
+		if ! $has_systemd_timedatectl
+		then
+			printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2
+			printf "Please contribute an implementation for it if you can.\n" >&2
+			exit 1
+		fi
+		;;
+esac
+
+# NOTE: timedatectl set-local-rtc for systemd is in gencode-remote
+# NOTE: /etc/adjtime is also updated in gencode-remote
diff --git a/cdist/conf/type/__hwclock/parameter/required b/cdist/conf/type/__hwclock/parameter/required
new file mode 100644
index 00000000..17ab372f
--- /dev/null
+++ b/cdist/conf/type/__hwclock/parameter/required
@@ -0,0 +1 @@
+mode
diff --git a/cdist/conf/type/__hwclock/singleton b/cdist/conf/type/__hwclock/singleton
new file mode 100644
index 00000000..e69de29b

From a07a45887136105cc8be316307805ceaecdd4cf3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 13 Nov 2020 06:43:01 +0100
Subject: [PATCH 176/411] ++changelog

---
 docs/changelog | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 9fa0926c..ff411a46 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,7 +4,8 @@ Changelog
 next:
 	* Documentation: Fix examples in best practice (Dennis Camera)
 	* Type __locale: Add state explorer (Matthias Stecher)
-	* Core: Reorganize scripts, version generation (Ander Punnar)
+	* Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera)
+	* New type: __hwclock (Dennis Camera)
 
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)

From 50927527863fd382f5200287a808d8b36028ff87 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 14 Nov 2020 09:59:30 +0100
Subject: [PATCH 177/411] Update build helper script in .gitattributes

---
 .gitattributes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitattributes b/.gitattributes
index 45c10d7b..01d20f30 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -4,5 +4,5 @@
 docs/speeches export-ignore
 docs/video export-ignore
 docs/src/man7 export-ignore
-bin/build-helper export-ignore
+bin/cdist-build-helper export-ignore
 README-maintainers export-ignore

From 76aa00b12e9b675c797e997a06ec7597c53a85f3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 14 Nov 2020 10:13:58 +0100
Subject: [PATCH 178/411] Fix importing cdist module

Resolve #845.
---
 bin/cdist | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index 1f92f157..ddaffa7f 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -25,25 +25,23 @@ import logging
 import os
 import sys
 
-# try to import cdist and if that fails,
-# then add this file's parent dir to
-# module search path and try again.
-try:
-    import cdist
-except ModuleNotFoundError:
-    cdist_dir = os.path.realpath(
-        os.path.join(
-            os.path.dirname(os.path.realpath(__file__)),
-            os.pardir))
+# See if this file's parent is cdist module
+# and if so add it to module search path.
+cdist_dir = os.path.realpath(
+    os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        os.pardir))
+cdist_init_dir = os.path.join(cdist_dir, 'cdist', '__init__.py')
+if os.path.exists(cdist_init_dir):
     sys.path.insert(0, cdist_dir)
-    import cdist
 
-import cdist.argparse
-import cdist.banner
-import cdist.config
-import cdist.install
-import cdist.shell
-import cdist.inventory
+import cdist            # noqa 402
+import cdist.argparse   # noqa 402
+import cdist.banner     # noqa 402
+import cdist.config     # noqa 402
+import cdist.install    # noqa 402
+import cdist.shell      # noqa 402
+import cdist.inventory  # noqa 402
 
 
 def commandline():

From f75d4772090d3a9cfd798dd193cd324083a9b3a7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 13:45:12 +0100
Subject: [PATCH 179/411] Deprecate __locale and replace with __localedef

---
 cdist/conf/type/__locale/deprecated           |  1 +
 cdist/conf/type/__localedef/gencode-remote    | 60 +++++++++++++++++++
 cdist/conf/type/__localedef/man.rst           | 55 +++++++++++++++++
 cdist/conf/type/__localedef/manifest          | 41 +++++++++++++
 .../type/__localedef/parameter/default/state  |  1 +
 .../conf/type/__localedef/parameter/optional  |  1 +
 6 files changed, 159 insertions(+)
 create mode 100644 cdist/conf/type/__locale/deprecated
 create mode 100755 cdist/conf/type/__localedef/gencode-remote
 create mode 100644 cdist/conf/type/__localedef/man.rst
 create mode 100755 cdist/conf/type/__localedef/manifest
 create mode 100644 cdist/conf/type/__localedef/parameter/default/state
 create mode 100644 cdist/conf/type/__localedef/parameter/optional

diff --git a/cdist/conf/type/__locale/deprecated b/cdist/conf/type/__locale/deprecated
new file mode 100644
index 00000000..5a06b28e
--- /dev/null
+++ b/cdist/conf/type/__locale/deprecated
@@ -0,0 +1 @@
+This type is deprecated. Please use __localedef instead.
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
new file mode 100755
index 00000000..1feb9884
--- /dev/null
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -0,0 +1,60 @@
+#!/bin/sh -e
+#
+# 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Let localedef do the magic
+#
+
+locale="$__object_id"
+
+# Hardcoded, create a pull request with
+# branching on $os in case it is at another location
+alias=/usr/share/locale/locale.alias
+
+input=$(echo "$locale" | cut -d . -f 1)
+charmap=$(echo "$locale" | cut -d . -f 2)
+
+# Adding locale? The name is de_CH.UTF-8
+# Removing locale? The name is de_CH.utf8.
+# W-T-F!
+locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
+
+state=$(cat "$__object/parameter/state")
+
+os=$(cat "$__global/explorer/os")
+
+# Nothing to be done on alpine
+case "$os" in
+    alpine)
+        exit 0
+        ;;
+esac
+
+case "$state" in
+    present)
+        echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
+    ;;
+    absent)
+        echo localedef --delete-from-archive "$locale_remove"
+    ;;
+    *)
+        echo "Unsupported state: $state" >&2
+        exit 1
+    ;;
+esac
diff --git a/cdist/conf/type/__localedef/man.rst b/cdist/conf/type/__localedef/man.rst
new file mode 100644
index 00000000..0ccf484a
--- /dev/null
+++ b/cdist/conf/type/__localedef/man.rst
@@ -0,0 +1,55 @@
+cdist-type__localedef(7)
+========================
+
+NAME
+----
+cdist-type__localedef - Define and remove system locales
+
+
+DESCRIPTION
+-----------
+This cdist type allows you to define locales on the system using
+:strong:`localedef`\ (1) or remove them.
+On systems that don't support definition of new locales, the type will raise an
+error.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+   ``present`` or ``absent``. Defaults to ``present``.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Add locale de_CH.UTF-8
+    __localedef de_CH.UTF-8
+
+    # Same as above, but more explicit
+    __localedef de_CH.UTF-8 --state present
+
+    # Remove colourful British English
+    __localedef en_GB.UTF-8 --state absent
+
+
+SEE ALSO
+--------
+:strong:`locale`\ (1),
+:strong:`localedef`\ (1),
+:strong:`cdist-type__locale_system`\ (7)
+
+
+AUTHORS
+-------
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+
+
+COPYING
+-------
+Copyright \(C) 2013-2019 Nico Schottelius, 2020 Dennis Camera. Free use of this
+software is granted under the terms of the GNU General Public License version 3
+or later (GPLv3+).
diff --git a/cdist/conf/type/__localedef/manifest b/cdist/conf/type/__localedef/manifest
new file mode 100755
index 00000000..9f1e17ac
--- /dev/null
+++ b/cdist/conf/type/__localedef/manifest
@@ -0,0 +1,41 @@
+#!/bin/sh -e
+#
+# 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2015 David Hürlimann (david at ungleich.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Install required packages
+#
+
+os=$(cat "$__global/explorer/os")
+
+
+case "$os" in
+    debian|devuan)
+        # Debian needs a seperate package
+        __package locales --state present
+    ;;
+    archlinux|suse|ubuntu|scientific|centos|alpine)
+        :
+    ;;
+    *)
+        echo "Sorry, do not know how to handle os: $os" >&2
+        echo "Please edit the type ${__type##*/} to fix this." >&2
+        exit 1
+    ;;
+esac
diff --git a/cdist/conf/type/__localedef/parameter/default/state b/cdist/conf/type/__localedef/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__localedef/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__localedef/parameter/optional b/cdist/conf/type/__localedef/parameter/optional
new file mode 100644
index 00000000..ff72b5c7
--- /dev/null
+++ b/cdist/conf/type/__localedef/parameter/optional
@@ -0,0 +1 @@
+state

From 54e689f7c2b3bc25aedb2aaf9f5aece289ec5b66 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 13:49:04 +0100
Subject: [PATCH 180/411] [type/__localedef] Add state explorer

---
 cdist/conf/type/__localedef/explorer/state | 71 ++++++++++++++++++++++
 cdist/conf/type/__localedef/gencode-remote | 13 +++-
 2 files changed, 82 insertions(+), 2 deletions(-)
 create mode 100755 cdist/conf/type/__localedef/explorer/state

diff --git a/cdist/conf/type/__localedef/explorer/state b/cdist/conf/type/__localedef/explorer/state
new file mode 100755
index 00000000..d8db29c5
--- /dev/null
+++ b/cdist/conf/type/__localedef/explorer/state
@@ -0,0 +1,71 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer determines if the locale is defined on the target system.
+# Will print nothing on error.
+#
+# Possible output:
+#  present:
+#    the main locale (and possibly aliases) is present
+#  absent:
+#    neither the main locale nor any aliases are present
+#
+
+command -v locale >/dev/null 2>&1 || exit 0
+
+locales=$(locale -a)
+
+parse_locale() {
+	# This function will split locales into their parts. Locale strings are
+	# usually of the form: [language[_territory][.codeset][@modifier]]
+	# For simplicity, language and territory are not separated by this function.
+	# Old Linux systems were also using "english" or "german" as locale strings.
+	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
+	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
+	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
+	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
+}
+
+format_locale() {
+	# Usage: format_locale language codeset modifier
+	printf '%s' "$1"
+	test -z "$2" || printf '.%s' "$2"
+	test -z "$3" || printf '@%s' "$3"
+	printf '\n'
+}
+
+gnu_normalize_codeset() {
+	# reimplementation of glibc/locale/programs/localedef.c normalize_codeset()
+	echo "$*" | tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]'
+}
+
+locale_available() (
+	echo "${locales}" | grep -qxF "$1" || {
+		# glibc uses "normalized" locale names in archives.
+		# If a locale is stored in an archive, the normalized name will be
+		# printed by locale, so that needs to be checked, too.
+		localename=$(
+			parse_locale "$1" _lang _codeset _modifier \
+			&& format_locale "${_lang:?}" "$(gnu_normalize_codeset "${_codeset?}")" \
+			   "${_modifier?}")
+		echo "${locales}" | grep -qxF "${localename}"
+	}
+)
+
+locale_available "${__object_id:?}" && echo present || echo absent
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 1feb9884..9ea3f6f1 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -35,7 +35,8 @@ charmap=$(echo "$locale" | cut -d . -f 2)
 # W-T-F!
 locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
 
-state=$(cat "$__object/parameter/state")
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 
 os=$(cat "$__global/explorer/os")
 
@@ -46,7 +47,15 @@ case "$os" in
         ;;
 esac
 
-case "$state" in
+# NOTE: If state explorer fails (e.g. locale(1) missing), the following check
+#       will always fail and let definition/removal run.
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
+
+case ${state_should}
+in
     present)
         echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
     ;;

From cc29e54b851212568c3f6eddf4bf0a64c1eabb0d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 23 Jul 2020 23:20:39 +0200
Subject: [PATCH 181/411] [type/__localedef] Differentiate between OSes and
 better handling of normalized locale names

---
 cdist/conf/type/__localedef/gencode-remote | 117 +++++++++++++++------
 1 file changed, 82 insertions(+), 35 deletions(-)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 9ea3f6f1..b9cc68e8 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,35 +18,16 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+# Manage system locales using localedef(1).
 #
-# Let localedef do the magic
-#
-
-locale="$__object_id"
-
-# Hardcoded, create a pull request with
-# branching on $os in case it is at another location
-alias=/usr/share/locale/locale.alias
-
-input=$(echo "$locale" | cut -d . -f 1)
-charmap=$(echo "$locale" | cut -d . -f 2)
-
-# Adding locale? The name is de_CH.UTF-8
-# Removing locale? The name is de_CH.utf8.
-# W-T-F!
-locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
 
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
-os=$(cat "$__global/explorer/os")
-
-# Nothing to be done on alpine
-case "$os" in
-    alpine)
-        exit 0
-        ;;
-esac
+test "${state_should}" = 'present' -o "${state_should}" = 'absent' || {
+	printf 'Invalid state: %s\n' "${state_should}" >&2
+	exit 1
+}
 
 # NOTE: If state explorer fails (e.g. locale(1) missing), the following check
 #       will always fail and let definition/removal run.
@@ -54,16 +36,81 @@ then
 	exit 0
 fi
 
-case ${state_should}
+locale=${__object_id:?}
+os=$(cat "${__global:?}/explorer/os")
+
+if expr "${locale}" : '.*/' >/dev/null
+then
+	printf 'Paths as locales are not supported.\n' >&2
+	printf '__object_id is: %s\n' "${locale}" >&2
+	exit 1
+fi
+
+parse_locale() {
+	# This function will split locales into their parts. Locale strings are
+	# usually of the form: [language[_territory][.codeset][@modifier]]
+	# For simplicity, language and territory are not separated by this function.
+	# Old Linux systems were also using "english" or "german" as locale strings.
+	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
+	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
+	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
+	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
+}
+
+format_locale() {
+	# Usage: format_locale language codeset modifier
+	printf '%s' "$1"
+	test -z "$2" || printf '.%s' "$2"
+	test -z "$3" || printf '@%s' "$3"
+	printf '\n'
+}
+
+gnu_normalize_codeset() {
+	echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]'
+}
+
+
+: "${lang=}" "${codeset=}" "${modifier=}"  # declare variables for shellcheck
+parse_locale "${locale}" lang codeset modifier
+
+
+case ${os}
 in
-    present)
-        echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
-    ;;
-    absent)
-        echo localedef --delete-from-archive "$locale_remove"
-    ;;
-    *)
-        echo "Unsupported state: $state" >&2
-        exit 1
-    ;;
+	(alpine|openwrt)
+		printf '%s does not support locales.\n' "${os}" >&2
+		exit 1
+		;;
+	(archlinux|debian|devuan|ubuntu|suse|centos|fedora|redhat|scientific)
+		# FIXME: The code below only works for glibc-based installations.
+
+		# NOTE: Hardcoded, create a pull request in case it is at another
+		#       location for some opther distro.
+		# NOTE: locale.alias can be symlinked (e.g. Debian)
+		aliasfile='/usr/share/locale/locale.alias'
+
+		case ${state_should}
+		in
+			(present)
+				input=$(format_locale "${lang}" '' "${modifier}")
+				cat <<-EOF
+				set --
+				if test -e '${aliasfile}'
+				then
+					set -- -A '${aliasfile}'
+				fi
+
+				localedef -i '${input}' -f '${codeset}' "\$@" '${locale}'
+				EOF
+				;;
+			(absent)
+				localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}")
+				printf "localedef --delete-from-archive '%s'\n" "${localename}"
+				;;
+		esac
+		;;
+	(*)
+		echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
+		echo "Please contribute an implementation for it if you can." >&2
+		exit 1
+		;;
 esac

From f44888f192d78d8ce1e3583f70f94b0c2039c1e2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 14:02:08 +0100
Subject: [PATCH 182/411] [type/__localedef] Only install dependencies in
 manifest. OS checking moved to gencode-remote

---
 cdist/conf/type/__localedef/manifest | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__localedef/manifest b/cdist/conf/type/__localedef/manifest
index 9f1e17ac..3ab3ad8c 100755
--- a/cdist/conf/type/__localedef/manifest
+++ b/cdist/conf/type/__localedef/manifest
@@ -2,6 +2,7 @@
 #
 # 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
 # 2015 David Hürlimann (david at ungleich.ch)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,24 +19,12 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# Install required packages
+# Install required packages.
 #
 
-os=$(cat "$__global/explorer/os")
-
-
-case "$os" in
-    debian|devuan)
-        # Debian needs a seperate package
-        __package locales --state present
-    ;;
-    archlinux|suse|ubuntu|scientific|centos|alpine)
-        :
-    ;;
-    *)
-        echo "Sorry, do not know how to handle os: $os" >&2
-        echo "Please edit the type ${__type##*/} to fix this." >&2
-        exit 1
-    ;;
+case $(cat "${__global:?}/explorer/os")
+in
+	(debian|devuan)
+		__package_apt locales --state present
+		;;
 esac

From dcef2c19f5b87e3a9594f01fefab66cc7488dce7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 14:07:42 +0100
Subject: [PATCH 183/411] [type/__localedef] Add support for FreeBSD

---
 cdist/conf/type/__localedef/gencode-remote | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index b9cc68e8..80e7eb1c 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -108,6 +108,25 @@ in
 				;;
 		esac
 		;;
+	(freebsd)
+		case ${state_should}
+		in
+			(present)
+				if expr "$(grep -oe '^[0-9]*' "${__global:?}/explorer/os_version")" '>=' 11 >/dev/null
+				then
+					# localedef(1) is available with FreeBSD >= 11
+					printf "localedef -i '%s' -f '%s' '%s'\n" "${input}" "${codeset}" "${locale}"
+				else
+					printf 'localedef(1) was added to FreeBSD starting with version 11.\n' >&2
+					printf 'Please upgrade your FreeBSD installation to use %s.\n' "${__type##*/}" >&2
+					exit 1
+				fi
+				;;
+			(absent)
+				printf "rm -R '/usr/share/locale/%s'\n" "${locale}"
+				;;
+		esac
+		;;
 	(*)
 		echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
 		echo "Please contribute an implementation for it if you can." >&2

From c1c60e3374e0cb93ea182c4cbcb813150756c3e4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 15:24:46 +0100
Subject: [PATCH 184/411] [type/__localedef] Blacklist OpenBSD and NetBSD

---
 cdist/conf/type/__localedef/gencode-remote | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 80e7eb1c..af1a77f7 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -127,6 +127,12 @@ in
 				;;
 		esac
 		;;
+	(netbsd|openbsd)
+		# NetBSD/OpenBSD are missing localedef(1).
+		# We also do not delete defined locales because they can't be recreated.
+		echo "${os} is lacking localedef(1). Locale management unavailable." >&2
+		exit 1
+		;;
 	(*)
 		echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
 		echo "Please contribute an implementation for it if you can." >&2

From 575bb62dc507fb8c294319f8db20ba45595e6869 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 13 Nov 2020 18:42:04 +0100
Subject: [PATCH 185/411] [type/__localedef] Externalise functions to separate
 files

---
 .../conf/type/__localedef/files/lib/glibc.sh  |  5 ++++
 .../conf/type/__localedef/files/lib/locale.sh | 20 +++++++++++++
 cdist/conf/type/__localedef/gencode-remote    | 29 ++++---------------
 3 files changed, 30 insertions(+), 24 deletions(-)
 create mode 100644 cdist/conf/type/__localedef/files/lib/glibc.sh
 create mode 100644 cdist/conf/type/__localedef/files/lib/locale.sh

diff --git a/cdist/conf/type/__localedef/files/lib/glibc.sh b/cdist/conf/type/__localedef/files/lib/glibc.sh
new file mode 100644
index 00000000..6ace80d4
--- /dev/null
+++ b/cdist/conf/type/__localedef/files/lib/glibc.sh
@@ -0,0 +1,5 @@
+# -*- mode: sh; indent-tabs-mode: t -*-
+
+gnu_normalize_codeset() {
+	echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]'
+}
diff --git a/cdist/conf/type/__localedef/files/lib/locale.sh b/cdist/conf/type/__localedef/files/lib/locale.sh
new file mode 100644
index 00000000..b5e61374
--- /dev/null
+++ b/cdist/conf/type/__localedef/files/lib/locale.sh
@@ -0,0 +1,20 @@
+# -*- mode: sh; indent-tabs-mode:t -*-
+
+parse_locale() {
+	# This function will split locales into their parts. Locale strings are
+	# usually of the form: [language[_territory][.codeset][@modifier]]
+	# For simplicity, language and territory are not separated by this function.
+	# Old Linux systems were also using "english" or "german" as locale strings.
+	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
+	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
+	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
+	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
+}
+
+format_locale() {
+	# Usage: format_locale language codeset modifier
+	printf '%s' "$1"
+	test -z "$2" || printf '.%s' "$2"
+	test -z "$3" || printf '@%s' "$3"
+	printf '\n'
+}
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index af1a77f7..17941c63 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -21,6 +21,11 @@
 # Manage system locales using localedef(1).
 #
 
+# shellcheck source=cdist/conf/type/__localedef/files/lib/locale.sh
+. "${__type:?}/files/lib/locale.sh"
+# shellcheck source=cdist/conf/type/__localedef/files/lib/glibc.sh
+. "${__type:?}/files/lib/glibc.sh"
+
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
@@ -46,30 +51,6 @@ then
 	exit 1
 fi
 
-parse_locale() {
-	# This function will split locales into their parts. Locale strings are
-	# usually of the form: [language[_territory][.codeset][@modifier]]
-	# For simplicity, language and territory are not separated by this function.
-	# Old Linux systems were also using "english" or "german" as locale strings.
-	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
-	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
-	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
-	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
-}
-
-format_locale() {
-	# Usage: format_locale language codeset modifier
-	printf '%s' "$1"
-	test -z "$2" || printf '.%s' "$2"
-	test -z "$3" || printf '@%s' "$3"
-	printf '\n'
-}
-
-gnu_normalize_codeset() {
-	echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]'
-}
-
-
 : "${lang=}" "${codeset=}" "${modifier=}"  # declare variables for shellcheck
 parse_locale "${locale}" lang codeset modifier
 

From eeb98719197a09691af60fa26c0b64b3aaf8960e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 14 Nov 2020 09:54:01 +0100
Subject: [PATCH 186/411] [type/__localedef] glibc: Also delete aliases when
 removing a locale

---
 cdist/conf/type/__localedef/gencode-remote | 15 +++++++++++++--
 cdist/conf/type/__localedef/man.rst        |  5 +++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 17941c63..d7fd6942 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -84,8 +84,19 @@ in
 				EOF
 				;;
 			(absent)
-				localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}")
-				printf "localedef --delete-from-archive '%s'\n" "${localename}"
+				main_localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}")
+
+				cat <<-EOF
+				while read -r _alias _localename
+				do
+				    if test "\${_localename}" = '$(format_locale "${lang}" "${codeset}")'
+				    then
+				        localedef --delete-from-archive "\${_alias}"
+				    fi
+				done <'${aliasfile}'
+
+				localedef --delete-from-archive '${main_localename}'
+				EOF
 				;;
 		esac
 		;;
diff --git a/cdist/conf/type/__localedef/man.rst b/cdist/conf/type/__localedef/man.rst
index 0ccf484a..454ce9d1 100644
--- a/cdist/conf/type/__localedef/man.rst
+++ b/cdist/conf/type/__localedef/man.rst
@@ -13,6 +13,11 @@ This cdist type allows you to define locales on the system using
 On systems that don't support definition of new locales, the type will raise an
 error.
 
+**NB:** This type respects the glibc ``locale.alias`` file,
+i.e. it defines alias locales or deletes aliases of a locale when it is removed.
+It is not possible, however, to use alias names to define locales or only remove
+certain aliases of a locale.
+
 
 OPTIONAL PARAMETERS
 -------------------

From 87faffd8750cac94190707c2b37233229c3fbde2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 14 Nov 2020 10:41:50 +0100
Subject: [PATCH 187/411] [type/__localdef] Also check for aliases in state
 explorer

---
 cdist/conf/type/__localedef/explorer/state | 31 +++++++++++++++++++++-
 cdist/conf/type/__localedef/gencode-remote |  7 +++--
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__localedef/explorer/state b/cdist/conf/type/__localedef/explorer/state
index d8db29c5..3ba57661 100755
--- a/cdist/conf/type/__localedef/explorer/state
+++ b/cdist/conf/type/__localedef/explorer/state
@@ -25,8 +25,14 @@
 #    the main locale (and possibly aliases) is present
 #  absent:
 #    neither the main locale nor any aliases are present
+#  alias-present:
+#    the main locale is absent, but at least one of its aliases is present
 #
 
+# Hardcoded, create a pull request in case it is at another location for
+# some other distro. (cf. gencode-remote)
+aliasfile='/usr/share/locale/locale.alias'
+
 command -v locale >/dev/null 2>&1 || exit 0
 
 locales=$(locale -a)
@@ -68,4 +74,27 @@ locale_available() (
 	}
 )
 
-locale_available "${__object_id:?}" && echo present || echo absent
+if locale_available "${__object_id:?}"
+then
+	echo present
+else
+	# NOTE: locale.alias can be symlinked.
+	if test -e "${aliasfile}"
+	then
+		# Check if one of the aliases of the locale is defined
+		baselocale=$(
+			parse_locale "${__object_id:?}" _lang _codeset _modifiers \
+			&& format_locale "${_lang}" "${_codeset}")
+		while read -r _alias _localename
+		do
+			if test "${_localename}" = "${baselocale}" \
+				&& echo "${locales}" | grep -qxF "${_alias}"
+			then
+				echo alias-present
+				exit 0
+			fi
+		done <"${aliasfile}"
+	fi
+
+	echo absent
+fi
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index d7fd6942..4538151f 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -94,9 +94,12 @@ in
 				        localedef --delete-from-archive "\${_alias}"
 				    fi
 				done <'${aliasfile}'
-
-				localedef --delete-from-archive '${main_localename}'
 				EOF
+
+				if test "${state_is}" = present
+				then
+					printf "localedef --delete-from-archive '%s'\n" "${main_localename}"
+				fi
 				;;
 		esac
 		;;

From 9d4f69250ea0e6bb733cde1e42183480f48b5d8f Mon Sep 17 00:00:00 2001
From: ssrq <ungleich@ssrq-sds-fds.ch>
Date: Thu, 19 Nov 2020 19:33:47 +0100
Subject: [PATCH 188/411] __sshd config: New type

---
 cdist/conf/type/__sshd_config/explorer/state  | 121 ++++++++
 .../files/update_sshd_config.awk              | 293 ++++++++++++++++++
 cdist/conf/type/__sshd_config/gencode-remote  |  97 ++++++
 cdist/conf/type/__sshd_config/man.rst         |  94 ++++++
 cdist/conf/type/__sshd_config/manifest        |  48 +++
 .../type/__sshd_config/parameter/default/file |   1 +
 .../__sshd_config/parameter/default/state     |   1 +
 .../type/__sshd_config/parameter/optional     |   4 +
 .../__sshd_config/parameter/optional_multiple |   1 +
 9 files changed, 660 insertions(+)
 create mode 100644 cdist/conf/type/__sshd_config/explorer/state
 create mode 100644 cdist/conf/type/__sshd_config/files/update_sshd_config.awk
 create mode 100755 cdist/conf/type/__sshd_config/gencode-remote
 create mode 100644 cdist/conf/type/__sshd_config/man.rst
 create mode 100755 cdist/conf/type/__sshd_config/manifest
 create mode 100644 cdist/conf/type/__sshd_config/parameter/default/file
 create mode 100644 cdist/conf/type/__sshd_config/parameter/default/state
 create mode 100644 cdist/conf/type/__sshd_config/parameter/optional
 create mode 100644 cdist/conf/type/__sshd_config/parameter/optional_multiple

diff --git a/cdist/conf/type/__sshd_config/explorer/state b/cdist/conf/type/__sshd_config/explorer/state
new file mode 100644
index 00000000..75c68b8a
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/explorer/state
@@ -0,0 +1,121 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Determines the current state of the config option.
+# Possible output:
+#  - present: "should" option present in config file
+#  - default: the "should" option is the default -> don’t know if present
+#  - absent: no such option present in config file
+#
+
+joinlines() { sed -n -e H -e "\${x;s/^\\n//;s/\\n/${1:?}/g;p;}"; }
+trlower() { tr '[:upper:]' '[:lower:]'; }
+tolower() { printf '%s' "$*" | trlower; }
+
+default_value() {
+	sshd -T -f /dev/null -C "$(make_conn_spec)" \
+	| sed -n -e 's/^'"$(tolower "${1:?}")"'[[:blank:]]\{1,\}//p'
+}
+
+make_conn_spec() {
+	if test -s "${__object:?}/parameter/match"
+	then
+		_match_file="${__object:?}/parameter/match"
+	else
+		_match_file='/dev/null'
+	fi
+
+	for _kw in \
+		addr=Address \
+		user=User \
+		host=Host \
+		laddr=LocalAddress \
+		lport=LocalPort \
+		rdomain=RDomain
+	do
+		_specname=${_kw%%=*}
+		_confname=$(tolower "${_kw#*=}")
+		while read -r _k _v
+		do
+			if test "$(tolower "${_k}")" = "${_confname}"
+			then
+				printf '%s=%s\n' "${_specname}" "${_v}"
+				continue 2
+			fi
+		done <"${_match_file}"
+
+		# NOTE: Print test spec even for empty keys to suppress errors like:
+		#  'Match User' in configuration but 'user' not in connection test specification.
+		# except lport:
+		#  Invalid port '' in test mode specification lport=
+		test "${_specname}" = 'lport' || printf '%s=\n' "${_specname}"
+	done \
+	| joinlines ','
+	unset _match_file
+}
+
+sshd_config_file=$(cat "${__object:?}/parameter/file")
+state_should=$(cat "${__object:?}/parameter/state")
+
+if test -s "${__object:?}/parameter/option"
+then
+	option_name=$(cat "${__object:?}/parameter/option")
+else
+	option_name=${__object_id:?}
+fi
+
+value_should=$(cat "${__object:?}/parameter/value" 2>/dev/null) \
+|| test "${state_should}" = absent || exit 0  # param optional if --state absent
+
+command -v sshd >/dev/null 2>&1 || {
+	echo 'Cannot find sshd.' >&2
+	exit 1
+}
+
+test -e "${sshd_config_file}" || {
+	echo 'absent'
+	exit 0
+}
+
+value_is=$(
+	sshd -T -f "${sshd_config_file}" -C "$(make_conn_spec)" \
+	| sed -n -e 's/^'"$(tolower "${option_name}")"'[[:blank:]]\{1,\}//p')
+
+if printf '%s\n' "${value_is}" | {
+		if test -n "${value_should}"
+		then
+			grep -q -x -F "${value_should}"
+		else
+			# if no value provided, assume "any" value
+			grep -q -e .
+		fi
+	}
+then
+	if default_value "${option_name}" | grep -q -x -F "${value_is}"
+	then
+		# Might produce false positives for default values.
+		# TODO: Manual checking should be done, but for simplicity, this case is
+		#       currently ignored here.
+		echo default
+	else
+		echo present
+	fi
+else
+	echo absent
+fi
diff --git a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
new file mode 100644
index 00000000..d0bc2b4b
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
@@ -0,0 +1,293 @@
+# -*- mode: awk; indent-tabs-mode: t -*-
+
+function usage() {
+	print_err("Usage: awk -f update_sshd_config.awk -- -o set|unset [-m 'User git'] -l 'X11Forwarding no' /etc/ssh/sshd_config")
+}
+
+function print_err(s) { print s | "cat >&2" }
+
+function alength(a,    i) {
+	for (i = 0; (i + 1) in a; ++i);
+	return i
+}
+
+function join(sep, a, i,    s) {
+	for (i = i ? i : 1; i in a; i++)
+		s = s sep a[i]
+	return substr(s, 2)
+}
+
+function getopt(opts, argv, target, files,    i, c, lv, idx, nf) {
+	# trivial getopt(3) implementation; only basic functionality
+	if (argv[1] == "--") i++
+	for (i += 1; i in argv; i++) {
+		if (lv) { target[c] = argv[i]; lv = 0; continue }
+		if (argv[i] ~ /^-/) {
+			c = substr(argv[i], 2, 1)
+			idx = index(opts, c)
+			if (!idx) {
+				print_err(sprintf("invalid option -%c\n", c))
+				continue
+			}
+			if (substr(opts, idx + 1, 1) == ":") {
+				# option takes argument
+				if (length(argv[i]) > 2)
+					target[c] = substr(argv[i], 3)
+				else
+					lv = 1
+			} else {
+				target[c] = 1
+			}
+		} else
+			files[++nf] = argv[i]
+	}
+}
+
+# tokenise configuration line
+# this function mimics the counterpart in OpenSSH (misc.c)
+# but it returns two (next token SUBSEP rest) because I didn’t want to have to
+# simulate any pointer magic.
+function strdelim_internal(s, split_equals,    old) {
+	if (!s)
+		return ""
+
+	old = s
+
+	if (!match(s, WHITESPACE "|" QUOTE "" (split_equals ? "|" EQUALS : "")))
+		return s
+
+	s = substr(s, RSTART)
+	old = substr(old, 1, RSTART - 1)
+
+	if (s ~ "^" QUOTE) {
+		old = substr(old, 2)
+
+		# Find matching quote
+		if (match(s, QUOTE)) {
+			old = substr(old, 1, RSTART)
+			# s = substr()
+			if (match(s, "^" WHITESPACE "*"))
+				s = substr(s, RLENGTH)
+			return old
+		} else {
+			# no matching quote
+			return ""
+		}
+	}
+
+	if (match(s, "^" WHITESPACE "+")) {
+		sub("^" WHITESPACE "+", "", s)
+		if (split_equals)
+			sub(EQUALS WHITESPACE "*", "", s)
+	} else if (s ~ "^" EQUALS) {
+		s = substr(s, 2)
+	}
+
+	return old SUBSEP s
+}
+function strdelim(s) { return strdelim_internal(s, 1) }
+function strdelimw(s) { return strdelim_internal(s, 0) }
+
+function singleton_option(opt) {
+	return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|authenticationmethods|authorizedkeysfile|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/
+}
+
+function print_update() {
+	if (mode) {
+		if (match_only) printf "\t"
+		printf "%s\n", line_should
+		updated = 1
+	}
+}
+
+BEGIN {
+	FS = "\n"  # disable field splitting
+
+	WHITESPACE = "[ \t]"  # servconf.c, misc.c:strdelim_internal (without line breaks, cf. bugs)
+	QUOTE = "[\"]"  # misc.c:strdelim_internal
+	EQUALS = "[=]"
+
+	split("", opts)
+	split("", files)
+	getopt("ho:l:m:", ARGV, opts, files)
+
+	if (opts["h"]) { usage(); exit (e="0") }
+
+	line_should = opts["l"]
+	match_only = opts["m"]
+	num_files = alength(files)
+
+	if (num_files != 1 || !opts["o"] || !line_should) {
+		usage()
+		exit (e=126)
+	}
+
+	if (opts["o"] == "set") {
+		mode = 1
+	} else if (opts["o"] == "unset") {
+		mode = 0
+	} else {
+		print_err(sprintf("invalid mode %s\n", mode))
+		exit (e=1)
+	}
+
+	if (mode) {
+		# loop over sshd_config twice!
+		ARGV[2] = ARGV[1] = files[1]
+		ARGC = 3
+	} else {
+		# only loop once
+		ARGV[1] = files[1]
+		ARGC = 2
+	}
+
+	split(strdelim(line_should), should, SUBSEP)
+	option_should = tolower(should[1])
+	value_should = should[2]
+}
+
+{
+	line = $0
+
+	# Strip trailing whitespace. Allow \f (form feed) at EOL only
+	sub("(" WHITESPACE "|\f)*$", "", line)
+
+	# Strip leading whitespace
+	sub("^" WHITESPACE "*", "", line)
+
+	if (match(line, "^#" WHITESPACE "*")) {
+		prefix = substr(line, RSTART, RLENGTH)
+		line = substr(line, RSTART + RLENGTH)
+	} else {
+		prefix = ""
+	}
+
+	line_type = "invalid"
+	option_is = value_is = ""
+
+	if (line) {
+		split(strdelim(line), toks, SUBSEP)
+
+		if (tolower(toks[1]) == "match") {
+			MATCH = (prefix ~ /^#/ ? "#" : "") join(" ", toks, 2)
+			line_type = "match"
+		} else if (toks[1] ~ /^[A-Za-z][A-Za-z0-9]+$/) {
+			# This could be an option line
+			line_type = "option"
+			option_is = tolower(toks[1])
+			value_is = toks[2]
+		}
+	} else {
+		line_type = "empty"
+	}
+}
+
+# mode: unset
+
+!mode {
+	# delete matching config
+	if (prefix !~ /^#/)
+		if (MATCH == match_only && option_is == option_should)
+			if (!value_should || value_should == value_is)
+				next
+
+	print
+	next
+}
+
+
+# mode: set
+
+mode && NR == FNR {
+	if (line_type == "option") {
+		if (MATCH !~ /^#/) {
+			if (prefix ~ /^#/) {
+				# comment line
+				last_occ[MATCH, "#" option_is] = FNR
+			} else {
+				# option line
+				last_occ[MATCH, option_is] = FNR
+			}
+			last_occ[MATCH] = FNR
+		}
+	} else if (line_type == "invalid" && !prefix) {
+		# INVALID LINE
+		print_err(sprintf("%s: syntax error on line %u\n", ARGV[0], FNR))
+	}
+
+	next
+}
+
+# before second pass prepare hashes containing location information to be used
+# in the second pass.
+mode && NR > FNR && FNR == 1 {
+	# First we drop the locations of commented-out options if a non-commented
+	# option is available. If a non-commented option is available, we will
+	# append new config options there to have them all at one place.
+	for (k in last_occ) {
+		if (k ~ /^#/) {
+			# delete entries of commented out match blocks
+			delete last_occ[k]
+			continue
+		}
+
+		split(k, parts, SUBSEP)
+
+		if (parts[2] ~ /^#/ && ((parts[1], substr(parts[2], 2)) in last_occ))
+			delete last_occ[k]
+	}
+
+	# Reverse the option => line mapping. The line_map allows for easier lookups
+	# in the second pass.
+	# We only keep options, not top-level keywords, because we can only have
+	# one entry per line and there are conflicts with last lines of "sections".
+	for (k in last_occ) {
+		if (!index(k, SUBSEP)) continue
+		line_map[last_occ[k]] = k
+	}
+}
+
+# Second pass
+mode && line_map[FNR] == match_only SUBSEP option_should && !updated {
+	split(line_map[FNR], parts, SUBSEP)
+
+	# If option allows multiple values, print current value
+	if (!singleton_option(parts[2])) {
+		if (value_should != value_is)
+			print
+	}
+
+	print_update()
+
+	next
+}
+
+mode { print }
+
+# Is a comment option
+mode && line_map[FNR] == match_only SUBSEP "#" option_should && !updated {
+	print_update()
+}
+
+# Last line of the should match section
+mode && last_occ[match_only] == FNR && !updated {
+	# NOTE: Inserting empty lines is only cosmetic.  It is only done if
+	#       different options are next to each other and not in a match block
+	#       (match blocks are usually not in the default config and thus don’t
+	#       contain commented blocks.)
+	if (line && option_is != option_should && !MATCH)
+		print ""
+	print_update()
+}
+
+END {
+	if (e) exit e
+
+	if (mode && !updated) {
+		if (match_only && MATCH != match_only) {
+			printf "\nMatch %s\n", match_only
+		}
+
+		print_update()
+	}
+}
diff --git a/cdist/conf/type/__sshd_config/gencode-remote b/cdist/conf/type/__sshd_config/gencode-remote
new file mode 100755
index 00000000..0b44dfa7
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/gencode-remote
@@ -0,0 +1,97 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+joinlines() { sed -n -e H -e "\${x;s/^\\n//;s/\\n/${1:?}/g;p;}"; }
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+if test "${state_is}" = "${state_should}" -o "${state_is}" = 'default'
+then
+	# nothing to do (if the value is the default, ignore its state)
+	exit 0
+fi
+
+case ${state_should}
+in
+	(present)
+		mode='set'
+		;;
+	(absent)
+		mode='unset'
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
+
+sshd_config_file=$(cat "${__object:?}/parameter/file")
+
+quote() { printf "'%s'" "$(printf '%s' "$*" | sed -e "s/'/'\\\\''/g")"; }
+drop_awk_comments() { quote "$(sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@")"; }
+
+# Ensure the sshd_config file is there
+cat <<EOF
+test -e $(quote "${sshd_config_file}") || {
+	 : >$(quote "${sshd_config_file}")
+	 chown 0:0 $(quote "${sshd_config_file}")
+	 chmod 0644 $(quote "${sshd_config_file}")
+}
+
+EOF
+
+match_only=
+if test -s "${__object:?}/parameter/match"
+then
+	match_only=$(joinlines ' ' <"${__object:?}/parameter/match")
+fi
+
+if test -s "${__object:?}/parameter/option"
+then
+	option_line=$(cat "${__object:?}/parameter/option")
+else
+	option_line=${__object_id:?}
+fi
+
+if test -s "${__object:?}/parameter/value"
+then
+	option_line="${option_line} $(cat "${__object:?}/parameter/value")"
+fi
+
+# Send message on config update
+printf '%s%s %s\n' "${mode}" "${match_only:+ [${match_only}]}" \
+	"${option_line}" >>"${__messages_out:?}"
+
+# Update sshd_config (remote code)
+cat <<EOF
+awk $(drop_awk_comments "${__type:?}/files/update_sshd_config.awk") \\
+	-o ${mode} \\
+	-m $(quote "${match_only}") \\
+	-l $(quote "${option_line}") \\
+	$(quote "${sshd_config_file}") >$(quote "${sshd_config_file}.tmp") \\
+|| exit
+
+cmp -s $(quote "${sshd_config_file}") $(quote "${sshd_config_file}.tmp") || {
+	sshd -t -f $(quote "${sshd_config_file}.tmp") \\
+	&& cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}")
+}
+rm -f $(quote "${sshd_config_file}.tmp")
+EOF
diff --git a/cdist/conf/type/__sshd_config/man.rst b/cdist/conf/type/__sshd_config/man.rst
new file mode 100644
index 00000000..8b0069ac
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/man.rst
@@ -0,0 +1,94 @@
+cdist-type__sshd_config(7)
+==========================
+
+NAME
+----
+cdist-type__sshd_config - Manage options in sshd_config
+
+
+DESCRIPTION
+-----------
+This space intentionally left blank.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+file
+    The path to the sshd_config file to edit.
+    Defaults to ``/etc/ssh/sshd_config``.
+match
+    Restrict this option to apply only for certain connections.
+    Allowed values are what would be allowed to be written after a ``Match``
+    keyword in ``sshd_config``, e.g. ``--match 'User anoncvs'``.
+
+    Can be used multiple times. All of the values are ANDed together.
+option
+    The name of the option to manipulate. Defaults to ``__object_id``.
+state
+    Can be:
+
+    - ``present``: ensure a matching config line is present (or the default
+      value).
+    - ``absent``: ensure no matching config line is present.
+value
+    The option's value to be assigned to the option (if ``--state present``) or
+    removed (if ``--state absent``).
+
+    This option is required if ``--state present``. If not specified and
+    ``--state absent``, all values for the given option are removed.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Disallow root logins with password
+    __sshd_config PermitRootLogin --value without-password
+
+    # Disallow password-based authentication
+    __sshd_config PasswordAuthentication --value no
+
+    # Accept the EDITOR environment variable
+    __sshd_config AcceptEnv:EDITOR --option AcceptEnv --value EDITOR
+
+    # Force command for connections as git user
+    __sshd_config git@ForceCommand --match 'User git' --option ForceCommand \
+        --value 'cd ~git && exec git-shell ${SSH_ORIGINAL_COMMAND:+-c "${SSH_ORIGINAL_COMMAND}"}'
+
+
+SEE ALSO
+--------
+:strong:`sshd_config`\ (5)
+
+
+BUGS
+----
+- This type assumes a nicely formatted config file,
+  i.e. no config options spanning multiple lines.
+- ``Include`` directives are ignored.
+- Config options are not added/removed to/from the config file if their value is
+  the default value.
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest
new file mode 100755
index 00000000..566bde90
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/manifest
@@ -0,0 +1,48 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$(cat "${__global:?}/explorer/os")
+
+state_should=$(cat "${__object:?}/parameter/state")
+
+case ${os}
+in
+	(alpine|centos|fedora|redhat|scientific|debian|devuan|ubuntu)
+		if test "${state_should}" != 'absent'
+		then
+			__package openssh-server --state present
+		fi
+		;;
+	(archlinux|gentoo|slackware|suse)
+		if test "${state_should}" != 'absent'
+		then
+			__package openssh --state present
+		fi
+		;;
+	(freebsd|netbsd|openbsd)
+		# whitelist
+		;;
+	(*)
+		printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \
+			"${os}" "${__type##*/}" >&2
+		printf 'Please contribute an implementation for it if you can.\n' >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__sshd_config/parameter/default/file b/cdist/conf/type/__sshd_config/parameter/default/file
new file mode 100644
index 00000000..d8ea5dfc
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/default/file
@@ -0,0 +1 @@
+/etc/ssh/sshd_config
diff --git a/cdist/conf/type/__sshd_config/parameter/default/state b/cdist/conf/type/__sshd_config/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__sshd_config/parameter/optional b/cdist/conf/type/__sshd_config/parameter/optional
new file mode 100644
index 00000000..922ab093
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/optional
@@ -0,0 +1,4 @@
+file
+option
+state
+value
diff --git a/cdist/conf/type/__sshd_config/parameter/optional_multiple b/cdist/conf/type/__sshd_config/parameter/optional_multiple
new file mode 100644
index 00000000..02b1d1a9
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/optional_multiple
@@ -0,0 +1 @@
+match

From 82eadb69948e631041263938994be5aeae161ab0 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 19 Nov 2020 19:34:43 +0100
Subject: [PATCH 189/411] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index ff411a46..331acbeb 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,8 @@ next:
 	* Type __locale: Add state explorer (Matthias Stecher)
 	* Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera)
 	* New type: __hwclock (Dennis Camera)
+	* Type __hostname: Fix guessing SuSE OS version (Dennis Camera)
+	* New type: __sshd_config (Dennis Camera)
 
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)

From 803a9d62a7a6b7fd11844f6ea273f4b692c2334e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 20 Nov 2020 19:46:03 +0100
Subject: [PATCH 190/411] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 331acbeb..cd5dbb4f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,8 @@ next:
 	* New type: __hwclock (Dennis Camera)
 	* Type __hostname: Fix guessing SuSE OS version (Dennis Camera)
 	* New type: __sshd_config (Dennis Camera)
+	* New type: __localedef (Dennis Camera)
+	* Type __locale: Deprecate in favor of __localedef (Dennis Camera)
 
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)

From 23e0da521c785715357b27d66007bd03fe173bf3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 20 Nov 2020 19:46:55 +0100
Subject: [PATCH 191/411] Release 6.9.2

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index cd5dbb4f..9a91bf57 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.2: 2020-11-20
 	* Documentation: Fix examples in best practice (Dennis Camera)
 	* Type __locale: Add state explorer (Matthias Stecher)
 	* Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera)

From 2b97e08ff29b3d966dbde0881f1aad825833efa9 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 29 Nov 2020 15:01:46 +0100
Subject: [PATCH 192/411] __lxc_container: remove default --quiet parameter

Because we want to see error messages, too. And lxc isn't that
verbose.
---
 cdist/conf/type/__lxc_container/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__lxc_container/gencode-remote b/cdist/conf/type/__lxc_container/gencode-remote
index 6bef504e..6f698768 100755
--- a/cdist/conf/type/__lxc_container/gencode-remote
+++ b/cdist/conf/type/__lxc_container/gencode-remote
@@ -33,7 +33,7 @@ lxcpath="$(cat "$__object/parameter/lxcpath" 2>/dev/null || true)"
 
 # Summerize common parameter arguments, listed in manual as "COMMON OPTIONS"
 #  will passed raw; must qouted manually
-LXC_PARAM="-q"
+LXC_PARAM=""
 
 # if lxcpath given
 if [ -n "$lxcpath" ]; then

From 6da0f9e0c53aa926ae27e7e38fd0ac37925e0847 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 29 Nov 2020 18:14:34 +0100
Subject: [PATCH 193/411] __lxc_container: updated todo.txt about type
 splitting

Because it will get a mess when adding every option to __lxc_container
..
---
 cdist/conf/type/__lxc_container/todo.txt | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__lxc_container/todo.txt b/cdist/conf/type/__lxc_container/todo.txt
index 41822842..05b4d95c 100644
--- a/cdist/conf/type/__lxc_container/todo.txt
+++ b/cdist/conf/type/__lxc_container/todo.txt
@@ -1,5 +1,5 @@
 # List of all featues that should be implemented
-template:
+template (common parameters):
   - type: a.E. debian, ubuntu -> what on change?
   - possible root_password?
   - possible packages?
@@ -7,10 +7,27 @@ template:
 auto.. (lxc autostart):
   - group
   - start
+handling networking?
+  - address
+  - bridge? (at default file?)
 
 config:
   - remove whole sections
   - keep prettier config files?
+  - own type for config?
+    does it conflict with the current one?
+own __lxc_default type?
+  `lxc-config lxc.default_config`
+  without lxcpath cause singleton?
+
+Splitting up type? (all times the library question :-) )
+  __lxc_container (umbrella; creating,cloning,destroying??)
+  __lxc_container_config (single key-values)
+  __lxc_container_state (only running,freezed,stopped??)
+  __lxc_container_network
+  __lxc_container_mount
+  ...
+All common library files (except explorer things sadly) could be kept under __lxc_container.
 
 
 if multi-valued object ids come to be a thing, following pattern is also used internal at lxc:

From 84172550df9e1800bc795dc51da01c9d3a0be9e0 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 30 Nov 2020 20:29:51 +0100
Subject: [PATCH 194/411] __iptables*: add IPv6 support

Because it currently only support IPv4. To implement this, it falls back
to IPv4 for backward compatibilty, but now supports rules for IPv6 and
both protocols at the same time.
---
 .../type/__iptables_apply/files/init-script   | 84 +++++++++++++++----
 cdist/conf/type/__iptables_rule/man.rst       | 38 ++++++++-
 cdist/conf/type/__iptables_rule/manifest      | 40 ++++++++-
 .../type/__iptables_rule/parameter/boolean    |  3 +
 4 files changed, 140 insertions(+), 25 deletions(-)
 create mode 100644 cdist/conf/type/__iptables_rule/parameter/boolean

diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script
index d9c79ef7..196f019b 100644
--- a/cdist/conf/type/__iptables_apply/files/init-script
+++ b/cdist/conf/type/__iptables_apply/files/init-script
@@ -1,6 +1,27 @@
 #!/bin/sh
-# Nico Schottelius
-# Zürisee, Mon Sep  2 18:38:27 CEST 2013
+#
+# 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is distributed with cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Originally written by:
+#   Nico Schottelius
+#   Zürisee, Mon Sep  2 18:38:27 CEST 2013
 #
 ### BEGIN INIT INFO
 # Provides:          iptables
@@ -15,33 +36,60 @@
 ### END INIT INFO
 
 
+# Read files and execute the content with the given commands
+#
+# Arguments:
+#  1:    Directory
+#  2..n: Commands which should be used to execute the file content
+gothrough() {
+    cd "$1" || return
+    shift
+
+    # iterate through all rules and continue if it's not a file
+    for rule in *; do
+        [ -f "$rule" ] || continue
+        echo "Appling iptables rule $rule ..."
+
+        # execute it with all commands specificed
+        ruleparam="$(cat "$rule")"
+        for cmd in "$@"; do
+            # Command and Rule should be split.
+            # shellcheck disable=SC2046
+            command $cmd $ruleparam
+        done
+    done
+}
+
+# Shortcut for iptables command to do IPv4 and v6
+iptables() {
+    command iptables "$@"
+    command ip6tables "$@"
+}
+
 basedir=/etc/iptables.d
-status="${basedir}/.pre-start"
+status4="${basedir}/.pre-start"
+status6="${basedir}/.pre-start6"
 
 case $1 in
     start)
         # Save status
-        iptables-save > "$status"
+        iptables-save > "$status4"
+        ip6tables-save > "$status6"
 
         # Apply our ruleset
-        cd "$basedir" || exit
-        count="$(find . ! -name . -prune | wc -l)"
-
-        # Only do something if there are rules
-        if [ "$count" -ge 1 ]; then
-            for rule in *; do
-                echo "Applying iptables rule $rule ..."
-                # Rule should be split.
-                # shellcheck disable=SC2046
-                iptables $(cat "$rule")
-            done
-        fi
+        gothrough "$basedir"     iptables
+        #gothrough "$basedir/v4"  iptables  # conflicts with $basedir
+        gothrough "$basedir/v6"  ip6tables
+        gothrough "$basedir/all" iptables  ip6tables
     ;;
 
     stop)
         # Restore from status before, if there is something to restore
-        if [ -f "$status" ]; then
-            iptables-restore < "$status"
+        if [ -f "$status4" ]; then
+            iptables-restore < "$status4"
+        fi
+        if [ -f "$status6" ]; then
+            ip6tables-restore < "$status6"
         fi
     ;;
     restart)
diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst
index 92d8859f..75d0740b 100644
--- a/cdist/conf/type/__iptables_rule/man.rst
+++ b/cdist/conf/type/__iptables_rule/man.rst
@@ -25,6 +25,24 @@ state
    'present' or 'absent', defaults to 'present'
 
 
+BOOLEAN PARAMETERS
+------------------
+All rules without any of this parameter will be threaten like ``--v4`` because
+of backward compatibility.
+
+v4
+    Explicitly set it as rule for IPv4. If IPv6 is set, too, it will be
+    threaten like ``--all``. Will be the default if nothing else is set.
+
+v6
+    Explicitly set it as rule for IPv6. If IPv4 is set, too, it will be
+    threaten like ``--all``.
+
+all
+    Set the rule for both IPv4 and IPv6. It will be saved separately from the
+    other rules.
+
+
 EXAMPLES
 --------
 
@@ -48,6 +66,16 @@ EXAMPLES
         --state absent
 
 
+    # IPv4-only rule for ICMPv4
+    __iptables_rule icmp-v4 --v4 --rule "-A INPUT -p icmp -j ACCEPT"
+    # IPv6-only rule for ICMPv6
+    __iptables_rule icmp-v6 --v6 --rule "-A INPUT -p icmpv6 -j ACCEPT"
+
+    # doing something for the dual stack
+    __iptables_rule fwd-eth0-eth1 --v4 --v6 --rule "-A INPUT -i eth0 -o eth1 -j ACCEPT"
+    __iptables_rule fwd-eth1-eth0 --all --rule "-A -o eth1 -i eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT"
+
+
 SEE ALSO
 --------
 :strong:`cdist-type__iptables_apply`\ (7), :strong:`iptables`\ (8)
@@ -56,11 +84,13 @@ SEE ALSO
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2013 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2013 Nico Schottelius.
+Copyright \(C) 2020 Matthias Stecher.
+You can redistribute it and/or modify it under the terms of the GNU
+General Public License as published by the Free Software Foundation,
+either version 3 of the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest
index ed78787f..27d00024 100755
--- a/cdist/conf/type/__iptables_rule/manifest
+++ b/cdist/conf/type/__iptables_rule/manifest
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
 #
 # This file is part of cdist.
 #
@@ -24,12 +25,36 @@ base_dir=/etc/iptables.d
 name="$__object_id"
 state="$(cat "$__object/parameter/state")"
 
+if [ -f "$__object/parameter/v4" ]; then
+    only_v4="yes"
+    # $specific_dir is $base_dir
+fi
+if [ -f "$__object/parameter/v6" ]; then
+    only_v6="yes"
+    specific_dir="$base_dir/v6"
+fi
+# If rules should be set for both protocols
+if ([ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]) \
+    || [ -f "$__object/parameter/all" ]; then
+
+    # all to a specific directory
+    specific_dir="$base_dir/all"
+fi
+
+# set rule directory based on if it's the base or subdirectory
+rule_dir="${specific_dir:-$base_dir}"
+
 ################################################################################
 # Basic setup
 #
 
 __directory "$base_dir" --state present
 
+# sub-directory if required
+if [ "$specific_dir" ]; then
+    require="__directory/$base_dir" __directory "$specific_dir" --state present
+fi
+
 # Have apply do the real job
 require="$__object_name" __iptables_apply
 
@@ -37,6 +62,15 @@ require="$__object_name" __iptables_apply
 # The rule
 #
 
-require="__directory/$base_dir" __file "$base_dir/${name}" \
-    --source "$__object/parameter/rule" \
-    --state "$state"
+for dir in "$base_dir" "$base_dir/v6" "$base_dir/all"; do
+    # defaults to absent except the directory that should contain the file
+    if [ "$rule_dir" = "$dir" ]; then
+        curr_state="$state"
+    else
+        curr_state="absent"
+    fi
+
+    require="__directory/$rule_dir" __file "$dir/$name" \
+        --source "$__object/parameter/rule" \
+        --state "$curr_state"
+done
diff --git a/cdist/conf/type/__iptables_rule/parameter/boolean b/cdist/conf/type/__iptables_rule/parameter/boolean
new file mode 100644
index 00000000..76882272
--- /dev/null
+++ b/cdist/conf/type/__iptables_rule/parameter/boolean
@@ -0,0 +1,3 @@
+all
+v4
+v6

From f568462e4981c8b6437c29f96963c3d8e7bed742 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 2 Dec 2020 17:48:41 +0100
Subject: [PATCH 195/411] __iptables_rule: fix shellcheck SC2235

---
 cdist/conf/type/__iptables_rule/manifest | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest
index 27d00024..d4394c25 100755
--- a/cdist/conf/type/__iptables_rule/manifest
+++ b/cdist/conf/type/__iptables_rule/manifest
@@ -34,8 +34,8 @@ if [ -f "$__object/parameter/v6" ]; then
     specific_dir="$base_dir/v6"
 fi
 # If rules should be set for both protocols
-if ([ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]) \
-    || [ -f "$__object/parameter/all" ]; then
+if { [ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]; } ||
+    [ -f "$__object/parameter/all" ]; then
 
     # all to a specific directory
     specific_dir="$base_dir/all"

From bee255c1ae0039e3d532317007f06fcd2e64e3bc Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 2 Dec 2020 18:04:50 +0100
Subject: [PATCH 196/411] __iptables_apply: man updates

---
 .../type/__iptables_apply/files/init-script   |  1 +
 cdist/conf/type/__iptables_apply/man.rst      | 21 +++++++++++++------
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script
index 196f019b..7faa2f92 100644
--- a/cdist/conf/type/__iptables_apply/files/init-script
+++ b/cdist/conf/type/__iptables_apply/files/init-script
@@ -61,6 +61,7 @@ gothrough() {
 }
 
 # Shortcut for iptables command to do IPv4 and v6
+# only applies to the "reset" target
 iptables() {
     command iptables "$@"
     command ip6tables "$@"
diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst
index 76e1f6bf..db0e7869 100644
--- a/cdist/conf/type/__iptables_apply/man.rst
+++ b/cdist/conf/type/__iptables_apply/man.rst
@@ -10,7 +10,14 @@ DESCRIPTION
 -----------
 This cdist type deploys an init script that triggers
 the configured rules and also re-applies them on
-configuration.
+configuration. Rules are written from __iptables_rule
+into the folder ``/etc/iptables.d/``.
+
+It reads all rules from the base folder as rules for IPv4.
+Rules in the subfolder ``v6/`` are IPv6 rules. Rules in
+the subfolder ``all/`` are applied to both rule tables. All
+files contain the arguments for a single ``iptables`` and/or
+``ip6tables`` command.
 
 
 REQUIRED PARAMETERS
@@ -24,7 +31,7 @@ None
 EXAMPLES
 --------
 
-None (__iptables_apply is used by __iptables_rule)
+None (__iptables_apply is used by __iptables_rule automaticly)
 
 
 SEE ALSO
@@ -35,11 +42,13 @@ SEE ALSO
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2013 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2013 Nico Schottelius.
+Copyright \(C) 2020 Matthias Stecher.
+You can redistribute it and/or modify it under the terms of the GNU
+General Public License as published by the Free Software Foundation,
+either version 3 of the License, or (at your option) any later version.

From a1db5c3d0e7b5899f1a877c5bab28ac4d4796f8c Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 2 Dec 2020 18:22:31 +0100
Subject: [PATCH 197/411] __iptables*: Update manpages for execution order

To make some thinks clear if someone needs it ..
---
 cdist/conf/type/__iptables_apply/man.rst | 10 ++++++++++
 cdist/conf/type/__iptables_rule/man.rst  |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst
index db0e7869..4109e789 100644
--- a/cdist/conf/type/__iptables_apply/man.rst
+++ b/cdist/conf/type/__iptables_apply/man.rst
@@ -19,6 +19,16 @@ the subfolder ``all/`` are applied to both rule tables. All
 files contain the arguments for a single ``iptables`` and/or
 ``ip6tables`` command.
 
+Rules are applied in the following order:
+1. All IPv4 rules
+2. All IPv6 rules
+2. All rules that should be applied to both tables
+
+The order of the rules that will be applied are definite
+from the result the shell glob returns, which should be
+alphabetical. If rules must be applied in a special order,
+prefix them with a number like ``02-some-rule``.
+
 
 REQUIRED PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst
index 75d0740b..86d38a34 100644
--- a/cdist/conf/type/__iptables_rule/man.rst
+++ b/cdist/conf/type/__iptables_rule/man.rst
@@ -11,6 +11,10 @@ DESCRIPTION
 This cdist type allows you to manage iptable rules
 in a distribution independent manner.
 
+See :strong:`cdist-type__iptables_apply`\ (7) for the
+execution order of these rules. It will be executed
+automaticly to apply all rules non-volaite.
+
 
 REQUIRED PARAMETERS
 -------------------

From 1055e92545e74cf7845ede8094e783b4103efbcc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 2 Dec 2020 19:54:41 +0100
Subject: [PATCH 198/411] [setup.py] Add cdist.scan to packages

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 858c2c17..bfc8b495 100644
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@ os.chdir(cur)
 
 setup(
     name="cdist",
-    packages=["cdist", "cdist.core", "cdist.exec", "cdist.util", ],
+    packages=["cdist", "cdist.core", "cdist.exec", "cdist.scan", "cdist.util"],
     package_data={'cdist': package_data},
     scripts=["bin/cdist", "bin/cdist-dump", "bin/cdist-new-type"],
     version=cdist.version.VERSION,

From c7fa2efe6b780d63a779eed310bfa4ebf926d27e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 4 Dec 2020 15:30:08 +0100
Subject: [PATCH 199/411] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 9a91bf57..ad0aa75d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)
+
 6.9.2: 2020-11-20
 	* Documentation: Fix examples in best practice (Dennis Camera)
 	* Type __locale: Add state explorer (Matthias Stecher)

From d44b5cfdc990ce020ffe3af5085bbb93b8ce64a8 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 4 Dec 2020 15:31:35 +0100
Subject: [PATCH 200/411] Release 6.9.3

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index ad0aa75d..6c272fd7 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)
 
 6.9.2: 2020-11-20

From ba7d16a155cef46230d1dc650119f3d542a7f7f4 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 4 Dec 2020 17:57:55 +0100
Subject: [PATCH 201/411] __iptables_*: correct manpage spelling

---
 cdist/conf/type/__iptables_apply/man.rst | 2 +-
 cdist/conf/type/__iptables_rule/man.rst  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst
index 4109e789..3bef92cc 100644
--- a/cdist/conf/type/__iptables_apply/man.rst
+++ b/cdist/conf/type/__iptables_apply/man.rst
@@ -41,7 +41,7 @@ None
 EXAMPLES
 --------
 
-None (__iptables_apply is used by __iptables_rule automaticly)
+None (__iptables_apply is used by __iptables_rule automatically)
 
 
 SEE ALSO
diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst
index 86d38a34..afb71e01 100644
--- a/cdist/conf/type/__iptables_rule/man.rst
+++ b/cdist/conf/type/__iptables_rule/man.rst
@@ -31,7 +31,7 @@ state
 
 BOOLEAN PARAMETERS
 ------------------
-All rules without any of this parameter will be threaten like ``--v4`` because
+All rules without any of these parameters will be treated like ``--v4`` because
 of backward compatibility.
 
 v4

From 2d19856840400af57bd6667cc868ab314a3e07c2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 4 Dec 2020 18:26:03 +0100
Subject: [PATCH 202/411] [type/__package_pkgng_freebsd] Set ASSUME_ALWAYS_YES
 instead of -y

---
 cdist/conf/type/__package_pkgng_freebsd/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
index b5944177..05ba4cb2 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
+++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
@@ -75,7 +75,7 @@ execcmd(){
    esac
 
    if [ -z "${pkg_bootstrapped}" ]; then
-       echo "pkg bootstrap -y >/dev/null 2>&1"
+       echo "ASSUME_ALWAYS_YES=yes pkg bootstrap >/dev/null 2>&1"
    fi
 
    echo "$_cmd >/dev/null 2>&1"   # Silence the output of the command

From 087be130fa67d3fe195387ae0ab079f39c5066e1 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 4 Dec 2020 19:23:49 +0100
Subject: [PATCH 203/411] __iptables_apply: shorten copyright header

Do we need all the copyright header or is this sufficient? The licence
is given for cdist, but not on the target host. But it should be clear
anyway.
---
 .../type/__iptables_apply/files/init-script   | 34 ++++++-------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script
index 7faa2f92..e42017ae 100644
--- a/cdist/conf/type/__iptables_apply/files/init-script
+++ b/cdist/conf/type/__iptables_apply/files/init-script
@@ -1,28 +1,4 @@
 #!/bin/sh
-#
-# 2013 Nico Schottelius (nico-cdist at schottelius.org)
-# 2020 Matthias Stecher (matthiasstecher at gmx.de)
-#
-# This file is distributed with cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Originally written by:
-#   Nico Schottelius
-#   Zürisee, Mon Sep  2 18:38:27 CEST 2013
-#
 ### BEGIN INIT INFO
 # Provides:          iptables
 # Required-Start:    $local_fs $remote_fs
@@ -35,6 +11,16 @@
 #                    and saves/restores previous status
 ### END INIT INFO
 
+# Originally written by:
+#   Nico Schottelius
+#   Zürisee, Mon Sep  2 18:38:27 CEST 2013
+#
+# 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is distributed with cdist and licenced under the
+# GNU GPLv3+ WITHOUT ANY WARRANTY.
+
 
 # Read files and execute the content with the given commands
 #

From 3930f69456fd5a0d108a107e4ee61d87c9a73a56 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 6 Dec 2020 16:45:58 +0100
Subject: [PATCH 204/411] __block: fix escaping in here-doc

This changes the here-document to do not interpret any shell-things. It
also single-quotes some more strings that are printed to code-remote.

Fixes #838
---
 cdist/conf/type/__block/gencode-remote | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__block/gencode-remote b/cdist/conf/type/__block/gencode-remote
index 1f5cc033..acdb3286 100755
--- a/cdist/conf/type/__block/gencode-remote
+++ b/cdist/conf/type/__block/gencode-remote
@@ -46,10 +46,10 @@ fi
 
 remove_block() {
    cat << DONE
-tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
+tmpfile=\$(mktemp '${file}.cdist.XXXXXXXXXX')
 # preserve ownership and permissions of existing file
-if [ -f "$file" ]; then
-   cp -p "$file" "\$tmpfile"
+if [ -f '$file' ]; then
+   cp -p '$file' "\$tmpfile"
 fi
 awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
 {
@@ -63,8 +63,8 @@ awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
    } else {
       print
    }
-}' "$file" > "\$tmpfile"
-mv -f "\$tmpfile" "$file"
+}' '$file' > "\$tmpfile"
+mv -f "\$tmpfile" '$file'
 DONE
 }
 
@@ -77,7 +77,7 @@ case "$state_should" in
          echo add >> "$__messages_out"
       fi
       cat << DONE
-cat >> "$file" << ${__type##*/}_DONE
+cat >> '$file' << '${__type##*/}_DONE'
 $(cat "$block")
 ${__type##*/}_DONE
 DONE

From bed08c2c5c71ec4252743e0b64b75e7d8fedbd92 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sun, 6 Dec 2020 20:24:00 +0100
Subject: [PATCH 205/411] Deal with deprecation of imp module.

importlib has been a thing since Python 3.1, and imp has been deprecated since
3.4.

Insert random complaint here about not being able to use f-strings because they
were introduced in Python 3.6 and apparently we support Python 3.5 >,<.

Output diff before to after for ./bin/cdist-build-helper test (on heavy load):
```
1,2d0
< /usr/home/evilham/s/cdist/cdist/cdist/test/__main__.py:23: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
<   import imp
72c70
< ERROR: cdisttesthost: __file/tmp/foobar requires object __file without object id. Defined at /tmp/tmp.cdist.test.g87lx7c8/tmp.cdist.test.6ramsakx
---
> ERROR: cdisttesthost: __file/tmp/foobar requires object __file without object id. Defined at /tmp/tmp.cdist.test.aqdf6vjz/tmp.cdist.test.jgv3udel
76c74
< test_nonexistent_type_requirement (cdist.test.emulator.EmulatorTestCase) ... ERROR: cdisttesthost: __file/tmp/foobar requires object __does-not-exist/some-id, but type __does-not-exist does not exist. Defined at /tmp/tmp.cdist.test.mma5j8ln/tmp.cdist.test.3zg4by4d
---
> test_nonexistent_type_requirement (cdist.test.emulator.EmulatorTestCase) ... ERROR: cdisttesthost: __file/tmp/foobar requires object __does-not-exist/some-id, but type __does-not-exist does not exist. Defined at /tmp/tmp.cdist.test.t8d6ockr/tmp.cdist.test.uimxurg9
86c84
< test_initial_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running initial manifest /tmp/tmp.cdist.test.uvid60ij/759547ff4356de6e3d9e08522b0d0807/data/conf/manifest/dump_environment
---
> test_initial_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running initial manifest /tmp/tmp.cdist.test._cttcnrj/759547ff4356de6e3d9e08522b0d0807/data/conf/manifest/dump_environment
89c87
< test_type_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running type manifest /tmp/tmp.cdist.test.k1i2onpb/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__dump_environment/manifest for object __dump_environment/whatever
---
> test_type_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running type manifest /tmp/tmp.cdist.test.ukr7lrzd/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__dump_environment/manifest for object __dump_environment/whatever
272c270
< Ran 225 tests in 44.457s
---
> Ran 225 tests in 43.750s
```
---
 cdist/test/__main__.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cdist/test/__main__.py b/cdist/test/__main__.py
index c8c7df3b..8049c752 100644
--- a/cdist/test/__main__.py
+++ b/cdist/test/__main__.py
@@ -20,7 +20,7 @@
 #
 #
 
-import imp
+import importlib
 import os
 import sys
 import unittest
@@ -37,8 +37,9 @@ for possible_test in os.listdir(base_dir):
 
 suites = []
 for test_module in test_modules:
-    module_parameters = imp.find_module(test_module, [base_dir])
-    module = imp.load_module("cdist.test." + test_module, *module_parameters)
+    module_spec = importlib.util.find_spec("cdist.test.{}".format(test_module))
+    module = importlib.util.module_from_spec(module_spec)
+    module_spec.loader.exec_module(module)
 
     suite = unittest.defaultTestLoader.loadTestsFromModule(module)
     # print("Got suite: " + suite.__str__())

From 29662961731ef07fbc20213d0d87da1469f0d27c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 7 Dec 2020 19:47:52 +0100
Subject: [PATCH 206/411] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 6c272fd7..8b35901b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
+	* Core: Deal with deprecated imp in unit tests (Evil Ham)
+
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)
 

From c5ca4cd2e13516dfb55371c1600e32297c3343e9 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 7 Dec 2020 19:59:05 +0100
Subject: [PATCH 207/411] __block: securly quote via the quote function

Because the function already exists, it will be used for the file to be
changed, too. Therefor, no quotes are required for that value.

The prefix and suffix match was also improved: There is no regex check
any more (the regex did checked the whole line); instead it will do it
simple.
---
 cdist/conf/type/__block/gencode-remote | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__block/gencode-remote b/cdist/conf/type/__block/gencode-remote
index acdb3286..7a1f4064 100755
--- a/cdist/conf/type/__block/gencode-remote
+++ b/cdist/conf/type/__block/gencode-remote
@@ -46,28 +46,29 @@ fi
 
 remove_block() {
    cat << DONE
-tmpfile=\$(mktemp '${file}.cdist.XXXXXXXXXX')
+tmpfile=\$(mktemp ${quoted_file}.cdist.XXXXXXXXXX)
 # preserve ownership and permissions of existing file
-if [ -f '$file' ]; then
-   cp -p '$file' "\$tmpfile"
+if [ -f $quoted_file ]; then
+   cp -p $quoted_file "\$tmpfile"
 fi
-awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
+awk -v prefix=$(quote "$prefix") -v suffix=$(quote "$suffix") '
 {
-   if (match(\$0,prefix)) {
+   if (\$0 == prefix) {
       triggered=1
    }
    if (triggered) {
-      if (match(\$0,suffix)) {
+      if (\$0 == suffix) {
          triggered=0
       }
    } else {
       print
    }
-}' '$file' > "\$tmpfile"
-mv -f "\$tmpfile" '$file'
+}' $quoted_file > "\$tmpfile"
+mv -f "\$tmpfile" $quoted_file
 DONE
 }
 
+quoted_file="$(quote "$file")"
 case "$state_should" in
    present)
       if [ "$state_is" = "changed" ]; then
@@ -77,7 +78,7 @@ case "$state_should" in
          echo add >> "$__messages_out"
       fi
       cat << DONE
-cat >> '$file' << '${__type##*/}_DONE'
+cat >> $quoted_file << '${__type##*/}_DONE'
 $(cat "$block")
 ${__type##*/}_DONE
 DONE

From 14c81d6c7e20a2ed04c7318f6e8ecb51098390e8 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 8 Dec 2020 07:16:26 +0100
Subject: [PATCH 208/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8b35901b..4be08ae9 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
+	* Type __iptables: Add IPv6 support (Matthias Stecher)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From a58f5ffa7f59a7e28ba0a414097f154e4a497e88 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 8 Dec 2020 19:36:44 +0100
Subject: [PATCH 209/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 4be08ae9..3d6084f1 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
 	* Type __iptables: Add IPv6 support (Matthias Stecher)
+	* Type __block: Fix escaping in here-doc (Matthias Stecher)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From a5169ad858a4d6e9c378184db5736f55a86306e0 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 10 Dec 2020 21:24:26 +0100
Subject: [PATCH 210/411] new type __debian_backports

This new type will setup the backports distribution for the current
Debian release.
---
 cdist/conf/type/__debian_backports/man.rst    | 90 +++++++++++++++++++
 cdist/conf/type/__debian_backports/manifest   | 59 ++++++++++++
 .../parameter/default/mirror                  |  1 +
 .../parameter/default/state                   |  1 +
 .../__debian_backports/parameter/optional     |  2 +
 cdist/conf/type/__debian_backports/singleton  |  0
 6 files changed, 153 insertions(+)
 create mode 100644 cdist/conf/type/__debian_backports/man.rst
 create mode 100755 cdist/conf/type/__debian_backports/manifest
 create mode 100644 cdist/conf/type/__debian_backports/parameter/default/mirror
 create mode 100644 cdist/conf/type/__debian_backports/parameter/default/state
 create mode 100644 cdist/conf/type/__debian_backports/parameter/optional
 create mode 100644 cdist/conf/type/__debian_backports/singleton

diff --git a/cdist/conf/type/__debian_backports/man.rst b/cdist/conf/type/__debian_backports/man.rst
new file mode 100644
index 00000000..ba353f4e
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/man.rst
@@ -0,0 +1,90 @@
+cdist-type__debian_backports(7)
+===============================
+
+NAME
+----
+cdist-type__debian_backports - Install backports for Debain systems
+
+
+DESCRIPTION
+-----------
+This singleton type installs backports for the current Debian version.
+It aborts if backports are not supported for the specified os or no
+version codename could be fetched (like Debian unstable).
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    Represents the state of the backports repository. ``present`` or
+    ``absent``, defaults to ``present``.
+
+    Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
+
+mirror
+    The mirror to fetch the backports from. Will defaults to the Debian default
+    `<http://deb.debian.org/debian/>`_.
+
+    Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+MESSAGES
+--------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+   # setup the backports
+   __debian_backports
+   __debian_backports --state absent
+   __debian_backports --state present --mirror "http://ftp.de.debian.org/debian/"
+
+   # update
+   require="__debian_backports" __apt_update_index
+
+   # install a backports package
+   # currently for the buster release backports
+   require="__apt_update_index" __package_apt wireguard \
+        --target-release buster-backports
+
+
+ABORTS
+------
+Aborts if the detected os is not Debian.
+
+Aborts if no distribuition codename could be detected. This is common for the
+unstable distribution, but there is no backports repository for it already.
+
+
+SEE ALSO
+--------
+`Official Debian Backports site <https://backports.debian.org/>`_
+
+:strong:`cdist-type__apt_source`\ (7)
+
+
+AUTHORS
+-------
+Matthias Stecher <matthiasstecher at gmx.de>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Matthias Stecher. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__debian_backports/manifest
new file mode 100755
index 00000000..29bf9a43
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/manifest
@@ -0,0 +1,59 @@
+#!/bin/sh -e
+# __debian_backports/manifest
+#
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Enables/disables backports repository. Utilies __apt_source for it.
+#
+
+
+# detect backport distribution
+os="$(cat "$__global/explorer/os")"
+case "$os" in
+    debian)
+        # distribution codename from /etc/os-release
+        # lsb_release may not be given in all debian installations
+        dist="$(
+            . "$__global/explorer/os-release"
+            printf "%s" "$VERSION_CODENAME"
+        )"
+        ;;
+    *)
+        printf "Backports for %s are not supported!\n" "$os" >&2
+        exit 1
+        ;;
+esac
+
+# error if no codename given (e.g. on Debian unstable)
+if [ -z "$dist" ]; then
+    printf "No backports for unkown version of distribution %s!\n" "$os" >&2
+    exit 1
+fi
+
+
+# parameters
+state="$(cat "$__object/parameter/state")"
+mirror="$(cat "$__object/parameter/mirror")"
+
+# install the given backports repository
+__apt_source "${dist}-backports" \
+    --state "$state" \
+    --distribution "${dist}-backports" \
+    --component main \
+    --uri "$mirror"
diff --git a/cdist/conf/type/__debian_backports/parameter/default/mirror b/cdist/conf/type/__debian_backports/parameter/default/mirror
new file mode 100644
index 00000000..0965ef04
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/parameter/default/mirror
@@ -0,0 +1 @@
+http://deb.debian.org/debian/
diff --git a/cdist/conf/type/__debian_backports/parameter/default/state b/cdist/conf/type/__debian_backports/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__debian_backports/parameter/optional b/cdist/conf/type/__debian_backports/parameter/optional
new file mode 100644
index 00000000..4b05c235
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/parameter/optional
@@ -0,0 +1,2 @@
+state
+mirror
diff --git a/cdist/conf/type/__debian_backports/singleton b/cdist/conf/type/__debian_backports/singleton
new file mode 100644
index 00000000..e69de29b

From 0d96b31b5696f95b559450e17c71914d0746c0ce Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 11 Dec 2020 18:13:44 +0100
Subject: [PATCH 211/411] __debian_backports: pass shellcheck for sourced file

Because the sourced explorer can't be detected by shellcheck, it will be
completely disabled. Changing the path to /etc/os-release isn't
deterministic either.

The shellcheck wiki page suggests to use `source=/dev/null` instead of
`disable=SC1090`, but it was choosen to completely avoid that check ..
---
 cdist/conf/type/__debian_backports/manifest | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__debian_backports/manifest
index 29bf9a43..661e5281 100755
--- a/cdist/conf/type/__debian_backports/manifest
+++ b/cdist/conf/type/__debian_backports/manifest
@@ -30,6 +30,7 @@ case "$os" in
         # distribution codename from /etc/os-release
         # lsb_release may not be given in all debian installations
         dist="$(
+            # shellcheck disable=SC1090
             . "$__global/explorer/os-release"
             printf "%s" "$VERSION_CODENAME"
         )"

From bc2948a8a552e3bda3ab9ae8e8e199e831d81af2 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 11 Dec 2020 19:37:53 +0100
Subject: [PATCH 212/411] ++scan stuff

---
 cdist/scan/scan.py           | 15 +++++++++++++++
 docs/dev/logs/2020-10-29.org | 10 +++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index fcbf1899..00858581 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -56,6 +56,8 @@ from scapy.all import *
 # Datetime overwrites scapy.all.datetime - needs to be imported AFTER
 import datetime
 
+import cdist.config
+
 log = logging.getLogger("scan")
 
 class Trigger(object):
@@ -120,6 +122,19 @@ class Scanner(object):
             with open(fname, "w") as fd:
                 fd.write(f"{now}\n")
 
+    def config(self):
+        """
+        Configure a host
+
+        - Assume we are only called if necessary
+        - However we need to ensure to not run in parallel
+        - Maybe keep dict storing per host processes
+        - Save the result
+        - Save the output -> probably aligned to config mode
+
+        """
+
+
     def start(self):
         self.process = Process(target=self.scan)
         self.process.start()
diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org
index 4461be8c..03d6b3f4 100644
--- a/docs/dev/logs/2020-10-29.org
+++ b/docs/dev/logs/2020-10-29.org
@@ -54,4 +54,12 @@ VERBOSE: scan: Host fe80::f29f:c2ff:fe7c:275e is alive
 VERBOSE: scan: Host fe80::ba69:f4ff:fec5:8db7 is alive
 VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f863 is alive
 VERBOSE: scan: Host fe80::21b:fcff:feee:f4bc is alive
-...
+** Better usage -> saving the env
+    sudo -E cdist scan -b -I wlan0 -vv
+** TODO Implement actual configuration step
+   - Also serves as a nice PoC
+   - Might need to escape literal IPv6 addresses for scp
+** TODO Define how to map link local address to something useful
+   - via reverse DNS?
+   - via link local in manifest?
+** TODO define ignorehosts?

From c4d19a23193ec13eda96a3d7536ba338d0801825 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 09:36:17 +0100
Subject: [PATCH 213/411] __debian_backports -> __apt_backports; add wider os
 support

As discussed in the chat, this type now supports a broader list of OSes
which it supports backports for. Because of this, it was renamed to
something more generic. "apt" should fit in.
---
 .../man.rst                                   | 27 ++++++++----
 .../manifest                                  | 41 ++++++++++++++-----
 .../parameter/default/state                   |  0
 .../parameter/optional                        |  0
 .../singleton                                 |  0
 .../parameter/default/mirror                  |  1 -
 6 files changed, 49 insertions(+), 20 deletions(-)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/man.rst (66%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/manifest (59%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/parameter/default/state (100%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/parameter/optional (100%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/singleton (100%)
 delete mode 100644 cdist/conf/type/__debian_backports/parameter/default/mirror

diff --git a/cdist/conf/type/__debian_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
similarity index 66%
rename from cdist/conf/type/__debian_backports/man.rst
rename to cdist/conf/type/__apt_backports/man.rst
index ba353f4e..7d269fbb 100644
--- a/cdist/conf/type/__debian_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -3,13 +3,13 @@ cdist-type__debian_backports(7)
 
 NAME
 ----
-cdist-type__debian_backports - Install backports for Debain systems
+cdist-type__apt_backports - Install backports
 
 
 DESCRIPTION
 -----------
-This singleton type installs backports for the current Debian version.
-It aborts if backports are not supported for the specified os or no
+This singleton type installs backports for the current OS release.
+It aborts if backports are not supported for the specified OS or no
 version codename could be fetched (like Debian unstable).
 
 
@@ -27,8 +27,8 @@ state
     Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
 
 mirror
-    The mirror to fetch the backports from. Will defaults to the Debian default
-    `<http://deb.debian.org/debian/>`_.
+    The mirror to fetch the backports from. Will defaults to the generic
+    mirror of the current OS.
 
     Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
 
@@ -49,12 +49,12 @@ EXAMPLES
 .. code-block:: sh
 
    # setup the backports
-   __debian_backports
-   __debian_backports --state absent
-   __debian_backports --state present --mirror "http://ftp.de.debian.org/debian/"
+   __apt_backports
+   __apt_backports --state absent
+   __apt_backports --state present --mirror "http://ftp.de.debian.org/debian/"
 
    # update
-   require="__debian_backports" __apt_update_index
+   require="__apt_backports" __apt_update_index
 
    # install a backports package
    # currently for the buster release backports
@@ -70,6 +70,15 @@ Aborts if no distribuition codename could be detected. This is common for the
 unstable distribution, but there is no backports repository for it already.
 
 
+CAVEATS
+-------
+For Ubuntu, it setup all componenents for the backports repository: ``main``,
+``restricted``, ``universe`` and ``multiverse``. The user may not want to
+install proprietary packages, which will only be installed if the user
+explicitly uses the backports target-release. The user may change this behavior
+to install backports packages without the need of explicitly select it.
+
+
 SEE ALSO
 --------
 `Official Debian Backports site <https://backports.debian.org/>`_
diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__apt_backports/manifest
similarity index 59%
rename from cdist/conf/type/__debian_backports/manifest
rename to cdist/conf/type/__apt_backports/manifest
index 661e5281..e5358dea 100755
--- a/cdist/conf/type/__debian_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -1,5 +1,5 @@
 #!/bin/sh -e
-# __debian_backports/manifest
+# __apt_backports/manifest
 #
 # 2020 Matthias Stecher (matthiasstecher at gmx.de)
 #
@@ -23,18 +23,34 @@
 #
 
 
+# Get the distribution codename by /etc/os-release.
+#  is already executed in a subshell by string substitution
+#  lsb_release may not be given in all installations
+codename_os_release() {
+    # shellcheck disable=SC1090
+    . "$__global/explorer/os-release"
+    printf "%s" "$VERSION_CODENAME"
+}
+
 # detect backport distribution
 os="$(cat "$__global/explorer/os")"
 case "$os" in
     debian)
-        # distribution codename from /etc/os-release
-        # lsb_release may not be given in all debian installations
-        dist="$(
-            # shellcheck disable=SC1090
-            . "$__global/explorer/os-release"
-            printf "%s" "$VERSION_CODENAME"
-        )"
+        dist="$( codename_os_release )"
+        components="main"
+        mirror="http://deb.debian.org/debian/"
         ;;
+    devuan)
+        dist="$( codename_os_release )"
+        components="main"
+        mirror="http://deb.devuan.org/merged"
+        ;;
+    ubuntu)
+        dist="$( codename_os_release )"
+        components="main restricted universe multiverse"
+        mirror="http://archive.ubuntu.com/ubuntu"
+        ;;
+
     *)
         printf "Backports for %s are not supported!\n" "$os" >&2
         exit 1
@@ -50,11 +66,16 @@ fi
 
 # parameters
 state="$(cat "$__object/parameter/state")"
-mirror="$(cat "$__object/parameter/mirror")"
+
+# mirror already set for the os, only override user-values
+if [ -f "$__object/parameter/mirror" ]; then
+    mirror="$(cat "$__object/parameter/mirror")"
+fi
+
 
 # install the given backports repository
 __apt_source "${dist}-backports" \
     --state "$state" \
     --distribution "${dist}-backports" \
-    --component main \
+    --component "$components" \
     --uri "$mirror"
diff --git a/cdist/conf/type/__debian_backports/parameter/default/state b/cdist/conf/type/__apt_backports/parameter/default/state
similarity index 100%
rename from cdist/conf/type/__debian_backports/parameter/default/state
rename to cdist/conf/type/__apt_backports/parameter/default/state
diff --git a/cdist/conf/type/__debian_backports/parameter/optional b/cdist/conf/type/__apt_backports/parameter/optional
similarity index 100%
rename from cdist/conf/type/__debian_backports/parameter/optional
rename to cdist/conf/type/__apt_backports/parameter/optional
diff --git a/cdist/conf/type/__debian_backports/singleton b/cdist/conf/type/__apt_backports/singleton
similarity index 100%
rename from cdist/conf/type/__debian_backports/singleton
rename to cdist/conf/type/__apt_backports/singleton
diff --git a/cdist/conf/type/__debian_backports/parameter/default/mirror b/cdist/conf/type/__debian_backports/parameter/default/mirror
deleted file mode 100644
index 0965ef04..00000000
--- a/cdist/conf/type/__debian_backports/parameter/default/mirror
+++ /dev/null
@@ -1 +0,0 @@
-http://deb.debian.org/debian/

From 49aec0b5e40ba05e7703565731aae6b0a9e5936a Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 09:40:47 +0100
Subject: [PATCH 214/411] __apt_backports: list supported OSes

The manpage now lists all OSes where this type supports backports.
---
 cdist/conf/type/__apt_backports/man.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
index 7d269fbb..fd311361 100644
--- a/cdist/conf/type/__apt_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -12,6 +12,12 @@ This singleton type installs backports for the current OS release.
 It aborts if backports are not supported for the specified OS or no
 version codename could be fetched (like Debian unstable).
 
+It supports backports from following OSes:
+
+- Debian
+- Devuan
+- Ubuntu
+
 
 REQUIRED PARAMETERS
 -------------------

From fafa3d9ea55f38ae0a350a463d64b7ef1d70c7bb Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 10:00:23 +0100
Subject: [PATCH 215/411] __apt_backports: update index if required

This type now automatically calls the type __apt_update_index to update
the package index if something changed.
---
 cdist/conf/type/__apt_backports/man.rst  | 13 +++++++------
 cdist/conf/type/__apt_backports/manifest |  5 ++++-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
index fd311361..c578ce6b 100644
--- a/cdist/conf/type/__apt_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -9,8 +9,11 @@ cdist-type__apt_backports - Install backports
 DESCRIPTION
 -----------
 This singleton type installs backports for the current OS release.
-It aborts if backports are not supported for the specified OS or no
-version codename could be fetched (like Debian unstable).
+It aborts if backports are not supported for the specified OS or
+no version codename could be fetched (like Debian unstable).
+
+The package index will be automatically updated by the type
+:strong:`cdist-type__apt_update_index`\ (7) if required.
 
 It supports backports from following OSes:
 
@@ -59,12 +62,9 @@ EXAMPLES
    __apt_backports --state absent
    __apt_backports --state present --mirror "http://ftp.de.debian.org/debian/"
 
-   # update
-   require="__apt_backports" __apt_update_index
-
    # install a backports package
    # currently for the buster release backports
-   require="__apt_update_index" __package_apt wireguard \
+   require="__apt_backports" __package_apt wireguard \
         --target-release buster-backports
 
 
@@ -90,6 +90,7 @@ SEE ALSO
 `Official Debian Backports site <https://backports.debian.org/>`_
 
 :strong:`cdist-type__apt_source`\ (7)
+:strong:`cdist-type__apt_update_index`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index e5358dea..c490a103 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -19,7 +19,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Enables/disables backports repository. Utilies __apt_source for it.
+# Enables/disables backports repository. Utilises __apt_source for it.
 #
 
 
@@ -79,3 +79,6 @@ __apt_source "${dist}-backports" \
     --distribution "${dist}-backports" \
     --component "$components" \
     --uri "$mirror"
+
+# update the index if the source changed
+require="__apt_source/${dist}-backports" __apt_update_index

From 645734c62959c694597826598be3b34c0edc79d2 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 12 Dec 2020 12:15:17 +0100
Subject: [PATCH 216/411] [explorer/os_version] Improve FreeBSD support.

It looks like uname -r is not the most reliable way to get the target patch
level for the target system.

For more information see:
https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
---
 cdist/conf/explorer/os_version | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index a7b1d3bc..3b02dedd 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -70,6 +70,11 @@ case "$("$__explorer/os")" in
    macosx)
       sw_vers -productVersion
    ;;
+   freebsd)
+      # Apparently uname -r is not a reliable way to get the patch level.
+      # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
+      freebsd-version
+   ;;
    *bsd|solaris)
       uname -r
    ;;

From fca35fc858d09fc649dc5e7f0964cef4af9e09f0 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 17:29:58 +0100
Subject: [PATCH 217/411] __apt_backports: fix explorer call

s/-/_/ because the explorers are following an other convention :-)
---
 cdist/conf/type/__apt_backports/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index c490a103..c2943725 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -28,7 +28,7 @@
 #  lsb_release may not be given in all installations
 codename_os_release() {
     # shellcheck disable=SC1090
-    . "$__global/explorer/os-release"
+    . "$__global/explorer/os_release"
     printf "%s" "$VERSION_CODENAME"
 }
 

From 27aca06fb893c84601e14bc30890ea6be9300dcd Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 17:34:51 +0100
Subject: [PATCH 218/411] __apt_backports: undo __apt_update_index call

Becuase it is already done by __apt_source.
---
 cdist/conf/type/__apt_backports/man.rst  | 4 +---
 cdist/conf/type/__apt_backports/manifest | 3 ---
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
index c578ce6b..7036fb84 100644
--- a/cdist/conf/type/__apt_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -12,8 +12,7 @@ This singleton type installs backports for the current OS release.
 It aborts if backports are not supported for the specified OS or
 no version codename could be fetched (like Debian unstable).
 
-The package index will be automatically updated by the type
-:strong:`cdist-type__apt_update_index`\ (7) if required.
+The package index will be automatically updated if required.
 
 It supports backports from following OSes:
 
@@ -90,7 +89,6 @@ SEE ALSO
 `Official Debian Backports site <https://backports.debian.org/>`_
 
 :strong:`cdist-type__apt_source`\ (7)
-:strong:`cdist-type__apt_update_index`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index c2943725..bc47d8de 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -79,6 +79,3 @@ __apt_source "${dist}-backports" \
     --distribution "${dist}-backports" \
     --component "$components" \
     --uri "$mirror"
-
-# update the index if the source changed
-require="__apt_source/${dist}-backports" __apt_update_index

From 71f22831175ec41e9348a759fcdaf0eceb1a2a52 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 13 Dec 2020 16:03:39 +0100
Subject: [PATCH 219/411] ++changelog

---
 docs/changelog | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 3d6084f1..290f94d0 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -2,10 +2,12 @@ Changelog
 ---------
 
 next:
-	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
+	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
 	* Type __iptables: Add IPv6 support (Matthias Stecher)
 	* Type __block: Fix escaping in here-doc (Matthias Stecher)
+	* Explorer os_version: Improve FreeBSD support (Evil Ham)
+	* New type: __apt_backports (Matthias Stecher)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From 932e2496ed87b830a468a263cf1b1f5e6a88e2ef Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 15 Dec 2020 18:40:39 +0100
Subject: [PATCH 220/411] [type/__postgres_role] Lint

---
 .../conf/type/__postgres_role/explorer/state  | 32 ++++----
 .../conf/type/__postgres_role/gencode-remote  | 78 ++++++++++---------
 2 files changed, 58 insertions(+), 52 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index c8e1fa9d..110d29d5 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -11,32 +11,32 @@
 #
 # cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$("${__explorer}/os")"
+case $("${__explorer:?}/os")
 in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
+	(netbsd)
+		postgres_user='pgsql'
+		;;
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(*)
+		postgres_user='postgres'
+		;;
 esac
 
 
-name="$__object_id"
+rolename=${__object_id:?}
 
-if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='$name'\"")"
+if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='${rolename}'\"")"
 then
-    echo 'present'
+	echo 'present'
 else
-    echo 'absent'
+	echo 'absent'
 fi
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 282294c9..4fb761ed 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -11,55 +11,61 @@
 #
 # cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
+case $(cat "${__global:?}/explorer/os")
 in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
+	(netbsd)
+		postgres_user='pgsql'
+		;;
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(*)
+		postgres_user='postgres'
+		;;
 esac
 
 
-name="$__object_id"
-state_is="$(cat "$__object/explorer/state")"
-state_should="$(cat "$__object/parameter/state")"
+rolename=${__object_id:?}
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 
-[ "$state_is" = "$state_should" ] && exit 0
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
 
-case "$state_should" in
-    present)
-        if [ -f "$__object/parameter/password" ]; then
-            password="$(cat "$__object/parameter/password")"
-        fi
-        booleans=""
-        for boolean in login createdb createrole superuser; do
-            if [ ! -f "$__object/parameter/$boolean" ]; then
-                boolean="no${boolean}"
-            fi
-            upper=$(echo $boolean | tr '[:lower:]' '[:upper:]')
-            booleans="$booleans $upper"
-        done
+case ${state_should}
+in
+	(present)
+		if test -f "${__object:?}/parameter/password"
+		then
+			password=$(cat "${__object:?}/parameter/password")
+		fi
+		booleans=
+		for boolean in login createdb createrole superuser
+		do
+			if test ! -f "${__object:?}/parameter/${boolean}"
+			then
+				boolean="no${boolean}"
+			fi
+			booleans="${booleans} $(echo ${boolean} | tr '[:lower:]' '[:upper:]')"
+		done
 
-        [ -n "$password" ] && password="PASSWORD '$password'"
-        cat << EOF
-su - '$postgres_user' -c "psql postgres -wc \"CREATE ROLE \\\\\"$name\\\\\" WITH $password $booleans;\""
+		[ -n "${password}" ] && password="PASSWORD '${password}'"
+		cat << EOF
+su - '${postgres_user}' -c "psql postgres -wc 'CREATE ROLE \\"${rolename}\\" WITH ${password} ${booleans};'"
 EOF
-    ;;
-    absent)
-        cat << EOF
-su - '$postgres_user' -c "dropuser \"$name\""
+		;;
+	(absent)
+		cat << EOF
+su - '${postgres_user}' -c "dropuser '${rolename}'"
 EOF
-    ;;
+		;;
 esac

From c36df82882fedb6ce3630e4718c46992a7c153a0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 15 Dec 2020 21:05:55 +0100
Subject: [PATCH 221/411] [type/__postgres_role] ALTER ROLE when parameters
 change

---
 .../conf/type/__postgres_role/explorer/state  | 42 +++++++++++-
 .../conf/type/__postgres_role/gencode-remote  | 65 +++++++++++++++----
 2 files changed, 91 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 110d29d5..033afcb2 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -31,11 +32,48 @@ in
 		;;
 esac
 
-
 rolename=${__object_id:?}
 
-if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='${rolename}'\"")"
+role_properties=$(
+	cmd=$(printf "psql -F '\034' -R '\036' -wAc \"SELECT * FROM pg_roles WHERE rolname='%s'\"" "${rolename}")
+	su -l "${postgres_user}" -c "${cmd}" \
+	| awk '
+	  BEGIN { RS = "\036"; FS = "\034" }
+	  /^\([0-9]+ rows?\)/ { exit }
+	  NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next }
+	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }
+	  '
+)
+
+if test -n "${role_properties}"
 then
+	# Check if the user's properties match the parameters
+	for prop in login createdb createrole superuser
+	do
+		bool_should=$(test -f "${__object:?}/parameter/${prop}" && echo 't' || echo 'f')
+		bool_is=$(
+			printf '%s\n' "${role_properties}" |
+			awk -F '=' -v key="${prop}" '
+			BEGIN {
+				if (key == "login")
+					key = "canlogin"
+				else if (key == "superuser")
+					key = "super"
+				key = "rol" key
+			}
+			$1 == key {
+				sub(/^[^=]*=/, "")
+				print
+			}
+			'
+		)
+
+		test "${bool_is}" = "${bool_should}" || {
+			echo 'different'
+			exit 0
+		}
+	done
+
 	echo 'present'
 else
 	echo 'absent'
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 4fb761ed..7837976c 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,6 +19,15 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+quote() {
+	if test $# -gt 0
+	then
+		printf '%s' "$*"
+	else
+		cat -
+	fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
+}
+
 case $(cat "${__global:?}/explorer/os")
 in
 	(netbsd)
@@ -44,28 +54,55 @@ fi
 case ${state_should}
 in
 	(present)
-		if test -f "${__object:?}/parameter/password"
+		if test -s "${__object:?}/parameter/password"
 		then
-			password=$(cat "${__object:?}/parameter/password")
+			quoted_password=$(
+				delim='$$'
+				while grep -q -F "${delim}" "${__object:?}/parameter/password"
+				do
+					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
+				done
+
+				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
+				# shellcheck disable=SC2016
+				printf '%s%s%s' "${delim}" "${raw_passwd%?.}" "${delim}"
+			)
 		fi
+
 		booleans=
 		for boolean in login createdb createrole superuser
 		do
-			if test ! -f "${__object:?}/parameter/${boolean}"
-			then
-				boolean="no${boolean}"
-			fi
-			booleans="${booleans} $(echo ${boolean} | tr '[:lower:]' '[:upper:]')"
+			booleans="${booleans}${booleans:+ }$(
+				if test -f "${__object:?}/parameter/${boolean}"
+				then
+					echo "${boolean}"
+				else
+					echo "no${boolean}"
+				fi \
+				| tr '[:lower:]' '[:upper:]')"
 		done
 
-		[ -n "${password}" ] && password="PASSWORD '${password}'"
-		cat << EOF
-su - '${postgres_user}' -c "psql postgres -wc 'CREATE ROLE \\"${rolename}\\" WITH ${password} ${booleans};'"
-EOF
+		case ${state_is}
+		in
+			(absent)
+				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				;;
+			(different)
+				query=$(printf 'ALTER ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				;;
+			(*)
+				exit 1	# TODO: error msg
+				;;
+		esac
+
+		psql_cmd=$(printf 'psql postgres -wc %s' "$(quote "${query}")" | quote)
+		printf "su -l '%s' -c %s\\n" "${postgres_user}" "${psql_cmd}"
 		;;
 	(absent)
-		cat << EOF
-su - '${postgres_user}' -c "dropuser '${rolename}'"
-EOF
+		printf "su -l '%s' -c 'dropuser '\\\\'%s\\\\'\\n" \
+			"${postgres_user}" \
+			"$(quote "${rolename}")"
 		;;
 esac

From 7b7ca4d385ccd025cbefced58419468651652da8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 16 Dec 2020 19:06:50 +0100
Subject: [PATCH 222/411] [type/__postgres_role] Handle password changes

---
 .../conf/type/__postgres_role/explorer/state  | 76 +++++++++++++++++--
 .../conf/type/__postgres_role/gencode-remote  | 23 ++++--
 2 files changed, 88 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 033afcb2..4aadbdff 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -34,9 +34,23 @@ esac
 
 rolename=${__object_id:?}
 
+
+psql_query() {
+	su -l "${postgres_user}" -c "$(
+		printf "psql -F '\034' -R '\036' -wAc '%s'" \
+		"$(printf %s "$*" | sed "s/'/'\\\\''/g")"
+	)"
+}
+
+password_check_login() (
+	PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .)
+	PGPASSWORD=${PGPASSWORD%?.}
+	export PGPASSWORD
+	psql -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
+)
+
 role_properties=$(
-	cmd=$(printf "psql -F '\034' -R '\036' -wAc \"SELECT * FROM pg_roles WHERE rolname='%s'\"" "${rolename}")
-	su -l "${postgres_user}" -c "${cmd}" \
+	psql_query "SELECT * FROM pg_roles WHERE rolname = '${rolename}'" \
 	| awk '
 	  BEGIN { RS = "\036"; FS = "\034" }
 	  /^\([0-9]+ rows?\)/ { exit }
@@ -69,12 +83,62 @@ then
 		)
 
 		test "${bool_is}" = "${bool_should}" || {
-			echo 'different'
-			exit 0
+			state='different properties'
 		}
 	done
 
-	echo 'present'
+	# Check password
+	passwd_stored=$(
+		psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \
+		| awk 'BEGIN { RS = "\036" } NR == 2'
+		printf .
+	)
+	passwd_stored=${passwd_stored%?.}
+
+	passwd_should=$(cat "${__object}/parameter/password"; printf .)
+	passwd_should=${passwd_should%?.}
+
+	if expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
+	then
+		# SCRAM-SHA-256 "encrypted" password
+		# NOTE: There is currently no easy way to check SCRAM passwords
+		password_check_login || state="${state:-different} password"
+	elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null
+	then
+		# MD5 "encrypted" password
+		if command -v md5sum >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| md5sum - | sed -e 's/[^0-9a-f]*$//')
+		elif command -v gmd5sum >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| gmd5sum - | sed -e 's/[^0-9a-f]*$//')
+		elif command -v openssl >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| openssl dgst -md5 | sed 's/^.* //')
+		fi
+
+		if test -n "${should_md5}"
+		then
+			test "${passwd_stored}" = "md5${should_md5}" \
+			|| state="${state:-different} password"
+		else
+			password_check_login || state="${state:-different} password"
+		fi
+	else
+		# unencrypted password (unsupported since PostgreSQL 10)
+		test "${passwd_stored}" = "${passwd_should}" \
+		|| state="${state:-different} password"
+	fi
+
+	test -n "${state}" || state='present'
 else
-	echo 'absent'
+	state='absent'
 fi
+
+echo "${state}"
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 7837976c..fd590a4b 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -58,7 +58,9 @@ in
 		then
 			quoted_password=$(
 				delim='$$'
-				while grep -q -F "${delim}" "${__object:?}/parameter/password"
+				# NOTE: Strip away trailing $ because with it the check breaks
+				#       if the password ends with $ + random value.
+				while grep -q -F "${delim%$}" "${__object:?}/parameter/password"
 				do
 					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
 				done
@@ -88,12 +90,23 @@ in
 				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
 					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
 				;;
-			(different)
-				query=$(printf 'ALTER ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+			(different*)
+				query="ALTER ROLE \"${rolename}\" WITH"
+
+				if expr "${state_is}" : 'different.*properties' >/dev/null
+				then
+					query="${query} ${booleans}"
+				fi
+				if expr "${state_is}" : 'different.*password' >/dev/null
+				then
+					query="${query} PASSWORD ${quoted_password:-NULL}"
+				fi
+
+				query="${query};"
 				;;
 			(*)
-				exit 1	# TODO: error msg
+				printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2
+				exit 1
 				;;
 		esac
 

From 4859c27900d125841c703e0270dd8c0eae05d504 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 16:57:03 +0100
Subject: [PATCH 223/411] [type/__postgres_role] Refactor gencode-remote

---
 .../conf/type/__postgres_role/gencode-remote  | 65 ++++++++++---------
 1 file changed, 34 insertions(+), 31 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index fd590a4b..540eb606 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -51,9 +51,6 @@ then
 	exit 0
 fi
 
-case ${state_should}
-in
-	(present)
 		if test -s "${__object:?}/parameter/password"
 		then
 			quoted_password=$(
@@ -64,6 +61,11 @@ in
 				do
 					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
 				done
+psql_query() {
+	printf 'su -l %s -c %s\n' \
+		"$(quote "${postgres_user}")" \
+		"$(quote "psql postgres -wc $(quote "$1")")"
+}
 
 				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
 				# shellcheck disable=SC2016
@@ -71,51 +73,52 @@ in
 			)
 		fi
 
-		booleans=
-		for boolean in login createdb createrole superuser
-		do
-			booleans="${booleans}${booleans:+ }$(
-				if test -f "${__object:?}/parameter/${boolean}"
-				then
-					echo "${boolean}"
-				else
-					echo "no${boolean}"
-				fi \
-				| tr '[:lower:]' '[:upper:]')"
-		done
+role_properties_should() {
+	_props=
+	for _prop in login createdb createrole superuser
+	do
+		_props="${_props}${_props:+ }$(
+			if test -f "${__object:?}/parameter/${_prop}"
+			then
+				echo "${_prop}"
+			else
+				echo "no${_prop}"
+			fi \
+			| tr '[:lower:]' '[:upper:]')"
+	done
+	printf '%s\n' "${_props}"
+	unset _prop _props
+}
 
+case ${state_should}
+in
+	(present)
 		case ${state_is}
 		in
 			(absent)
-				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				psql_query "$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "$(role_properties_should)" "${quoted_password:-NULL}")"
 				;;
 			(different*)
-				query="ALTER ROLE \"${rolename}\" WITH"
-
 				if expr "${state_is}" : 'different.*properties' >/dev/null
 				then
-					query="${query} ${booleans}"
-				fi
-				if expr "${state_is}" : 'different.*password' >/dev/null
-				then
-					query="${query} PASSWORD ${quoted_password:-NULL}"
+					psql_query "ALTER ROLE \"${rolename}\" WITH $(role_properties_should);"
 				fi
 
-				query="${query};"
+				if expr "${state_is}" : 'different.*password' >/dev/null
+				then
+					psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD ${quoted_password:-NULL};"
+				fi
 				;;
 			(*)
 				printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2
 				exit 1
 				;;
 		esac
-
-		psql_cmd=$(printf 'psql postgres -wc %s' "$(quote "${query}")" | quote)
-		printf "su -l '%s' -c %s\\n" "${postgres_user}" "${psql_cmd}"
 		;;
 	(absent)
-		printf "su -l '%s' -c 'dropuser '\\\\'%s\\\\'\\n" \
-			"${postgres_user}" \
-			"$(quote "${rolename}")"
+		printf 'su -l %s -c %s\n' \
+			"$(quote "${postgres_user}")" \
+			"$(quote "dropuser $(quote "${rolename}")")"
 		;;
 esac

From 1180f13ed6c12e0277bebdf32bdd9840320d0fb9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 16:58:32 +0100
Subject: [PATCH 224/411] [type/__postgres_role] Fix setting password

We need to make sure that the password does not end up in ~/.psql_history.
---
 .../conf/type/__postgres_role/gencode-remote  | 38 ++++++++++---------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 540eb606..15c3e692 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -51,27 +51,29 @@ then
 	exit 0
 fi
 
-		if test -s "${__object:?}/parameter/password"
-		then
-			quoted_password=$(
-				delim='$$'
-				# NOTE: Strip away trailing $ because with it the check breaks
-				#       if the password ends with $ + random value.
-				while grep -q -F "${delim%$}" "${__object:?}/parameter/password"
-				do
-					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
-				done
 psql_query() {
 	printf 'su -l %s -c %s\n' \
 		"$(quote "${postgres_user}")" \
 		"$(quote "psql postgres -wc $(quote "$1")")"
 }
 
-				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
-				# shellcheck disable=SC2016
-				printf '%s%s%s' "${delim}" "${raw_passwd%?.}" "${delim}"
-			)
-		fi
+psql_set_password() {
+	# NOTE: Always make sure that the password does not end up in psql_history!
+	if test -s "${__object:?}/parameter/password"
+	then
+		cat <<-EOF
+		exec 3< "\${__object:?}/parameter/password"
+		su -l '${postgres_user}' -c 'psql -q postgres -w' <<'SQL'
+		\set HISTFILE /dev/null
+		\set pw \`cat <&3\`
+		ALTER ROLE "${rolename}" WITH PASSWORD :'pw';
+		SQL
+		exec 3<&-
+		EOF
+	else
+		psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD NULL;"
+	fi
+}
 
 role_properties_should() {
 	_props=
@@ -96,8 +98,8 @@ in
 		case ${state_is}
 		in
 			(absent)
-				psql_query "$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "$(role_properties_should)" "${quoted_password:-NULL}")"
+				psql_query "CREATE ROLE \"${rolename}\" WITH $(role_properties_should);"
+				psql_set_password
 				;;
 			(different*)
 				if expr "${state_is}" : 'different.*properties' >/dev/null
@@ -107,7 +109,7 @@ in
 
 				if expr "${state_is}" : 'different.*password' >/dev/null
 				then
-					psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD ${quoted_password:-NULL};"
+					psql_set_password
 				fi
 				;;
 			(*)

From 99d82fd0d5ab6d6ccd95fe8a7d3b24f5d82a4cce Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 17:05:58 +0100
Subject: [PATCH 225/411] [type/__postgres_role] Always set psql -q

---
 cdist/conf/type/__postgres_role/explorer/state | 4 ++--
 cdist/conf/type/__postgres_role/gencode-remote | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 4aadbdff..77779662 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -37,7 +37,7 @@ rolename=${__object_id:?}
 
 psql_query() {
 	su -l "${postgres_user}" -c "$(
-		printf "psql -F '\034' -R '\036' -wAc '%s'" \
+		printf "psql -q -F '\034' -R '\036' -wAc '%s'" \
 		"$(printf %s "$*" | sed "s/'/'\\\\''/g")"
 	)"
 }
@@ -46,7 +46,7 @@ password_check_login() (
 	PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .)
 	PGPASSWORD=${PGPASSWORD%?.}
 	export PGPASSWORD
-	psql -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
+	psql -q -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
 )
 
 role_properties=$(
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 15c3e692..877c8135 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -54,7 +54,7 @@ fi
 psql_query() {
 	printf 'su -l %s -c %s\n' \
 		"$(quote "${postgres_user}")" \
-		"$(quote "psql postgres -wc $(quote "$1")")"
+		"$(quote "psql postgres -q -w -c $(quote "$1")")"
 }
 
 psql_set_password() {
@@ -63,7 +63,7 @@ psql_set_password() {
 	then
 		cat <<-EOF
 		exec 3< "\${__object:?}/parameter/password"
-		su -l '${postgres_user}' -c 'psql -q postgres -w' <<'SQL'
+		su -l '${postgres_user}' -c 'psql -q -w postgres' <<'SQL'
 		\set HISTFILE /dev/null
 		\set pw \`cat <&3\`
 		ALTER ROLE "${rolename}" WITH PASSWORD :'pw';

From 8dc2c4207cdd826b9e64cc91cfc7c918cddd5f35 Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Fri, 18 Dec 2020 11:16:28 +0100
Subject: [PATCH 226/411] Added optional dirmode parameter to set the mode of
 (optional) the directory.

---
 cdist/conf/type/__dot_file/man.rst                   | 3 +++
 cdist/conf/type/__dot_file/manifest                  | 2 ++
 cdist/conf/type/__dot_file/parameter/default/dirmode | 1 +
 cdist/conf/type/__dot_file/parameter/optional        | 1 +
 4 files changed, 7 insertions(+)
 create mode 100644 cdist/conf/type/__dot_file/parameter/default/dirmode

diff --git a/cdist/conf/type/__dot_file/man.rst b/cdist/conf/type/__dot_file/man.rst
index ae65eb95..ba7621a1 100644
--- a/cdist/conf/type/__dot_file/man.rst
+++ b/cdist/conf/type/__dot_file/man.rst
@@ -25,6 +25,9 @@ user
 OPTIONAL PARAMETERS
 -------------------
 
+dirmode
+    forwarded to :strong:`__directory` type as mode
+
 mode
     forwarded to :strong:`__file` type
 
diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest
index 5e4957e5..02dadf05 100755
--- a/cdist/conf/type/__dot_file/manifest
+++ b/cdist/conf/type/__dot_file/manifest
@@ -19,6 +19,7 @@ set -eu
 user="$(cat "${__object}/parameter/user")"
 home="$(cat "${__object}/explorer/home")"
 primary_group="$(cat "${__object}/explorer/primary_group")"
+dirmode="$(cat "${__object}/parameter/dirmode")"
 
 # Create parent directory. Type __directory has flag 'parents', but it
 # will leave us with root-owned directory in user home, which is not
@@ -36,6 +37,7 @@ export CDIST_ORDER_DEPENDENCY
 for dir ; do
 	__directory "${home}/${dir}"   \
 	    --group "${primary_group}" \
+            --mode "${dirmode}" \
 	    --owner "${user}"
 done
 
diff --git a/cdist/conf/type/__dot_file/parameter/default/dirmode b/cdist/conf/type/__dot_file/parameter/default/dirmode
new file mode 100644
index 00000000..e9745d1f
--- /dev/null
+++ b/cdist/conf/type/__dot_file/parameter/default/dirmode
@@ -0,0 +1 @@
+0700
diff --git a/cdist/conf/type/__dot_file/parameter/optional b/cdist/conf/type/__dot_file/parameter/optional
index ccab9fa6..9f7f83fb 100644
--- a/cdist/conf/type/__dot_file/parameter/optional
+++ b/cdist/conf/type/__dot_file/parameter/optional
@@ -1,3 +1,4 @@
 state
 mode
 source
+dirmode

From 4bae2863dbc1f6b60b8e797c2f121e1616a94c94 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 18 Dec 2020 12:54:33 +0100
Subject: [PATCH 227/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 290f94d0..3a623236 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __block: Fix escaping in here-doc (Matthias Stecher)
 	* Explorer os_version: Improve FreeBSD support (Evil Ham)
 	* New type: __apt_backports (Matthias Stecher)
+	* Type __dot_file: Add dirmode parameter (Mark Verboom)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From 7cf85c465980c74049f8e0258758c0fd8ea178b2 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Dec 2020 19:21:51 +0100
Subject: [PATCH 228/411] Release 6.9.4

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 3a623236..35953a88 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
 	* Type __iptables: Add IPv6 support (Matthias Stecher)

From 766198912d558e4630b258bba62e78587b3f0efe Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 5 Jan 2021 15:50:21 +0100
Subject: [PATCH 229/411] [type/__sshd_config] Produce error if invalid config
 file is generated

Previously, cdist would silently swallow the error (no invalid config file was
generated).

Reason: `set -e` does not exit if a command in a sub-command group fails,
it merely returns with a non-zero exit status.

e.g. the following snippet does not abort the script if sshd -t returns with a
non-zero exit status:

    set -e
    cmp -s old new || {
        # check config file and update it
        sshd -t -f new \
        && cat new >old
    }

or compressed:

    set -e
    false || { false && true; }
    echo $?
    # prints 1
---
 cdist/conf/type/__sshd_config/gencode-remote | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sshd_config/gencode-remote b/cdist/conf/type/__sshd_config/gencode-remote
index 0b44dfa7..275db4aa 100755
--- a/cdist/conf/type/__sshd_config/gencode-remote
+++ b/cdist/conf/type/__sshd_config/gencode-remote
@@ -91,7 +91,8 @@ awk $(drop_awk_comments "${__type:?}/files/update_sshd_config.awk") \\
 
 cmp -s $(quote "${sshd_config_file}") $(quote "${sshd_config_file}.tmp") || {
 	sshd -t -f $(quote "${sshd_config_file}.tmp") \\
-	&& cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}")
+	&& cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}") \\
+	|| exit  # stop if sshd_config file check fails
 }
 rm -f $(quote "${sshd_config_file}.tmp")
 EOF

From 8753b7eedf022a052ddc0b27fbc58ebf8fd638e1 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 5 Jan 2021 15:57:39 +0100
Subject: [PATCH 230/411] [type/__sshd_config] Make AuthenticationMethods and
 AuthorizedKeysFile singleton options

They were incorrectly treated as non-singleton options before.

cf. https://github.com/openssh/openssh-portable/blob/V_8_4/servconf.c#L2273
and https://github.com/openssh/openssh-portable/blob/V_8_4/servconf.c#L1899 resp.
---
 cdist/conf/type/__sshd_config/files/update_sshd_config.awk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
index d0bc2b4b..f7f30e87 100644
--- a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
+++ b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
@@ -89,7 +89,7 @@ function strdelim(s) { return strdelim_internal(s, 1) }
 function strdelimw(s) { return strdelim_internal(s, 0) }
 
 function singleton_option(opt) {
-	return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|authenticationmethods|authorizedkeysfile|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/
+	return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/
 }
 
 function print_update() {

From bd8ab8f26fdc242f2eb77e58049d640f96d51a69 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 5 Jan 2021 17:00:55 +0100
Subject: [PATCH 231/411] [type/__sshd_config] Document "bug" in state explorer

---
 cdist/conf/type/__sshd_config/man.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cdist/conf/type/__sshd_config/man.rst b/cdist/conf/type/__sshd_config/man.rst
index 8b0069ac..c8e6b8ad 100644
--- a/cdist/conf/type/__sshd_config/man.rst
+++ b/cdist/conf/type/__sshd_config/man.rst
@@ -79,6 +79,10 @@ BUGS
 - ``Include`` directives are ignored.
 - Config options are not added/removed to/from the config file if their value is
   the default value.
+- | The explorer will incorrectly report ``absent`` if OpenSSH internally
+    transforms one value to another (e.g. ``permitrootlogin prohibit-password``
+    is transformed to ``permitrootlogin without-password``).
+  | Workaround: Use the value that OpenSSH uses internally.
 
 
 AUTHORS

From c819548343ad7bb24d023f94a32575e1b2520328 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 11 Jan 2021 09:50:12 +0100
Subject: [PATCH 232/411] Fix debug parameter

-d was removed from cdist in favor of mulitple -v and -l parameters, but
-d was not removed from preos.

Resolve #849.
---
 cdist/preos/debootstrap/files/code | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/preos/debootstrap/files/code b/cdist/preos/debootstrap/files/code
index 9e37003b..d836848c 100755
--- a/cdist/preos/debootstrap/files/code
+++ b/cdist/preos/debootstrap/files/code
@@ -22,7 +22,7 @@ set -e
 if [ "${debug}" ]
 then
     set -x
-    cdist_params="${cdist_params} -d"
+    cdist_params="${cdist_params} -l 3"
 fi
 
 bootstrap_dir="${target_dir}"

From 2954347771e00610a084c7b033016765f7a3d6d7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 14 Jan 2021 13:46:40 +0100
Subject: [PATCH 233/411] [type/__postgres_role] Add note regarding empty
 passwords

---
 cdist/conf/type/__postgres_role/gencode-remote | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 877c8135..d7631fbd 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -59,6 +59,8 @@ psql_query() {
 
 psql_set_password() {
 	# NOTE: Always make sure that the password does not end up in psql_history!
+	# NOTE: Never set an empty string as the password, because they can be
+	#       interpreted differently by different tooling.
 	if test -s "${__object:?}/parameter/password"
 	then
 		cat <<-EOF

From 6e9b13d94962e1bfb365efa849ebefc32854364b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 18 Jan 2021 06:22:32 +0100
Subject: [PATCH 234/411] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 35953a88..b2b35616 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
+	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
+
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)

From 92a50da4873ff4bfa85dde3b85e272704cda02c7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 18 Jan 2021 06:28:09 +0100
Subject: [PATCH 235/411] Fix pycodestyle issues

---
 cdist/scan/scan.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 5ca8fae2..b1d0e9e1 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -139,7 +139,6 @@ class Scanner(object):
 
         """
 
-
     def start(self):
         self.process = Process(target=self.scan)
         self.process.start()

From 35cde3e666c229ceb94a1c5074083f7ba905768a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 13:09:29 +0100
Subject: [PATCH 236/411] [type/__postgres_role] Fix state explorer when stored
 password is empty

---
 cdist/conf/type/__postgres_role/explorer/state | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 77779662..34069de9 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -95,13 +95,20 @@ then
 	)
 	passwd_stored=${passwd_stored%?.}
 
-	passwd_should=$(cat "${__object}/parameter/password"; printf .)
+	if test -f "${__object:?}/parameter/password"
+	then
+		passwd_should=$(cat "${__object:?}/parameter/password"; printf .)
+	fi
 	passwd_should=${passwd_should%?.}
 
-	if expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
+	if test -z "${passwd_stored}"
+	then
+		test -z "${passwd_should}" || state="${state:-different} password"
+	elif expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
 	then
 		# SCRAM-SHA-256 "encrypted" password
-		# NOTE: There is currently no easy way to check SCRAM passwords
+		# NOTE: There is currently no easy way to check SCRAM passwords without
+		#       logging in
 		password_check_login || state="${state:-different} password"
 	elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null
 	then

From c51d68a7375915154a90c22e716e832f635ca878 Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Thu, 3 Dec 2020 18:48:04 +0100
Subject: [PATCH 237/411] [type/__postgres_conf] New type based on ALTER SYSTEM
 command

---
 .../conf/type/__postgres_conf/gencode-remote  | 92 +++++++++++++++++++
 cdist/conf/type/__postgres_conf/man.rst       | 55 +++++++++++
 .../__postgres_conf/parameter/default/state   |  1 +
 .../type/__postgres_conf/parameter/optional   |  2 +
 4 files changed, 150 insertions(+)
 create mode 100755 cdist/conf/type/__postgres_conf/gencode-remote
 create mode 100644 cdist/conf/type/__postgres_conf/man.rst
 create mode 100644 cdist/conf/type/__postgres_conf/parameter/default/state
 create mode 100644 cdist/conf/type/__postgres_conf/parameter/optional

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
new file mode 100755
index 00000000..a09e1873
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -0,0 +1,92 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+# 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$(cat "${__global}/explorer/os")
+state_should=$(cat "${__object}/parameter/state")
+
+if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ]
+then
+	echo "Invalid state '${state_should}'." \
+	     'Only "present" and "absent" are acceptable' >&2
+	exit 1
+fi
+
+if [ "${state_should}" = 'present' ] && [ ! -f "${__object}/parameter/value" ]
+then
+	echo 'Missing required parameter "value"' >&2
+	exit 1
+fi
+
+# Parameters
+conf_name="${__object_id}"
+if [ -f "${__object}/parameter/value" ]
+then
+	conf_value=$(cat "${__object}/parameter/value")
+fi
+
+if [ "${state_should}" = 'present' ]
+then
+	set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'"
+	check_command="SHOW ${conf_name}"
+else
+	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
+fi
+
+case $os
+in
+	openbsd|devuan)
+		case $os
+		in
+			openbsd)
+				postgres_user='_postgresql'
+				restart_command='/etc/rc.d/postgresql restart'
+				;;
+			devuan)
+				postgres_user='postgres'
+				restart_command='/etc/init.d/postgresql restart'
+				;;
+		esac
+		# needs two separate psql commands because ALTER SYSTEM
+		# cannot run inside a transaction block
+		cat <<-EOF
+		su - ${postgres_user} -c "psql postgres -twAc \
+			\"${set_command}\""
+		su - ${postgres_user} -c "psql postgres -twAc \
+			\"SELECT pg_reload_conf()\""
+		EOF
+		# check success (makes only sense if setting to a non-default value)
+		# and restart server if needed
+		if [ "${state_should}" = 'present' ]
+		then
+			cat <<-EOF
+			if [ \$(su - ${postgres_user} -c "psql postgres -twAc \"SHOW ${conf_name}\"") != '${conf_value}' ]
+			then
+				${restart_command}
+			fi
+			EOF
+		fi
+		;;
+	*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst
new file mode 100644
index 00000000..76016c6c
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/man.rst
@@ -0,0 +1,55 @@
+cdist-type__postgres_conf(7)
+============================
+
+Configure a PostgreSQL server.
+
+NOTE: This type might need to be run multiple times to apply all bits of the
+configuration due to ordering requirements.
+
+SSRQ <cdist--@--ssrq-sds-fds.ch>
+
+
+DESCRIPTION
+-----------
+Configure a PostgreSQL server using ALTER SYSTEM.
+
+
+REQUIRED PARAMETERS
+-------------------
+value
+    The value to setup (can be omitted when state is set to "absent").
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    "present" or "absent". Defaults to "present".
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+	# set timezone
+    __postgres_conf timezone --value Europe/Zurich
+
+    # reset maximum number of concurrent connections to default (normally 100)
+    __postgres_conf max_connections --state absent
+
+
+SEE ALSO
+--------
+- `cdist-type(7) <cdist-type.html>`_
+
+
+COPYING
+-------
+Copyright \(C) 2020 SSRQ (www.ssrq-sds-fds.ch).
+Free use of this software is granted under the terms
+of the GNU General Public License version 3 (GPLv3).
diff --git a/cdist/conf/type/__postgres_conf/parameter/default/state b/cdist/conf/type/__postgres_conf/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__postgres_conf/parameter/optional b/cdist/conf/type/__postgres_conf/parameter/optional
new file mode 100644
index 00000000..d0460d86
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/parameter/optional
@@ -0,0 +1,2 @@
+state
+value

From 534d5f6bb5dd3d8e9cb2256e75ed182d6530a892 Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Fri, 4 Dec 2020 14:31:04 +0100
Subject: [PATCH 238/411] [type/__postgres_conf] Fix errors found by ShellCheck

---
 cdist/conf/type/__postgres_conf/gencode-remote | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index a09e1873..e25514d0 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -20,8 +20,8 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$(cat "${__global}/explorer/os")
-state_should=$(cat "${__object}/parameter/state")
+os=$(cat "${__global:?}/explorer/os")
+state_should=$(cat "${__object:?}/parameter/state")
 
 if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ]
 then
@@ -37,7 +37,7 @@ then
 fi
 
 # Parameters
-conf_name="${__object_id}"
+conf_name="${__object_id:?}"
 if [ -f "${__object}/parameter/value" ]
 then
 	conf_value=$(cat "${__object}/parameter/value")
@@ -46,7 +46,6 @@ fi
 if [ "${state_should}" = 'present' ]
 then
 	set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'"
-	check_command="SHOW ${conf_name}"
 else
 	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
 fi

From 50bcd951055d7599ec340b4dea80a3a5d11dd9b0 Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Thu, 10 Dec 2020 11:45:32 +0100
Subject: [PATCH 239/411] [type/__postgres_conf] Remove faulty quotes

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index e25514d0..8ccb3b42 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -45,7 +45,7 @@ fi
 
 if [ "${state_should}" = 'present' ]
 then
-	set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'"
+	set_command="ALTER SYSTEM SET ${conf_name} = ${conf_value}"
 else
 	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
 fi

From b4060720dc3c7cc58aa992285877e4c009caa6be Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Fri, 11 Dec 2020 11:12:16 +0100
Subject: [PATCH 240/411] [type/__postgres_conf] Fix psql options for ALTER
 command

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 8ccb3b42..7d86028e 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -67,7 +67,7 @@ in
 		# needs two separate psql commands because ALTER SYSTEM
 		# cannot run inside a transaction block
 		cat <<-EOF
-		su - ${postgres_user} -c "psql postgres -twAc \
+		su - ${postgres_user} -c "psql postgres -qwc \
 			\"${set_command}\""
 		su - ${postgres_user} -c "psql postgres -twAc \
 			\"SELECT pg_reload_conf()\""

From 1b49fec972ce5f486f2794571c8e892d1fa6cb51 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 19:15:49 +0100
Subject: [PATCH 241/411] [type/__postgres_conf] Refactor

---
 .../type/__postgres_conf/explorer/context     |  42 ++++++
 .../conf/type/__postgres_conf/explorer/state  |  60 ++++++++
 .../conf/type/__postgres_conf/gencode-remote  | 135 ++++++++++--------
 3 files changed, 180 insertions(+), 57 deletions(-)
 create mode 100644 cdist/conf/type/__postgres_conf/explorer/context
 create mode 100644 cdist/conf/type/__postgres_conf/explorer/state

diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context
new file mode 100644
index 00000000..3a2fa504
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/explorer/context
@@ -0,0 +1,42 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Returns the "context" of the configuration setting.
+# cf. also https://www.postgresql.org/docs/10/view-pg-settings.html
+
+os=$("${__explorer:?}/os")
+
+case ${os}
+in
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(devuan)
+		postgres_user='postgres'
+		;;
+	(*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
+
+conf_name=${__object_id:?}
+
+su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE name = '${conf_name}'\""
diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
new file mode 100644
index 00000000..da904b56
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -0,0 +1,60 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$("${__explorer:?}/os")
+
+case ${os}
+in
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(devuan)
+		postgres_user='postgres'
+		;;
+	(*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
+
+conf_name=${__object_id:?}
+
+if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
+	| cmp -s "${__object:?}/parameter/value" -
+then
+	echo present
+else
+	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT source FROM pg_settings WHERE name = '${conf_name}'\"")
+	in
+		('')
+			# invalid configuration parameter
+			# (error message was already printed by SHOW command above.
+			#  Yes, it's a hack)
+			exit 1
+			;;
+		(default)
+			echo absent
+			;;
+		(*)
+			echo different
+			;;
+	esac
+fi
diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 7d86028e..998b8582 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -1,7 +1,7 @@
 #!/bin/sh -e
 # -*- mode: sh; indent-tabs-mode: t -*-
 #
-# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+# 2019-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 # 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
@@ -21,71 +21,92 @@
 #
 
 os=$(cat "${__global:?}/explorer/os")
+state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
-if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ]
+conf_name=${__object_id:?}
+
+if test "${state_is}" = "${state_should}"
 then
-	echo "Invalid state '${state_should}'." \
-	     'Only "present" and "absent" are acceptable' >&2
-	exit 1
+	exit 0
 fi
 
-if [ "${state_should}" = 'present' ] && [ ! -f "${__object}/parameter/value" ]
-then
-	echo 'Missing required parameter "value"' >&2
-	exit 1
-fi
-
-# Parameters
-conf_name="${__object_id:?}"
-if [ -f "${__object}/parameter/value" ]
-then
-	conf_value=$(cat "${__object}/parameter/value")
-fi
-
-if [ "${state_should}" = 'present' ]
-then
-	set_command="ALTER SYSTEM SET ${conf_name} = ${conf_value}"
-else
-	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
-fi
-
-case $os
-in
-	openbsd|devuan)
-		case $os
-		in
-			openbsd)
-				postgres_user='_postgresql'
-				restart_command='/etc/rc.d/postgresql restart'
-				;;
-			devuan)
-				postgres_user='postgres'
-				restart_command='/etc/init.d/postgresql restart'
-				;;
-		esac
-		# needs two separate psql commands because ALTER SYSTEM
-		# cannot run inside a transaction block
-		cat <<-EOF
-		su - ${postgres_user} -c "psql postgres -qwc \
-			\"${set_command}\""
-		su - ${postgres_user} -c "psql postgres -twAc \
-			\"SELECT pg_reload_conf()\""
-		EOF
-		# check success (makes only sense if setting to a non-default value)
-		# and restart server if needed
-		if [ "${state_should}" = 'present' ]
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
 		then
-			cat <<-EOF
-			if [ \$(su - ${postgres_user} -c "psql postgres -twAc \"SHOW ${conf_name}\"") != '${conf_value}' ]
-			then
-				${restart_command}
-			fi
-			EOF
+			# needs quoting
+			set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'"
+		else
+			set -- "$@" "${_arg}"
 		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
+
+case ${os}
+in
+	(openbsd)
+		postgres_user='_postgresql'
+		restart_command='/etc/rc.d/postgresql restart'
 		;;
-	*)
+	(devuan)
+		postgres_user='postgres'
+		restart_command='/etc/init.d/postgresql restart'
+		;;
+	(*)
 		echo "Unsupported OS: ${os}" >&2
 		exit 1
 		;;
 esac
+
+
+psql_cmd() {
+	printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")"
+}
+
+case ${state_should}
+in
+	(present)
+		test -s "${__object:?}/parameter/value" || {
+			echo 'Missing required parameter --value' >&2
+			exit 1
+		}
+
+		cat <<-EOF
+		exec 3< "\${__object:?}/parameter/value"
+		$(psql_cmd postgres -tAw) <<'SQL'
+		\\set conf_value \`cat <&3\`
+		ALTER SYSTEM SET ${conf_name} = :'conf_value';
+		SELECT pg_reload_conf();
+		SQL
+		exec 3<&-
+		EOF
+		;;
+	(absent)
+		psql_cmd postgres -qwc "ALTER SYSTEM SET ${conf_name} TO DEFAULT"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		printf 'Only "present" and "absent" are acceptable.\n' >&2
+		exit 1
+		;;
+esac
+
+# check success (makes only sense if setting to a non-default value)
+# and restart server if needed
+if test "${state_should}" = 'present'
+then
+	cat <<-EOF
+
+	$(psql_cmd postgres -twAc "SHOW ${conf_name}") \\
+	| cmp -s "\${__object:?}/parameter/value" - || {
+		${restart_command}
+	}
+	EOF
+fi

From 803367b316f5c72c51153172a41b7a745f2d2d83 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 19:29:42 +0100
Subject: [PATCH 242/411] [type/__postgres_conf] Fix default detection when
 default is also set in config file

e.g. port is usually also set to the default value in postgresql.conf
---
 cdist/conf/type/__postgres_conf/explorer/state | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index da904b56..400b27e8 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -42,7 +42,7 @@ if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
 then
 	echo present
 else
-	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT source FROM pg_settings WHERE name = '${conf_name}'\"")
+	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE name = '${conf_name}'\"")
 	in
 		('')
 			# invalid configuration parameter

From 891c98567e1d94577dc4f20d3ec9f73f4927fd24 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 19:39:56 +0100
Subject: [PATCH 243/411] [type/__postgres_conf] Compare configuration
 parameter names case insensitively

---
 cdist/conf/type/__postgres_conf/explorer/context | 3 ++-
 cdist/conf/type/__postgres_conf/explorer/state   | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context
index 3a2fa504..def5f676 100644
--- a/cdist/conf/type/__postgres_conf/explorer/context
+++ b/cdist/conf/type/__postgres_conf/explorer/context
@@ -39,4 +39,5 @@ esac
 
 conf_name=${__object_id:?}
 
-su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE name = '${conf_name}'\""
+# NOTE: SHOW/SET are case-insentitive, so this command should also be.
+su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE lower(name) = lower('${conf_name}')\""
diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 400b27e8..a4930296 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -42,7 +42,8 @@ if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
 then
 	echo present
 else
-	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE name = '${conf_name}'\"")
+	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
+	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"")
 	in
 		('')
 			# invalid configuration parameter

From 5051d4f40b7097f58431ce334f9ddf5b171dbc7f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 16:36:44 +0100
Subject: [PATCH 244/411] [type/__postgres_conf] Catch invalid values

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 998b8582..cc9b4e99 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -80,7 +80,7 @@ in
 
 		cat <<-EOF
 		exec 3< "\${__object:?}/parameter/value"
-		$(psql_cmd postgres -tAw) <<'SQL'
+		$(psql_cmd postgres -tAw -v ON_ERROR_STOP=on) <<'SQL'
 		\\set conf_value \`cat <&3\`
 		ALTER SYSTEM SET ${conf_name} = :'conf_value';
 		SELECT pg_reload_conf();

From 0f2ff477381d67de27149a92744f873322e44fd7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 16:37:43 +0100
Subject: [PATCH 245/411] [type/__postgres_conf] Restart PostgreSQL server
 based on pending_restart column of pg_settings

---
 .../type/__postgres_conf/explorer/context     | 43 -------------------
 .../conf/type/__postgres_conf/gencode-remote  | 16 +++----
 2 files changed, 6 insertions(+), 53 deletions(-)
 delete mode 100644 cdist/conf/type/__postgres_conf/explorer/context

diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context
deleted file mode 100644
index def5f676..00000000
--- a/cdist/conf/type/__postgres_conf/explorer/context
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/sh -e
-# -*- mode: sh; indent-tabs-mode: t -*-
-#
-# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-# Returns the "context" of the configuration setting.
-# cf. also https://www.postgresql.org/docs/10/view-pg-settings.html
-
-os=$("${__explorer:?}/os")
-
-case ${os}
-in
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(devuan)
-		postgres_user='postgres'
-		;;
-	(*)
-		echo "Unsupported OS: ${os}" >&2
-		exit 1
-		;;
-esac
-
-conf_name=${__object_id:?}
-
-# NOTE: SHOW/SET are case-insentitive, so this command should also be.
-su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE lower(name) = lower('${conf_name}')\""
diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index cc9b4e99..fa930cc4 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -98,15 +98,11 @@ in
 		;;
 esac
 
-# check success (makes only sense if setting to a non-default value)
-# and restart server if needed
-if test "${state_should}" = 'present'
-then
-	cat <<-EOF
+# Restart PostgreSQL server if required to apply new configuration value
+cat <<EOF
 
-	$(psql_cmd postgres -twAc "SHOW ${conf_name}") \\
-	| cmp -s "\${__object:?}/parameter/value" - || {
-		${restart_command}
-	}
-	EOF
+if test 't' = "\$($(psql_cmd postgres -twAc "SELECT pending_restart FROM pg_settings WHERE lower(name) = lower('${conf_name}')"))"
+then
+	${restart_command}
 fi
+EOF

From 3f605c31ac7538d188a11b5d9d84cf2ed485bf23 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 17:18:27 +0100
Subject: [PATCH 246/411] [type/__postgres_conf] Add support for more init
 systems to restart service

---
 .../conf/type/__postgres_conf/gencode-remote  | 35 +++++++++++++++++--
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index fa930cc4..15d5bb3e 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -53,11 +53,9 @@ case ${os}
 in
 	(openbsd)
 		postgres_user='_postgresql'
-		restart_command='/etc/rc.d/postgresql restart'
 		;;
 	(devuan)
 		postgres_user='postgres'
-		restart_command='/etc/init.d/postgresql restart'
 		;;
 	(*)
 		echo "Unsupported OS: ${os}" >&2
@@ -103,6 +101,37 @@ cat <<EOF
 
 if test 't' = "\$($(psql_cmd postgres -twAc "SELECT pending_restart FROM pg_settings WHERE lower(name) = lower('${conf_name}')"))"
 then
-	${restart_command}
+	$(
+		init=$(cat "${__global:?}/explorer/init")
+		case ${init}
+		in
+			(systemd)
+				echo 'systemctl restart postgresql.service'
+				;;
+			(*openrc*)
+				echo 'rc-service postgresql restart'
+				;;
+			(sysvinit)
+				echo '/etc/init.d/postgresql restart'
+				;;
+			(init)
+				case $(cat "${__global:?}/explorer/kernel_name")
+				in
+					(FreeBSD)
+						echo 'service postgresql restart'
+						;;
+					(OpenBSD|NetBSD)
+						echo '/etc/rc.d/postgresql restart'
+						;;
+					(*)
+						echo "Unsupported operating system. Don't know how to restart services." >&2
+						exit 1
+				esac
+				;;
+			(*)
+				printf "Don't know how to restart services with your init (%s)\n" "${init}" >&2
+				exit 1
+		esac
+	)
 fi
 EOF

From 4967c7ebbb7e98af6aa2c7f8bc91511bcbbf9820 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 17:20:27 +0100
Subject: [PATCH 247/411] [type/__postgres_conf] Silence psql output

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 15d5bb3e..9b8227ee 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -78,7 +78,7 @@ in
 
 		cat <<-EOF
 		exec 3< "\${__object:?}/parameter/value"
-		$(psql_cmd postgres -tAw -v ON_ERROR_STOP=on) <<'SQL'
+		$(psql_cmd postgres -tAwq -o /dev/null -v ON_ERROR_STOP=on) <<'SQL'
 		\\set conf_value \`cat <&3\`
 		ALTER SYSTEM SET ${conf_name} = :'conf_value';
 		SELECT pg_reload_conf();

From f9ebb4333c85e41439635cb00a40c3e07d8a3683 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 17:21:50 +0100
Subject: [PATCH 248/411] [type/__postgres_conf] Add NetBSD PostgreSQL UNIX
 user

---
 cdist/conf/type/__postgres_conf/gencode-remote | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 9b8227ee..8f8a2bfe 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -51,15 +51,14 @@ quote() {
 
 case ${os}
 in
+	(netbsd)
+		postgres_user='pgsql'
+		;;
 	(openbsd)
 		postgres_user='_postgresql'
 		;;
-	(devuan)
-		postgres_user='postgres'
-		;;
 	(*)
-		echo "Unsupported OS: ${os}" >&2
-		exit 1
+		postgres_user='postgres'
 		;;
 esac
 

From 6b18cace759d1345cec6c2168c3bd8f238f23bca Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 21 Jan 2021 19:29:07 +0100
Subject: [PATCH 249/411] [type/__postgres_conf] Catch connection errors early

---
 cdist/conf/type/__postgres_conf/explorer/state | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index a4930296..589925de 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -37,6 +37,11 @@ esac
 
 conf_name=${__object_id:?}
 
+su - "${postgres_user}" -c 'psql postgres -c "SELECT 1"' >/dev/null || {
+	echo 'Connection to PostgreSQL server failed' >&2
+	exit 1
+}
+
 if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
 	| cmp -s "${__object:?}/parameter/value" -
 then

From 8eccacec59b0badd7a6e2a40a27f02f0ff355ccf Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 4 Feb 2021 19:09:26 +0100
Subject: [PATCH 250/411] __package_pip: add optional dependencies

This is a poor implementation of optional dependencies for pip packages.
It ensures to install them if the package will be installed, but does
not take into account if they must be added/removed after the package is
already installed. Also, it will not be autoremoved, as all dependencies
will not be removed.
---
 cdist/conf/type/__package_pip/gencode-remote        | 13 +++++++++++++
 cdist/conf/type/__package_pip/man.rst               | 13 +++++++++++++
 .../type/__package_pip/parameter/optional_multiple  |  1 +
 3 files changed, 27 insertions(+)
 create mode 100644 cdist/conf/type/__package_pip/parameter/optional_multiple

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index a1375c2d..58c89040 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -56,6 +56,19 @@ fi
 
 case "$state_should" in
     present)
+        if [ -f "$__object/parameter/extra" ]
+        then
+            while read extra
+            do
+                if [ "$extras" ]; then
+                    extras="$extras,$extra"
+                else
+                    extras="$extra"
+                fi
+            done < "$__object/parameter/extra"
+            name="${name}[${extras}]"
+        fi
+
         if [ "$runas" ]
         then
             echo "su -c '$pip install -q $name' $runas"
diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index 234ceee2..64ad0358 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -22,6 +22,11 @@ OPTIONAL PARAMETERS
 name
     If supplied, use the name and not the object id as the package name.
 
+extra
+    Extra optional dependencies which should be installed along the selected
+    package. Can be specified multiple times. Will only be applied if the
+    package actually will be installed, but will not explicitly checked.
+
 pip
     Instead of using pip from PATH, use the specific pip path.
 
@@ -46,6 +51,14 @@ EXAMPLES
     # Use pip in a virtualenv located at /foo/shinken_virtualenv as user foo
     __package_pip pyro --state present --pip /foo/shinken_virtualenv/bin/pip --runas foo
 
+    # Install package with optional dependencies
+    __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails
+    # or do a little cheating
+    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails
+
+    # or take all extras
+    __package_pip mautrix-telegram --extra all
+
 
 SEE ALSO
 --------
diff --git a/cdist/conf/type/__package_pip/parameter/optional_multiple b/cdist/conf/type/__package_pip/parameter/optional_multiple
new file mode 100644
index 00000000..0f228715
--- /dev/null
+++ b/cdist/conf/type/__package_pip/parameter/optional_multiple
@@ -0,0 +1 @@
+extra

From 73a03d75d7d23fea292b5fd60aeb059d8c72c4ba Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 4 Feb 2021 19:18:02 +0100
Subject: [PATCH 251/411] __package_pip: fix shellcheck

---
 cdist/conf/type/__package_pip/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 58c89040..925eaf24 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -58,7 +58,7 @@ case "$state_should" in
     present)
         if [ -f "$__object/parameter/extra" ]
         then
-            while read extra
+            while read -r extra
             do
                 if [ "$extras" ]; then
                     extras="$extras,$extra"

From cda17be38a1030742ed224b2536fca517bf8c09b Mon Sep 17 00:00:00 2001
From: ssrq <ungleich@ssrq-sds-fds.ch>
Date: Mon, 8 Feb 2021 08:27:03 +0100
Subject: [PATCH 252/411] [explorer/memory] Clean up, return kiB for all
 systems, add SunOS

BSDs were MiB before.
---
 cdist/conf/explorer/memory | 85 ++++++++++++++++++++++++++++++--------
 1 file changed, 68 insertions(+), 17 deletions(-)

diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory
index 5ea15ada..63aba9c6 100755
--- a/cdist/conf/explorer/memory
+++ b/cdist/conf/explorer/memory
@@ -1,8 +1,9 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2014 Daniel Heule  (hda at sfs.biz)
 # 2014 Thomas Oettli (otho at sfs.biz)
 # Copyright 2017, Philippe Gregoire <pg@pgregoire.xyz>
+# 2020 Dennis Camera <dennis.camera at ssrq-sds-fds.ch>
 #
 # This file is part of cdist.
 #
@@ -19,24 +20,74 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
+# Returns the amount of memory physically installed in the system, or if that
+# cannot be determined the amount available to the operating system kernel,
+# in kibibytes (kiB).
 
-# FIXME: other system types (not linux ...)
+str2bytes() {
+	awk -F' ' '
+	$2 ==   "B" || !$2 { print $1 }
+	$2 ==  "kB" { print $1 * 1000 }
+	$2 ==  "MB" { print $1 * 1000 * 1000 }
+	$2 ==  "GB" { print $1 * 1000 * 1000 * 1000 }
+	$2 ==  "TB" { print $1 * 1000 * 1000 * 1000 * 1000 }
+	$2 == "kiB" { print $1 * 1024 }
+	$2 == "MiB" { print $1 * 1024 * 1024 }
+	$2 == "GiB" { print $1 * 1024 * 1024 * 1024 }
+	$2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }'
+}
 
-os=$("$__explorer/os")
-case "$os" in
-    "macosx")
-        echo "$(sysctl -n hw.memsize)/1024" | bc
-    ;;
+bytes2kib() {
+	set -- "$(cat)"
+	test "$1" -gt 0 && echo $(($1 / 1024))
+}
 
-    *"bsd")
-        PATH=$(getconf PATH)
-        echo "$(sysctl -n hw.physmem) / 1048576" | bc
-    ;;
 
-    *)
-    if [ -r /proc/meminfo ]; then
-        grep "MemTotal:" /proc/meminfo | awk '{print $2}'
-    fi
-    ;;
+case $(uname -s)
+in
+	(Darwin)
+		sysctl -n hw.memsize | bytes2kib
+		;;
+	(FreeBSD)
+		sysctl -n hw.realmem | bytes2kib
+		;;
+	(NetBSD|OpenBSD)
+		# NOTE: This reports "usable" memory, not physically installed memory.
+		command -p sysctl -n hw.physmem | bytes2kib
+		;;
+	(SunOS)
+		# Make sure that awk from xpg4 is used for the scripts to work
+		export PATH="/usr/xpg4/bin:${PATH}"
+		prtconf \
+		| awk -F ': ' '
+		  $1 == "Memory size" { sub(/Megabytes/, "MiB", $2); print $2 }
+		  /^$/ { exit }' \
+		| str2bytes \
+		| bytes2kib
+		;;
+	(Linux)
+		if test -d /sys/devices/system/memory
+		then
+			# Use memory blocks if the architecture (e.g. x86, PPC64, s390)
+			# supports them (they denote physical memory)
+			num_mem_blocks=$(cat /sys/devices/system/memory/memory[0-9]*/state | grep -cxF online)
+			mem_block_size=$(cat /sys/devices/system/memory/block_size_bytes)
+
+			echo $((num_mem_blocks * 0x$mem_block_size)) | bytes2kib && exit
+		fi
+		if test -r /proc/meminfo
+		then
+			# Fall back to meminfo file on other architectures (e.g. ARM, MIPS,
+			# PowerPC)
+			# NOTE: This is "usable" memory, not physically installed memory.
+			awk -F ': +' '$1 == "MemTotal" { sub(/B$/, "iB", $2); print $2 }' /proc/meminfo \
+			| str2bytes \
+			| bytes2kib
+		fi
+		;;
+	(*)
+		printf "Your kernel (%s) is currently not supported by the memory explorer\n" "$(uname -s)" >&2
+		printf "Please contribute an implementation for it if you can.\n" >&2
+		exit 1
+		;;
 esac

From 65a6a2ed52c56f553d65ad0aa0aaba3ee4579e3c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 8 Feb 2021 08:28:31 +0100
Subject: [PATCH 253/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index b2b35616..52686617 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
+	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)

From bc145bbc27a45e382adb744e6774fb803f4fda0a Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 9 Feb 2021 19:58:47 +0100
Subject: [PATCH 254/411] [__letsencrypt_cert] Fix various issues with hooks.

Closes #853, see issue for full description / discussion.

Short summary:
- There was about 6.53% chances of `--renewal-hook` not being applied
- Using --automatic-renewal in one cert and not in another was an error.
- It was not possible to use different hooks for different certificates.
- FreeBSD support was utterly broken.
---
 cdist/conf/type/__letsencrypt_cert/man.rst    |  84 +++++++--
 cdist/conf/type/__letsencrypt_cert/manifest   | 171 ++++++++++++++----
 .../parameter/deprecated/automatic-renewal    |   2 +
 .../parameter/deprecated/renew-hook           |   2 +
 .../parameter/optional_multiple               |   3 +
 5 files changed, 220 insertions(+), 42 deletions(-)
 mode change 100755 => 100644 cdist/conf/type/__letsencrypt_cert/manifest
 create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal
 create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook

diff --git a/cdist/conf/type/__letsencrypt_cert/man.rst b/cdist/conf/type/__letsencrypt_cert/man.rst
index 85eb88ea..43be8424 100644
--- a/cdist/conf/type/__letsencrypt_cert/man.rst
+++ b/cdist/conf/type/__letsencrypt_cert/man.rst
@@ -1,16 +1,33 @@
 cdist-type__letsencrypt_cert(7)
 ===============================
 
+
 NAME
 ----
 
 cdist-type__letsencrypt_cert - Get an SSL certificate from Let's Encrypt
 
+
 DESCRIPTION
 -----------
 
 Automatically obtain a Let's Encrypt SSL certificate using Certbot.
 
+This type attempts to setup automatic renewals always. In many Linux
+distributions, that is the case out of the box, see:
+https://certbot.eff.org/docs/using.html#automated-renewals
+
+For Alpine Linux and Arch Linux, we setup a system-wide cronjob that
+attempts to renew certificates daily.
+
+If you are using FreeBSD, we configure periodic(8) as recommended by
+the port mantainer, so there will be a weekly attempt at renewal.
+
+If your OS is not mentioned here or on Certbot's docs as having
+support for automated renewals, please make sure you check your OS
+and possibly patch this type so the system-wide cronjob is installed.
+
+
 REQUIRED PARAMETERS
 -------------------
 
@@ -21,6 +38,7 @@ object id
 admin-email
     Where to send Let's Encrypt emails like "certificate needs renewal".
 
+
 OPTIONAL PARAMETERS
 -------------------
 
@@ -36,25 +54,68 @@ webroot
     The path to your webroot, as set up in your webserver config. If this
     parameter is not present, Certbot will be run in standalone mode.
 
+
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
 
-renew-hook
-    Renew hook command directly passed to Certbot in cron job.
-
 domain
     Domains to be included in the certificate. When specified then object id
     is not used as a domain.
 
+deploy-hook
+    Command to be executed only when the certificate associated with this
+    ``$__object_id`` is issued or renewed.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    For this command, the
+    shell variable ``$RENEWED_LINEAGE`` will point to the
+    config live subdirectory (for example,
+    ``/etc/letsencrypt/live/${__object_id}``) containing the
+    new certificates and keys; the shell variable
+    ``$RENEWED_DOMAINS`` will contain a space-delimited list
+    of renewed certificate domains (for example,
+    ``example.com www.example.com``)
+
+pre-hook
+    Command to be run in a shell before obtaining any
+    certificates.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    Note these run regardless of which certificate is attempted, you may want to
+    manage these system-wide hooks with ``__file`` in
+    ``/etc/letsencrypt/renewal-hooks/pre/``.
+
+    Intended primarily for renewal, where it
+    can be used to temporarily shut down a webserver that
+    might conflict with the standalone plugin. This will
+    only be called if a certificate is actually to be
+    obtained/renewed.
+
+post-hook
+    Command to be run in a shell after attempting to
+    obtain/renew certificates.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    Note these run regardless of which certificate was attempted, you may want to
+    manage these system-wide hooks with ``__file`` in
+    ``/etc/letsencrypt/renewal-hooks/post/``.
+
+    Can be used to deploy
+    renewed certificates, or to restart any servers that
+    were stopped by --pre-hook. This is only run if an
+    attempt was made to obtain/renew a certificate.
+
+
 BOOLEAN PARAMETERS
 ------------------
 
-automatic-renewal
-    Install a cron job, which attempts to renew certificates daily.
-
 staging
     Obtain a test certificate from a staging server.
 
+
 MESSAGES
 --------
 
@@ -67,6 +128,7 @@ create
 remove
     Certificate was removed.
 
+
 EXAMPLES
 --------
 
@@ -75,8 +137,7 @@ EXAMPLES
     # use object id as domain
     __letsencrypt_cert example.com \
         --admin-email root@example.com \
-        --automatic-renewal \
-        --renew-hook "service nginx reload" \
+        --deploy-hook "service nginx reload" \
         --webroot /data/letsencrypt/root
 
 .. code-block:: sh
@@ -85,11 +146,10 @@ EXAMPLES
     # and example.com needs to be included again with domain parameter
     __letsencrypt_cert example.com \
         --admin-email root@example.com \
-        --automatic-renewal \
         --domain example.com \
         --domain foo.example.com \
         --domain bar.example.com \
-        --renew-hook "service nginx reload" \
+        --deploy-hook "service nginx reload" \
         --webroot /data/letsencrypt/root
 
 AUTHORS
@@ -99,11 +159,13 @@ AUTHORS
 | Kamila Součková <kamila--@--ksp.sk>
 | Darko Poljak <darko.poljak--@--gmail.com>
 | Ľubomír Kučera <lubomir.kucera.jr at gmail.com>
+| Evilham <contact@evilham.com>
+
 
 COPYING
 -------
 
-Copyright \(C) 2017-2018 Nico Schottelius, Kamila Součková, Darko Poljak and
+Copyright \(C) 2017-2021 Nico Schottelius, Kamila Součková, Darko Poljak and
 Ľubomír Kučera. You can redistribute it and/or modify it under the terms of
 the GNU General Public License as published by the Free Software Foundation,
 either version 3 of the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
old mode 100755
new mode 100644
index b4464366..9c8cc043
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -1,18 +1,20 @@
 #!/bin/sh
 
 certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")"
+state=$(cat "${__object}/parameter/state")
+os="$(cat "${__global:?}/explorer/os")"
 
 if [ -z "${certbot_fullpath}" ]; then
-	os="$(cat "${__global:?}/explorer/os")"
 	os_version="$(cat "${__global}/explorer/os_version")"
-
+	# Use this, very common value, as a default. It is OS-dependent
+	certbot_fullpath="/usr/bin/certbot"
 	case "$os" in
-        archlinux)
-            __package certbot
-            ;;
-        alpine)
-            __package certbot
-            ;;
+		archlinux)
+			__package certbot
+		;;
+		alpine)
+			__package certbot
+		;;
 		debian)
 			case "$os_version" in
 				8*)
@@ -48,9 +50,7 @@ if [ -z "${certbot_fullpath}" ]; then
 					exit 1
 				;;
 			esac
-
-			certbot_fullpath=/usr/bin/certbot
-			;;
+		;;
 		devuan)
 			case "$os_version" in
 				jessie)
@@ -83,17 +83,14 @@ if [ -z "${certbot_fullpath}" ]; then
 					exit 1
 				;;
 			esac
-
-			certbot_fullpath=/usr/bin/certbot
 		;;
 		freebsd)
-			__package py27-certbot
-
-			certbot_fullpath=/usr/local/bin/certbot
+			__package py37-certbot
+			certbot_fullpath="/usr/local/bin/certbot"
 		;;
 		ubuntu)
-            __package certbot
-            ;;
+			__package certbot
+		;;
 		*)
 			echo "Unsupported os: $os" >&2
 			exit 1
@@ -101,18 +98,130 @@ if [ -z "${certbot_fullpath}" ]; then
 	esac
 fi
 
-if [ -f "${__object}/parameter/automatic-renewal" ]; then
-	renew_hook_param="${__object}/parameter/renew-hook"
-	renew_hook=""
-	if [ -f "${renew_hook_param}" ]; then
-		while read -r hook; do
-			renew_hook="${renew_hook} --renew-hook \"${hook}\""
-		done < "${renew_hook_param}"
-	fi
+# Other OS-dependent values that we want to set every time
+LE_DIR="/etc/letsencrypt"
+certbot_cronjob_state="absent"
+case "$os" in
+	archlinux|alpine)
+		certbot_cronjob_state="present"
+	;;
+	freebsd)
+		LE_DIR="/usr/local/etc/letsencrypt"
+		# FreeBSD uses periodic(8) instead of crontabs for this
+		__line "periodic.conf_weekly_certbot" \
+			--file "/etc/periodic.conf" \
+			--regex "^(#[[:space:]]*)?weekly_certbot_enable=.*" \
+			--state "replace" \
+			--line 'weekly_certbot_enable="YES"'
+	;;
+	*)
+	;;
+esac
 
-	__cron letsencrypt-certbot  \
-		--user root \
-		--command "${certbot_fullpath} renew -q ${renew_hook}" \
-		--hour 0 \
-		--minute 47
+# This is only necessary in certain OS
+__cron letsencrypt-certbot \
+	--user root \
+	--command "${certbot_fullpath} renew -q" \
+	--hour 0 \
+	--minute 47 \
+	--state "${certbot_cronjob_state}"
+
+# Ensure hook directories
+HOOKS_DIR="${LE_DIR}/renewal-hooks"
+__directory "${LE_DIR}" --mode 0755
+require="__directory/${LE_DIR}" __directory "${HOOKS_DIR}" --mode 0755
+
+if [ -f "${__object}/parameter/domain" ]; then
+	domains="$(sort "${__object}/parameter/domain")"
+else
+	domains="${__object_id}"
 fi
+
+# Install hooks as needed
+for hook in deploy pre post; do
+	# Using something unique and specific to this object
+	hook_file="${HOOKS_DIR}/${hook}/${__object_id}.cdist.sh"
+	# Reasonable defaults
+	hook_source="${__object}/parameter/${hook}-hook"
+	hook_state="absent"
+	hook_contents_head="#!/bin/sh -e"
+	hook_contents_logic=""
+	hook_contents_tail=""
+
+	# Backwards compatibility
+	# Remove this when renew-hook is removed
+	# Falling back to renew-hook if deploy-hook is not passed
+	if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then
+		hook_source="${__object}/parameter/renew-hook"
+	fi
+	if [ "${state}" = "present" ] && \
+		[ -f "${hook_source}" ]; then
+		# This hook is to be installed, let's generate it with some
+		# safety boilerplate
+		# Since certbot runs all hooks for all renewal processes
+		# (at each state for deploy, pre, post), it is up to us to
+		# differentiate whether or not the hook must run
+		hook_state="present"
+		hook_contents_head="$(cat <<EOF
+#!/bin/sh -e
+#
+# Managed remotely with https://cdi.st
+#
+# Domains for which this hook is supposed to apply
+lineage="${LE_DIR}/live/${__object_id}"
+domains="\$(cat <<eof
+${domains}
+eof
+)"
+EOF
+)"
+		case "${hook}" in
+			pre|post)
+				# Certbot is kind of terrible, we have
+				# no way of knowing what domain/lineage the
+				# hook is running for
+				hook_contents_logic="$(cat <<EOF
+# pre/post-hooks apply always due to a certbot limitation
+APPLY_HOOK="YES"
+EOF
+)"
+			;;
+			deploy)
+				hook_contents_logic="$(cat <<EOF
+# certbot defines these:
+# RENEWED_DOMAINS: DOMAIN1,DOMAIN2
+# RENEWED_LINEAGE: /etc/letsencrypt/live/__object_id
+# It feels more stable to use RENEWED_LINEAGE
+if [ "\${lineage}" = "\${RENEWED_LINEAGE}" ]; then
+	APPLY_HOOK="YES"
+fi
+EOF
+)"
+			;;
+			*)
+				echo "Unknown hook '${hook}'" >> /dev/stderr
+				exit 1
+			;;
+		esac
+
+		hook_contents_tail="$(cat <<EOF
+if [ -n "\${APPLY_HOOK}" ]; then
+$(sed -e 's/^/	//' "${hook_source}")
+fi
+EOF
+)"
+	fi
+	# Ensure hook directory exists
+	require="__directory/${HOOKS_DIR}" __directory "${HOOKS_DIR}/${hook}" \
+		--mode 0755
+	require="__directory/${HOOKS_DIR}/${hook}" __file "${hook_file}" \
+		--mode 0555 \
+		--source '-' \
+		--state "${hook_state}" <<EOF
+${hook_contents_head}
+
+${hook_contents_logic}
+
+${hook_contents_tail}
+EOF
+done
diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal
new file mode 100644
index 00000000..dbfc5fa7
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal
@@ -0,0 +1,2 @@
+Deprecated in favour of consistent behaviour. It has no effect, see:
+https://code.ungleich.ch/ungleich-public/cdist/-/issues/853
diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook
new file mode 100644
index 00000000..be4a3460
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook
@@ -0,0 +1,2 @@
+This parameter has been deprecated in favour of --deploy-hook.
+See: https://code.ungleich.ch/ungleich-public/cdist/-/issues/853
diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple b/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple
index 0e866d45..bebb19bd 100644
--- a/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple
+++ b/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple
@@ -1,2 +1,5 @@
+deploy-hook
 domain
+post-hook
+pre-hook
 renew-hook

From e49da474c4e92775477c132833fb8c517db6921a Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 9 Feb 2021 20:29:17 +0100
Subject: [PATCH 255/411] [__letsencrypt_cert] Remove problematic trailing
 slash in sed.

Happy fingers are happy and like adding slashes places.
---
 cdist/conf/type/__letsencrypt_cert/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 9c8cc043..04edea1e 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -206,7 +206,7 @@ EOF
 
 		hook_contents_tail="$(cat <<EOF
 if [ -n "\${APPLY_HOOK}" ]; then
-$(sed -e 's/^/	//' "${hook_source}")
+$(sed -e 's/^/	/' "${hook_source}")
 fi
 EOF
 )"

From b832af5e3bce7d631dc1aafd6c4d9c2786391684 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 9 Feb 2021 20:53:58 +0100
Subject: [PATCH 256/411] [__letsencrypt_cert] Don't mess with user script
 indentation

This could break in odd ways if they passed sth like:
cat <<eof
bla bla
eof
---
 cdist/conf/type/__letsencrypt_cert/manifest | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 04edea1e..84221a80 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -206,7 +206,8 @@ EOF
 
 		hook_contents_tail="$(cat <<EOF
 if [ -n "\${APPLY_HOOK}" ]; then
-$(sed -e 's/^/	/' "${hook_source}")
+# Messing with indentation can eff up the users' scripts, let's not
+$(cat "${hook_source}")
 fi
 EOF
 )"

From aa80c09c803e132c3a05a642e9ada82ccb0e7dd8 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Wed, 10 Feb 2021 10:10:21 +0100
Subject: [PATCH 257/411] [__letsencrypt_cert] Move hook contents generation
 out of manifest

While there address some minor issues in the comments in the hook contents.
---
 .../type/__letsencrypt_cert/files/gen_hook.sh | 84 +++++++++++++++++++
 cdist/conf/type/__letsencrypt_cert/manifest   | 78 +----------------
 2 files changed, 88 insertions(+), 74 deletions(-)
 create mode 100644 cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh

diff --git a/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh b/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh
new file mode 100644
index 00000000..81ea4856
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh
@@ -0,0 +1,84 @@
+#!/bin/sh -e
+
+# It is expected that this defines hook_contents
+
+# Reasonable defaults
+hook_source="${__object}/parameter/${hook}-hook"
+hook_state="absent"
+hook_contents_head="#!/bin/sh -e"
+hook_contents_logic=""
+hook_contents_tail=""
+
+# Backwards compatibility
+# Remove this when renew-hook is removed
+# Falling back to renew-hook if deploy-hook is not passed
+if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then
+	hook_source="${__object}/parameter/renew-hook"
+fi
+if [ "${state}" = "present" ] && \
+	[ -f "${hook_source}" ]; then
+	# This hook is to be installed, let's generate it with some
+	# safety boilerplate
+	# Since certbot runs all hooks for all renewal processes
+	# (at each state for deploy, pre, post), it is up to us to
+	# differentiate whether or not the hook must run
+	hook_state="present"
+	hook_contents_head="$(cat <<EOF
+#!/bin/sh -e
+#
+# Managed remotely with https://cdi.st
+#
+# Domains for which this hook is supposed to apply
+lineage="${LE_DIR}/live/${__object_id}"
+domains="\$(cat <<eof
+${domains}
+eof
+)"
+EOF
+)"
+	case "${hook}" in
+		pre|post)
+			# Certbot is kind of terrible, we have
+			# no way of knowing what domain/lineage the
+			# hook is running for
+			hook_contents_logic="$(cat <<EOF
+# pre/post-hooks apply always due to a certbot limitation
+APPLY_HOOK="YES"
+EOF
+)"
+		;;
+		deploy)
+			hook_contents_logic="$(cat <<EOF
+# certbot defines these environment variables:
+# RENEWED_DOMAINS="DOMAIN1 DOMAIN2"
+# RENEWED_LINEAGE="/etc/letsencrypt/live/__object_id"
+# It feels more stable to use RENEWED_LINEAGE
+if [ "\${lineage}" = "\${RENEWED_LINEAGE}" ]; then
+	APPLY_HOOK="YES"
+fi
+EOF
+)"
+		;;
+		*)
+			echo "Unknown hook '${hook}'" >> /dev/stderr
+			exit 1
+		;;
+	esac
+
+	hook_contents_tail="$(cat <<EOF
+if [ -n "\${APPLY_HOOK}" ]; then
+	# Messing with indentation can eff up the users' scripts, let's not
+$(cat "${hook_source}")
+fi
+EOF
+)"
+fi
+
+hook_contents="$(cat <<EOF
+${hook_contents_head}
+
+${hook_contents_logic}
+
+${hook_contents_tail}
+EOF
+)"
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 84221a80..1df3574a 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -141,77 +141,11 @@ fi
 for hook in deploy pre post; do
 	# Using something unique and specific to this object
 	hook_file="${HOOKS_DIR}/${hook}/${__object_id}.cdist.sh"
-	# Reasonable defaults
-	hook_source="${__object}/parameter/${hook}-hook"
-	hook_state="absent"
-	hook_contents_head="#!/bin/sh -e"
-	hook_contents_logic=""
-	hook_contents_tail=""
 
-	# Backwards compatibility
-	# Remove this when renew-hook is removed
-	# Falling back to renew-hook if deploy-hook is not passed
-	if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then
-		hook_source="${__object}/parameter/renew-hook"
-	fi
-	if [ "${state}" = "present" ] && \
-		[ -f "${hook_source}" ]; then
-		# This hook is to be installed, let's generate it with some
-		# safety boilerplate
-		# Since certbot runs all hooks for all renewal processes
-		# (at each state for deploy, pre, post), it is up to us to
-		# differentiate whether or not the hook must run
-		hook_state="present"
-		hook_contents_head="$(cat <<EOF
-#!/bin/sh -e
-#
-# Managed remotely with https://cdi.st
-#
-# Domains for which this hook is supposed to apply
-lineage="${LE_DIR}/live/${__object_id}"
-domains="\$(cat <<eof
-${domains}
-eof
-)"
-EOF
-)"
-		case "${hook}" in
-			pre|post)
-				# Certbot is kind of terrible, we have
-				# no way of knowing what domain/lineage the
-				# hook is running for
-				hook_contents_logic="$(cat <<EOF
-# pre/post-hooks apply always due to a certbot limitation
-APPLY_HOOK="YES"
-EOF
-)"
-			;;
-			deploy)
-				hook_contents_logic="$(cat <<EOF
-# certbot defines these:
-# RENEWED_DOMAINS: DOMAIN1,DOMAIN2
-# RENEWED_LINEAGE: /etc/letsencrypt/live/__object_id
-# It feels more stable to use RENEWED_LINEAGE
-if [ "\${lineage}" = "\${RENEWED_LINEAGE}" ]; then
-	APPLY_HOOK="YES"
-fi
-EOF
-)"
-			;;
-			*)
-				echo "Unknown hook '${hook}'" >> /dev/stderr
-				exit 1
-			;;
-		esac
+	# This defines hook_contents
+	# shellcheck source=cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh
+	. "${__type}/files/gen_hook.sh"
 
-		hook_contents_tail="$(cat <<EOF
-if [ -n "\${APPLY_HOOK}" ]; then
-# Messing with indentation can eff up the users' scripts, let's not
-$(cat "${hook_source}")
-fi
-EOF
-)"
-	fi
 	# Ensure hook directory exists
 	require="__directory/${HOOKS_DIR}" __directory "${HOOKS_DIR}/${hook}" \
 		--mode 0755
@@ -219,10 +153,6 @@ EOF
 		--mode 0555 \
 		--source '-' \
 		--state "${hook_state}" <<EOF
-${hook_contents_head}
-
-${hook_contents_logic}
-
-${hook_contents_tail}
+${hook_contents}
 EOF
 done

From 4717e5ceff078111748824e4fb365df7979e5fe7 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 10:31:07 +0100
Subject: [PATCH 258/411] __package_pip: add extras explorer

The two new explorers detect all installed extras for this package.
---
 .../type/__package_pip/explorer/distinfo-dir  | 45 +++++++++++++++++++
 cdist/conf/type/__package_pip/explorer/extras | 26 +++++++++++
 cdist/conf/type/__package_pip/explorer/state  |  0
 3 files changed, 71 insertions(+)
 create mode 100755 cdist/conf/type/__package_pip/explorer/distinfo-dir
 create mode 100755 cdist/conf/type/__package_pip/explorer/extras
 mode change 100644 => 100755 cdist/conf/type/__package_pip/explorer/state

diff --git a/cdist/conf/type/__package_pip/explorer/distinfo-dir b/cdist/conf/type/__package_pip/explorer/distinfo-dir
new file mode 100755
index 00000000..18e169ae
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/distinfo-dir
@@ -0,0 +1,45 @@
+#!/bin/sh
+#
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+nameparam="$__object/parameter/name"
+if [ -f "$nameparam" ]; then
+    name=$(cat "$nameparam")
+else
+    name="$__object_id"
+fi
+
+pipparam="$__object/parameter/pip"
+if [ -f "$pipparam" ]; then
+    pip=$(cat "$pipparam")
+else
+    pip="$( "$__type_explorer/pip" )"
+fi
+
+
+if command -v "$pip" >/dev/null 2>&1; then
+    # assemble the path where pip stores all pip package info
+    "$pip" show "$name" \
+        | awk -F': ' '
+            $1 == "Name" {name=$2; gsub(/-/,"_",name); next}
+            $1 == "Version" {version=$2; next}
+            $1 == "Location" {location=$2; next}
+            END {if (version != "") printf "%s/%s-%s.dist-info", location, name, version}'
+fi
diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
new file mode 100755
index 00000000..92fc4732
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -0,0 +1,26 @@
+#!/bin/sh
+#
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+distinfo_dir="$("$__type_explorer/distinfo-dir")"
+if [ "$distinfo_dir" ]; then
+    # output all extras that are installed
+    awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA"
+fi
diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state
old mode 100644
new mode 100755

From 8dc6ab97384f2240e9218fbd4463ba12b004f44c Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 13:49:53 +0100
Subject: [PATCH 259/411] __package_pip: install not found extras

Compares the explorer against the parameters and install those extras
that are not already installed.
---
 cdist/conf/type/__package_pip/gencode-remote | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 925eaf24..ad327e37 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -2,6 +2,7 @@
 #
 # 2012 Nico Schottelius (nico-cdist at schottelius.org)
 # 2016 Darko Poljak (darko.poljak at gmail.com)
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
 #
 # This file is part of cdist.
 #
@@ -58,14 +59,17 @@ case "$state_should" in
     present)
         if [ -f "$__object/parameter/extra" ]
         then
-            while read -r extra
-            do
-                if [ "$extras" ]; then
-                    extras="$extras,$extra"
-                else
-                    extras="$extra"
-                fi
-            done < "$__object/parameter/extra"
+            mkdir "$__object/files"
+            # sort and generalize all extras
+            sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params"
+            sort "$__object/explorer/extras" > "$__object/files/extras-explorer"
+
+            # assemble extras
+            # all extras are passed to pip in a comma-separated list
+            extras="$(comm -23 \
+                "$__object/files/extras-params" "$__object/files/extras-explorer" \
+                | sed ':a; $!N; s/\n/,/; ta'  # loops through all input lines and add commas between them
+            )"
             name="${name}[${extras}]"
         fi
 

From 2db0ef7c98226661e5e4b55c89180b31ba65b8bc Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 22:53:26 +0100
Subject: [PATCH 260/411] __package_pip: updating real detection of extras

As the previous detection took the wrong values, this explorer now
checks if packages for an extra are installed or not. If not, the extra
is not installed.

Based on the information of the explorer, it will install the package
again with the absent extras.
---
 cdist/conf/type/__package_pip/explorer/extras | 37 +++++++++++++++++--
 cdist/conf/type/__package_pip/gencode-remote  | 21 ++++-------
 2 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 92fc4732..2f5fcca6 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -17,10 +17,41 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+#
+# Checks if the given extras are really installed or not. It will be
+# done by querring all dependencies for that extra and return it as
+# "to be installed" if no dependency was found.
+#
 
 
 distinfo_dir="$("$__type_explorer/distinfo-dir")"
-if [ "$distinfo_dir" ]; then
-    # output all extras that are installed
-    awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA"
+
+# check if we have something to check
+if [ "$distinfo_dir" ] && [ -s "$__object/parameter/extra" ]
+then
+    # save cause freezing is slow
+    mkdir "$__object/files"
+    pip_freeze="$__object/files/pip-freeze.tmp"
+    pip3 freeze > "$pip_freeze"
+
+    for extra in $(cat "$__object/parameter/extra" | tr ',' '\n')
+    do
+        # create a grep BRE pattern to search all packages
+        grep_pattern="$(
+            awk -F'(: | ; )' -v check="$extra" '
+                $1 == "Requires-Dist" {
+                    split($2, r, " ");
+                    sub("extra == ", "", $3); gsub("'"'"'", "", $3);
+                    if($3 == check) print r[1]
+                }' "$distinfo_dir/METADATA" \
+                | sed ':a; $!N; s/\n/\\|/; ta'
+        )"
+
+        # echo the extra if no packages where found for it
+        # if there is no pattern, we don't need to search ;-)
+        if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze"
+        then
+            echo "$extra"
+        fi
+    done
 fi
diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index ad327e37..9abe28bf 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -26,7 +26,10 @@
 state_is=$(cat "$__object/explorer/state")
 state_should="$(cat "$__object/parameter/state")"
 
-[ "$state_is" = "$state_should" ] && exit 0
+# short circuit if state is the same and no extras to install
+[ "$state_is" = "$state_should" ] && ! [ -s "$__object/explorer/extras" ] \
+    && exit 0
+
 
 nameparam="$__object/parameter/name"
 if [ -f "$nameparam" ]; then
@@ -57,19 +60,11 @@ fi
 
 case "$state_should" in
     present)
-        if [ -f "$__object/parameter/extra" ]
+        if [ -s "$__object/explorer/extras" ]
         then
-            mkdir "$__object/files"
-            # sort and generalize all extras
-            sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params"
-            sort "$__object/explorer/extras" > "$__object/files/extras-explorer"
-
-            # assemble extras
-            # all extras are passed to pip in a comma-separated list
-            extras="$(comm -23 \
-                "$__object/files/extras-params" "$__object/files/extras-explorer" \
-                | sed ':a; $!N; s/\n/,/; ta'  # loops through all input lines and add commas between them
-            )"
+            # all extras are passed to pip in a comma-separated list in the name
+            # sed loops through all input lines and add commas between them
+            extras="$(sed ':a; $!N; s/\n/,/; ta' "$__object/explorer/extras")"
             name="${name}[${extras}]"
         fi
 

From 7398382890a4368e588e700183e6c777e90fb069 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 23:12:10 +0100
Subject: [PATCH 261/411] __package_pip: fix shellcheck

Useless `cat $file`, use `< $file` instead.
---
 cdist/conf/type/__package_pip/explorer/extras | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 2f5fcca6..d57c69db 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -34,7 +34,7 @@ then
     pip_freeze="$__object/files/pip-freeze.tmp"
     pip3 freeze > "$pip_freeze"
 
-    for extra in $(cat "$__object/parameter/extra" | tr ',' '\n')
+    for extra in $(tr ',' '\n' < "$__object/parameter/extra")
     do
         # create a grep BRE pattern to search all packages
         grep_pattern="$(

From a9d7dfb2ed40d0a61978cc706ff469c354a6c8f5 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 12 Feb 2021 09:17:02 +0100
Subject: [PATCH 262/411] __package_pip: split extra 'all' to a list of all
 extras

This will fix if a package will be upgraded from some extras to all
extras. Previously, it will not work because some dependencies of 'all'
are already installed, so the feature 'all' is already installed.

Now, it will use a list of all extras to iterate over them separatly. This
will result it will never install all extras via `[all]`, but rather
`[foo,bar]`.
---
 cdist/conf/type/__package_pip/explorer/extras | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index d57c69db..be945e36 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -34,7 +34,14 @@ then
     pip_freeze="$__object/files/pip-freeze.tmp"
     pip3 freeze > "$pip_freeze"
 
-    for extra in $(tr ',' '\n' < "$__object/parameter/extra")
+    # If all is set, it searches all available extras to separatly check them.
+    # It would work with just 'all' (cause dependencies are given), but will
+    # not update if one extra is already present. Side effect is that it will
+    # not use [all] but instead name all extras seperatly.
+    for extra in $(if grep -qFx all "$__object/parameter/extra";
+        then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA";
+        else tr ',' '\n' < "$__object/parameter/extra";
+        fi)
     do
         # create a grep BRE pattern to search all packages
         grep_pattern="$(

From 951712740f95f8f1277f1295bcd97c0b9bafdc94 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 12 Feb 2021 13:42:51 +0100
Subject: [PATCH 263/411] __package_pip: update man.rst

Adjusted comments for `explorer/extras` and updated the man page for the
new behaviour of updating the extras.
---
 cdist/conf/type/__package_pip/explorer/extras |  7 +++---
 cdist/conf/type/__package_pip/man.rst         | 25 ++++++++++++-------
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index be945e36..8104ed47 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -35,15 +35,16 @@ then
     pip3 freeze > "$pip_freeze"
 
     # If all is set, it searches all available extras to separatly check them.
-    # It would work with just 'all' (cause dependencies are given), but will
-    # not update if one extra is already present. Side effect is that it will
-    # not use [all] but instead name all extras seperatly.
+    # It would work with just 'all' (cause dependencies are specified for
+    # 'all'), but will not update if one extra is already present. Side effect
+    # is that it will not use [all] but instead name all extras seperatly.
     for extra in $(if grep -qFx all "$__object/parameter/extra";
         then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA";
         else tr ',' '\n' < "$__object/parameter/extra";
         fi)
     do
         # create a grep BRE pattern to search all packages
+        # maybe a file full of patterns for -F could be written
         grep_pattern="$(
             awk -F'(: | ; )' -v check="$extra" '
                 $1 == "Requires-Dist" {
diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index 64ad0358..c013695c 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -24,8 +24,14 @@ name
 
 extra
     Extra optional dependencies which should be installed along the selected
-    package. Can be specified multiple times. Will only be applied if the
-    package actually will be installed, but will not explicitly checked.
+    package. Can be specified multiple times. Multiple extra optional
+    dependencies can also be specified in a comma-separated list to provide
+    a more pip-natvie style.
+
+    Extra optional dependencies will be installed even when the base package
+    is already installed. Notice that the type will not remove installed extras
+    that are not explicitly named for the type because pip does not offer a
+    management for orphaned packages and they may be used by other packages.
 
 pip
     Instead of using pip from PATH, use the specific pip path.
@@ -53,8 +59,8 @@ EXAMPLES
 
     # Install package with optional dependencies
     __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails
-    # or do a little cheating
-    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails
+    # the extras can also be specified comma-separated
+    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails --extra postgres
 
     # or take all extras
     __package_pip mautrix-telegram --extra all
@@ -67,12 +73,13 @@ SEE ALSO
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2012 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2012 Nico Schottelius, 2021 Matthias Stecher. You can
+redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either
+version 3 of the License, or (at your option) any later version.

From 2ce1fce7677b64a8d17a86f29f676b1ba8475db2 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 15 Feb 2021 16:17:46 +0100
Subject: [PATCH 264/411] __package_pip: match package names case insensitive

Pip matches them insensitive, so we need to do the same to avoid
problems by saying extras are not installed but already is there in
place.
---
 cdist/conf/type/__package_pip/explorer/extras | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 8104ed47..bbdc17ab 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -57,7 +57,8 @@ then
 
         # echo the extra if no packages where found for it
         # if there is no pattern, we don't need to search ;-)
-        if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze"
+        # pip matches packages case-insensetive, we need to do that, too
+        if [ "$grep_pattern" ] && ! grep -qi "$grep_pattern" "$pip_freeze"
         then
             echo "$extra"
         fi

From 0835f414a5b2ce41309ee9b265cb80abb1362563 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 16 Feb 2021 16:03:23 +0100
Subject: [PATCH 265/411] [type/__postgres_conf] Extract PostgreSQL service
 user detection to separate explorer

---
 .../__postgres_conf/explorer/postgres_user    | 64 +++++++++++++++++++
 .../conf/type/__postgres_conf/explorer/state  | 16 +----
 .../conf/type/__postgres_conf/gencode-remote  | 17 +----
 3 files changed, 67 insertions(+), 30 deletions(-)
 create mode 100644 cdist/conf/type/__postgres_conf/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_conf/explorer/postgres_user b/cdist/conf/type/__postgres_conf/explorer/postgres_user
new file mode 100644
index 00000000..c6582dc4
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/explorer/postgres_user
@@ -0,0 +1,64 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$("${__explorer:?}/os")
+
+case ${os}
+in
+	(alpine)
+		echo 'postgres'
+		;;
+	(centos|rhel|scientific)
+		echo 'postgres'
+		;;
+	(debian|devuan|ubuntu)
+		echo 'postgres'
+		;;
+	(freebsd)
+		test -x /usr/local/etc/rc.d/postgresql || {
+			printf 'could not find postgresql rc script./n' >&2
+			exit 1
+		}
+		pg_status=$(/usr/local/etc/rc.d/postgresql onestatus) || {
+			printf 'postgresql daemon is not running.\n' >&2
+			exit 1
+		}
+		pg_pid=$(printf '%s\n' "${pg_status}" \
+			| sed -n 's/^pg_ctl:.*(PID: *\([0-9]*\))$/\1/p')
+
+		# PostgreSQL < 9.6: pgsql
+		# PostgreSQL >= 9.6: postgres
+		ps -o user -p "${pg_pid}" | sed -n '2p'
+		;;
+	(netbsd)
+		echo 'pgsql'
+		;;
+	(openbsd)
+		echo '_postgresql'
+		;;
+	(suse)
+		echo 'postgres'
+		;;
+	(*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 589925de..de6e8aa0 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -19,21 +19,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$("${__explorer:?}/os")
-
-case ${os}
-in
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(devuan)
-		postgres_user='postgres'
-		;;
-	(*)
-		echo "Unsupported OS: ${os}" >&2
-		exit 1
-		;;
-esac
+postgres_user=$("${__type_explorer:?}/postgres_user")
 
 conf_name=${__object_id:?}
 
diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 8f8a2bfe..d0d247b4 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -20,9 +20,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$(cat "${__global:?}/explorer/os")
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
 conf_name=${__object_id:?}
 
@@ -49,19 +49,6 @@ quote() {
 	printf '%s' "$*"
 }
 
-case ${os}
-in
-	(netbsd)
-		postgres_user='pgsql'
-		;;
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(*)
-		postgres_user='postgres'
-		;;
-esac
-
 
 psql_cmd() {
 	printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")"
@@ -117,7 +104,7 @@ then
 				case $(cat "${__global:?}/explorer/kernel_name")
 				in
 					(FreeBSD)
-						echo 'service postgresql restart'
+						echo '/usr/local/etc/rc.d/postgresql restart'
 						;;
 					(OpenBSD|NetBSD)
 						echo '/etc/rc.d/postgresql restart'

From d1f45d3524f8a025488d0d2f020433b6efb7edb6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 19 Feb 2021 09:03:56 +0100
Subject: [PATCH 266/411] __package_pip: corrected typo in man

.. by fully replacing it with a smaller sentence.
---
 cdist/conf/type/__package_pip/man.rst | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index c013695c..5a2bc673 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -24,9 +24,8 @@ name
 
 extra
     Extra optional dependencies which should be installed along the selected
-    package. Can be specified multiple times. Multiple extra optional
-    dependencies can also be specified in a comma-separated list to provide
-    a more pip-natvie style.
+    package. Can be specified multiple times. Multiple extras can be passed
+    in one `--extra` as a comma-separated list.
 
     Extra optional dependencies will be installed even when the base package
     is already installed. Notice that the type will not remove installed extras

From 5e0572189fb930764318227fa9fd9f4966b0da99 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 22 Feb 2021 09:11:22 +0100
Subject: [PATCH 267/411] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 52686617..59c58e65 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,8 @@ next:
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
+	* Type __postgres_role: Implement modification of roles (Dennis Camera)
+	* Type __letsencrypt_cert: Fix issues with hooks (Evil Ham)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)

From 22f637c15b6616899c848e2a08c4e1c1b13f0f0b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 23 Feb 2021 06:29:24 +0100
Subject: [PATCH 268/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 59c58e65..88dda0aa 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
 	* Type __postgres_role: Implement modification of roles (Dennis Camera)
 	* Type __letsencrypt_cert: Fix issues with hooks (Evil Ham)
+	* Type __package_pip: Add optional extra dependencies param (Matthias Stecher)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)

From 0734288483700e8e10cebd87c797b625aa83d55e Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Sun, 21 Feb 2021 19:59:57 +0000
Subject: [PATCH 269/411] First draft of __apt_pin

---
 cdist/conf/type/__apt_pin/man.rst             | 53 ++++++++++++++++
 cdist/conf/type/__apt_pin/manifest            | 63 +++++++++++++++++++
 .../type/__apt_pin/parameter/default/package  |  1 +
 .../type/__apt_pin/parameter/default/state    |  1 +
 cdist/conf/type/__apt_pin/parameter/optional  |  2 +
 cdist/conf/type/__apt_pin/parameter/required  |  2 +
 6 files changed, 122 insertions(+)
 create mode 100644 cdist/conf/type/__apt_pin/man.rst
 create mode 100755 cdist/conf/type/__apt_pin/manifest
 create mode 100644 cdist/conf/type/__apt_pin/parameter/default/package
 create mode 100644 cdist/conf/type/__apt_pin/parameter/default/state
 create mode 100644 cdist/conf/type/__apt_pin/parameter/optional
 create mode 100644 cdist/conf/type/__apt_pin/parameter/required

diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst
new file mode 100644
index 00000000..7fcae6f8
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/man.rst
@@ -0,0 +1,53 @@
+cdist-type__apt_pin(7)
+======================
+
+NAME
+----
+cdist-type__apt_pin - TODO
+
+
+DESCRIPTION
+-----------
+This space intentionally left blank.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # TODO
+    __apt_pin
+
+
+SEE ALSO
+--------
+:strong:`TODO`\ (7)
+
+
+AUTHORS
+-------
+Daniel Fancsali <fancsali@gmail.com>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Daniel Fancsali. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
new file mode 100755
index 00000000..8dd9770d
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -0,0 +1,63 @@
+#!/bin/sh -e
+#
+# 2021 Daniel Fancsali (fancsali@gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+os=$(cat "$__global/explorer/os")
+state="$(cat "$__object/parameter/state")"
+package="$(cat "$__object/parameter/package")"
+distribution="$(cat "$__object/parameter/distribution")"
+priority="$(cat "$__object/parameter/priority")"
+
+
+case "$os" in
+   debian|ubuntu|devuan)
+   ;;
+   *)
+      printf "This type is specific to Debian and it's derivatives" >&2
+      printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2
+      exit 1
+   ;;
+esac
+
+if [ "$package" = "*" ]; then
+    name="default"
+
+else
+    name="$__object_id"
+fi
+
+case $distribution in
+    stabletesting|unsatbel|experimental)
+        pin="release a=$distribution"
+        ;;
+    *)
+        pin="release n=$distribution"
+        ;;
+esac
+
+
+__file /etc/apt/preferences.d/$name \
+    --owner root --group root --mode 0644 \
+    --state "$state" \
+    --source - << EOF
+Package: $package
+Pin: $pin
+Pin-Priority: $priority
+EOF
diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package
new file mode 100644
index 00000000..72e8ffc0
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/default/package
@@ -0,0 +1 @@
+*
diff --git a/cdist/conf/type/__apt_pin/parameter/default/state b/cdist/conf/type/__apt_pin/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional
new file mode 100644
index 00000000..52f01fd2
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/optional
@@ -0,0 +1,2 @@
+state
+package
diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required
new file mode 100644
index 00000000..4b4e9741
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/required
@@ -0,0 +1,2 @@
+distribution
+priority

From 1a74470c4d9b30e41a72fbfc084dc54ea44e643b Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Tue, 23 Feb 2021 09:37:36 +0000
Subject: [PATCH 270/411] __apt_pin: Always use $__object_id as preferences.d
 filename

---
 cdist/conf/type/__apt_pin/manifest | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index 8dd9770d..162b523f 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -36,12 +36,7 @@ case "$os" in
    ;;
 esac
 
-if [ "$package" = "*" ]; then
-    name="default"
-
-else
-    name="$__object_id"
-fi
+name="$__object_id"
 
 case $distribution in
     stabletesting|unsatbel|experimental)

From dc66efa690e15ff32d6836e23baa1cbec5eee1ed Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Tue, 23 Feb 2021 11:59:09 +0000
Subject: [PATCH 271/411] Fix shellcheck issues

---
 cdist/conf/type/__apt_pin/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index 162b523f..b1372ad0 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -48,7 +48,7 @@ case $distribution in
 esac
 
 
-__file /etc/apt/preferences.d/$name \
+__file "/etc/apt/preferences.d/$name" \
     --owner root --group root --mode 0644 \
     --state "$state" \
     --source - << EOF

From 60fd7ba1f38382761c366cbc0425ddaec62f1164 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 28 Feb 2021 13:37:23 +0100
Subject: [PATCH 272/411] Release 6.9.5

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 88dda0aa..95e36b88 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)

From 8ef19d47f6b5fb6cdcba7a9a3b8890388dbd7023 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 1 Mar 2021 17:59:25 +0100
Subject: [PATCH 273/411] [type/__pyvenv] Fix example (--user -> --owner)

---
 cdist/conf/type/__pyvenv/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__pyvenv/man.rst b/cdist/conf/type/__pyvenv/man.rst
index 8085ff12..e2e4a1e6 100644
--- a/cdist/conf/type/__pyvenv/man.rst
+++ b/cdist/conf/type/__pyvenv/man.rst
@@ -61,7 +61,7 @@ EXAMPLES
     __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4
 
     # Create python virtualenv for user foo.
-    __pyvenv /home/foo/fooenv --group foo --user foo
+    __pyvenv /home/foo/fooenv --group foo --owner foo
 
     # Create python virtualenv with specific parameters.
     __pyvenv /home/services/djangoenv --venvparams "--copies --system-site-packages"

From e7d33891df945831d54d164eb13f2a9ae4f9dd8e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 2 Mar 2021 09:29:33 +0100
Subject: [PATCH 274/411] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 95e36b88..072fcb6b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __pyvenv: Fix user example in man page (Dennis Camera)
+
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)

From ea0126dd8117c051612e6b6abfe895f5f722c584 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Fri, 5 Mar 2021 16:11:49 +0100
Subject: [PATCH 275/411] Make local state dir available to custom remote
 scripts

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/config.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cdist/config.py b/cdist/config.py
index e84f6f84..19d5bd70 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -420,6 +420,9 @@ class Config:
                 exec_path=sys.argv[0],
                 save_output_streams=args.save_output_streams)
 
+            # Make __global state dir available to custom remote scripts.
+            os.environ['__global'] = local.base_path
+
             remote = cdist.exec.remote.Remote(
                 target_host=target_host,
                 remote_exec=remote_exec,

From ecba284fc8ce68486c5f1f87a863409fcad0f106 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Fri, 5 Mar 2021 16:13:02 +0100
Subject: [PATCH 276/411] changelog++

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 072fcb6b..ad93a595 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
+	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From fb19f342669d18e6decb21ee4cfb2b22e46748d9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 9 Mar 2021 21:15:26 +0100
Subject: [PATCH 277/411] [type/__ssh_authorized_key] Only grep if file exists

---
 cdist/conf/type/__ssh_authorized_key/explorer/entry | 1 +
 cdist/conf/type/__ssh_authorized_key/gencode-remote | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__ssh_authorized_key/explorer/entry b/cdist/conf/type/__ssh_authorized_key/explorer/entry
index ccab0afc..aca0f2b9 100755
--- a/cdist/conf/type/__ssh_authorized_key/explorer/entry
+++ b/cdist/conf/type/__ssh_authorized_key/explorer/entry
@@ -25,6 +25,7 @@ type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ]
 if [ -n "${type_and_key}" ]
 then
     file="$(cat "$__object/parameter/file")"
+    test -e "$file" || exit 0
 
     # get any entries that match the type and key
 
diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote
index f37aa565..61c77fb9 100755
--- a/cdist/conf/type/__ssh_authorized_key/gencode-remote
+++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote
@@ -37,9 +37,9 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
 # preserve ownership and permissions of existing file
 if [ -f "$file" ]; then
    cp -p "$file" "\$tmpfile"
+   grep -v -F -x '$line' '$file' >\$tmpfile
 fi
-grep -v -F -x '$line' '$file' > \$tmpfile || true
-mv -f "\$tmpfile" "$file"
+cat "\$tmpfile" >"$file"
 DONE
 }
 

From 31cc592aa10902c3e83357ee4a0b9300b5e34efa Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 10 Mar 2021 19:25:04 +0100
Subject: [PATCH 278/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index ad93a595..c63b901a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
+	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From e47c4dd8a4cc441b94fe73d362d79525d01a4bb1 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 9 Mar 2021 19:59:05 +0100
Subject: [PATCH 279/411] [type/__sshd_config] Whitelist OpenBMC in manifest

---
 cdist/conf/type/__sshd_config/manifest | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest
index 566bde90..e37afebb 100755
--- a/cdist/conf/type/__sshd_config/manifest
+++ b/cdist/conf/type/__sshd_config/manifest
@@ -39,7 +39,14 @@ in
 	(freebsd|netbsd|openbsd)
 		# whitelist
 		;;
+	(openbmc-phosphor)
+		# whitelist
+		# OpenBMC can be configured with dropbear and OpenSSH.
+		# If dropbear is used, the state explorer will already fail because it
+		# cannot find the sshd binary.
+		;;
 	(*)
+		: "${__type:?}"  # make shellcheck happy
 		printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \
 			"${os}" "${__type##*/}" >&2
 		printf 'Please contribute an implementation for it if you can.\n' >&2

From 10ca1c12fd25a9f2b92cebbc5d375a1723ed5e2f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 12 Mar 2021 08:21:03 +0100
Subject: [PATCH 280/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index c63b901a..42a74d04 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
+	* Type __sshd_config: Whitelist OpenBMC (Dennis Camera)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From 7a0b697f4c394f1ae8a8941a59937007cfe474aa Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 21 Mar 2021 18:10:06 +0100
Subject: [PATCH 281/411] Implement maintaining object relationship graph

For each object maintain parent-child relationship graph, i.e. list of
parent objects ('parents' property) and list of children objects ('children'
property).

Objects without parent(s) are objects specified in init manifest.
Objects without children are object of types that do not reuse other types.
---
 cdist/core/cdist_object.py |  7 +++++++
 cdist/emulator.py          | 19 +++++++++++++++++++
 docs/src/cdist-cache.rst   | 17 +++++++++++++++++
 3 files changed, 43 insertions(+)

diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index 51d61e04..eea2c255 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -247,6 +247,13 @@ class CdistObject:
             lambda obj: os.path.join(obj.absolute_path, 'typeorder'))
     typeorder_dep = fsproperty.FileListProperty(
             lambda obj: os.path.join(obj.absolute_path, 'typeorder_dep'))
+    # objects without parents are objects specified in init manifest
+    parents = fsproperty.FileListProperty(
+            lambda obj: os.path.join(obj.absolute_path, 'parents'))
+    # objects without children are object of types that do not reuse other
+    # types
+    children = fsproperty.FileListProperty(
+            lambda obj: os.path.join(obj.absolute_path, 'children'))
 
     def cleanup(self):
         try:
diff --git a/cdist/emulator.py b/cdist/emulator.py
index a2bdc3d4..f1db862e 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -106,6 +106,7 @@ class Emulator:
             self.save_stdin()
             self.record_requirements()
             self.record_auto_requirements()
+            self.record_parent_child_relationships()
             self.log.trace("Finished %s %s" % (
                 self.cdist_object.path, self.parameters))
 
@@ -420,3 +421,21 @@ class Emulator:
                 self.log.debug("Recording autorequirement %s for %s",
                                current_object.name, parent.name)
                 parent.autorequire.append(current_object.name)
+
+    def record_parent_child_relationships(self):
+        # __object_name is the name of the object whose type manifest is
+        # currently executed
+        __object_name = self.env.get('__object_name', None)
+        if __object_name:
+            # The object whose type manifest is currently run
+            parent = self.cdist_object.object_from_name(__object_name)
+            # The object currently being defined
+            current_object = self.cdist_object
+            if current_object.name not in parent.children:
+                self.log.debug("Recording child %s for %s",
+                               current_object.name, parent.name)
+                parent.children.append(current_object.name)
+            if parent.name not in current_object.parents:
+                self.log.debug("Recording parent %s for %s",
+                               parent.name, current_object.name)
+                current_object.parents.append(parent.name)
diff --git a/docs/src/cdist-cache.rst b/docs/src/cdist-cache.rst
index d2d2d56c..d4159e77 100644
--- a/docs/src/cdist-cache.rst
+++ b/docs/src/cdist-cache.rst
@@ -61,6 +61,14 @@ Object cache overview
 ~~~~~~~~~~~~~~~~~~~~~
 Each object under :strong:`object` directory has its own structure.
 
+autorequire
+    file containing a list of object auto requirements
+
+children
+    file containing a list of object children, i.e. objects of types that this
+    type reuses (along with 'parents' it is used for maintaining parent-child
+    relationship graph)
+
 code-local
     code generated from gencode-local, present only if something is
     generated
@@ -80,6 +88,15 @@ parameter
     directory containing type parameter named files containing parameter
     values   
 
+parents
+    file containing a list of object parents, i.e. objects of types that reuse
+    this type (along with 'children' it is used for maintaining parent-child
+    relationship graph); objects without parents are objects specified in init
+    manifest
+
+require
+    file containing a list of object requirements
+
 source
     this type's source (init manifest)
 

From 167c2ad7ea2b895cdc7d89c72968d88c7d1ab34f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 30 Mar 2021 13:24:56 +0200
Subject: [PATCH 282/411] [type/__git] Fix if --owner / --group is numeric

Before, if --owner and/or --group was numeric, gencode-remote would generate
`chown` code every time.
---
 cdist/conf/type/__git/explorer/group | 20 +++++++++++++++++---
 cdist/conf/type/__git/explorer/owner | 20 +++++++++++++++++---
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group
index 3ddf9656..1365c60d 100644
--- a/cdist/conf/type/__git/explorer/group
+++ b/cdist/conf/type/__git/explorer/group
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id/.git"
+destination="/${__object_id:?}/.git"
 
-stat --print "%G" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((group_gid + 1)) -ge 0
+then
+	group_should=$(cat "${__object:?}/parameter/group")
+
+	if expr "${group_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${group_gid}"
+	else
+		printf '%s\n' "$(id -u -n "${group_gid}")"
+	fi
+fi
diff --git a/cdist/conf/type/__git/explorer/owner b/cdist/conf/type/__git/explorer/owner
index 4c3cd431..4a4d0d13 100644
--- a/cdist/conf/type/__git/explorer/owner
+++ b/cdist/conf/type/__git/explorer/owner
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id/.git"
+destination="/${__object_id:?}/.git"
 
-stat --print "%U" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((owner_uid + 1)) -ge 0
+then
+	owner_should=$(cat "${__object:?}/parameter/owner")
+
+	if expr "${owner_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${owner_uid}"
+	else
+		printf '%s\n' "$(id -u -n "${owner_uid}")"
+	fi
+fi

From 985252585ce83605787d2612fdc2a84b9788d943 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 30 Mar 2021 13:26:21 +0200
Subject: [PATCH 283/411] [type/__pyvenv] Fix if --owner / --group is numeric

Before, if --owner and/or --group was numeric, gencode-remote would generate
`chown` code every time.
---
 cdist/conf/type/__pyvenv/explorer/group | 20 +++++++++++++++++---
 cdist/conf/type/__pyvenv/explorer/owner | 20 +++++++++++++++++---
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group
index a655bda7..f31a1cb7 100755
--- a/cdist/conf/type/__pyvenv/explorer/group
+++ b/cdist/conf/type/__pyvenv/explorer/group
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id"
+destination="/${__object_id:?}"
 
-stat --print "%G" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((group_gid + 1)) -ge 0
+then
+	group_should=$(cat "${__object:?}/parameter/group")
+
+	if expr "${group_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${group_gid}"
+	else
+		printf '%s\n' "$(id -u -n "${group_gid}")"
+	fi
+fi
diff --git a/cdist/conf/type/__pyvenv/explorer/owner b/cdist/conf/type/__pyvenv/explorer/owner
index 8b3c7f8e..ebec751f 100755
--- a/cdist/conf/type/__pyvenv/explorer/owner
+++ b/cdist/conf/type/__pyvenv/explorer/owner
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id"
+destination="/${__object_id:?}"
 
-stat --print "%U" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((owner_uid + 1)) -ge 0
+then
+	owner_should=$(cat "${__object:?}/parameter/owner")
+
+	if expr "${owner_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${owner_uid}"
+	else
+		printf '%s\n' "$(id -u -n "${owner_uid}")"
+	fi
+fi

From 1e765fcab7bae02d6b380ca02e7cb04ec69ebdc5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 31 Mar 2021 07:54:00 +0200
Subject: [PATCH 284/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 42a74d04..03ae7566 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
 	* Type __sshd_config: Whitelist OpenBMC (Dennis Camera)
+	* Core: Maintain object relationship graph in cdist cache (Darko Poljak)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From f984a918b959a46e49fe8456a327eca8d44313de Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 30 Mar 2021 07:56:38 +0200
Subject: [PATCH 285/411] Fix log message string formatting

Use logging message format with args, instead of direct `%` or `str.format`.

Resolve #855.
---
 cdist/argparse.py                      |  8 +--
 cdist/config.py                        | 87 ++++++++++++--------------
 cdist/core/cdist_type.py               |  6 +-
 cdist/core/explorer.py                 | 26 ++++----
 cdist/core/manifest.py                 |  2 +-
 cdist/emulator.py                      | 37 +++++------
 cdist/exec/local.py                    | 16 +++--
 cdist/exec/remote.py                   | 14 ++---
 cdist/install.py                       |  2 +-
 cdist/inventory.py                     | 36 +++++------
 cdist/preos.py                         |  6 +-
 cdist/preos/debootstrap/debootstrap.py | 41 ++++++------
 cdist/scan/scan.py                     |  4 +-
 cdist/util/ipaddr.py                   |  6 +-
 14 files changed, 140 insertions(+), 151 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 88759d7b..cadac39a 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -533,10 +533,10 @@ def parse_and_configure(argv, singleton=True):
 
     log = logging.getLogger("cdist")
 
-    log.verbose("version %s" % cdist.VERSION)
-    log.trace('command line args: {}'.format(cfg.command_line_args))
-    log.trace('configuration: {}'.format(cfg.get_config()))
-    log.trace('configured args: {}'.format(args))
+    log.verbose("version %s", cdist.VERSION)
+    log.trace('command line args: %s', cfg.command_line_args)
+    log.trace('configuration: %s', cfg.get_config())
+    log.trace('configured args: %s', args)
 
     check_beta(vars(args))
 
diff --git a/cdist/config.py b/cdist/config.py
index 19d5bd70..d6fec55f 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -273,15 +273,15 @@ class Config:
                     host_tags = None
             host_base_path, hostdir = cls.create_host_base_dirs(
                 host, base_root_path)
-            log.debug("Base root path for target host \"{}\" is \"{}\"".format(
-                host, host_base_path))
+            log.debug("Base root path for target host \"%s\" is \"%s\"",
+                      host, host_base_path)
 
             hostcnt += 1
             if args.parallel:
                 pargs = (host, host_tags, host_base_path, hostdir, args, True,
                          configuration)
-                log.trace(("Args for multiprocessing operation "
-                           "for host {}: {}".format(host, pargs)))
+                log.trace("Args for multiprocessing operation for host %s: %s",
+                          host, pargs)
                 process_args.append(pargs)
             else:
                 try:
@@ -298,10 +298,10 @@ class Config:
             except cdist.Error:
                 failed_hosts.append(host)
         elif args.parallel:
-            log.trace("Multiprocessing start method is {}".format(
-                multiprocessing.get_start_method()))
-            log.trace(("Starting multiprocessing Pool for {} "
-                       "parallel host operation".format(args.parallel)))
+            log.trace("Multiprocessing start method is %s",
+                      multiprocessing.get_start_method())
+            log.trace("Starting multiprocessing Pool for %d parallel host"
+                      " operation", args.parallel)
 
             results = mp_pool_run(cls.onehost,
                                   process_args,
@@ -396,16 +396,13 @@ class Config:
 
             remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds(
                 args)
-            log.debug("remote_exec for host \"{}\": {}".format(
-                host, remote_exec))
-            log.debug("remote_copy for host \"{}\": {}".format(
-                host, remote_copy))
+            log.debug("remote_exec for host \"%s\": %s", host, remote_exec)
+            log.debug("remote_copy for host \"%s\": %s", host, remote_copy)
 
             family = cls._address_family(args)
-            log.debug("address family: {}".format(family))
+            log.debug("address family: %s", family)
             target_host = cls.resolve_target_addresses(host, family)
-            log.debug("target_host for host \"{}\": {}".format(
-                host, target_host))
+            log.debug("target_host for host \"%s\": %s", host, target_host)
 
             local = cdist.exec.local.Local(
                 target_host=target_host,
@@ -474,8 +471,8 @@ class Config:
         """Do what is most often done: deploy & cleanup"""
         start_time = time.time()
 
-        self.log.info("Starting {} run".format(
-            'dry' if self.dry_run else 'configuration'))
+        self.log.info("Starting %s run",
+                      'dry' if self.dry_run else 'configuration')
 
         self._init_files_dirs()
 
@@ -493,9 +490,9 @@ class Config:
         self._remove_files_dirs()
 
         self.local.save_cache(start_time)
-        self.log.info("Finished {} run in {:.2f} seconds".format(
+        self.log.info("Finished %s run in %.2f seconds",
                       'dry' if self.dry_run else 'successful',
-                      time.time() - start_time))
+                      time.time() - start_time)
 
     def cleanup(self):
         self.log.debug("Running cleanup commands")
@@ -519,8 +516,8 @@ class Config:
                 self.local.object_path, self.local.type_path,
                 self.local.object_marker_name):
             if cdist_object.cdist_type.is_install:
-                self.log.debug(("Running in config mode, ignoring install "
-                                "object: {0}").format(cdist_object))
+                self.log.debug("Running in config mode, ignoring install "
+                               "object: %s", cdist_object)
             else:
                 yield cdist_object
 
@@ -565,8 +562,7 @@ class Config:
         return objects_changed
 
     def _iterate_once_parallel(self):
-        self.log.debug("Iteration in parallel mode in {} jobs".format(
-            self.jobs))
+        self.log.debug("Iteration in parallel mode in %d jobs", self.jobs)
         objects_changed = False
 
         cargo = []
@@ -588,8 +584,8 @@ class Config:
             self.object_prepare(cargo[0])
             objects_changed = True
         elif cargo:
-            self.log.trace("Multiprocessing start method is {}".format(
-                multiprocessing.get_start_method()))
+            self.log.trace("Multiprocessing start method is %s",
+                           multiprocessing.get_start_method())
 
             self.log.trace("Multiprocessing cargo: %s", cargo)
 
@@ -603,9 +599,8 @@ class Config:
                                 "sequentially"))
                 self.explorer.transfer_type_explorers(cargo_types.pop())
             else:
-                self.log.trace(("Starting multiprocessing Pool for {} "
-                                "parallel types explorers transferring".format(
-                                    nt)))
+                self.log.trace("Starting multiprocessing Pool for %d "
+                               "parallel types explorers transferring", nt)
                 args = [
                     (ct, ) for ct in cargo_types
                 ]
@@ -614,8 +609,8 @@ class Config:
                 self.log.trace(("Multiprocessing for parallel transferring "
                                 "types' explorers finished"))
 
-            self.log.trace(("Starting multiprocessing Pool for {} parallel "
-                            "objects preparation".format(n)))
+            self.log.trace("Starting multiprocessing Pool for %d parallel "
+                           "objects preparation", n)
             args = [
                 (c, False, ) for c in cargo
             ]
@@ -667,10 +662,10 @@ class Config:
                 self.object_run(chunk[0])
                 objects_changed = True
             elif chunk:
-                self.log.trace("Multiprocessing start method is {}".format(
-                    multiprocessing.get_start_method()))
-                self.log.trace(("Starting multiprocessing Pool for {} "
-                               "parallel object run".format(n)))
+                self.log.trace("Multiprocessing start method is %s",
+                               multiprocessing.get_start_method())
+                self.log.trace("Starting multiprocessing Pool for %d "
+                               "parallel object run", n)
                 args = [
                     (c, ) for c in chunk
                 ]
@@ -794,9 +789,9 @@ class Config:
     def object_prepare(self, cdist_object, transfer_type_explorers=True):
         """Prepare object: Run type explorer + manifest"""
         self._handle_deprecation(cdist_object)
-        self.log.verbose("Preparing object {}".format(cdist_object.name))
-        self.log.verbose(
-            "Running manifest and explorers for " + cdist_object.name)
+        self.log.verbose("Preparing object %s", cdist_object.name)
+        self.log.verbose("Running manifest and explorers for %s",
+                         cdist_object.name)
         self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
         try:
             self.manifest.run_type_manifest(cdist_object)
@@ -810,13 +805,13 @@ class Config:
     def object_run(self, cdist_object):
         """Run gencode and code for an object"""
         try:
-            self.log.verbose("Running object " + cdist_object.name)
+            self.log.verbose("Running object %s", cdist_object.name)
             if cdist_object.state == core.CdistObject.STATE_DONE:
                 raise cdist.Error(("Attempting to run an already finished "
-                                   "object: %s"), cdist_object)
+                                   "object: {}").format(cdist_object))
 
             # Generate
-            self.log.debug("Generating code for %s" % (cdist_object.name))
+            self.log.debug("Generating code for %s", cdist_object.name)
             cdist_object.code_local = self.code.run_gencode_local(cdist_object)
             cdist_object.code_remote = self.code.run_gencode_remote(
                 cdist_object)
@@ -825,20 +820,20 @@ class Config:
 
             # Execute
             if cdist_object.code_local or cdist_object.code_remote:
-                self.log.info("Processing %s" % (cdist_object.name))
+                self.log.info("Processing %s", cdist_object.name)
             if not self.dry_run:
                 if cdist_object.code_local:
-                    self.log.trace("Executing local code for %s"
-                                   % (cdist_object.name))
+                    self.log.trace("Executing local code for %s",
+                                   cdist_object.name)
                     self.code.run_code_local(cdist_object)
                 if cdist_object.code_remote:
-                    self.log.trace("Executing remote code for %s"
-                                   % (cdist_object.name))
+                    self.log.trace("Executing remote code for %s",
+                                   cdist_object.name)
                     self.code.transfer_code_remote(cdist_object)
                     self.code.run_code_remote(cdist_object)
 
             # Mark this object as done
-            self.log.trace("Finishing run of " + cdist_object.name)
+            self.log.trace("Finishing run of %s", cdist_object.name)
             cdist_object.state = core.CdistObject.STATE_DONE
         except cdist.Error as e:
             raise cdist.CdistObjectError(cdist_object, e)
diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py
index c0329c8a..274de989 100644
--- a/cdist/core/cdist_type.py
+++ b/cdist/core/cdist_type.py
@@ -82,9 +82,9 @@ class CdistType:
                 yield cls(base_path, name)
             except InvalidTypeError as e:
                 # ignore invalid type, log warning and continue
-                msg = "Ignoring invalid type '%s' at '%s' defined at '%s'" % (
-                    e.type_path, e.type_absolute_path, e.source_path)
-                cls.log.warning(msg)
+                cls.log.warning("Ignoring invalid type '%s' at '%s' defined"
+                                " at '%s'", e.type_path, e.type_absolute_path,
+                                e.source_path)
                 # remove invalid from runtime conf dir
                 os.remove(e.type_absolute_path)
 
diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py
index a3baa959..caa12a7d 100644
--- a/cdist/core/explorer.py
+++ b/cdist/core/explorer.py
@@ -131,18 +131,17 @@ class Explorer:
             self._run_global_explorer(explorer, out_path)
 
     def _run_global_explorers_parallel(self, out_path):
-        self.log.debug("Running global explorers in {} parallel jobs".format(
-            self.jobs))
-        self.log.trace("Multiprocessing start method is {}".format(
-            multiprocessing.get_start_method()))
-        self.log.trace(("Starting multiprocessing Pool for global "
-                       "explorers run"))
+        self.log.debug("Running global explorers in %s parallel jobs",
+                       self.jobs)
+        self.log.trace("Multiprocessing start method is %s",
+                       multiprocessing.get_start_method())
+        self.log.trace("Starting multiprocessing Pool for global explorers"
+                       " run")
         args = [
             (e, out_path, ) for e in self.list_global_explorer_names()
         ]
         mp_pool_run(self._run_global_explorer, args, jobs=self.jobs)
-        self.log.trace(("Multiprocessing run for global explorers "
-                        "finished"))
+        self.log.trace("Multiprocessing run for global explorers finished")
 
     # logger is not pickable, so remove it when we pickle
     def __getstate__(self):
@@ -184,15 +183,14 @@ class Explorer:
         in the object.
 
         """
-        self.log.verbose("Running type explorers for {}".format(
-            cdist_object.cdist_type))
+        self.log.verbose("Running type explorers for %s",
+                         cdist_object.cdist_type)
         if transfer_type_explorers:
             self.log.trace("Transferring type explorers for type: %s",
                            cdist_object.cdist_type)
             self.transfer_type_explorers(cdist_object.cdist_type)
         else:
-            self.log.trace(("No need for transferring type explorers for "
-                            "type: %s"),
+            self.log.trace("No need for transferring type explorers for %s",
                            cdist_object.cdist_type)
         self.log.trace("Transferring object parameters for object: %s",
                        cdist_object.name)
@@ -236,8 +234,8 @@ class Explorer:
            remote side."""
         if cdist_type.explorers:
             if cdist_type.name in self._type_explorers_transferred:
-                self.log.trace(("Skipping retransfer of type explorers "
-                                "for: %s"), cdist_type)
+                self.log.trace("Skipping retransfer of type explorers for: %s",
+                               cdist_type)
             else:
                 source = os.path.join(self.local.type_path,
                                       cdist_type.explorer_path)
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 390340d4..3148d66c 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -161,7 +161,7 @@ class Manifest:
             raise NoInitialManifestError(initial_manifest, user_supplied)
 
         message_prefix = "initialmanifest"
-        self.log.verbose("Running initial manifest " + initial_manifest)
+        self.log.verbose("Running initial manifest %s", initial_manifest)
         which = "init"
         if self.local.save_output_streams:
             stderr_path = os.path.join(self.local.stderr_base_path, which)
diff --git a/cdist/emulator.py b/cdist/emulator.py
index f1db862e..f09c282d 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -107,8 +107,8 @@ class Emulator:
             self.record_requirements()
             self.record_auto_requirements()
             self.record_parent_child_relationships()
-            self.log.trace("Finished %s %s" % (
-                self.cdist_object.path, self.parameters))
+            self.log.trace("Finished %s %s", self.cdist_object.path,
+                           self.parameters)
 
     def __init_log(self):
         """Setup logging facility"""
@@ -170,7 +170,7 @@ class Emulator:
 
         # And finally parse/verify parameter
         self.args = parser.parse_args(self.argv[1:])
-        self.log.trace('Args: %s' % self.args)
+        self.log.trace('Args: %s', self.args)
 
     def init_object(self):
         # Initialize object - and ensure it is not in args
@@ -241,8 +241,8 @@ class Emulator:
                 raise cdist.Error(errmsg)
         else:
             if self.cdist_object.exists:
-                self.log.debug(('Object %s override forced with '
-                                'CDIST_OVERRIDE'), self.cdist_object.name)
+                self.log.debug('Object %s override forced with CDIST_OVERRIDE',
+                               self.cdist_object.name)
                 self.cdist_object.create(True)
             else:
                 self.cdist_object.create()
@@ -260,8 +260,8 @@ class Emulator:
             parent = self.cdist_object.object_from_name(__object_name)
             parent.typeorder.append(self.cdist_object.name)
             if self._order_dep_on():
-                self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
-                                ' for %s'), depname, parent.name)
+                self.log.trace('[ORDER_DEP] Adding %s to typeorder dep for %s',
+                               depname, parent.name)
                 parent.typeorder_dep.append(depname)
         elif self._order_dep_on():
             self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
@@ -301,16 +301,14 @@ class Emulator:
         try:
             cdist_object = self.cdist_object.object_from_name(requirement)
         except core.cdist_type.InvalidTypeError as e:
-            self.log.error(("%s requires object %s, but type %s does not"
-                            " exist. Defined at %s" % (
-                                self.cdist_object.name,
-                                requirement, e.name, self.object_source)))
+            self.log.error("%s requires object %s, but type %s does not"
+                           " exist. Defined at %s", self.cdist_object.name,
+                           requirement, e.name, self.object_source)
             raise
         except core.cdist_object.MissingObjectIdError:
-            self.log.error(("%s requires object %s without object id."
-                            " Defined at %s" % (self.cdist_object.name,
-                                                requirement,
-                                                self.object_source)))
+            self.log.error("%s requires object %s without object id."
+                           " Defined at %s", self.cdist_object.name,
+                           requirement, self.object_source)
             raise
 
         self.log.debug("Recording requirement %s for %s",
@@ -380,10 +378,9 @@ class Emulator:
                         self.env['require'] += " " + lastcreatedtype
                 else:
                     self.env['require'] = lastcreatedtype
-                self.log.debug(("Injecting require for "
-                                "CDIST_ORDER_DEPENDENCY: %s for %s"),
-                               lastcreatedtype,
-                               self.cdist_object.name)
+                self.log.debug("Injecting require for"
+                               " CDIST_ORDER_DEPENDENCY: %s for %s",
+                               lastcreatedtype, self.cdist_object.name)
             except IndexError:
                 # if no second last line, we are on the first type,
                 # so do not set a requirement
@@ -391,7 +388,7 @@ class Emulator:
 
         if "require" in self.env:
             requirements = self.env['require']
-            self.log.debug("reqs = " + requirements)
+            self.log.debug("reqs = %s", requirements)
             for requirement in self._parse_require(requirements):
                 # Ignore empty fields - probably the only field anyway
                 if len(requirement) == 0:
diff --git a/cdist/exec/local.py b/cdist/exec/local.py
index e0aab190..6713cd13 100644
--- a/cdist/exec/local.py
+++ b/cdist/exec/local.py
@@ -154,8 +154,8 @@ class Local:
         with open(self.object_marker_file, 'w') as fd:
             fd.write("%s\n" % self.object_marker_name)
 
-        self.log.trace("Object marker %s saved in %s" % (
-            self.object_marker_name, self.object_marker_file))
+        self.log.trace("Object marker %s saved in %s",
+                       self.object_marker_name, self.object_marker_file)
 
     def _init_cache_dir(self, cache_dir):
         home_dir = cdist.home_dir()
@@ -289,14 +289,12 @@ class Local:
         return cache_subpath
 
     def save_cache(self, start_time=time.time()):
-        self.log.trace("cache subpath pattern: {}".format(
-            self.cache_path_pattern))
+        self.log.trace("cache subpath pattern: %s", self.cache_path_pattern)
         cache_subpath = self._cache_subpath(start_time,
                                             self.cache_path_pattern)
-        self.log.debug("cache subpath: {}".format(cache_subpath))
+        self.log.debug("cache subpath: %s", cache_subpath)
         destination = os.path.join(self.cache_path, cache_subpath)
-        self.log.trace(("Saving cache: " + self.base_path + " to " +
-                        destination))
+        self.log.trace("Saving cache %s to %s", self.base_path, destination)
 
         if not os.path.exists(destination):
             shutil.move(self.base_path, destination)
@@ -332,7 +330,7 @@ class Local:
 
         # Iterate over all directories and link the to the output dir
         for conf_dir in self.conf_dirs:
-            self.log.debug("Checking conf_dir %s ..." % (conf_dir))
+            self.log.debug("Checking conf_dir %s ...", conf_dir)
             for sub_dir in CONF_SUBDIRS_LINKED:
                 current_dir = os.path.join(conf_dir, sub_dir)
 
@@ -350,7 +348,7 @@ class Local:
                     if os.path.exists(dst):
                         os.unlink(dst)
 
-                    self.log.trace("Linking %s to %s ..." % (src, dst))
+                    self.log.trace("Linking %s to %s ...", src, dst)
                     try:
                         os.symlink(src, dst)
                     except OSError as e:
diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py
index e5af2f34..ea85da4c 100644
--- a/cdist/exec/remote.py
+++ b/cdist/exec/remote.py
@@ -176,19 +176,19 @@ class Remote:
                 # create archive
                 tarpath, fcnt = autil.tar(source, self.archiving_mode)
                 if tarpath is None:
-                    self.log.trace(("Files count {} is lower than {} limit, "
-                                    "skipping archiving").format(
-                                        fcnt, autil.FILES_LIMIT))
+                    self.log.trace("Files count %d is lower than %d limit, "
+                                   "skipping archiving",
+                                   fcnt, autil.FILES_LIMIT)
                 else:
-                    self.log.trace(("Archiving mode, tarpath: %s, file count: "
-                                    "%s"), tarpath, fcnt)
+                    self.log.trace("Archiving mode, tarpath: %s, file count: "
+                                   "%s", tarpath, fcnt)
                     # get archive name
                     tarname = os.path.basename(tarpath)
                     self.log.trace("Archiving mode tarname: %s", tarname)
                     # archive path at the remote
                     desttarpath = os.path.join(destination, tarname)
-                    self.log.trace(
-                        "Archiving mode desttarpath: %s", desttarpath)
+                    self.log.trace("Archiving mode desttarpath: %s",
+                                   desttarpath)
                     # transfer archive to the remote side
                     self.log.trace("Archiving mode: transferring")
                     self._transfer_file(tarpath, desttarpath)
diff --git a/cdist/install.py b/cdist/install.py
index 7c894fe5..d077baef 100644
--- a/cdist/install.py
+++ b/cdist/install.py
@@ -47,4 +47,4 @@ class Install(cdist.config.Config):
                 yield cdist_object
             else:
                 self.log.debug("Running in install mode, ignoring non install"
-                               "object: {0}".format(cdist_object))
+                               "object: %s", cdist_object)
diff --git a/cdist/inventory.py b/cdist/inventory.py
index 0387f326..106052a2 100644
--- a/cdist/inventory.py
+++ b/cdist/inventory.py
@@ -92,7 +92,7 @@ class Inventory:
         self.init_db()
 
     def init_db(self):
-        self.log.trace("Init db: {}".format(self.db_basedir))
+        self.log.trace("Init db: %s", self.db_basedir)
         if not os.path.exists(self.db_basedir):
             os.makedirs(self.db_basedir, exist_ok=True)
         elif not os.path.isdir(self.db_basedir):
@@ -182,9 +182,9 @@ class Inventory:
         configuration = cfg.get_config(section='GLOBAL')
         determine_default_inventory_dir(args, configuration)
 
-        log.debug("Using inventory: {}".format(args.inventory_dir))
-        log.trace("Inventory args: {}".format(vars(args)))
-        log.trace("Inventory command: {}".format(args.subcommand))
+        log.debug("Using inventory: %s", args.inventory_dir)
+        log.trace("Inventory args: %s", vars(args))
+        log.trace("Inventory command: %s", args.subcommand)
 
         if args.subcommand == "list":
             c = InventoryList(hosts=args.host, istag=args.tag,
@@ -237,16 +237,16 @@ class InventoryList(Inventory):
     def _do_list(self, it_tags, it_hosts, check_func):
         if (it_tags is not None):
             param_tags = set(it_tags)
-            self.log.trace("param_tags: {}".format(param_tags))
+            self.log.trace("param_tags: %s", param_tags)
         else:
             param_tags = set()
         for host in it_hosts:
-            self.log.trace("host: {}".format(host))
+            self.log.trace("host: %s", host)
             tags = self._get_host_tags(host)
             if tags is None:
-                self.log.debug("Host \'{}\' not found, skipped".format(host))
+                self.log.debug("Host \'%s\' not found, skipped", host)
                 continue
-            self.log.trace("tags: {}".format(tags))
+            self.log.trace("tags: %s", tags)
             if check_func(tags, param_tags):
                 yield host, tags
 
@@ -308,11 +308,11 @@ class InventoryHost(Inventory):
 
     def _action(self, host):
         if self.action == "add":
-            self.log.debug("Adding host \'{}\'".format(host))
+            self.log.debug("Adding host \'%s\'", host)
         elif self.action == "del":
-            self.log.debug("Deleting host \'{}\'".format(host))
+            self.log.debug("Deleting host \'%s\'", host)
         hostpath = self._host_path(host)
-        self.log.trace("hostpath: {}".format(hostpath))
+        self.log.trace("hostpath: %s", hostpath)
         if self.action == "add" and not os.path.exists(hostpath):
             self._new_hostpath(hostpath)
         else:
@@ -372,23 +372,23 @@ class InventoryTag(Inventory):
             print("Host \'{}\' does not exist, skipping".format(host),
                   file=sys.stderr)
             return
-        self.log.trace("existing host_tags: {}".format(host_tags))
+        self.log.trace("existing host_tags: %s", host_tags)
         if self.action == "del" and self.all:
             host_tags = set()
         else:
             for tag in self.input_tags:
                 if self.action == "add":
-                    self.log.debug("Adding tag \'{}\' for host \'{}\'".format(
-                        tag, host))
+                    self.log.debug("Adding tag \'%s\' for host \'%s\'",
+                                   tag, host)
                     host_tags.add(tag)
                 elif self.action == "del":
-                    self.log.debug("Deleting tag \'{}\' for host "
-                                   "\'{}\'".format(tag, host))
+                    self.log.debug("Deleting tag \'%s\' for host \'%s\'",
+                                   tag, host)
                     if tag in host_tags:
                         host_tags.remove(tag)
-        self.log.trace("new host tags: {}".format(host_tags))
+        self.log.trace("new host tags: %s", host_tags)
         if not self._write_host_tags(host, host_tags):
-            self.log.trace("{} does not exist, skipped".format(host))
+            self.log.trace("%s does not exist, skipped", host)
 
     def run(self):
         if self.allhosts:
diff --git a/cdist/preos.py b/cdist/preos.py
index f8a5dd67..45711a41 100644
--- a/cdist/preos.py
+++ b/cdist/preos.py
@@ -49,7 +49,7 @@ def scan_preos_dir_plugins(dir):
                 c = cm[1]
                 yield from preos_plugin(c)
         except ImportError as e:
-            log.warning("Cannot import '{}': {}".format(module_name, e))
+            log.warning("Cannot import '%s': %s", module_name, e)
 
 
 def find_preos_plugins():
@@ -102,7 +102,7 @@ class PreOS:
         parser.add_argument('remainder_args', nargs=argparse.REMAINDER)
         args = parser.parse_args(argv[1:])
         cdist.argparse.handle_loglevel(args)
-        log.debug("preos args : {}".format(args))
+        log.debug("preos args : %s", args)
 
         conf_dirs = util.resolve_conf_dirs_from_config_and_args(args)
 
@@ -122,7 +122,7 @@ class PreOS:
                 func_args = [preos, args.remainder_args, ]
             else:
                 func_args = [args.remainder_args, ]
-            log.info("Running preos : {}".format(preos_name))
+            log.info("Running preos : %s", preos_name)
             func(*func_args)
         else:
             raise cdist.Error(
diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py
index a20cdb9c..f1f750ee 100644
--- a/cdist/preos/debootstrap/debootstrap.py
+++ b/cdist/preos/debootstrap/debootstrap.py
@@ -166,7 +166,7 @@ class Debian:
             args.pxe_boot_dir = os.path.realpath(args.pxe_boot_dir)
 
         cdist.argparse.handle_loglevel(args)
-        log.debug("preos: {}, args: {}".format(cls._preos_name, args))
+        log.debug("preos: %s, args: %s", cls._preos_name, args)
         try:
             env = vars(args)
             new_env = {}
@@ -190,27 +190,30 @@ class Debian:
             env = new_env
             env.update(os.environ)
             cls.update_env(env)
-            log.debug("preos: {} env: {}".format(cls._preos_name, env))
+            log.debug("preos: %s env: %s", cls._preos_name, env)
+
+            if log.getEffectiveLevel() <= logging.INFO:
+                info_msg = ["Running preos: {}, suite: {}, arch: {}".format(
+                    cls._preos_name, args.suite, args.arch), ]
+                if args.mirror:
+                    info_msg.append("mirror: {}".format(args.mirror))
+                if args.script:
+                    info_msg.append("script: {}".format(args.script))
+                if args.bootstrap:
+                    info_msg.append("bootstrapping")
+                if args.configure:
+                    info_msg.append("configuring")
+                if args.pxe_boot_dir:
+                    info_msg.append("creating PXE")
+                if args.drive:
+                    info_msg.append("creating bootable drive")
+                log.info(info_msg)
+
             cmd = os.path.join(cls._files_dir, "code")
-            info_msg = ["Running preos: {}, suite: {}, arch: {}".format(
-                cls._preos_name, args.suite, args.arch), ]
-            if args.mirror:
-                info_msg.append("mirror: {}".format(args.mirror))
-            if args.script:
-                info_msg.append("script: {}".format(args.script))
-            if args.bootstrap:
-                info_msg.append("bootstrapping")
-            if args.configure:
-                info_msg.append("configuring")
-            if args.pxe_boot_dir:
-                info_msg.append("creating PXE")
-            if args.drive:
-                info_msg.append("creating bootable drive")
-            log.info(info_msg)
-            log.debug("cmd={}".format(cmd))
+            log.debug("cmd=%s", cmd)
             subprocess.check_call(cmd, env=env, shell=True)
         except subprocess.CalledProcessError as e:
-            log.error("preos {} failed: {}".format(cls._preos_name, e))
+            log.error("preos %s failed: %s", cls._preos_name, e)
 
 
 class Ubuntu(Debian):
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index b1d0e9e1..9a0bcc52 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -94,7 +94,7 @@ class Trigger(object):
 
     def trigger(self, interface):
         packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
-        log.debug(f"Sending request on {interface}")
+        log.debug("Sending request on %s", interface)
         send(packet, verbose=self.verbose)
 
 
@@ -114,7 +114,7 @@ class Scanner(object):
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
             host = pkg['IPv6'].src
-            log.verbose(f"Host {host} is alive")
+            log.verbose("Host %s is alive", host)
 
             dir = os.path.join(self.outdir, host)
             fname = os.path.join(dir, "last_seen")
diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py
index 95ca74ee..d9e5f498 100644
--- a/cdist/util/ipaddr.py
+++ b/cdist/util/ipaddr.py
@@ -42,8 +42,7 @@ def resolve_target_host_name(host, family=0):
         # gethostbyaddr returns triple
         # (hostname, aliaslist, ipaddrlist)
         host_name = socket.gethostbyaddr(ip_addr)[0]
-        log.debug("derived host_name for host \"{}\": {}".format(
-            host, host_name))
+        log.debug("derived host_name for host \"%s\": %s", host, host_name)
     except (socket.gaierror, socket.herror) as e:
         # in case of error provide empty value
         host_name = ''
@@ -54,8 +53,7 @@ def resolve_target_fqdn(host):
     log = logging.getLogger(host)
     try:
         host_fqdn = socket.getfqdn(host)
-        log.debug("derived host_fqdn for host \"{}\": {}".format(
-            host, host_fqdn))
+        log.debug("derived host_fqdn for host \"%s\": %s", host, host_fqdn)
     except socket.herror as e:
         # in case of error provide empty value
         host_fqdn = ''

From 4c2d273f07e6cedb3dcb539d132cb0c3f8176d42 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 30 Mar 2021 07:56:38 +0200
Subject: [PATCH 286/411] Unify string formatting

Use one way of string formatting: replace old `%` style with new `str.format`.

Resolve #855.
---
 cdist/config.py                     |  4 ++--
 cdist/core/cdist_object.py          | 14 +++++++-------
 cdist/core/cdist_type.py            |  4 ++--
 cdist/core/code.py                  |  8 ++++----
 cdist/core/explorer.py              |  6 +++---
 cdist/core/manifest.py              | 11 +++++------
 cdist/emulator.py                   | 12 ++++++------
 cdist/exec/local.py                 | 11 ++++++-----
 cdist/exec/remote.py                | 11 +++++------
 cdist/message.py                    |  2 +-
 cdist/scan/scan.py                  |  2 +-
 cdist/shell.py                      |  2 +-
 cdist/test/cdist_object/__init__.py | 27 +++++++++++++++------------
 cdist/test/emulator/__init__.py     |  4 ++--
 cdist/test/exec/remote.py           |  8 ++++----
 cdist/test/message/__init__.py      |  2 +-
 cdist/util/fsproperty.py            |  4 ++--
 17 files changed, 67 insertions(+), 65 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index d6fec55f..adc460de 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -190,7 +190,7 @@ class Config:
                     fd.write(sys.stdin.read())
             except (IOError, OSError) as e:
                 raise cdist.Error(("Creating tempfile for stdin data "
-                                   "failed: %s" % e))
+                                   "failed: {}").format(e))
 
             args.manifest = initial_manifest_temp_path
             atexit.register(lambda: os.remove(initial_manifest_temp_path))
@@ -764,7 +764,7 @@ class Config:
 
             raise cdist.UnresolvableRequirementsError(
                     ("The requirements of the following objects could not be "
-                     "resolved:\n%s") % ("\n".join(info_string)))
+                     "resolved:\n{}").format("\n".join(info_string)))
 
     def _handle_deprecation(self, cdist_object):
         cdist_type = cdist_object.cdist_type
diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index eea2c255..ccf7392d 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -34,17 +34,17 @@ class IllegalObjectIdError(cdist.Error):
         self.message = message or 'Illegal object id'
 
     def __str__(self):
-        return '%s: %s' % (self.message, self.object_id)
+        return '{}: {}'.format(self.message, self.object_id)
 
 
 class MissingObjectIdError(cdist.Error):
     def __init__(self, type_name):
         self.type_name = type_name
-        self.message = ("Type %s requires object id (is not a "
-                        "singleton type)") % self.type_name
+        self.message = ("Type {} requires object id (is not a "
+                        "singleton type)").format(self.type_name)
 
     def __str__(self):
-        return '%s' % (self.message)
+        return '{}'.format(self.message)
 
 
 class CdistObject:
@@ -142,7 +142,7 @@ class CdistObject:
             if self.object_marker in self.object_id.split(os.sep):
                 raise IllegalObjectIdError(
                         self.object_id, ('object_id may not contain '
-                                         '\'%s\'') % self.object_marker)
+                                         '\'{}\'').format(self.object_marker))
             if '//' in self.object_id:
                 raise IllegalObjectIdError(
                         self.object_id, 'object_id may not contain //')
@@ -189,7 +189,7 @@ class CdistObject:
                               object_id=object_id)
 
     def __repr__(self):
-        return '<CdistObject %s>' % self.name
+        return '<CdistObject {}>'.format(self.name)
 
     def __eq__(self, other):
         """define equality as 'name is the same'"""
@@ -277,7 +277,7 @@ class CdistObject:
                 os.makedirs(path, exist_ok=allow_overwrite)
         except EnvironmentError as error:
             raise cdist.Error(('Error creating directories for cdist object: '
-                               '%s: %s') % (self, error))
+                               '{}: {}').format(self, error))
 
     def requirements_unfinished(self, requirements):
         """Return state whether requirements are satisfied"""
diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py
index 274de989..b68448bc 100644
--- a/cdist/core/cdist_type.py
+++ b/cdist/core/cdist_type.py
@@ -34,7 +34,7 @@ class InvalidTypeError(cdist.Error):
         self.source_path = os.path.realpath(self.type_absolute_path)
 
     def __str__(self):
-        return "Invalid type '%s' at '%s' defined at '%s'" % (
+        return "Invalid type '{}' at '{}' defined at '{}'".format(
                 self.type_path, self.type_absolute_path, self.source_path)
 
 
@@ -109,7 +109,7 @@ class CdistType:
         return cls._instances[name]
 
     def __repr__(self):
-        return '<CdistType %s>' % self.name
+        return '<CdistType {}>'.format(self.name)
 
     def __eq__(self, other):
         return isinstance(other, self.__class__) and self.name == other.name
diff --git a/cdist/core/code.py b/cdist/core/code.py
index 1e9b4f80..12888ba4 100644
--- a/cdist/core/code.py
+++ b/cdist/core/code.py
@@ -122,8 +122,8 @@ class Code:
 
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.cdist_type
-        script = os.path.join(self.local.type_path,
-                              getattr(cdist_type, 'gencode_%s_path' % which))
+        gencode_attr = getattr(cdist_type, 'gencode_{}_path'.format(which))
+        script = os.path.join(self.local.type_path, gencode_attr)
         if os.path.isfile(script):
             env = os.environ.copy()
             env.update(self.env)
@@ -167,8 +167,8 @@ class Code:
 
     def _run_code(self, cdist_object, which, env=None):
         which_exec = getattr(self, which)
-        script = os.path.join(which_exec.object_path,
-                              getattr(cdist_object, 'code_%s_path' % which))
+        code_attr = getattr(cdist_object, 'code_{}_path'.format(which))
+        script = os.path.join(which_exec.object_path, code_attr)
         if which_exec.save_output_streams:
             stderr_path = os.path.join(cdist_object.stderr_path,
                                        'code-' + which)
diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py
index caa12a7d..48414ade 100644
--- a/cdist/core/explorer.py
+++ b/cdist/core/explorer.py
@@ -160,8 +160,8 @@ class Explorer:
         self.remote.transfer(self.local.global_explorer_path,
                              self.remote.global_explorer_path,
                              self.jobs)
-        self.remote.run(["chmod", "0700",
-                         "%s/*" % (self.remote.global_explorer_path)])
+        self.remote.run(["chmod", "0700", "{}/*".format(
+            self.remote.global_explorer_path)])
 
     def run_global_explorer(self, explorer):
         """Run the given global explorer and return it's output."""
@@ -242,7 +242,7 @@ class Explorer:
                 destination = os.path.join(self.remote.type_path,
                                            cdist_type.explorer_path)
                 self.remote.transfer(source, destination)
-                self.remote.run(["chmod", "0700", "%s/*" % (destination)])
+                self.remote.run(["chmod", "0700", "{}/*".format(destination)])
                 self._type_explorers_transferred.append(cdist_type.name)
 
     def transfer_object_parameters(self, cdist_object):
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 3148d66c..09e74dac 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -80,13 +80,12 @@ class NoInitialManifestError(cdist.Error):
 
         if user_supplied:
             if os.path.islink(manifest_path):
-                self.message = "%s: %s -> %s" % (
-                        msg_header, manifest_path,
-                        os.path.realpath(manifest_path))
+                self.message = "{}: {} -> {}".format(
+                    msg_header, manifest_path, os.path.realpath(manifest_path))
             else:
-                self.message = "%s: %s" % (msg_header, manifest_path)
+                self.message = "{}: {}".format(msg_header, manifest_path)
         else:
-            self.message = "%s" % (msg_header)
+            self.message = "{}".format(msg_header)
 
     def __str__(self):
         return repr(self.message)
@@ -107,7 +106,7 @@ class Manifest:
         self._open_logger()
 
         self.env = {
-            'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
+            'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']),
             # for use in type emulator
             '__cdist_type_base_path': self.local.type_path,
             '__global': self.local.base_path,
diff --git a/cdist/emulator.py b/cdist/emulator.py
index f09c282d..c55b47d2 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -36,8 +36,8 @@ from cdist.core.manifest import Manifest
 class MissingRequiredEnvironmentVariableError(cdist.Error):
     def __init__(self, name):
         self.name = name
-        self.message = ("Emulator requires the environment variable %s to be "
-                        "setup" % self.name)
+        self.message = ("Emulator requires the environment variable {} to be "
+                        "setup").format(self.name)
 
     def __str__(self):
         return self.message
@@ -231,13 +231,13 @@ class Emulator:
         if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env:
             obj_params = self._object_params_in_context()
             if obj_params != self.parameters:
-                errmsg = ("Object %s already exists with conflicting "
-                          "parameters:\n%s: %s\n%s: %s" % (
+                errmsg = ("Object {} already exists with conflicting "
+                          "parameters:\n{}: {}\n{}: {}").format(
                               self.cdist_object.name,
                               " ".join(self.cdist_object.source),
                               obj_params,
                               self.object_source,
-                              self.parameters))
+                              self.parameters)
                 raise cdist.Error(errmsg)
         else:
             if self.cdist_object.exists:
@@ -292,7 +292,7 @@ class Emulator:
                         fd.write(chunk)
                         chunk = self._read_stdin()
             except EnvironmentError as e:
-                raise cdist.Error('Failed to read from stdin: %s' % e)
+                raise cdist.Error('Failed to read from stdin: {}'.format(e))
 
     def record_requirement(self, requirement):
         """record requirement and return recorded requirement"""
diff --git a/cdist/exec/local.py b/cdist/exec/local.py
index 6713cd13..370336ad 100644
--- a/cdist/exec/local.py
+++ b/cdist/exec/local.py
@@ -152,7 +152,7 @@ class Local:
 
     def _setup_object_marker_file(self):
         with open(self.object_marker_file, 'w') as fd:
-            fd.write("%s\n" % self.object_marker_name)
+            fd.write("{}\n".format(self.object_marker_name))
 
         self.log.trace("Object marker %s saved in %s",
                        self.object_marker_name, self.object_marker_file)
@@ -184,7 +184,7 @@ class Local:
 
         """
         assert isinstance(command, (list, tuple)), (
-                "list or tuple argument expected, got: %s" % command)
+                "list or tuple argument expected, got: {}".format(command))
 
         quiet = self.quiet_mode or quiet_mode
         do_save_output = save_output and not quiet and self.save_output_streams
@@ -352,8 +352,9 @@ class Local:
                     try:
                         os.symlink(src, dst)
                     except OSError as e:
-                        raise cdist.Error("Linking %s %s to %s failed: %s" % (
-                            sub_dir, src, dst, e.__str__()))
+                        raise cdist.Error(
+                            "Linking {} {} to {} failed: {}".format(
+                                sub_dir, src, dst, e.__str__()))
 
     def _link_types_for_emulator(self):
         """Link emulator to types"""
@@ -366,5 +367,5 @@ class Local:
                 os.symlink(src, dst)
             except OSError as e:
                 raise cdist.Error(
-                        "Linking emulator from %s to %s failed: %s" % (
+                        "Linking emulator from {} to {} failed: {}".format(
                             src, dst, e.__str__()))
diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py
index ea85da4c..af9e70cb 100644
--- a/cdist/exec/remote.py
+++ b/cdist/exec/remote.py
@@ -24,12 +24,10 @@ import os
 import glob
 import subprocess
 import logging
-import multiprocessing
 
 import cdist
 import cdist.exec.util as util
 import cdist.util.ipaddr as ipaddr
-from cdist.mputil import mp_pool_run
 
 
 def _wrap_addr(addr):
@@ -262,9 +260,10 @@ class Remote:
         # remotely in e.g. csh and setting up CDIST_REMOTE_SHELL to e.g.
         # /bin/csh will execute this script in the right way.
         if env:
-            remote_env = [" export %s=%s;" % item for item in env.items()]
-            string_cmd = ("/bin/sh -c '" + " ".join(remote_env) +
-                          " ".join(command) + "'")
+            remote_env = [" export {env[0]}={env[1]};".format(env=item)
+                          for item in env.items()]
+            string_cmd = ("/bin/sh -c '{}{}'").format(" ".join(remote_env),
+                                                      " ".join(command))
             cmd.append(string_cmd)
         else:
             cmd.extend(command)
@@ -278,7 +277,7 @@ class Remote:
 
         """
         assert isinstance(command, (list, tuple)), (
-                "list or tuple argument expected, got: %s" % command)
+                "list or tuple argument expected, got: {}".format(command))
 
         close_stdout = False
         close_stderr = False
diff --git a/cdist/message.py b/cdist/message.py
index ffa8c2bb..0c4e21a6 100644
--- a/cdist/message.py
+++ b/cdist/message.py
@@ -70,7 +70,7 @@ class Message:
 
         with open(self.global_messages, 'a') as fd:
             for line in content:
-                fd.write("%s:%s" % (self.prefix, line))
+                fd.write("{}:{}".format(self.prefix, line))
 
     def merge_messages(self):
         self._merge_messages()
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 9a0bcc52..faee8a56 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -93,7 +93,7 @@ class Trigger(object):
             time.sleep(self.sleeptime)
 
     def trigger(self, interface):
-        packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
+        packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest()
         log.debug("Sending request on %s", interface)
         send(packet, verbose=self.verbose)
 
diff --git a/cdist/shell.py b/cdist/shell.py
index 04a68937..05803556 100644
--- a/cdist/shell.py
+++ b/cdist/shell.py
@@ -65,7 +65,7 @@ class Shell:
     def _init_environment(self):
         self.env = os.environ.copy()
         additional_env = {
-            'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
+            'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']),
             # for use in type emulator
             '__cdist_type_base_path': self.local.type_path,
             '__cdist_manifest': "cdist shell",
diff --git a/cdist/test/cdist_object/__init__.py b/cdist/test/cdist_object/__init__.py
index a9c20cd3..11619e96 100644
--- a/cdist/test/cdist_object/__init__.py
+++ b/cdist/test/cdist_object/__init__.py
@@ -86,8 +86,7 @@ class ObjectClassTestCase(test.CdistTestCase):
 
     def test_create_singleton(self):
         """Check whether creating an object without id (singleton) works"""
-        singleton = self.expected_objects[0].object_from_name(
-                "__test_singleton")
+        self.expected_objects[0].object_from_name("__test_singleton")
         # came here - everything fine
 
     def test_create_singleton_not_singleton_type(self):
@@ -126,16 +125,16 @@ class ObjectIdTestCase(test.CdistTestCase):
 
     def test_object_id_contains_object_marker(self):
         cdist_type = core.CdistType(type_base_path, '__third')
-        illegal_object_id = (
-            'object_id/may/not/contain/%s/anywhere' % OBJECT_MARKER_NAME)
+        illegal_object_id = 'object_id/may/not/contain/{}/anywhere'.format(
+            OBJECT_MARKER_NAME)
         with self.assertRaises(core.IllegalObjectIdError):
             core.CdistObject(cdist_type, self.object_base_path,
                              OBJECT_MARKER_NAME, illegal_object_id)
 
     def test_object_id_contains_object_marker_string(self):
         cdist_type = core.CdistType(type_base_path, '__third')
-        illegal_object_id = (
-            'object_id/may/contain_%s_in_filename' % OBJECT_MARKER_NAME)
+        illegal_object_id = 'object_id/may/contain_{}_in_filename'.format(
+            OBJECT_MARKER_NAME)
         core.CdistObject(cdist_type, self.object_base_path,
                          OBJECT_MARKER_NAME, illegal_object_id)
         # if we get here, the test passed
@@ -195,28 +194,32 @@ class ObjectTestCase(test.CdistTestCase):
 
     def test_path(self):
         self.assertEqual(self.cdist_object.path,
-                         "__third/moon/%s" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}".format(OBJECT_MARKER_NAME))
 
     def test_absolute_path(self):
         self.assertEqual(self.cdist_object.absolute_path,
                          os.path.join(self.object_base_path,
-                                      "__third/moon/%s" % OBJECT_MARKER_NAME))
+                                      "__third/moon/{}".format(
+                                          OBJECT_MARKER_NAME)))
 
     def test_code_local_path(self):
         self.assertEqual(self.cdist_object.code_local_path,
-                         "__third/moon/%s/code-local" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/code-local".format(
+                             OBJECT_MARKER_NAME))
 
     def test_code_remote_path(self):
         self.assertEqual(self.cdist_object.code_remote_path,
-                         "__third/moon/%s/code-remote" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/code-remote".format(
+                             OBJECT_MARKER_NAME))
 
     def test_parameter_path(self):
         self.assertEqual(self.cdist_object.parameter_path,
-                         "__third/moon/%s/parameter" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/parameter".format(
+                             OBJECT_MARKER_NAME))
 
     def test_explorer_path(self):
         self.assertEqual(self.cdist_object.explorer_path,
-                         "__third/moon/%s/explorer" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/explorer".format(OBJECT_MARKER_NAME))
 
     def test_parameters(self):
         expected_parameters = {'planet': 'Saturn', 'name': 'Prometheus'}
diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py
index befd7b57..4b2bc3ba 100644
--- a/cdist/test/emulator/__init__.py
+++ b/cdist/test/emulator/__init__.py
@@ -84,8 +84,8 @@ class EmulatorTestCase(test.CdistTestCase):
 
     def test_illegal_object_id_requirement(self):
         argv = ['__file', '/tmp/foobar']
-        self.env['require'] = (
-                "__file/bad/id/with/%s/inside") % self.local.object_marker_name
+        self.env['require'] = "__file/bad/id/with/{}/inside".format(
+            self.local.object_marker_name)
         emu = emulator.Emulator(argv, env=self.env)
         self.assertRaises(core.IllegalObjectIdError, emu.run)
 
diff --git a/cdist/test/exec/remote.py b/cdist/test/exec/remote.py
index a7fe384d..b23f8447 100644
--- a/cdist/test/exec/remote.py
+++ b/cdist/test/exec/remote.py
@@ -47,8 +47,8 @@ class RemoteTestCase(test.CdistTestCase):
             args = (self.target_host,)
         kwargs.setdefault('base_path', self.base_path)
         user = getpass.getuser()
-        kwargs.setdefault('remote_exec', 'ssh -o User=%s -q' % user)
-        kwargs.setdefault('remote_copy', 'scp -o User=%s -q' % user)
+        kwargs.setdefault('remote_exec', 'ssh -o User={} -q'.format(user))
+        kwargs.setdefault('remote_copy', 'scp -o User={} -q'.format(user))
         if 'stdout_base_path' not in kwargs:
             stdout_path = os.path.join(self.temp_dir, 'stdout')
             os.makedirs(stdout_path, exist_ok=True)
@@ -170,7 +170,7 @@ class RemoteTestCase(test.CdistTestCase):
         r = self.create_remote(remote_exec=remote_exec,
                                remote_copy=remote_copy)
         self.assertEqual(r.run('true', return_output=True),
-                         "%s\n" % self.target_host[0])
+                         "{}\n".format(self.target_host[0]))
 
     def test_run_script_target_host_in_env(self):
         handle, remote_exec_path = self.mkstemp(dir=self.temp_dir)
@@ -185,7 +185,7 @@ class RemoteTestCase(test.CdistTestCase):
         with os.fdopen(handle, "w") as fd:
             fd.writelines(["#!/bin/sh\n", "true"])
         self.assertEqual(r.run_script(script, return_output=True),
-                         "%s\n" % self.target_host[0])
+                         "{}\n".format(self.target_host[0]))
 
     def test_run_script_with_env_target_host_in_env(self):
         handle, script = self.mkstemp(dir=self.temp_dir)
diff --git a/cdist/test/message/__init__.py b/cdist/test/message/__init__.py
index 61cd5d97..55040fc9 100644
--- a/cdist/test/message/__init__.py
+++ b/cdist/test/message/__init__.py
@@ -67,7 +67,7 @@ class MessageTestCase(test.CdistTestCase):
     def test_message_merge_prefix(self):
         """Ensure messages are merged and are prefixed"""
 
-        expectedcontent = "%s:%s" % (self.prefix, self.content)
+        expectedcontent = "{}:{}".format(self.prefix, self.content)
 
         out = self.message.env['__messages_out']
 
diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py
index 1d76fd76..09e9cc19 100644
--- a/cdist/util/fsproperty.py
+++ b/cdist/util/fsproperty.py
@@ -30,7 +30,7 @@ class AbsolutePathRequiredError(cdist.Error):
         self.path = path
 
     def __str__(self):
-        return 'Absolute path required, got: %s' % self.path
+        return 'Absolute path required, got: {}'.format(self.path)
 
 
 class FileList(collections.MutableSequence):
@@ -218,7 +218,7 @@ class FileBasedProperty:
 
     def _get_attribute(self, instance, owner):
         name = self._get_property_name(owner)
-        attribute_name = '__%s' % name
+        attribute_name = '__{}'.format(name)
         if not hasattr(instance, attribute_name):
             path = self._get_path(instance)
             attribute_instance = self.attribute_class(path)

From ab811ad282882558a4639986e75c800e1b63c2e0 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 1 Apr 2021 15:37:11 +0200
Subject: [PATCH 287/411] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 03ae7566..db60c0c4 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,8 @@ next:
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
 	* Type __sshd_config: Whitelist OpenBMC (Dennis Camera)
 	* Core: Maintain object relationship graph in cdist cache (Darko Poljak)
+	* Type __git: Fix numeric owner and group handling (Dennis Camera)
+	* Type __pyvenv: Fix numeric owner and group handling (Dennis Camera)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From 199effb7ef23980bc69f0233ff6d7ee465623f40 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 6 Apr 2021 19:35:14 +0200
Subject: [PATCH 288/411] Improve unfinished object requirements bool check

When we need only boolean value for unfinished object requirements then
we don't need to determine the whole list of unfinished objects.
---
 cdist/config.py            | 12 +++++++-----
 cdist/core/cdist_object.py | 13 ++++++++++++-
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index adc460de..f7da9b1d 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -537,7 +537,7 @@ class Config:
         objects_changed = False
 
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(
+            if cdist_object.has_requirements_unfinished(
                     cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
@@ -548,7 +548,7 @@ class Config:
                 self.object_prepare(cdist_object)
                 objects_changed = True
 
-            if cdist_object.requirements_unfinished(
+            if cdist_object.has_requirements_unfinished(
                     cdist_object.autorequire):
                 """The previous step created objects we depend on -
                     wait for them
@@ -567,7 +567,8 @@ class Config:
 
         cargo = []
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(cdist_object.requirements):
+            if cdist_object.has_requirements_unfinished(
+                    cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
 
@@ -621,12 +622,13 @@ class Config:
 
         del cargo[:]
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(cdist_object.requirements):
+            if cdist_object.has_requirements_unfinished(
+                    cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
 
             if cdist_object.state == core.CdistObject.STATE_PREPARED:
-                if cdist_object.requirements_unfinished(
+                if cdist_object.has_requirements_unfinished(
                         cdist_object.autorequire):
                     """The previous step created objects we depend on -
                     wait for them
diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index ccf7392d..bb3a65bd 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -280,7 +280,7 @@ class CdistObject:
                                '{}: {}').format(self, error))
 
     def requirements_unfinished(self, requirements):
-        """Return state whether requirements are satisfied"""
+        """Return unsatisfied requirements"""
 
         object_list = []
 
@@ -291,3 +291,14 @@ class CdistObject:
                 object_list.append(cdist_object)
 
         return object_list
+
+    def has_requirements_unfinished(self, requirements):
+        """Return whether requirements are satisfied"""
+
+        for requirement in requirements:
+            cdist_object = self.object_from_name(requirement)
+
+            if cdist_object.state != self.STATE_DONE:
+                return True
+
+        return False

From 750c71fb5a9fcf4a30af0700b7a09e3501988a27 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 7 Apr 2021 09:07:29 +0200
Subject: [PATCH 289/411] Minor refactoring and remove code duplication

---
 cdist/config.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index f7da9b1d..638fdf0e 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -699,20 +699,22 @@ class Config:
             check for cycles.
         '''
         graph = {}
-        for cdist_object in self.object_list():
+
+        def _add_requirements(cdist_object, requirements):
             obj_name = cdist_object.name
             if obj_name not in graph:
                 graph[obj_name] = []
+
+            for requirement in cdist_object.requirements_unfinished(
+                    requirements):
+                graph[obj_name].append(requirement.name)
+
+        for cdist_object in self.object_list():
             if cdist_object.state == cdist_object.STATE_DONE:
                 continue
 
-            for requirement in cdist_object.requirements_unfinished(
-                    cdist_object.requirements):
-                graph[obj_name].append(requirement.name)
-
-            for requirement in cdist_object.requirements_unfinished(
-                    cdist_object.autorequire):
-                graph[obj_name].append(requirement.name)
+            _add_requirements(cdist_object, cdist_object.requirements)
+            _add_requirements(cdist_object, cdist_object.autorequire)
         return graph_check_cycle(graph)
 
     def iterate_until_finished(self):

From d2eec6066876eeb4a295b81a670c70b0ff8ad2a7 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 11 Apr 2021 23:05:48 +0300
Subject: [PATCH 290/411] [__download] make --sum optional

---
 cdist/conf/type/__download/explorer/state     |  6 ++++++
 cdist/conf/type/__download/man.rst            | 15 ++++++---------
 cdist/conf/type/__download/parameter/optional |  1 +
 cdist/conf/type/__download/parameter/required |  1 -
 4 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 00362545..68b517c5 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -8,6 +8,12 @@ then
     exit 0
 fi
 
+if [ ! -f "$__object/parameter/sum" ]
+then
+    echo 'present'
+    exit 0
+fi
+
 sum_should="$( cat "$__object/parameter/sum" )"
 
 if [ -f "$__object/parameter/cmd-sum" ]
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index 54503470..a1278cfb 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -8,9 +8,6 @@ cdist-type__download - Download a file
 
 DESCRIPTION
 -----------
-Destination (``$__object_id``) in target host must be persistent storage
-in order to calculate checksum and decide if file must be (re-)downloaded.
-
 By default type will try to use ``wget``, ``curl`` or ``fetch``.
 If download happens in target (see ``--download``) then type will
 fallback to (and install) ``wget``.
@@ -25,14 +22,14 @@ REQUIRED PARAMETERS
 url
    File's URL.
 
-sum
-   Checksum of file going to be downloaded.
-   By default output of ``cksum`` without filename is expected.
-   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
-
 
 OPTIONAL PARAMETERS
 -------------------
+sum
+   Checksum is used to decide if existing destination file must be redownloaded.
+   By default output of ``cksum`` without filename is expected.
+   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
+
 download
    If ``local`` (default), then download file to local storage and copy
    it to target host. If ``remote``, then download happens in target.
@@ -81,7 +78,7 @@ Ander Punnar <ander-at-kvlt-dot-ee>
 
 COPYING
 -------
-Copyright \(C) 2020 Ander Punnar. You can redistribute it
+Copyright \(C) 2021 Ander Punnar. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index 838e2fbf..d69e083e 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,3 +1,4 @@
+sum
 cmd-get
 cmd-sum
 download
diff --git a/cdist/conf/type/__download/parameter/required b/cdist/conf/type/__download/parameter/required
index 6ea4c38f..96cdd3b9 100644
--- a/cdist/conf/type/__download/parameter/required
+++ b/cdist/conf/type/__download/parameter/required
@@ -1,2 +1 @@
 url
-sum

From 9ec01d9f971339ee67c866c4a248047966f73ce1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 13 Apr 2021 12:22:45 +0200
Subject: [PATCH 291/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index db60c0c4..c8cb44da 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,7 @@ next:
 	* Core: Maintain object relationship graph in cdist cache (Darko Poljak)
 	* Type __git: Fix numeric owner and group handling (Dennis Camera)
 	* Type __pyvenv: Fix numeric owner and group handling (Dennis Camera)
+	* Type __download: Make sum parameter optional (Ander Punnar)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From 92b8942a8c37bb985ef42991f3b7aaef0cd8073f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 09:00:32 +0200
Subject: [PATCH 292/411] [type/__postgres_conf] Add psql_exec function to
 state explorer

---
 cdist/conf/type/__postgres_conf/explorer/state | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index de6e8aa0..1a58751a 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -20,21 +20,24 @@
 #
 
 postgres_user=$("${__type_explorer:?}/postgres_user")
-
 conf_name=${__object_id:?}
 
-su - "${postgres_user}" -c 'psql postgres -c "SELECT 1"' >/dev/null || {
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_exec() {
+	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
+}
+
+psql_exec 'SELECT 1' >/dev/null || {
 	echo 'Connection to PostgreSQL server failed' >&2
 	exit 1
 }
 
-if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
-	| cmp -s "${__object:?}/parameter/value" -
+if psql_exec "SHOW ${conf_name}" | cmp -s "${__object:?}/parameter/value" -
 then
 	echo present
 else
 	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
-	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"")
+	case $(psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')")
 	in
 		('')
 			# invalid configuration parameter

From 2ccc03fef1e28c759a7918fa00ffcb45c76dc57a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 09:02:40 +0200
Subject: [PATCH 293/411] [type/__postgres_conf] Add psql_conf_cmp function to
 state explorer

---
 cdist/conf/type/__postgres_conf/explorer/state | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 1a58751a..be881be6 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -27,12 +27,16 @@ psql_exec() {
 	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
 }
 
+psql_conf_cmp() {
+	test "$(psql_exec "SHOW $1")" = "$2"
+}
+
 psql_exec 'SELECT 1' >/dev/null || {
 	echo 'Connection to PostgreSQL server failed' >&2
 	exit 1
 }
 
-if psql_exec "SHOW ${conf_name}" | cmp -s "${__object:?}/parameter/value" -
+if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
 then
 	echo present
 else

From e0416403c4862cb8bc0f41a7c6f8a33d1a8656a5 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 09:04:07 +0200
Subject: [PATCH 294/411] [type/__postgres_conf] Add psql_conf_source function
 to state explorer

---
 cdist/conf/type/__postgres_conf/explorer/state | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index be881be6..9f54724a 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -27,6 +27,10 @@ psql_exec() {
 	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
 }
 
+psql_conf_source() {
+	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
+	psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')"
+}
 psql_conf_cmp() {
 	test "$(psql_exec "SHOW $1")" = "$2"
 }
@@ -40,8 +44,7 @@ if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
 then
 	echo present
 else
-	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
-	case $(psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')")
+	case $(psql_conf_source "${conf_name}")
 	in
 		('')
 			# invalid configuration parameter

From 12c2995494ce582cb3adf1d942b36f6df035370a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 14:46:36 +0200
Subject: [PATCH 295/411] [type/__postgres_conf] Implement complex state
 compare logic

---
 .../conf/type/__postgres_conf/explorer/state  | 166 +++++++++++++++++-
 1 file changed, 163 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 9f54724a..e76540c5 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -22,6 +22,52 @@
 postgres_user=$("${__type_explorer:?}/postgres_user")
 conf_name=${__object_id:?}
 
+tolower() { printf '%s' "$*" | tr '[:upper:]' '[:lower:]'; }
+
+tobytes() {
+	# NOTE: This function treats everything as base 2.
+	#       It is not compatible with SI units.
+	awk 'BEGIN { FS = "\n" }
+	/TB$/ { $0 = ($0 * 1024) "GB" }
+	/GB$/ { $0 = ($0 * 1024) "MB" }
+	/MB$/ { $0 = ($0 * 1024) "kB" }
+	/kB$/ { $0 = ($0 * 1024) "B" }
+	 /B?$/ { sub(/ *B?$/, "") }
+	($0*1) == $0  # is number
+	' <<-EOF
+	$1
+	EOF
+}
+
+tomillisecs() {
+	awk 'BEGIN { FS = "\n" }
+	    /d$/ { $0 = ($0 * 24) "h" }
+	    /h$/ { $0 = ($0 * 60) "min" }
+	  /min$/ { $0 = ($0 * 60) "s" }
+	/[^m]s$/ { $0 = ($0 * 1000) "ms" }
+	   /ms$/ { $0 *= 1 }
+	($0*1) == $0  # is number
+	' <<-EOF
+	$1
+	EOF
+}
+
+tobool() {
+	# prints either 'on' or 'off'
+	case $(tolower "$1")
+	in
+		(t|true|y|yes|on|1)
+			echo 'on' ;;
+		(f|false|n|no|off|0)
+			echo 'off' ;;
+		(*)
+			printf 'Inavlid bool value: %s\n' "$2" >&2
+			return 1
+			;;
+	esac
+	return 0
+}
+
 quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
 psql_exec() {
 	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
@@ -31,9 +77,123 @@ psql_conf_source() {
 	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
 	psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')"
 }
-psql_conf_cmp() {
-	test "$(psql_exec "SHOW $1")" = "$2"
-}
+psql_conf_cmp() (
+	IFS='|' read -r lower_name vartype setting unit <<-EOF
+	$(psql_exec "SELECT lower(name), vartype, setting, unit FROM pg_settings WHERE lower(name) = lower('$1')")
+	EOF
+
+	should_value=$2
+	is_value=${setting}
+
+	# The following case contains special cases for special settings.
+	case ${lower_name}
+	in
+		(archive_command)
+			if test "${setting}" = '(disabled)'
+			then
+				# DAFUQ PostgreSQL?!
+				# PostgreSQL returns (disabled) if the feature is inactive.
+				# We cannot compare the values unless it is enabled, first.
+				return 0
+			fi
+			;;
+		(archive_mode|backslash_quote|constraint_exclusion|force_parallel_mode|huge_pages|synchronous_commit)
+			# Although only 'on', 'off' are documented, PostgreSQL accepts all
+			# the "likely" variants of "on" and "off".
+			case $(tolower "${should_value}")
+			in
+				(on|off|true|false|yes|no|1|0)
+					should_value=$(tobool "${should_value}")
+					;;
+			esac
+			;;
+	esac
+
+	case ${vartype}
+	in
+		(bool)
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported for vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			should_value=$(tobool "${should_value}")
+
+			test "${is_value}" = "${should_value}"
+			;;
+		(enum)
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported with vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			# NOTE: All enums that are currently defined are lower case, but
+			#       PostgreSQL also accepts upper case spelling.
+			should_value=$(tolower "$2")
+
+			test "${is_value}" = "${should_value}"
+			;;
+		(integer)
+			# split multiples from unit, first (e.g. 8kB -> 8, kB)
+			case ${unit}
+			in
+				([0-9]*)
+					multiple=${unit%%[!0-9]*}
+					unit=${unit##*[0-9 ]}
+					;;
+				(*) multiple=1 ;;
+			esac
+
+			is_value=$((setting * multiple))${unit}
+
+			if expr "${should_value}" : '-\{0,1\}[0-9]*$' >/dev/null
+			then
+				# default unit
+				should_value=$((should_value * multiple))${unit}
+			fi
+
+			# then, do conversion
+			# NOTE: these conversions work for integers only!
+			case ${unit}
+			in
+				(B|[kMGT]B)
+					# bytes
+					is_bytes=$(tobytes "${is_value}")
+					should_bytes=$(tobytes "${should_value}")
+
+					test $((is_bytes)) -eq $((should_bytes))
+					;;
+				(ms|s|min|h|d)
+					# seconds
+					is_ms=$(tomillisecs "${is_value}")
+					should_ms=$(tomillisecs "${should_value}")
+
+					test $((is_ms)) -eq $((should_ms))
+					;;
+				('')
+					# no unit
+					is_int=${is_value}
+					should_int=${should_value}
+
+					test $((is_int)) -eq $((should_int))
+					;;
+			esac
+			;;
+		(real|string)
+			# NOTE: reals could possibly have units, but currently there none.
+
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported with vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			test "${is_value}" = "${should_value}"
+			;;
+	esac
+)
 
 psql_exec 'SELECT 1' >/dev/null || {
 	echo 'Connection to PostgreSQL server failed' >&2

From bef1433ba3adb022f8b50eb9df0fbbbf90757d76 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 14:46:50 +0200
Subject: [PATCH 296/411] [type/__postgres_conf] Accept empty values

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index d0d247b4..27651600 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -57,7 +57,7 @@ psql_cmd() {
 case ${state_should}
 in
 	(present)
-		test -s "${__object:?}/parameter/value" || {
+		test -n "${__object:?}/parameter/value" || {
 			echo 'Missing required parameter --value' >&2
 			exit 1
 		}

From 686e4f0f2d498ee39a69e43f7535171f16f5fcbc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 15:01:38 +0200
Subject: [PATCH 297/411] [type/__postgres_conf] Reverse state logic (decide
 based on source first)

---
 .../conf/type/__postgres_conf/explorer/state  | 39 ++++++++++---------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index e76540c5..4b7b0a43 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -200,23 +200,24 @@ psql_exec 'SELECT 1' >/dev/null || {
 	exit 1
 }
 
-if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
-then
-	echo present
-else
-	case $(psql_conf_source "${conf_name}")
-	in
-		('')
-			# invalid configuration parameter
-			# (error message was already printed by SHOW command above.
-			#  Yes, it's a hack)
-			exit 1
-			;;
-		(default)
-			echo absent
-			;;
-		(*)
+case $(psql_conf_source "${conf_name}")
+in
+	('')
+		printf 'Invalid configuration parameter: %s\n' "${conf_name}" >&2
+		exit 1
+		;;
+	(default)
+		echo absent
+		;;
+	(*)
+		if ! test -f "${__object:?}/parameter/value"
+		then
+			echo present
+		elif psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
+		then
+			echo present
+		else
 			echo different
-			;;
-	esac
-fi
+		fi
+		;;
+esac

From 19bf37be1a8b0b099a7199d1565b74598010f614 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 15:55:52 +0200
Subject: [PATCH 298/411] [type/__postgres_conf] Update man.rst

---
 cdist/conf/type/__postgres_conf/man.rst | 39 ++++++++++++++-----------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst
index 76016c6c..e035f080 100644
--- a/cdist/conf/type/__postgres_conf/man.rst
+++ b/cdist/conf/type/__postgres_conf/man.rst
@@ -1,29 +1,27 @@
 cdist-type__postgres_conf(7)
 ============================
 
-Configure a PostgreSQL server.
-
-NOTE: This type might need to be run multiple times to apply all bits of the
-configuration due to ordering requirements.
-
-SSRQ <cdist--@--ssrq-sds-fds.ch>
+NAME
+----
+cdist-type__postgres_conf - Alter PostgreSQL configuration
 
 
 DESCRIPTION
 -----------
-Configure a PostgreSQL server using ALTER SYSTEM.
+Configure a running PostgreSQL server using ``ALTER SYSTEM``.
 
 
 REQUIRED PARAMETERS
 -------------------
 value
-    The value to setup (can be omitted when state is set to "absent").
+   The value to set (can be omitted if ``--state`` is set to ``absent``).
 
 
 OPTIONAL PARAMETERS
 -------------------
 state
-    "present" or "absent". Defaults to "present".
+   ``present`` or ``absent``.
+   Defaults to ``present``.
 
 
 BOOLEAN PARAMETERS
@@ -36,20 +34,27 @@ EXAMPLES
 
 .. code-block:: sh
 
-	# set timezone
-    __postgres_conf timezone --value Europe/Zurich
+   # set timezone
+   __postgres_conf timezone --value Europe/Zurich
 
-    # reset maximum number of concurrent connections to default (normally 100)
-    __postgres_conf max_connections --state absent
+   # reset maximum number of concurrent connections to default (normally 100)
+   __postgres_conf max_connections --state absent
 
 
 SEE ALSO
 --------
-- `cdist-type(7) <cdist-type.html>`_
+None.
+
+
+AUTHORS
+-------
+Beni Ruef (bernhard.ruef--@--ssrq-sds-fds.ch)
+Dennis Camera (dennis.camera--@--ssrq-sds-fds.ch)
 
 
 COPYING
 -------
-Copyright \(C) 2020 SSRQ (www.ssrq-sds-fds.ch).
-Free use of this software is granted under the terms
-of the GNU General Public License version 3 (GPLv3).
+Copyright \(C) 2019-2021 SSRQ (www.ssrq-sds-fds.ch).
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.

From 1c047353a95e7ac079ccf89af8fc232451b8f891 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 17 Apr 2021 09:57:10 +0200
Subject: [PATCH 299/411] [bin/cdist] Fix Python version check

---
 bin/cdist         | 8 +++++---
 cdist/__init__.py | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index ddaffa7f..adb06a8d 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -72,9 +72,11 @@ def commandline():
 
 
 if __name__ == "__main__":
-    if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION:
-        print('Python >= {} is required on the source host.'.format(
-                cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr)
+    if sys.version_info[:3] < cdist.MIN_SUPPORTED_PYTHON_VERSION:
+        print(
+            'Python >= {} is required on the source host.'.format(
+                ".".join(map(str, cdist.MIN_SUPPORTED_PYTHON_VERSION))),
+            file=sys.stderr)
         sys.exit(1)
 
     exit_code = 0
diff --git a/cdist/__init__.py b/cdist/__init__.py
index 44366cd0..31d49889 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -64,7 +64,7 @@ REMOTE_EXEC = "ssh -o User=root"
 REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
 
 
-MIN_SUPPORTED_PYTHON_VERSION = '3.5'
+MIN_SUPPORTED_PYTHON_VERSION = (3, 5)
 
 
 class Error(Exception):

From 1bb696a4100f161afada9b1e8d8ad2b8ab190f54 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 20 Apr 2021 07:33:07 +0200
Subject: [PATCH 300/411] Release 6.9.6

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index c8cb44da..70d0b960 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)

From acf9bf91f17fb4ad0e5813c6e4b0b40fc7e027b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Thu, 22 Apr 2021 08:55:14 +0200
Subject: [PATCH 301/411] [scanner] error to stderr and exit when scapy is not
 available

---
 cdist/scan/commandline.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index eca4cf13..0d7fb0ca 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -20,6 +20,7 @@
 #
 
 import logging
+import sys
 
 log = logging.getLogger("scan")
 
@@ -31,7 +32,9 @@ def commandline(args):
     try:
         import cdist.scan.scan as scan
     except ModuleNotFoundError:
-        print('cdist scan requires scapy to be installed')
+        print('cdist scan requires scapy to be installed! Exiting.',
+                file=sys.stderr)
+        sys.exit(1)
 
     processes = []
 

From a4464209b6741f2c1764aa432c56cfa8128852a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Thu, 22 Apr 2021 09:29:53 +0200
Subject: [PATCH 302/411] [scanner] add minimal error handling, consolidate CLI
 args processing

---
 cdist/argparse.py         | 10 ++++--
 cdist/scan/commandline.py | 19 ++++++-----
 cdist/scan/scan.py        | 71 +++++++++++----------------------------
 3 files changed, 37 insertions(+), 63 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index cadac39a..153e7864 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -492,12 +492,16 @@ def get_parsers():
         help='Try to configure detected hosts')
     parser['scan'].add_argument(
         '-I', '--interfaces',
-        action='append',  default=[],
+        action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
     parser['scan'].add_argument(
         '-d', '--delay',
-        action='store',  default=3600,
-        help='How long to wait before reconfiguring after last try')
+        action='store',  default=3600, type=int,
+        help='How long (seconds) to wait before reconfiguring after last try')
+    parser['scan'].add_argument(
+        '-t', '--trigger-delay',
+        action='store',  default=5, type=int,
+        help='How long (seconds) to wait between ICMPv6 echo requests')
     parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
 
     for p in parser:
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 0d7fb0ca..dead5292 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -24,26 +24,29 @@ import sys
 
 log = logging.getLogger("scan")
 
-
-# define this outside of the class to not handle scapy import errors by default
+# CLI processing is defined outside of the main scan class to handle
+# non-available optional scapy dependency (instead of crashing mid-flight).
 def commandline(args):
     log.debug(args)
 
+    # Check if we have the optional scapy dependency available.
     try:
         import cdist.scan.scan as scan
     except ModuleNotFoundError:
-        print('cdist scan requires scapy to be installed! Exiting.',
-                file=sys.stderr)
+        log.error('cdist scan requires scapy to be installed. Exiting.')
         sys.exit(1)
 
-    processes = []
-
+    # Default operation mode.
     if not args.mode:
-        # By default scan and trigger, but do not call any action
+        # By default scan and trigger, but do not call any action.
         args.mode = ['scan', 'trigger', ]
 
+    # We run each component in a separate process since they
+    # must not block on each other.
+    processes = []
+
     if 'trigger' in args.mode:
-        t = scan.Trigger(interfaces=args.interfaces)
+        t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay)
         t.start()
         processes.append(t)
         log.debug("Trigger started")
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index faee8a56..633a5c06 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -61,20 +61,22 @@ import datetime
 
 import cdist.config
 
+logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("scan")
 
-
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
     """
 
-    def __init__(self, interfaces=None, verbose=False):
+    def __init__(self, interfaces, sleeptime, verbose=False):
         self.interfaces = interfaces
+
+        # Used by scapy / send in trigger/2.
         self.verbose = verbose
 
-        # Wait 5 seconds before triggering again - FIXME: add parameter
-        self.sleeptime = 5
+        # Delay in seconds between sent ICMPv6EchoRequests.
+        self.sleeptime = sleeptime
 
     def start(self):
         self.processes = []
@@ -93,9 +95,12 @@ class Trigger(object):
             time.sleep(self.sleeptime)
 
     def trigger(self, interface):
-        packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest()
-        log.debug("Sending request on %s", interface)
-        send(packet, verbose=self.verbose)
+        try:
+            log.debug("Sending ICMPv6EchoRequest on %s", interface)
+            packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest()
+            send(packet, verbose=self.verbose)
+        except Exception as e:
+            log.error( "Could not send ICMPv6EchoRequest: %s", e)
 
 
 class Scanner(object):
@@ -103,7 +108,7 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces=None, args=None, outdir=None):
+    def __init__(self, interfaces, args=None, outdir=None):
         self.interfaces = interfaces
 
         if outdir:
@@ -148,47 +153,9 @@ class Scanner(object):
 
     def scan(self):
         log.debug("Scanning - zzzzz")
-        sniff(iface=self.interfaces,
-              filter="icmp6",
-              prn=self.handle_pkg)
-
-
-if __name__ == '__main__':
-    t = Trigger(interfaces=["wlan0"])
-    t.start()
-
-    # Scanner can listen on many interfaces at the same time
-    s = Scanner(interfaces=["wlan0"])
-    s.scan()
-
-    # Join back the trigger processes
-    t.join()
-
-    # Test in my lan shows:
-    # [18:48] bridge:cdist% ls -1d fe80::*
-    # fe80::142d:f0a5:725b:1103
-    # fe80::20d:b9ff:fe49:ac11
-    # fe80::20d:b9ff:fe4c:547d
-    # fe80::219:d2ff:feb2:2e12
-    # fe80::21b:fcff:feee:f446
-    # fe80::21b:fcff:feee:f45c
-    # fe80::21b:fcff:feee:f4b1
-    # fe80::21b:fcff:feee:f4ba
-    # fe80::21b:fcff:feee:f4bc
-    # fe80::21b:fcff:feee:f4c1
-    # fe80::21d:72ff:fe86:46b
-    # fe80::42b0:34ff:fe6f:f6f0
-    # fe80::42b0:34ff:fe6f:f863
-    # fe80::42b0:34ff:fe6f:f9b2
-    # fe80::4a5d:60ff:fea1:e55f
-    # fe80::77a3:5e3f:82cc:f2e5
-    # fe80::9e93:4eff:fe6c:c1f4
-    # fe80::ba69:f4ff:fec5:6041
-    # fe80::ba69:f4ff:fec5:8db7
-    # fe80::bad8:12ff:fe65:313d
-    # fe80::bad8:12ff:fe65:d9b1
-    # fe80::ce2d:e0ff:fed4:2611
-    # fe80::ce32:e5ff:fe79:7ea7
-    # fe80::d66d:6dff:fe33:e00
-    # fe80::e2ff:f7ff:fe00:20e6
-    # fe80::f29f:c2ff:fe7c:275e
+        try:
+            sniff(iface=self.interfaces,
+                  filter="icmp6",
+                  prn=self.handle_pkg)
+        except Exception as e:
+            log.error( "Could not start listener: %s", e)

From bb24d632d62da8390dd1a724fc325a57526da40d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Thu, 22 Apr 2021 10:20:49 +0200
Subject: [PATCH 303/411] [scanner] implement the --list flag

---
 cdist/argparse.py         |  4 +++
 cdist/scan/commandline.py | 63 ++++++++++++++++++++++++++++-----------
 cdist/scan/scan.py        | 17 +++++++++++
 3 files changed, 67 insertions(+), 17 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 153e7864..f390a974 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -486,6 +486,10 @@ def get_parsers():
         '-m', '--mode', help='Which modes should run',
         action='append', default=[],
         choices=['scan', 'trigger'])
+    parser['scan'].add_argument(
+        '--list',
+        action='store_true',
+        help='List the known hosts and exit')
     parser['scan'].add_argument(
         '--config',
         action='store_true',
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index dead5292..331694e4 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -21,26 +21,11 @@
 
 import logging
 import sys
+from datetime import datetime
 
 log = logging.getLogger("scan")
 
-# CLI processing is defined outside of the main scan class to handle
-# non-available optional scapy dependency (instead of crashing mid-flight).
-def commandline(args):
-    log.debug(args)
-
-    # Check if we have the optional scapy dependency available.
-    try:
-        import cdist.scan.scan as scan
-    except ModuleNotFoundError:
-        log.error('cdist scan requires scapy to be installed. Exiting.')
-        sys.exit(1)
-
-    # Default operation mode.
-    if not args.mode:
-        # By default scan and trigger, but do not call any action.
-        args.mode = ['scan', 'trigger', ]
-
+def run(scan, args):
     # We run each component in a separate process since they
     # must not block on each other.
     processes = []
@@ -59,3 +44,47 @@ def commandline(args):
 
     for process in processes:
         process.join()
+
+def list(scan, args):
+    s = scan.Scanner(interfaces=args.interfaces, args=args)
+    hosts = s.list()
+
+    # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
+    # 6 colons.
+    ipv6_max_size = 8 * 4 + 10
+    # We format dates as follow: YYYY-MM-DD HH:MM:SS
+    date_max_size =              8 + 2    + 6 + 2
+
+    print("{} | {}".format(
+        'link-local address'.ljust(ipv6_max_size),
+        'last seen'.ljust(date_max_size)))
+    print('=' * (ipv6_max_size + 3 + date_max_size))
+    for addr in hosts:
+        last_seen = datetime.strftime(
+                datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'),
+                '%Y-%m-%d %H:%M:%S')
+        print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size)))
+
+# CLI processing is defined outside of the main scan class to handle
+# non-available optional scapy dependency (instead of crashing mid-flight).
+def commandline(args):
+    log.debug(args)
+
+    # Check if we have the optional scapy dependency available.
+    try:
+        import cdist.scan.scan as scan
+    except ModuleNotFoundError:
+        log.error('cdist scan requires scapy to be installed. Exiting.')
+        sys.exit(1)
+
+    # Set default operation mode.
+    if not args.mode:
+        # By default scan and trigger, but do not call any action.
+        args.mode = ['scan', 'trigger', ]
+
+    # Print known hosts and exit is --list is specified - do not start
+    # the scanner.
+    if args.list:
+        list(scan, args)
+    else:
+        run(scan, args)
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 633a5c06..f3976370 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -132,6 +132,23 @@ class Scanner(object):
             with open(fname, "w") as fd:
                 fd.write(f"{now}\n")
 
+    def list(self):
+        hosts = dict()
+        for linklocal_addr in os.listdir(self.outdir):
+            workdir = os.path.join(self.outdir, linklocal_addr)
+            # We ignore any (unexpected) file in this directory.
+            if os.path.isdir(workdir):
+                last_seen='-'
+                last_seen_file = os.path.join(workdir, 'last_seen')
+                if os.path.isfile(last_seen_file):
+                    with open(last_seen_file, "r") as fd:
+                        last_seen = fd.readline()
+
+                hosts[linklocal_addr] = {'last_seen': last_seen}
+
+        return hosts
+
+
     def config(self):
         """
         Configure a host

From 13e2ad175f01b6cfbdb21ee50e85b6f3ba6fe750 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Sun, 25 Apr 2021 12:45:34 +0200
Subject: [PATCH 304/411] [scanner] add host class, name mapper and pre-config
 logic

---
 cdist/argparse.py         |   6 +-
 cdist/scan/commandline.py |  37 ++++++++----
 cdist/scan/scan.py        | 122 +++++++++++++++++++++++++++++---------
 3 files changed, 124 insertions(+), 41 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index f390a974..bedb23ac 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -485,7 +485,7 @@ def get_parsers():
     parser['scan'].add_argument(
         '-m', '--mode', help='Which modes should run',
         action='append', default=[],
-        choices=['scan', 'trigger'])
+        choices=['scan', 'trigger', 'config'])
     parser['scan'].add_argument(
         '--list',
         action='store_true',
@@ -498,6 +498,10 @@ def get_parsers():
         '-I', '--interfaces',
         action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
+    parser['scan'].add_argument(
+        '--name-mapper',
+        action='store',  default=None,
+        help='Map addresses to names, required for config mode')
     parser['scan'].add_argument(
         '-d', '--delay',
         action='store',  default=3600, type=int,
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 331694e4..1a0ab0a7 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -37,7 +37,10 @@ def run(scan, args):
         log.debug("Trigger started")
 
     if 'scan' in args.mode:
-        s = scan.Scanner(interfaces=args.interfaces, args=args)
+        s = scan.Scanner(
+                autoconfigure='config' in args.mode,
+                interfaces=args.interfaces,
+                name_mapper=args.name_mapper)
         s.start()
         processes.append(s)
         log.debug("Scanner started")
@@ -46,24 +49,27 @@ def run(scan, args):
         process.join()
 
 def list(scan, args):
-    s = scan.Scanner(interfaces=args.interfaces, args=args)
+    s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper)
     hosts = s.list()
 
     # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
     # 6 colons.
     ipv6_max_size = 8 * 4 + 10
-    # We format dates as follow: YYYY-MM-DD HH:MM:SS
-    date_max_size =              8 + 2    + 6 + 2
+    date_max_size = len(datetime.now().strftime(scan.datetime_format))
+    name_max_size = 25
 
-    print("{} | {}".format(
-        'link-local address'.ljust(ipv6_max_size),
-        'last seen'.ljust(date_max_size)))
-    print('=' * (ipv6_max_size + 3 + date_max_size))
-    for addr in hosts:
-        last_seen = datetime.strftime(
-                datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'),
-                '%Y-%m-%d %H:%M:%S')
-        print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size)))
+    print("{} | {} | {} | {}".format(
+        'name'.ljust(name_max_size),
+        'address'.ljust(ipv6_max_size),
+        'last seen'.ljust(date_max_size),
+        'last configured'.ljust(date_max_size)))
+    print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
+    for host in hosts:
+        print("{} | {} | {} | {}".format(
+            host.name(default='-').ljust(name_max_size),
+            host.address().ljust(ipv6_max_size),
+            host.last_seen().ljust(date_max_size),
+            host.last_configured().ljust(date_max_size)))
 
 # CLI processing is defined outside of the main scan class to handle
 # non-available optional scapy dependency (instead of crashing mid-flight).
@@ -82,6 +88,11 @@ def commandline(args):
         # By default scan and trigger, but do not call any action.
         args.mode = ['scan', 'trigger', ]
 
+    if 'config' in args.mode and args.name_mapper == None:
+        print('--name-mapper must be specified for scanner config mode.',
+                file=sys.stderr)
+        sys.exit(1)
+
     # Print known hosts and exit is --list is specified - do not start
     # the scanner.
     if args.list:
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index f3976370..459138e2 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -63,6 +63,69 @@ import cdist.config
 
 logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("scan")
+datetime_format = '%Y-%m-%d %H:%M:%S'
+
+class Host(object):
+    def __init__(self, addr, outdir, name_mapper=None):
+        self.addr = addr
+        self.workdir = os.path.join(outdir, addr)
+        self.name_mapper = name_mapper
+
+        os.makedirs(self.workdir, exist_ok=True)
+
+    def __get(self, key, default=None):
+        fname = os.path.join(self.workdir, key)
+        value=default
+        if os.path.isfile(fname):
+            with open(fname, "r") as fd:
+                value = fd.readline()
+        return value
+
+    def __set(self, key, value):
+        fname = os.path.join(self.workdir, key)
+        with open(fname, "w") as fd:
+            fd.write(f"{value}")
+
+    def name(self, default=None):
+        if self.name_mapper == None:
+            return default
+
+        fpath = os.path.join(os.getcwd(), self.name_mapper)
+        if os.path.isfile(fpath) and os.access(fpath, os.X_OK):
+             out = subprocess.run([fpath, self.addr], capture_output=True)
+             if out.returncode != 0:
+                 return default
+             else:
+                value = out.stdout.decode()
+                return (None if len(value) == 0 else value)
+        else:
+            return default
+
+    def address(self):
+        return self.addr
+
+    def last_seen(self, default=None):
+        raw = self.__get('last_seen')
+        if raw:
+            return datetime.datetime.strptime(raw, datetime_format)
+        else:
+            return default
+
+    def last_configured(self, default=None):
+        raw = self.__get('last_configured')
+        if raw:
+            return datetime.datetime.strptime(raw, datetime_format)
+        else:
+            return default
+
+    def seen(self):
+        now = datetime.datetime.now().strftime(datetime_format)
+        self.__set('last_seen', now)
+
+    def configure(self):
+        # TODO: configure.
+        now = datetime.datetime.now().strftime(datetime_format)
+        self.__set('last_configured', now)
 
 class Trigger(object):
     """
@@ -108,48 +171,43 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces, args=None, outdir=None):
+    def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None):
         self.interfaces = interfaces
+        self.autoconfigure=autoconfigure
+        self.name_mapper = name_mapper
+        self.config_delay = datetime.timedelta(seconds=3600)
 
         if outdir:
             self.outdir = outdir
         else:
             self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan')
+        os.makedirs(self.outdir, exist_ok=True)
+
+        self.running_configs = {}
 
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
-            host = pkg['IPv6'].src
-            log.verbose("Host %s is alive", host)
+            host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper)
+            if host.name():
+                log.verbose("Host %s (%s) is alive", host.name(), host.address())
+            else:
+                log.verbose("Host %s is alive", host.address())
+            host.seen()
 
-            dir = os.path.join(self.outdir, host)
-            fname = os.path.join(dir, "last_seen")
-
-            now = datetime.datetime.now()
-
-            os.makedirs(dir, exist_ok=True)
-
-            # FIXME: maybe adjust the format so we can easily parse again
-            with open(fname, "w") as fd:
-                fd.write(f"{now}\n")
+            # TODO check last config.
+            if self.autoconfigure and \
+                    host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now():
+                self.config(host)
 
     def list(self):
-        hosts = dict()
-        for linklocal_addr in os.listdir(self.outdir):
-            workdir = os.path.join(self.outdir, linklocal_addr)
-            # We ignore any (unexpected) file in this directory.
-            if os.path.isdir(workdir):
-                last_seen='-'
-                last_seen_file = os.path.join(workdir, 'last_seen')
-                if os.path.isfile(last_seen_file):
-                    with open(last_seen_file, "r") as fd:
-                        last_seen = fd.readline()
-
-                hosts[linklocal_addr] = {'last_seen': last_seen}
+        hosts = []
+        for addr in os.listdir(self.outdir):
+            hosts.append(Host(addr, self.outdir, self.name_mapper))
 
         return hosts
 
 
-    def config(self):
+    def config(self, host):
         """
         Configure a host
 
@@ -158,9 +216,19 @@ class Scanner(object):
         - Maybe keep dict storing per host processes
         - Save the result
         - Save the output -> probably aligned to config mode
-
         """
 
+        if host.name() == None:
+            log.debug("config - could not resolve name for %s, aborting.", host.address())
+            return
+
+        if self.running_configs.get(host.name()) != None:
+            log.debug("config - is already running for %s, aborting.", host.name())
+
+        log.info("config - running against host %s.", host.name())
+        p = host.configure()
+        self.running_configs[host.name()] = p
+
     def start(self):
         self.process = Process(target=self.scan)
         self.process.start()

From 512e9b23c0035b09ac1e7e76f706fee2e2ec481f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 25 Apr 2021 15:53:40 +0200
Subject: [PATCH 305/411] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 70d0b960..c41743cd 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
+
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong

From 6ac8cbf98f5d5018373db6eb9c038ef1c599ff2f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:33:56 +0200
Subject: [PATCH 306/411] [type/__postgres_database] Include postgres_user
 explorer from __postgres_conf

---
 .../explorer/postgres_user                    |  1 +
 .../type/__postgres_database/explorer/state   | 23 +++++--------------
 .../type/__postgres_database/gencode-remote   | 13 +----------
 3 files changed, 8 insertions(+), 29 deletions(-)
 create mode 120000 cdist/conf/type/__postgres_database/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_database/explorer/postgres_user b/cdist/conf/type/__postgres_database/explorer/postgres_user
new file mode 120000
index 00000000..714e7237
--- /dev/null
+++ b/cdist/conf/type/__postgres_database/explorer/postgres_user
@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user
\ No newline at end of file
diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state
index d68d4120..bb044e0e 100755
--- a/cdist/conf/type/__postgres_database/explorer/state
+++ b/cdist/conf/type/__postgres_database/explorer/state
@@ -1,6 +1,7 @@
 #!/bin/sh
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,25 +19,13 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$("${__explorer}/os")"
-in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
-esac
+postgres_user=$("${__type_explorer:?}/postgres_user")
 
+name=${__object_id:?}
 
-name="$__object_id"
-
-if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='$name'\"")"
+if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='${name}'\"")"
 then
-    echo 'present'
+	echo 'present'
 else
-    echo 'absent'
+	echo 'absent'
 fi
diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote
index 0f11cff4..d70fc436 100755
--- a/cdist/conf/type/__postgres_database/gencode-remote
+++ b/cdist/conf/type/__postgres_database/gencode-remote
@@ -18,23 +18,12 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
-in
-   netbsd)
-      postgres_user='pgsql'
-      ;;
-   openbsd)
-      postgres_user='_postgresql'
-      ;;
-   *)
-      postgres_user='postgres'
-      ;;
-esac
 
 
 name="$__object_id"
 state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
 if [ "$state_should" != "$state_is" ]; then
    case "$state_should" in

From 58b279a8d0121dcbef05a4999d1d88208656aeed Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:34:17 +0200
Subject: [PATCH 307/411] [type/__postgres_database] Improve quoting

---
 .../type/__postgres_database/gencode-remote   | 95 +++++++++++--------
 1 file changed, 55 insertions(+), 40 deletions(-)

diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote
index d70fc436..7d7d6fa2 100755
--- a/cdist/conf/type/__postgres_database/gencode-remote
+++ b/cdist/conf/type/__postgres_database/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,49 +19,63 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
 
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
 
-name="$__object_id"
-state_should="$(cat "$__object/parameter/state")"
-state_is="$(cat "$__object/explorer/state")"
 postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
-if [ "$state_should" != "$state_is" ]; then
-   case "$state_should" in
-      present)
-         owner=""
-         if [ -f "$__object/parameter/owner" ]; then
-            owner="-O \"$(cat "$__object/parameter/owner")\""
-         fi
+dbname=${__object_id:?}
+state_should=$(cat "${__object:?}/parameter/state")
+state_is=$(cat "${__object:?}/explorer/state")
 
-         template=""
-         if [ -f "$__object/parameter/template" ]; then
-            template="--template \"$(cat "$__object/parameter/template")\""
-         fi
-
-         encoding=""
-         if [ -f "$__object/parameter/encoding" ]; then
-            encoding="--encoding \"$(cat "$__object/parameter/encoding")\""
-         fi
-
-         lc_collate=""
-         if [ -f "$__object/parameter/lc-collate" ]; then
-            lc_collate="--lc-collate \"$(cat "$__object/parameter/lc-collate")\""
-         fi
-
-         lc_ctype=""
-         if [ -f "$__object/parameter/lc-ctype" ]; then
-            lc_ctype="--lc-ctype \"$(cat "$__object/parameter/lc-ctype")\""
-         fi
-
-         cat << EOF
-su - '$postgres_user' -c "createdb $owner \"$name\" $template $encoding $lc_collate $lc_ctype"
-EOF
-      ;;
-      absent)
-         cat << EOF
-su - '$postgres_user' -c "dropdb \"$name\""
-EOF
-      ;;
-   esac
+if test "${state_should}" = "$state_is"
+then
+	exit 0
 fi
+
+case ${state_should}
+in
+	(present)
+		set --
+
+		while read -r param_name opt
+		do
+			if test -f "${__object:?}/parameter/${param_name}"
+			then
+				set -- "$@" "${opt}" "$(cat "${__object:?}/parameter/${param_name}")"
+			fi
+		done <<-'EOF'
+		owner -O
+		template --template
+		encoding --encoding
+		lc_collate --lc-collate
+		lc_ctype --lc-ctype
+		EOF
+
+		set -- "$@" "${dbname}"
+
+		cat <<-EOF
+		su - $(quote "${postgres_user}") -c $(quote "$(quote createdb "$@")")
+		EOF
+		;;
+	(absent)
+		cat <<-EOF
+		su - $(quote "${postgres_user}") -c $(quote "$(quote dropdb "${dbname}")")
+		EOF
+		;;
+esac

From beb8da6d5fd141ac63eacfb526cfcc43cbf22d8f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:48:51 +0200
Subject: [PATCH 308/411] [type/__postgres_role] Include postgres_user explorer
 from __postgres_conf

---
 .../type/__postgres_role/explorer/postgres_user   |  1 +
 cdist/conf/type/__postgres_role/explorer/state    | 14 +-------------
 cdist/conf/type/__postgres_role/gencode-remote    | 15 +--------------
 3 files changed, 3 insertions(+), 27 deletions(-)
 create mode 120000 cdist/conf/type/__postgres_role/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_role/explorer/postgres_user b/cdist/conf/type/__postgres_role/explorer/postgres_user
new file mode 120000
index 00000000..714e7237
--- /dev/null
+++ b/cdist/conf/type/__postgres_role/explorer/postgres_user
@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user
\ No newline at end of file
diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 34069de9..0b264e6f 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -19,19 +19,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case $("${__explorer:?}/os")
-in
-	(netbsd)
-		postgres_user='pgsql'
-		;;
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(*)
-		postgres_user='postgres'
-		;;
-esac
-
+postgres_user=$("${__type_explorer:?}/postgres_user")
 rolename=${__object_id:?}
 
 
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index d7631fbd..7324b80c 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -28,20 +28,7 @@ quote() {
 	fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
 }
 
-case $(cat "${__global:?}/explorer/os")
-in
-	(netbsd)
-		postgres_user='pgsql'
-		;;
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(*)
-		postgres_user='postgres'
-		;;
-esac
-
-
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 rolename=${__object_id:?}
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")

From 3cf93249c38a6f27c097877add619b5d6499a049 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:55:11 +0200
Subject: [PATCH 309/411] [type/__postgres_extension] Include postgres_user
 explorer from __postgres_conf

---
 .../__postgres_extension/explorer/postgres_user     |  1 +
 cdist/conf/type/__postgres_extension/gencode-remote | 13 +------------
 2 files changed, 2 insertions(+), 12 deletions(-)
 create mode 120000 cdist/conf/type/__postgres_extension/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_extension/explorer/postgres_user b/cdist/conf/type/__postgres_extension/explorer/postgres_user
new file mode 120000
index 00000000..714e7237
--- /dev/null
+++ b/cdist/conf/type/__postgres_extension/explorer/postgres_user
@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user
\ No newline at end of file
diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote
index af9c97f1..a1c84828 100755
--- a/cdist/conf/type/__postgres_extension/gencode-remote
+++ b/cdist/conf/type/__postgres_extension/gencode-remote
@@ -22,18 +22,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
-in
-   netbsd)
-      postgres_user='pgsql'
-      ;;
-   openbsd)
-      postgres_user='_postgresql'
-      ;;
-   *)
-      postgres_user='postgres'
-      ;;
-esac
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
 
 dbname=$(    echo "$__object_id" | cut -d":" -f1 )

From 8296051653a8187374ad51a3a487315767b73a19 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 16 Apr 2021 19:22:58 +0200
Subject: [PATCH 310/411] [type/__postgres_extension] Add state explorer

---
 .../type/__postgres_extension/explorer/state  | 41 ++++++++++++++
 .../type/__postgres_extension/gencode-remote  | 44 ++++++++++-----
 cdist/conf/type/__postgres_extension/man.rst  | 55 +++++++++++--------
 3 files changed, 104 insertions(+), 36 deletions(-)
 create mode 100644 cdist/conf/type/__postgres_extension/explorer/state

diff --git a/cdist/conf/type/__postgres_extension/explorer/state b/cdist/conf/type/__postgres_extension/explorer/state
new file mode 100644
index 00000000..9d156be7
--- /dev/null
+++ b/cdist/conf/type/__postgres_extension/explorer/state
@@ -0,0 +1,41 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Prints "present" if the extension is currently installed.
+# "absent" otherwise.
+
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+
+postgres_user=$("${__type_explorer:?}/postgres_user")
+
+IFS=: read -r dbname extname <<EOF
+${__object_id:?}
+EOF
+
+psql_exec() {
+	su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")"
+}
+
+if psql_exec "${dbname}" 'SELECT extname FROM pg_extension' | grep -qFx "${extname}"
+then
+	echo present
+else
+	echo absent
+fi
diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote
index a1c84828..4ce72622 100755
--- a/cdist/conf/type/__postgres_extension/gencode-remote
+++ b/cdist/conf/type/__postgres_extension/gencode-remote
@@ -2,9 +2,10 @@
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2013 Tomas Pospisek (tpo_deb at sourcepole.ch)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This type was created by Tomas Pospisek based on the
-#__postgres_role type by Steven Armstrong
+# __postgres_role type by Steven Armstrong.
 #
 # This file is part of cdist.
 #
@@ -24,19 +25,36 @@
 
 postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_cmd() {
+	printf 'su - %s -c %s\n' \
+		"$(quote "${postgres_user}")" \
+		"$(quote psql "$(quote "$1")" -c "$(quote "$2")")"
+}
 
-dbname=$(    echo "$__object_id" | cut -d":" -f1 )
-extension=$( echo "$__object_id" | cut -d":" -f2 )
 
-state_should=$( cat "$__object/parameter/state" )
+IFS=: read -r dbname extname <<EOF
+${__object_id:?}
+EOF
 
-case "$state_should" in
-    present)
-         cmd="CREATE EXTENSION IF NOT EXISTS $extension"
-         echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'"
-    ;;
-    absent)
-         cmd="DROP EXTENSION IF EXISTS $extension"
-         echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'"
-    ;;
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
+
+case ${state_should}
+in
+	(present)
+		psql_cmd "${dbname}" "CREATE EXTENSION ${extname}"
+		;;
+	(absent)
+		psql_cmd "${dbname}" "DROP EXTENSION ${extname}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
 esac
diff --git a/cdist/conf/type/__postgres_extension/man.rst b/cdist/conf/type/__postgres_extension/man.rst
index 79645b2b..442239f6 100644
--- a/cdist/conf/type/__postgres_extension/man.rst
+++ b/cdist/conf/type/__postgres_extension/man.rst
@@ -3,32 +3,36 @@ cdist-type__postgres_extension(7)
 
 NAME
 ----
-cdist-type__postgres_extension - manage postgres extensions
+cdist-type__postgres_extension - Manage PostgreSQL extensions
 
 
 DESCRIPTION
 -----------
-This cdist type allows you to create or drop postgres extensions.
+This cdist type allows you to manage PostgreSQL extensions.
 
-The object you need to pass to __postgres_extension consists of
-the database name and the extension name joined by a colon in the
-following form:
-
-.. code-block:: sh
-
-    dbname:extension
-
-f.ex.
+The ``__object_id`` to pass to ``__postgres_extension`` is of the form
+``dbname:extension``, e.g.:
 
 .. code-block:: sh
 
     rails_test:unaccent
 
 
+**CAUTION!** Be careful when installing extensions from (untrusted) third-party
+sources:
+
+   | Installing an extension as superuser requires trusting that the extension's
+     author wrote the extension installation script in a secure fashion. It is
+     not terribly difficult for a malicious user to create trojan-horse objects
+     that will compromise later execution of a carelessly-written extension
+     script, allowing that user to acquire superuser privileges.
+   | – `<https://www.postgresql.org/docs/13/sql-createextension.html#id-1.9.3.64.7>`_
+
+
 OPTIONAL PARAMETERS
 -------------------
 state
-    either "present" or "absent", defaults to "present"
+    either ``present`` or ``absent``, defaults to ``present``.
 
 
 EXAMPLES
@@ -36,24 +40,29 @@ EXAMPLES
 
 .. code-block:: sh
 
-    __postgres_extension           rails_test:unaccent
-    __postgres_extension --present rails_test:unaccent
-    __postgres_extension --absent  rails_test:unaccent
+   # Install extension unaccent into database rails_test
+   __postgres_extension rails_test:unaccent
+
+   # Drop extension unaccent from database fails_test
+   __postgres_extension rails_test:unaccent --state absent
 
 
 SEE ALSO
 --------
-:strong:`cdist-type__postgre_database`\ (7)
+- :strong:`cdist-type__postgres_database`\ (7)
+- PostgreSQL "CREATE EXTENSION" documentation at:
+  `<http://www.postgresql.org/docs/current/static/sql-createextension.html>`_.
 
-Postgres "Create Extension" documentation at: <http://www.postgresql.org/docs/current/static/sql-createextension.html>.
 
-AUTHOR
+AUTHORS
 -------
-Tomas Pospisek <tpo_deb--@--sourcepole.ch>
+| Tomas Pospisek <tpo_deb--@--sourcepole.ch>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+
 
 COPYING
 -------
-Copyright \(C) 2014 Tomas Pospisek. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2014 Tomas Pospisek, 2021 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.

From 0d33407b182964672b1cfc3ffc49c7fa7cf462a0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 16 Apr 2021 19:35:26 +0200
Subject: [PATCH 311/411] [type/__postgres_database] Proper quoting in state
 explorer

---
 cdist/conf/type/__postgres_database/explorer/state | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state
index bb044e0e..6a25df86 100755
--- a/cdist/conf/type/__postgres_database/explorer/state
+++ b/cdist/conf/type/__postgres_database/explorer/state
@@ -21,9 +21,14 @@
 
 postgres_user=$("${__type_explorer:?}/postgres_user")
 
-name=${__object_id:?}
+dbname=${__object_id:?}
 
-if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='${name}'\"")"
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_exec() {
+	su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")"
+}
+
+if psql_exec postgres "SELECT datname FROM pg_database" | grep -qFx "${dbname}"
 then
 	echo 'present'
 else

From 0f05f38384c94b79232ca0917d1323eeefd6e1ed Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 19 Apr 2021 19:09:46 +0200
Subject: [PATCH 312/411] [type/__postgres_role] Treat --password '' like no
 --password

---
 cdist/conf/type/__postgres_role/explorer/state | 11 ++++-------
 cdist/conf/type/__postgres_role/gencode-remote |  2 +-
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 0b264e6f..822816c1 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -43,8 +43,7 @@ role_properties=$(
 	  BEGIN { RS = "\036"; FS = "\034" }
 	  /^\([0-9]+ rows?\)/ { exit }
 	  NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next }
-	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }
-	  '
+	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }'
 )
 
 if test -n "${role_properties}"
@@ -78,12 +77,10 @@ then
 	# Check password
 	passwd_stored=$(
 		psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \
-		| awk 'BEGIN { RS = "\036" } NR == 2'
-		printf .
-	)
-	passwd_stored=${passwd_stored%?.}
+		| awk 'BEGIN { RS = "\036" } NR == 2 { printf "%s.", $0 }')
+	passwd_stored=${passwd_stored%.}
 
-	if test -f "${__object:?}/parameter/password"
+	if test -s "${__object:?}/parameter/password"
 	then
 		passwd_should=$(cat "${__object:?}/parameter/password"; printf .)
 	fi
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 7324b80c..4cb78330 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -46,7 +46,7 @@ psql_query() {
 
 psql_set_password() {
 	# NOTE: Always make sure that the password does not end up in psql_history!
-	# NOTE: Never set an empty string as the password, because they can be
+	# NOTE: Never set an empty string as the password, because it can be
 	#       interpreted differently by different tooling.
 	if test -s "${__object:?}/parameter/password"
 	then

From 92fff7cb77271009f25d999b6451dc1e43e9bb6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 26 Apr 2021 12:09:20 +0200
Subject: [PATCH 313/411] [scanner] fix crash on --list with name mapper
 provided

---
 cdist/scan/commandline.py | 10 ++++++++--
 cdist/scan/scan.py        |  2 +-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 1a0ab0a7..3eb7eec4 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -65,11 +65,17 @@ def list(scan, args):
         'last configured'.ljust(date_max_size)))
     print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
     for host in hosts:
+        last_seen = host.last_seen()
+        last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-'
+
+        last_configured = host.last_configured()
+        last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-'
+
         print("{} | {} | {} | {}".format(
             host.name(default='-').ljust(name_max_size),
             host.address().ljust(ipv6_max_size),
-            host.last_seen().ljust(date_max_size),
-            host.last_configured().ljust(date_max_size)))
+            last_seen.ljust(date_max_size),
+            last_configured.ljust(date_max_size)))
 
 # CLI processing is defined outside of the main scan class to handle
 # non-available optional scapy dependency (instead of crashing mid-flight).
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 459138e2..69a4121d 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -97,7 +97,7 @@ class Host(object):
                  return default
              else:
                 value = out.stdout.decode()
-                return (None if len(value) == 0 else value)
+                return (default if len(value) == 0 else value)
         else:
             return default
 

From 3a9dd5b1669fa29e1713cdadec1596d859d377f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 26 Apr 2021 12:09:55 +0200
Subject: [PATCH 314/411] [scanner] add minimal (non-configurable) config mode

---
 cdist/scan/scan.py | 35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 69a4121d..152abb4e 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -122,8 +122,20 @@ class Host(object):
         now = datetime.datetime.now().strftime(datetime_format)
         self.__set('last_seen', now)
 
+    # XXX: There's no easy way to use the config module without feeding it with
+    # CLI args. Might as well call everything from scratch!
     def configure(self):
-        # TODO: configure.
+        target = self.name() or self.address()
+        cmd = ['cdist', 'config', '-v', target ]
+
+        fname = os.path.join(self.workdir, 'last_configuration_log')
+        with open(fname, "w") as fd:
+            log.debug("Executing: %s", cmd)
+            completed_process = subprocess.run(cmd, stdout=fd, stderr=fd)
+            if completed_process.returncode != 0:
+                log.error("%s return with non-zero code %i - see %s for details.",
+                        cmd, completed_process.returncode, fname)
+
         now = datetime.datetime.now().strftime(datetime_format)
         self.__set('last_configured', now)
 
@@ -194,7 +206,7 @@ class Scanner(object):
                 log.verbose("Host %s is alive", host.address())
             host.seen()
 
-            # TODO check last config.
+            # Configure if needed.
             if self.autoconfigure and \
                     host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now():
                 self.config(host)
@@ -206,27 +218,18 @@ class Scanner(object):
 
         return hosts
 
-
     def config(self, host):
-        """
-        Configure a host
-
-        - Assume we are only called if necessary
-        - However we need to ensure to not run in parallel
-        - Maybe keep dict storing per host processes
-        - Save the result
-        - Save the output -> probably aligned to config mode
-        """
-
         if host.name() == None:
             log.debug("config - could not resolve name for %s, aborting.", host.address())
             return
 
-        if self.running_configs.get(host.name()) != None:
+        previous_config_process = self.running_configs.get(host.name())
+        if previous_config_process != None and previous_config_process.is_alive():
             log.debug("config - is already running for %s, aborting.", host.name())
 
-        log.info("config - running against host %s.", host.name())
-        p = host.configure()
+        log.info("config - running against host %s (%s).", host.name(), host.address())
+        p = Process(target=host.configure())
+        p.start()
         self.running_configs[host.name()] = p
 
     def start(self):

From 2232435c2206b2b3a9a5f2b976df16540b3f1eeb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 26 Apr 2021 14:39:26 +0200
Subject: [PATCH 315/411] [scanner] initial documentation

Note: still needs to patch main cdist(1) manpage
---
 cdist/scan/scan.py      | 32 -------------
 docs/src/cdist-scan.rst | 99 +++++++++++++++++++++++++++++++++++++++++
 docs/src/index.rst      |  1 +
 3 files changed, 100 insertions(+), 32 deletions(-)
 create mode 100644 docs/src/cdist-scan.rst

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 152abb4e..2912dab3 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -19,38 +19,6 @@
 #
 #
 
-#
-# Interface to be implemented:
-# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated
-#   scan: scan / listen for icmp6 replies
-#   trigger: send trigger to multicast
-#   config: configure newly detected hosts
-#   install: install newly detected hosts
-#
-# Scanner logic
-#  - save results to configdir:
-#     basedir = ~/.cdist/scan/<ipv6-address>
-#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time
-#           or similar
-#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record
-#           unix time or similar
-#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record
-#           unix time or similar
-#
-#
-#
-#
-# cdist scan --list
-#       Show all known hosts including last seen flag
-#
-# Logic for reconfiguration:
-#
-#  - record when configured last time
-#  - introduce a parameter --reconfigure-after that takes time argument
-#  - reconfigure if a) host alive and b) reconfigure-after time passed
-#
-
-
 from multiprocessing import Process
 import os
 import logging
diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst
new file mode 100644
index 00000000..02193456
--- /dev/null
+++ b/docs/src/cdist-scan.rst
@@ -0,0 +1,99 @@
+Scan
+=====
+
+Description
+-----------
+Runs cdist as a daemon that discover/watch on hosts and reconfigure them
+periodically. It is especially useful in netboot-based environment where hosts
+boot unconfigured, and to ensure your infrastructure stays in sync with your
+configuration.
+
+This feature is still consider to be in **beta** stage.
+
+Usage (Examples)
+----------------
+
+Discover hosts on local network and configure those whose name is resolved by
+the name mapper script.
+
+.. code-block:: sh
+
+    $ cdist scan --beta --interface eth0 \
+      --mode scan --name-mapper path/to/script \
+      --mode trigger --mode config
+
+List known hosts and exit.
+
+.. code-block:: sh
+
+    $ cdist scan --beta --list --name-mapper path/to/script
+
+Please refer to `cdist(1)` for a detailed list of parameters.
+
+Modes
+-----
+
+The scanner has 3 modes that can be independently toggled. If the `--mode`
+parameter is not specified, only `tigger` and `scan` are enabled (= hosts are
+not configured).
+
+trigger
+  Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to
+  trigger detection by the `scan` module.
+
+scan
+  Watch for incoming ICMPv6 replies and optionally configure detected hosts.
+
+config
+  Enable configuration of hosts detected by `scan`.
+
+Name Mapper Script
+------------------
+
+The name mapper script takes an IPv6 address as first argument and writes the
+resolved name to stdout - if any. The script must be executable.
+
+Simplest script:
+
+.. code-block:: sh
+  #!/bin/sh
+
+  case "$1" in
+  	"fe80::20d:b9ff:fe57:3524")
+  		printf "my-host-01"
+  		;;
+  	"fe80::7603:bdff:fe05:89bb")
+  		printf "my-host-02"
+  		;;
+  esac
+
+Resolving name from `PTR` DNS record:
+
+.. code-block:: sh
+  #!/bin/sh
+
+  for cmd in dig sed; do
+  	if ! command -v $cmd > /dev/null; then
+  		exit 1
+  	fi
+  done
+
+  dig +short -x "$1" | sed -e 's/.$//'
+
+
+Trigger Source Script
+---------------------
+
+This script returns a list of addresses (separated by a newline) to be used by
+`trigger` mode. It is not used to map names. The script must be executable.
+
+Simplest script:
+
+.. code-block:: sh
+  #!/bin/sh
+
+  cat << EOF
+  server1.domain.tld
+  server2.domain.tld
+  server3.domain.tld
+  EOF
diff --git a/docs/src/index.rst b/docs/src/index.rst
index 31c044dc..369d5309 100644
--- a/docs/src/index.rst
+++ b/docs/src/index.rst
@@ -34,6 +34,7 @@ It natively supports IPv6 since the first release.
    cdist-parallelization
    cdist-inventory
    cdist-preos
+   cdist-scan
    cdist-integration
    cdist-reference
    cdist-best-practice

From a4122882f2a14034d1a219bd0314892176c5cb64 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Apr 2021 16:39:51 +0200
Subject: [PATCH 316/411] [type/__debconf_set_selections] Add state explorer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…and to make it work, replace --file with --line.

--file is deprecated because it does not work with the state explorer as the
contents of the file are not available on the target.
---
 .../__debconf_set_selections/explorer/state   | 124 ++++++++++++++++++
 .../__debconf_set_selections/gencode-remote   |  37 ++++--
 .../type/__debconf_set_selections/man.rst     |  49 ++++---
 .../type/__debconf_set_selections/manifest    |  21 +++
 .../parameter/{required => deprecated}        |   0
 .../parameter/optional_multiple               |   1 +
 6 files changed, 206 insertions(+), 26 deletions(-)
 create mode 100644 cdist/conf/type/__debconf_set_selections/explorer/state
 create mode 100755 cdist/conf/type/__debconf_set_selections/manifest
 rename cdist/conf/type/__debconf_set_selections/parameter/{required => deprecated} (100%)
 create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/optional_multiple

diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state
new file mode 100644
index 00000000..68d28941
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/explorer/state
@@ -0,0 +1,124 @@
+#!/bin/sh -e
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Determine current debconf selections' state.
+# Prints one of:
+#   present: all selections are already set as they should.
+#   different: one or more of the selections have a different value.
+#   absent: one or more of the selections are not (currently) defined.
+#
+
+test -x /usr/bin/perl || {
+	# cannot find perl (no perl ~ no debconf)
+	echo 'absent'
+	exit 0
+}
+
+linesfile="${__object:?}/parameter/line"
+test -s "${linesfile}" || {
+	if test -s "${__object:?}/parameter/file"
+	then
+		echo absent
+	else
+		echo present
+	fi
+	exit 0
+}
+
+/usr/bin/perl -- - "${linesfile}" <<'EOF'
+use strict;
+use warnings "all";
+
+use Debconf::Db;
+use Debconf::Question;
+
+# Extract @known... arrays from debconf-set-selections
+# These values are required to distinguish flags and values in the given lines.
+# DC: I couldn't think of a more ugly solution to the problem…
+my @knownflags;
+my @knowntypes;
+my $debconf_set_selections = '/usr/bin/debconf-set-selections';
+if (-e $debconf_set_selections) {
+	my $sed_known = 's/^my \(@known\(flags\|types\) = qw([a-z ]*);\).*$/\1/p';
+	eval `sed -n '$sed_known' '$debconf_set_selections'`;
+}
+
+sub mungeline ($) {
+	my $line = shift;
+	chomp $line;
+	$line =~ s/\r$//;
+	return $line;
+}
+
+sub fatal { printf STDERR @_; exit 1; }
+
+my $state = 'present';
+
+sub state {
+	my $new = shift;
+	if ($state eq 'present'
+	    or ($state eq 'different' and $new eq 'absent')) {
+		$state = $new;
+	}
+}
+
+Debconf::Db->load(readonly => 'true');
+
+while (<>) {
+	# Read and process lines (taken from debconf-set-selections)
+	$_ = mungeline($_);
+	while (/\\$/ && ! eof) {
+		s/\\$//;
+		$_ .= mungeline(<>);
+	}
+	next if /^\s*$/ || /^\s*\#/;
+
+	my ($owner, $label, $type, $content) = /^\s*(\S+)\s+(\S+)\s+(\S+)(?:\s(.*))?/
+		or fatal "invalid line: %s\n", $_;
+	$content = '' unless defined $content;
+
+
+	# Compare is and should state
+	my $q = Debconf::Question->get($label);
+
+	unless (defined $q) {
+		# probably a preseed
+		state 'absent';
+		next;
+	}
+
+	if (grep { $_ eq $q->type } @knownflags) {
+		# This line wants to set a flag, presumably.
+		if ($q->flag($q->type) ne $content) {
+			state 'different';
+		}
+	} else {
+		# Otherwise, it's probably a value…
+		if ($q->value ne $content) {
+			state 'different';
+		}
+
+		unless (grep { $_ eq $owner } (split /, /, $q->owners)) {
+			state 'different';
+		}
+	}
+}
+
+printf "%s\n", $state;
+EOF
diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote
index e99aef40..50d898c6 100755
--- a/cdist/conf/type/__debconf_set_selections/gencode-remote
+++ b/cdist/conf/type/__debconf_set_selections/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,16 +18,32 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# Setup selections
-#
 
-filename="$(cat "$__object/parameter/file")"
-
-if [ "$filename" = "-" ]; then
-    filename="$__object/stdin"
+if test -f "${__object:?}/parameter/line"
+then
+	filename="${__object:?}/parameter/line"
+elif test -s "${__object:?}/parameter/file"
+then
+	filename=$(cat "${__object:?}/parameter/file")
+	if test "${filename}" = '-'
+	then
+		filename="${__object:?}/stdin"
+	fi
+else
+	printf 'Neither --line nor --file set.\n' >&2
+	exit 1
 fi
 
-echo "debconf-set-selections << __file-eof"
-cat "$filename"
-echo "__file-eof"
+# setting no lines makes no sense
+test -s "${filename}" || exit 0
+
+state_is=$(cat "${__object:?}/explorer/state")
+
+if test "${state_is}" != 'present'
+then
+	cat <<-CODE
+	debconf-set-selections <<'EOF'
+	$(cat "${filename}")
+	EOF
+	CODE
+fi
diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst
index 58c25b81..690e3e49 100644
--- a/cdist/conf/type/__debconf_set_selections/man.rst
+++ b/cdist/conf/type/__debconf_set_selections/man.rst
@@ -8,15 +8,33 @@ cdist-type__debconf_set_selections - Setup debconf selections
 
 DESCRIPTION
 -----------
-On Debian and alike systems debconf-set-selections(1) can be used
+On Debian and alike systems :strong:`debconf-set-selections`\ (1) can be used
 to setup configuration parameters.
 
 
 REQUIRED PARAMETERS
 -------------------
+cf. ``--line``.
+
+
+OPTIONAL PARAMETERS
+-------------------
 file
-   Use the given filename as input for debconf-set-selections(1)
-   If filename is "-", read from stdin.
+   Use the given filename as input for :strong:`debconf-set-selections`\ (1)
+   If filename is ``-``, read from stdin.
+
+   **This parameter is deprecated, because it doesn't work with state detection.**
+line
+   A line in :strong:`debconf-set-selections`\ (1) compatible format.
+   This parameter can be used multiple times to set multiple options.
+
+   (This parameter is actually required, but marked optional because the
+   deprecated ``--file`` is still accepted.)
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
 
 
 EXAMPLES
@@ -24,30 +42,29 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # Setup configuration for nslcd
-    __debconf_set_selections nslcd --file /path/to/file
+    # Setup gitolite's gituser
+    __debconf_set_selections nslcd --line 'gitolite gitolite/gituser string git'
 
-    # Setup configuration for nslcd from another type
-    __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd"
-
-    __debconf_set_selections nslcd --file - << eof
-    gitolite gitolite/gituser string git
-    eof
+    # Setup configuration for nslcd from a file.
+    # NB: Multiple lines can be passed to --line, although this can be considered a hack.
+    __debconf_set_selections nslcd --line "$(cat "${__files:?}/preseed/nslcd.debconf")"
 
 
 SEE ALSO
 --------
-:strong:`debconf-set-selections`\ (1), :strong:`cdist-type__update_alternatives`\ (7)
+- :strong:`cdist-type__update_alternatives`\ (7)
+- :strong:`debconf-set-selections`\ (1)
 
 
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2014 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2011-2014 Nico Schottelius, 2021 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__debconf_set_selections/manifest b/cdist/conf/type/__debconf_set_selections/manifest
new file mode 100755
index 00000000..0f4fb2e2
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/manifest
@@ -0,0 +1,21 @@
+#!/bin/sh -e
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+__package_apt debconf
diff --git a/cdist/conf/type/__debconf_set_selections/parameter/required b/cdist/conf/type/__debconf_set_selections/parameter/deprecated
similarity index 100%
rename from cdist/conf/type/__debconf_set_selections/parameter/required
rename to cdist/conf/type/__debconf_set_selections/parameter/deprecated
diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple
new file mode 100644
index 00000000..a999a0c2
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple
@@ -0,0 +1 @@
+line

From 9cf19388abe95a5f8df9f5c94d2e54fa1060b2ef Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Apr 2021 16:47:44 +0200
Subject: [PATCH 317/411] [type/__debconf_set_selections] Send message about
 each debconf setting that is changed

---
 cdist/conf/type/__debconf_set_selections/gencode-remote | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote
index 50d898c6..9ba28f09 100755
--- a/cdist/conf/type/__debconf_set_selections/gencode-remote
+++ b/cdist/conf/type/__debconf_set_selections/gencode-remote
@@ -46,4 +46,9 @@ then
 	$(cat "${filename}")
 	EOF
 	CODE
+
+	awk '
+		{
+			printf "set %s %s %s %s\n", $1, $2, $3, $4
+		}' "${filename}" >>"${__messages_out:?}"
 fi

From 3a25b804664963e092d1f75c1fdded25d7016ef3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 26 Apr 2021 21:27:15 +0200
Subject: [PATCH 318/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index c41743cd..497ae91a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
+	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From a42ebc7a78e09e138954b1376376f7cb643420e0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 27 Apr 2021 19:41:10 +0200
Subject: [PATCH 319/411] [type/__debconf_set_selections] Synchronise objects

Works around locking error:

	debconf: DbDriver "config": /var/cache/debconf/config.dat is locked by another process: Resource temporarily unavailable
---
 .../__debconf_set_selections/explorer/state   | 20 ++++++++++++++++++-
 .../type/__debconf_set_selections/nonparallel |  0
 2 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 cdist/conf/type/__debconf_set_selections/nonparallel

diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state
index 68d28941..f8a3f6c8 100644
--- a/cdist/conf/type/__debconf_set_selections/explorer/state
+++ b/cdist/conf/type/__debconf_set_selections/explorer/state
@@ -41,10 +41,15 @@ test -s "${linesfile}" || {
 	exit 0
 }
 
+# assert __type_explorer is set (because it is used by the Perl script)
+: "${__type_explorer:?}"
+
 /usr/bin/perl -- - "${linesfile}" <<'EOF'
 use strict;
 use warnings "all";
 
+use Fcntl qw(:DEFAULT :flock);
+
 use Debconf::Db;
 use Debconf::Question;
 
@@ -78,7 +83,20 @@ sub state {
 	}
 }
 
-Debconf::Db->load(readonly => 'true');
+
+# Load Debconf DB but manually lock on the state explorer script,
+# because Debconf aborts immediately if executed concurrently.
+# This is not really an ideal solution because the Debconf DB could be locked by
+# another process (e.g. apt-get), but no way to achieve this could be found.
+# If you know how to, please provide a patch.
+my $lockfile = "%ENV{'__type_explorer'}/state";
+if (open my $lock_fh, '+<', $lockfile) {
+   flock $lock_fh, LOCK_EX or die "Cannot lock $lockfile";
+}
+{
+	Debconf::Db->load(readonly => 'true');
+}
+
 
 while (<>) {
 	# Read and process lines (taken from debconf-set-selections)
diff --git a/cdist/conf/type/__debconf_set_selections/nonparallel b/cdist/conf/type/__debconf_set_selections/nonparallel
new file mode 100644
index 00000000..e69de29b

From c00c8c20127af9881b4e295b7732524e7e1c5031 Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Mon, 10 May 2021 12:08:22 +0200
Subject: [PATCH 320/411] [__apt_key*] Deprecate __apt_key_uri and improve
 __apt_key

Previously this type was falling back to using the deprecated apt-key(8) by
checking for existence of files/directories on the controller host in
gencode-remote.

Adding `--use-deprecated-apt-key` as an explicit boolean serves two purposes:
1. It prevents fallbacks that might end up doing the wrong thing
   (as was the case)
2. It allows for a simple way to remove keys from the keyring that were
   previously added with apt-key(8) to /etc/apt/trusted.gpg

This parameter is added marked as deprecated as is only intended use is to
migrate to directory-based keyrings as recommended by Debian for a few releases.
It will be removed when Debian 11 stops being supported.

During the review process of this merge request, it was noted that the state of
PGP Key Servers is somewhat suboptimal, that the examples encouraged bad
practise (it is trivial to produce collisions for short key IDs), and that
this use does not require the Web of Trust, but instead only the public key
that is signing the repository.

That is why this also adds `--source` as an argument allowing for in-type or
in-manifest provision of such public keys by the type/manifest maintainer and
the use of Key Servers is still supported, but discouraged.
---
 cdist/conf/type/__apt_key/explorer/state      |  27 +++--
 cdist/conf/type/__apt_key/gencode-remote      |  53 ++++-----
 cdist/conf/type/__apt_key/man.rst             |  84 ++++++++++----
 cdist/conf/type/__apt_key/manifest            | 104 +++++++++++++++++-
 cdist/conf/type/__apt_key/parameter/boolean   |   1 +
 .../deprecated/use-deprecated-apt-key         |   3 +
 cdist/conf/type/__apt_key/parameter/optional  |   5 +-
 cdist/conf/type/__apt_key_uri/deprecated      |   1 +
 8 files changed, 209 insertions(+), 69 deletions(-)
 create mode 100644 cdist/conf/type/__apt_key/parameter/boolean
 create mode 100644 cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key
 create mode 100644 cdist/conf/type/__apt_key_uri/deprecated

diff --git a/cdist/conf/type/__apt_key/explorer/state b/cdist/conf/type/__apt_key/explorer/state
index 38f1bd3c..8ab268c1 100755
--- a/cdist/conf/type/__apt_key/explorer/state
+++ b/cdist/conf/type/__apt_key/explorer/state
@@ -27,18 +27,25 @@ else
    keyid="$__object_id"
 fi
 
+# From apt-key(8):
+#   Use of apt-key is deprecated, except for the use of apt-key del in
+#   maintainer scripts to remove existing keys from the main keyring.
+#   If such usage of apt-key is desired the additional installation of
+#   the GNU Privacy Guard suite (packaged in gnupg) is required.
+if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then
+   if apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK"
+      then echo present
+      else echo absent
+   fi
+   exit
+fi
+
 keydir="$(cat "$__object/parameter/keydir")"
 keyfile="$keydir/$__object_id.gpg"
 
-if [ -d "$keydir" ]
+if [ -f "$keyfile" ]
 then
-   if [ -f "$keyfile" ]
-   then echo present
-   else echo absent
-   fi
-else
-   # fallback to deprecated apt-key
-   apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \
-      && echo present \
-      || echo absent
+   echo present
+   exit
 fi
+echo absent
diff --git a/cdist/conf/type/__apt_key/gencode-remote b/cdist/conf/type/__apt_key/gencode-remote
index 0c96ff67..17dc9bfc 100755
--- a/cdist/conf/type/__apt_key/gencode-remote
+++ b/cdist/conf/type/__apt_key/gencode-remote
@@ -25,11 +25,7 @@ else
 fi
 state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
-
-if [ "$state_should" = "$state_is" ]; then
-   # nothing to do
-   exit 0
-fi
+method="$(cat "$__object/key_method")"
 
 keydir="$(cat "$__object/parameter/keydir")"
 keyfile="$keydir/$__object_id.gpg"
@@ -37,30 +33,18 @@ keyfile="$keydir/$__object_id.gpg"
 case "$state_should" in
    present)
       keyserver="$(cat "$__object/parameter/keyserver")"
-
-      if [ -f "$__object/parameter/uri" ]; then
-         uri="$(cat "$__object/parameter/uri")"
-
-         if [ -d "$keydir" ]; then
-            cat << EOF
-
-curl -s -L \\
-    -o "$keyfile" \\
-    "$uri"
-
-key="\$( cat "$keyfile" )"
-
-if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK'
-then
-    echo "\$key" | gpg --dearmor > "$keyfile"
-fi
-
-EOF
-         else
-            # fallback to deprecated apt-key
-            echo "curl -s -L '$uri' | apt-key add -"
+      # Using __download or __file as key source
+      # Propagate messages if needed
+      if [ "${method}" = "uri" ] || [ "${method}" = "source" ]; then
+         if grep -Eq "^__(file|download)$keyfile" "$__messages_in"; then
+            echo "added '$keyid'" >> "$__messages_out"
          fi
-      elif [ -d "$keydir" ]; then
+         exit 0
+      elif [ "${state_is}" = "present" ]; then
+         exit 0
+      fi
+      # Using key servers to fetch the key
+      if [ ! -f "$__object/parameter/use-deprecated-apt-key" ]; then
          # we need to kill gpg after 30 seconds, because gpg
          # can get stuck if keyserver is not responding.
          # exporting env var and not exit 1,
@@ -100,13 +84,16 @@ EOF
       echo "added '$keyid'" >> "$__messages_out"
    ;;
    absent)
-      if [ -f "$keyfile" ]; then
-         echo "rm '$keyfile'"
-      else
+      # Removal for keys added from a keyserver without this flag
+      # is done in the manifest
+      if [ "$state_is" != "absent" ] && \
+            [ -f "$__object/parameter/use-deprecated-apt-key" ]; then
          # fallback to deprecated apt-key
          echo "apt-key del \"$keyid\""
+         echo "removed '$keyid'" >> "$__messages_out"
+      # Propagate messages if needed
+      elif grep -Eq "^__file$keyfile" "$__messages_in"; then
+         echo "removed '$keyid'" >> "$__messages_out"
       fi
-
-      echo "removed '$keyid'" >> "$__messages_out"
    ;;
 esac
diff --git a/cdist/conf/type/__apt_key/man.rst b/cdist/conf/type/__apt_key/man.rst
index 234bc715..e35eaa0f 100644
--- a/cdist/conf/type/__apt_key/man.rst
+++ b/cdist/conf/type/__apt_key/man.rst
@@ -10,6 +10,14 @@ DESCRIPTION
 -----------
 Manages the list of keys used by apt to authenticate packages.
 
+This is done by placing the requested key in a file named
+``$__object_id.gpg`` in the ``keydir`` directory.
+
+This is supported by modern releases of Debian-based distributions.
+
+In order of preference, exactly one of: ``source``, ``uri`` or ``keyid``
+must be specified.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -18,21 +26,49 @@ None.
 
 OPTIONAL PARAMETERS
 -------------------
+keydir
+   keyring directory, defaults to ``/etc/apt/trusted.pgp.d``, which is
+   enabled system-wide by default.
+
+source
+   path to a file containing the GPG key of the repository.
+   Using this is recommended as it ensures that the manifest/type manintainer
+   has validated the key.
+   If ``-``, the GPG key is read from the type's stdin.
+
 state
    'present' or 'absent'. Defaults to 'present'
 
+uri
+   the URI from which to download the key.
+   It is highly recommended that you only use protocols with TLS like HTTPS.
+   This uses ``__download`` but does not use checksums, if you want to ensure
+   that the key doesn't change, you are better off downloading it and using
+   ``--source``.
+
+
+DEPRECATED OPTIONAL PARAMETERS
+------------------------------
 keyid
-   the id of the key to add. Defaults to __object_id
+   the id of the key to download from the ``keyserver``.
+   This is to be used in absence of ``--source`` and ``--uri`` or together
+   with ``--use-deprecated-apt-key`` for key removal.
+   Defaults to ``$__object_id``.
 
 keyserver
-   the keyserver from which to fetch the key. If omitted the default set
-   in ./parameter/default/keyserver is used.
+   the keyserver from which to fetch the key.
+   Defaults to ``pool.sks-keyservers.net``.
 
-keydir
-   key save location, defaults to ``/etc/apt/trusted.pgp.d``
 
-uri
-   the URI from which to download the key
+DEPRECATED BOOLEAN PARAMETERS
+-----------------------------
+use-deprecated-apt-key
+   ``apt-key(8)`` will last be available in Debian 11 and Ubuntu 22.04.
+   You can use this parameter to force usage of ``apt-key(8)``.
+   Please only use this parameter to *remove* keys from the keyring,
+   in order to prepare for removal of ``apt-key``.
+   Adding keys should be done without this parameter.
+   This parameter will be removed when Debian 11 stops being supported.
 
 
 EXAMPLES
@@ -40,33 +76,39 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # Add Ubuntu Archive Automatic Signing Key
-    __apt_key 437D05B5
-    # Same thing
-    __apt_key 437D05B5 --state present
-    # Get rid of it
-    __apt_key 437D05B5 --state absent
+    # add a key that has been verified by a type maintainer
+    __apt_key jitsi_meet_2021 \
+       --source cdist-contrib/type/__jitsi_meet/files/apt_2021.gpg
 
-    # same thing with human readable name and explicit keyid
-    __apt_key UbuntuArchiveKey --keyid 437D05B5
+    # remove an old, deprecated or expired key
+    __apt_key jitsi_meet_2016 --state absent
 
-    # same thing with other keyserver
-    __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com
+    # Get rid of a key that might have been added to
+    # /etc/apt/trusted.gpg with apt-key
+    __apt_key 0x40976EAF437D05B5 --use-deprecated-apt-key --state absent
 
-    # download key from the internet
-    __apt_key rabbitmq \
-       --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc
+    # add a key that we define in-line
+    __apt_key jitsi_meet_2021 --source '-' <<EOF
+    -----BEGIN PGP PUBLIC KEY BLOCK-----
+    [...]
+    -----END PGP PUBLIC KEY BLOCK-----
+    EOF
+
+    # download or update key from the internet
+    __apt_key rabbitmq_2007 \
+       --uri https://www.rabbitmq.com/rabbitmq-signing-key-public.asc
 
 
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
 Ander Punnar <ander-at-kvlt-dot-ee>
+Evilham <contact~~@~~evilham.com>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can
+Copyright \(C) 2011-2021 Steven Armstrong, Ander Punnar and Evilham. You can
 redistribute it and/or modify it under the terms of the GNU General Public
 License as published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__apt_key/manifest b/cdist/conf/type/__apt_key/manifest
index 010357cd..889a764a 100755
--- a/cdist/conf/type/__apt_key/manifest
+++ b/cdist/conf/type/__apt_key/manifest
@@ -2,7 +2,105 @@
 
 __package gnupg
 
-if [ -f "$__object/parameter/uri" ]
-then __package curl
-else __package dirmngr
+state_should="$(cat "${__object}/parameter/state")"
+
+incompatible_args()
+{
+	cat >> /dev/stderr <<-EOF
+	This type does not support --${1} and --${method} simultaneously.
+	EOF
+	exit 1
+}
+
+if [ -f "${__object}/parameter/source" ]; then
+	method="source"
+	src="$(cat "${__object}/parameter/source")"
+	if [ "${src}" = "-" ]; then
+		src="${__object}/stdin"
+	fi
+fi
+if [ -f "${__object}/parameter/uri" ]; then
+	if [ -n "${method}" ]; then
+		incompatible_args uri
+	fi
+	method="uri"
+	src="$(cat "${__object}/parameter/uri")"
+fi
+if [ -f "${__object}/parameter/keyid" ]; then
+	if [ -n "${method}" ]; then
+		incompatible_args keyid
+	fi
+	method="keyid"
+fi
+# Keep old default
+if [ -z "${method}" ]; then
+	method="keyid"
+fi
+# Save this for later in gencode-remote
+echo "${method}" > "${__object}/key_method"
+
+# Required remotely (most likely already installed)
+__package dirmngr
+# We need this in case a key has to be dearmor'd
+__package gnupg
+export require="__package/gnupg"
+
+if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then
+	# This is required if apt-key(8) is to be used
+	if [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then
+		incompatible_args use-deprecated-apt-key
+	fi
+else
+	if [ "${state_should}" = "absent" ] && \
+			[ -f "${__object}/parameter/keyid" ]; then
+		cat >> /dev/stderr <<EOF
+You can't reliably remove by keyid without --use-deprecated-apt-key.
+This would very likely do something you do not intend.
+EOF
+		exit 1
+	fi
+fi
+
+keydir="$(cat "${__object}/parameter/keydir")"
+keyfile="${keydir}/${__object_id}.gpg"
+keyfilecdist="${keyfile}.cdist"
+if [ "${state_should}" != "absent" ]; then
+	# Ensure keydir exists
+	__directory "${keydir}" --state exists --mode 0755
+fi
+
+if [ "${state_should}" = "absent" ]; then
+	__file "${keyfile}" --state "absent"
+	__file "${keyfilecdist}" --state "absent"
+elif [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then
+	dearmor="$(cat <<-EOF
+	if [ '${state_should}' = 'present' ]; then
+		# Dearmor if necessary
+		if grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK' '${keyfilecdist}'; then
+			gpg --dearmor < '${keyfilecdist}' > '${keyfile}'
+		else
+			cp '${keyfilecdist}' '${keyfile}'
+		fi
+		# Ensure permissions
+		chown root '${keyfile}'
+		chmod 0444 '${keyfile}'
+	fi
+	EOF
+	)"
+
+	if [ "${method}" = "uri" ]; then
+		__download "${keyfilecdist}" \
+			--url "${src}" \
+			--onchange "${dearmor}"
+		require="__download${keyfilecdist}" \
+			__file "${keyfile}" \
+				--owner root \
+				--mode 0444 \
+				--state pre-exists
+	else
+		__file "${keyfilecdist}" --state "${state_should}" \
+			--mode 0444 \
+			--source "${src}" \
+			--onchange "${dearmor}"
+	fi
 fi
diff --git a/cdist/conf/type/__apt_key/parameter/boolean b/cdist/conf/type/__apt_key/parameter/boolean
new file mode 100644
index 00000000..7263e1b2
--- /dev/null
+++ b/cdist/conf/type/__apt_key/parameter/boolean
@@ -0,0 +1 @@
+use-deprecated-apt-key
diff --git a/cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key b/cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key
new file mode 100644
index 00000000..d18ebe57
--- /dev/null
+++ b/cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key
@@ -0,0 +1,3 @@
+apt-key(8) will last be available in Debian 11 and Ubuntu 22.04.
+Use this flag *only* to migrate to placing a keyring directly in the
+/etc/apt/trusted.gpg.d/ directory with a descriptive name.
diff --git a/cdist/conf/type/__apt_key/parameter/optional b/cdist/conf/type/__apt_key/parameter/optional
index de647375..975d1fd8 100644
--- a/cdist/conf/type/__apt_key/parameter/optional
+++ b/cdist/conf/type/__apt_key/parameter/optional
@@ -1,5 +1,6 @@
-state
+keydir
 keyid
 keyserver
-keydir
+source
+state
 uri
diff --git a/cdist/conf/type/__apt_key_uri/deprecated b/cdist/conf/type/__apt_key_uri/deprecated
new file mode 100644
index 00000000..05550587
--- /dev/null
+++ b/cdist/conf/type/__apt_key_uri/deprecated
@@ -0,0 +1 @@
+Please migrate to using __apt_key key_id --uri URI.

From a696f3cf0026ec96e4cf7235ee8634391ccfecdb Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Mon, 10 May 2021 12:10:00 +0200
Subject: [PATCH 321/411] [__letsencrypt_cert] Revamp explorers, add locking.

This would fix #839

Certbot uses locking [1] even for read-only operations and does not properly
use exit codes, which means that sometimes it would print:
"Another instance of Certbot is already running" and exit with success.

However, the previous explorers would take that as the certificate being absent
and would trigger code generation.

The issue was made worse by having many explorers running certbot, so for N
certificates, we'd run certbot N*4 times, potentially "in parallel".

[1]: https://certbot.eff.org/docs/using.html#id5

This patch joins all explorers in one to avoid starting multiple remote python
processes and uses a cdist-specific lock in /tmp/certbot.cdist.lock with a
60 seconds timeout.

It has been tested with certbot 0.31.0 and 0.17 that the:

    from certbot.main import main

trick works. It is somewhat well documented so it can be somewhat relied upon.
---
 .../__letsencrypt_cert/explorer/certbot-path  |  3 -
 .../explorer/certificate-data                 | 78 +++++++++++++++++++
 .../explorer/certificate-domains              |  8 --
 .../explorer/certificate-exists               | 13 ----
 .../explorer/certificate-is-test              | 14 ----
 .../type/__letsencrypt_cert/gencode-remote    | 11 ++-
 cdist/conf/type/__letsencrypt_cert/manifest   |  2 +-
 7 files changed, 87 insertions(+), 42 deletions(-)
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certbot-path
 create mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-data
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test

diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path
deleted file mode 100755
index 3c6076df..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh -e
-
-command -v certbot 2>/dev/null || true
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data
new file mode 100755
index 00000000..ff62e742
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data
@@ -0,0 +1,78 @@
+#!/bin/sh -e
+certbot_path="$(command -v certbot 2>/dev/null || true)"
+# Defaults
+certificate_exists="no"
+certificate_is_test="no"
+
+if [ -n "${certbot_path}" ]; then
+	# Find python executable that has access to certbot's module
+	python_path=$(sed -n '1s/^#! *//p' "${certbot_path}")
+
+	# Use a lock for cdist due to certbot not exiting with failure
+	# or having any flags for concurrent use.
+	_certbot() {
+		${python_path} - 2>/dev/null <<EOF
+from certbot.main import main
+import fcntl
+lock_file = "/tmp/certbot.cdist.lock"
+timeout=60
+with open(lock_file, 'w') as fd:
+    for i in range(timeout):
+        try:
+            # Get exclusive lock
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            break
+        except:
+            # Wait if that fails
+            import time
+            time.sleep(1)
+    else:
+        # Timed out, exit with failure
+        import sys
+        sys.exit(1)
+    # Do list certificates
+    main(["certificates", "--cert-name", "${__object_id:?}"])
+EOF
+	}
+
+
+	_certificate_exists() {
+		if grep -q "  Certificate Name: ${__object_id:?}$"; then
+			echo yes
+		else
+			echo no
+		fi
+	}
+
+	_certificate_is_test() {
+		if grep -q 'INVALID: TEST_CERT'; then
+			echo yes
+		else
+			echo no
+		fi
+	}
+
+	_certificate_domains() {
+		grep '    Domains: ' | cut -d ' ' -f 6- | tr ' ' '\n'
+	}
+
+	# Get data about all available certificates
+	certificates="$(_certbot)"
+
+	# Check whether or not the certificate exists
+	certificate_exists="$(echo "${certificates}" | _certificate_exists)"
+
+	# Check whether or not the certificate is for testing
+	certificate_is_test="$(echo "${certificates}" | _certificate_is_test)"
+
+	# Get domains for certificate
+	certificate_domains="$(echo "${certificates}" | _certificate_domains)"
+fi
+
+# Return received data
+cat <<EOF
+certbot_path:${certbot_path}
+certificate_exists:${certificate_exists}
+certificate_is_test:${certificate_is_test}
+${certificate_domains}
+EOF
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains
deleted file mode 100755
index db605b63..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	certbot certificates --cert-name "${__object_id:?}" | grep '    Domains: ' | \
-		cut -d ' ' -f 6- | tr ' ' '\n'
-fi
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists
deleted file mode 100755
index 4e6f44db..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	if certbot certificates | grep -q "  Certificate Name: ${__object_id:?}$"; then
-		echo yes
-	else
-		echo no
-	fi
-else
-	echo no
-fi
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test
deleted file mode 100755
index 9b445059..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	if certbot certificates --cert-name "${__object_id:?}" | \
-		grep -q 'INVALID: TEST_CERT'; then
-		echo yes
-	else
-		echo no
-	fi
-else
-	echo no
-fi
diff --git a/cdist/conf/type/__letsencrypt_cert/gencode-remote b/cdist/conf/type/__letsencrypt_cert/gencode-remote
index 375570a4..60a77ec3 100755
--- a/cdist/conf/type/__letsencrypt_cert/gencode-remote
+++ b/cdist/conf/type/__letsencrypt_cert/gencode-remote
@@ -1,6 +1,10 @@
 #!/bin/sh -e
 
-certificate_exists=$(cat "${__object:?}/explorer/certificate-exists")
+_explorer_var() {
+	grep "^$1:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-
+}
+
+certificate_exists="$(_explorer_var certificate_exists)"
 name="${__object_id:?}"
 state=$(cat "${__object}/parameter/state")
 
@@ -29,8 +33,9 @@ case "${state}" in
 		fi
 
 		if [ "${certificate_exists}" = "yes" ]; then
-			existing_domains="${__object}/explorer/certificate-domains"
-			certificate_is_test=$(cat "${__object}/explorer/certificate-is-test")
+			existing_domains=$(mktemp "${TMPDIR:-/tmp}/existing_domains.cdist.XXXXXXXXXX")
+			tail -n +4 "${__object:?}/explorer/certificate-data" | grep -v '^$' > "${existing_domains}"
+			certificate_is_test="$(_explorer_var certificate_is_test)"
 
 			sort -uo "${requested_domains}" "${requested_domains}"
 			sort -uo "${existing_domains}" "${existing_domains}"
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 1df3574a..6394f629 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")"
+certbot_fullpath="$(grep "^certbot_path:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-)"
 state=$(cat "${__object}/parameter/state")
 os="$(cat "${__global:?}/explorer/os")"
 

From f14623e45f56aec4bdce53bc2b367e11ff157865 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 10 May 2021 12:17:08 +0200
Subject: [PATCH 322/411] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 497ae91a..5c3b547a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,8 @@ Changelog
 next:
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
+	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)
+	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 6210cccb28ace8ec2807ffe6ca9f9e4d9007990b Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 10 May 2021 12:34:04 +0200
Subject: [PATCH 323/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 5c3b547a..a7310788 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)
 	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
+	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking.
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 503a06ed28f743aad47797a0989be735c67b07a6 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 28 Apr 2021 13:32:10 +0300
Subject: [PATCH 324/411] [__git] fix group explorer

group name from numberic id wasn't resolved correctly.

try to use getent and fallback to reading /etc/group directly.
---
 cdist/conf/type/__git/explorer/group | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group
index 1365c60d..ab4396b1 100644
--- a/cdist/conf/type/__git/explorer/group
+++ b/cdist/conf/type/__git/explorer/group
@@ -14,6 +14,11 @@ then
 	then
 		printf '%u\n' "${group_gid}"
 	else
-		printf '%s\n' "$(id -u -n "${group_gid}")"
+		if command -v getent > /dev/null
+		then
+			getent group "${group_gid}" | cut -d : -f 1
+		else
+			awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group
+		fi
 	fi
 fi

From 75c71f69c1fed4371c5891ddcca0eaf28ad928e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 10:17:48 +0200
Subject: [PATCH 325/411] [scanner] pycodestyle compliance

---
 cdist/scan/commandline.py | 20 +++++++++++----
 cdist/scan/scan.py        | 54 ++++++++++++++++++++++++---------------
 2 files changed, 48 insertions(+), 26 deletions(-)

diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 3eb7eec4..b42bc7b2 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -25,13 +25,15 @@ from datetime import datetime
 
 log = logging.getLogger("scan")
 
+
 def run(scan, args):
     # We run each component in a separate process since they
     # must not block on each other.
     processes = []
 
     if 'trigger' in args.mode:
-        t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay)
+        t = scan.Trigger(interfaces=args.interfaces,
+                         sleeptime=args.trigger_delay)
         t.start()
         processes.append(t)
         log.debug("Trigger started")
@@ -48,6 +50,7 @@ def run(scan, args):
     for process in processes:
         process.join()
 
+
 def list(scan, args):
     s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper)
     hosts = s.list()
@@ -66,10 +69,16 @@ def list(scan, args):
     print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
     for host in hosts:
         last_seen = host.last_seen()
-        last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-'
+        if last_seen:
+            last_seen = last_seen.strftime(scan.datetime_format)
+        else:
+            last_seen = '-'
 
         last_configured = host.last_configured()
-        last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-'
+        if last_configured:
+            last_configured = last_configured.strftime(scan.datetime_format)
+        else:
+            '-'
 
         print("{} | {} | {} | {}".format(
             host.name(default='-').ljust(name_max_size),
@@ -77,6 +86,7 @@ def list(scan, args):
             last_seen.ljust(date_max_size),
             last_configured.ljust(date_max_size)))
 
+
 # CLI processing is defined outside of the main scan class to handle
 # non-available optional scapy dependency (instead of crashing mid-flight).
 def commandline(args):
@@ -94,9 +104,9 @@ def commandline(args):
         # By default scan and trigger, but do not call any action.
         args.mode = ['scan', 'trigger', ]
 
-    if 'config' in args.mode and args.name_mapper == None:
+    if 'config' in args.mode and args.name_mapper is None:
         print('--name-mapper must be specified for scanner config mode.',
-                file=sys.stderr)
+              file=sys.stderr)
         sys.exit(1)
 
     # Print known hosts and exit is --list is specified - do not start
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 2912dab3..4a20f511 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -33,6 +33,7 @@ logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("scan")
 datetime_format = '%Y-%m-%d %H:%M:%S'
 
+
 class Host(object):
     def __init__(self, addr, outdir, name_mapper=None):
         self.addr = addr
@@ -43,7 +44,7 @@ class Host(object):
 
     def __get(self, key, default=None):
         fname = os.path.join(self.workdir, key)
-        value=default
+        value = default
         if os.path.isfile(fname):
             with open(fname, "r") as fd:
                 value = fd.readline()
@@ -55,15 +56,15 @@ class Host(object):
             fd.write(f"{value}")
 
     def name(self, default=None):
-        if self.name_mapper == None:
+        if self.name_mapper is None:
             return default
 
         fpath = os.path.join(os.getcwd(), self.name_mapper)
         if os.path.isfile(fpath) and os.access(fpath, os.X_OK):
-             out = subprocess.run([fpath, self.addr], capture_output=True)
-             if out.returncode != 0:
-                 return default
-             else:
+            out = subprocess.run([fpath, self.addr], capture_output=True)
+            if out.returncode != 0:
+                return default
+            else:
                 value = out.stdout.decode()
                 return (default if len(value) == 0 else value)
         else:
@@ -94,19 +95,20 @@ class Host(object):
     # CLI args. Might as well call everything from scratch!
     def configure(self):
         target = self.name() or self.address()
-        cmd = ['cdist', 'config', '-v', target ]
+        cmd = ['cdist', 'config', '-v', target]
 
         fname = os.path.join(self.workdir, 'last_configuration_log')
         with open(fname, "w") as fd:
             log.debug("Executing: %s", cmd)
             completed_process = subprocess.run(cmd, stdout=fd, stderr=fd)
             if completed_process.returncode != 0:
-                log.error("%s return with non-zero code %i - see %s for details.",
-                        cmd, completed_process.returncode, fname)
+                log.error("%s return with non-zero code %i - see %s for \
+                        details.", cmd, completed_process.returncode, fname)
 
         now = datetime.datetime.now().strftime(datetime_format)
         self.__set('last_configured', now)
 
+
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
@@ -140,10 +142,12 @@ class Trigger(object):
     def trigger(self, interface):
         try:
             log.debug("Sending ICMPv6EchoRequest on %s", interface)
-            packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest()
+            packet = IPv6(
+                    dst="ff02::1%{}".format(interface)
+                    ) / ICMPv6EchoRequest()
             send(packet, verbose=self.verbose)
         except Exception as e:
-            log.error( "Could not send ICMPv6EchoRequest: %s", e)
+            log.error("Could not send ICMPv6EchoRequest: %s", e)
 
 
 class Scanner(object):
@@ -151,9 +155,10 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None):
+    def __init__(self, interfaces, autoconfigure=False, outdir=None,
+                 name_mapper=None):
         self.interfaces = interfaces
-        self.autoconfigure=autoconfigure
+        self.autoconfigure = autoconfigure
         self.name_mapper = name_mapper
         self.config_delay = datetime.timedelta(seconds=3600)
 
@@ -169,14 +174,17 @@ class Scanner(object):
         if ICMPv6EchoReply in pkg:
             host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper)
             if host.name():
-                log.verbose("Host %s (%s) is alive", host.name(), host.address())
+                log.verbose("Host %s (%s) is alive", host.name(),
+                            host.address())
             else:
                 log.verbose("Host %s is alive", host.address())
+
             host.seen()
 
             # Configure if needed.
             if self.autoconfigure and \
-                    host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now():
+                    host.last_configured(default=datetime.datetime.min) + \
+                    self.config_delay < datetime.datetime.now():
                 self.config(host)
 
     def list(self):
@@ -187,15 +195,19 @@ class Scanner(object):
         return hosts
 
     def config(self, host):
-        if host.name() == None:
-            log.debug("config - could not resolve name for %s, aborting.", host.address())
+        if host.name() is None:
+            log.debug("config - could not resolve name for %s, aborting.",
+                      host.address())
             return
 
         previous_config_process = self.running_configs.get(host.name())
-        if previous_config_process != None and previous_config_process.is_alive():
-            log.debug("config - is already running for %s, aborting.", host.name())
+        if previous_config_process is not None and \
+                previous_config_process.is_alive():
+            log.debug("config - is already running for %s, aborting.",
+                      host.name())
 
-        log.info("config - running against host %s (%s).", host.name(), host.address())
+        log.info("config - running against host %s (%s).", host.name(),
+                 host.address())
         p = Process(target=host.configure())
         p.start()
         self.running_configs[host.name()] = p
@@ -214,4 +226,4 @@ class Scanner(object):
                   filter="icmp6",
                   prn=self.handle_pkg)
         except Exception as e:
-            log.error( "Could not start listener: %s", e)
+            log.error("Could not start listener: %s", e)

From ab10b453f275d4289f3d37988b3caec204227194 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 11:15:41 +0200
Subject: [PATCH 326/411] [scanner] populate cdist(1)

---
 docs/src/man1/cdist.rst | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 0ecb4a61..599ec3b7 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -88,6 +88,9 @@ SYNOPSIS
     cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t]
                [pattern]
 
+    cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list]
+               [-d CONFIG_DELAY] [-t TRIGGER_DELAY]
+
 
 DESCRIPTION
 -----------
@@ -641,6 +644,31 @@ Display information for cdist (global explorers, types).
 **-t, --types**
     Display info for types.
 
+SCAN
+----
+
+Runs cdist as a daemon that discover/watch on hosts and reconfigure them
+periodically.
+
+**-I INTERFACE, --interfaces INTERFACE**
+    Interface to listen on. Can be specified multiple times.
+
+**-m MODE, --mode MODE**
+    Scanner components to enable. Can be specified multiple time to enable more
+    than one component. Supported modes are: scan, trigger and config. Defaults
+    to tiggger and scan.
+
+**--name-mapper PATH_TO_SCRIPT**
+    Path to script used to resolve a remote host name from an IPv6 address.
+
+**--list**
+    List known hosts and exit.
+
+**-d CONFIG_DELAY, --config-delay CONFIG_DELAY**
+    How long (seconds) to wait before reconfiguring after last try (config mode only).
+
+**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY**
+    How long (seconds) to wait between ICMPv6 echo requests (trigger mode only).
 
 CONFIGURATION
 -------------

From b8733c65f52776facce7a8de0265538a856ead64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 11:26:35 +0200
Subject: [PATCH 327/411] [scanner] fix minor CLI handling and --list bugs /
 typo

---
 cdist/argparse.py         |  4 ++--
 cdist/scan/commandline.py | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index bedb23ac..f17315e7 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -495,7 +495,7 @@ def get_parsers():
         action='store_true',
         help='Try to configure detected hosts')
     parser['scan'].add_argument(
-        '-I', '--interfaces',
+        '-I', '--interface',
         action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
     parser['scan'].add_argument(
@@ -503,7 +503,7 @@ def get_parsers():
         action='store',  default=None,
         help='Map addresses to names, required for config mode')
     parser['scan'].add_argument(
-        '-d', '--delay',
+        '-d', '--config-delay',
         action='store',  default=3600, type=int,
         help='How long (seconds) to wait before reconfiguring after last try')
     parser['scan'].add_argument(
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index b42bc7b2..ddbe4933 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -32,7 +32,7 @@ def run(scan, args):
     processes = []
 
     if 'trigger' in args.mode:
-        t = scan.Trigger(interfaces=args.interfaces,
+        t = scan.Trigger(interfaces=args.interface,
                          sleeptime=args.trigger_delay)
         t.start()
         processes.append(t)
@@ -41,7 +41,7 @@ def run(scan, args):
     if 'scan' in args.mode:
         s = scan.Scanner(
                 autoconfigure='config' in args.mode,
-                interfaces=args.interfaces,
+                interfaces=args.interface,
                 name_mapper=args.name_mapper)
         s.start()
         processes.append(s)
@@ -52,7 +52,7 @@ def run(scan, args):
 
 
 def list(scan, args):
-    s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper)
+    s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper)
     hosts = s.list()
 
     # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
@@ -75,10 +75,10 @@ def list(scan, args):
             last_seen = '-'
 
         last_configured = host.last_configured()
-        if last_configured:
+        if last_configured is not None:
             last_configured = last_configured.strftime(scan.datetime_format)
         else:
-            '-'
+            last_configured = '-'
 
         print("{} | {} | {} | {}".format(
             host.name(default='-').ljust(name_max_size),

From e0c52d0e1dfbaa3814a2ff26482177c2909a15ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 11:27:11 +0200
Subject: [PATCH 328/411] [scanner] remove mention of non-implemented trigger
 soruce script

---
 docs/src/cdist-scan.rst | 21 ++-------------------
 1 file changed, 2 insertions(+), 19 deletions(-)

diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst
index 02193456..064e65ff 100644
--- a/docs/src/cdist-scan.rst
+++ b/docs/src/cdist-scan.rst
@@ -8,7 +8,8 @@ periodically. It is especially useful in netboot-based environment where hosts
 boot unconfigured, and to ensure your infrastructure stays in sync with your
 configuration.
 
-This feature is still consider to be in **beta** stage.
+This feature is still consider to be in **beta** stage, and only operate on
+IPv6 (including link-local).
 
 Usage (Examples)
 ----------------
@@ -79,21 +80,3 @@ Resolving name from `PTR` DNS record:
   done
 
   dig +short -x "$1" | sed -e 's/.$//'
-
-
-Trigger Source Script
----------------------
-
-This script returns a list of addresses (separated by a newline) to be used by
-`trigger` mode. It is not used to map names. The script must be executable.
-
-Simplest script:
-
-.. code-block:: sh
-  #!/bin/sh
-
-  cat << EOF
-  server1.domain.tld
-  server2.domain.tld
-  server3.domain.tld
-  EOF

From defa3c22eaef88652a2bec6135d038a84710c41e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 29 May 2021 11:21:34 +0200
Subject: [PATCH 329/411] ++changelog

---
 docs/changelog | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index a7310788..403e7653 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,7 +6,8 @@ next:
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)
 	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
-	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking.
+	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham)
+	* Type __git: Fix group explorer (Ander Punnar)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From d596986af894749877fb3dcadf90628c9cfa9d13 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 31 May 2021 09:06:52 +0200
Subject: [PATCH 330/411] [type/__pyvenv] Fix group explorer

---
 cdist/conf/type/__pyvenv/explorer/group | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group
index f31a1cb7..922ce3df 100755
--- a/cdist/conf/type/__pyvenv/explorer/group
+++ b/cdist/conf/type/__pyvenv/explorer/group
@@ -14,6 +14,11 @@ then
 	then
 		printf '%u\n' "${group_gid}"
 	else
-		printf '%s\n' "$(id -u -n "${group_gid}")"
+		if command -v getent >/dev/null 2>&1
+		then
+			getent group "${group_gid}" | cut -d : -f 1
+		else
+			awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group
+		fi
 	fi
 fi

From 6ede76b08b96b7504c65ca2dc6737e31be7e7f24 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 8 Jun 2021 16:20:55 +0200
Subject: [PATCH 331/411] [type/__debconf_set_selections] man.rst: Fix line
 break in AUTHORS

---
 cdist/conf/type/__debconf_set_selections/man.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst
index 690e3e49..fd0040ae 100644
--- a/cdist/conf/type/__debconf_set_selections/man.rst
+++ b/cdist/conf/type/__debconf_set_selections/man.rst
@@ -58,8 +58,8 @@ SEE ALSO
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
-Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING

From c308a2896971ac4808be0152630ef41b7f96a2de Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 10 Jun 2021 06:39:55 +0200
Subject: [PATCH 332/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 403e7653..97ea204b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
 	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham)
 	* Type __git: Fix group explorer (Ander Punnar)
+	* Type __pyvenv: Fix group explorer (Dennis Camera)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From a3102022e18a23ce9b54eeaf7415b55361f80bd1 Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Fri, 11 Jun 2021 15:05:17 +0100
Subject: [PATCH 333/411] More sensible defaults; reword debian-only error
 message

---
 cdist/conf/type/__apt_pin/manifest                  | 13 +++++++++----
 cdist/conf/type/__apt_pin/nonparallel               |  0
 cdist/conf/type/__apt_pin/parameter/default/package |  1 -
 3 files changed, 9 insertions(+), 5 deletions(-)
 create mode 100644 cdist/conf/type/__apt_pin/nonparallel
 delete mode 100644 cdist/conf/type/__apt_pin/parameter/default/package

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index b1372ad0..909bc80d 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -19,9 +19,17 @@
 #
 
 
+name="$__object_id"
+
 os=$(cat "$__global/explorer/os")
 state="$(cat "$__object/parameter/state")"
-package="$(cat "$__object/parameter/package")"
+
+if [ -f "$__object/parameter/package" ]; then
+    package="$(cat "$__object/parameter/package")"
+else
+    package=$name
+fi
+
 distribution="$(cat "$__object/parameter/distribution")"
 priority="$(cat "$__object/parameter/priority")"
 
@@ -31,13 +39,10 @@ case "$os" in
    ;;
    *)
       printf "This type is specific to Debian and it's derivatives" >&2
-      printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2
       exit 1
    ;;
 esac
 
-name="$__object_id"
-
 case $distribution in
     stabletesting|unsatbel|experimental)
         pin="release a=$distribution"
diff --git a/cdist/conf/type/__apt_pin/nonparallel b/cdist/conf/type/__apt_pin/nonparallel
new file mode 100644
index 00000000..e69de29b
diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package
deleted file mode 100644
index 72e8ffc0..00000000
--- a/cdist/conf/type/__apt_pin/parameter/default/package
+++ /dev/null
@@ -1 +0,0 @@
-*

From b726697e070e5266eae38c7951ced14a2305acb2 Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Fri, 11 Jun 2021 15:05:33 +0100
Subject: [PATCH 334/411] Add documentation

---
 cdist/conf/type/__apt_pin/man.rst | 37 ++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst
index 7fcae6f8..0c91cdec 100644
--- a/cdist/conf/type/__apt_pin/man.rst
+++ b/cdist/conf/type/__apt_pin/man.rst
@@ -13,11 +13,21 @@ This space intentionally left blank.
 
 REQUIRED PARAMETERS
 -------------------
-None.
+distribution
+   Specifies what distribution the package should be pinned to. Accepts both codenames (buster/bullseye/sid) and suite names (stable/testing/...).
 
 
 OPTIONAL PARAMETERS
 -------------------
+package
+   Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used.
+
+priority
+   The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority)
+
+state
+   Will be passed to underlying `__file` type; see there for valid values and defaults.
+
 None.
 
 
@@ -31,14 +41,31 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # TODO
-    __apt_pin
+   # Add the bullseye repo to buster, but do not install any pacakges by default
+   # only if explicitely asked for
+    __apt_pin bullseye-default \
+       --package "*" \
+       --distribution bullseye \
+       --priority -1
+
+    require="__apt_pin/bullseye-default" __apt_source bullseye \
+       --uri http://deb.debian.org/debian/ \
+       --distribution bullseye \
+       --component main
+       # TODO
+       __apt_pin
+
+    __apt_pin foo --package "foo foo-*" --distribution bullseye
+
+    __foo # Installs the `foo` package internally
+
+    __package foo-plugin-extras
 
 
 SEE ALSO
 --------
-:strong:`TODO`\ (7)
-
+:strong:`apt_preferences`\ (7)
+:strong:`cdist-type__file`\ (7)
 
 AUTHORS
 -------

From 7b3f268df25922c515174862da95569aea547367 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 22 Jun 2021 16:36:30 +0300
Subject: [PATCH 335/411] [__download] improvements

1. post download checksum verification
2. detect hashes without prefix
3. add optional --destination
4. updated man
---
 .../conf/type/__download/explorer/remote_cmd  |  19 ---
 .../type/__download/explorer/remote_cmd_get   |  16 +++
 .../type/__download/explorer/remote_cmd_sum   |  82 +++++++++++
 cdist/conf/type/__download/explorer/state     |  60 ++------
 cdist/conf/type/__download/gencode-local      | 131 +++++++++++++++---
 cdist/conf/type/__download/gencode-remote     |  46 +++++-
 cdist/conf/type/__download/man.rst            |  31 ++++-
 cdist/conf/type/__download/manifest           |   2 +-
 cdist/conf/type/__download/parameter/optional |   3 +-
 9 files changed, 292 insertions(+), 98 deletions(-)
 delete mode 100755 cdist/conf/type/__download/explorer/remote_cmd
 create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_get
 create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_sum

diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd
deleted file mode 100755
index e3e35b45..00000000
--- a/cdist/conf/type/__download/explorer/remote_cmd
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh -e
-
-if [ -f "$__object/parameter/cmd-get" ]
-then
-    cmd="$( cat "$__object/parameter/cmd-get" )"
-
-elif command -v curl > /dev/null
-then
-    cmd="curl -L -o - '%s'"
-
-elif command -v fetch > /dev/null
-then
-    cmd="fetch -o - '%s'"
-
-else
-    cmd="wget -O - '%s'"
-fi
-
-echo "$cmd"
diff --git a/cdist/conf/type/__download/explorer/remote_cmd_get b/cdist/conf/type/__download/explorer/remote_cmd_get
new file mode 100755
index 00000000..9f1cd59c
--- /dev/null
+++ b/cdist/conf/type/__download/explorer/remote_cmd_get
@@ -0,0 +1,16 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cat "$__object/parameter/cmd-get"
+elif
+    command -v curl > /dev/null
+then
+    echo "curl -sSL -o - '%s'"
+elif
+    command -v fetch > /dev/null
+then
+    echo "fetch -o - '%s'"
+else
+    echo "wget -O - '%s'"
+fi
diff --git a/cdist/conf/type/__download/explorer/remote_cmd_sum b/cdist/conf/type/__download/explorer/remote_cmd_sum
new file mode 100755
index 00000000..84df663c
--- /dev/null
+++ b/cdist/conf/type/__download/explorer/remote_cmd_sum
@@ -0,0 +1,82 @@
+#!/bin/sh -e
+
+if [ ! -f "$__object/parameter/sum" ]
+then
+    exit 0
+fi
+
+if [ -f "$__object/parameter/cmd-sum" ]
+then
+    cat "$__object/parameter/cmd-sum"
+    exit 0
+fi
+
+sum_should="$( cat "$__object/parameter/sum" )"
+
+if echo "$sum_should" | grep -Fq ':'
+then
+    sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
+else
+    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+    then
+        sum_hash='cksum'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
+    then
+        sum_hash='md5'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
+    then
+        sum_hash='sha1'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
+    then
+        sum_hash='sha256'
+    else
+        echo 'hash format detection failed' >&2
+        exit 1
+    fi
+fi
+
+os="$( "$__explorer/os" )"
+
+case "$sum_hash" in
+    cksum)
+        echo "cksum %s | awk '{print \$1\" \"\$2}'"
+    ;;
+    md5)
+        case "$os" in
+            freebsd)
+                echo "md5 -q %s"
+            ;;
+            *)
+                echo "md5sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    sha1)
+        case "$os" in
+            freebsd)
+                echo "sha1 -q %s"
+            ;;
+            *)
+                echo "sha1sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    sha256)
+        case "$os" in
+            freebsd)
+                echo "sha256 -q %s"
+            ;;
+            *)
+                echo "sha256sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    *)
+        # we arrive here only if --sum is given with unknown format prefix
+        echo "unknown hash format: $sum_hash" >&2
+        exit 1
+    ;;
+esac
diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 68b517c5..881a1c09 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -1,6 +1,11 @@
 #!/bin/sh -e
 
-dst="/$__object_id"
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
+else
+    dst="/$__object_id"
+fi
 
 if [ ! -f "$dst" ]
 then
@@ -16,57 +21,18 @@ fi
 
 sum_should="$( cat "$__object/parameter/sum" )"
 
-if [ -f "$__object/parameter/cmd-sum" ]
+if echo "$sum_should" | grep -Fq ':'
 then
-    # shellcheck disable=SC2059
-    sum_is="$( eval "$( printf \
-        "$( cat "$__object/parameter/cmd-sum" )" \
-        "$dst" )" )"
-else
-    os="$( "$__explorer/os" )"
-
-    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
-    then
-        sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
-
-    elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="md5:$( md5 -q "$dst" )"
-            ;;
-            *)
-                sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-
-    elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="sha1:$( sha1 -q "$dst" )"
-            ;;
-            *)
-                sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-
-    elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="sha256:$( sha256 -q "$dst" )"
-            ;;
-            *)
-                sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-    fi
+    sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
 fi
 
+sum_cmd="$( "$__type_explorer/remote_cmd_sum" )"
+
+sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )"
+
 if [ -z "$sum_is" ]
 then
-    echo 'no checksum from target' >&2
+    echo 'existing destination checksum failed' >&2
     exit 1
 fi
 
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 571d2c3c..d1b0d0d5 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -11,34 +11,133 @@ fi
 
 url="$( cat "$__object/parameter/url" )"
 
-tmp="$( mktemp )"
-
-dst="/$__object_id"
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
+else
+    dst="/$__object_id"
+fi
 
 if [ -f "$__object/parameter/cmd-get" ]
 then
     cmd="$( cat "$__object/parameter/cmd-get" )"
 
-elif command -v wget > /dev/null
-then
-    cmd="wget -O - '%s'"
-
 elif command -v curl > /dev/null
 then
-    cmd="curl -L -o - '%s'"
+    cmd="curl -sSL -o - '%s'"
 
 elif command -v fetch > /dev/null
 then
     cmd="fetch -o - '%s'"
 
+elif command -v wget > /dev/null
+then
+    cmd="wget -O - '%s'"
+
 else
-    echo 'no usable locally installed utility for downloading' >&2
+    echo 'local download failed, no usable utility' >&2
     exit 1
 fi
 
-printf "$cmd > %s\n" \
-    "$url" \
-    "$tmp"
+echo "download_tmp=\"\$( mktemp )\""
+
+# shellcheck disable=SC2059
+printf "$cmd > \"\$download_tmp\"\n" "$url"
+
+if [ -f "$__object/parameter/sum" ]
+then
+    sum_should="$( cat "$__object/parameter/sum" )"
+
+    if [ -f "$__object/parameter/cmd-sum" ]
+    then
+        local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
+    else
+        if echo "$sum_should" | grep -Fq ':'
+        then
+            sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
+
+            sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
+        else
+            if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+            then
+                sum_hash='cksum'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
+            then
+                sum_hash='md5'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
+            then
+                sum_hash='sha1'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
+            then
+                sum_hash='sha256'
+            else
+                echo 'hash format detection failed' >&2
+                exit 1
+            fi
+        fi
+
+        case "$sum_hash" in
+            cksum)
+                local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'"
+            ;;
+            md5)
+                if command -v md5 > /dev/null
+                then
+                    local_cmd_sum="md5 -q %s"
+                elif
+                    command -v md5sum > /dev/null
+                then
+                    local_cmd_sum="md5sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            sha1)
+                if command -v sha1 > /dev/null
+                then
+                    local_cmd_sum="sha1 -q %s"
+                elif
+                    command -v sha1sum > /dev/null
+                then
+                    local_cmd_sum="sha1sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            sha256)
+                if command -v sha256 > /dev/null
+                then
+                    local_cmd_sum="sha256 -q %s"
+                elif
+                    command -v sha256sum > /dev/null
+                then
+                    local_cmd_sum="sha256sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            *)
+                # we arrive here only if --sum is given with unknown format prefix
+                echo "unknown hash format: $sum_hash" >&2
+                exit 1
+            ;;
+        esac
+
+        if [ -z "$local_cmd_sum" ]
+        then
+            echo 'local checksum verification failed, no usable utility' >&2
+            exit 1
+        fi
+    fi
+
+    # shellcheck disable=SC2059
+    echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\""
+
+    echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
+
+    echo "echo 'local download checksum mismatch' >&2"
+
+    echo "rm -f \"\$download_tmp\""
+
+    echo 'exit 1; fi'
+fi
 
 if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$'
 then
@@ -47,12 +146,10 @@ else
     target_host="$__target_host"
 fi
 
-printf '%s %s %s:%s\n' \
+# shellcheck disable=SC2016
+printf '%s "$download_tmp" %s:%s\n' \
     "$__remote_copy" \
-    "$tmp" \
     "$target_host" \
     "$dst"
 
-echo "rm -f '$tmp'"
-
-echo 'downloaded' > "$__messages_out"
+echo "rm -f \"\$download_tmp\""
diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
index 029a0801..e49bcec3 100755
--- a/cdist/conf/type/__download/gencode-remote
+++ b/cdist/conf/type/__download/gencode-remote
@@ -6,17 +6,51 @@ state_is="$( cat "$__object/explorer/state" )"
 
 if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
 then
-    cmd="$( cat "$__object/explorer/remote_cmd" )"
+    cmd_get="$( cat "$__object/explorer/remote_cmd_get" )"
 
     url="$( cat "$__object/parameter/url" )"
 
-    dst="/$__object_id"
+    if [ -f "$__object/parameter/destination" ]
+    then
+        dst="$( cat "$__object/parameter/destination" )"
+    else
+        dst="/$__object_id"
+    fi
 
-    printf "$cmd > %s\n" \
-        "$url" \
-        "$dst"
+    echo "download_tmp=\"\$( mktemp )\""
 
-    echo 'downloaded' > "$__messages_out"
+    # shellcheck disable=SC2059
+    printf "$cmd_get > \"\$download_tmp\"\n" "$url"
+
+    if [ -f "$__object/parameter/sum" ]
+    then
+        sum_should="$( cat "$__object/parameter/sum" )"
+
+        if [ -f "$__object/parameter/cmd-sum" ]
+        then
+            remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
+        else
+            remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )"
+
+            if echo "$sum_should" | grep -Fq ':'
+            then
+                sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
+            fi
+        fi
+
+        # shellcheck disable=SC2059
+        echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\""
+
+        echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
+
+        echo "echo 'remote download checksum mismatch' >&2"
+
+        echo "rm -f \"\$download_tmp\""
+
+        echo 'exit 1; fi'
+    fi
+
+    echo "mv \"\$download_tmp\" '$dst'"
 fi
 
 if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ]
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index a1278cfb..c16510a9 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -8,7 +8,7 @@ cdist-type__download - Download a file
 
 DESCRIPTION
 -----------
-By default type will try to use ``wget``, ``curl`` or ``fetch``.
+By default type will try to use ``curl``, ``fetch`` or ``wget``.
 If download happens in target (see ``--download``) then type will
 fallback to (and install) ``wget``.
 
@@ -16,6 +16,8 @@ If download happens in local machine, then environment variables like
 ``{http,https,ftp}_proxy`` etc can be used on cdist execution
 (``http_proxy=foo cdist config ...``).
 
+To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -25,14 +27,29 @@ url
 
 OPTIONAL PARAMETERS
 -------------------
+destination
+   Downloaded file's destination in target. If unset, ``$__object_id`` is used.
+
 sum
-   Checksum is used to decide if existing destination file must be redownloaded.
-   By default output of ``cksum`` without filename is expected.
-   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
+   Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256.
+
+   Type tries to detect hash format with regexes, but prefixes
+   ``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported.
+
+   Checksum have two purposes - state check and post-download verification.
+   In state check, if destination checksum mismatches, then content of URL
+   will be downloaded to temporary file. If downloaded temporary file's
+   checksum matches, then it will be moved to destination (overwritten).
+
+   For local downloads it is expected that usable utilities for checksum
+   calculation exist in the system.
 
 download
-   If ``local`` (default), then download file to local storage and copy
-   it to target host. If ``remote``, then download happens in target.
+   If ``local`` (default), then file is downloaded to local storage and copied
+   to target host. If ``remote``, then download happens in target.
+
+   For local downloads it is expected that usable utilities for downloading
+   exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``.
 
 cmd-get
    Command used for downloading.
@@ -62,7 +79,7 @@ EXAMPLES
     require='__directory/opt/cpma' \
         __download /opt/cpma/cnq3.zip \
             --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
-            --sum md5:46da3021ca9eace277115ec9106c5b46
+            --sum 46da3021ca9eace277115ec9106c5b46
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest
index 7ec8d86d..3d4c498b 100755
--- a/cdist/conf/type/__download/manifest
+++ b/cdist/conf/type/__download/manifest
@@ -1,6 +1,6 @@
 #!/bin/sh -e
 
-if grep -Eq '^wget' "$__object/explorer/remote_cmd"
+if grep -Eq '^wget' "$__object/explorer/remote_cmd_get"
 then
     __package wget
 fi
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index d69e083e..e809ef78 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,5 +1,6 @@
-sum
 cmd-get
 cmd-sum
+destination
 download
 onchange
+sum

From 2db40d8d704b427768307fbea29384bd3dc8dbd7 Mon Sep 17 00:00:00 2001
From: fancsali <fancsali@gmail.com>
Date: Mon, 28 Jun 2021 12:54:20 +0200
Subject: [PATCH 336/411] Use $__remote_exec and thus the ssh multiplexing

---
 cdist/conf/type/__rsync/gencode-local | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index e36ded2f..36addc36 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -36,4 +36,5 @@ fi
 
 echo rsync -a \
     --no-owner --no-group \
+    -e "$__remote_exec" \
     -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"

From d937d53f3dfd10830a07aee0450596eea62f2a1a Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Mon, 28 Jun 2021 18:09:35 +0100
Subject: [PATCH 337/411] Add quotes to rsync command

---
 cdist/conf/type/__rsync/gencode-local | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index 36addc36..f1bddc16 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -36,5 +36,5 @@ fi
 
 echo rsync -a \
     --no-owner --no-group \
-    -e "$__remote_exec" \
+    -e \"$__remote_exec\" \
     -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"

From 60753ddfcc3ac49303d572da7f6a68f398c02227 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 1 Jul 2021 14:42:10 +0300
Subject: [PATCH 338/411] fix shellcheck

---
 cdist/conf/type/__download/explorer/state | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 881a1c09..8c5d5ce1 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -28,6 +28,7 @@ fi
 
 sum_cmd="$( "$__type_explorer/remote_cmd_sum" )"
 
+# shellcheck disable=SC2059
 sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )"
 
 if [ -z "$sum_is" ]

From a90e642c1354cf01be7c9a1ba8a468ad624c4202 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 1 Jul 2021 14:50:40 +0300
Subject: [PATCH 339/411] update README

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index de6901c7..a468dd86 100644
--- a/README.md
+++ b/README.md
@@ -24,8 +24,8 @@ For community-maintained types there is
 
 ## Participating
 
-IRC: ``#cdist`` @ freenode
+IRC: ``#cdist`` @ [libera](https://libera.chat)
 
 Matrix: ``#cdist:ungleich.ch``
 
-Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist
+Matrix and IRC are bridged.

From 243a4b904a2de638d00bac57f6e762a076f9ae54 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 2 Jul 2021 06:50:02 +0200
Subject: [PATCH 340/411] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 97ea204b..d48c657e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,8 @@ next:
 	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham)
 	* Type __git: Fix group explorer (Ander Punnar)
 	* Type __pyvenv: Fix group explorer (Dennis Camera)
+	* Type __download: Improve checksum verification, add optional --destination (Ander Punnar)
+	* Type __debconf_set_selections: Add state explorer (Dennis Camera)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 30ba796d060fd3d0c4affd4167f14784156b5354 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 1 Jul 2021 11:49:07 +0300
Subject: [PATCH 341/411] new type: __snakeoil_cert

---
 .../__snakeoil_cert/explorer/ssl-cert-group   |  8 ++
 .../conf/type/__snakeoil_cert/explorer/state  | 24 ++++++
 .../conf/type/__snakeoil_cert/gencode-remote  | 73 +++++++++++++++++++
 cdist/conf/type/__snakeoil_cert/man.rst       | 60 +++++++++++++++
 .../parameter/default/cert-path               |  1 +
 .../parameter/default/key-path                |  1 +
 .../parameter/default/key-type                |  1 +
 .../type/__snakeoil_cert/parameter/optional   |  4 +
 8 files changed, 172 insertions(+)
 create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group
 create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/state
 create mode 100755 cdist/conf/type/__snakeoil_cert/gencode-remote
 create mode 100644 cdist/conf/type/__snakeoil_cert/man.rst
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/cert-path
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-path
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-type
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/optional

diff --git a/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group
new file mode 100755
index 00000000..a6cb3dfd
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+if grep -Eq '^ssl-cert:' /etc/group
+then
+    echo 'present'
+else
+    echo 'absent'
+fi
diff --git a/cdist/conf/type/__snakeoil_cert/explorer/state b/cdist/conf/type/__snakeoil_cert/explorer/state
new file mode 100755
index 00000000..cc5aae0b
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/explorer/state
@@ -0,0 +1,24 @@
+#!/bin/sh -e
+
+key_path="$( cat "$__object/parameter/key-path" )"
+
+if echo "$key_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    key_path="$( printf "$key_path" "$__object_id" )"
+fi
+
+cert_path="$( cat "$__object/parameter/cert-path" )"
+
+if echo "$cert_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    cert_path="$( printf "$cert_path" "$__object_id" )"
+fi
+
+if [ ! -f "$key_path" ] || [ ! -f "$cert_path" ]
+then
+    echo 'absent'
+else
+    echo 'present'
+fi
diff --git a/cdist/conf/type/__snakeoil_cert/gencode-remote b/cdist/conf/type/__snakeoil_cert/gencode-remote
new file mode 100755
index 00000000..8ffbfad1
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/gencode-remote
@@ -0,0 +1,73 @@
+#!/bin/sh -e
+
+state="$( cat "$__object/explorer/state" )"
+
+if [ "$state" = 'present' ]
+then
+    exit 0
+fi
+
+if [ -f "$__object/parameter/common-name" ]
+then
+    common_name="$( cat "$__object/parameter/common-name" )"
+else
+    common_name="$__object_id"
+fi
+
+key_path="$( cat "$__object/parameter/key-path" )"
+
+if echo "$key_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    key_path="$( printf "$key_path" "$__object_id" )"
+fi
+
+cert_path="$( cat "$__object/parameter/cert-path" )"
+
+if echo "$cert_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    cert_path="$( printf "$cert_path" "$__object_id" )"
+fi
+
+key_type="$( cat "$__object/parameter/key-type" )"
+
+key_type_arg="$( echo "$key_type" | cut -d : -f 2 )"
+
+case "$key_type" in
+    rsa:*)
+        echo "openssl genrsa -out '$key_path' $key_type_arg"
+    ;;
+    ec:*)
+        echo "openssl ecparam -name $key_type_arg -genkey -noout -out '$key_path'"
+    ;;
+esac
+
+# shellcheck disable=SC2016
+echo 'csr_path="$( mktemp )"'
+
+echo "openssl req -new -subj '/CN=$common_name' -key '$key_path' -out \"\$csr_path\""
+
+echo "openssl x509 -req -sha256 -days 3650 -in \"\$csr_path\" -signkey '$key_path' -out '$cert_path'"
+
+# shellcheck disable=SC2016
+echo 'rm -f "$csr_path"'
+
+if [ "$( cat "$__object/explorer/ssl-cert-group" )" = 'present' ]
+then
+    key_group='ssl-cert'
+else
+    key_group='root'
+fi
+
+echo "chmod 640 '$key_path'"
+
+echo "chown root '$key_path'"
+
+echo "chgrp $key_group '$key_path'"
+
+echo "chmod 644 '$cert_path'"
+
+echo "chown root '$cert_path'"
+
+echo "chgrp root '$cert_path'"
diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst
new file mode 100644
index 00000000..0b547804
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/man.rst
@@ -0,0 +1,60 @@
+cdist-type__snakeoil_cert(7)
+============================
+
+NAME
+----
+cdist-type__snakeoil_cert - Generate self-signed certificate
+
+
+DESCRIPTION
+-----------
+The purpose of this type is to generate **self-signed** certificate and private key
+for **testing purposes**. Certificate will expire in 3650 days.
+
+Certificate's and key's access bits will be ``644`` and ``640`` respectively.
+If target system has ``ssl-cert`` group, then it will be used as key's group.
+Use ``require='__snakeoil_cert/...' __file ...`` to override.
+
+
+OPTIONAL PARAMETERS
+-------------------
+common-name
+   Defaults to ``$__object_id``.
+
+key-path
+   ``%s`` in path will be replaced with ``$__object_id``.
+   Defaults to ``/etc/ssl/private/%s.pem``.
+
+key-type
+   Possible values are ``rsa:$bits`` and ``ec:$name``.
+   For possible EC names see ``openssl ecparam -list_curves``.
+   Defaults to ``rsa:2048``.
+
+cert-path
+   ``%s`` in path will be replaced with ``$__object_id``.
+   Defaults to ``/etc/ssl/certs/%s.pem``.
+
+
+EXAMPLES
+--------
+.. code-block:: sh
+	__snakeoil_cert localhost-rsa \
+	    --common-name localhost \
+	    --key-type rsa:4096
+
+	__snakeoil_cert localhost-ec \
+	    --common-name localhost \
+	    --key-type ec:prime256v1
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path
new file mode 100644
index 00000000..4bbae089
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path
@@ -0,0 +1 @@
+/etc/ssl/certs/%s.pem
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-path b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path
new file mode 100644
index 00000000..86eb9359
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path
@@ -0,0 +1 @@
+/etc/ssl/private/%s.pem
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-type b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type
new file mode 100644
index 00000000..f13f8ada
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type
@@ -0,0 +1 @@
+rsa:2048
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/optional b/cdist/conf/type/__snakeoil_cert/parameter/optional
new file mode 100644
index 00000000..76d08c0a
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/optional
@@ -0,0 +1,4 @@
+common-name
+key-path
+key-type
+cert-path

From 853e5cf7b4f615e1470c7ec2cfa8fe68cfa85ce7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 5 Jul 2021 09:07:06 +0200
Subject: [PATCH 342/411] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index d48c657e..0c3c64e1 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,8 @@ next:
 	* Type __pyvenv: Fix group explorer (Dennis Camera)
 	* Type __download: Improve checksum verification, add optional --destination (Ander Punnar)
 	* Type __debconf_set_selections: Add state explorer (Dennis Camera)
+	* Core: Implement usable cdist scan (Timothée Floure)
+	* New type: __snakeoil_cert (Ander Punnar)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From be92731c5c8a8543448f0d87fafae67e22ac76a1 Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Mon, 5 Jul 2021 12:38:26 +0100
Subject: [PATCH 343/411] Shell check quoting

We're actually echo-ing the command, hence the escape in front of the
quotes - the issue Shellcheck alludes too would actually occur, had the
escaping bakcslashes been omitted.
---
 cdist/conf/type/__rsync/gencode-local | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index f1bddc16..be4feabb 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -34,7 +34,8 @@ if [ -f "$__object/parameter/rsync-opts" ]; then
     done < "$__object/parameter/rsync-opts"
 fi
 
+# shellcheck disable=SC2086
 echo rsync -a \
     --no-owner --no-group \
-    -e \"$__remote_exec\" \
+    -e \"${__remote_exec}\" \
     -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"

From 521241d74102e37fae5f55552e1ef565d26ea9d2 Mon Sep 17 00:00:00 2001
From: fancsali <fancsali@gmail.com>
Date: Mon, 5 Jul 2021 15:28:05 +0200
Subject: [PATCH 344/411] Refine docs even more

---
 cdist/conf/type/__apt_pin/man.rst | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst
index 0c91cdec..4229c0cd 100644
--- a/cdist/conf/type/__apt_pin/man.rst
+++ b/cdist/conf/type/__apt_pin/man.rst
@@ -3,12 +3,12 @@ cdist-type__apt_pin(7)
 
 NAME
 ----
-cdist-type__apt_pin - TODO
+cdist-type__apt_pin - Manage apt pinning rules
 
 
 DESCRIPTION
 -----------
-This space intentionally left blank.
+Adds/removes/edits rules to pin some packages to a specific distribution. Useful if using multiple debian repositories at the same time. (Useful, if one wants to use a few specific packages from backports or perhaps Debain testing... or even sid.)
 
 
 REQUIRED PARAMETERS
@@ -20,7 +20,7 @@ distribution
 OPTIONAL PARAMETERS
 -------------------
 package
-   Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used.
+   Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used.
 
 priority
    The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority)
@@ -28,7 +28,6 @@ priority
 state
    Will be passed to underlying `__file` type; see there for valid values and defaults.
 
-None.
 
 
 BOOLEAN PARAMETERS
@@ -41,8 +40,8 @@ EXAMPLES
 
 .. code-block:: sh
 
-   # Add the bullseye repo to buster, but do not install any pacakges by default
-   # only if explicitely asked for
+   # Add the bullseye repo to buster, but do not install any packages by default,
+   # only if explicitely asked for (-1 means "never" for apt)
     __apt_pin bullseye-default \
        --package "*" \
        --distribution bullseye \
@@ -52,19 +51,19 @@ EXAMPLES
        --uri http://deb.debian.org/debian/ \
        --distribution bullseye \
        --component main
-       # TODO
-       __apt_pin
 
     __apt_pin foo --package "foo foo-*" --distribution bullseye
 
-    __foo # Installs the `foo` package internally
+    __foo # Assuming, this installs the `foo` package internally
 
-    __package foo-plugin-extras
+    __package foo-plugin-extras # Assuming we also need some extra stuff
 
 
 SEE ALSO
 --------
-:strong:`apt_preferences`\ (7)
+:strong:`apt_preferences`\ (5)
+:strong:`cdist-type__apt_source`\ (7)
+:strong:`cdist-type__apt_backports`\ (7)
 :strong:`cdist-type__file`\ (7)
 
 AUTHORS

From 166b58aeea09da41086525849291c70a3c3a571c Mon Sep 17 00:00:00 2001
From: fancsali <fancsali@gmail.com>
Date: Mon, 5 Jul 2021 15:32:27 +0200
Subject: [PATCH 345/411] Fix typo in distro names...

---
 cdist/conf/type/__apt_pin/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index 909bc80d..e72a8fdd 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -44,7 +44,7 @@ case "$os" in
 esac
 
 case $distribution in
-    stabletesting|unsatbel|experimental)
+    stable|testing|unstable|experimental)
         pin="release a=$distribution"
         ;;
     *)

From 485283f2e531f5fa23a876fa52411e0030f32589 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 7 Jul 2021 20:47:22 +0300
Subject: [PATCH 346/411] new type: __sed

---
 cdist/conf/type/__sed/explorer/file           | 22 +++++++++
 cdist/conf/type/__sed/gencode-remote          | 46 +++++++++++++++++
 cdist/conf/type/__sed/man.rst                 | 49 +++++++++++++++++++
 cdist/conf/type/__sed/parameter/boolean       |  1 +
 cdist/conf/type/__sed/parameter/optional      |  1 +
 .../type/__sed/parameter/required_multiple    |  1 +
 6 files changed, 120 insertions(+)
 create mode 100755 cdist/conf/type/__sed/explorer/file
 create mode 100755 cdist/conf/type/__sed/gencode-remote
 create mode 100644 cdist/conf/type/__sed/man.rst
 create mode 100644 cdist/conf/type/__sed/parameter/boolean
 create mode 100644 cdist/conf/type/__sed/parameter/optional
 create mode 100644 cdist/conf/type/__sed/parameter/required_multiple

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
new file mode 100755
index 00000000..a40f71b3
--- /dev/null
+++ b/cdist/conf/type/__sed/explorer/file
@@ -0,0 +1,22 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/file" ]
+then
+    file="$( cat "$__object/parameter/file" )"
+else
+    file="/$__object_id"
+fi
+
+if [ -f "$file" ]
+then
+    if [ -s "$file" ]
+    then
+        cat "$file"
+    else
+        echo "$file is empty" >&2
+        exit 1
+    fi
+else
+    echo "$file do not exist" >&2
+    exit 1
+fi
diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
new file mode 100755
index 00000000..391e3bc1
--- /dev/null
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -0,0 +1,46 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/file" ]
+then
+    file="$( cat "$__object/parameter/file" )"
+else
+    file="/$__object_id"
+fi
+
+script="$( cat "$__object/parameter/script" )"
+
+if [ "$script" = '-' ]
+then
+    script="$( cat "$__object/stdin" )"
+elif
+    [ -f "$script" ]
+then
+    script="$( cat "$script" )"
+fi
+
+file_from_target="$__object/explorer/file"
+
+sed_cmd='sed'
+
+if [ -f "$__object/parameter/regexp-extended" ]
+then
+    sed_cmd="$sed_cmd --regexp-extended"
+fi
+
+if ! echo "$script" \
+    | "$sed_cmd" -f - "$file_from_target" \
+    | diff "$file_from_target" - \
+    > /dev/null
+then
+    echo 'tmp="$( mktemp )"'
+
+    echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
+
+    echo "$script"
+
+    echo 'EOF'
+
+    echo "cp \"\$tmp\" '$file'"
+    
+    echo 'rm -f "$tmp"'
+fi
diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst
new file mode 100644
index 00000000..fbac212d
--- /dev/null
+++ b/cdist/conf/type/__sed/man.rst
@@ -0,0 +1,49 @@
+cdist-type__sed(7)
+==================
+
+NAME
+----
+cdist-type__sed - Transform text files with ``sed``
+
+
+DESCRIPTION
+-----------
+TODO
+
+
+REQUIRED MULTIPLE PARAMETERS
+----------------------------
+script
+   TODO
+
+
+OPTIONAL PARAMETERS
+-------------------
+file
+   TODO
+
+
+BOOLEAN PARAMETERS
+------------------
+regexp-extended
+   TODO
+
+
+EXAMPLES
+--------
+.. code-block:: sh
+
+   true
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
diff --git a/cdist/conf/type/__sed/parameter/boolean b/cdist/conf/type/__sed/parameter/boolean
new file mode 100644
index 00000000..1ad75c5d
--- /dev/null
+++ b/cdist/conf/type/__sed/parameter/boolean
@@ -0,0 +1 @@
+regexp-extended
diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional
new file mode 100644
index 00000000..f73f3093
--- /dev/null
+++ b/cdist/conf/type/__sed/parameter/optional
@@ -0,0 +1 @@
+file
diff --git a/cdist/conf/type/__sed/parameter/required_multiple b/cdist/conf/type/__sed/parameter/required_multiple
new file mode 100644
index 00000000..84f7e31d
--- /dev/null
+++ b/cdist/conf/type/__sed/parameter/required_multiple
@@ -0,0 +1 @@
+script

From 7a5896acfad0f992d6e7fe9cbe0bd08035ce19db Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 7 Jul 2021 21:23:25 +0300
Subject: [PATCH 347/411] add --onchange, fix shellcheck

---
 cdist/conf/type/__sed/gencode-remote     | 9 ++++++++-
 cdist/conf/type/__sed/man.rst            | 3 +++
 cdist/conf/type/__sed/parameter/optional | 1 +
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 391e3bc1..6b1bd216 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -32,6 +32,7 @@ if ! echo "$script" \
     | diff "$file_from_target" - \
     > /dev/null
 then
+    # shellcheck disable=SC2016
     echo 'tmp="$( mktemp )"'
 
     echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
@@ -41,6 +42,12 @@ then
     echo 'EOF'
 
     echo "cp \"\$tmp\" '$file'"
-    
+
+    # shellcheck disable=SC2016
     echo 'rm -f "$tmp"'
+
+    if [ -f "$__object/parameter/onchange" ]
+    then
+        cat "$__object/parameter/onchange"
+    fi
 fi
diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst
index fbac212d..4a728184 100644
--- a/cdist/conf/type/__sed/man.rst
+++ b/cdist/conf/type/__sed/man.rst
@@ -22,6 +22,9 @@ OPTIONAL PARAMETERS
 file
    TODO
 
+onchange
+   TODO
+
 
 BOOLEAN PARAMETERS
 ------------------
diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional
index f73f3093..fa86f917 100644
--- a/cdist/conf/type/__sed/parameter/optional
+++ b/cdist/conf/type/__sed/parameter/optional
@@ -1 +1,2 @@
 file
+onchange

From cf0032d667eb4bd8b7fe1bebacec341bbf71ed42 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 7 Jul 2021 21:28:00 +0300
Subject: [PATCH 348/411] add messaging and exit earlier

---
 cdist/conf/type/__sed/gencode-remote | 42 +++++++++++++++-------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 6b1bd216..836506f0 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -27,27 +27,31 @@ then
     sed_cmd="$sed_cmd --regexp-extended"
 fi
 
-if ! echo "$script" \
+if echo "$script" \
     | "$sed_cmd" -f - "$file_from_target" \
     | diff "$file_from_target" - \
     > /dev/null
 then
-    # shellcheck disable=SC2016
-    echo 'tmp="$( mktemp )"'
-
-    echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
-
-    echo "$script"
-
-    echo 'EOF'
-
-    echo "cp \"\$tmp\" '$file'"
-
-    # shellcheck disable=SC2016
-    echo 'rm -f "$tmp"'
-
-    if [ -f "$__object/parameter/onchange" ]
-    then
-        cat "$__object/parameter/onchange"
-    fi
+    exit 0
+fi
+
+# shellcheck disable=SC2016
+echo 'tmp="$( mktemp )"'
+
+echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
+
+echo "$script"
+
+echo 'EOF'
+
+echo "cp \"\$tmp\" '$file'"
+
+# shellcheck disable=SC2016
+echo 'rm -f "$tmp"'
+
+echo 'change' >> "$__messages_out"
+
+if [ -f "$__object/parameter/onchange" ]
+then
+    cat "$__object/parameter/onchange"
 fi

From 3e76d1cd3fbe53fa77c460cb2ce8416698b101bd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 8 Jul 2021 08:09:05 +0200
Subject: [PATCH 349/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 0c3c64e1..04f826f0 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -13,6 +13,7 @@ next:
 	* Type __debconf_set_selections: Add state explorer (Dennis Camera)
 	* Core: Implement usable cdist scan (Timothée Floure)
 	* New type: __snakeoil_cert (Ander Punnar)
+	* Type __rsync: Honour $__remote_exec env var (Daniel Fancsali)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 77dab4c5c63070aef962875af7fe8b1565f5ba78 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 10 Jul 2021 20:37:02 +0200
Subject: [PATCH 350/411] Release 6.9.7

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 04f826f0..284293c1 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)

From 65c43d3c1db938ac0063f54b1cb6b5090bb0a665 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 10 Jul 2021 21:02:27 +0200
Subject: [PATCH 351/411] Fix docs code block errors

---
 cdist/conf/type/__snakeoil_cert/man.rst | 1 +
 docs/src/cdist-scan.rst                 | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst
index 0b547804..b0b0a2e9 100644
--- a/cdist/conf/type/__snakeoil_cert/man.rst
+++ b/cdist/conf/type/__snakeoil_cert/man.rst
@@ -38,6 +38,7 @@ cert-path
 EXAMPLES
 --------
 .. code-block:: sh
+
 	__snakeoil_cert localhost-rsa \
 	    --common-name localhost \
 	    --key-type rsa:4096
diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst
index 064e65ff..86b7fab6 100644
--- a/docs/src/cdist-scan.rst
+++ b/docs/src/cdist-scan.rst
@@ -57,6 +57,7 @@ resolved name to stdout - if any. The script must be executable.
 Simplest script:
 
 .. code-block:: sh
+
   #!/bin/sh
 
   case "$1" in
@@ -71,6 +72,7 @@ Simplest script:
 Resolving name from `PTR` DNS record:
 
 .. code-block:: sh
+
   #!/bin/sh
 
   for cmd in dig sed; do

From 0e611af2a6388572eef5112c2ffaed082803965c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 13 Jul 2021 00:13:22 +0300
Subject: [PATCH 352/411] [__rsync] rewrite

---
 cdist/conf/type/__rsync/gencode-local         | 123 +++++++++++++-----
 cdist/conf/type/__rsync/gencode-remote        |  37 ------
 cdist/conf/type/__rsync/man.rst               | 107 +++++----------
 cdist/conf/type/__rsync/manifest              |  18 ---
 .../type/__rsync/parameter/default/options    |   1 +
 cdist/conf/type/__rsync/parameter/optional    |   4 +-
 .../type/__rsync/parameter/optional_multiple  |   2 +-
 7 files changed, 130 insertions(+), 162 deletions(-)
 delete mode 100755 cdist/conf/type/__rsync/gencode-remote
 create mode 100644 cdist/conf/type/__rsync/parameter/default/options

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index be4feabb..612d237e 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -1,41 +1,100 @@
 #!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
-source=$(cat "$__object/parameter/source")
-remote_user=$(cat "$__object/parameter/remote-user")
+if ! command -v rsync > /dev/null
+then
+    echo 'rsync is missing in local machine' >&2
+    exit 1
+fi
 
-if [ -f "$__object/parameter/destination" ]; then
-    destination=$(cat "$__object/parameter/destination")
+src="$( cat "$__object/parameter/source" )"
+
+if [ ! -e "$src" ]
+then
+    echo "$src not found" >&2
+    exit 1
+fi
+
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
 else
-    destination="/$__object_id"
+    dst="/$__object_id"
 fi
 
-set --
-if [ -f "$__object/parameter/rsync-opts" ]; then
-    while read -r opts; do
-        set -- "$@" "--$opts"
-    done < "$__object/parameter/rsync-opts"
+# if source is directory, then make sure that
+# source and destination are ending with slash,
+# because this is what you almost always want when
+# rsyncing two directories.
+
+if [ -d "$src" ]
+then
+    if ! echo "$src" | grep -Eq '/$'
+    then
+        src="$src/"
+    fi
+
+    if ! echo "$dst" | grep -Eq '/$'
+    then
+        dst="$dst/"
+    fi
 fi
 
+remote_user="$( cat "$__object/parameter/remote-user" )"
+
+options="$( cat "$__object/parameter/options" )"
+
+if [ -f "$__object/parameter/option" ]
+then
+    while read -r l
+    do
+        # there's a limitation in argparse: value can't begin with '-'.
+        # to workaround this, let's prefix opts with '\' in manifest and remove here.
+        # read more about argparse issue: https://bugs.python.org/issue9334
+
+        options="$options $( echo "$l" | sed 's/\\//g' )"
+    done \
+        < "$__object/parameter/option"
+fi
+
+if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ]
+then
+    options="$options --chown="
+
+    if [ -f "$__object/parameter/owner" ]
+    then
+        owner="$( cat "$__object/parameter/owner" )"
+        options="$options$owner"
+    fi
+
+    if [ -f "$__object/parameter/group" ]
+    then
+        group="$( cat "$__object/parameter/group" )"
+        options="$options:$group"
+    fi
+fi
+
+if [ -f "$__object/parameter/mode" ]
+then
+    mode="$( cat "$__object/parameter/mode" )"
+    options="$options --chmod=$mode"
+fi
+
+# IMPORTANT
+#
+# 1. we first dry-run rsync with change summary to find out
+#    if there are any changes and code generation is needed.
+# 2. normally, to get current state or target host, we run
+#    such operations in type explorers, but that's not
+#    possible due to how rsync works.
+# 3. redirecting output of dry-run to stderr to ease debugging.
+# 4. to understand how that cryptic regex works, please
+#    open rsync manpage and read about --itemize-changes.
+
 # shellcheck disable=SC2086
-echo rsync -a \
-    --no-owner --no-group \
-    -e \"${__remote_exec}\" \
-    -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"
+if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \
+    | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2
+then
+    exit 0
+fi
+
+echo "rsync $options $src $remote_user@$__target_host:$dst"
diff --git a/cdist/conf/type/__rsync/gencode-remote b/cdist/conf/type/__rsync/gencode-remote
deleted file mode 100755
index 074246af..00000000
--- a/cdist/conf/type/__rsync/gencode-remote
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-if [ -f "$__object/parameter/destination" ]; then
-    destination=$(cat "$__object/parameter/destination")
-else
-    destination="/$__object_id"
-fi
-
-ownergroup=""
-if [ -f "$__object/parameter/owner" ]; then
-    ownergroup=$(cat "$__object/parameter/owner")
-fi
-if [ -f "$__object/parameter/group" ]; then
-    ownergroup="${ownergroup}:$(cat "$__object/parameter/group")"
-fi
-
-if [ "$ownergroup" ]; then
-    echo chown -R "$ownergroup" "$destination"
-fi
diff --git a/cdist/conf/type/__rsync/man.rst b/cdist/conf/type/__rsync/man.rst
index 94b06d63..88019c92 100644
--- a/cdist/conf/type/__rsync/man.rst
+++ b/cdist/conf/type/__rsync/man.rst
@@ -3,112 +3,73 @@ cdist-type__rsync(7)
 
 NAME
 ----
-cdist-type__rsync - Mirror directories using rsync
+cdist-type__rsync - Mirror directories using ``rsync``
 
 
 DESCRIPTION
 -----------
-WARNING: This type is of BETA quality:
-
-- it has not been tested widely
-- interfaces *may* change
-- if there is a better approach to solve the problem -> the type may even vanish
-
-If you are fine with these constraints, please read on.
-
-
-This cdist type allows you to mirror local directories to the
-target host using rsync. Rsync will be installed in the manifest of the type.
-If group or owner are giveng, a recursive chown will be executed on the 
-target host.
-
-A slash will be appended to the source directory so that only the contents
-of the directory are taken and not the directory name itself.
+The purpose of this type is to bring power of ``rsync`` into ``cdist``.
 
 
 REQUIRED PARAMETERS
 -------------------
 source
-    Where to take files from
+   Source directory in local machine.
+   If source is directory, slash (``/``) will be added to source and destination paths.
 
 
 OPTIONAL PARAMETERS
 -------------------
-group
-   Group to chgrp to.
+destination
+   Destination directory. Defaults to ``$__object_id``.
 
 owner
-   User to chown to.
+   Will be passed to ``rsync`` as ``--chown=OWNER``.
+   Read ``rsync(1)`` for more details.
 
-destination
-    Use this as the base destination instead of the object id
+group
+   Will be passed to ``rsync`` as ``--chown=:GROUP``.
+   Read ``rsync(1)`` for more details.
+
+mode
+   Will be passed to ``rsync`` as ``--chmod=MODE``.
+   Read ``rsync(1)`` for more details.
+
+options
+   Defaults to ``--recursive --links --perms --times``.
+   Due to `bug in Python's argparse<https://bugs.python.org/issue9334>`_, value must be prefixed with ``\``.
 
 remote-user
-    Use this user instead of the default "root" for rsync operations.
+   Defaults to ``root``.
 
 
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
-rsync-opts
-    Use this option to give rsync options with.
-    See rsync(1) for available options.
-    Only "--" options are supported.
-    Write the options without the beginning "--"
-    Can be specified multiple times.
-
-
-MESSAGES
---------
-NONE
+option
+   Pass additional options to ``rsync``.
+   See ``rsync(1)`` for all possible options.
+   Due to `bug in Python's argparse<https://bugs.python.org/issue9334>`_, value must be prefixed with ``\``.
 
 
 EXAMPLES
 --------
-
 .. code-block:: sh
 
-    # You can use any source directory
-    __rsync /tmp/testdir \
-        --source /etc
-
-    # Use source from type
-    __rsync /etc \
-        --source "$__type/files/package"
-
-    # Allow multiple __rsync objects to write to the same dir
-    __rsync mystuff \
-        --destination /usr/local/bin \
-        --source "$__type/files/package"
-
-    __rsync otherstuff \
-        --destination /usr/local/bin \
-        --source "$__type/files/package2"
-
-    # Use rsync option --exclude
-    __rsync /tmp/testdir \
-        --source /etc \
-        --rsync-opts exclude=sshd_conf
-
-    # Use rsync with multiple options --exclude --dry-run
-    __rsync /tmp/testing \
-        --source /home/tester \
-        --rsync-opts exclude=id_rsa \
-        --rsync-opts dry-run
-
-
-SEE ALSO
---------
-:strong:`rsync`\ (1)
+    __rsync /var/www/example.com \
+        --owner root \
+        --group www-data \
+        --mode 'D750,F640' \
+        --source "$__files/example.com/www"
 
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+Ander Punnar <ander-at-kvlt-dot-ee>
 
 
 COPYING
 -------
-Copyright \(C) 2015 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
diff --git a/cdist/conf/type/__rsync/manifest b/cdist/conf/type/__rsync/manifest
index 9bd44c6d..64fa804e 100755
--- a/cdist/conf/type/__rsync/manifest
+++ b/cdist/conf/type/__rsync/manifest
@@ -1,21 +1,3 @@
 #!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
 __package rsync
diff --git a/cdist/conf/type/__rsync/parameter/default/options b/cdist/conf/type/__rsync/parameter/default/options
new file mode 100644
index 00000000..d967b110
--- /dev/null
+++ b/cdist/conf/type/__rsync/parameter/default/options
@@ -0,0 +1 @@
+--recursive --links --perms --times
diff --git a/cdist/conf/type/__rsync/parameter/optional b/cdist/conf/type/__rsync/parameter/optional
index ac2b2390..833e9bbe 100644
--- a/cdist/conf/type/__rsync/parameter/optional
+++ b/cdist/conf/type/__rsync/parameter/optional
@@ -1,4 +1,6 @@
 destination
-owner
 group
+mode
+options
+owner
 remote-user
diff --git a/cdist/conf/type/__rsync/parameter/optional_multiple b/cdist/conf/type/__rsync/parameter/optional_multiple
index fdb7cd88..01925a15 100644
--- a/cdist/conf/type/__rsync/parameter/optional_multiple
+++ b/cdist/conf/type/__rsync/parameter/optional_multiple
@@ -1 +1 @@
-rsync-opts
+option

From 46b5c24cd240cd9006d93cfcc12a4d81b46a5238 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 18 Jul 2021 16:25:00 +0300
Subject: [PATCH 353/411] use $__remote_exec for RSYNC_RSH

---
 cdist/conf/type/__rsync/gencode-local | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index 612d237e..e9f3c131 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -90,6 +90,8 @@ fi
 # 4. to understand how that cryptic regex works, please
 #    open rsync manpage and read about --itemize-changes.
 
+export RSYNC_RSH="$__remote_exec"
+
 # shellcheck disable=SC2086
 if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \
     | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2
@@ -97,4 +99,6 @@ then
     exit 0
 fi
 
+echo "export RSYNC_RSH='$__remote_exec'"
+
 echo "rsync $options $src $remote_user@$__target_host:$dst"

From 5229337611b7e7afb2597729a786c637a8bec1f6 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 18 Jul 2021 17:41:29 +0200
Subject: [PATCH 354/411] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 284293c1..f0746218 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __rsync: Rewrite (Ander Punnar)
+
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)

From de116661613002e91b54194ac8f0d5c3dd3eebbd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 18 Jul 2021 17:45:19 +0200
Subject: [PATCH 355/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index f0746218..ea51ac1b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __rsync: Rewrite (Ander Punnar)
+	* New type: __apt_pin (Daniel Fancsali)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 24c9406ea0f2c6c8edc87e5bbc1be25b5e8e1572 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 19 Jul 2021 12:13:23 +0200
Subject: [PATCH 356/411] [explorer/os_version] Convert Devuan ceres to version
 number

Conversion of Devuan ceres to version numbers is done based on Devuan codenames.
The version number is the version number of the final release - 0.01.

Analogous to Debian.
---
 cdist/conf/explorer/os_version | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 3b02dedd..6c94915c 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -1,6 +1,7 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,12 +18,11 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
 # All os variables are lower case
 #
-#
 
-case "$("$__explorer/os")" in
+case $("${__explorer:?}/os")
+in
    amazon)
       cat /etc/system-release
    ;;
@@ -59,7 +59,23 @@ case "$("$__explorer/os")" in
       esac
    ;;
    devuan)
-      cat /etc/devuan_version
+      devuan_version=$(cat /etc/devuan_version)
+      case ${devuan_version}
+      in
+         (*/ceres)
+            # ceres versions don't have a number, so we decode by codename:
+            case ${devuan_version}
+            in
+               (chimaera/ceres) echo 3.99 ;;
+               (beowulf/ceres) echo 2.99 ;;
+               (ascii/ceres) echo 1.99 ;;
+               (*) exit 1
+            esac
+            ;;
+         (*)
+            echo "${devuan_version}"
+            ;;
+      esac
    ;;
    fedora)
       cat /etc/fedora-release

From fbc9594729ece74b2f8612fe8046ddea966e8c05 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 20 Jul 2021 06:38:46 +0200
Subject: [PATCH 357/411] ++changelog

---
 docs/changelog | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/docs/changelog b/docs/changelog
index ea51ac1b..55d7cb73 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)
+	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
@@ -146,7 +147,7 @@ next:
 	* Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham)
 	* Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham)
 	* Configuration: Add notes to cdist.cfg.skeleton (Evil Ham)
-	* Explorers cpu_cores, memory: Improve *BSD support (Evil Ham)
+	* Explorers cpu_cores, memory: Improve BSD support (Evil Ham)
 	* Core: Remove debug logging noise (Evil Ham)
 
 6.5.4: 2020-04-11
@@ -211,7 +212,7 @@ next:
 	* Documentation: PreOS english nitpicking (Evil Ham)
 	* Documentation: Add installing from source with signature verification (Darko Poljak)
 	* Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak)
-	* Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham)
+	* Type __start_on_boot: Docs: remove unsupported BSD claim (Evil Ham)
 	* New type: __openldap_server (Evil Ham)
 
 6.2.0: 2019-11-30
@@ -1070,9 +1071,9 @@ next:
 	* Removed type __removeline (replaced by __line) (Nico Schottelius)
 	* Type __directory: Parameter --parents and --recursive are now boolean (Nico Schottelius)
 	* Type __package_apt, __package_luarocks, __package_opkg,
-		__package_pacman, __package_pkg_freebsd, __package_pkg_openbsd,
-		__package_rubygem, __package_yum, __process:
-			Parameter state accepts only "present" and "absent" (Nico Schottelius)
+	  __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd,
+	  __package_rubygem, __package_yum, __process:
+	  Parameter state accepts only "present" and "absent" (Nico Schottelius)
 	* Dist: Initial support for pypi packaging (Nico Schottelius)
 
 2.0.15: 2012-11-02

From c7daaabc6c28cbfaebaf6c620cc3cac130684d2c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 20 Jul 2021 09:03:16 +0200
Subject: [PATCH 358/411] [docs] Bump copyright year to 2021

---
 docs/src/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/conf.py b/docs/src/conf.py
index 47765413..a3dfafca 100644
--- a/docs/src/conf.py
+++ b/docs/src/conf.py
@@ -56,7 +56,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'cdist'
-copyright = 'ungleich GmbH 2020'
+copyright = 'ungleich GmbH 2021'
 # author = 'Darko Poljak'
 
 # The version info for the project you're documenting, acts as replacement for

From fed01ded83b217a3ad147f2403ca863be373eeaa Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 22 Jul 2021 11:17:41 +0200
Subject: [PATCH 359/411] [cdist.log] Define custom log functions on
 logging.Logger

Define out custom logger functions on logging.Logger so that they are passed on
to all other loggers.

Also, the logger functions need to take a self argument so that they can log on
the corrent Logger.
---
 cdist/log.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/cdist/log.py b/cdist/log.py
index 113f3b4c..62e457fe 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -36,25 +36,27 @@ import threading
 logging.OFF = logging.CRITICAL + 10  # disable logging
 logging.addLevelName(logging.OFF, 'OFF')
 
+
 logging.VERBOSE = logging.INFO - 5
 logging.addLevelName(logging.VERBOSE, 'VERBOSE')
 
 
-def _verbose(msg, *args, **kwargs):
-    logging.log(logging.VERBOSE, msg, *args, **kwargs)
+def _verbose(self, msg, *args, **kwargs):
+    self.log(logging.VERBOSE, msg, args, **kwargs)
 
 
-logging.verbose = _verbose
+logging.Logger.verbose = _verbose
+
 
 logging.TRACE = logging.DEBUG - 5
 logging.addLevelName(logging.TRACE, 'TRACE')
 
 
-def _trace(msg, *args, **kwargs):
-    logging.log(logging.TRACE, msg, *args, **kwargs)
+def _trace(self, msg, *args, **kwargs):
+    self.log(logging.TRACE, msg, *args, **kwargs)
 
 
-logging.trace = _trace
+logging.Logger.trace = _trace
 
 
 class CdistFormatter(logging.Formatter):

From 71fee1fd6b4874abffee5173a96c31842a9583bd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 23 Jul 2021 08:06:45 +0200
Subject: [PATCH 360/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 55d7cb73..1273e432 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
+	* Core: Fix logging bug (Dennis Camera)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 67bcc6cae38e2d480c3336b6a4d59d01196190e3 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 24 Jul 2021 02:37:58 +0200
Subject: [PATCH 361/411] Improve Makefile compatibility and build docs

We now use `$(MAKE)` for subsequent calls to `make`.
This means that systems that do not default to GNU make can run `gmake man` and
produce the man pages.

While there also document a dependency on the rtd theme for sphinx.
---
 Makefile                   | 6 +++---
 docs/src/cdist-install.rst | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 3712511c..89286310 100644
--- a/Makefile
+++ b/Makefile
@@ -35,9 +35,9 @@ DOCS_SRC_DIR=./docs/src
 SPEECHDIR=./docs/speeches
 TYPEDIR=./cdist/conf/type
 
-SPHINXM=make -C $(DOCS_SRC_DIR) man
-SPHINXH=make -C $(DOCS_SRC_DIR) html
-SPHINXC=make -C $(DOCS_SRC_DIR) clean
+SPHINXM=$(MAKE) -C $(DOCS_SRC_DIR) man
+SPHINXH=$(MAKE) -C $(DOCS_SRC_DIR) html
+SPHINXC=$(MAKE) -C $(DOCS_SRC_DIR) clean
 
 ################################################################################
 # Manpages
diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst
index 18863145..390ab9ec 100644
--- a/docs/src/cdist-install.rst
+++ b/docs/src/cdist-install.rst
@@ -12,7 +12,7 @@ This is the machine from which you will configure target hosts.
  * /bin/sh: A POSIX like shell (for instance bash, dash, zsh)
  * Python >= 3.5
  * SSH client
- * sphinx (for building html docs and/or the man pages)
+ * sphinx with the rtd theme (for building html docs and/or the man pages)
 
 Target Hosts
 ~~~~~~~~~~~~

From cb8695cc88478640d7e76992438d90d3d6a68a90 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 24 Jul 2021 12:53:39 +0200
Subject: [PATCH 362/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 1273e432..181c4139 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* New type: __apt_pin (Daniel Fancsali)
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
 	* Core: Fix logging bug (Dennis Camera)
+	* Build: Improve Makefile compatibility (Evilham)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 4156fea9001aa267c8b173247cada2f919511c1b Mon Sep 17 00:00:00 2001
From: Joachim Desroches <joachim.desroches@epfl.ch>
Date: Wed, 28 Jul 2021 12:56:39 +0200
Subject: [PATCH 363/411] [filesystem] Add ubuntu as supported distribution.

---
 cdist/conf/type/__filesystem/explorer/lsblk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk
index 9be3c575..d376c09f 100644
--- a/cdist/conf/type/__filesystem/explorer/lsblk
+++ b/cdist/conf/type/__filesystem/explorer/lsblk
@@ -27,7 +27,7 @@ else
 fi
 
 case "$os" in
-    alpine|centos|fedora|redhat|suse|gentoo)
+    alpine|centos|fedora|gentoo|redhat|suse|ubuntu)
         if [ ! -x "$(command -v lsblk)" ]; then
             echo "lsblk is required for __filesystem type" >&2
             exit 1

From 542674dae81ab03a4de2c4ad0b2eaf264ee2c442 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 30 Jul 2021 10:30:33 +0200
Subject: [PATCH 364/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 181c4139..9fc10b20 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
 	* Core: Fix logging bug (Dennis Camera)
 	* Build: Improve Makefile compatibility (Evilham)
+	* Type __filesystem: Support ubuntu (Joachim Desroches)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 53334fb4eb311550af7f5b73f279a2e86fa1c504 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 4 Aug 2021 19:50:10 +0200
Subject: [PATCH 365/411] [explorer/os_version] Fix for FreeBSD < 10.0 (again)

---
 cdist/conf/explorer/os_version | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 6c94915c..cc976608 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -89,7 +89,14 @@ in
    freebsd)
       # Apparently uname -r is not a reliable way to get the patch level.
       # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
-      freebsd-version
+      if command -v freebsd-version >/dev/null 2>&1
+      then
+         # get userland version
+         freebsd-version -u
+      else
+         # fallback to kernel release for FreeBSD < 10.0
+         uname -r
+      fi
    ;;
    *bsd|solaris)
       uname -r

From e108cbc205cb5e7ac0d2e07b82cef4c83eaa285f Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Tue, 3 Aug 2021 13:20:43 +0200
Subject: [PATCH 366/411] [explorer/os_version] Ubuntu: fall back to
 os-release/lsb-release files

---
 cdist/conf/explorer/os_version | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 6c94915c..e70fe7f4 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -21,6 +21,17 @@
 # All os variables are lower case
 #
 
+rc_getvar() {
+   awk -F= -v varname="$2" '
+      function unquote(s) {
+         if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/)
+            return substr(s, 2, length(s) - 2)
+         else
+            return s
+      }
+      $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1"
+}
+
 case $("${__explorer:?}/os")
 in
    amazon)
@@ -114,7 +125,20 @@ in
       fi
    ;;
    ubuntu)
-      lsb_release -sr
+      if command -v lsb_release >/dev/null 2>&1
+      then
+         lsb_release -sr
+      elif test -r /usr/lib/os-release
+      then
+         # fallback to /usr/lib/os-release if lsb_release is not present (like
+         # on minimized Ubuntu installations)
+         rc_getvar /usr/lib/os-release VERSION_ID
+      elif test -r /etc/lsb-release
+      then
+         # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old
+         # versions without /usr/lib/os-release.
+         rc_getvar /etc/lsb-release DISTRIB_RELEASE
+      fi
    ;;
    alpine)
        cat /etc/alpine-release

From 83fe6e9f5b2537db73d5c6a142b6d24eef75ac58 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Tue, 3 Aug 2021 19:26:55 +0200
Subject: [PATCH 367/411] [explorer/memory] Fix conversion of large numbers (>=
 2GiB)

At least mawk uses scientific notation when using print for
numbers >=2^31 (INT_MAX of a signed 32-bit int).

`printf "%.f\n"` works around this.
---
 cdist/conf/explorer/memory | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory
index 63aba9c6..c6d113cf 100755
--- a/cdist/conf/explorer/memory
+++ b/cdist/conf/explorer/memory
@@ -27,19 +27,18 @@
 str2bytes() {
 	awk -F' ' '
 	$2 ==   "B" || !$2 { print $1 }
-	$2 ==  "kB" { print $1 * 1000 }
-	$2 ==  "MB" { print $1 * 1000 * 1000 }
-	$2 ==  "GB" { print $1 * 1000 * 1000 * 1000 }
-	$2 ==  "TB" { print $1 * 1000 * 1000 * 1000 * 1000 }
-	$2 == "kiB" { print $1 * 1024 }
-	$2 == "MiB" { print $1 * 1024 * 1024 }
-	$2 == "GiB" { print $1 * 1024 * 1024 * 1024 }
-	$2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }'
+	$2 ==  "kB" { printf "%.f\n", ($1 * 1000) }
+	$2 ==  "MB" { printf "%.f\n", ($1 * 1000 * 1000) }
+	$2 ==  "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) }
+	$2 ==  "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) }
+	$2 == "kiB" { printf "%.f\n", ($1 * 1024) }
+	$2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) }
+	$2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) }
+	$2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }'
 }
 
 bytes2kib() {
-	set -- "$(cat)"
-	test "$1" -gt 0 && echo $(($1 / 1024))
+	awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }'
 }
 
 

From a7d6481a7ddc7cb72b1a55bfca7fdfed20514a62 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Mon, 2 Aug 2021 21:23:50 +0200
Subject: [PATCH 368/411] [type/__update_alternatives] Secure cdist-defined
 environment variables with :?

---
 .../__update_alternatives/explorer/alternatives     |  2 +-
 cdist/conf/type/__update_alternatives/explorer/link |  6 +++---
 .../type/__update_alternatives/explorer/path_is     |  4 ++--
 .../explorer/path_should_state                      |  2 +-
 .../conf/type/__update_alternatives/gencode-remote  | 13 ++++++-------
 5 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives
index 34aaca56..ecc62f4b 100755
--- a/cdist/conf/type/__update_alternatives/explorer/alternatives
+++ b/cdist/conf/type/__update_alternatives/explorer/alternatives
@@ -1,4 +1,4 @@
 #!/bin/sh -e
 
-update-alternatives --display "$__object_id" 2>/dev/null \
+update-alternatives --display "${__object_id:?}" 2>/dev/null \
     | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link
index 6519e7c2..c6fd1c98 100755
--- a/cdist/conf/type/__update_alternatives/explorer/link
+++ b/cdist/conf/type/__update_alternatives/explorer/link
@@ -18,12 +18,12 @@ for altdir in \
     /var/lib/dpkg/alternatives \
     /var/lib/alternatives
 do
-    if [ ! -f "$altdir/$__object_id" ]
+    if [ ! -f "$altdir/${__object_id:?}" ]
     then
         continue
     fi
 
-    link="$( awk 'NR==2' "$altdir/$__object_id" )"
+    link="$( awk 'NR==2' "$altdir/${__object_id:?}" )"
 
     if [ -n "$link" ]
     then
@@ -33,7 +33,7 @@ done
 
 if [ -z "$link" ]
 then
-    echo "unable to get link for $__object_id" >&2
+    echo "unable to get link for ${__object_id:?}" >&2
     exit 1
 fi
 
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
index fc304d5d..a24bd40e 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_is
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -1,11 +1,11 @@
 #!/bin/sh -e
 
-path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \
+path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \
     | awk '/link currently points to/ {print $5}' )"
 
 if [ -z "$path_is" ]
 then
-    echo "unable to get current path for $__object_id" >&2
+    echo "unable to get current path for ${__object_id:?}" >&2
     exit 1
 fi
 
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state
index 59e015c5..b74a7ee8 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_should_state
+++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state
@@ -1,6 +1,6 @@
 #!/bin/sh -e
 
-if [ -f "$( cat "$__object/parameter/path" )" ]
+if [ -f "$( cat "${__object:?}/parameter/path" )" ]
 then
     echo 'present'
 else
diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index e393cdef..13666805 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -18,26 +18,25 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 
-path_is="$( cat "$__object/explorer/path_is" )"
+path_is="$( cat "${__object:?}/explorer/path_is" )"
 
-path_should="$( cat "$__object/parameter/path" )"
+path_should="$( cat "${__object:?}/parameter/path" )"
 
 if [ "$path_is" = "$path_should" ]
 then
     exit 0
 fi
 
-if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ]
+if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \
+    && [ -z "${__cdist_dry_run+dry run}" ]
 then
     echo "$path_should does not exist in target" >&2
     exit 1
 fi
 
-name="$__object_id"
+name=${__object_id:?}
 
-alternatives="$( cat "$__object/explorer/alternatives" )"
-
-if ! echo "$alternatives" | grep -Fxq "$path_should"
+if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives"
 then
     if [ ! -f "$__object/parameter/install" ]
     then

From 0b3b47396f2aafa377e3d5d9a13f51ace2303d41 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Mon, 2 Aug 2021 21:25:08 +0200
Subject: [PATCH 369/411] [type/__update_alternatives] dry-run fixes

---
 cdist/conf/type/__update_alternatives/explorer/link |  5 ++++-
 .../type/__update_alternatives/explorer/path_is     |  5 ++++-
 .../conf/type/__update_alternatives/gencode-remote  | 13 ++++++++-----
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link
index c6fd1c98..d1087c75 100755
--- a/cdist/conf/type/__update_alternatives/explorer/link
+++ b/cdist/conf/type/__update_alternatives/explorer/link
@@ -31,8 +31,11 @@ do
     fi
 done
 
-if [ -z "$link" ]
+if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then
+    # NOTE: ignore error for dry-runs because a package providing the link
+    #       might be managed by another cdist object (which wasn't executed,
+    #       because dry run…).
     echo "unable to get link for ${__object_id:?}" >&2
     exit 1
 fi
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
index a24bd40e..9208df7b 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_is
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -3,8 +3,11 @@
 path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \
     | awk '/link currently points to/ {print $5}' )"
 
-if [ -z "$path_is" ]
+if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then
+    # NOTE: ignore error for dry-runs because a package providing the
+    #       alternative might be managed by another cdist object (which
+    #       wasn't executed, because dry run…).
     echo "unable to get current path for ${__object_id:?}" >&2
     exit 1
 fi
diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index 13666805..e91ea78f 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -38,16 +38,19 @@ name=${__object_id:?}
 
 if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives"
 then
-    if [ ! -f "$__object/parameter/install" ]
+    if [ -f "${__object:?}/parameter/install" ]
     then
+        link="$( cat "${__object:?}/explorer/link" )"
+        echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
+    elif [ -z "${__cdist_dry_run+dry run}" ]
+    then
+        # NOTE: ignore error for dry-runs because a package providing the link
+        #       to be installed might be managed by another cdist object (which
+        #       wasn't executed, because dry run…).
         echo "$path_should is not in $name alternatives." >&2
         echo 'Please install missing packages or use --install to add path to alternatives.' >&2
         exit 1
     fi
-
-    link="$( cat "$__object/explorer/link" )"
-
-    echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
 fi
 
 echo "update-alternatives --set '$name' '$path_should'"

From bbcc81a9841f2619e1b9e13b25a941337489a681 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 4 Aug 2021 21:44:04 +0200
Subject: [PATCH 370/411] [type/__update_alternatives] Fix for non-English
 locales

Since update-alternatives(1) is localized, screen scraping its output breaks
if the locale is set to non-English.
---
 cdist/conf/type/__update_alternatives/explorer/alternatives | 4 ++--
 cdist/conf/type/__update_alternatives/explorer/path_is      | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives
index ecc62f4b..bb1619a9 100755
--- a/cdist/conf/type/__update_alternatives/explorer/alternatives
+++ b/cdist/conf/type/__update_alternatives/explorer/alternatives
@@ -1,4 +1,4 @@
 #!/bin/sh -e
 
-update-alternatives --display "${__object_id:?}" 2>/dev/null \
-    | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
+LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \
+| awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
index 9208df7b..5cf4fa4b 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_is
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -1,7 +1,8 @@
 #!/bin/sh -e
 
-path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \
-    | awk '/link currently points to/ {print $5}' )"
+path_is=$(
+    LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \
+    | awk '/link currently points to/ { print $5 }')
 
 if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then

From 2a0c073d4021206c7459015cefaba218004235ce Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 4 Aug 2021 21:54:17 +0200
Subject: [PATCH 371/411] [explorer/os_version] Fix for legacy Mac OS X
 versions

---
 cdist/conf/explorer/os_version | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 3b02dedd..96eca1ee 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -68,7 +68,8 @@ case "$("$__explorer/os")" in
       cat /etc/gentoo-release
    ;;
    macosx)
-      sw_vers -productVersion
+      # NOTE: Legacy versions (< 10.3) do not support options
+      sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }'
    ;;
    freebsd)
       # Apparently uname -r is not a reliable way to get the patch level.

From 3ae5a606ca7182ec7fe13134670400506e198ec1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 5 Aug 2021 10:27:51 +0200
Subject: [PATCH 372/411] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 9fc10b20..f9409d7e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,10 @@ next:
 	* Core: Fix logging bug (Dennis Camera)
 	* Build: Improve Makefile compatibility (Evilham)
 	* Type __filesystem: Support ubuntu (Joachim Desroches)
+	* Explorer os_version: Fall back to os-release/lsb-release file on Ubuntu (Dennis Camera)
+	* Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera)
+	* Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera)
+	* Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From edcac70b2a99ce532dd5bca9a0b8fc5bf6dc5148 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 21 Jul 2021 13:22:34 +0200
Subject: [PATCH 373/411] [explorer/machine_type] Reimplement

---
 cdist/conf/explorer/machine_type | 986 ++++++++++++++++++++++++++++---
 1 file changed, 904 insertions(+), 82 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 1c84f4d7..29f98849 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -1,8 +1,6 @@
-#!/bin/sh
+#!/bin/sh -e
 #
-# 2014 Daniel Heule  (hda at sfs.biz)
-# 2014 Thomas Oettli (otho at sfs.biz)
-# 2020 Evilham (contact at evilham.com)
+# 2021 Dennis Camera (cdist at dtnr.ch)
 #
 # This file is part of cdist.
 #
@@ -19,91 +17,915 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+# This explorer tries to determine what type of machine the target to be
+# configured is (container, virtual machine, bare-metal).
+#
+# It will print one line for each layer it can detect.
+# The format of all lines is: TYPE[ VERB VENDOR]
+#
+# VERB does not have a special meaning, it is just for better readability.
+#
+# e.g.
+# container on lxc
+# virtual by kvm-spapr
+#
+# The third word of each line can be composed of different parts concatenated with a `-'
+# (minus) character, with each component being a specification of the previous,
+# e.g.:
+#  - lxc-libvirt             (LXC container, managed by libvirt)
+#  - lpar-s390 / lpar-power  (LPAR running on IBM S/390 or POWER, respectively)
+#  - xen-hvm / xen-pv        (Xen HVM vs para-virtualization)
+#
+# If this explorer cannot determine any information it will print nothing.
+#
 
-os=$("$__explorer/os")
+# Add /sbin and /usr/sbin to the path so we can find system
+# binaries like dmidecode.
+PATH=$(getconf PATH 2>/dev/null) || PATH='/usr/bin:/bin'
+PATH="/sbin:/usr/sbin:${PATH}"
+export PATH
 
-vendor_string_to_machine_type() {
-    for vendor in vmware bochs kvm qemu virtualbox bhyve; do
-        if echo "${1}" | grep -q -i "${vendor}"; then
-            if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then
-                vendor="kvm"
-            fi
-            echo "virtual_by_${vendor}"
-            exit
-        fi
-    done
+arch=$(uname -m | sed -e 's/i.86/i386/' -e 's/arm.*/arm/')
+uname_s=$(uname -s)
+
+
+is_command() { command -v "$1" >/dev/null 2>&1; }
+
+is_oneof() (
+	x=$1; shift
+	for y
+	do
+		test "${x}" = "${y}" || continue
+		return 0
+	done
+	return 1
+)
+
+tolower() { LC_ALL=C tr '[:upper:]' '[:lower:]'; }
+
+# shellcheck disable=SC2086
+glob_exists() { set -- $1; test -e "$1"; }
+
+get_dmi_field() {
+	if is_oneof "${uname_s}" NetBSD
+	then
+		case $1
+		in
+			(system-manufacturer) _mib=machdep.dmi.system-vendor ;;
+			(system-product-name) _mib=machdep.dmi.system-product ;;
+			(system-version|system-uuid) _mib=machdep.dmi.$1 ;;
+			(bios-vendor|bios-version) _mib=machdep.dmi.$1 ;;
+			(biod-release-date) _mib=machdep.dmi.bios-date ;;
+			(*) _mib= ;;
+		esac
+
+		test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return
+	fi
+
+	if is_command dmidecode
+	then
+		dmidecode -s "$1"
+	elif test -d "${dmi_sysfs-}"
+	then
+		case $1
+		in
+			(system-manufacturer) _filename=sys_vendor ;;
+			(system-product-name) _filename=product_name ;;
+			(*) _filename=$(echo "$1" | tr - _) ;;
+		esac
+		if test -r "${dmi_sysfs-}/${_filename}"
+		then
+			cat "${dmi_sysfs}/${_filename}"
+		fi
+		unset _filename
+	elif test "${uname_s}" = OpenBSD
+	then
+		# NOTE: something similar to system-manufacutrer and system-product-name
+		#       is available on OpenBSD in sysctl
+		case $1
+		in
+			(system-manufacturer) _mib=hw.vendor ;;
+			(system-product-name) _mib=hw.product ;;
+			(*) _mib= ;;
+		esac
+
+		test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return
+	fi
+
+	return 1
 }
 
-case "$os" in
-    "freebsd")
-        # FreeBSD does not have /proc/cpuinfo even when procfs is used.
-        # Instead there is a sysctl kern.vm_guest.
-        # Which is 'none' if physical, else the virtualisation.
-        vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)"
-        if [ -n "${vm_guest}" ]; then
-            if [ "${vm_guest}" = "none" ]; then
-                echo "physical"
-                exit
-            fi
-            echo "virtual_by_${vm_guest}"
-            exit
-        fi
-    ;;
+has_cpuinfo() { test -e /proc/cpuinfo; }
 
-    "openbsd")
-        # OpenBSD can also use the sysctl's: hw.vendor or hw.product.
-        # Note we can be reasonably sure about a machine being virtualised
-        # as long as we can identify the virtualisation technology.
-        # But not so much about it being physical...
-        # Patches are welcome / reach out if you have better ideas.
-        for sysctl in hw.vendor hw.product; do
-            # This exits if we can make a reasonable judgement
-            vendor_string_to_machine_type "$(sysctl -n "${sysctl}")"
-        done
-    ;;
+get_sysctl() {
+	is_command sysctl && sysctl -n "$1" 2>/dev/null
+}
 
-    *)
-        # Defaulting to linux for compatibility with previous cdist behaviour
 
-        if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then
-            echo openvz
-            exit
-        fi
+# Check for container
 
-        if [ -e "/proc/1/environ" ] &&
-            tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then
-            echo lxc
-            exit
-        fi
+has_ct_pid_1() {
+	test -r /run/systemd/container -o -r /proc/1/environ
+}
 
-        if [ -r /proc/cpuinfo ]; then
-            # this should only exist on virtual guest machines,
-            # tested on vmware, xen, kvm, bhyve
-            if grep -q "hypervisor" /proc/cpuinfo; then
-                # this file is aviable in xen guest systems
-                if [ -r /sys/hypervisor/type ]; then
-                    if grep -q -i "xen" /sys/hypervisor/type; then
-                        echo virtual_by_xen
-                        exit
-                    fi
-                else
-                    for vendor_file in /sys/class/dmi/id/product_name \
-                                       /sys/class/dmi/id/sys_vendor \
-                                       /sys/class/dmi/id/chasis_vendor; do
-                        if [ -r ${vendor_file} ]; then
-                            # This exits if we can make a reasonable judgement
-                            vendor_string_to_machine_type "$(cat "${vendor_file}")"
-                        fi
-                    done
-                fi
-                echo "virtual_by_unknown"
-                exit
-            else
-                echo "physical"
-                exit
-            fi
-        fi
-    ;;
-esac
+translate_container_name() {
+	case $1
+	in
+		('lxc')
+			echo lxc ;;
+		('lxc-libvirt')
+			echo lxc-libvirt ;;
+		('podman')
+			echo podman ;;
+		('systemd-nspawn')
+			echo systemd_nspawn ;;
+		(*)
+			return 1 ;;
+	esac
+	return 0
+}
 
-echo "unknown"
+check_ct_pid_1() {
+	if test -r /run/systemd/container
+	then
+		translate_container_name "$(head -n1 /run/systemd/container)" \
+		&& return 0
+	fi
+
+	if test -r /proc/1/environ
+	then
+		translate_container_name "$(
+			LC_ALL=C tr '\000' '\n' </proc/1/environ \
+			| sed -n -e 's/^container=//p')" \
+		&& return 0
+	fi
+
+	return 1
+}
+
+has_ct_cgroup() {
+	test -r /proc/self/cgroup
+}
+
+check_ct_cgroup() {
+	if grep -q -e ':/docker/' /proc/self/cgroup
+	then
+		echo docker
+	elif grep -q -e ':/lxc/.*\.libvirt-lxc$' /proc/self/cgroup
+	then
+		echo libvirt-lxc
+	elif grep -q -F '/libpod-' /proc/self/cgroup
+	then
+		echo podman
+	else
+		return 1
+	fi
+}
+
+check_ct_files() {
+	if test -e /.dockerenv -o -e /run/.dockerenv -o -e /.dockerinit
+	then
+		echo docker
+	elif test -f /run/.containerenv
+	then
+		# https://github.com/containers/podman/issues/3586#issuecomment-661918679
+		echo podman
+	elif test -d /proc/vz -a ! -d /proc/bc -a -f /proc/user_beancounters \
+		|| test -d /.cpt_hardlink_dir_*/
+	then
+		# Virtuozzo / OpenVZ
+		#
+		# /proc/vz: always exists if OpenVZ kernel is running
+		# /proc/bc: exists only on node (not inside container)
+		# /proc/user_beancounters - exists inside container
+		#
+		# /.cpt_hardlink_dir_*/ may be (?) created by vzctl:
+		# https://github.com/kolyshkin/vzctl/commit/09e974f
+
+		echo openvz
+	elif grep -qie 'Microsoft\|WSL' /proc/sys/kernel/osrelease /proc/version 2>/dev/null
+	then
+		# https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364
+		echo wsl
+	elif test -d /var/.cagefs
+	then
+		# https://docs.cloudlinux.com/cloudlinux_os_components/#cagefs
+		# CageFS is not "really" a container, but it isn't a chroot either.
+		echo cagefs
+	elif test -e /proc/self/status && grep -q -e '^VxID: [0-9]\{1,\}' /proc/self/status
+	then
+		# Linux-VServer
+		if grep -q -x -F 'VxID: 0' /proc/self/status
+		then
+			# host
+			return 1
+		else
+			# guest
+			echo linux_vserver
+		fi
+	else
+		return 1
+	fi
+}
+
+check_ct_os_specific() (
+	if jailed=$(get_sysctl security.jail.jailed) && test "${jailed}" = 1
+	then
+		# FreeBSD jail
+		echo jail
+		return 0
+	fi
+
+	if is_command zonename && test "$(zonename)" != global
+	then
+		# Solaris zone
+		echo zone
+		return 0
+	fi
+
+	return 1
+)
+
+
+# Check for hypervisor
+
+guess_hypervisor_from_cpu_model() {
+	case $1
+	in
+		(*\ KVM\ *)
+			echo kvm ;;
+		(*\ QEMU\ *|QEMU\ *)
+			echo qemu ;;
+		(*)
+			return 1 ;;
+	esac
+}
+
+has_vm_cpuinfo() { has_cpuinfo; }
+
+check_vm_cpuinfo() {
+	if grep -q -F 'User Mode Linux' /proc/cpuinfo \
+		|| grep -q -F 'UML' /proc/cpuinfo
+	then
+		# User Mode Linux
+		echo uml
+	elif grep -q -e '^vendor_id.*: PowerVM Lx86' /proc/cpuinfo
+	then
+		# IBM PowerVM Lx86 (Linux/x86 emulator)
+		echo powervm_lx86
+	elif grep -q -e '^vendor_id.*: IBM/S390' /proc/cpuinfo
+	then
+		# IBM SystemZ (S/390)
+		if test -f /proc/sysinfo
+		then
+			if grep -q -e '^VM[0-9]* Control Program: KVM/Linux' /proc/sysinfo
+			then
+				echo kvm-s390
+				return 0
+			elif grep -q -e '^VM[0-9]* Control Program: z/VM' /proc/sysinfo
+			then
+				echo zvm
+				return 0
+			elif grep -q -e '^LPAR ' /proc/sysinfo
+			then
+				echo zvm-lpar
+				return 0
+			fi
+		fi
+		return 1
+	else
+		if grep -q -e '^model name.*:' /proc/cpuinfo
+		then
+			sed -n -e 's/^model name[^:]*: *//p' /proc/cpuinfo \
+			| while read -r _cpu_model
+			  do
+				  guess_hypervisor_from_cpu_model "${_cpu_model}"
+			  done \
+			| sort \
+			| uniq -c \
+			| awk '
+			  { if ($1 > most_c) { most_c = $1; most_s = $2 } }
+			  END {
+				if (most_s) print most_s
+				exit !most_s
+			  }' \
+			&& return 0
+		fi
+		return 1
+	fi
+}
+
+check_vm_arch_specific() {
+	case ${arch}
+	in
+		(ppc64|ppc64le)
+			# Check PPC64 LPAR, KVM
+
+			# example /proc/cpuinfo line indicating 'not baremetal'
+			# platform	: pSeries
+			#
+			# example /proc/ppc64/lparcfg systemtype line
+			# system_type=IBM pSeries (emulated by qemu)
+
+			if has_cpuinfo && grep -q -e 'platform.**pSeries' /proc/cpuinfo
+			then
+				if test -e /proc/ppc64/lparcfg
+				then
+					# Assume LPAR, now detect shared or dedicated
+					if grep -q -x -F 'shared_processor_mode=1' /proc/ppc64/lparcfg
+					then
+						echo powervm-shared
+						return 0
+					else
+						echo powervm-dedicated
+						return 0
+					fi
+				fi
+			fi
+			;;
+		(sparc*)
+			# Check for SPARC LDoms
+
+			if test -e /dev/mdesc
+			then
+				if test -d /sys/class/vlds/ctrl -a -d /sys/class/vlds/sp
+				then
+					# control LDom
+					return 1
+				else
+					# guest LDom
+					echo ldom-sparc
+				fi
+
+				# MDPROP=/usr/lib/ldoms/mdprop.py
+				# if test -x "${MDPROP}"
+				# then
+				# 	if test -n "$("${MDPROP}" -v iodevice device-type=pciex)"
+				# 	then
+				# 		echo ldoms-root
+				# 		echo ldoms-io
+				# 	elif test -n "$("${MDPROP}" -v iov-device vf-id=0)"
+				# 	then
+				# 		echo ldoms-io
+				# 	fi
+				# fi
+				return 0
+			fi
+			;;
+		(i?86|x86*|amd64|i86pc)
+			# Check CPUID
+			#
+			# Many fullvirt hypervisors give an indication through CPUID.  Use
+			# the virt-what helper program to get this information if available.
+
+			for CPUID_HELPER in \
+				$(command -v virt-what-cpuid-helper 2>/dev/null) \
+				/usr/lib/x86_64-*/virt-what-cpuid-helper \
+				/usr/lib/i?86-*/virt-what-cpuid-helper \
+				/usr/lib/virt-what/virt-what-cpuid-helper
+			do
+				if test -x "${CPUID_HELPER:?}"; then break; fi
+			done
+
+			if test -x "${CPUID_HELPER-}"
+			then
+				case $(command "${CPUID_HELPER}")
+				in
+					('bhyve bhyve ')
+						echo bhyve
+						;;
+					('LKVMLKVMLKVM')
+						echo lkvm
+						;;
+					('KVMKVMKVM')
+						echo kvm
+						;;
+					('TCGTCGTCGTCG')
+						echo qemu-tcg
+						;;
+					('Microsoft Hv')
+						# http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx
+						echo hyperv
+						;;
+					('OpenBSDVMM58')
+						# OpenBSD/VMM
+						echo openbsd_vmm
+						;;
+					('VMwareVMware')
+						# check added by Chetan Loke.
+						echo vmware
+						;;
+					('XenVMMXenVMM')
+						if has dmi
+						then
+							# https://access.redhat.com/solutions/222903
+							echo xen-hvm
+						else
+							echo xen-paravirt
+						fi
+						;;
+					(*)
+						return 1 ;;
+				esac
+				return 0
+			fi
+
+			unset CPUID_HELPER
+
+			# VMM CPUID flag denotes that this system is running under a VMM
+			if is_oneof "${uname_s}" Darwin
+			then
+				get_sysctl machdep.cpu.features | tr ' ' '\n' | grep -qixF VMM \
+				&& return 0
+			fi
+			if has_cpuinfo \
+				&& grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo
+			then
+				return 0
+			fi
+			;;
+		(ia64)
+			if test -d /sys/bus/xen -a ! -d /sys/bus/xen-backend
+			then
+				# PV-on-HVM drivers installed in a Xen guest
+				echo xen-hvm
+				return 0
+			fi
+			;;
+	esac
+	return 1
+}
+
+has_vm_dmi() {
+	# Check for various products in SMBIOS/DMI.
+	# Note that DMI doesn't exist on all architectures (only x86 and some ARM).
+	# On other architectures the $dmi variable will be empty.
+
+	if test -d /sys/class/dmi/id/
+	then
+		dmi_sysfs=/sys/class/dmi/id
+	elif test -d /sys/devices/virtual/dmi/id/
+	then
+		dmi_sysfs=/sys/devices/virtual/dmi/id
+	fi
+
+	# shellcheck disable=SC2015
+	{
+		is_command dmidecode \
+		&& (
+			# dmidecode needs to exit 0 and not print the No SMBIOS/DMI line
+			dmi_out=$(dmidecode 2>&1) \
+			&& ! printf '%s\n' "${dmi_out}" \
+				 | grep -qF 'No SMBIOS nor DMI entry point found, sorry.'
+			) \
+		|| test -d "${dmi_sysfs}"
+	}
+}
+
+check_vm_dmi() {
+	case $(get_dmi_field system-product-name)
+	in
+		(*.metal)
+			if test "$(get_dmi_field system-manufacturer)" = 'Amazon EC2'
+			then
+				# AWS EC2 bare metal -> no virtualisation
+				return 1
+			fi
+			;;
+		('BHYVE')
+			echo bhyve
+			return 0
+			;;
+		('Google Compute Engine')
+			echo gce
+			return 0
+			;;
+		('RHEV Hypervisor')
+			# Red Hat Enterprise Virtualization
+			echo rhev
+			return 0
+			;;
+		('KVM'|'Bochs'|'KVM Virtual Machine')
+			echo kvm
+			return 0
+			;;
+		('Parallels Virtual Platform')
+			echo parallels
+			return 0
+			;;
+		('VirtualBox')
+			echo virtualbox
+			return 0
+			;;
+		('VMware Virtual Platform')
+			echo vmware
+			return 0
+			;;
+	esac
+
+	case $(get_dmi_field system-manufacturer)
+	in
+		('Alibaba'*)
+			case $(get_dmi_field system-product-name)
+			in
+				('Alibaba Cloud ECS')
+					echo alibaba-ecs
+					;;
+				(*)
+					echo alibaba
+					;;
+			esac
+			return 0
+			;;
+		('Amazon EC2')
+			# AWS on bare-metal or KVM
+			echo aws-ec2
+			return 0
+			;;
+		('innotek GmbH'|'Oracle Corporation')
+			echo virtualbox
+			return 0
+			;;
+		('Joyent')
+			if test "$(get_dmi_field system-product-name)" = 'SmartDC HVM'
+			then
+				# SmartOS KVM
+				echo kvm-smartdc_hvm
+				return 0
+			fi
+			;;
+		('Microsoft Corporation'*)
+			if test "$(get_dmi_field system-product-name)" = 'Virtual Machine'
+			then
+				if test -e /proc/irq/7/hyperv \
+				|| expr "$(get_dmi_field bios-version)" : 'VRTUAL.*' >/dev/null
+				then
+					echo hyperv
+					return 0
+				fi
+
+				case $(get_dmi_field system-version)
+				in
+					(VPC[0-9]*|VS2005*|*[Vv]irtual*[Pp][Cc]*)
+						echo virtualpc
+						return 0
+						;;
+					(*)
+						echo hyperv
+						return 0
+						;;
+				esac
+			fi
+			;;
+		('Nutanix')
+			# Nutanix AHV. Similar to KVM.
+			if test "$(get_dmi_field system-product-name)" = 'AHV'
+			then
+				echo nutanix_ahv
+				return 0
+			fi
+			;;
+		('oVirt')
+			echo ovirt
+			return 0
+			;;
+		('Parallels Software International Inc.')
+			echo parallels
+			return 0
+			;;
+		('QEMU')
+			echo qemu
+			return 0
+			;;
+		('VMware, Inc.')
+			echo vmware
+			return 0
+			;;
+	esac
+
+	case $(get_dmi_field bios-vendor)
+	in
+		('Amazon EC2')
+			# AWS on bare-metal or KVM
+			echo aws-ec2
+			return 0
+			;;
+		('BHYVE')
+			echo bhyve
+			return 0
+			;;
+		('innotek GmbH')
+			echo virtualbox
+			return 0
+			;;
+		('Parallels Software International Inc.')
+			echo parallels
+			return 0
+			;;
+		('Xen')
+			if get_dmi_field bios-version | grep -q -e '\([0-9]\{1,\}\.\)\{2\}amazon'
+			then
+				# AWS on Xen
+				echo aws-xen
+				return 0
+			fi
+			;;
+	esac
+
+	return 1
+}
+
+check_vm_hyp_specific() {
+	if is_command vmware-checkvm && vmware-checkvm >/dev/null
+	then
+		# vmware-checkvm is provided by VMware's open-vm-tools
+		echo vmware
+		return 0
+	elif test -d /proc/xen
+	then
+		test -r /proc/xen/capabilities &&
+		if grep -q -F 'control_d' /proc/xen/capabilities 2>/dev/null
+		then
+			# Xen dom0
+			return 1
+		else
+			# Xen domU
+			echo xen
+			return 0
+		fi
+	fi
+	return 1
+}
+
+has_vm_dt() {
+	# OpenFirmware/Das U-Boot device-tree
+	test -d /proc/device-tree
+}
+
+check_vm_dt() {
+	case ${arch}
+	in
+		(arm|aarch64)
+			if test -r /proc/device-tree/hypervisor/compatible
+			then
+				if grep -q -F 'xen' /proc/device-tree/hypervisor/compatible
+				then
+					echo xen
+					return 0
+				elif grep -q -F 'vmware' /proc/device-tree/hypervisor/compatible
+				then
+					# e.g. VMware ESXi on ARM
+					echo vmware
+					return 0
+				fi
+			fi
+			if glob_exists /proc/device-tree/fw-cfg@*/compatible
+			then
+				# qemu,fw-cfg-mmio
+				sed -e 's/,.*$//' /proc/device-tree/fw-cfg@*/compatible | head -n1
+				return 0
+			fi
+			if grep -q -F 'dummy-virt' /proc/device-tree/compatible
+			then
+				echo lkvm
+				return 0
+			fi
+			;;
+		(ppc64*)
+			if test -d /proc/device-tree/hypervisor \
+				&& grep -qF 'linux,kvm' /proc/device-tree/hypervisor/compatible
+			then
+				# We are running as a spapr KVM guest on ppc64
+				echo kvm-spapr
+				return 0
+			fi
+			if test -r /proc/device-tree/ibm,partition-name \
+				&& test -r /proc/device-tree/hmc-managed\? \
+				&& test -r /proc/device-tree/chosen/qemu,graphic-width
+			then
+				echo powervm
+			fi
+			;;
+	esac
+	return 1
+}
+
+has_vm_sys_hypervisor() {
+	test -d /sys/hypervisor/
+}
+
+check_vm_sys_hypervisor() {
+	test -r /sys/hypervisor/type &&
+	case $(head -n1 /sys/hypervisor/type)
+	in
+		(xen)
+			# Ordinary kernel with pv_ops.	There does not seem to be
+			# enough information at present to tell whether this is dom0
+			# or domU.
+			echo xen
+			return 0
+			;;
+	esac
+	return 1
+}
+
+check_vm_os_specific() {
+	_hyp_generic=false
+
+	case ${uname_s}
+	in
+		(Darwin)
+			if hv_vmm_present=$(get_sysctl kern.hv_vmm_present) \
+				&& test "${hv_vmm_present}" -ne 0
+			then
+				_hyp_generic=true
+			fi
+			;;
+		(FreeBSD)
+			# FreeBSD does not have /proc/cpuinfo even when procfs is used.
+			# Instead there is a sysctl kern.vm_guest.
+			# Which is 'none' if physical, else the virtualisation.
+			vm_guest=$(get_sysctl kern.vm_guest | tolower) &&
+			case ${vm_guest}
+			in
+				(none) ;;
+				(generic) _hyp_generic=true ;;
+				(*)
+					# kernel could detect hypervisor
+					case ${vm_guest}
+					in
+						(hv) echo hyperv ;;
+						(vbox) echo virtualbox ;;
+						(*) echo "${vm_guest}" ;;
+					esac
+					return 0
+					;;
+			esac
+			;;
+		(NetBSD)
+			machdep_hv=$(get_sysctl machdep.hypervisor | tolower) &&
+			case ${machdep_hv}
+			in
+				(none) ;;
+				(generic) _hyp_generic=true ;;
+				(*)
+					# kernel could detect hypervisor
+					case ${machdep_hv}
+					in
+						(hyper-v) echo hyperv ;;
+						(xenhvm*) echo xen-hvm ;;
+						(xenpv*) echo xen-pv ;;
+						(xen*) echo xen ;;
+						(*) echo "${machdep_hv}" ;;
+					esac
+					return 0
+					;;
+			esac
+			;;
+		(OpenBSD)
+			if is_command hostctl && glob_exists /dev/pvbus[0-9]*
+			then
+				for _pvbus in /dev/pvbus[0-9]*
+				do
+					_h_out=$(hostctl -f "${_pvbus}" -t 2>/dev/null) || continue
+					case $(expr "${_h_out}" : '[^:]*: *\(.*\)$')
+					in
+						(KVM) echo kvm ;;
+						(Hyper-V) echo hyperv ;;
+						(VMware) echo vmware ;;
+						(Xen) echo xen ;;
+						(bhyve) echo bhyve ;;
+						(OpenBSD) echo openbsd_vmm ;;
+					esac
+					return 0
+				done
+			fi
+			;;
+		(SunOS)
+			diag_conf=$(prtdiag | sed -n -e 's/.*Configuration: *//p' -e '/^$/q')
+			# NOTE: Don't use -e or -F in Solaris grep
+			if printf '%s\n' "${diag_conf}" | grep -q -i QEMU
+			then
+				echo qemu
+				return 0
+			elif printf '%s\n' "${diag_conf}" | grep -q -i VMware
+			then
+				echo vmware
+				return 0
+			fi
+			;;
+		(Linux)
+			if is_command dmesg
+			then
+				while read -r line
+				do
+					case ${line}
+					in
+						('Booting paravirtualized kernel on ')
+							case $(expr "${line}" : '.* kernel on \(.*\)')
+							in
+								('Xen')
+									echo xen-pv; return 0 ;;
+								('bare hardware')
+									return 1 ;;
+							esac
+							;;
+						('Hypervisor detected')
+							case $(expr "${line}" : '.*: *\(.*\)')
+							in
+								('ACRN')
+									echo acrn ;;
+								('Jailhouse')
+									echo jailhouse ;;
+								('KVM')
+									echo kvm ;;
+								('Microsoft Hyper-V')
+									echo hyperv ;;
+								('VMware')
+									echo vmware ;;
+								('Xen HVM')
+									echo xen-hvm ;;
+								('Xen PV')
+									echo xen-pv ;;
+							esac
+							return 0
+							;;
+						(lpar:*' under hypervisor')
+							return 0 ;;
+					esac
+				done <<-EOF
+				$(dmesg 2>/dev/null | awk '
+				/Booting paravirtualized kernel on /
+				/Hypervisor detected: /
+				/lpar: .* under hypervisor/
+				')
+				EOF
+			fi
+	esac
+
+	# Try to guess hypervisor based on CPU model (sysctl hw.model if available)
+	if cpu_model=$(get_sysctl hw.model)
+	then
+		guess_hypervisor_from_cpu_model "${cpu_model}" && return 0
+	fi
+
+	if ${_hyp_generic}
+	then
+		# cannot say which hypervisor, but one was detected
+		return 0
+	else
+		return 1
+	fi
+}
+
+run_stage() {
+	if type "has_$1_$2" >/dev/null 2>&1
+	then
+		"has_$1_$2"
+	else
+		true
+	fi \
+	&& "check_$1_$2"
+}
+
+
+# Execute container stages
+
+for stage in \
+	pid_1 cgroup files os_specific
+do
+	ctengine=$(run_stage ct ${stage}) || continue
+	is_contained=true
+	if test -n "${ctengine}"
+	then
+		echo container on "${ctengine}"
+		break
+	fi
+done
+if ${is_contained:-false} && test -z "${ctengine}"
+then
+	# none of the stages could determine the specific container engine, but
+	# we are running in some container.
+	echo container
+fi
+
+
+# Execute virtual machine / hypervisor stages
+
+for stage in \
+	os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
+do
+	hypervisor=$(run_stage vm ${stage}) || continue
+	is_virtual=true
+	if test -n "${hypervisor}"
+	then
+		echo virtual by "${hypervisor}"
+		break
+	fi
+done
+if ${is_virtual:-false} && test -z "${hypervisor}"
+then
+	# none of the stages could determine the specific hypervisor, but
+	# we are virtual.
+	echo virtual
+fi

From abc6d009b21b0d1ce3fc5107201e30740f127200 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 19:29:41 +0200
Subject: [PATCH 374/411] [explorer/machine_type] Print top most machine layer
 as first line (fallback to physical)

---
 cdist/conf/explorer/machine_type | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 29f98849..90c441da 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -26,17 +26,19 @@
 # VERB does not have a special meaning, it is just for better readability.
 #
 # e.g.
+# container
 # container on lxc
 # virtual by kvm-spapr
 #
-# The third word of each line can be composed of different parts concatenated with a `-'
-# (minus) character, with each component being a specification of the previous,
-# e.g.:
+# The third word of each line (except the first) can be composed of different
+# parts concatenated with a `-' (minus) character, with each component being
+# a specification of the previous, e.g.:
 #  - lxc-libvirt             (LXC container, managed by libvirt)
 #  - lpar-s390 / lpar-power  (LPAR running on IBM S/390 or POWER, respectively)
 #  - xen-hvm / xen-pv        (Xen HVM vs para-virtualization)
 #
-# If this explorer cannot determine any information it will print nothing.
+# If this explorer cannot collect enough information about virtualization it
+# will fall back to 'physical'.
 #
 
 # Add /sbin and /usr/sbin to the path so we can find system
@@ -121,6 +123,10 @@ get_sysctl() {
 	is_command sysctl && sysctl -n "$1" 2>/dev/null
 }
 
+detected_layer() {
+	test -n "${_toplayer:-}" || echo "${_toplayer:=${1:?}}"
+}
+
 
 # Check for container
 
@@ -895,6 +901,7 @@ for stage in \
 	pid_1 cgroup files os_specific
 do
 	ctengine=$(run_stage ct ${stage}) || continue
+	detected_layer 'container'
 	is_contained=true
 	if test -n "${ctengine}"
 	then
@@ -916,6 +923,7 @@ for stage in \
 	os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
 do
 	hypervisor=$(run_stage vm ${stage}) || continue
+	detected_layer 'virtual machine'
 	is_virtual=true
 	if test -n "${hypervisor}"
 	then
@@ -929,3 +937,8 @@ then
 	# we are virtual.
 	echo virtual
 fi
+
+
+# Fallback
+
+detected_layer physical

From 2ffa895f578f12d615ed65b9a93f3e2ae07f1d07 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 19:57:24 +0200
Subject: [PATCH 375/411] [explorer/machine_type] Remove CPUID check

it's a lot of code and depends on a binary helper unlikely to be installed.
---
 cdist/conf/explorer/machine_type | 59 --------------------------------
 1 file changed, 59 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 90c441da..7ce035e3 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -379,65 +379,6 @@ check_vm_arch_specific() {
 			fi
 			;;
 		(i?86|x86*|amd64|i86pc)
-			# Check CPUID
-			#
-			# Many fullvirt hypervisors give an indication through CPUID.  Use
-			# the virt-what helper program to get this information if available.
-
-			for CPUID_HELPER in \
-				$(command -v virt-what-cpuid-helper 2>/dev/null) \
-				/usr/lib/x86_64-*/virt-what-cpuid-helper \
-				/usr/lib/i?86-*/virt-what-cpuid-helper \
-				/usr/lib/virt-what/virt-what-cpuid-helper
-			do
-				if test -x "${CPUID_HELPER:?}"; then break; fi
-			done
-
-			if test -x "${CPUID_HELPER-}"
-			then
-				case $(command "${CPUID_HELPER}")
-				in
-					('bhyve bhyve ')
-						echo bhyve
-						;;
-					('LKVMLKVMLKVM')
-						echo lkvm
-						;;
-					('KVMKVMKVM')
-						echo kvm
-						;;
-					('TCGTCGTCGTCG')
-						echo qemu-tcg
-						;;
-					('Microsoft Hv')
-						# http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx
-						echo hyperv
-						;;
-					('OpenBSDVMM58')
-						# OpenBSD/VMM
-						echo openbsd_vmm
-						;;
-					('VMwareVMware')
-						# check added by Chetan Loke.
-						echo vmware
-						;;
-					('XenVMMXenVMM')
-						if has dmi
-						then
-							# https://access.redhat.com/solutions/222903
-							echo xen-hvm
-						else
-							echo xen-paravirt
-						fi
-						;;
-					(*)
-						return 1 ;;
-				esac
-				return 0
-			fi
-
-			unset CPUID_HELPER
-
 			# VMM CPUID flag denotes that this system is running under a VMM
 			if is_oneof "${uname_s}" Darwin
 			then

From 23fbfaf0352eadaabd43504d3ee074bb9f696fcd Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 21:29:24 +0200
Subject: [PATCH 376/411] [explorer/machine_type] Use systemd-detect-virt (if
 available) to detect containers and VMs

---
 cdist/conf/explorer/machine_type | 49 ++++++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 7ce035e3..1a38fda0 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -130,6 +130,25 @@ detected_layer() {
 
 # Check for container
 
+has_ct_systemd() {
+	is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^  -c'
+}
+
+check_ct_systemd() (
+	_ctengine=$(systemd-detect-virt -c 2>/dev/null) &&
+	case ${_ctengine}
+	in
+		(''|'none')
+			return 1 ;;
+		('container-other')
+			return 0 ;;
+		('systemd-nspawn')
+			echo systemd_nspawn ;;
+		(*)
+			echo "${_ctengine}" ;;
+	esac
+)
+
 has_ct_pid_1() {
 	test -r /run/systemd/container -o -r /proc/1/environ
 }
@@ -267,6 +286,32 @@ guess_hypervisor_from_cpu_model() {
 	esac
 }
 
+has_vm_systemd() {
+	is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^  -v'
+}
+
+check_vm_systemd() (
+	_hypervisor=$(systemd-detect-virt -v 2>/dev/null) &&
+	case ${_hypervisor}
+	in
+		(''|'none')
+			return 1 ;;
+		('amazon')
+			echo aws ;;
+		('bochs')
+			echo kvm ;;
+		('microsoft')
+			# assumption
+			echo hyperv ;;
+		('oracle')
+			echo virtualbox ;;
+		('vm-other')
+			return 0 ;;
+		(*)
+			echo "${_hypervisor}" ;;
+	esac
+)
+
 has_vm_cpuinfo() { has_cpuinfo; }
 
 check_vm_cpuinfo() {
@@ -839,7 +884,7 @@ run_stage() {
 # Execute container stages
 
 for stage in \
-	pid_1 cgroup files os_specific
+	systemd pid_1 cgroup files os_specific
 do
 	ctengine=$(run_stage ct ${stage}) || continue
 	detected_layer 'container'
@@ -861,7 +906,7 @@ fi
 # Execute virtual machine / hypervisor stages
 
 for stage in \
-	os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
+	systemd os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
 do
 	hypervisor=$(run_stage vm ${stage}) || continue
 	detected_layer 'virtual machine'

From 4a05669765c2c00c19af0ef1b607b0f2efa10f42 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 22:01:28 +0200
Subject: [PATCH 377/411] [explorer/machine_type] Implement chroot detection

---
 cdist/conf/explorer/machine_type | 53 ++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 1a38fda0..10c914ba 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -53,6 +53,21 @@ uname_s=$(uname -s)
 
 is_command() { command -v "$1" >/dev/null 2>&1; }
 
+files_same() {
+	# shellcheck disable=SC2012
+	LC_ALL=C df -P "$1" "$2" 2>/dev/null | {
+		read -r _  # skip header line
+		read -r fs1 _ _ _ _ mp1
+		read -r fs2 _ _ _ _ mp2
+		test "${fs1}" = "${fs2}" -a "${mp1}" = "${mp2}" || return 1
+	} &&
+	ls -1Ldi "$1" "$2" 2>/dev/null | {
+		read -r ino1 _
+		read -r ino2 _
+		test "${ino1}" = "${ino2}" || return 1
+	}
+}
+
 is_oneof() (
 	x=$1; shift
 	for y
@@ -128,6 +143,32 @@ detected_layer() {
 }
 
 
+# Check for chroot
+
+has_chroot_systemd() {
+	is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^  -r'
+}
+
+check_chroot_systemd() {
+	systemd-detect-virt -r
+}
+
+has_chroot_debian_ischroot() {
+	is_command ischroot
+}
+
+check_chroot_debian_ischroot() {
+	ischroot --default-false
+}
+
+has_chroot_procfs() {
+	test -d /proc/
+}
+
+check_chroot_procfs() {
+	test -e /proc/1/root && ! files_same /proc/1/root /
+}
+
 # Check for container
 
 has_ct_systemd() {
@@ -881,6 +922,18 @@ run_stage() {
 }
 
 
+# Execute chroot stages
+
+for stage in \
+	procfs debian_ischroot systemd
+do
+	run_stage chroot ${stage} || continue
+	detected_layer 'chroot'
+	echo chroot
+	break
+done
+
+
 # Execute container stages
 
 for stage in \

From 5af1317c2969995871da1d17b951de60dbc3fd09 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sun, 1 Aug 2021 16:40:20 +0200
Subject: [PATCH 378/411] [explorer/machine_type] Try to detect chroot path

---
 cdist/conf/explorer/machine_type | 38 ++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 10c914ba..fa68ec4d 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -166,7 +166,28 @@ has_chroot_procfs() {
 }
 
 check_chroot_procfs() {
-	test -e /proc/1/root && ! files_same /proc/1/root /
+	if test -e /proc/1/root && ! files_same /proc/1/root /
+	then
+		# try to determine where the chroot has been mounted
+		(
+			rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}')
+
+			if test -e "${rootdev}"
+			then
+				# escape chroot to determine where the device containing the
+				# chroot's / is mounted
+				rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}')
+
+				# shellcheck disable=SC2012
+				root_ino=$(ls -1di / | awk '{print $1}')
+
+				# Get mount point
+				chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}"
+			fi
+		)
+		return 0
+	fi
+	return 1
 }
 
 # Check for container
@@ -927,11 +948,20 @@ run_stage() {
 for stage in \
 	procfs debian_ischroot systemd
 do
-	run_stage chroot ${stage} || continue
+	chrootpnt=$(run_stage chroot ${stage}) || continue
+	is_chrooted=true
 	detected_layer 'chroot'
-	echo chroot
-	break
+	if test -n "${chrootpnt}"
+	then
+		echo chroot at "${chrootpnt}"
+		break
+	fi
 done
+if ${is_chrooted:-false} && test -z "${chrootpnt}"
+then
+	# could determine chroot, but not its mount point
+	echo chroot
+fi
 
 
 # Execute container stages

From 05c2a62191f533ceff1e074df571278a103d75ab Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sun, 1 Aug 2021 23:09:02 +0200
Subject: [PATCH 379/411] [explorer/machine_type] Implement chroot detection
 using /proc/.../mountinfo

---
 cdist/conf/explorer/machine_type | 64 +++++++++++++++++++++-----------
 1 file changed, 43 insertions(+), 21 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index fa68ec4d..00646c75 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -165,30 +165,52 @@ has_chroot_procfs() {
 	test -d /proc/
 }
 
-check_chroot_procfs() {
+check_chroot_procfs() (
+	is_chroot=false  # default
 	if test -e /proc/1/root && ! files_same /proc/1/root /
 	then
-		# try to determine where the chroot has been mounted
-		(
-			rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}')
-
-			if test -e "${rootdev}"
-			then
-				# escape chroot to determine where the device containing the
-				# chroot's / is mounted
-				rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}')
-
-				# shellcheck disable=SC2012
-				root_ino=$(ls -1di / | awk '{print $1}')
-
-				# Get mount point
-				chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}"
-			fi
-		)
-		return 0
+		is_chroot=true
 	fi
-	return 1
-}
+	if test -e /proc/1/mountinfo -a -e /proc/self/mountinfo
+	then
+		has_mountinfo=true
+		cmp -s /proc/1/mountinfo /proc/self/mountinfo || is_chroot=true
+	fi
+
+	if ${is_chroot}
+	then
+		# try to determine where the chroot has been mounted
+		rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}')
+
+		if test -e "${rootdev}"
+		then
+			# escape chroot to determine where the device containing the
+			# chroot's / is mounted
+			rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}')
+
+			# shellcheck disable=SC2012
+			root_ino=$(ls -1di / | awk '{print $1}')
+
+			# escape chroot and find mount point by inode
+			chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}"
+		elif ${has_mountinfo}
+		then
+			while read -r mntid _ _ _ cmntpnt _
+			do
+				read -r _ _ _ _ hmntpnt _ <<-EOF
+				$(grep -e "^$((mntid)) " /proc/1/mountinfo)
+				EOF
+				printf '%s\n' "${hmntpnt%${cmntpnt}}"
+			done </proc/self/mountinfo \
+			| sort -u \
+			| head -n 1  # just take the first...
+		fi
+
+		return 0
+	else
+		return 1
+	fi
+)
 
 # Check for container
 

From 67f85546ec3bf865d368f268fdbbc121ccfd5742 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 23 Aug 2021 09:57:20 +0300
Subject: [PATCH 380/411] [explorer/os_version] add new debian code names:
 bookworm and trixie

---
 cdist/conf/explorer/os_version | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 7bc6dd6b..aea3c43f 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -54,6 +54,8 @@ in
               # sid versions don't have a number, so we decode by codename:
               case $(expr "$debian_version" : '\([a-z]\{1,\}\)/')
               in
+                  trixie) echo 12.99 ;;
+                  bookworm) echo 11.99 ;;
                   bullseye) echo 10.99 ;;
                   buster) echo 9.99 ;;
                   stretch) echo 8.99 ;;

From e1e134899811e51e4694e89e421790e41cb894bf Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 23 Aug 2021 10:44:48 +0300
Subject: [PATCH 381/411] [explorer/os_version] use 99.99 as fallback for
 unknown code names in */sid

---
 cdist/conf/explorer/os_version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index aea3c43f..bbc9e4f0 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -63,7 +63,7 @@ in
                   wheezy) echo 6.99 ;;
                   squeeze) echo 5.99 ;;
                   lenny) echo 4.99 ;;
-                  *) exit 1
+                  *) echo 99.99 ;;
               esac
               ;;
           *)

From 46ed48d546af6ff663b131c803a9027b19798ba1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 08:09:47 +0200
Subject: [PATCH 382/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index f9409d7e..8507c663 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,7 @@ next:
 	* Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera)
 	* Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera)
 	* Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera)
+	* Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 0546283d0ebc0058dbff68cec9d18b9281c26edd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 20:32:44 +0200
Subject: [PATCH 383/411] Update shellcheck disable

---
 cdist/conf/type/__apt_backports/manifest             | 1 +
 cdist/conf/type/__package_pkg_freebsd/gencode-remote | 1 +
 cdist/conf/type/__ssh_authorized_keys/explorer/keys  | 1 +
 3 files changed, 3 insertions(+)

diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index bc47d8de..6fcd9212 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -28,6 +28,7 @@
 #  lsb_release may not be given in all installations
 codename_os_release() {
     # shellcheck disable=SC1090
+    # shellcheck disable=SC1091
     . "$__global/explorer/os_release"
     printf "%s" "$VERSION_CODENAME"
 }
diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote
index 3f88f6bc..ca9aa45a 100755
--- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote
+++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote
@@ -37,6 +37,7 @@ assert ()                 #  If condition false,
 	then
 		echo "Assertion failed:  \"$1\""
 		# shellcheck disable=SC2039
+		# shellcheck disable=SC3044
 		echo "File \"$0\", line $lineno, called by $(caller 0)"
 		exit $E_ASSERT_FAILED
 	fi
diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
index cec25746..9694a64b 100755
--- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys
+++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 
 # shellcheck disable=SC1090
+# shellcheck disable=SC1091
 file="$( . "$__type_explorer/file" )"
 
 if [ -f "$file" ]

From 44741e714b16f7a00bf84bd54ae33eacc7593192 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 20:25:49 +0200
Subject: [PATCH 384/411] Release 6.9.8

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 8507c663..dcdc4b3d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)

From b8eb6e984c1638e8e167394c6b3aa482cb8aad49 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 20:47:50 +0200
Subject: [PATCH 385/411] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index dcdc4b3d..693d028f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Explorer machine_type: Rewrite (Dennis Camera)
+
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)

From d7fdc8006f747be86b64a2790266b50219bab507 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 21:54:45 +0300
Subject: [PATCH 386/411] allow empty file

---
 cdist/conf/type/__sed/explorer/file | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
index a40f71b3..5f0f4edd 100755
--- a/cdist/conf/type/__sed/explorer/file
+++ b/cdist/conf/type/__sed/explorer/file
@@ -9,13 +9,7 @@ fi
 
 if [ -f "$file" ]
 then
-    if [ -s "$file" ]
-    then
-        cat "$file"
-    else
-        echo "$file is empty" >&2
-        exit 1
-    fi
+    cat "$file"
 else
     echo "$file do not exist" >&2
     exit 1

From 0f6e48dbc657a9d9145b965eacf79f021a1419f5 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:24:26 +0300
Subject: [PATCH 387/411] use $__object/tempfile in target instead of mktemp,
 add comments

---
 cdist/conf/type/__sed/gencode-remote | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 836506f0..df929b42 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -35,8 +35,11 @@ then
     exit 0
 fi
 
+# we can't use -i, because it's not posix, so we fly with tempfile and cp
+# and we use cp because we want to preserve destination file's attributes
+
 # shellcheck disable=SC2016
-echo 'tmp="$( mktemp )"'
+echo 'tmp="$__object/tempfile"'
 
 echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
 

From 90488fcebcc00f3b4ea260867f8b31625df61e46 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:27:42 +0300
Subject: [PATCH 388/411] use -e

---
 cdist/conf/type/__sed/explorer/file | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
index 5f0f4edd..9262b613 100755
--- a/cdist/conf/type/__sed/explorer/file
+++ b/cdist/conf/type/__sed/explorer/file
@@ -7,10 +7,10 @@ else
     file="/$__object_id"
 fi
 
-if [ -f "$file" ]
+if [ ! -e "$file" ]
 then
-    cat "$file"
-else
     echo "$file do not exist" >&2
     exit 1
 fi
+
+cat "$file"

From b7f392fa371ff78591abb910f7e224e7587c81be Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:38:55 +0300
Subject: [PATCH 389/411] use -E for better compat (not really sure if it is
 posix at all)

---
 cdist/conf/type/__sed/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index df929b42..625448e6 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -24,7 +24,7 @@ sed_cmd='sed'
 
 if [ -f "$__object/parameter/regexp-extended" ]
 then
-    sed_cmd="$sed_cmd --regexp-extended"
+    sed_cmd="$sed_cmd -E"
 fi
 
 if echo "$script" \

From aabef7f44a98ad144a7e1fe68bfb562079916ef8 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:40:06 +0300
Subject: [PATCH 390/411] remove reading script from file

---
 cdist/conf/type/__sed/gencode-remote | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 625448e6..be7690e1 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -12,10 +12,6 @@ script="$( cat "$__object/parameter/script" )"
 if [ "$script" = '-' ]
 then
     script="$( cat "$__object/stdin" )"
-elif
-    [ -f "$script" ]
-then
-    script="$( cat "$script" )"
 fi
 
 file_from_target="$__object/explorer/file"

From 5bf0c71e7af49ff6e7c1cccec192f763a019503e Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:45:36 +0300
Subject: [PATCH 391/411] update man

---
 cdist/conf/type/__sed/man.rst | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst
index 4a728184..86789363 100644
--- a/cdist/conf/type/__sed/man.rst
+++ b/cdist/conf/type/__sed/man.rst
@@ -8,35 +8,40 @@ cdist-type__sed - Transform text files with ``sed``
 
 DESCRIPTION
 -----------
-TODO
+Transform text files with ``sed``.
 
 
 REQUIRED MULTIPLE PARAMETERS
 ----------------------------
 script
-   TODO
+   ``sed`` script.
+   If ``-`` then the script is read from ``stdin``.
 
 
 OPTIONAL PARAMETERS
 -------------------
 file
-   TODO
+   Path to the file. Defaults to ``$__object_id``.
 
 onchange
-   TODO
+   Execute this command if ``sed`` changes file.
 
 
 BOOLEAN PARAMETERS
 ------------------
 regexp-extended
-   TODO
+   Use extended regular expressions in the script.
+   Might not be supported with every ``sed`` version.
 
 
 EXAMPLES
 --------
+
 .. code-block:: sh
 
-   true
+   __sed /tmp/foobar --script 's/foo/bar/'
+
+   echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script -
 
 
 AUTHORS

From cd4acde67ec9c3967e6e08450734cf65040e787f Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 15 Sep 2021 09:22:27 +0300
Subject: [PATCH 392/411] grammar

---
 cdist/conf/type/__sed/explorer/file | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
index 9262b613..ec3d0fe8 100755
--- a/cdist/conf/type/__sed/explorer/file
+++ b/cdist/conf/type/__sed/explorer/file
@@ -9,7 +9,7 @@ fi
 
 if [ ! -e "$file" ]
 then
-    echo "$file do not exist" >&2
+    echo "$file does not exist" >&2
     exit 1
 fi
 

From 7b6789ddeb74450f70991efe957e3eb0f11043f6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 15:04:12 +0200
Subject: [PATCH 393/411] __package_update_index: fix complain about suite
 change

2 of 4th fix for ticket #861
---
 .../type/__package_update_index/gencode-remote     | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote
index 803468b5..a10c16d3 100755
--- a/cdist/conf/type/__package_update_index/gencode-remote
+++ b/cdist/conf/type/__package_update_index/gencode-remote
@@ -41,7 +41,19 @@ fi
 case "$type" in
     yum) ;;
     apt)
-        echo "apt-get --quiet update"
+        # There are special arguments to apt(8) to prevent aborts if apt woudn't been
+        # updated after the 19th April 2021 till the bullseye release. The additional
+        # arguments acknoledge the happend suite change (the apt(8) update does the
+        # same by itself).
+        #
+        # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+        # allows backward compatablility to pre-buster Debian versions.
+        #
+        # See more: ticket #861
+        # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+        apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
+        echo "apt-get --quiet $apt_opts update"
         echo "apt-cache updated (age was: $currage)" >> "$__messages_out"
         ;;
     pacman)

From 12787ffe2c48f5c6d534d10f8faadde30db477ac Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 15:12:20 +0200
Subject: [PATCH 394/411] __apt_source: fix complain about suite change

3 of 4th fix for ticket #861
---
 cdist/conf/type/__apt_source/gencode-remote | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_source/gencode-remote b/cdist/conf/type/__apt_source/gencode-remote
index 1e8592c6..973b0f6c 100755
--- a/cdist/conf/type/__apt_source/gencode-remote
+++ b/cdist/conf/type/__apt_source/gencode-remote
@@ -22,7 +22,21 @@
 name="$__object_id"
 destination="/etc/apt/sources.list.d/${name}.list"
 
+# There are special arguments to apt(8) to prevent aborts if apt woudn't been
+# updated after the 19th April 2021 till the bullseye release. The additional
+# arguments acknoledge the happend suite change (the apt(8) update does the
+# same by itself).
+#
+# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+# allows backward compatablility to pre-buster Debian versions.
+#
+# See more: ticket #861
+# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
+# run 'apt-get update' only if something changed with our sources.list file
+#   it will be run a second time on error as a redundancy messure to success
 if grep -q "^__file${destination}" "$__messages_in"; then
-   printf 'apt-get update || apt-get update\n'
+   printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts"
 fi
 

From d246e067109dc8b85ea89668f742affbb221d1a6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 14:49:23 +0200
Subject: [PATCH 395/411] __apt_update_index: fix complain about suite change

1 of 4th fix for ticket #861
---
 .../conf/type/__apt_update_index/gencode-remote  | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_update_index/gencode-remote b/cdist/conf/type/__apt_update_index/gencode-remote
index 70b59710..2d7f9030 100755
--- a/cdist/conf/type/__apt_update_index/gencode-remote
+++ b/cdist/conf/type/__apt_update_index/gencode-remote
@@ -18,9 +18,23 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+
+# There are special arguments to apt(8) to prevent aborts if apt woudn't been
+# updated after the 19th April 2021 till the bullseye release. The additional
+# arguments acknoledge the happend suite change (the apt(8) update does the
+# same by itself).
+#
+# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+# allows backward compatablility to pre-buster Debian versions.
+#
+# See more: ticket #861
+# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
 # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists
+#   it will be run a second time on error as a redundancy messure to success
 cat << DONE
 if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then
-   apt-get update || apt-get update
+   apt-get $apt_opts update || apt-get $apt_opts update
 fi
 DONE

From 3d7b31cbb43067271c2c510281f90f19dc5ec5fb Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 15:22:16 +0200
Subject: [PATCH 396/411] __package_apt: fix complain about suite change

the last fix for ticket #861 :-)
---
 cdist/conf/type/__package_apt/gencode-remote | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote
index fbfca330..79c0d9d3 100755
--- a/cdist/conf/type/__package_apt/gencode-remote
+++ b/cdist/conf/type/__package_apt/gencode-remote
@@ -81,12 +81,24 @@ aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::=
 
 case "$state_should" in
     present)
+        # There are special arguments to apt(8) to prevent aborts if apt woudn't been
+        # updated after the 19th April 2021 till the bullseye release. The additional
+        # arguments acknoledge the happend suite change (the apt(8) update does the
+        # same by itself).
+        #
+        # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+        # allows backward compatablility to pre-buster Debian versions.
+        #
+        # See more: ticket #861
+        # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+        apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
         # following is bit ugly, but important hack.
         # due to how cdist config run works, there isn't
         # currently better way to do it :(
         cat << EOF
 if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ]
-then echo apt-get update > /dev/null 2>&1 || true
+then echo apt-get $apt_opts update > /dev/null 2>&1 || true
 fi
 EOF
         if [ -n "$version" ]; then

From 72ff48154c958ebdedc8b6d919b6f372efa63be2 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 16 Sep 2021 21:36:39 +0300
Subject: [PATCH 397/411] add comments, add -u to diff

---
 cdist/conf/type/__sed/gencode-remote | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index be7690e1..f99c5a88 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -14,6 +14,8 @@ then
     script="$( cat "$__object/stdin" )"
 fi
 
+# since stdin is not available in explorer, we pull file from target with explorer
+
 file_from_target="$__object/explorer/file"
 
 sed_cmd='sed'
@@ -23,10 +25,10 @@ then
     sed_cmd="$sed_cmd -E"
 fi
 
-if echo "$script" \
-    | "$sed_cmd" -f - "$file_from_target" \
-    | diff "$file_from_target" - \
-    > /dev/null
+# do sed dry run, diff result and if no change, then there's nothing to do
+# also redirect diff's output to stderr for debugging purposes
+
+if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2
 then
     exit 0
 fi

From bf222d05435000063930ff199bd0dec94d7d8cd5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 21 Sep 2021 08:55:54 +0200
Subject: [PATCH 398/411] ++changelog

---
 docs/changelog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 693d028f..ab637f5f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,11 @@ Changelog
 
 next:
 	* Explorer machine_type: Rewrite (Dennis Camera)
+	* New type: __sed (Ander Punnar)
+	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From 15c642a9b7af42c368eb5a8e22cab5f07b39b95e Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 12:06:45 +0200
Subject: [PATCH 399/411] [__debconf_set_selections] Fix --file not being
 supported

Even if deprecated, the parameter *must* be supported, which isn't the case
right now.

This was due to a misunderstanding of how deprecating parameters work, see:
https://www.cdi.st/manual/latest/cdist-type.html#deprecated-parameters
---
 .../conf/type/__debconf_set_selections/parameter/deprecated/file | 1 +
 .../__debconf_set_selections/parameter/{deprecated => optional}  | 0
 2 files changed, 1 insertion(+)
 create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/deprecated/file
 rename cdist/conf/type/__debconf_set_selections/parameter/{deprecated => optional} (100%)

diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file
new file mode 100644
index 00000000..09db545a
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file
@@ -0,0 +1 @@
+'file' has been deprecated in favour of 'line' in order to provide idempotency.
diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated b/cdist/conf/type/__debconf_set_selections/parameter/optional
similarity index 100%
rename from cdist/conf/type/__debconf_set_selections/parameter/deprecated
rename to cdist/conf/type/__debconf_set_selections/parameter/optional

From 5b7cca99f73e4ec3dd5cdb90e4dbfd40ec8fe349 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 12:09:42 +0200
Subject: [PATCH 400/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index ab637f5f..142602c8 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
+	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From fc9bd40c9a34cf54e2d6cfd2fec089bd25e1b172 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 13:14:57 +0200
Subject: [PATCH 401/411] Improve bullseye support, perticularly
 __letsencrypt_cert

---
 cdist/conf/type/__grafana_dashboard/manifest | 2 +-
 cdist/conf/type/__letsencrypt_cert/manifest  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest
index d145c4c3..0d944482 100755
--- a/cdist/conf/type/__grafana_dashboard/manifest
+++ b/cdist/conf/type/__grafana_dashboard/manifest
@@ -15,7 +15,7 @@ case $os in
                 # Differntation not needed anymore
                 apt_source_distribution=stable
                 ;;
-            10*)
+            10*|11*)
                 # Differntation not needed anymore
                 apt_source_distribution=stable
                 ;;
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 6394f629..638a99e0 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -41,7 +41,7 @@ if [ -z "${certbot_fullpath}" ]; then
 					require="__apt_source/stretch-backports" __package_apt certbot \
 						--target-release stretch-backports
 				;;
-				10*)
+				10*|11*)
 					__package_apt certbot
 				;;
 

From 560374a6861281695b61c8c2298123be41d92326 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 13:16:11 +0200
Subject: [PATCH 402/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 142602c8..6f717cf4 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,7 @@ next:
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
+	* Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From c33d99ee120ad180daba144be80a25d77b473f56 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sun, 31 Oct 2021 17:38:10 +0100
Subject: [PATCH 403/411] [__haproxy_dualstack] New type with PROXY protocol
 support

This is backwards compatible with what is already used internally @ungleich, but
adds on top of that the ability to customise ports and, most importantly, it
adds PROXY protocol support.
---
 .../conf/type/__haproxy_dualstack/files/http  |   8 +
 .../conf/type/__haproxy_dualstack/files/https |  10 ++
 .../conf/type/__haproxy_dualstack/files/imaps |  12 ++
 .../conf/type/__haproxy_dualstack/files/smtps |  12 ++
 cdist/conf/type/__haproxy_dualstack/man.rst   | 121 ++++++++++++++
 cdist/conf/type/__haproxy_dualstack/manifest  | 155 ++++++++++++++++++
 .../parameter/default/protocol                |   1 +
 .../parameter/optional_multiple               |   3 +
 cdist/conf/type/__haproxy_dualstack/singleton |   0
 9 files changed, 322 insertions(+)
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/http
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/https
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/imaps
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/smtps
 create mode 100644 cdist/conf/type/__haproxy_dualstack/man.rst
 create mode 100644 cdist/conf/type/__haproxy_dualstack/manifest
 create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/default/protocol
 create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple
 create mode 100644 cdist/conf/type/__haproxy_dualstack/singleton

diff --git a/cdist/conf/type/__haproxy_dualstack/files/http b/cdist/conf/type/__haproxy_dualstack/files/http
new file mode 100644
index 00000000..0508a465
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/http
@@ -0,0 +1,8 @@
+frontend http
+	bind	BIND@:80
+	mode	http
+	option	httplog
+	default_backend	http
+
+backend http
+	mode	http
diff --git a/cdist/conf/type/__haproxy_dualstack/files/https b/cdist/conf/type/__haproxy_dualstack/files/https
new file mode 100644
index 00000000..73deac46
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/https
@@ -0,0 +1,10 @@
+frontend https
+	bind	BIND@:443
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	https
+
+backend https
+	mode	tcp
diff --git a/cdist/conf/type/__haproxy_dualstack/files/imaps b/cdist/conf/type/__haproxy_dualstack/files/imaps
new file mode 100644
index 00000000..b1ec3793
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/imaps
@@ -0,0 +1,12 @@
+frontend imaps
+	bind	BIND@:143
+	bind	BIND@:993
+
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	imaps
+
+backend imaps
+	mode	tcp
diff --git a/cdist/conf/type/__haproxy_dualstack/files/smtps b/cdist/conf/type/__haproxy_dualstack/files/smtps
new file mode 100644
index 00000000..dce6ed4a
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/smtps
@@ -0,0 +1,12 @@
+frontend smtps
+	bind	BIND@:25
+	bind	BIND@:465
+
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	smtps
+
+backend smtps
+	mode	tcp
diff --git a/cdist/conf/type/__haproxy_dualstack/man.rst b/cdist/conf/type/__haproxy_dualstack/man.rst
new file mode 100644
index 00000000..6c131cbe
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/man.rst
@@ -0,0 +1,121 @@
+cdist-type__haproxy_dualstack(7)
+================================
+
+
+NAME
+----
+cdist-type__haproxy_dualstack - Proxy services from a dual-stack server
+
+
+DESCRIPTION
+-----------
+This (singleton) type installs and configures haproxy to act as a dual-stack
+proxy for single-stack services.
+
+This can be useful to add IPv4 support to IPv6-only services while only using
+one IPv4 for many such services.
+
+By default this type uses the plain TCP proxy mode, which means that there is no
+need for TLS termination on this host when SNI is supported.
+This also means that proxied services will not receive the client's IP address,
+but will see the proxy's IP address instead (that of `$__target_host`).
+
+This can be solved by using the PROXY protocol, but do take into account that,
+e.g. nginx cannot serve both regular HTTP(S) and PROXY protocols on the same
+port, so you will need to use other ports for that.
+
+As a recommendation in this type: use TCP ports 8080 and 591 respectively to
+serve HTTP and HTTPS using the PROXY protocol.
+
+See the EXAMPLES for more details.
+
+
+OPTIONAL PARAMETERS
+-------------------
+v4proxy
+    Proxy incoming IPv4 connections to the equivalent IPv6 endpoint.
+    In its simplest use, it must be a NAME with an `AAAA` DNS entry, which is
+    the IP address actually providing the proxied services.
+    The full format of this argument is:
+    `[proxy:]NAME[[:PROTOCOL_1=PORT_1]...[:PROTOCOL_N=PORT_N]]`
+    Where starting with `proxy:` determines that the PROXY protocol must be
+    used and each `:PROTOCOL=PORT` (e.g. `:http=8080` or `:https=591`) is a PORT
+    override for the given PROTOCOL (see `--protocol`), if not present the
+    PROTOCOL's default port will be used.
+
+
+v6proxy
+    Proxy incoming IPv6 connections to the equivalent IPv4 endpoint.
+    In its simplest use, it must be a NAME with an `A` DNS entry, which is
+    the IP address actually providing the proxied services.
+    See `--v4proxy` for more options and details.
+
+protocol
+    Can be passed multiple times or as a space-separated list of protocols.
+    Currently supported protocols are: `http`, `https`, `imaps`, `smtps`.
+    This defaults to: `http https imaps smtps`.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Proxy the IPv6-only services so IPv4-only clients can access them
+    # This uses HAProxy's TCP mode for http, https, imaps and smtps
+    __haproxy_dualstack \
+        --v4proxy ipv6.chat \
+        --v4proxy matrix.ungleich.ch
+
+    # Proxy the IPv6-only HTTP(S) services so IPv4-only clients can access them
+    # Note this means that the backend IPv6-only server will only see
+    # the IPv6 address of the haproxy host managed by cdist, which can be
+    # troublesome if this information is relevant for analytics/security/...
+    # See the PROXY example below
+    __haproxy_dualstack \
+        --protocol http --protocol https \
+        --v4proxy ipv6.chat \
+        --v4proxy matrix.ungleich.ch
+
+    # Use the PROXY protocol to proxy the IPv6-only HTTP(S) services enabling
+    # IPv4-only clients to access them while maintaining the client's IP address
+    __haproxy_dualstack \
+        --protocol http --protocol https \
+        --v4proxy proxy:ipv6.chat:http=8080:https=591 \
+        --v4proxy proxy:matrix.ungleich.ch:http=8080:https=591
+    # Note however that the PROXY protocol is not compatible with regular
+    # HTTP(S) protocols, so your nginx will have to listen on different ports
+    # with the PROXY settings.
+    # Note that you will need to restrict access to the 8080 port to prevent
+    # Client IP spoofing.
+    # This can be something like:
+    # server {
+    #     # listen for regular HTTP connections
+    #     listen [::]:80 default_server;
+    #     listen 80 default_server;
+    #     # listen for PROXY HTTP connections
+    #     listen [::]:8080 proxy_protocol;
+    #     # Accept the Client's IP from the PROXY protocol
+    #     real_ip_header proxy_protocol;
+    # }
+
+
+SEE ALSO
+--------
+- https://www.haproxy.com/blog/enhanced-ssl-load-balancing-with-server-name-indication-sni-tls-extension/
+- https://www.haproxy.com/blog/haproxy/proxy-protocol/
+- https://docs.nginx.com/nginx/admin-guide/load-balancer/using-proxy-protocol/
+
+
+AUTHORS
+-------
+ungleich <foss--@--ungleich.ch>
+Evilham <cvs--@--evilham.com>
+
+
+COPYING
+-------
+Copyright \(C) 2021 ungleich glarus ag. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__haproxy_dualstack/manifest b/cdist/conf/type/__haproxy_dualstack/manifest
new file mode 100644
index 00000000..d110eea6
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/manifest
@@ -0,0 +1,155 @@
+#!/bin/sh -eu
+
+__package haproxy
+require="__package/haproxy" __start_on_boot haproxy
+
+tmpdir="$__object/files"
+mkdir "$tmpdir"
+configtmp="$__object/files/haproxy.cfg"
+
+os=$(cat "$__global/explorer/os")
+case $os in
+	freebsd)
+		CONFIG_FILE="/usr/local/etc/haproxy.conf"
+		cat <<EOF > "$configtmp"
+global
+	maxconn	4000
+	user	nobody
+	group	nogroup
+	daemon
+
+EOF
+
+		;;
+	*)
+		CONFIG_FILE="/etc/haproxy/haproxy.cfg"
+		cat <<EOF > "$configtmp"
+global
+	log	[::1] local2
+	chroot	/var/lib/haproxy
+	pidfile	/var/run/haproxy.pid
+	maxconn	4000
+	user	haproxy
+	group	haproxy
+	daemon
+
+	# turn on stats unix socket
+	stats socket	/var/lib/haproxy/stats
+
+EOF
+		;;
+esac
+
+cat <<EOF >> "$configtmp"
+defaults
+	retries	3
+	log	global
+	timeout http-request	10s
+	timeout queue		1m
+	timeout connect		10s
+	timeout client		1m
+	timeout server		1m
+	timeout http-keep-alive	10s
+	timeout check		10s
+EOF
+
+dig_cmd="$(command -v dig || true)"
+get_ip() {
+	# Usage: get_ip (ipv4|ipv6) NAME
+	# uses "dig" if available, else fallback to "host"
+	case $1 in
+		ipv4)
+			if [ -n "${dig_cmd}" ]; then
+				${dig_cmd} +short A "$2"
+			else
+				host -t A "$2" | cut -d ' ' -f 4 | grep -v 'found:'
+			fi
+			;;
+		ipv6)
+			if [ -n "${dig_cmd}" ]; then
+				${dig_cmd} +short AAAA "$2"
+			else
+				host -t AAAA "$2" | cut -d ' ' -f 5 | grep -v 'NXDOMAIN'
+			fi
+			;;
+	esac
+}
+
+PROTOCOLS="$(cat "$__object/parameter/protocol")"
+
+for proxy in v4proxy v6proxy; do
+	param=$__object/parameter/$proxy
+	# no backend? skip generating code
+	if [ ! -f "$param" ]; then
+		continue
+	fi
+
+	# turn backend name into bind parameter: v4backend -> ipv4@
+	bind=$(echo $proxy | sed -e 's/^/ip/' -e 's/proxy//')
+
+	case $bind in
+		ipv4)
+			backendproto=ipv6
+			;;
+		ipv6)
+			backendproto=ipv4
+			;;
+	esac
+
+	for proto in ${PROTOCOLS}; do
+		# Add protocol "header"
+		printf "\n# %s %s \n" "${bind}" "${proto}" >> "$configtmp"
+
+		sed -e "s/BIND/$bind/" \
+			-e "s/\(frontend[[:space:]].*\)/\1$bind/" \
+			-e "s/\(backend[[:space:]].*\)/\\1$bind/" \
+			"$__type/files/$proto" >> "$configtmp"
+
+		while read -r hostdefinition; do
+			if echo "$hostdefinition" | grep -qE '^proxy:'; then
+				# Proxy protocol was requested
+				host="$(echo "$hostdefinition" | sed -E 's/^proxy:([^:]+).*$/\1/')"
+				send_proxy=" send-proxy"
+			else
+				# Just use tcp proxy mode
+				host="$hostdefinition"
+				send_proxy=""
+			fi
+			if echo "$hostdefinition" | grep -qE ":${proto}="; then
+				# Use custom port definition if requested
+				port="$(echo "$hostdefinition" | sed -E "s/^(.*:)?${proto}=([0-9]+).*$/:\2/")"
+			else
+				# Else use the default
+				port=""
+			fi
+			servername=$host
+
+			res=$(get_ip "$bind" "$servername")
+
+			if [ -z "$res" ]; then
+				echo "$servername does not resolve - aborting config" >&2
+				exit 1
+			fi
+
+			# Treat protocols without TLS+SNI specially
+			if [ "$proto" = http ]; then
+				echo "	use-server $servername if { hdr(host) -i $host }" >> "$configtmp"
+			else
+				echo "	use-server $servername if { req_ssl_sni -i $host }" >> "$configtmp"
+			fi
+
+			# Create the "server" itself.
+			# Note that port and send_proxy will be empty unless
+			# they were requested by the type user
+			echo "	server $servername ${backendproto}@${host}${port}${send_proxy}" >> "$configtmp"
+
+		done < "$param"
+	done
+done
+
+# Create config file
+require="__package/haproxy" __file ${CONFIG_FILE} --source "$configtmp" --mode 0644
+
+require="__file${CONFIG_FILE}" __check_messages "haproxy_reload" \
+	--pattern "^__file${CONFIG_FILE}" \
+	--execute "service haproxy reload || service haproxy restart"
diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol
new file mode 100644
index 00000000..dc8bb7bf
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol
@@ -0,0 +1 @@
+http https imaps smtps
diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple
new file mode 100644
index 00000000..8c482bd4
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple
@@ -0,0 +1,3 @@
+protocol
+v4proxy
+v6proxy
diff --git a/cdist/conf/type/__haproxy_dualstack/singleton b/cdist/conf/type/__haproxy_dualstack/singleton
new file mode 100644
index 00000000..e69de29b

From e2500248f2ddc83129e77f2e6b8dffb64904dbae Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Wed, 3 Nov 2021 11:03:33 +0100
Subject: [PATCH 404/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 6f717cf4..99a8c08b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Explorer machine_type: Rewrite (Dennis Camera)
 	* New type: __sed (Ander Punnar)
+	* New type: __haproxy_dualstack (Evilham and ungleich)
 	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)

From 3a321469a8ba5aea55220bd70bd4900de732e917 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 16 Nov 2021 11:11:45 +0100
Subject: [PATCH 405/411] Python 3.10: collections.X -> collections.abc.X

---
 cdist/integration.py     | 2 +-
 cdist/util/fsproperty.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/integration.py b/cdist/integration.py
index 17b65f09..04470ea7 100644
--- a/cdist/integration.py
+++ b/cdist/integration.py
@@ -84,7 +84,7 @@ def _process_hosts_simple(action, host, manifest, verbose,
     """
     if isinstance(host, str):
         hosts = [host, ]
-    elif isinstance(host, collections.Iterable):
+    elif isinstance(host, collections.abc.Iterable):
         hosts = host
     else:
         raise cdist.Error('Invalid host argument: {}'.format(host))
diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py
index 09e9cc19..6bf935e8 100644
--- a/cdist/util/fsproperty.py
+++ b/cdist/util/fsproperty.py
@@ -33,7 +33,7 @@ class AbsolutePathRequiredError(cdist.Error):
         return 'Absolute path required, got: {}'.format(self.path)
 
 
-class FileList(collections.MutableSequence):
+class FileList(collections.abc.MutableSequence):
     """A list that stores it's state in a file.
 
     """
@@ -102,7 +102,7 @@ class FileList(collections.MutableSequence):
         self.__write(lines)
 
 
-class DirectoryDict(collections.MutableMapping):
+class DirectoryDict(collections.abc.MutableMapping):
     """A dict that stores it's items as files in a directory.
 
     """

From 6e3ad11ea0177865e7e0288b1765267df8d5d020 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Thu, 23 Dec 2021 20:07:28 +0100
Subject: [PATCH 406/411] [__package_upgrade_all] Add new --apt-with-new-pkgs
 argument

---
 cdist/conf/type/__package_upgrade_all/gencode-remote    | 6 +++++-
 cdist/conf/type/__package_upgrade_all/man.rst           | 8 ++++++++
 cdist/conf/type/__package_upgrade_all/parameter/boolean | 1 +
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote
index 38aa001e..d332e851 100755
--- a/cdist/conf/type/__package_upgrade_all/gencode-remote
+++ b/cdist/conf/type/__package_upgrade_all/gencode-remote
@@ -28,6 +28,10 @@ apt_clean="$__object/parameter/apt-clean"
 
 apt_dist_upgrade="$__object/parameter/apt-dist-upgrade"
 
+if [ -f "$__object/parameter/apt-with-new-pkgs" ]; then
+	apt_with_new_pkgs="--with-new-pkgs"
+fi
+
 if [ -f "$type" ]; then
     type="$(cat "$type")"
 else
@@ -54,7 +58,7 @@ case "$type" in
     apt)
         if [ -f "$apt_dist_upgrade" ]
         then echo "$aptget dist-upgrade"
-        else echo "$aptget upgrade"
+        else echo "$aptget $apt_with_new_pkgs upgrade"
         fi
 
         if [ -f "$apt_clean" ]
diff --git a/cdist/conf/type/__package_upgrade_all/man.rst b/cdist/conf/type/__package_upgrade_all/man.rst
index e9e2b8ce..0c116bac 100644
--- a/cdist/conf/type/__package_upgrade_all/man.rst
+++ b/cdist/conf/type/__package_upgrade_all/man.rst
@@ -33,6 +33,14 @@ BOOLEAN PARAMETERS
 apt-dist-upgrade
     Do dist-upgrade instead of upgrade.
 
+apt-with-new-pkg
+    Allow installing new packages when used in conjunction with
+    upgrade. This is useful if the update of an installed package
+    requires new dependencies to be installed. Instead of holding the
+    package back upgrade will upgrade the package and install the new
+    dependencies. Note that upgrade with this option will never remove
+    packages, only allow adding new ones.
+
 apt-clean
     Clean out the local repository of retrieved package files.
 
diff --git a/cdist/conf/type/__package_upgrade_all/parameter/boolean b/cdist/conf/type/__package_upgrade_all/parameter/boolean
index 7a56a34b..cd22eb90 100644
--- a/cdist/conf/type/__package_upgrade_all/parameter/boolean
+++ b/cdist/conf/type/__package_upgrade_all/parameter/boolean
@@ -1,2 +1,3 @@
 apt-clean
 apt-dist-upgrade
+apt-with-new-pkgs

From c2c5668b704e1648ff6c8fb88219badddd028346 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Thu, 23 Dec 2021 20:08:49 +0100
Subject: [PATCH 407/411] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 99a8c08b..26d89057 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* New type: __haproxy_dualstack (Evilham and ungleich)
 	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_upgrade_all: Add new --apt-with-new-pkgs argument (Evilham)
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)

From 08ff41efded6e3112fc462ba13d9166e620b4082 Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Tue, 8 Mar 2022 12:04:58 +0100
Subject: [PATCH 408/411] Added rm of tmpfile.

---
 cdist/conf/type/__ssh_authorized_key/gencode-remote | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote
index 61c77fb9..cbffde94 100755
--- a/cdist/conf/type/__ssh_authorized_key/gencode-remote
+++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote
@@ -40,6 +40,7 @@ if [ -f "$file" ]; then
    grep -v -F -x '$line' '$file' >\$tmpfile
 fi
 cat "\$tmpfile" >"$file"
+rm -f "\$tmpfile"
 DONE
 }
 

From e0150e779681e232f95bdbefd957c666f05daa89 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Wed, 9 Mar 2022 16:16:44 +0100
Subject: [PATCH 409/411] ++changes

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 26d89057..81be51f6 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,8 @@ next:
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
 	* Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham)
+	* Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom)
+	* Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From bd44c023d33eb51a09afad12b18c082f6a3ae36d Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Fri, 11 Jun 2021 11:22:31 +0100
Subject: [PATCH 410/411] Fix typos; add default priority; comments in
 generated files

---
 cdist/conf/type/__apt_pin/manifest                   | 5 +++++
 cdist/conf/type/__apt_pin/parameter/default/priority | 1 +
 cdist/conf/type/__apt_pin/parameter/optional         | 1 +
 cdist/conf/type/__apt_pin/parameter/required         | 1 -
 4 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 cdist/conf/type/__apt_pin/parameter/default/priority

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index e72a8fdd..983b2b42 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -57,6 +57,11 @@ __file "/etc/apt/preferences.d/$name" \
     --owner root --group root --mode 0644 \
     --state "$state" \
     --source - << EOF
+# Created by cdist ${__type##*/}
+# Do not change. Changes will be overwritten.
+#
+
+# $name
 Package: $package
 Pin: $pin
 Pin-Priority: $priority
diff --git a/cdist/conf/type/__apt_pin/parameter/default/priority b/cdist/conf/type/__apt_pin/parameter/default/priority
new file mode 100644
index 00000000..1b79f38e
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/default/priority
@@ -0,0 +1 @@
+500
diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional
index 52f01fd2..847e703d 100644
--- a/cdist/conf/type/__apt_pin/parameter/optional
+++ b/cdist/conf/type/__apt_pin/parameter/optional
@@ -1,2 +1,3 @@
 state
 package
+priority
diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required
index 4b4e9741..c8572d92 100644
--- a/cdist/conf/type/__apt_pin/parameter/required
+++ b/cdist/conf/type/__apt_pin/parameter/required
@@ -1,2 +1 @@
 distribution
-priority

From 22039284f57f575defa0bc65c46b8bbcbe016cd8 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@armstrong.cc>
Date: Sun, 10 Apr 2022 23:52:53 +0200
Subject: [PATCH 411/411] __file: make file uploading and attribute changes
 more atomic

Fixes https://code.ungleich.ch/ungleich-public/cdist/pulls/331

Signed-off-by: Steven Armstrong <steven@armstrong.cc>
---
 cdist/conf/type/__file/gencode-local  | 28 +++++++++++++++++++--------
 cdist/conf/type/__file/gencode-remote | 14 +++++++++++++-
 2 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local
index 231b6927..bea3d79c 100755
--- a/cdist/conf/type/__file/gencode-local
+++ b/cdist/conf/type/__file/gencode-local
@@ -1,7 +1,7 @@
 #!/bin/sh -e
 #
 # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
-# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -89,10 +89,26 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then
       touch "$__object/files/set-attributes"
 
       # upload file to temp location
-      tempfile_template="${destination}.cdist.XXXXXXXXXX"
+      upload_destination="$(mktemp -u "${destination}.cdist.XXXXXXXXXX")"
+      # Yes, we are aware that this is a race condition.
+      # However:
+      # a) cdist usually writes to directories that are not user writable
+      #    (probably > 99.9%)
+      # b) if they are user owned, the user / attacker always wins
+      #    (probably < 0.1%)
+      # c) the only case which we could improve are tmp directories and we
+      #    don't think managing tmp directories with cdist is a typical case
+      #    ("the rest %)"
       cat << DONE
-destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template")"
+$__remote_exec $__target_host test -e $upload_destination && {
+   echo "Refusing to upload file to existing destination: $upload_destination" >&2
+   exit 1
+}
 DONE
+      # Tell gencode-remote to where we uploaded the file so it can move
+      # it to its final destination.
+      echo "$upload_destination" > "$__object/files/upload-destination"
+
       if [ "$upload_file" ]; then
          echo upload >> "$__messages_out"
          # IPv6 fix
@@ -103,12 +119,8 @@ DONE
              my_target_host="${__target_host}"
          fi
          cat << DONE
-$__remote_copy "$source" "${my_target_host}:\$destination_upload"
+$__remote_copy "$source" "${my_target_host}:${upload_destination}"
 DONE
       fi
-# move uploaded file into place
-cat << DONE
-$__remote_exec $__target_host "rm -rf \"$destination\"; mv \"\$destination_upload\" \"$destination\""
-DONE
    fi
 fi
diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index f7a528fd..136520a7 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -1,7 +1,7 @@
 #!/bin/sh -e
 #
 # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
-# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -62,6 +62,13 @@ set_mode() {
 
 case "$state_should" in
     present|exists)
+        if [ -f "$__object/files/upload-destination" ]; then
+            final_destination="$destination"
+            # We change the 'global' $destination variable here so we can
+            # change attributes of the new/uploaded file before moving it
+            # to it's final destination.
+            destination="$(cat "$__object/files/upload-destination")"
+        fi
         # Note: Mode - needs to happen last as a chown/chgrp can alter mode by
         #  clearing S_ISUID and S_ISGID bits (see chown(2))
         for attribute in group owner mode; do
@@ -81,6 +88,11 @@ case "$state_should" in
                 fi
             fi
         done
+        if [ -f "$__object/files/upload-destination" ]; then
+            # move uploaded file into place
+            printf 'rm -rf "%s"\n' "$final_destination"
+            printf 'mv -T "%s" "%s"\n' "$destination" "$final_destination"
+        fi
         if [ -f "$__object/files/set-attributes" ]; then
             # set-attributes is created if file is created or uploaded in gencode-local
             fire_onchange=1