From 64c247026a4fe02db1d31e2bd45839f577c7c1bb Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 30 Sep 2019 14:20:26 +0200
Subject: [PATCH 001/743] [__locale_system] Support Devuan

---
 cdist/conf/type/__locale_system/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 80f7401b..4a1fdeed 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -26,7 +26,7 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-    debian|ubuntu)
+    debian|devuan|ubuntu)
         locale_conf="/etc/default/locale"
         ;;
     archlinux)

From 3c9eb7bce769e7a902e437cdb411be9f5179fe27 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Aug 2019 16:15:47 +0200
Subject: [PATCH 002/743] [__hostname] Support all systemd OSes and prefer
 hostnamectl

---
 cdist/conf/type/__hostname/explorer/hostname | 48 ++++++++++++++++++++
 cdist/conf/type/__hostname/gencode-remote    | 32 +++++++++----
 cdist/conf/type/__hostname/manifest          |  4 +-
 3 files changed, 74 insertions(+), 10 deletions(-)
 create mode 100644 cdist/conf/type/__hostname/explorer/hostname

diff --git a/cdist/conf/type/__hostname/explorer/hostname b/cdist/conf/type/__hostname/explorer/hostname
new file mode 100644
index 00000000..aaaa3a30
--- /dev/null
+++ b/cdist/conf/type/__hostname/explorer/hostname
@@ -0,0 +1,48 @@
+#!/bin/sh
+#
+# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Retrieve the running hostname
+#
+
+# Firstly, check hostnamectl if available.
+if command -v hostnamectl >/dev/null
+then
+    hostnamectl --static && exit 0
+fi
+
+
+# Almost any distribution
+if [ -f /etc/hostname ]; then
+    cat /etc/hostname
+# SuSE
+elif [ -f /etc/HOSTNAME ]; then
+    cat /etc/HOSTNAME
+# OpenBSD
+elif [ -f /etc/myname ]; then
+    cat /etc/myname
+# FreeBSD
+elif [ "$(uname -s)" == FreeBSD ]; then
+    (. /etc/rc.conf && echo $hostname)
+# Mac OS X
+elif [ "$(uname -s)" == Darwin -a -f "$(command -v scutil)" ]; then
+    scutil --get HostName 2>/dev/null || hostname -s
+else
+    command -v hostname && hostname -s
+fi
diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 8b5797dd..20de7078 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -2,6 +2,7 @@
 #
 # 2014-2017 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2014 Nico Schottelius (nico-cdist at schottelius.org)
+# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -46,8 +47,9 @@ case "$os" in
         fi
     ;;
     *)
-        echo "Unsupported os: $os" >&2
-        exit 1
+        if [ "$name_running" -a "$name_running" = "$name_should" ]; then
+            exit 0
+        fi
     ;;
 esac
 
@@ -56,23 +58,35 @@ esac
 #
 echo changed >> "$__messages_out"
 
-# Use the good old way to set the hostname even on machines running systemd.
+
+# First try to set the hostname using hostnamectl, if available.
+if [ "$has_hostnamectl" ]; then
+    # Allow hostnamectl to fail silently.
+    # Who the fuck invented a tool that needs dbus to set the hostname anyway ...
+
+    cat <<EOF
+if hostnamectl set-hostname '$name_should'; then
+    printf '%s\\n' '$name_should' > /etc/hostname
+    exit 0
+fi
+EOF
+fi
+
+# Use the good old way to set the hostname. Also if hostnamectl fails.
 case "$os" in
     archlinux|debian|ubuntu|devuan|centos|coreos|alpine)
         printf "printf '%%s\\\\n' '$name_should' > /etc/hostname\\n"
         echo "hostname -F /etc/hostname"
     ;;
     freebsd|openbsd)
+        # NOTE: Writing the hostname to file is done in the manifest.
         echo "hostname '$name_should'"
     ;;
     suse)
         echo "hostname '$name_should'"
         printf "printf '%%s\\\\n' '$name_should' > /etc/HOSTNAME\\n"
     ;;
+    *)
+        echo "echo 'Unsupported OS: $os' >&2; exit 1"
+    ;;
 esac
-
-if [ "$has_hostnamectl" ]; then
-    # Allow hostnamectl set-hostname to fail silently.
-    # Who the fuck invented a tool that needs dbus to set the hostname anyway ...
-    echo "hostnamectl set-hostname '$name_should' || true"
-fi
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 8f1adf12..5ea3b8ff 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -2,6 +2,7 @@
 #
 # 2012 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2014 Nico Schottelius (nico-cdist at schottelius.org)
+# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -63,6 +64,7 @@ case "$os" in
         echo "$name_should" | __file /etc/myname --source -
     ;;
     *)
-        not_supported
+        # On other operating systems we fall back to hostnamectl if available…
+        test -n "$(cat "$__object/explorer/has_hostnamectl")" || not_supported
     ;;
 esac

From 2804a8bae6de1f0c3ae376092a3102d8a6964373 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Aug 2019 16:16:38 +0200
Subject: [PATCH 003/743] [__hostname] Add support for macosx

---
 cdist/conf/type/__hostname/explorer/hostname | 2 +-
 cdist/conf/type/__hostname/gencode-remote    | 3 +++
 cdist/conf/type/__hostname/manifest          | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__hostname/explorer/hostname b/cdist/conf/type/__hostname/explorer/hostname
index aaaa3a30..0b562d11 100644
--- a/cdist/conf/type/__hostname/explorer/hostname
+++ b/cdist/conf/type/__hostname/explorer/hostname
@@ -41,7 +41,7 @@ elif [ -f /etc/myname ]; then
 elif [ "$(uname -s)" == FreeBSD ]; then
     (. /etc/rc.conf && echo $hostname)
 # Mac OS X
-elif [ "$(uname -s)" == Darwin -a -f "$(command -v scutil)" ]; then
+elif [ "$(uname -s)" == Darwin -a -x "$(command -v scutil)" ]; then
     scutil --get HostName 2>/dev/null || hostname -s
 else
     command -v hostname && hostname -s
diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 20de7078..ee12f822 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -86,6 +86,9 @@ case "$os" in
         echo "hostname '$name_should'"
         printf "printf '%%s\\\\n' '$name_should' > /etc/HOSTNAME\\n"
     ;;
+    macosx)
+        echo "scutil --set HostName '$name_should'"
+    ;;
     *)
         echo "echo 'Unsupported OS: $os' >&2; exit 1"
     ;;
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 5ea3b8ff..fe067718 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -42,7 +42,7 @@ not_supported() {
 }
 
 case "$os" in
-    archlinux|debian|suse|ubuntu|devuan|coreos|alpine)
+    archlinux|debian|suse|ubuntu|devuan|coreos|alpine|macosx)
         # handled in gencode-remote
         :
     ;;

From abc765bd99fae9e479dfed2a3d39e780a1ab3c34 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 30 Sep 2019 12:52:43 +0200
Subject: [PATCH 004/743] [__hostname/explorer/hostname] Darwin: Always fall
 back to hostname if scutil is missing or fails

---
 cdist/conf/type/__hostname/explorer/hostname | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__hostname/explorer/hostname b/cdist/conf/type/__hostname/explorer/hostname
index 0b562d11..8aab1ec0 100644
--- a/cdist/conf/type/__hostname/explorer/hostname
+++ b/cdist/conf/type/__hostname/explorer/hostname
@@ -41,8 +41,8 @@ elif [ -f /etc/myname ]; then
 elif [ "$(uname -s)" == FreeBSD ]; then
     (. /etc/rc.conf && echo $hostname)
 # Mac OS X
-elif [ "$(uname -s)" == Darwin -a -x "$(command -v scutil)" ]; then
-    scutil --get HostName 2>/dev/null || hostname -s
+elif [ "$(uname -s)" == Darwin ]; then
+    [ -x "$(command -v scutil)" ] && scutil --get HostName 2>/dev/null || hostname -s
 else
     command -v hostname && hostname -s
 fi

From 81ba849af8340ff3d59646d602a429b9170b9701 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 3 Oct 2019 19:19:06 +0200
Subject: [PATCH 005/743] Move __hostname hostname explorer to global scope

---
 cdist/conf/explorer/hostname                  | 24 ++++++++--
 cdist/conf/type/__hostname/explorer/hostname  | 48 -------------------
 .../__hostname/explorer/hostname_sysconfig    |  2 +-
 3 files changed, 21 insertions(+), 53 deletions(-)
 delete mode 100644 cdist/conf/type/__hostname/explorer/hostname

diff --git a/cdist/conf/explorer/hostname b/cdist/conf/explorer/hostname
index 7715c6b0..13ac91c2 100755
--- a/cdist/conf/explorer/hostname
+++ b/cdist/conf/explorer/hostname
@@ -1,7 +1,6 @@
 #!/bin/sh
 #
-# 2010-2014 Nico Schottelius (nico-cdist at schottelius.org)
-# 2012 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,7 +18,24 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
+# Retrieve the running hostname
+#
 
-if command -v uname >/dev/null; then
-   uname -n
+# Firstly, check hostnamectl if available.
+if command -v hostnamectl >/dev/null
+then
+	hostnamectl status --static && exit 0
+fi
+
+# In case hostnamectl is missing or failed:
+# We fall back to alternative approaches
+if [ "$(uname -s)" = Darwin ] && command -v scutil >/dev/null
+then
+	# Mac OS X
+	scutil --get HostName 2>/dev/null
+elif command -v hostname >/dev/null
+then
+	hostname -s
+else
+	uname -n
 fi
diff --git a/cdist/conf/type/__hostname/explorer/hostname b/cdist/conf/type/__hostname/explorer/hostname
deleted file mode 100644
index 8aab1ec0..00000000
--- a/cdist/conf/type/__hostname/explorer/hostname
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/bin/sh
-#
-# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Retrieve the running hostname
-#
-
-# Firstly, check hostnamectl if available.
-if command -v hostnamectl >/dev/null
-then
-    hostnamectl --static && exit 0
-fi
-
-
-# Almost any distribution
-if [ -f /etc/hostname ]; then
-    cat /etc/hostname
-# SuSE
-elif [ -f /etc/HOSTNAME ]; then
-    cat /etc/HOSTNAME
-# OpenBSD
-elif [ -f /etc/myname ]; then
-    cat /etc/myname
-# FreeBSD
-elif [ "$(uname -s)" == FreeBSD ]; then
-    (. /etc/rc.conf && echo $hostname)
-# Mac OS X
-elif [ "$(uname -s)" == Darwin ]; then
-    [ -x "$(command -v scutil)" ] && scutil --get HostName 2>/dev/null || hostname -s
-else
-    command -v hostname && hostname -s
-fi
diff --git a/cdist/conf/type/__hostname/explorer/hostname_sysconfig b/cdist/conf/type/__hostname/explorer/hostname_sysconfig
index d0d7b4e7..4d6b8513 100755
--- a/cdist/conf/type/__hostname/explorer/hostname_sysconfig
+++ b/cdist/conf/type/__hostname/explorer/hostname_sysconfig
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Retrieve the contents of /etc/hostname
+# Retrieve the contents of /etc/sysconfig/network
 #
 
 if [ -f /etc/sysconfig/network ]; then

From ee440ec61996b53889dff3327d1fd4c4fa0f2a57 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 3 Oct 2019 21:41:51 +0200
Subject: [PATCH 006/743] [__hostname] Compress state check

---
 cdist/conf/type/__hostname/gencode-remote | 29 +++++++++--------------
 1 file changed, 11 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index ee12f822..c6779a22 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -33,25 +33,18 @@ name_sysconfig=$(cat "$__object/explorer/hostname_sysconfig")
 has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
 
 ################################################################################
-# If everything is ok -> exit
+# Check if the hostname is already correct
 #
-case "$os" in
-    archlinux|debian|suse|ubuntu|devuan|coreos|alpine)
-        if [ "$name_config" = "$name_should" ] && [ "$name_running" = "$name_should" ]; then
-            exit 0
-        fi
-    ;;
-    scientific|centos|freebsd|openbsd)
-        if [ "$name_sysconfig" = "$name_should" ] && [ "$name_running" = "$name_should" ]; then
-            exit 0
-        fi
-    ;;
-    *)
-        if [ "$name_running" -a "$name_running" = "$name_should" ]; then
-            exit 0
-        fi
-    ;;
-esac
+if [ "$name_running" = "$name_should" ]; then
+    case "$os" in
+        archlinux|debian|suse|ubuntu|devuan|coreos|alpine)
+            [ "$name_config" != "$name_should" ] || exit 0
+        ;;
+        scientific|centos|freebsd|openbsd)
+            [ "$name_sysconfig" != "$name_should" ] || exit 0
+        ;;
+    esac
+fi
 
 ################################################################################
 # Setup hostname

From f5342e9a35c6660004392f400c0843b26e65562a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 4 Oct 2019 20:03:19 +0200
Subject: [PATCH 007/743] [explorer/hostname] Remove code reading persistent
 hostname

---
 cdist/conf/explorer/hostname | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/cdist/conf/explorer/hostname b/cdist/conf/explorer/hostname
index 13ac91c2..a16afb65 100755
--- a/cdist/conf/explorer/hostname
+++ b/cdist/conf/explorer/hostname
@@ -21,19 +21,7 @@
 # Retrieve the running hostname
 #
 
-# Firstly, check hostnamectl if available.
-if command -v hostnamectl >/dev/null
-then
-	hostnamectl status --static && exit 0
-fi
-
-# In case hostnamectl is missing or failed:
-# We fall back to alternative approaches
-if [ "$(uname -s)" = Darwin ] && command -v scutil >/dev/null
-then
-	# Mac OS X
-	scutil --get HostName 2>/dev/null
-elif command -v hostname >/dev/null
+if command -v hostname >/dev/null
 then
 	hostname -s
 else

From 56c654808d84630e82f60ae0376cf3e5b54274c2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 4 Oct 2019 20:35:14 +0200
Subject: [PATCH 008/743] [__hostname] Unify persistent hostname explorers

---
 .../type/__hostname/explorer/hostname_file    | 30 -----------
 .../__hostname/explorer/hostname_sysconfig    | 26 ----------
 .../__hostname/explorer/persistent_hostname   | 52 +++++++++++++++++++
 cdist/conf/type/__hostname/gencode-remote     | 14 ++---
 4 files changed, 55 insertions(+), 67 deletions(-)
 delete mode 100755 cdist/conf/type/__hostname/explorer/hostname_file
 delete mode 100755 cdist/conf/type/__hostname/explorer/hostname_sysconfig
 create mode 100755 cdist/conf/type/__hostname/explorer/persistent_hostname

diff --git a/cdist/conf/type/__hostname/explorer/hostname_file b/cdist/conf/type/__hostname/explorer/hostname_file
deleted file mode 100755
index 6a00aa9f..00000000
--- a/cdist/conf/type/__hostname/explorer/hostname_file
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/sh
-#
-# 2014 Nico Schottelius (nico-cdist at schottelius.org)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Retrieve the contents of /etc/hostname
-#
-
-# Almost any distribution
-if [ -f /etc/hostname ]; then
-    cat /etc/hostname
-# SuSE
-elif [ -f /etc/HOSTNAME ]; then
-    cat /etc/HOSTNAME
-fi
diff --git a/cdist/conf/type/__hostname/explorer/hostname_sysconfig b/cdist/conf/type/__hostname/explorer/hostname_sysconfig
deleted file mode 100755
index 4d6b8513..00000000
--- a/cdist/conf/type/__hostname/explorer/hostname_sysconfig
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/sh
-#
-# 2014 Nico Schottelius (nico-cdist at schottelius.org)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Retrieve the contents of /etc/sysconfig/network
-#
-
-if [ -f /etc/sysconfig/network ]; then
-    awk -F= '/^HOSTNAME=/ { print $2 }' /etc/sysconfig/network
-fi
diff --git a/cdist/conf/type/__hostname/explorer/persistent_hostname b/cdist/conf/type/__hostname/explorer/persistent_hostname
new file mode 100755
index 00000000..9ec5639b
--- /dev/null
+++ b/cdist/conf/type/__hostname/explorer/persistent_hostname
@@ -0,0 +1,52 @@
+#!/bin/sh
+#
+# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Retrieve the persistent hostname
+#
+
+# Firstly, check hostnamectl if available.
+if command -v hostnamectl >/dev/null
+then
+	hostnamectl status --static && exit 0
+fi
+
+# In case hostnamectl is missing or failed:
+# Fall back to alternative OS-specific approaches.
+case $("$__explorer/os")
+in
+	alpine|archlinux|coreos|debian|devuan|suse|ubuntu)
+		[ -f /etc/hostname ] && cat /etc/hostname
+		;;
+	suse)
+		[ -f /etc/HOSTNAME ] && cat /etc/HOSTNAME
+		;;
+	scientific|centos)
+		[ -f /etc/sysconfig/network ] && awk -F= '/^HOSTNAME=/ { print $2 }' /etc/sysconfig/network
+		;;
+	freebsd)
+		(. /etc/rc.conf && echo "$hostname")
+		;;
+	openbsd)
+		cat /etc/myname
+		;;
+	macosx)
+		scutil --get HostName 2>/dev/null
+		;;
+esac
diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index c6779a22..e7f32323 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -28,22 +28,14 @@ fi
 
 os=$(cat "$__global/explorer/os")
 name_running=$(cat "$__global/explorer/hostname")
-name_config=$(cat "$__object/explorer/hostname_file")
-name_sysconfig=$(cat "$__object/explorer/hostname_sysconfig")
+name_config=$(cat "$__object/explorer/persistent_hostname")
 has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
 
 ################################################################################
 # Check if the hostname is already correct
 #
-if [ "$name_running" = "$name_should" ]; then
-    case "$os" in
-        archlinux|debian|suse|ubuntu|devuan|coreos|alpine)
-            [ "$name_config" != "$name_should" ] || exit 0
-        ;;
-        scientific|centos|freebsd|openbsd)
-            [ "$name_sysconfig" != "$name_should" ] || exit 0
-        ;;
-    esac
+if [ "$name_running" = "$name_should" ] && [ ! "$name_config" -o "$name_config" = "$name_should" ]; then
+    exit 0
 fi
 
 ################################################################################

From 1bab641c948174d82653736bfbbd39b09aa61801 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 5 Oct 2019 18:43:06 +0200
Subject: [PATCH 009/743] [explorer/hostname] Do not shorten hostname in any
 case

---
 cdist/conf/explorer/hostname | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/hostname b/cdist/conf/explorer/hostname
index a16afb65..dca004d1 100755
--- a/cdist/conf/explorer/hostname
+++ b/cdist/conf/explorer/hostname
@@ -23,7 +23,7 @@
 
 if command -v hostname >/dev/null
 then
-	hostname -s
+	hostname
 else
 	uname -n
 fi

From d43eb5b22f11696d6992cbb6375806f7a8af133a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 5 Oct 2019 20:10:52 +0200
Subject: [PATCH 010/743] [__hostname] Add support for NetBSD

---
 .../conf/type/__hostname/explorer/persistent_hostname  | 10 ++++++++++
 cdist/conf/type/__hostname/manifest                    |  9 +++++++++
 2 files changed, 19 insertions(+)

diff --git a/cdist/conf/type/__hostname/explorer/persistent_hostname b/cdist/conf/type/__hostname/explorer/persistent_hostname
index 9ec5639b..8909ebaa 100755
--- a/cdist/conf/type/__hostname/explorer/persistent_hostname
+++ b/cdist/conf/type/__hostname/explorer/persistent_hostname
@@ -43,6 +43,16 @@ in
 	freebsd)
 		(. /etc/rc.conf && echo "$hostname")
 		;;
+	netbsd)
+		if grep -q '^hostname=' /etc/rc.conf
+		then
+			(. /etc/rc.conf && echo "$hostname")
+		elif [ -f /etc/myname ]
+		then
+			# Fall back to /etc/myname file
+			cat /etc/myname
+		fi
+		;;
 	openbsd)
 		cat /etc/myname
 		;;
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index fe067718..b0948d24 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -60,6 +60,15 @@ case "$os" in
             --key 'hostname' \
             --value "$name_should"
     ;;
+    netbsd)
+        __key_value rcconf-hostname \
+            --file /etc/rc.conf \
+            --delimiter '=' \
+            --key 'hostname' \
+            --value "\"$name_should\""
+        # To avoid confusion, ensure that the hostname is only stored once
+        __file /etc/myname --state absent
+    ;;
     openbsd)
         echo "$name_should" | __file /etc/myname --source -
     ;;

From 77210c349e70bd1f54394f1873c768ddb62d7ea9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 6 Oct 2019 17:56:37 +0200
Subject: [PATCH 011/743] [__hostname] Extend os switches with
 RedHat-derivatives (CentOS, Fedora, RedHat, Scientific)

---
 cdist/conf/type/__hostname/explorer/persistent_hostname | 2 +-
 cdist/conf/type/__hostname/gencode-remote               | 6 +++---
 cdist/conf/type/__hostname/manifest                     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__hostname/explorer/persistent_hostname b/cdist/conf/type/__hostname/explorer/persistent_hostname
index 8909ebaa..ce71d06a 100755
--- a/cdist/conf/type/__hostname/explorer/persistent_hostname
+++ b/cdist/conf/type/__hostname/explorer/persistent_hostname
@@ -37,7 +37,7 @@ in
 	suse)
 		[ -f /etc/HOSTNAME ] && cat /etc/HOSTNAME
 		;;
-	scientific|centos)
+	centos|fedora|redhat|scientific)
 		[ -f /etc/sysconfig/network ] && awk -F= '/^HOSTNAME=/ { print $2 }' /etc/sysconfig/network
 		;;
 	freebsd)
diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index e7f32323..e3f1bee4 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -58,12 +58,12 @@ EOF
 fi
 
 # Use the good old way to set the hostname. Also if hostnamectl fails.
-case "$os" in
-    archlinux|debian|ubuntu|devuan|centos|coreos|alpine)
+case $os in
+    archlinux|debian|ubuntu|devuan|coreos|alpine)
         printf "printf '%%s\\\\n' '$name_should' > /etc/hostname\\n"
         echo "hostname -F /etc/hostname"
     ;;
-    freebsd|openbsd)
+    centos|fedora|redhat|scientific|freebsd|openbsd)
         # NOTE: Writing the hostname to file is done in the manifest.
         echo "hostname '$name_should'"
     ;;
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index b0948d24..ccd11c4e 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -46,7 +46,7 @@ case "$os" in
         # handled in gencode-remote
         :
     ;;
-    scientific|centos)
+    centos|fedora|redhat|scientific)
         __key_value sysconfig-hostname \
             --file /etc/sysconfig/network \
             --delimiter '=' \

From 7bf203509bdaf42c88fbc67fbdd8dd1a15a6e21d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 6 Oct 2019 18:01:58 +0200
Subject: [PATCH 012/743] [__hostname] Use __key_value exact_delimiter for
 shell-like files

---
 cdist/conf/type/__hostname/manifest | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index ccd11c4e..f1c17653 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -49,21 +49,21 @@ case "$os" in
     centos|fedora|redhat|scientific)
         __key_value sysconfig-hostname \
             --file /etc/sysconfig/network \
-            --delimiter '=' \
+            --delimiter '=' --exact_delimiter \
             --key HOSTNAME \
-            --value "$name_should" --exact_delimiter
+            --value "\"$name_should\""
     ;;
     freebsd)
         __key_value rcconf-hostname \
             --file /etc/rc.conf \
-            --delimiter '=' \
+            --delimiter '=' --exact_delimiter \
             --key 'hostname' \
             --value "$name_should"
     ;;
     netbsd)
         __key_value rcconf-hostname \
             --file /etc/rc.conf \
-            --delimiter '=' \
+            --delimiter '=' --exact_delimiter \
             --key 'hostname' \
             --value "\"$name_should\""
         # To avoid confusion, ensure that the hostname is only stored once

From 74568e8a0aa0f1e8222aff49c9b00f667694b0c3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 6 Oct 2019 18:02:24 +0200
Subject: [PATCH 013/743] [__hostname] Use FQDN as hostname on RedHat
 derivatives and BSDs

---
 cdist/conf/type/__hostname/manifest | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index f1c17653..5d9eeeeb 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -25,10 +25,13 @@ if [ -f "$__object/parameter/name" ]; then
     name_should="$(cat "$__object/parameter/name")"
 else
     case "$os" in
-    openbsd)
+    # RedHat-derivatives and BSDs
+    centos|fedora|redhat|scientific|freebsd|netbsd|openbsd)
+        # Hostname is FQDN
         name_should="${__target_host}"
     ;;
     *)
+        # Hostname is only first component of FQDN
         name_should="${__target_host%%.*}"
     ;;
     esac

From 3e7cf68de590c755c74bd3542166a698b86b27d4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 6 Oct 2019 18:14:35 +0200
Subject: [PATCH 014/743] [__hostname] Source sysconfig file instead of
 processing through AWK

This way quotes and expansions will be handled correctly.
---
 cdist/conf/type/__hostname/explorer/persistent_hostname | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__hostname/explorer/persistent_hostname b/cdist/conf/type/__hostname/explorer/persistent_hostname
index ce71d06a..6bc0786f 100755
--- a/cdist/conf/type/__hostname/explorer/persistent_hostname
+++ b/cdist/conf/type/__hostname/explorer/persistent_hostname
@@ -38,7 +38,8 @@ in
 		[ -f /etc/HOSTNAME ] && cat /etc/HOSTNAME
 		;;
 	centos|fedora|redhat|scientific)
-		[ -f /etc/sysconfig/network ] && awk -F= '/^HOSTNAME=/ { print $2 }' /etc/sysconfig/network
+		[ -f /etc/sysconfig/network ] \
+			&& (. /etc/sysconfig/network && echo "$HOSTNAME")
 		;;
 	freebsd)
 		(. /etc/rc.conf && echo "$hostname")

From cbb108d61c20d2699946f819aff77de99b7bb94f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 6 Oct 2019 18:23:56 +0200
Subject: [PATCH 015/743] [__hostname] Copy FQDN-logic to gencode-remote

Otherwise, the code-remote gets run on the remote every time.
---
 cdist/conf/type/__hostname/gencode-remote | 23 +++++++++++++++++------
 cdist/conf/type/__hostname/manifest       | 20 ++++++++++----------
 2 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index e3f1bee4..973d1d97 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -20,17 +20,28 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-if [ -f "$__object/parameter/name" ]; then
-    name_should="$(cat "$__object/parameter/name")"
-else
-    name_should="${__target_host%%.*}"
-fi
-
 os=$(cat "$__global/explorer/os")
 name_running=$(cat "$__global/explorer/hostname")
 name_config=$(cat "$__object/explorer/persistent_hostname")
 has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
 
+
+if [ -f "$__object/parameter/name" ]; then
+    name_should="$(cat "$__object/parameter/name")"
+else
+    case $os in
+        # RedHat-derivatives and BSDs
+        centos|fedora|redhat|scientific|freebsd|netbsd|openbsd)
+            # Hostname is FQDN
+            name_should="${__target_host}"
+        ;;
+        *)
+            # Hostname is only first component of FQDN
+            name_should="${__target_host%%.*}"
+        ;;
+    esac
+fi
+
 ################################################################################
 # Check if the hostname is already correct
 #
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 5d9eeeeb..699f3e93 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -24,16 +24,16 @@ os=$(cat "$__global/explorer/os")
 if [ -f "$__object/parameter/name" ]; then
     name_should="$(cat "$__object/parameter/name")"
 else
-    case "$os" in
-    # RedHat-derivatives and BSDs
-    centos|fedora|redhat|scientific|freebsd|netbsd|openbsd)
-        # Hostname is FQDN
-        name_should="${__target_host}"
-    ;;
-    *)
-        # Hostname is only first component of FQDN
-        name_should="${__target_host%%.*}"
-    ;;
+    case $os in
+        # RedHat-derivatives and BSDs
+        centos|fedora|redhat|scientific|freebsd|netbsd|openbsd)
+            # Hostname is FQDN
+            name_should="${__target_host}"
+        ;;
+        *)
+            # Hostname is only first component of FQDN
+            name_should="${__target_host%%.*}"
+        ;;
     esac
 fi
 

From 6e1a105c22021e0f98e63930eeade65b416aaa2d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 6 Oct 2019 18:36:43 +0200
Subject: [PATCH 016/743] [__hostname] Lint

---
 cdist/conf/type/__hostname/gencode-remote | 12 ++++++++----
 cdist/conf/type/__hostname/manifest       |  6 ++++--
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 973d1d97..b258165d 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -29,7 +29,8 @@ has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
 if [ -f "$__object/parameter/name" ]; then
     name_should="$(cat "$__object/parameter/name")"
 else
-    case $os in
+    case $os
+    in
         # RedHat-derivatives and BSDs
         centos|fedora|redhat|scientific|freebsd|netbsd|openbsd)
             # Hostname is FQDN
@@ -45,14 +46,16 @@ fi
 ################################################################################
 # Check if the hostname is already correct
 #
-if [ "$name_running" = "$name_should" ] && [ ! "$name_config" -o "$name_config" = "$name_should" ]; then
+if [ "$name_running" = "$name_should" ] \
+  && [ -z "$name_config" -o "$name_config" = "$name_should" ]
+then
     exit 0
 fi
 
 ################################################################################
 # Setup hostname
 #
-echo changed >> "$__messages_out"
+echo 'changed' >> "$__messages_out"
 
 
 # First try to set the hostname using hostnamectl, if available.
@@ -69,7 +72,8 @@ EOF
 fi
 
 # Use the good old way to set the hostname. Also if hostnamectl fails.
-case $os in
+case $os
+in
     archlinux|debian|ubuntu|devuan|coreos|alpine)
         printf "printf '%%s\\\\n' '$name_should' > /etc/hostname\\n"
         echo "hostname -F /etc/hostname"
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 699f3e93..18152c3e 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -24,7 +24,8 @@ os=$(cat "$__global/explorer/os")
 if [ -f "$__object/parameter/name" ]; then
     name_should="$(cat "$__object/parameter/name")"
 else
-    case $os in
+    case $os
+    in
         # RedHat-derivatives and BSDs
         centos|fedora|redhat|scientific|freebsd|netbsd|openbsd)
             # Hostname is FQDN
@@ -44,7 +45,8 @@ not_supported() {
    exit 1
 }
 
-case "$os" in
+case $os
+in
     archlinux|debian|suse|ubuntu|devuan|coreos|alpine|macosx)
         # handled in gencode-remote
         :

From da1d70e16ac56511fb03f30bf9c76645c01caab8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 6 Oct 2019 18:43:27 +0200
Subject: [PATCH 017/743] [__hostname] Also quote hostname in FreeBSD rc.conf

---
 cdist/conf/type/__hostname/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 18152c3e..53e530d7 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -63,7 +63,7 @@ in
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
-            --value "$name_should"
+            --value "\"$name_should\""
     ;;
     netbsd)
         __key_value rcconf-hostname \

From 064e992a7aec8916a2bc65e5279f65bab4464140 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 6 Oct 2019 21:03:49 +0200
Subject: [PATCH 018/743] [__hostname] Support openSUSE and replace printf with
 echo

---
 .../type/__hostname/explorer/persistent_hostname     | 12 ++++++------
 cdist/conf/type/__hostname/gencode-remote            | 12 +++++++-----
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__hostname/explorer/persistent_hostname b/cdist/conf/type/__hostname/explorer/persistent_hostname
index 6bc0786f..ec52bbd2 100755
--- a/cdist/conf/type/__hostname/explorer/persistent_hostname
+++ b/cdist/conf/type/__hostname/explorer/persistent_hostname
@@ -31,12 +31,9 @@ fi
 # Fall back to alternative OS-specific approaches.
 case $("$__explorer/os")
 in
-	alpine|archlinux|coreos|debian|devuan|suse|ubuntu)
+	alpine|archlinux|coreos|debian|devuan|ubuntu)
 		[ -f /etc/hostname ] && cat /etc/hostname
 		;;
-	suse)
-		[ -f /etc/HOSTNAME ] && cat /etc/HOSTNAME
-		;;
 	centos|fedora|redhat|scientific)
 		[ -f /etc/sysconfig/network ] \
 			&& (. /etc/sysconfig/network && echo "$HOSTNAME")
@@ -44,6 +41,9 @@ in
 	freebsd)
 		(. /etc/rc.conf && echo "$hostname")
 		;;
+	macosx)
+		scutil --get HostName 2>/dev/null
+		;;
 	netbsd)
 		if grep -q '^hostname=' /etc/rc.conf
 		then
@@ -57,7 +57,7 @@ in
 	openbsd)
 		cat /etc/myname
 		;;
-	macosx)
-		scutil --get HostName 2>/dev/null
+	suse)
+		cat /etc/HOSTNAME 2>/dev/null
 		;;
 esac
diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index b258165d..37ace1b7 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -75,20 +75,22 @@ fi
 case $os
 in
     archlinux|debian|ubuntu|devuan|coreos|alpine)
-        printf "printf '%%s\\\\n' '$name_should' > /etc/hostname\\n"
+        echo "printf '%s\\n' '$name_should' > /etc/hostname"
         echo "hostname -F /etc/hostname"
     ;;
     centos|fedora|redhat|scientific|freebsd|openbsd)
         # NOTE: Writing the hostname to file is done in the manifest.
         echo "hostname '$name_should'"
     ;;
-    suse)
-        echo "hostname '$name_should'"
-        printf "printf '%%s\\\\n' '$name_should' > /etc/HOSTNAME\\n"
-    ;;
     macosx)
         echo "scutil --set HostName '$name_should'"
     ;;
+    suse)
+        # We write into /etc/HOSTNAME for backwards-compatibility.  Modern SuSE
+        # has hostnamectl anyway and symlinks /etc/HOSTNAME to /etc/hostname.
+        echo "printf '%s\\n' '$name_should' > /etc/HOSTNAME"
+        echo 'hostname -F /etc/HOSTNAME'
+    ;;
     *)
         echo "echo 'Unsupported OS: $os' >&2; exit 1"
     ;;

From c11e757dfac865fc54f2bbe7b5ad015d25e813ad Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 6 Oct 2019 23:34:59 +0200
Subject: [PATCH 019/743] [__hostname] Add support for Gentoo

---
 .../conf/type/__hostname/explorer/persistent_hostname  |  3 +++
 cdist/conf/type/__hostname/gencode-remote              | 10 +++++-----
 cdist/conf/type/__hostname/manifest                    |  7 +++++++
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__hostname/explorer/persistent_hostname b/cdist/conf/type/__hostname/explorer/persistent_hostname
index ec52bbd2..141c44fe 100755
--- a/cdist/conf/type/__hostname/explorer/persistent_hostname
+++ b/cdist/conf/type/__hostname/explorer/persistent_hostname
@@ -41,6 +41,9 @@ in
 	freebsd)
 		(. /etc/rc.conf && echo "$hostname")
 		;;
+	gentoo)
+		(. /etc/conf.d/hostname && echo "$hostname")
+		;;
 	macosx)
 		scutil --get HostName 2>/dev/null
 		;;
diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 37ace1b7..64fff7ad 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -74,14 +74,10 @@ fi
 # Use the good old way to set the hostname. Also if hostnamectl fails.
 case $os
 in
-    archlinux|debian|ubuntu|devuan|coreos|alpine)
+    alpine|archlinux|coreos|debian|devuan|ubuntu)
         echo "printf '%s\\n' '$name_should' > /etc/hostname"
         echo "hostname -F /etc/hostname"
     ;;
-    centos|fedora|redhat|scientific|freebsd|openbsd)
-        # NOTE: Writing the hostname to file is done in the manifest.
-        echo "hostname '$name_should'"
-    ;;
     macosx)
         echo "scutil --set HostName '$name_should'"
     ;;
@@ -91,6 +87,10 @@ in
         echo "printf '%s\\n' '$name_should' > /etc/HOSTNAME"
         echo 'hostname -F /etc/HOSTNAME'
     ;;
+    centos|fedora|freebsd|gentoo|netbsd|openbsd|redhat|scientific)
+        # NOTE: Writing the hostname to file is done in the manifest.
+        echo "hostname '$name_should'"
+    ;;
     *)
         echo "echo 'Unsupported OS: $os' >&2; exit 1"
     ;;
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 53e530d7..0ffd7678 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -58,6 +58,13 @@ in
             --key HOSTNAME \
             --value "\"$name_should\""
     ;;
+    gentoo)
+        __key_value confd-hostname \
+            --file /etc/conf.d/hostname \
+            --delimiter '=' --exact_delimiter \
+            --key 'hostname' \
+            --value "\"$name_should\""
+    ;;
     freebsd)
         __key_value rcconf-hostname \
             --file /etc/rc.conf \

From 30c7d153e828afd52d1e76328622deb06aca6aaa Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 12 Oct 2019 18:47:27 +0200
Subject: [PATCH 020/743] [__hostname] Only write to distro specific file when
 hostnamectl is missing

---
 cdist/conf/type/__hostname/gencode-remote |  7 +++++-
 cdist/conf/type/__hostname/manifest       | 30 +++++++++++++++--------
 2 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 64fff7ad..182d37ae 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -78,6 +78,11 @@ in
         echo "printf '%s\\n' '$name_should' > /etc/hostname"
         echo "hostname -F /etc/hostname"
     ;;
+    gentoo)
+        # NOTE: Writing the hostname to file is done in the manifest for OpenRC.
+        #       For systemd hostnamectl should take care of that.
+        echo "hostname '$name_should'"
+    ;;
     macosx)
         echo "scutil --set HostName '$name_should'"
     ;;
@@ -87,7 +92,7 @@ in
         echo "printf '%s\\n' '$name_should' > /etc/HOSTNAME"
         echo 'hostname -F /etc/HOSTNAME'
     ;;
-    centos|fedora|freebsd|gentoo|netbsd|openbsd|redhat|scientific)
+    centos|fedora|freebsd|netbsd|openbsd|redhat|scientific)
         # NOTE: Writing the hostname to file is done in the manifest.
         echo "hostname '$name_should'"
     ;;
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 0ffd7678..58034a19 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -52,18 +52,28 @@ in
         :
     ;;
     centos|fedora|redhat|scientific)
-        __key_value sysconfig-hostname \
-            --file /etc/sysconfig/network \
-            --delimiter '=' --exact_delimiter \
-            --key HOSTNAME \
-            --value "\"$name_should\""
+        if test -z "$(cat "$__object/explorer/has_hostnamectl")"
+        then
+            # Only write to /etc/sysconfig/network on non-systemd versions.
+            # On systemd-based versions this entry is ignored.
+            __key_value sysconfig-hostname \
+                --file /etc/sysconfig/network \
+                --delimiter '=' --exact_delimiter \
+                --key HOSTNAME \
+                --value "\"$name_should\""
+        fi
     ;;
     gentoo)
-        __key_value confd-hostname \
-            --file /etc/conf.d/hostname \
-            --delimiter '=' --exact_delimiter \
-            --key 'hostname' \
-            --value "\"$name_should\""
+        if test -z "$(cat "$__object/explorer/has_hostnamectl")"
+        then
+            # Only write to /etc/conf.d/hostname on OpenRC-based installations.
+            # On systemd use hostnamectl(1) in gencode-remote.
+            __key_value confd-hostname \
+                --file /etc/conf.d/hostname \
+                --delimiter '=' --exact_delimiter \
+                --key 'hostname' \
+                --value "\"$name_should\""
+        fi
     ;;
     freebsd)
         __key_value rcconf-hostname \

From cee553a6dd27e2099e60763dec8bb9d061ef3aad Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 12 Oct 2019 18:49:31 +0200
Subject: [PATCH 021/743] [__hostname] Improve systemd hostname setting

This means we write to /etc/hostname before running hostnamectl, so that
it can overwrite the file if it wants to.
---
 cdist/conf/type/__hostname/gencode-remote | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 182d37ae..36593ef2 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -61,23 +61,29 @@ echo 'changed' >> "$__messages_out"
 # First try to set the hostname using hostnamectl, if available.
 if [ "$has_hostnamectl" ]; then
     # Allow hostnamectl to fail silently.
-    # Who the fuck invented a tool that needs dbus to set the hostname anyway ...
+    # Who the fuck invented a tool that needs dbus to set the hostname anyway…
 
-    cat <<EOF
-if hostnamectl set-hostname '$name_should'; then
-    printf '%s\\n' '$name_should' > /etc/hostname
-    exit 0
-fi
-EOF
+    # NOTE: We write the static hostname to /etc/hostname first in case
+    #       hostnamectl fails. Fallback-code below will then adjust the running
+    #       hostname below.
+    echo "printf '%s\\n' '$name_should' > /etc/hostname"
+
+    echo "hostnamectl set-hostname '$name_should' && exit 0"
 fi
 
 # Use the good old way to set the hostname. Also if hostnamectl fails.
 case $os
 in
-    alpine|archlinux|coreos|debian|devuan|ubuntu)
+    alpine|archlinux|debian|devuan|ubuntu)
         echo "printf '%s\\n' '$name_should' > /etc/hostname"
         echo "hostname -F /etc/hostname"
     ;;
+    coreos|openbmc-phosphor)
+        # NOTE: systemd-only distros have the hostname already written above.
+        #       But since hostamectl failed, we update the running hostname
+        #       manually.
+        echo "hostname -F /etc/hostname"
+        ;;
     gentoo)
         # NOTE: Writing the hostname to file is done in the manifest for OpenRC.
         #       For systemd hostnamectl should take care of that.

From 689b5a299265ae4626348073afe409336893a175 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 17 Oct 2019 16:44:26 +0200
Subject: [PATCH 022/743] [letsencrypt_cert] add support for alpine

---
 cdist/conf/type/__letsencrypt_cert/manifest | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 35962d31..4fa5f119 100755
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -8,6 +8,9 @@ if [ -z "${certbot_fullpath}" ]; then
 
 	case "$os" in
         archlinux)
+            __package certbot
+            ;;
+        alpine)
             __package certbot
             ;;
 		debian)

From 076133028f181d2fe0811dc9d915cfba36d07421 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 17 Oct 2019 16:44:56 +0200
Subject: [PATCH 023/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index f1b37065..10aeafc9 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* New types: __xymon_server, __xymon_apache, __xymon_config, __xymon_client (Thomas Eckert)
 	* Type __letsencrypt_cert: Add Arch Linux support (Nico Schottelius)
+	* Type __letsencrypt_cert: Add Alpine support (Nico Schottelius)
 
 6.0.1: 2019-10-08
 	* Type __group: Support OSes without getent (Dennis Camera)

From f6a45808f9796630aa415209d711424d570e145e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 18 Oct 2019 13:11:59 +0200
Subject: [PATCH 024/743] Fix building man pages command

---
 docs/src/cdist-upgrade.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/cdist-upgrade.rst b/docs/src/cdist-upgrade.rst
index e57ed63c..67fd4934 100644
--- a/docs/src/cdist-upgrade.rst
+++ b/docs/src/cdist-upgrade.rst
@@ -11,7 +11,7 @@ To upgrade cdist in the current branch use
     git pull
 
     # Also update the manpages
-    ./build man
+    make man
     export MANPATH=$MANPATH:$(pwd -P)/doc/man
 
 If you stay on a version branche (i.e. 1.0, 1.1., ...), nothing should break.

From ecc77e55ed7f7c76a9f2f1b6af6e5d7fd44bef99 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 18 Oct 2019 19:45:19 +0200
Subject: [PATCH 025/743] Add timing hack / test1

---
 hacking/timing-tests/benchmark-files.sh | 79 +++++++++++++++++++++++++
 1 file changed, 79 insertions(+)
 create mode 100644 hacking/timing-tests/benchmark-files.sh

diff --git a/hacking/timing-tests/benchmark-files.sh b/hacking/timing-tests/benchmark-files.sh
new file mode 100644
index 00000000..c71d1c7e
--- /dev/null
+++ b/hacking/timing-tests/benchmark-files.sh
@@ -0,0 +1,79 @@
+#!/bin/sh
+
+num=50000
+dsthost=localhost
+
+tmp=$(mktemp -d)
+remote_tmp=${tmp}-remote
+
+cd "$tmp"
+
+create_files() {
+    i=0
+    while [ $i -lt $num ]; do
+        echo $i > file-${i}
+        i=$((i+1))
+    done
+}
+
+delete_remote() {
+    ssh "${dsthost}" "rm -rf ${remote_tmp}"
+}
+
+
+tar_remote() {
+    cd "${tmp}"
+    tar c . | ssh "${dsthost}" "mkdir ${remote_tmp}; cd ${remote_tmp}; tar x"
+}
+
+cdist_remote()
+{
+    (
+        while [ $i -lt $num ]; do
+            echo __file ${remote_tmp}/file-${i} --source "${tmp}/file-${i}"
+            i=$((i+1))
+        done
+    ) | cdist config -i - -vv "${dsthost}"
+
+}
+
+cdist_remote_parallel()
+{
+    (
+        while [ $i -lt $num ]; do
+            echo __file ${remote_tmp}/file-${i} --source "${tmp}/file-${i}"
+            i=$((i+1))
+        done
+    ) | cdist config -j10 -i - -vv "${dsthost}"
+
+}
+
+echo "Creating ${num} files"
+time create_files
+
+echo "scping files"
+time scp -r "${tmp}" "${dsthost}:$remote_tmp" >/dev/null
+
+echo "delete remote"
+time delete_remote
+
+echo "taring files"
+time tar_remote
+
+echo "delete remote"
+time delete_remote
+
+echo "cdisting files"
+time cdist_remote
+
+echo "delete remote"
+time delete_remote
+
+echo "cdisting files (parallel)!"
+time cdist_remote
+
+echo "delete remote"
+time delete_remote
+
+echo "delete local"
+rm -rf "$tmp"

From 32c15f2ecbc9a4cf29258f9d8e28ef537bcc2ac1 Mon Sep 17 00:00:00 2001
From: Dmitry Bogatov <KAction@disroot.org>
Date: Sun, 20 Oct 2019 18:59:48 +0000
Subject: [PATCH 026/743] Fix spelling error in manpage

---
 cdist/conf/type/__xymon_client/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__xymon_client/man.rst b/cdist/conf/type/__xymon_client/man.rst
index 6f90c15b..6660b0ef 100644
--- a/cdist/conf/type/__xymon_client/man.rst
+++ b/cdist/conf/type/__xymon_client/man.rst
@@ -23,7 +23,7 @@ state
    'present', 'absent', defaults to 'present'.
 
 servers
-   One or more IP adresses (space separated) of the Xymon server(s) to report
+   One or more IP addresses (space separated) of the Xymon server(s) to report
    to. While DNS-names are ok it is discouraged, defaults to 127.0.0.1.
 
 

From 50a3130b0ae280526ac13fd8a3619717d29c57ac Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 21 Oct 2019 12:02:53 +0200
Subject: [PATCH 027/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 176c5d40..5020c3c2 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __letsencrypt_cert: Add Alpine support (Nico Schottelius)
+	* Type __xymon_client: Fix spelling error in manpage (Dmitry Bogatov)
 
 6.0.2: 2019-10-17
 	* New types: __xymon_server, __xymon_apache, __xymon_config, __xymon_client (Thomas Eckert)

From c3b066c14733fd62969abd7a36e3ac881ed71227 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 24 Oct 2019 19:44:51 +0200
Subject: [PATCH 028/743] [__hostname] Check hostname length if getconf(1) is
 available

---
 cdist/conf/type/__hostname/explorer/max_len | 10 ++++++++++
 cdist/conf/type/__hostname/manifest         | 18 ++++++++++++------
 2 files changed, 22 insertions(+), 6 deletions(-)
 create mode 100644 cdist/conf/type/__hostname/explorer/max_len

diff --git a/cdist/conf/type/__hostname/explorer/max_len b/cdist/conf/type/__hostname/explorer/max_len
new file mode 100644
index 00000000..97d8a142
--- /dev/null
+++ b/cdist/conf/type/__hostname/explorer/max_len
@@ -0,0 +1,10 @@
+#!/bin/sh -e
+
+command -v getconf >/dev/null || exit 0
+
+val=$(getconf HOST_NAME_MAX 2>/dev/null)
+
+if test -n "${val}" -a "${val}" != 'undefined'
+then
+	echo "${val}"
+fi
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 58034a19..2c6057d6 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -20,6 +20,13 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+not_supported() {
+   echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
+   echo "Please contribute an implementation for it if you can." >&2
+   exit 1
+}
+
+
 os=$(cat "$__global/explorer/os")
 if [ -f "$__object/parameter/name" ]; then
     name_should="$(cat "$__object/parameter/name")"
@@ -38,12 +45,11 @@ else
     esac
 fi
 
-
-not_supported() {
-   echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
-   echo "Please contribute an implementation for it if you can." >&2
-   exit 1
-}
+if test -n "${max_len}" -a "$(wc -c "${name_should}")" -gt "${max_len}"a
+then
+	printf "Host name too long. Up to %u characters allowed.\n" "${max_len}" >&2
+	exit 1
+fi
 
 case $os
 in

From 629d443f5aa61eaf8d41c066011108e2b3b9caa1 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 25 Oct 2019 01:40:48 +0200
Subject: [PATCH 029/743] [__hostname] Fix "command not found" errors with old
 Bash

---
 cdist/conf/type/__hostname/explorer/has_hostnamectl     | 2 +-
 cdist/conf/type/__hostname/explorer/max_len             | 2 +-
 cdist/conf/type/__hostname/explorer/persistent_hostname | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__hostname/explorer/has_hostnamectl b/cdist/conf/type/__hostname/explorer/has_hostnamectl
index 9040023d..2f531f30 100755
--- a/cdist/conf/type/__hostname/explorer/has_hostnamectl
+++ b/cdist/conf/type/__hostname/explorer/has_hostnamectl
@@ -21,4 +21,4 @@
 # Check whether system has hostnamectl
 #
 
-command -v hostnamectl || true
+command -v hostnamectl 2>/dev/null || true
diff --git a/cdist/conf/type/__hostname/explorer/max_len b/cdist/conf/type/__hostname/explorer/max_len
index 97d8a142..fb863949 100644
--- a/cdist/conf/type/__hostname/explorer/max_len
+++ b/cdist/conf/type/__hostname/explorer/max_len
@@ -2,7 +2,7 @@
 
 command -v getconf >/dev/null || exit 0
 
-val=$(getconf HOST_NAME_MAX 2>/dev/null)
+val=$(getconf HOST_NAME_MAX 2>/dev/null) || exit 0
 
 if test -n "${val}" -a "${val}" != 'undefined'
 then
diff --git a/cdist/conf/type/__hostname/explorer/persistent_hostname b/cdist/conf/type/__hostname/explorer/persistent_hostname
index 141c44fe..0feb0809 100755
--- a/cdist/conf/type/__hostname/explorer/persistent_hostname
+++ b/cdist/conf/type/__hostname/explorer/persistent_hostname
@@ -22,7 +22,7 @@
 #
 
 # Firstly, check hostnamectl if available.
-if command -v hostnamectl >/dev/null
+if command -v hostnamectl >/dev/null 2>&1
 then
 	hostnamectl status --static && exit 0
 fi
@@ -32,10 +32,10 @@ fi
 case $("$__explorer/os")
 in
 	alpine|archlinux|coreos|debian|devuan|ubuntu)
-		[ -f /etc/hostname ] && cat /etc/hostname
+		test -f /etc/hostname && cat /etc/hostname
 		;;
 	centos|fedora|redhat|scientific)
-		[ -f /etc/sysconfig/network ] \
+		test -f /etc/sysconfig/network \
 			&& (. /etc/sysconfig/network && echo "$HOSTNAME")
 		;;
 	freebsd)

From d4313c7501b39c09e01cee530ec36a7452bcfc53 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 25 Oct 2019 01:41:26 +0200
Subject: [PATCH 030/743] [__hostname] Fix checking of maximum hostname length
 against sysconf(3)

---
 cdist/conf/type/__hostname/manifest | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 2c6057d6..f3739f6c 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -28,6 +28,8 @@ not_supported() {
 
 
 os=$(cat "$__global/explorer/os")
+max_len=$(cat "$__object/explorer/max_len")
+
 if [ -f "$__object/parameter/name" ]; then
     name_should="$(cat "$__object/parameter/name")"
 else
@@ -45,7 +47,7 @@ else
     esac
 fi
 
-if test -n "${max_len}" -a "$(wc -c "${name_should}")" -gt "${max_len}"a
+if test -n "${max_len}" -a "$(printf "${name_should}" | wc -c)" -gt "${max_len}"
 then
 	printf "Host name too long. Up to %u characters allowed.\n" "${max_len}" >&2
 	exit 1

From b65ceba56936b581870754afa879284b686e057b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 25 Oct 2019 02:16:43 +0200
Subject: [PATCH 031/743] [__hostame] Resolve shellcheck errors

---
 cdist/conf/type/__hostname/gencode-remote | 10 +++++-----
 cdist/conf/type/__hostname/manifest       |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 36593ef2..5942b2e3 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -55,18 +55,18 @@ fi
 ################################################################################
 # Setup hostname
 #
-echo 'changed' >> "$__messages_out"
+echo 'changed' >>"$__messages_out"
 
 
 # First try to set the hostname using hostnamectl, if available.
-if [ "$has_hostnamectl" ]; then
+if [ -n "$has_hostnamectl" ]; then
     # Allow hostnamectl to fail silently.
     # Who the fuck invented a tool that needs dbus to set the hostname anyway…
 
     # NOTE: We write the static hostname to /etc/hostname first in case
     #       hostnamectl fails. Fallback-code below will then adjust the running
     #       hostname below.
-    echo "printf '%s\\n' '$name_should' > /etc/hostname"
+    echo "echo '$name_should' >/etc/hostname"
 
     echo "hostnamectl set-hostname '$name_should' && exit 0"
 fi
@@ -75,7 +75,7 @@ fi
 case $os
 in
     alpine|archlinux|debian|devuan|ubuntu)
-        echo "printf '%s\\n' '$name_should' > /etc/hostname"
+        echo "echo '$name_should' >/etc/hostname"
         echo "hostname -F /etc/hostname"
     ;;
     coreos|openbmc-phosphor)
@@ -95,7 +95,7 @@ in
     suse)
         # We write into /etc/HOSTNAME for backwards-compatibility.  Modern SuSE
         # has hostnamectl anyway and symlinks /etc/HOSTNAME to /etc/hostname.
-        echo "printf '%s\\n' '$name_should' > /etc/HOSTNAME"
+        echo "echo '$name_should' >/etc/HOSTNAME"
         echo 'hostname -F /etc/HOSTNAME'
     ;;
     centos|fedora|freebsd|netbsd|openbsd|redhat|scientific)
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index f3739f6c..a4068cc2 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -47,7 +47,7 @@ else
     esac
 fi
 
-if test -n "${max_len}" -a "$(printf "${name_should}" | wc -c)" -gt "${max_len}"
+if test -n "${max_len}" -a "$(printf '%s' "${name_should}" | wc -c)" -gt "${max_len}"
 then
 	printf "Host name too long. Up to %u characters allowed.\n" "${max_len}" >&2
 	exit 1

From 00852cb17d375da39ae585ec337da18a4f63587d Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 25 Oct 2019 14:26:08 +0200
Subject: [PATCH 032/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 5020c3c2..cdc4e8f5 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __letsencrypt_cert: Add Alpine support (Nico Schottelius)
 	* Type __xymon_client: Fix spelling error in manpage (Dmitry Bogatov)
+	* Build: Support pip from git (Darko Poljak, Ľubomír Kučera)
 
 6.0.2: 2019-10-17
 	* New types: __xymon_server, __xymon_apache, __xymon_config, __xymon_client (Thomas Eckert)

From 6eec5f87a4e2c3b006a2bded1e85ec565cd7acc6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 26 Oct 2019 17:11:35 +0200
Subject: [PATCH 033/743] [__hostame] Simplify and revert hostnamectl to be a
 last resort

---
 .../__hostname/explorer/persistent_hostname   | 66 ----------------
 cdist/conf/type/__hostname/gencode-remote     | 73 +++++++----------
 cdist/conf/type/__hostname/manifest           | 78 +++++++++++++++----
 3 files changed, 90 insertions(+), 127 deletions(-)
 delete mode 100755 cdist/conf/type/__hostname/explorer/persistent_hostname

diff --git a/cdist/conf/type/__hostname/explorer/persistent_hostname b/cdist/conf/type/__hostname/explorer/persistent_hostname
deleted file mode 100755
index 0feb0809..00000000
--- a/cdist/conf/type/__hostname/explorer/persistent_hostname
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/bin/sh
-#
-# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Retrieve the persistent hostname
-#
-
-# Firstly, check hostnamectl if available.
-if command -v hostnamectl >/dev/null 2>&1
-then
-	hostnamectl status --static && exit 0
-fi
-
-# In case hostnamectl is missing or failed:
-# Fall back to alternative OS-specific approaches.
-case $("$__explorer/os")
-in
-	alpine|archlinux|coreos|debian|devuan|ubuntu)
-		test -f /etc/hostname && cat /etc/hostname
-		;;
-	centos|fedora|redhat|scientific)
-		test -f /etc/sysconfig/network \
-			&& (. /etc/sysconfig/network && echo "$HOSTNAME")
-		;;
-	freebsd)
-		(. /etc/rc.conf && echo "$hostname")
-		;;
-	gentoo)
-		(. /etc/conf.d/hostname && echo "$hostname")
-		;;
-	macosx)
-		scutil --get HostName 2>/dev/null
-		;;
-	netbsd)
-		if grep -q '^hostname=' /etc/rc.conf
-		then
-			(. /etc/rc.conf && echo "$hostname")
-		elif [ -f /etc/myname ]
-		then
-			# Fall back to /etc/myname file
-			cat /etc/myname
-		fi
-		;;
-	openbsd)
-		cat /etc/myname
-		;;
-	suse)
-		cat /etc/HOSTNAME 2>/dev/null
-		;;
-esac
diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 5942b2e3..253bf5ea 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -22,17 +22,17 @@
 
 os=$(cat "$__global/explorer/os")
 name_running=$(cat "$__global/explorer/hostname")
-name_config=$(cat "$__object/explorer/persistent_hostname")
 has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
 
 
-if [ -f "$__object/parameter/name" ]; then
-    name_should="$(cat "$__object/parameter/name")"
+if test -s "$__object/parameter/name"
+then
+    name_should=$(cat "$__object/parameter/name")
 else
     case $os
     in
         # RedHat-derivatives and BSDs
-        centos|fedora|redhat|scientific|freebsd|netbsd|openbsd)
+        centos|fedora|redhat|scientific|freebsd|macosx|netbsd|openbsd)
             # Hostname is FQDN
             name_should="${__target_host}"
         ;;
@@ -43,66 +43,49 @@ else
     esac
 fi
 
+
 ################################################################################
-# Check if the hostname is already correct
+# Check if the (running) hostname is already correct
 #
-if [ "$name_running" = "$name_should" ] \
-  && [ -z "$name_config" -o "$name_config" = "$name_should" ]
-then
-    exit 0
-fi
+test "$name_running" != "$name_should" || exit 0
+
 
 ################################################################################
 # Setup hostname
 #
 echo 'changed' >>"$__messages_out"
 
-
-# First try to set the hostname using hostnamectl, if available.
-if [ -n "$has_hostnamectl" ]; then
-    # Allow hostnamectl to fail silently.
-    # Who the fuck invented a tool that needs dbus to set the hostname anyway…
-
-    # NOTE: We write the static hostname to /etc/hostname first in case
-    #       hostnamectl fails. Fallback-code below will then adjust the running
-    #       hostname below.
-    echo "echo '$name_should' >/etc/hostname"
-
-    echo "hostnamectl set-hostname '$name_should' && exit 0"
-fi
-
-# Use the good old way to set the hostname. Also if hostnamectl fails.
+# Use the good old way to set the hostname.
 case $os
 in
-    alpine|archlinux|debian|devuan|ubuntu)
-        echo "echo '$name_should' >/etc/hostname"
-        echo "hostname -F /etc/hostname"
+    alpine|debian|devuan|ubuntu)
+        echo 'hostname -F /etc/hostname'
     ;;
-    coreos|openbmc-phosphor)
-        # NOTE: systemd-only distros have the hostname already written above.
-        #       But since hostamectl failed, we update the running hostname
-        #       manually.
-        echo "hostname -F /etc/hostname"
-        ;;
-    gentoo)
-        # NOTE: Writing the hostname to file is done in the manifest for OpenRC.
-        #       For systemd hostnamectl should take care of that.
+    archlinux|centos|fedora|redhat|scientific|freebsd|netbsd|gentoo)
         echo "hostname '$name_should'"
     ;;
     macosx)
         echo "scutil --set HostName '$name_should'"
     ;;
+    openbsd)
+        echo 'hostname -F /etc/myname'
+    ;;
     suse)
-        # We write into /etc/HOSTNAME for backwards-compatibility.  Modern SuSE
-        # has hostnamectl anyway and symlinks /etc/HOSTNAME to /etc/hostname.
-        echo "echo '$name_should' >/etc/HOSTNAME"
         echo 'hostname -F /etc/HOSTNAME'
     ;;
-    centos|fedora|freebsd|netbsd|openbsd|redhat|scientific)
-        # NOTE: Writing the hostname to file is done in the manifest.
-        echo "hostname '$name_should'"
-    ;;
     *)
-        echo "echo 'Unsupported OS: $os' >&2; exit 1"
+        # Fall back to set the hostname using hostnamectl, if available.
+        if test -n "$has_hostnamectl"
+        then
+            # Don't use hostnamectl as the primary means to set the hostname for
+            # systemd systems, because it cannot be trusted to work reliably and
+            # exit with non-zero when it fails.
+            # Who invented a tool that needs dbus to set the hostname anyway…
+
+            echo "hostnamectl set-hostname \"\$(cat /etc/hostname)\""
+            echo "test \"\$(hostname)\" = \"\$(cat /etc/hostname)\" || hostname -F /etc/hostname"
+        else
+            printf "echo 'Unsupported OS: %s' >&2\nexit 1\n" "$os"
+        fi
     ;;
 esac
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index a4068cc2..78adc20b 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -20,18 +20,25 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+onchange_cmd="echo changed >>'$__messages_out'"
+
 not_supported() {
-   echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
-   echo "Please contribute an implementation for it if you can." >&2
-   exit 1
+    echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
+    echo "Please contribute an implementation for it if you can." >&2
+    exit 1
 }
 
+set_hostname_systemd() {
+    echo "$1" | __file /etc/hostname --source - --onchange "$onchange_cmd"
+}
 
 os=$(cat "$__global/explorer/os")
 max_len=$(cat "$__object/explorer/max_len")
+has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
 
-if [ -f "$__object/parameter/name" ]; then
-    name_should="$(cat "$__object/parameter/name")"
+if test -s "$__object/parameter/name"
+then
+    name_should=$(cat "$__object/parameter/name")
 else
     case $os
     in
@@ -49,18 +56,37 @@ fi
 
 if test -n "${max_len}" -a "$(printf '%s' "${name_should}" | wc -c)" -gt "${max_len}"
 then
-	printf "Host name too long. Up to %u characters allowed.\n" "${max_len}" >&2
-	exit 1
+    printf "Host name too long. Up to %u characters allowed.\n" "${max_len}" >&2
+    exit 1
 fi
 
 case $os
 in
-    archlinux|debian|suse|ubuntu|devuan|coreos|alpine|macosx)
+    alpine|debian|devuan|ubuntu)
+        echo "$name_should" \
+            | __file /etc/hostname --source - --onchange "$onchange_cmd"
+    ;;
+    macosx)
         # handled in gencode-remote
         :
     ;;
+    archlinux)
+        if test -n "$has_hostnamectl"
+        then
+            set_hostname_systemd "$name_should"
+        else
+            # Only for ancient ArchLinux, write to /etc/rc.conf on pre-systemd
+            # versions.
+            __key_value sysconfig-hostname \
+                --file /etc/rc.conf \
+                --delimiter '=' --exact_delimiter \
+                --key 'HOSTNAME' \
+                --value "\"$name_should\"" \
+                --onchange "$onchange_cmd"
+        fi
+        ;;
     centos|fedora|redhat|scientific)
-        if test -z "$(cat "$__object/explorer/has_hostnamectl")"
+        if test -z "$has_hostnamectl"
         then
             # Only write to /etc/sysconfig/network on non-systemd versions.
             # On systemd-based versions this entry is ignored.
@@ -68,11 +94,14 @@ in
                 --file /etc/sysconfig/network \
                 --delimiter '=' --exact_delimiter \
                 --key HOSTNAME \
-                --value "\"$name_should\""
+                --value "\"$name_should\"" \
+                --onchange "$onchange_cmd"
+        else
+            set_hostname_systemd "$name_should"
         fi
     ;;
     gentoo)
-        if test -z "$(cat "$__object/explorer/has_hostnamectl")"
+        if test -z "$has_hostnamectl"
         then
             # Only write to /etc/conf.d/hostname on OpenRC-based installations.
             # On systemd use hostnamectl(1) in gencode-remote.
@@ -81,6 +110,8 @@ in
                 --delimiter '=' --exact_delimiter \
                 --key 'hostname' \
                 --value "\"$name_should\""
+        else
+            set_hostname_systemd "$name_should"
         fi
     ;;
     freebsd)
@@ -88,22 +119,37 @@ in
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
-            --value "\"$name_should\""
+            --value "\"$name_should\"" \
+            --onchange "$onchange_cmd"
     ;;
     netbsd)
         __key_value rcconf-hostname \
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
-            --value "\"$name_should\""
+            --value "\"$name_should\"" \
+            --onchange "$onchange_cmd"
+
         # To avoid confusion, ensure that the hostname is only stored once
         __file /etc/myname --state absent
     ;;
     openbsd)
-        echo "$name_should" | __file /etc/myname --source -
+        echo "$name_should" \
+            | __file /etc/myname --source - --onchange "$onchange_cmd"
+    ;;
+    suse)
+        # We write into /etc/HOSTNAME for backwards-compatibility.  Modern SuSE
+        # has hostnamectl anyway and symlinks /etc/HOSTNAME to /etc/hostname.
+        echo "$name_should" \
+            | __file /etc/HOSTNAME --source - --onchange "$onchange_cmd"
     ;;
     *)
-        # On other operating systems we fall back to hostnamectl if available…
-        test -n "$(cat "$__object/explorer/has_hostnamectl")" || not_supported
+        # On other operating systems we fall back to systemd's hostnamectl if available…
+        if test -n "$has_hostnamectl"
+        then
+            set_hostname_systemd "$name_should"
+        else
+            not_supported
+        fi
     ;;
 esac

From e7279680edbd645249eca3c2d363d233dd1cd269 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 26 Oct 2019 17:12:08 +0200
Subject: [PATCH 034/743] [__hostname/man.rst] Add link to RFC 1178

---
 cdist/conf/type/__hostname/man.rst | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__hostname/man.rst b/cdist/conf/type/__hostname/man.rst
index d23a3b8a..72aefbab 100644
--- a/cdist/conf/type/__hostname/man.rst
+++ b/cdist/conf/type/__hostname/man.rst
@@ -8,7 +8,10 @@ cdist-type__hostname - Set the hostname
 
 DESCRIPTION
 -----------
-Set's the hostname on various operating systems.
+Sets the hostname on various operating systems.
+
+**Tip:** For advice on choosing a hostname, see
+`RFC 1178 <https://tools.ietf.org/html/rfc1178>`_.
 
 
 REQUIRED PARAMETERS
@@ -18,7 +21,7 @@ None.
 OPTIONAL PARAMETERS
 -------------------
 name
-   The hostname to set. Defaults to the first segment of __target_host 
+   The hostname to set. Defaults to the first segment of __target_host
    (${__target_host%%.*})
 
 

From f276813f7f1fe0af7775bb41adf37dcd3d377992 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 26 Oct 2019 17:25:44 +0200
Subject: [PATCH 035/743] [__hostname] Remove incorrectly sent messages (again)

The --onchage sent the message on the target where messaging is not supported.

Now the result is that a "changed" message is only emitted when the running
hostname changes, not when the persistent hostname changes.
---
 cdist/conf/type/__hostname/manifest | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 78adc20b..49591926 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -20,8 +20,6 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-onchange_cmd="echo changed >>'$__messages_out'"
-
 not_supported() {
     echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
     echo "Please contribute an implementation for it if you can." >&2
@@ -29,7 +27,7 @@ not_supported() {
 }
 
 set_hostname_systemd() {
-    echo "$1" | __file /etc/hostname --source - --onchange "$onchange_cmd"
+    echo "$1" | __file /etc/hostname --source -
 }
 
 os=$(cat "$__global/explorer/os")
@@ -64,7 +62,7 @@ case $os
 in
     alpine|debian|devuan|ubuntu)
         echo "$name_should" \
-            | __file /etc/hostname --source - --onchange "$onchange_cmd"
+            | __file /etc/hostname --source -
     ;;
     macosx)
         # handled in gencode-remote
@@ -81,8 +79,7 @@ in
                 --file /etc/rc.conf \
                 --delimiter '=' --exact_delimiter \
                 --key 'HOSTNAME' \
-                --value "\"$name_should\"" \
-                --onchange "$onchange_cmd"
+                --value "\"$name_should\""
         fi
         ;;
     centos|fedora|redhat|scientific)
@@ -94,8 +91,7 @@ in
                 --file /etc/sysconfig/network \
                 --delimiter '=' --exact_delimiter \
                 --key HOSTNAME \
-                --value "\"$name_should\"" \
-                --onchange "$onchange_cmd"
+                --value "\"$name_should\""
         else
             set_hostname_systemd "$name_should"
         fi
@@ -119,29 +115,27 @@ in
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
-            --value "\"$name_should\"" \
-            --onchange "$onchange_cmd"
+            --value "\"$name_should\""
     ;;
     netbsd)
         __key_value rcconf-hostname \
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
-            --value "\"$name_should\"" \
-            --onchange "$onchange_cmd"
+            --value "\"$name_should\""
 
         # To avoid confusion, ensure that the hostname is only stored once
         __file /etc/myname --state absent
     ;;
     openbsd)
         echo "$name_should" \
-            | __file /etc/myname --source - --onchange "$onchange_cmd"
+            | __file /etc/myname --source -
     ;;
     suse)
         # We write into /etc/HOSTNAME for backwards-compatibility.  Modern SuSE
         # has hostnamectl anyway and symlinks /etc/HOSTNAME to /etc/hostname.
         echo "$name_should" \
-            | __file /etc/HOSTNAME --source - --onchange "$onchange_cmd"
+            | __file /etc/HOSTNAME --source -
     ;;
     *)
         # On other operating systems we fall back to systemd's hostnamectl if available…

From cbae534af58859da299972b95d9f5c05040c33e5 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 26 Oct 2019 21:01:54 +0200
Subject: [PATCH 036/743] [__hostname] Patch for modern ArchLinux without
 hostname(1)

---
 cdist/conf/type/__hostname/gencode-remote | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 253bf5ea..6840ca9e 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -61,7 +61,12 @@ in
     alpine|debian|devuan|ubuntu)
         echo 'hostname -F /etc/hostname'
     ;;
-    archlinux|centos|fedora|redhat|scientific|freebsd|netbsd|gentoo)
+    archlinux)
+        echo 'command -v hostnamectl >/dev/null 2>&1' \
+            "&& hostnamectl set-hostname '$name_should'" \
+            "|| hostname '$name_should'"
+    ;;
+    centos|fedora|redhat|scientific|freebsd|netbsd|gentoo)
         echo "hostname '$name_should'"
     ;;
     macosx)

From 0ec52fd1bd1a5b20aa5d8503da62d18b43399403 Mon Sep 17 00:00:00 2001
From: Ahmed Bilal <49-ahmedbilal@users.noreply.code.ungleich.ch>
Date: Sun, 27 Oct 2019 18:11:08 +0100
Subject: [PATCH 037/743] alpine support added in __package_update_index

---
 cdist/conf/type/__package_update_index/explorer/currage | 3 +++
 cdist/conf/type/__package_update_index/explorer/type    | 1 +
 cdist/conf/type/__package_update_index/gencode-remote   | 4 ++++
 3 files changed, 8 insertions(+)

diff --git a/cdist/conf/type/__package_update_index/explorer/currage b/cdist/conf/type/__package_update_index/explorer/currage
index 3539b8e1..cfb778d5 100644
--- a/cdist/conf/type/__package_update_index/explorer/currage
+++ b/cdist/conf/type/__package_update_index/explorer/currage
@@ -34,6 +34,9 @@ case "$type" in
             echo 0
         fi
         ;;
+    alpine)
+        echo 0
+        ;;
     *)  echo "Your specified type ($type) is currently not supported." >&2
         echo "Please contribute an implementation for it if you can." >&2
         ;;
diff --git a/cdist/conf/type/__package_update_index/explorer/type b/cdist/conf/type/__package_update_index/explorer/type
index 35254c5f..c98e1e67 100644
--- a/cdist/conf/type/__package_update_index/explorer/type
+++ b/cdist/conf/type/__package_update_index/explorer/type
@@ -26,6 +26,7 @@ else
         amazon|scientific|centos|fedora|redhat) echo "yum" ;;
         debian|ubuntu|devuan) echo "apt" ;;
         archlinux) echo "pacman" ;;
+        alpine) echo "apk" ;;
         *)
             echo "Don't know how to manage packages on: $os" >&2
             exit 1
diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote
index 738d38eb..9b2ecba2 100755
--- a/cdist/conf/type/__package_update_index/gencode-remote
+++ b/cdist/conf/type/__package_update_index/gencode-remote
@@ -47,6 +47,10 @@ case "$type" in
         echo "pacman --noprogressbar --sync --refresh"
         echo "pacman package database synced (age was: $currage)" >> "$__messages_out"
         ;;
+    alpine)
+        echo "apk update"
+        echo "apk package database updated."
+        ;;
     *)
         echo "Don't know how to manage packages for type: $type" >&2
         exit 1

From 39b320a19a14c1e7653ff14e7e824f2d1ac762cb Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sun, 27 Oct 2019 19:12:52 +0100
Subject: [PATCH 038/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index cdc4e8f5..d4b55336 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __letsencrypt_cert: Add Alpine support (Nico Schottelius)
 	* Type __xymon_client: Fix spelling error in manpage (Dmitry Bogatov)
 	* Build: Support pip from git (Darko Poljak, Ľubomír Kučera)
+	* Type __package_update_index: Add Alpine support (Ahmed Bilal Khalid)
 
 6.0.2: 2019-10-17
 	* New types: __xymon_server, __xymon_apache, __xymon_config, __xymon_client (Thomas Eckert)

From b9571a2bbdf3cb9cca52444f5da2db16327801c9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 26 Oct 2019 22:17:08 +0200
Subject: [PATCH 039/743] [__hostname] Add support for Void Linux

---
 cdist/conf/type/__hostname/gencode-remote | 2 +-
 cdist/conf/type/__hostname/manifest       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 6840ca9e..90312fad 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -66,7 +66,7 @@ in
             "&& hostnamectl set-hostname '$name_should'" \
             "|| hostname '$name_should'"
     ;;
-    centos|fedora|redhat|scientific|freebsd|netbsd|gentoo)
+    centos|fedora|redhat|scientific|freebsd|netbsd|gentoo|void)
         echo "hostname '$name_should'"
     ;;
     macosx)
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 49591926..4465038d 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -60,7 +60,7 @@ fi
 
 case $os
 in
-    alpine|debian|devuan|ubuntu)
+    alpine|debian|devuan|ubuntu|void)
         echo "$name_should" \
             | __file /etc/hostname --source -
     ;;

From 66c85230a0aa35a64323e2cfc68490ef87055670 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 27 Oct 2019 13:16:05 +0100
Subject: [PATCH 040/743] [__hostname] Make __key_value names more meaningful

---
 cdist/conf/type/__hostname/manifest | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 4465038d..64f90492 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -61,12 +61,7 @@ fi
 case $os
 in
     alpine|debian|devuan|ubuntu|void)
-        echo "$name_should" \
-            | __file /etc/hostname --source -
-    ;;
-    macosx)
-        # handled in gencode-remote
-        :
+        echo "$name_should" | __file /etc/hostname --source -
     ;;
     archlinux)
         if test -n "$has_hostnamectl"
@@ -75,7 +70,8 @@ in
         else
             # Only for ancient ArchLinux, write to /etc/rc.conf on pre-systemd
             # versions.
-            __key_value sysconfig-hostname \
+			# There are some versions which use /etc/hostname but not systemd. It is unclear which ones these are.
+            __key_value '/etc/rc.conf:HOSTNAME' \
                 --file /etc/rc.conf \
                 --delimiter '=' --exact_delimiter \
                 --key 'HOSTNAME' \
@@ -87,7 +83,7 @@ in
         then
             # Only write to /etc/sysconfig/network on non-systemd versions.
             # On systemd-based versions this entry is ignored.
-            __key_value sysconfig-hostname \
+            __key_value '/etc/sysconfig/network:HOSTNAME' \
                 --file /etc/sysconfig/network \
                 --delimiter '=' --exact_delimiter \
                 --key HOSTNAME \
@@ -101,7 +97,7 @@ in
         then
             # Only write to /etc/conf.d/hostname on OpenRC-based installations.
             # On systemd use hostnamectl(1) in gencode-remote.
-            __key_value confd-hostname \
+            __key_value '/etc/conf.d/hostname:hostname' \
                 --file /etc/conf.d/hostname \
                 --delimiter '=' --exact_delimiter \
                 --key 'hostname' \
@@ -111,14 +107,18 @@ in
         fi
     ;;
     freebsd)
-        __key_value rcconf-hostname \
+        __key_value '/etc/rc.conf:hostname' \
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
             --value "\"$name_should\""
     ;;
+    macosx)
+        # handled in gencode-remote
+        :
+    ;;
     netbsd)
-        __key_value rcconf-hostname \
+        __key_value '/etc/rc.conf:hostname' \
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
@@ -128,14 +128,12 @@ in
         __file /etc/myname --state absent
     ;;
     openbsd)
-        echo "$name_should" \
-            | __file /etc/myname --source -
+        echo "$name_should" | __file /etc/myname --source -
     ;;
     suse)
         # We write into /etc/HOSTNAME for backwards-compatibility.  Modern SuSE
         # has hostnamectl anyway and symlinks /etc/HOSTNAME to /etc/hostname.
-        echo "$name_should" \
-            | __file /etc/HOSTNAME --source -
+        echo "$name_should" | __file /etc/HOSTNAME --source -
     ;;
     *)
         # On other operating systems we fall back to systemd's hostnamectl if available…

From 5eb9fec550907d3ae65ce9ef62849093afc7c9f4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 27 Oct 2019 15:14:09 +0100
Subject: [PATCH 041/743] [__hostname] Add support for Slackware

---
 cdist/conf/type/__hostname/gencode-remote | 2 +-
 cdist/conf/type/__hostname/manifest       | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 90312fad..1e798036 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -66,7 +66,7 @@ in
             "&& hostnamectl set-hostname '$name_should'" \
             "|| hostname '$name_should'"
     ;;
-    centos|fedora|redhat|scientific|freebsd|netbsd|gentoo|void)
+    centos|fedora|redhat|scientific|freebsd|netbsd|gentoo|slackware|void)
         echo "hostname '$name_should'"
     ;;
     macosx)
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 64f90492..8b4d26b1 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -41,7 +41,7 @@ else
     case $os
     in
         # RedHat-derivatives and BSDs
-        centos|fedora|redhat|scientific|freebsd|netbsd|openbsd)
+        centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware)
             # Hostname is FQDN
             name_should="${__target_host}"
         ;;
@@ -130,6 +130,12 @@ in
     openbsd)
         echo "$name_should" | __file /etc/myname --source -
     ;;
+    slackware)
+        # We write the FQDN into /etc/HOSTNAME.
+        # But /etc/rc.d/rc.M will only read the first component from this file
+        # and set it as the running hostname on boot.
+        echo "$name_should" | __file /etc/HOSTNAME --source -
+    ;;
     suse)
         # We write into /etc/HOSTNAME for backwards-compatibility.  Modern SuSE
         # has hostnamectl anyway and symlinks /etc/HOSTNAME to /etc/hostname.

From c939bd6bf676b186ced82bebf925ce83b7c79df4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 27 Oct 2019 20:12:10 +0100
Subject: [PATCH 042/743] [__hostname] Small fixes

---
 cdist/conf/type/__hostname/gencode-remote | 5 +----
 cdist/conf/type/__hostname/manifest       | 6 +++---
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 1e798036..4532bacf 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -66,15 +66,12 @@ in
             "&& hostnamectl set-hostname '$name_should'" \
             "|| hostname '$name_should'"
     ;;
-    centos|fedora|redhat|scientific|freebsd|netbsd|gentoo|slackware|void)
+    centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|slackware|void)
         echo "hostname '$name_should'"
     ;;
     macosx)
         echo "scutil --set HostName '$name_should'"
     ;;
-    openbsd)
-        echo 'hostname -F /etc/myname'
-    ;;
     suse)
         echo 'hostname -F /etc/HOSTNAME'
     ;;
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 8b4d26b1..31d43264 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -52,7 +52,7 @@ else
     esac
 fi
 
-if test -n "${max_len}" -a "$(printf '%s' "${name_should}" | wc -c)" -gt "${max_len}"
+if test -n "${max_len}" && test "$(printf '%s' "${name_should}" | wc -c)" -gt "${max_len}"
 then
     printf "Host name too long. Up to %u characters allowed.\n" "${max_len}" >&2
     exit 1
@@ -69,8 +69,8 @@ in
             set_hostname_systemd "$name_should"
         else
             # Only for ancient ArchLinux, write to /etc/rc.conf on pre-systemd
-            # versions.
-			# There are some versions which use /etc/hostname but not systemd. It is unclear which ones these are.
+            # versions.  There are some versions which use /etc/hostname but not
+			# systemd. It is unclear which ones these are.
             __key_value '/etc/rc.conf:HOSTNAME' \
                 --file /etc/rc.conf \
                 --delimiter '=' --exact_delimiter \

From 5fcd4bc09f1f34c20c53a9a5cc25d084d86c36d8 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 31 Oct 2019 08:56:13 +0100
Subject: [PATCH 043/743] Release 6.0.3

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index d4b55336..1ba56e73 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.0.3: 2019-10-31
 	* Type __letsencrypt_cert: Add Alpine support (Nico Schottelius)
 	* Type __xymon_client: Fix spelling error in manpage (Dmitry Bogatov)
 	* Build: Support pip from git (Darko Poljak, Ľubomír Kučera)

From 7b8f1f09d9cbe6d0da721597055b68e2efd9980d Mon Sep 17 00:00:00 2001
From: Kirill Miazine <km@krot.org>
Date: Fri, 8 Nov 2019 09:00:29 +0100
Subject: [PATCH 044/743] Correct a couple of typos.

---
 docs/src/cdist-real-world.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/src/cdist-real-world.rst b/docs/src/cdist-real-world.rst
index 8ccb0fc9..ba118d63 100644
--- a/docs/src/cdist-real-world.rst
+++ b/docs/src/cdist-real-world.rst
@@ -27,7 +27,7 @@ for that. This type will:
 - configure nginx.
 
 Our type will not create the actual python application. Its intention is only
-to configure hosing for specified user and project. It is up to the user to
+to configure hosting for specified user and project. It is up to the user to
 create his/her applications.
 
 So let's start.
@@ -480,7 +480,7 @@ Creating python bottle application
 
 We now need to create Bottle application. As you remember from the beginning
 of this walkthrough our type does not create the actual python application,
-its intention is only to configure hosing for specified user and project.
+its intention is only to configure hosting for specified user and project.
 It is up to the user to create his/her applications.
 
 Become app user::

From 8b0686766a0f6f6eb4219ec54a68ef270703bff5 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 8 Nov 2019 13:27:38 +0100
Subject: [PATCH 045/743] ++changes

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 1ba56e73..1c56ec74 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Doc: Fix typos (Kirill Miazine)
+
 6.0.3: 2019-10-31
 	* Type __letsencrypt_cert: Add Alpine support (Nico Schottelius)
 	* Type __xymon_client: Fix spelling error in manpage (Dmitry Bogatov)

From 88947d45b3a6a8496b568e0880a94afd6553bbed Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 16 Nov 2019 14:57:27 +0100
Subject: [PATCH 046/743] [__hostname] Add support for Solaris

---
 cdist/conf/type/__hostname/gencode-remote | 3 +++
 cdist/conf/type/__hostname/manifest       | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 4532bacf..f06ee145 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -72,6 +72,9 @@ in
     macosx)
         echo "scutil --set HostName '$name_should'"
     ;;
+    solaris)
+        echo "uname -S '$name_should'"
+    ;;
     suse)
         echo 'hostname -F /etc/HOSTNAME'
     ;;
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 31d43264..6b26cada 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -136,6 +136,9 @@ in
         # and set it as the running hostname on boot.
         echo "$name_should" | __file /etc/HOSTNAME --source -
     ;;
+    solaris)
+        echo "$name_should" | __file /etc/nodename --source -
+    ;;
     suse)
         # We write into /etc/HOSTNAME for backwards-compatibility.  Modern SuSE
         # has hostnamectl anyway and symlinks /etc/HOSTNAME to /etc/hostname.

From a885082534dedb56a78f2b2ee7c3ff7a20348b59 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 16 Nov 2019 14:58:30 +0100
Subject: [PATCH 047/743] [__hostname] Little bit of clean up

---
 cdist/conf/type/__hostname/manifest | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 6b26cada..f2f65155 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -52,7 +52,7 @@ else
     esac
 fi
 
-if test -n "${max_len}" && test "$(printf '%s' "${name_should}" | wc -c)" -gt "${max_len}"
+if test -n "$max_len" && test "$(printf '%s' "$name_should" | wc -c)" -gt "$max_len"
 then
     printf "Host name too long. Up to %u characters allowed.\n" "${max_len}" >&2
     exit 1
@@ -93,10 +93,10 @@ in
         fi
     ;;
     gentoo)
+        # Only write to /etc/conf.d/hostname on OpenRC-based installations.
+        # On systemd use hostnamectl(1) in gencode-remote.
         if test -z "$has_hostnamectl"
         then
-            # Only write to /etc/conf.d/hostname on OpenRC-based installations.
-            # On systemd use hostnamectl(1) in gencode-remote.
             __key_value '/etc/conf.d/hostname:hostname' \
                 --file /etc/conf.d/hostname \
                 --delimiter '=' --exact_delimiter \

From f18bdd1fade8b2caea80bdeb4d0c099f421a3d18 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 16 Nov 2019 23:31:58 +0100
Subject: [PATCH 048/743] [__hostname] Comment out support for ancient
 ArchLinux versions

---
 cdist/conf/type/__hostname/manifest | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index f2f65155..7e36cb53 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -68,14 +68,17 @@ in
         then
             set_hostname_systemd "$name_should"
         else
+            echo 'Ancient ArchLinux variants without hostnamectl are not supported.' >&2
+            exit 1
             # Only for ancient ArchLinux, write to /etc/rc.conf on pre-systemd
             # versions.  There are some versions which use /etc/hostname but not
-			# systemd. It is unclear which ones these are.
-            __key_value '/etc/rc.conf:HOSTNAME' \
-                --file /etc/rc.conf \
-                --delimiter '=' --exact_delimiter \
-                --key 'HOSTNAME' \
-                --value "\"$name_should\""
+            # systemd. It is unclear which ones these are.
+
+            # __key_value '/etc/rc.conf:HOSTNAME' \
+            #     --file /etc/rc.conf \
+            #     --delimiter '=' --exact_delimiter \
+            #     --key 'HOSTNAME' \
+            #     --value "\"$name_should\""
         fi
         ;;
     centos|fedora|redhat|scientific)

From a45e30612301a93e1156f97309c9b0f090c4a4ce Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 17 Nov 2019 14:46:37 +0100
Subject: [PATCH 049/743] [__hostname] Better support different versions of
 SuSE

---
 cdist/conf/type/__hostname/gencode-remote | 16 +++++---
 cdist/conf/type/__hostname/manifest       | 50 ++++++++++++++++++-----
 2 files changed, 50 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index f06ee145..ae224611 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -66,7 +66,7 @@ in
             "&& hostnamectl set-hostname '$name_should'" \
             "|| hostname '$name_should'"
     ;;
-    centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|slackware|void)
+    centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void)
         echo "hostname '$name_should'"
     ;;
     macosx)
@@ -75,8 +75,11 @@ in
     solaris)
         echo "uname -S '$name_should'"
     ;;
-    suse)
-        echo 'hostname -F /etc/HOSTNAME'
+    slackware|suse|opensuse-leap)
+        # We do not read from /etc/HOSTNAME, because the running
+        # hostname is the first component only while the file contains
+        # the FQDN.
+        echo "hostname '$name_should'"
     ;;
     *)
         # Fall back to set the hostname using hostnamectl, if available.
@@ -84,11 +87,12 @@ in
         then
             # Don't use hostnamectl as the primary means to set the hostname for
             # systemd systems, because it cannot be trusted to work reliably and
-            # exit with non-zero when it fails.
-            # Who invented a tool that needs dbus to set the hostname anyway…
+            # exit with non-zero when it fails (e.g. hostname too long,
+            # D-Bus failure, etc.).
 
             echo "hostnamectl set-hostname \"\$(cat /etc/hostname)\""
-            echo "test \"\$(hostname)\" = \"\$(cat /etc/hostname)\" || hostname -F /etc/hostname"
+            echo "test \"\$(hostname)\" = \"\$(cat /etc/hostname)\"" \
+                 " || hostname -F /etc/hostname"
         else
             printf "echo 'Unsupported OS: %s' >&2\nexit 1\n" "$os"
         fi
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 7e36cb53..75a90027 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -31,6 +31,9 @@ set_hostname_systemd() {
 }
 
 os=$(cat "$__global/explorer/os")
+os_version=$(cat "$__global/explorer/os_version")
+os_major=$(echo "$os_version" | grep -o '^[0-9][0-9]*')
+
 max_len=$(cat "$__object/explorer/max_len")
 has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
 
@@ -45,8 +48,21 @@ else
             # Hostname is FQDN
             name_should="${__target_host}"
         ;;
+        suse|opensuse-leap)
+            # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
+            # systemd does not. The running hostname is the first
+            # component in both cases.
+            # In versions before 15.x, the FQDN is stored in /etc/hostname.
+            if test -n "$has_hostnamectl" && test "$os_major" -ge 15 \
+                    && test "$os_major" -ne 42
+            then
+                name_should="${__target_host%%.*}"
+            else
+                name_should="${__target_host}"
+            fi
+        ;;
         *)
-            # Hostname is only first component of FQDN
+            # Hostname is only first component of FQDN on all other systems.
             name_should="${__target_host%%.*}"
         ;;
     esac
@@ -127,28 +143,42 @@ in
             --key 'hostname' \
             --value "\"$name_should\""
 
-        # To avoid confusion, ensure that the hostname is only stored once
+        # To avoid confusion, ensure that the hostname is only stored once.
         __file /etc/myname --state absent
     ;;
     openbsd)
         echo "$name_should" | __file /etc/myname --source -
     ;;
     slackware)
-        # We write the FQDN into /etc/HOSTNAME.
-        # But /etc/rc.d/rc.M will only read the first component from this file
-        # and set it as the running hostname on boot.
+        # We write the FQDN into /etc/HOSTNAME.  But /etc/rc.d/rc.M will only
+        # read the first component from this file and set it as the running
+        # hostname on boot.
         echo "$name_should" | __file /etc/HOSTNAME --source -
     ;;
     solaris)
         echo "$name_should" | __file /etc/nodename --source -
     ;;
-    suse)
-        # We write into /etc/HOSTNAME for backwards-compatibility.  Modern SuSE
-        # has hostnamectl anyway and symlinks /etc/HOSTNAME to /etc/hostname.
-        echo "$name_should" | __file /etc/HOSTNAME --source -
+    suse|opensuse-leap)
+        # Modern SuSE provides /etc/HOSTNAME as a symlink for
+        # backwards-compatibility. Unfortunately it cannot be used
+        # here as __file does not follow the symlink.
+        # Therefore, we use the presence of the hostnamectl binary as
+        # an indication of which file to use.  This unfortunately does
+        # not work correctly on openSUSE 12.x which provides
+        # hostnamectl but not /etc/hostname.
+
+        if test -n "$has_hostnamectl" -a "$os_major" -gt 12
+        then
+            hostname_file='/etc/hostname'
+        else
+            hostname_file='/etc/HOSTNAME'
+        fi
+
+        echo "$name_should" | __file "$hostname_file" --source -
     ;;
     *)
-        # On other operating systems we fall back to systemd's hostnamectl if available…
+        # On other operating systems we fall back to systemd's
+        # hostnamectl if available…
         if test -n "$has_hostnamectl"
         then
             set_hostname_systemd "$name_should"

From c1633d9301c053b604cceb39b57621636e80b9cd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 19 Nov 2019 21:29:45 +0100
Subject: [PATCH 050/743] Release 6.0.4

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 1c56ec74..b0a52eb6 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.0.4: 2019-11-19
 	* Doc: Fix typos (Kirill Miazine)
 
 6.0.3: 2019-10-31

From 6c67d7194af8166882da6169e681ca5a7836d534 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 19 Nov 2019 21:53:40 +0100
Subject: [PATCH 051/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index b0a52eb6..49416d6d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Explorer hostname, type __hostname: Support more operating systems, rewrite type and hostname explorer (Dennis Camera)
+
 6.0.4: 2019-11-19
 	* Doc: Fix typos (Kirill Miazine)
 

From 0bc00477aeea206e80a1e1fb887024ca2e43d0ab Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 19 Nov 2019 21:54:56 +0100
Subject: [PATCH 052/743] Release 6.1.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 49416d6d..2342b0b0 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.1.0: 2019-11-19
 	* Explorer hostname, type __hostname: Support more operating systems, rewrite type and hostname explorer (Dennis Camera)
 
 6.0.4: 2019-11-19

From 34bd96db4b61c9c80e41d0a91ddb51ef9dbb08c3 Mon Sep 17 00:00:00 2001
From: Thomas Eckert <tom@it-eckert.de>
Date: Wed, 20 Nov 2019 15:47:25 +0100
Subject: [PATCH 053/743] add `--msgcache`-parameter to support passive clients
 (__xymon_client)

Fix a typo along the way, too.
---
 cdist/conf/type/__xymon_client/man.rst           | 13 +++++++++++--
 cdist/conf/type/__xymon_client/manifest          |  9 +++++++--
 cdist/conf/type/__xymon_client/parameter/boolean |  1 +
 3 files changed, 19 insertions(+), 4 deletions(-)
 create mode 100644 cdist/conf/type/__xymon_client/parameter/boolean

diff --git a/cdist/conf/type/__xymon_client/man.rst b/cdist/conf/type/__xymon_client/man.rst
index 6660b0ef..05d085dc 100644
--- a/cdist/conf/type/__xymon_client/man.rst
+++ b/cdist/conf/type/__xymon_client/man.rst
@@ -27,21 +27,30 @@ servers
    to. While DNS-names are ok it is discouraged, defaults to 127.0.0.1.
 
 
+BOOLEAN PARAMETERS
+------------------
+msgcache
+    Enable xymon `msgcache`. Note: XYMONSERVER has to be `127.0.0.1` for using
+    `msgcache` (see `msgcache (8)` of the xymon documentation for details).
+
 EXAMPLES
 --------
 
 .. code-block:: sh
 
-    # minmal, report to 127.0.0.1
+    # minimal, report to 127.0.0.1
     __xymon_client
 
     # specify server:
     __xymon_client --servers "192.168.1.1"
 
+    # activate `msgcache` for passive client:
+    __xymon_client --msgcache
+
 
 SEE ALSO
 --------
-:strong:`cdist__xymon_server`\ (7), :strong:`xymon`\ (7)
+:strong:`cdist__xymon_server`\ (7), :strong:`xymon`\ (7), :strong:`msgcache`\ (8)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__xymon_client/manifest b/cdist/conf/type/__xymon_client/manifest
index 92ad079e..88293a12 100755
--- a/cdist/conf/type/__xymon_client/manifest
+++ b/cdist/conf/type/__xymon_client/manifest
@@ -34,13 +34,18 @@ esac
 
 __package xymon-client --state "$state"
 
+if [ -f "$__object/parameter/msgcache" ]; then
+	require="__package/xymon-client" __line /etc/xymon/clientlaunch.cfg \
+		--regex DISABLED --state absent
+fi
+
 require="__package/xymon-client" __key_value CLIENTHOSTNAME \
-       	--file /etc/default/xymon-client \
+	--file /etc/default/xymon-client \
 	--value "'$__target_hostname'" \
 	--delimiter '=' \
 	--state "$state"
 require="__package/xymon-client" __key_value XYMONSERVERS \
-       	--file /etc/default/xymon-client \
+	--file /etc/default/xymon-client \
 	--value "'$servers'" \
 	--delimiter '=' \
 	--state "$state"
diff --git a/cdist/conf/type/__xymon_client/parameter/boolean b/cdist/conf/type/__xymon_client/parameter/boolean
new file mode 100644
index 00000000..0dd7839d
--- /dev/null
+++ b/cdist/conf/type/__xymon_client/parameter/boolean
@@ -0,0 +1 @@
+msgcache

From 552860b9cd801af0dc3cd63969f935dca1e54491 Mon Sep 17 00:00:00 2001
From: Thomas Eckert <tom@it-eckert.de>
Date: Wed, 20 Nov 2019 15:49:25 +0100
Subject: [PATCH 054/743] add parameters for ownership and generic
 rsync-options (__xymon_config)

---
 cdist/conf/type/__xymon_config/man.rst        | 23 ++++++++++++++++++-
 cdist/conf/type/__xymon_config/manifest       | 21 ++++++++++++++++-
 .../type/__xymon_config/parameter/optional    |  2 ++
 .../parameter/optional_multiple               |  1 +
 4 files changed, 45 insertions(+), 2 deletions(-)
 create mode 100644 cdist/conf/type/__xymon_config/parameter/optional
 create mode 100644 cdist/conf/type/__xymon_config/parameter/optional_multiple

diff --git a/cdist/conf/type/__xymon_config/man.rst b/cdist/conf/type/__xymon_config/man.rst
index 8b427ca0..8adfbe1f 100644
--- a/cdist/conf/type/__xymon_config/man.rst
+++ b/cdist/conf/type/__xymon_config/man.rst
@@ -23,6 +23,21 @@ confdir
    deployed.
 
 
+OPTIONAL PARAMETERS
+-------------------
+owner
+   passed as-is as `--owner` to `__rsync`
+
+group
+   passed as-is as `--group` to `__rsync`
+
+
+OPTIONAL MULTIPLE PARAMETERS
+----------------------------
+rsync-opts
+   identical to __rsync type, only `--`-options are supported
+
+
 REQUIRED FILES
 --------------
 The directory specified by `confdir` has to contain a valid xymon-configuration
@@ -39,10 +54,16 @@ EXAMPLES
     # this will replace /etc/xymon/ on the target host with
     # the contents from __xymon_config/files/xymon.example.com/
 
+    ## the same but set ownership to `xymon:xymon` and exclude
+    ## the `netrc`-file:
+    __xymon_config --confdir=xymon.example.com \
+       --owner xymon --group xymon \
+       --rsync-opts "exclude=netrc"
+
 
 SEE ALSO
 --------
-:strong:`cdist__xymon_server`\ (7), :strong:`xymon`\ (7)
+:strong:`cdist__xymon_server`\ (7), :strong:`cdist__rsync`\ (7), :strong:`xymon`\ (7)
 
 AUTHORS
 -------
diff --git a/cdist/conf/type/__xymon_config/manifest b/cdist/conf/type/__xymon_config/manifest
index fb1bce54..4a5fb6c9 100644
--- a/cdist/conf/type/__xymon_config/manifest
+++ b/cdist/conf/type/__xymon_config/manifest
@@ -18,7 +18,26 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 
 confdir=$(cat "$__object/parameter/confdir")
+set --
+if [ -f "$__object/parameter/owner" ]; then
+	owner=$(cat "$__object/parameter/owner")
+	set -- "$@" "--owner $owner"
+fi
+if [ -f "$__object/parameter/group" ]; then
+	group=$(cat "$__object/parameter/group")
+	set -- "$@" "--group $group"
+fi
 
+## pass `--rsync-opts` as-is to `__rsync`:
+if [ -f "$__object/parameter/rsync-opts" ]; then
+	while read -r opts; do
+		# shellcheck disable=SC2089
+		set -- "$@" "--rsync-opts '$opts'"
+	done < "$__object/parameter/rsync-opts"
+fi
+
+# shellcheck disable=SC2068,SC2090
 __rsync /etc/xymon/ \
 	--source "$__type/files/$confdir/" \
-	--rsync-opts "delete"
+	--rsync-opts "delete" \
+	$@
diff --git a/cdist/conf/type/__xymon_config/parameter/optional b/cdist/conf/type/__xymon_config/parameter/optional
new file mode 100644
index 00000000..866b4bde
--- /dev/null
+++ b/cdist/conf/type/__xymon_config/parameter/optional
@@ -0,0 +1,2 @@
+owner
+group
diff --git a/cdist/conf/type/__xymon_config/parameter/optional_multiple b/cdist/conf/type/__xymon_config/parameter/optional_multiple
new file mode 100644
index 00000000..fdb7cd88
--- /dev/null
+++ b/cdist/conf/type/__xymon_config/parameter/optional_multiple
@@ -0,0 +1 @@
+rsync-opts

From ac2463fe31fcf74da47e6a159c67e041f7dc617e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 20 Nov 2019 20:58:42 +0100
Subject: [PATCH 055/743] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 2342b0b0..dc317ac9 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* Type __xymon_config: Add parameters for ownership and generic rsync options (Thomas Eckert)
+	* Type __xymon_client: Add msgcache parameter to support passive clients (Thomas Eckert)
+
 6.1.0: 2019-11-19
 	* Explorer hostname, type __hostname: Support more operating systems, rewrite type and hostname explorer (Dennis Camera)
 

From 01bd01573e6cb6fb3ba5d279e10bf58ab2cab929 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 21 Nov 2019 14:15:51 +0200
Subject: [PATCH 056/743] __apt_key: use mktemp for unique temporary gpg home

---
 cdist/conf/type/__apt_key/gencode-remote | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__apt_key/gencode-remote b/cdist/conf/type/__apt_key/gencode-remote
index e9daa524..0c96ff67 100755
--- a/cdist/conf/type/__apt_key/gencode-remote
+++ b/cdist/conf/type/__apt_key/gencode-remote
@@ -61,31 +61,29 @@ EOF
             echo "curl -s -L '$uri' | apt-key add -"
          fi
       elif [ -d "$keydir" ]; then
-         tmp='/tmp/cdist_apt_key_tmp'
-
          # we need to kill gpg after 30 seconds, because gpg
          # can get stuck if keyserver is not responding.
          # exporting env var and not exit 1,
          # because we need to clean up and kill dirmngr.
          cat << EOF
 
-mkdir -m 700 -p "$tmp"
+gpgtmphome="\$( mktemp -d )"
 
 if timeout 30s \\
-    gpg --homedir "$tmp" \\
+    gpg --homedir "\$gpgtmphome" \\
         --keyserver "$keyserver" \\
         --recv-keys "$keyid"
 then
-    gpg --homedir "$tmp" \\
+    gpg --homedir "\$gpgtmphome" \\
         --export "$keyid" \\
         > "$keyfile"
 else
     export GPG_GOT_STUCK=1
 fi
 
-GNUPGHOME="$tmp" gpgconf --kill dirmngr
+GNUPGHOME="\$gpgtmphome" gpgconf --kill dirmngr
 
-rm -rf "$tmp"
+rm -rf "\$gpgtmphome"
 
 if [ -n "\$GPG_GOT_STUCK" ]
 then

From b8c6f8c8f504c52062561b18921346733cce7dca Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 21 Nov 2019 17:01:27 +0100
Subject: [PATCH 057/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index dc317ac9..71509004 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __xymon_config: Add parameters for ownership and generic rsync options (Thomas Eckert)
 	* Type __xymon_client: Add msgcache parameter to support passive clients (Thomas Eckert)
+	* Type __apt_key: Use mktemp for unique temporary gpg home (Ander Punnar)
 
 6.1.0: 2019-11-19
 	* Explorer hostname, type __hostname: Support more operating systems, rewrite type and hostname explorer (Dennis Camera)

From 4ed18e3446a6eae022ac099a34ef4f25401874ae Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 22 Nov 2019 18:56:11 +0200
Subject: [PATCH 058/743] __hostname: silence grep exit 1, when os_version is
 not numeric

on Debian Sid os_version returns 'bullseye/sid'
---
 cdist/conf/type/__hostname/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 75a90027..e1e356a0 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -32,7 +32,7 @@ set_hostname_systemd() {
 
 os=$(cat "$__global/explorer/os")
 os_version=$(cat "$__global/explorer/os_version")
-os_major=$(echo "$os_version" | grep -o '^[0-9][0-9]*')
+os_major=$(echo "$os_version" | grep -o '^[0-9][0-9]*' || true)
 
 max_len=$(cat "$__object/explorer/max_len")
 has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")

From b876ebb16e2500b00a2ab73ff909b8067e8d055a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 22 Nov 2019 19:59:52 +0100
Subject: [PATCH 059/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 71509004..d287a2af 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __xymon_config: Add parameters for ownership and generic rsync options (Thomas Eckert)
 	* Type __xymon_client: Add msgcache parameter to support passive clients (Thomas Eckert)
 	* Type __apt_key: Use mktemp for unique temporary gpg home (Ander Punnar)
+	* Type __hostname: Silence grep exit 1 when os_version is not numeric (Ander Punnar)
 
 6.1.0: 2019-11-19
 	* Explorer hostname, type __hostname: Support more operating systems, rewrite type and hostname explorer (Dennis Camera)

From 64b07af4ab21483ebe6b03adb4d3f12c26544135 Mon Sep 17 00:00:00 2001
From: Daniel Tschada <daniel.tschada@ungleich.ch>
Date: Sun, 24 Nov 2019 17:18:34 +0100
Subject: [PATCH 060/743] give error msg to stderr

---
 cdist/conf/type/__ufw/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__ufw/manifest b/cdist/conf/type/__ufw/manifest
index 54309ff5..370b7ff5 100755
--- a/cdist/conf/type/__ufw/manifest
+++ b/cdist/conf/type/__ufw/manifest
@@ -31,7 +31,7 @@ case "$state" in
                     __package epel-release
                     require='__package/epel-release' __package ufw
                 else
-                    echo 'CentOS version 7 is required!'
+                    echo 'CentOS version 7 is required!' >&2
                     exit 1
                 fi
                 ;;

From 56435492a0e8a66c7437d93823e88f058d48f307 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sun, 24 Nov 2019 17:18:02 +0100
Subject: [PATCH 061/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index d287a2af..82941895 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Type __xymon_client: Add msgcache parameter to support passive clients (Thomas Eckert)
 	* Type __apt_key: Use mktemp for unique temporary gpg home (Ander Punnar)
 	* Type __hostname: Silence grep exit 1 when os_version is not numeric (Ander Punnar)
+	* Type __ufw: Print error message to stderr (Daniel Tschada)
 
 6.1.0: 2019-11-19
 	* Explorer hostname, type __hostname: Support more operating systems, rewrite type and hostname explorer (Dennis Camera)

From 2848a6a0f4e31e5e2fe19ab1340785b0cc332baa Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 25 Nov 2019 21:52:00 +0200
Subject: [PATCH 062/743] __clean_path: use __directory and __file types for
 better clarity

---
 cdist/conf/type/__clean_path/explorer/list  |  6 ++--
 cdist/conf/type/__clean_path/gencode-remote | 26 +---------------
 cdist/conf/type/__clean_path/manifest       | 33 +++++++++++++++++++++
 3 files changed, 38 insertions(+), 27 deletions(-)
 create mode 100755 cdist/conf/type/__clean_path/manifest

diff --git a/cdist/conf/type/__clean_path/explorer/list b/cdist/conf/type/__clean_path/explorer/list
index 07d38127..58d0b411 100755
--- a/cdist/conf/type/__clean_path/explorer/list
+++ b/cdist/conf/type/__clean_path/explorer/list
@@ -29,7 +29,9 @@ then
     exclude="$( cat "$__object/parameter/exclude" )"
 
     find "$path" -mindepth 1 -maxdepth 1 -regex "$pattern" \
-        -and -not -regex "$exclude"
+        -and -not -regex "$exclude" \
+        \( -type d -printf '%p/\n' , -type f -print \)
 else
-    find "$path" -mindepth 1 -maxdepth 1 -regex "$pattern"
+    find "$path" -mindepth 1 -maxdepth 1 -regex "$pattern" \
+        \( -type d -printf '%p/\n' , -type f -print \)
 fi
diff --git a/cdist/conf/type/__clean_path/gencode-remote b/cdist/conf/type/__clean_path/gencode-remote
index 998a70d8..0157733f 100755
--- a/cdist/conf/type/__clean_path/gencode-remote
+++ b/cdist/conf/type/__clean_path/gencode-remote
@@ -18,31 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-[ ! -s "$__object/explorer/list" ] && exit 0
-
-path="/$__object_id"
-
-pattern="$( cat "$__object/parameter/pattern" )"
-
-if [ -f "$__object/parameter/exclude" ]
-then
-    exclude="$( cat "$__object/parameter/exclude" )"
-
-    echo "find '$path' -mindepth 1 -maxdepth 1 -regex '$pattern'" \
-        "-and -not -regex '$exclude'" \
-        '-exec rm -rf {} \;'
-else
-    echo "find '$path' -mindepth 1 -maxdepth 1 -regex '$pattern'" \
-        '-exec rm -rf {} \;'
-fi
-
-while read -r f
-do
-    echo "removed '$f'" >> "$__messages_out"
-done \
-< "$__object/explorer/list"
-
-if [ -f "$__object/parameter/onchange" ]
+if [ -f "$__object/explorer/list" ] && [ -f "$__object/parameter/onchange" ]
 then
     cat "$__object/parameter/onchange"
 fi
diff --git a/cdist/conf/type/__clean_path/manifest b/cdist/conf/type/__clean_path/manifest
new file mode 100755
index 00000000..065ecb36
--- /dev/null
+++ b/cdist/conf/type/__clean_path/manifest
@@ -0,0 +1,33 @@
+#!/bin/sh -e
+#
+# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+[ ! -f "$__object/explorer/list" ] && exit 0
+
+for l in $( cat "$__object/explorer/list" )
+do
+    if echo "$l" | grep -Eq '/$'
+    then
+        __directory "$l" --state absent
+        echo "removed directory '$l'" >> "$__messages_out"
+    else
+        __file "$l" --state absent
+        echo "removed file '$l'" >> "$__messages_out"
+    fi
+done

From e44219903a11954f0078782bd234d2913633032e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 25 Nov 2019 21:01:20 +0100
Subject: [PATCH 063/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 82941895..d70cd72c 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Type __apt_key: Use mktemp for unique temporary gpg home (Ander Punnar)
 	* Type __hostname: Silence grep exit 1 when os_version is not numeric (Ander Punnar)
 	* Type __ufw: Print error message to stderr (Daniel Tschada)
+	* Type __clean_path: Use __directory and __file types for better clarity (Ander Punnar)
 
 6.1.0: 2019-11-19
 	* Explorer hostname, type __hostname: Support more operating systems, rewrite type and hostname explorer (Dennis Camera)

From 8236de763f1194c8ca80f44f1bc4d607d3dbb754 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 26 Nov 2019 17:57:19 +0100
Subject: [PATCH 064/743] Revert "++changelog"

This reverts commit e44219903a11954f0078782bd234d2913633032e.
---
 docs/changelog | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index d70cd72c..82941895 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,7 +7,6 @@ next:
 	* Type __apt_key: Use mktemp for unique temporary gpg home (Ander Punnar)
 	* Type __hostname: Silence grep exit 1 when os_version is not numeric (Ander Punnar)
 	* Type __ufw: Print error message to stderr (Daniel Tschada)
-	* Type __clean_path: Use __directory and __file types for better clarity (Ander Punnar)
 
 6.1.0: 2019-11-19
 	* Explorer hostname, type __hostname: Support more operating systems, rewrite type and hostname explorer (Dennis Camera)

From d1c3264cce6716f4950bef832c57923994457c31 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 26 Nov 2019 17:58:25 +0100
Subject: [PATCH 065/743] Revert "Merge branch '__clean_path_use_types' into
 'master'"

This reverts commit 073523be38742578a1bb47e030869a8f141517d3, reversing
changes made to 56435492a0e8a66c7437d93823e88f058d48f307.
---
 cdist/conf/type/__clean_path/explorer/list  |  6 ++--
 cdist/conf/type/__clean_path/gencode-remote | 26 +++++++++++++++-
 cdist/conf/type/__clean_path/manifest       | 33 ---------------------
 3 files changed, 27 insertions(+), 38 deletions(-)
 delete mode 100755 cdist/conf/type/__clean_path/manifest

diff --git a/cdist/conf/type/__clean_path/explorer/list b/cdist/conf/type/__clean_path/explorer/list
index 58d0b411..07d38127 100755
--- a/cdist/conf/type/__clean_path/explorer/list
+++ b/cdist/conf/type/__clean_path/explorer/list
@@ -29,9 +29,7 @@ then
     exclude="$( cat "$__object/parameter/exclude" )"
 
     find "$path" -mindepth 1 -maxdepth 1 -regex "$pattern" \
-        -and -not -regex "$exclude" \
-        \( -type d -printf '%p/\n' , -type f -print \)
+        -and -not -regex "$exclude"
 else
-    find "$path" -mindepth 1 -maxdepth 1 -regex "$pattern" \
-        \( -type d -printf '%p/\n' , -type f -print \)
+    find "$path" -mindepth 1 -maxdepth 1 -regex "$pattern"
 fi
diff --git a/cdist/conf/type/__clean_path/gencode-remote b/cdist/conf/type/__clean_path/gencode-remote
index 0157733f..998a70d8 100755
--- a/cdist/conf/type/__clean_path/gencode-remote
+++ b/cdist/conf/type/__clean_path/gencode-remote
@@ -18,7 +18,31 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-if [ -f "$__object/explorer/list" ] && [ -f "$__object/parameter/onchange" ]
+[ ! -s "$__object/explorer/list" ] && exit 0
+
+path="/$__object_id"
+
+pattern="$( cat "$__object/parameter/pattern" )"
+
+if [ -f "$__object/parameter/exclude" ]
+then
+    exclude="$( cat "$__object/parameter/exclude" )"
+
+    echo "find '$path' -mindepth 1 -maxdepth 1 -regex '$pattern'" \
+        "-and -not -regex '$exclude'" \
+        '-exec rm -rf {} \;'
+else
+    echo "find '$path' -mindepth 1 -maxdepth 1 -regex '$pattern'" \
+        '-exec rm -rf {} \;'
+fi
+
+while read -r f
+do
+    echo "removed '$f'" >> "$__messages_out"
+done \
+< "$__object/explorer/list"
+
+if [ -f "$__object/parameter/onchange" ]
 then
     cat "$__object/parameter/onchange"
 fi
diff --git a/cdist/conf/type/__clean_path/manifest b/cdist/conf/type/__clean_path/manifest
deleted file mode 100755
index 065ecb36..00000000
--- a/cdist/conf/type/__clean_path/manifest
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/sh -e
-#
-# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-[ ! -f "$__object/explorer/list" ] && exit 0
-
-for l in $( cat "$__object/explorer/list" )
-do
-    if echo "$l" | grep -Eq '/$'
-    then
-        __directory "$l" --state absent
-        echo "removed directory '$l'" >> "$__messages_out"
-    else
-        __file "$l" --state absent
-        echo "removed file '$l'" >> "$__messages_out"
-    fi
-done

From 3b5aa8654d2b6912bc26b98ffdecaaf5d4979d70 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 26 Nov 2019 19:24:15 +0100
Subject: [PATCH 066/743] Release 6.1.1

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 82941895..f6e9357c 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.1.1: 2019-11-26
 	* Type __xymon_config: Add parameters for ownership and generic rsync options (Thomas Eckert)
 	* Type __xymon_client: Add msgcache parameter to support passive clients (Thomas Eckert)
 	* Type __apt_key: Use mktemp for unique temporary gpg home (Ander Punnar)

From c01aa576de4e7fe9920ea7b116a5376145e6d433 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 27 Nov 2019 14:51:28 +0100
Subject: [PATCH 067/743] Add/keep empty directories

---
 cdist/test/cdist_type/fixtures/__not_deprecated/.keep             | 0
 .../cdist_type/fixtures/__without_deprecated_parameters/.keep     | 0
 cdist/test/inventory/fixtures/.keep                               | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 cdist/test/cdist_type/fixtures/__not_deprecated/.keep
 create mode 100644 cdist/test/cdist_type/fixtures/__without_deprecated_parameters/.keep
 create mode 100644 cdist/test/inventory/fixtures/.keep

diff --git a/cdist/test/cdist_type/fixtures/__not_deprecated/.keep b/cdist/test/cdist_type/fixtures/__not_deprecated/.keep
new file mode 100644
index 00000000..e69de29b
diff --git a/cdist/test/cdist_type/fixtures/__without_deprecated_parameters/.keep b/cdist/test/cdist_type/fixtures/__without_deprecated_parameters/.keep
new file mode 100644
index 00000000..e69de29b
diff --git a/cdist/test/inventory/fixtures/.keep b/cdist/test/inventory/fixtures/.keep
new file mode 100644
index 00000000..e69de29b

From 3d3b59ab87e72bf407e4c7f8880f19fc0e009f25 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 27 Nov 2019 14:59:25 +0100
Subject: [PATCH 068/743] Add pycodestyle ignores

---
 bin/build-helper  | 2 +-
 cdist/config.py   | 2 +-
 cdist/emulator.py | 2 +-
 setup.py          | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/bin/build-helper b/bin/build-helper
index 9a776491..7cfc4d55 100755
--- a/bin/build-helper
+++ b/bin/build-helper
@@ -405,7 +405,7 @@ eof
     ;;
 
     pycodestyle|pep8)
-        pycodestyle "${basedir}" "${basedir}/scripts/cdist" | less
+        pycodestyle "${basedir}" "${basedir}/scripts/cdist"
     ;;
 
     check-pycodestyle)
diff --git a/cdist/config.py b/cdist/config.py
index 26d07fc4..6c30db2e 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -368,7 +368,7 @@ class Config(object):
     def resolve_target_addresses(host, family):
         try:
             return ipaddr.resolve_target_addresses(host, family)
-        except:
+        except:  # noqa
             e = sys.exc_info()[1]
             raise cdist.Error(("Error resolving target addresses for host '{}'"
                                ": {}").format(host, e))
diff --git a/cdist/emulator.py b/cdist/emulator.py
index 3cf82f84..417f2cdd 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -44,7 +44,7 @@ class MissingRequiredEnvironmentVariableError(cdist.Error):
 class DefaultList(list):
     """Helper class to allow default values for optional_multiple parameters.
 
-       @see https://groups.google.com/forum/#!msg/comp.lang.python/sAUvkJEDpRc/RnRymrzJVDYJ
+       @see https://groups.google.com/forum/#!msg/comp.lang.python/sAUvkJEDpRc/RnRymrzJVDYJ  # noqa
     """
     def __copy__(self):
         return []
diff --git a/setup.py b/setup.py
index 2bb1e16d..7b000041 100644
--- a/setup.py
+++ b/setup.py
@@ -21,7 +21,7 @@ else:
         raise DistutilsError("Missing version file {}".format(version_file))
 
 
-import cdist
+import cdist  # noqa
 
 
 def data_finder(data_dir):
@@ -66,7 +66,7 @@ setup(
         "Development Status :: 6 - Mature",
         "Environment :: Console",
         "Intended Audience :: System Administrators",
-        "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+        "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",  # noqa
         "Operating System :: MacOS :: MacOS X",
         "Operating System :: POSIX",
         "Operating System :: POSIX :: BSD",

From f3bd439c43ca7fd6a3f2d5f2f58d4df141780935 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 27 Nov 2019 14:22:13 +0100
Subject: [PATCH 069/743] Add gitlab CI

---
 .gitlab-ci.yml | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 .gitlab-ci.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 00000000..44de6b0b
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,28 @@
+stages:
+    - test
+
+unit_tests:
+    stage: test
+    before_script:
+        - 'apk update'
+        - 'apk add python3'
+    script:
+        - ./bin/build-helper version
+        - ./bin/build-helper test
+
+pycodestyle:
+    stage: test
+    before_script:
+        - 'apk update'
+        - 'apk add python3 py3-pycodestyle'
+        - 'ln -f -s /usr/bin/pycodestyle-3 /usr/bin/pycodestyle'
+    script:
+        - ./bin/build-helper pycodestyle
+
+shellcheck:
+    stage: test
+    before_script:
+        - 'wget https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz'
+        - 'tar xf shellcheck-stable.linux.x86_64.tar.xz && mv shellcheck-stable/shellcheck /usr/bin/'
+    script:
+        - ./bin/build-helper shellcheck

From da274e5ef3e8c1eb47bb9fde003248711439fc8d Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 23 Nov 2019 00:25:51 +0100
Subject: [PATCH 070/743] Redefine/reimplement CDIST_ORDER_DEPENDENCY

CDIST_ORDER_DEPENDENCY now defines type order dependency context.
cdist (emulator) maintains global state variables, as files,
order_dep_state and typeorder_dep, and per object state variable,
as file, typeorder_dep.

If order_dep_state exists then this defines that order dependency is
turned on.
If order_dep_state does not exist then order dependency is turned off.

If order dependency is on then objects created after it is turned on are
recorded into:
    * global typeorder_dep, in case of init manifest
    * object's typeorder_dep, in case of type's manifest.

If order dependency is on then requirement is injected, where object
created before current, is read from:
    * global typeorder_dep, in case of init manifest
    * object's typeorder_dep, in case of type's manifest.

Every time order dependency is turned off, typeorder_dep files are
removed, which means that type order list is cleared, context is
cleaned.

In the end cdist cleans after itself, i.e. mentioned files are removed.

When running type manifest is finished typeorder_dep file is removed.
When running config finishes global typeorder_dep and order_dep_state
files are removed.

Global type order recording is untouched.
Furthermore, for completeness, type order is now recorded for each object
too.
---
 cdist/config.py                 |   4 ++
 cdist/core/cdist_object.py      |  10 +++
 cdist/core/manifest.py          |  14 ++++
 cdist/emulator.py               | 124 +++++++++++++++++++++++---------
 cdist/test/emulator/__init__.py |  43 +++++++++--
 5 files changed, 157 insertions(+), 38 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index 6c30db2e..97cc1da6 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -124,6 +124,7 @@ class Config(object):
         """Remove files and directories for the run"""
         if self.remove_remote_files_dirs:
             self._remove_remote_files_dirs()
+        self.manifest.cleanup()
 
     @staticmethod
     def hosts(source):
@@ -787,6 +788,9 @@ class Config(object):
         self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
         try:
             self.manifest.run_type_manifest(cdist_object)
+            self.log.trace("[ORDER_DEP] Removing order dep files for %s",
+                           cdist_object)
+            cdist_object.cleanup()
             cdist_object.state = core.CdistObject.STATE_PREPARED
         except cdist.Error as e:
             raise cdist.CdistObjectError(cdist_object, e)
diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index 237f0ddd..114a47e0 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -243,6 +243,16 @@ class CdistObject(object):
             lambda obj: os.path.join(obj.base_path, obj.code_local_path))
     code_remote = fsproperty.FileStringProperty(
             lambda obj: os.path.join(obj.base_path, obj.code_remote_path))
+    typeorder = fsproperty.FileListProperty(
+            lambda obj: os.path.join(obj.absolute_path, 'typeorder'))
+    typeorder_dep = fsproperty.FileListProperty(
+            lambda obj: os.path.join(obj.absolute_path, 'typeorder_dep'))
+
+    def cleanup(self):
+        try:
+            os.remove(os.path.join(self.absolute_path, 'typeorder_dep'))
+        except FileNotFoundError:
+            pass
 
     @property
     def exists(self):
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 07af0ef8..8aeaf860 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -96,6 +96,10 @@ class Manifest(object):
     """Executes cdist manifests.
 
     """
+
+    ORDER_DEP_STATE_NAME = 'order_dep_state'
+    TYPEORDER_DEP_NAME = 'typeorder_dep'
+
     def __init__(self, target_host, local, dry_run=False):
         self.target_host = target_host
         self.local = local
@@ -212,3 +216,13 @@ class Manifest(object):
                     type_manifest,
                     env=self.env_type_manifest(cdist_object),
                     message_prefix=message_prefix)
+
+    def cleanup(self):
+        def _rm_file(fname):
+            try:
+                self.log.trace("[ORDER_DEP] Removing %s", fname)
+                os.remove(os.path.join(self.local.base_path, fname))
+            except FileNotFoundError:
+                pass
+        _rm_file(Manifest.ORDER_DEP_STATE_NAME)
+        _rm_file(Manifest.TYPEORDER_DEP_NAME)
diff --git a/cdist/emulator.py b/cdist/emulator.py
index 417f2cdd..4800e2a3 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -29,6 +29,7 @@ import sys
 import cdist
 from cdist import core
 from cdist import flock
+from cdist.core.manifest import Manifest
 
 
 class MissingRequiredEnvironmentVariableError(cdist.Error):
@@ -82,6 +83,11 @@ class Emulator(object):
         self.object_base_path = os.path.join(self.global_path, "object")
         self.typeorder_path = os.path.join(self.global_path, "typeorder")
 
+        self.typeorder_dep_path = os.path.join(self.global_path,
+                                               Manifest.TYPEORDER_DEP_NAME)
+        self.order_dep_state_path = os.path.join(self.global_path,
+                                                 Manifest.ORDER_DEP_STATE_NAME)
+
         self.type_name = os.path.basename(argv[0])
         self.cdist_type = core.CdistType(self.type_base_path, self.type_name)
 
@@ -206,6 +212,14 @@ class Emulator(object):
         return params
 
     def setup_object(self):
+        # CDIST_ORDER_DEPENDENCY state
+        order_dep_on = self._order_dep_on()
+        order_dep_defined = "CDIST_ORDER_DEPENDENCY" in self.env
+        if not order_dep_defined and order_dep_on:
+            self._set_order_dep_state_off()
+        if order_dep_defined and not order_dep_on:
+            self._set_order_dep_state_on()
+
         # Create object with given parameters
         self.parameters = {}
         for key, value in vars(self.args).items():
@@ -237,6 +251,20 @@ class Emulator(object):
             # record the created object in typeorder file
             with open(self.typeorder_path, 'a') as typeorderfile:
                 print(self.cdist_object.name, file=typeorderfile)
+            # record the created object in parent object typeorder file
+            __object_name = self.env.get('__object_name', None)
+            depname = self.cdist_object.name
+            if __object_name:
+                parent = self.cdist_object.object_from_name(__object_name)
+                parent.typeorder.append(self.cdist_object.name)
+                if self._order_dep_on():
+                    self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
+                                    ' for %s'), depname, parent.name)
+                    parent.typeorder_dep.append(depname)
+            elif self._order_dep_on():
+                self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
+                               depname)
+                self._add_typeorder_dep(depname)
 
         # Record / Append source
         self.cdist_object.source.append(self.object_source)
@@ -293,45 +321,73 @@ class Emulator(object):
 
         return cdist_object.name
 
+    def _order_dep_on(self):
+        return os.path.exists(self.order_dep_state_path)
+
+    def _set_order_dep_state_on(self):
+        self.log.trace('[ORDER_DEP] Setting order dep state on')
+        with open(self.order_dep_state_path, 'w'):
+            pass
+
+    def _set_order_dep_state_off(self):
+        self.log.trace('[ORDER_DEP] Setting order dep state off')
+        # remove order dep state file
+        try:
+            os.remove(self.order_dep_state_path)
+        except FileNotFoundError:
+            pass
+        # remove typeorder dep file
+        try:
+            os.remove(self.typeorder_dep_path)
+        except FileNotFoundError:
+            pass
+
+    def _add_typeorder_dep(self, name):
+        with open(self.typeorder_dep_path, 'a') as f:
+            print(name, file=f)
+
+    def _read_typeorder_dep(self):
+        try:
+            with open(self.typeorder_dep_path, 'r') as f:
+                return f.readlines()
+        except FileNotFoundError:
+            return []
+
     def record_requirements(self):
         """Record requirements."""
 
+        order_dep_on = self._order_dep_on()
+
         # Inject the predecessor, but not if its an override
         # (this would leed to an circular dependency)
-        if ("CDIST_ORDER_DEPENDENCY" in self.env and
-                'CDIST_OVERRIDE' not in self.env):
-            # load object name created befor this one from typeorder file ...
-            with open(self.typeorder_path, 'r') as typecreationfile:
-                typecreationorder = typecreationfile.readlines()
-                # get the type created before this one ...
-                try:
-                    lastcreatedtype = typecreationorder[-2].strip()
-                    # __object_name is the name of the object whose type
-                    # manifest is currently executed
-                    __object_name = self.env.get('__object_name', None)
-                    if lastcreatedtype == __object_name:
-                        self.log.debug(("Not injecting require for "
-                                        "CDIST_ORDER_DEPENDENCY: %s for %s,"
-                                        " %s's type manifest is currently"
-                                        " being executed"),
-                                       lastcreatedtype,
-                                       self.cdist_object.name,
-                                       lastcreatedtype)
-                    else:
-                        if 'require' in self.env:
-                            appendix = " " + lastcreatedtype
-                            if appendix not in self.env['require']:
-                                self.env['require'] += appendix
-                        else:
-                            self.env['require'] = lastcreatedtype
-                        self.log.debug(("Injecting require for "
-                                        "CDIST_ORDER_DEPENDENCY: %s for %s"),
-                                       lastcreatedtype,
-                                       self.cdist_object.name)
-                except IndexError:
-                    # if no second last line, we are on the first type,
-                    # so do not set a requirement
-                    pass
+        if (order_dep_on and 'CDIST_OVERRIDE' not in self.env):
+            try:
+                # __object_name is the name of the object whose type
+                # manifest is currently executed
+                __object_name = self.env.get('__object_name', None)
+                # load object name created befor this one from typeorder
+                # dep file
+                if __object_name:
+                    parent = self.cdist_object.object_from_name(
+                        __object_name)
+                    typeorder = parent.typeorder_dep
+                else:
+                    typeorder = self._read_typeorder_dep()
+                # get the type created before this one
+                lastcreatedtype = typeorder[-2].strip()
+                if 'require' in self.env:
+                    if lastcreatedtype not in self.env['require']:
+                        self.env['require'] += " " + lastcreatedtype
+                else:
+                    self.env['require'] = lastcreatedtype
+                self.log.debug(("Injecting require for "
+                                "CDIST_ORDER_DEPENDENCY: %s for %s"),
+                               lastcreatedtype,
+                               self.cdist_object.name)
+            except IndexError:
+                # if no second last line, we are on the first type,
+                # so do not set a requirement
+                pass
 
         reqs = set()
         if "require" in self.env:
diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py
index 5691093c..e375676c 100644
--- a/cdist/test/emulator/__init__.py
+++ b/cdist/test/emulator/__init__.py
@@ -24,8 +24,6 @@
 import io
 import os
 import shutil
-import string
-import filecmp
 import random
 import logging
 
@@ -34,7 +32,6 @@ from cdist import test
 from cdist.exec import local
 from cdist import emulator
 from cdist import core
-from cdist import config
 
 import os.path as op
 my_dir = op.abspath(op.dirname(__file__))
@@ -115,7 +112,7 @@ class EmulatorTestCase(test.CdistTestCase):
     def test_requirement_pattern(self):
         argv = ['__file', '/tmp/foobar']
         self.env['require'] = '__file/etc/*'
-        emu = emulator.Emulator(argv, env=self.env)
+        emulator.Emulator(argv, env=self.env)
         # if we get here all is fine
 
     def test_loglevel(self):
@@ -172,6 +169,44 @@ class EmulatorTestCase(test.CdistTestCase):
         self.assertEqual(list(file_object.requirements), ['__planet/mars'])
         # if we get here all is fine
 
+    def test_order_dependency_context(self):
+        test_seq = ('A', True, 'B', 'C', 'D', False, 'E', 'F', True, 'G',
+                    'H', False, 'I', )
+        expected_requirements = {
+            'C': set(('__planet/B', )),
+            'D': set(('__planet/C', )),
+            'H': set(('__planet/G', )),
+        }
+        # Ensure env var is not in env
+        if 'CDIST_ORDER_DEPENDENCY' in self.env:
+            del self.env['CDIST_ORDER_DEPENDENCY']
+
+        for x in test_seq:
+            if isinstance(x, str):
+                # Clear because of order dep injection
+                # In real world, this is not shared over instances
+                if 'require' in self.env:
+                    del self.env['require']
+                argv = ['__planet', x]
+                emu = emulator.Emulator(argv, env=self.env)
+                emu.run()
+            elif isinstance(x, bool):
+                if x:
+                    self.env['CDIST_ORDER_DEPENDENCY'] = 'on'
+                elif 'CDIST_ORDER_DEPENDENCY' in self.env:
+                    del self.env['CDIST_ORDER_DEPENDENCY']
+        cdist_type = core.CdistType(self.local.type_path, '__planet')
+        for x in test_seq:
+            if isinstance(x, str):
+                obj = core.CdistObject(cdist_type, self.local.object_path,
+                                       self.local.object_marker_name, x)
+                reqs = set(obj.requirements)
+                if x in expected_requirements:
+                    self.assertEqual(reqs, expected_requirements[x])
+                else:
+                    self.assertTrue(len(reqs) == 0)
+        # if we get here all is fine
+
 
 class EmulatorConflictingRequirementsTestCase(test.CdistTestCase):
 

From 332f5dcff9b27e110c60bc94aa9c2f3c40f44121 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 23 Nov 2019 00:25:51 +0100
Subject: [PATCH 071/743] Redefine/reimplement CDIST_ORDER_DEPENDENCY

Update documentation.
---
 docs/src/cdist-best-practice.rst |  91 +----------------------
 docs/src/cdist-manifest.rst      | 121 ++++++++++++++++++++++++++++++-
 docs/src/cdist-reference.rst.sh  |   2 +-
 docs/src/man1/cdist.rst          |   2 +
 4 files changed, 126 insertions(+), 90 deletions(-)

diff --git a/docs/src/cdist-best-practice.rst b/docs/src/cdist-best-practice.rst
index a91f2cc0..39ec453e 100644
--- a/docs/src/cdist-best-practice.rst
+++ b/docs/src/cdist-best-practice.rst
@@ -226,8 +226,8 @@ and also to store all important files in one
 repository.
 
 
-Perils of CDIST_ORDER_DEPENDENCY
---------------------------------
+Notes on CDIST_ORDER_DEPENDENCY
+-------------------------------
 With CDIST_ORDER_DEPENDENCY all types are executed in the order in which they
 are created in the manifest. The current created object automatically depends
 on the previously created object.
@@ -235,96 +235,11 @@ on the previously created object.
 It essentially helps you to build up blocks of code that build upon each other
 (like first creating the directory xyz than the file below the directory).
 
-This can be helpful, but it can also be the source of *evil*.
-
-
-CDIST_ORDER_DEPENDENCY easily causes unobvious dependency cycles
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Let's see an example. Suppose you have special init manifest where among other
-things you are assuring that remote host has packages `sudo` and `curl`
-installed.
-
-**init1**
-
-.. code-block:: sh
-
-   CDIST_ORDER_DEPENDENCY=1
-   export CDIST_ORDER_DEPENDENCY
-
-   for p in sudo curl
-   do
-      __package "${p}"
-   done
-
-Then you have some other special init manifest where among other things you are
-assuring `sudo` package is installed.
-
-**init2**
-
-.. code-block:: sh
-
-   CDIST_ORDER_DEPENDENCY=1
-   export CDIST_ORDER_DEPENDENCY
-
-   __package sudo
-
-Then you have third init manifest where you combine those two init manifests,
-by including them:
-
-**init**
-
-.. code-block:: sh
-
-   sh -e "$__manifest/init1"
-   sh -e "$__manifest/init2"
-
-The resulting init manifest is then equal to:
-
-.. code-block:: sh
-
-   CDIST_ORDER_DEPENDENCY=1
-   export CDIST_ORDER_DEPENDENCY
-
-   for p in sudo curl
-   do
-      __package "${p}"
-   done
-
-   CDIST_ORDER_DEPENDENCY=1
-   export CDIST_ORDER_DEPENDENCY
-
-   __package sudo
-
-In the end you get the following dependencies:
-
-* `__package/curl` depends on `__package/sudo`
-* `__package/sudo` depends on `__package/curl`
-
-And here you have a circular dependency!
-
-In the real world manifest can be quite complex, dependencies can become
-complicated and circual dependencies are not so obvious. Resolving it can
-become cumbersome.
-
-**Practical solution?**
-
-Instead of managing complex init manifests you can write custom types.
-Each custom type can do one thing, it has well defined dependencies that will
-not leak into init manifest. In custom type you can also add special explorers
-and gencode.
-
-Then, in init manifest you combine your complex types. It is:  
-
-* cleaner
-* easier to follow
-* easier to maintain
-* easier to debug.
+This can be helpful, but one must be aware of its side effects.
 
 
 CDIST_ORDER_DEPENDENCY kills parallelization
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
 Suppose you have defined CDIST_ORDER_DEPENDENCY and then, among other things,
 you specify creation of three, by nature independent, files.
 
diff --git a/docs/src/cdist-manifest.rst b/docs/src/cdist-manifest.rst
index 4dd3e74b..5dbca479 100644
--- a/docs/src/cdist-manifest.rst
+++ b/docs/src/cdist-manifest.rst
@@ -163,7 +163,126 @@ automatically depends on the previously created object.
 It essentially helps you to build up blocks of code that build upon each other
 (like first creating the directory xyz than the file below the directory).
 
-Read also about `perils of CDIST_ORDER_DEPENDENCY <cdist-best-practice.html#perils-of-cdist-order-dependency>`_.
+Read also about `notes on CDIST_ORDER_DEPENDENCY <cdist-best-practice.html#notes-on-cdist-order-dependency>`_.
+
+In version 6.2.0 semantic CDIST_ORDER_DEPENDENCY is finally fixed and well defined.
+
+CDIST_ORDER_DEPENDENCY defines type order dependency context. Order dependency context
+starts when CDIST_ORDER_DEPENDENCY is set, and ends when it is unset. After each
+manifest execution finishes, any existing order dependency context is automatically
+unset. This ensures that CDIST_ORDER_DEPENDENCY is valid within the manifest where it
+is used. When order dependency context is defined then cdist executes types in the
+order in which they are created in the manifest inside order dependency context.
+
+Sometimes the best way to see how something works is to see examples.
+
+Suppose you have defined **initial manifest**:
+
+.. code-block:: sh
+
+    __cycle1 cycle1
+    export CDIST_ORDER_DEPENDENCY=1
+    __cycle2 cycle2
+    __cycle3 cycle3
+
+with types **__cycle1**:
+
+.. code-block:: sh
+
+    export CDIST_ORDER_DEPENDENCY=1
+    __file /tmp/cycle11
+    __file /tmp/cycle12
+    __file /tmp/cycle13
+
+**__cycle2**:
+
+.. code-block:: sh
+
+    __file /tmp/cycle21
+    export CDIST_ORDER_DEPENDENCY=1
+    __file /tmp/cycle22
+    __file /tmp/cycle23
+    unset CDIST_ORDER_DEPENDENCY
+    __file /tmp/cycle24
+
+**__cycle3**:
+
+.. code-block:: sh
+
+    __file /tmp/cycle31
+    __file /tmp/cycle32
+    export CDIST_ORDER_DEPENDENCY=1
+    __file /tmp/cycle33
+    __file /tmp/cycle34
+
+For the above config, cdist results in the following expected *dependency graph*
+(type *__cycleX* is shown as *cX*, *__file/tmp/cycleXY* is shown as *fcXY*):
+
+::
+
+    c1---->fc11
+    |      /\
+    |       |
+    +----->fc12
+    |      /\
+    |       |
+    +----->fc13
+
+    c2--+--->fc21
+    /\  |
+    |   |
+    |   +----->fc22
+    |   |      /\
+    |   |       |
+    |   +----->fc23
+    |   |
+    |   |
+    |   +----->fc24
+    |
+    |
+    c3---->fc31
+    |
+    |
+    +----->fc32
+    |
+    |
+    +----->fc33
+    |      /\
+    |       |
+    +----->fc34
+
+Before version 6.2.0 the above configuration would result in cycle:
+
+::
+
+    ERROR: 185.203.112.26: Cycle detected in object dependencies:
+    __file/tmp/cycle11 -> __cycle3/cycle3 -> __cycle2/cycle2 -> __cycle1/cycle1 -> __file/tmp/cycle11!
+
+The following manifest shows an example for order dependency contexts:
+
+.. code-block:: sh
+
+    __file /tmp/fileA
+    export CDIST_ORDER_DEPENDENCY=1
+    __file /tmp/fileB
+    __file /tmp/fileC
+    __file /tmp/fileD
+    unset CDIST_ORDER_DEPENDENCY
+    __file /tmp/fileE
+    __file /tmp/fileF
+    export CDIST_ORDER_DEPENDENCY=1
+    __file /tmp/fileG
+    __file /tmp/fileH
+    unset CDIST_ORDER_DEPENDENCY
+    __file /tmp/fileI
+
+This means:
+
+* C depends on B
+* D depends on C
+* H depends on G
+
+and there are no other dependencies from this manifest.
 
 
 Overrides
diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh
index 3ab12fe2..e77d98f6 100755
--- a/docs/src/cdist-reference.rst.sh
+++ b/docs/src/cdist-reference.rst.sh
@@ -330,7 +330,7 @@ CDIST_OVERRIDE
 
 CDIST_ORDER_DEPENDENCY
     Create dependencies based on the execution order (see  \`cdist manifest <cdist-manifest.html>\`_).
-    Read also about \`perils of CDIST_ORDER_DEPENDENCY <cdist-best-practice.html#perils-of-cdist-order-dependency>\`_.
+    Note that in version 6.2.0 semantic of this processing mode is finally fixed and well defined.
 
 CDIST_REMOTE_EXEC
     Use this command for remote execution (should behave like ssh).
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 7f368e68..f40491e9 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -827,6 +827,8 @@ CDIST_OVERRIDE
 
 CDIST_ORDER_DEPENDENCY
     Create dependencies based on the execution order.
+    Note that in version 6.2.0 semantic of this processing mode is
+    finally fixed and well defined.
 
 CDIST_REMOTE_EXEC
     Use this command for remote execution (should behave like ssh).

From b3012b99114f586da045e37a2ac74985c33bcc82 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 29 Nov 2019 13:56:32 +0100
Subject: [PATCH 072/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index f6e9357c..175d9b61 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak)
+
 6.1.1: 2019-11-26
 	* Type __xymon_config: Add parameters for ownership and generic rsync options (Thomas Eckert)
 	* Type __xymon_client: Add msgcache parameter to support passive clients (Thomas Eckert)

From a1f33ca8ebbeb6faae1b7f14af7fb83700a329e4 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 30 Nov 2019 23:11:36 +0100
Subject: [PATCH 073/743] Release 6.2.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 175d9b61..09e30e79 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.2.0: 2019-11-30
 	* Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak)
 
 6.1.1: 2019-11-26

From 648cdf8e29c886652c0ea20599e7b17370ed81de Mon Sep 17 00:00:00 2001
From: Darko Poljak <darko.poljak@ungleich.ch>
Date: Sun, 1 Dec 2019 14:23:02 +0100
Subject: [PATCH 074/743] Add LICENSE

---
 LICENSE | 674 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 674 insertions(+)
 create mode 100644 LICENSE

diff --git a/LICENSE b/LICENSE
new file mode 100644
index 00000000..14682ad6
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    cdist
+    Copyright (C) 2019  ungleich-public
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    cdist  Copyright (C) 2019  ungleich-public
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<http://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.

From 8c0ee28b9940b765e48e839008b8f3765567da8e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 2 Dec 2019 12:35:43 +0100
Subject: [PATCH 075/743] gitlab CI runner should have necessary tools

---
 .gitlab-ci.yml | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 44de6b0b..1cc17995 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,26 +3,16 @@ stages:
 
 unit_tests:
     stage: test
-    before_script:
-        - 'apk update'
-        - 'apk add python3'
     script:
         - ./bin/build-helper version
         - ./bin/build-helper test
 
 pycodestyle:
     stage: test
-    before_script:
-        - 'apk update'
-        - 'apk add python3 py3-pycodestyle'
-        - 'ln -f -s /usr/bin/pycodestyle-3 /usr/bin/pycodestyle'
     script:
         - ./bin/build-helper pycodestyle
 
 shellcheck:
     stage: test
-    before_script:
-        - 'wget https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz'
-        - 'tar xf shellcheck-stable.linux.x86_64.tar.xz && mv shellcheck-stable/shellcheck /usr/bin/'
     script:
         - ./bin/build-helper shellcheck

From d25c72e678f8d4fb311a9800a3097e1bf6e9bf16 Mon Sep 17 00:00:00 2001
From: Dominique Roux <dominique.roux4@gmail.com>
Date: Tue, 3 Dec 2019 10:13:46 +0100
Subject: [PATCH 076/743] [PACKAGE-UPDATE-INDEX]: Bugfix of the alpine part of
 the __package_update_index type

---
 cdist/conf/type/__package_update_index/gencode-remote | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote
index 9b2ecba2..6c51cbed 100755
--- a/cdist/conf/type/__package_update_index/gencode-remote
+++ b/cdist/conf/type/__package_update_index/gencode-remote
@@ -47,9 +47,9 @@ case "$type" in
         echo "pacman --noprogressbar --sync --refresh"
         echo "pacman package database synced (age was: $currage)" >> "$__messages_out"
         ;;
-    alpine)
+    apk)
         echo "apk update"
-        echo "apk package database updated."
+        echo "apk package database updated." >>"$__messages_out"
         ;;
     *)
         echo "Don't know how to manage packages for type: $type" >&2

From 320f962e1df7f27909e6157e8dfc6033706c8ddd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 4 Dec 2019 22:27:58 +0100
Subject: [PATCH 077/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 09e30e79..8bdee928 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __package_update_index: Fix Alpine part (Dominique Roux)
+
 6.2.0: 2019-11-30
 	* Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak)
 

From 4f21bf534dd91c7395845beb08715ad6c84bafef Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Thu, 5 Dec 2019 23:14:27 +0100
Subject: [PATCH 078/743] [docs] PreOS: English nitpicking.

---
 cdist/argparse.py        |  2 +-
 docs/src/cdist-preos.rst | 70 ++++++++++++++++++++--------------------
 2 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index ca69cdae..ed94d266 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -424,7 +424,7 @@ def get_parsers():
     parser['inventory'].set_defaults(
             func=cdist.inventory.Inventory.commandline)
 
-    # PreOs
+    # PreOS
     parser['preos'] = parser['sub'].add_parser('preos', add_help=False)
 
     # Shell
diff --git a/docs/src/cdist-preos.rst b/docs/src/cdist-preos.rst
index e85af2de..2f102fef 100644
--- a/docs/src/cdist-preos.rst
+++ b/docs/src/cdist-preos.rst
@@ -4,9 +4,9 @@ PreOS
 Description
 -----------
 With cdist you can install and configure new machines. You can use cdist to
-create PreOS, minimal OS which purpose is to boot new machine.
-After PreOS is booted machine is ready for installing desired OS and
-then it is ready for configuration.
+create PreOS, minimal OS whose purpose is to boot a new machine.
+After PreOS is booted, the machine is ready for installing the desired OS and
+afterwards it is ready for configuration.
 
 PreOS creation
 --------------
@@ -15,62 +15,62 @@ Currently supported PreOS-es include:
 
 * debian
 * ubuntu
-* devuan.
+* devuan
 
-PreOS is created using cdist preos command. preos command has subcommands that
-create the desired PreOS.
+PreOS is created using the ``cdist preos`` command.
+This command has subcommands that determine the desired PreOS.
 
-For example, to create ubuntu PreOS:
+For example, to create an ubuntu PreOS:
 
 .. code-block:: sh
 
     $ cdist preos ubuntu /preos/preos-ubuntu -B -C \
         -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu
 
-For more info about available options see cdist manual page.
+For more info about the available options see the cdist manual page.
 
-This will bootstrap (``-B``) ubuntu PreOS in ``/preos/preos-ubuntu`` directory, it
-will be configured (``-C``) using default built-in initial manifest and with
-specified ssh authorized key (``-k``).
-After bootstrapping and configuration PXE
-boot directory will be created (``-p``) in ``/preos/pxe-ubuntu``.
+This will bootstrap (``-B``) ubuntu PreOS in the ``/preos/preos-ubuntu``
+directory, it will be configured (``-C``) using default the built-in initial
+manifest and with specified ssh authorized key (``-k``).
+After bootstrapping and configuration, the PXE boot directory will be
+created (``-p``) in ``/preos/pxe-ubuntu``.
 
-After PreOS is created new machines can be booted using created PXE (after
-proper dhcp, tftp setting).
+After PreOS is created, new machines can be booted using the created PXE
+(after proper dhcp and tftp settings).
 
-Since PreOS is configured with ssh authorized key it can be accessed throguh
+Since PreOS is configured with ssh authorized key it can be accessed through
 ssh, i.e. it can be further installed and configured with cdist.
 
-Implementing new PreOS sub-command
-----------------------------------
+Implementing a new PreOS sub-command
+------------------------------------
 preos command is implemented as a plugin system. This plugin system scans for
-preos subcommands in ``cdist/preos/`` distribution directory and also in
+preos subcommands in the ``cdist/preos/`` distribution directory and also in
 ``~/.cdist/preos/`` directory if it exists.
 
 preos subcommand is a module or a class that satisfies the following:
 
-* it has attribute ``_cdist_preos`` set to ``True``
-* it has function/method ``commandline``.
+* it has the attribute ``_cdist_preos`` set to ``True``
+* it defines a function/method ``commandline``.
 
-For a module based preos subcommand ``commandline`` function accepts a module
-object as its first argument and the list of command line
+For a module-based preos subcommand, the ``commandline`` function accepts a
+module object as its first argument and the list of command line
 arguments (``sys.argv[2:]``).
 
-For a class preos subcommand ``commandline`` method should be staticmethod and
-it accepts a class object as its first argument and the list of command line
-arguments(``sys.argv[2:]``).
+For a class-based preos subcommand ``commandline`` method should be
+static-method and must accept a class as its first argument and the
+list of command line arguments (``sys.argv[2:]``).
 
 If preos scanning finds a module/class that has ``_cdist_preos`` set
-to ``True`` and it has function/method ``commandline`` then this module/class is
+to ``True`` and a function/method ``commandline`` then this module/class is
 registered to preos subcommands. The name of the command is set to ``_preos_name``
-attribute if it exists, otherwise it is set to the module/class name, lowercase.
-When registered preos subcommand is specified as preos command then ``commandline``
-will be called with first argument set to module/class object and second argument
-set to ``sys.argv[2:]``.
+attribute if defined in the module/class, defaulting to the module/class name in lowercase.
+When a registered preos subcommand is specified, ``commandline``
+will be called with the first argument set to module/class and the second
+argument set to ``sys.argv[2:]``.
 
-Example writing new dummy preos sub-command
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Module based preos:
+Example of writing new dummy preos sub-command
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Module-based preos:
 ^^^^^^^^^^^^^^^^^^^
 
 #. Create directory ``~/.cdist/preos/`` if it does not exist
@@ -134,4 +134,4 @@ When you try to run this new preos you will get:
     FreeBSD dummy preos: []
 
 In the ``commandline`` function/method you have all the freedom to actually create
-PreOS.
+a PreOS.

From f22349ce8a6571601478a76bfe3b68bb574154ba Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 6 Dec 2019 08:56:57 +0100
Subject: [PATCH 079/743] Fix documentation for install types

Fixes gitlab #790.
---
 cdist/conf/type/__install_directory/man.rst | 102 +++++++++++++++++++-
 cdist/conf/type/__install_file/man.rst      |  14 ++-
 docs/changelog                              |   1 +
 3 files changed, 115 insertions(+), 2 deletions(-)
 mode change 120000 => 100644 cdist/conf/type/__install_directory/man.rst

diff --git a/cdist/conf/type/__install_directory/man.rst b/cdist/conf/type/__install_directory/man.rst
deleted file mode 120000
index 1ad7fa84..00000000
--- a/cdist/conf/type/__install_directory/man.rst
+++ /dev/null
@@ -1 +0,0 @@
-../__directory/man.rst
\ No newline at end of file
diff --git a/cdist/conf/type/__install_directory/man.rst b/cdist/conf/type/__install_directory/man.rst
new file mode 100644
index 00000000..c402cbad
--- /dev/null
+++ b/cdist/conf/type/__install_directory/man.rst
@@ -0,0 +1,101 @@
+cdist-type__install_directory(7)
+================================
+
+NAME
+----
+cdist-type__install_directory - Manage a directory with install command
+
+
+DESCRIPTION
+-----------
+This cdist type allows you to create or remove directories on the target.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+   'present' or 'absent', defaults to 'present'
+
+group
+   Group to chgrp to.
+
+mode
+   Unix permissions, suitable for chmod.
+
+owner
+   User to chown to.
+
+
+BOOLEAN PARAMETERS
+------------------
+parents
+   Whether to create parents as well (mkdir -p behaviour).
+   Warning: all intermediate directory permissions default
+   to whatever mkdir -p does. 
+
+   Usually this means root:root, 0700.
+
+recursive
+   If supplied the chgrp and chown call will run recursively.
+   This does *not* influence the behaviour of chmod.
+
+MESSAGES
+--------
+chgrp <group>
+    Changed group membership
+chown <owner>
+    Changed owner
+chmod <mode>
+    Changed mode
+create
+    Empty directory was created
+remove
+    Directory exists, but state is absent, directory will be removed by generated code.
+remove non directory
+    Something other than a directory with the same name exists and was removed prior to create.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # A silly example
+    __install_directory /tmp/foobar
+
+    # Remove a directory
+    __install_directory /tmp/foobar --state absent
+
+    # Ensure /etc exists correctly
+    __install_directory /etc --owner root --group root --mode 0755
+
+    # Create nfs service directory, including parents
+    __install_directory /home/services/nfs --parents
+
+    # Change permissions recursively
+    __install_directory /home/services --recursive --owner root --group root
+
+    # Setup a temp directory
+    __install_directory /local --mode 1777
+
+    # Take it all
+    __install_directory /home/services/kvm --recursive --parents \
+        --owner root --group root --mode 0755 --state present
+
+
+AUTHORS
+-------
+Nico Schottelius <nico-cdist--@--schottelius.org>
+
+
+COPYING
+-------
+Copyright \(C) 2011 Nico Schottelius. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__install_file/man.rst b/cdist/conf/type/__install_file/man.rst
index c5409167..977ed77c 100644
--- a/cdist/conf/type/__install_file/man.rst
+++ b/cdist/conf/type/__install_file/man.rst
@@ -23,6 +23,10 @@ symlink
 directory
   replace it with the source file
 
+One exception is that when state is pre-exists, an error is raised if
+the file would have been created otherwise (e.g. it is not present or
+not a regular file).
+
 In any case, make sure that the file attributes are as specified.
 
 
@@ -33,7 +37,7 @@ None.
 OPTIONAL PARAMETERS
 -------------------
 state
-   'present', 'absent' or 'exists', defaults to 'present' where:
+   'present', 'absent', 'exists' or 'pre-exists', defaults to 'present' where:
 
    present
       the file is exactly the one from source
@@ -41,6 +45,9 @@ state
       the file does not exist
    exists
       the file from source but only if it doesn't already exist
+   pre-exists
+      check that the file exists and is a regular file, but do not
+      create or modify it
 
 group
    Group to chgrp to.
@@ -56,6 +63,9 @@ source
    If not supplied, an empty file or directory will be created.
    If source is '-' (dash), take what was written to stdin as the file content.
 
+onchange
+   The code to run if file is modified.
+
 MESSAGES
 --------
 chgrp <group>
@@ -93,6 +103,8 @@ EXAMPLES
     __install_file /home/frodo/.bashrc --source "/etc/skel/.bashrc" \
        --state exists \
        --owner frodo --mode 0600
+    # Check that the file is present, show an error when it is not
+    __install_file /etc/somefile --state pre-exists
     # Take file content from stdin
     __install_file /tmp/whatever --owner root --group root --mode 644 --source - << DONE
         Here goes the content for /tmp/whatever
diff --git a/docs/changelog b/docs/changelog
index 8bdee928..ed993609 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __package_update_index: Fix Alpine part (Dominique Roux)
+	* Documentation: Fix man pages for install types (Darko Poljak)
 
 6.2.0: 2019-11-30
 	* Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak)

From 90e85d81285a1ed60085b4df0a1618e4aeaa8c6f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 6 Dec 2019 09:37:21 +0100
Subject: [PATCH 080/743] Embed config skeleton instead of rewriting it

Resolves #796.
---
 .gitignore                       |  1 +
 Makefile                         |  8 +++-
 docs/changelog                   |  1 +
 docs/src/cdist-configuration.rst | 70 ++------------------------------
 4 files changed, 13 insertions(+), 67 deletions(-)

diff --git a/.gitignore b/.gitignore
index ed8b453a..fb831051 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,7 @@ docs/src/man1/*.1
 docs/src/man7/*.7
 docs/src/man7/cdist-type__*.rst
 docs/src/cdist-reference.rst
+docs/src/cdist.cfg.skeleton
 
 # Ignore cdist cache for version control
 /cache/
diff --git a/Makefile b/Makefile
index fa3327d1..e6a2a28e 100644
--- a/Makefile
+++ b/Makefile
@@ -63,6 +63,11 @@ DOCSREFSH=$(DOCS_SRC_DIR)/cdist-reference.rst.sh
 $(DOCSREF): $(DOCSREFSH)
 	$(DOCSREFSH)
 
+DOCSCFGSKEL=./configuration/cdist.cfg.skeleton
+
+configskel: $(DOCSCFGSKEL)
+	cp -f "$(DOCSCFGSKEL)" "$(DOCS_SRC_DIR)/"
+
 version:
 	@[ -f "cdist/version.py" ] || { \
 		printf "Missing 'cdist/version.py', please generate it first.\n" && exit 1; \
@@ -72,7 +77,7 @@ version:
 man: version $(MANTYPES) $(DOCSREF)
 	$(SPHINXM)
 
-html: version $(MANTYPES) $(DOCSREF)
+html: version configskel $(MANTYPES) $(DOCSREF)
 	$(SPHINXH)
 
 docs: man html
@@ -114,6 +119,7 @@ speeches: $(SPEECHES)
 #
 clean: docs-clean
 	rm -f $(DOCS_SRC_DIR)/cdist-reference.rst
+	rm -f $(DOCS_SRC_DIR)/cdist.cfg.skeleton
 
 	find "$(DOCS_SRC_DIR)" -mindepth 2 -type l \
 	| xargs rm -f
diff --git a/docs/changelog b/docs/changelog
index ed993609..ca33d7bd 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __package_update_index: Fix Alpine part (Dominique Roux)
 	* Documentation: Fix man pages for install types (Darko Poljak)
+	* Documentation: Embed config skeleton instead of rewriting it (Darko Poljak)
 
 6.2.0: 2019-11-30
 	* Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak)
diff --git a/docs/src/cdist-configuration.rst b/docs/src/cdist-configuration.rst
index 4c9b4d33..706ed761 100644
--- a/docs/src/cdist-configuration.rst
+++ b/docs/src/cdist-configuration.rst
@@ -31,73 +31,11 @@ can be used.
 
 Config file format
 ------------------
+
 cdist configuration file is in the INI file format. Currently it supports
 only [GLOBAL] section.
-The possible keywords and their meanings are as follows:
 
-:strong:`archiving`
-    Use specified archiving. Valid values include:
-    'none', 'tar', 'tgz', 'tbz2' and 'txz'.
+Here you can find configuration file skeleton:
 
-:strong:`beta`
-    Enable beta functionality. It recognizes boolean values from
-    'yes'/'no', 'on'/'off', 'true'/'false' and '1'/'0'.
-
-:strong:`cache_path_pattern`
-    Specify cache path pattern.
-
-:strong:`conf_dir`
-    List of configuration directories separated with the character conventionally
-    used by the operating system to separate search path components (as in PATH),
-    such as ':' for POSIX or ';' for Windows.
-    If also specified at command line then values from command line are
-    appended to this value.
-
-:strong:`init_manifest`
-    Specify default initial manifest.
-
-:strong:`inventory_dir`
-    Specify inventory directory.
-
-:strong:`jobs`
-    Specify number of jobs for parallel processing. If -1 then the default,
-    number of CPU's in the system is used. If 0 then parallel processing in
-    jobs is disabled. If set to positive number then specified maximum
-    number of processes will be used.
-
-:strong:`local_shell`
-    Shell command used for local execution.
-
-:strong:`out_path`
-    Directory to save cdist output in.
-
-:strong:`parallel`
-    Process hosts in parallel. If -1 then the default, number of CPU's in
-    the system is used. If 0 then parallel processing of hosts is disabled.
-    If set to positive number then specified maximum number of processes
-    will be used.
-
-:strong:`remote_copy`
-    Command to use for remote copy (should behave like scp).
-
-:strong:`remote_exec`
-    Command to use for remote execution (should behave like ssh).
-
-:strong:`remote_out_path`
-    Directory to save cdist output in on the target host.
-
-:strong:`remote_shell`
-    Shell command at remote host used for remote execution.
-
-:strong:`save_output_streams`
-    Enable/disable saving output streams (enabled by default).
-    It recognizes boolean values from 'yes'/'no', 'on'/'off', 'true'/'false'
-    and '1'/'0'.
-
-:strong:`timestamp`
-    Timestamp log messages with the current local date and time
-    in the format: YYYYMMDDHHMMSS.us.
-
-:strong:`verbosity`
-    Set verbosity level. Valid values are: 
-    'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'.
+.. literalinclude:: cdist.cfg.skeleton
+    :language: ini

From 71b6646b38799f7316d791a4d9b511098d879ab0 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 6 Dec 2019 10:00:32 +0100
Subject: [PATCH 081/743] Rm cdist-type prefix and man page ref from docs

E.g.

16.10. __block
16.11. __ccollect_source
16.12. __cdist
16.13. __cdistmarker
16.14. __check_messages

insted of

16.10. cdist-type__block(7)
16.11. cdist-type__ccollect_source(7)
16.12. cdist-type__cdist(7)
16.13. cdist-type__cdistmarker(7)
16.14. cdist-type__check_messages(7)
---
 .gitignore                  |  1 +
 Makefile                    | 10 ++++++-
 docs/src/cdist-types.rst    |  8 ------
 docs/src/cdist-types.rst.sh | 55 +++++++++++++++++++++++++++++++++++++
 4 files changed, 65 insertions(+), 9 deletions(-)
 delete mode 100644 docs/src/cdist-types.rst
 create mode 100755 docs/src/cdist-types.rst.sh

diff --git a/.gitignore b/.gitignore
index fb831051..85a8ccc7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,7 @@ docs/src/man1/*.1
 docs/src/man7/*.7
 docs/src/man7/cdist-type__*.rst
 docs/src/cdist-reference.rst
+docs/src/cdist-types.rst
 docs/src/cdist.cfg.skeleton
 
 # Ignore cdist cache for version control
diff --git a/Makefile b/Makefile
index e6a2a28e..f89ac1e7 100644
--- a/Makefile
+++ b/Makefile
@@ -63,6 +63,13 @@ DOCSREFSH=$(DOCS_SRC_DIR)/cdist-reference.rst.sh
 $(DOCSREF): $(DOCSREFSH)
 	$(DOCSREFSH)
 
+# Html types list with references
+DOCSTYPESREF=$(MAN7DSTDIR)/cdist-types.rst
+DOCSTYPESREFSH=$(DOCS_SRC_DIR)/cdist-types.rst.sh
+
+$(DOCSTYPESREF): $(DOCSTYPESREFSH)
+	$(DOCSTYPESREFSH)
+
 DOCSCFGSKEL=./configuration/cdist.cfg.skeleton
 
 configskel: $(DOCSCFGSKEL)
@@ -77,7 +84,7 @@ version:
 man: version $(MANTYPES) $(DOCSREF)
 	$(SPHINXM)
 
-html: version configskel $(MANTYPES) $(DOCSREF)
+html: version configskel $(MANTYPES) $(DOCSREF) $(DOCSTYPESREF)
 	$(SPHINXH)
 
 docs: man html
@@ -119,6 +126,7 @@ speeches: $(SPEECHES)
 #
 clean: docs-clean
 	rm -f $(DOCS_SRC_DIR)/cdist-reference.rst
+	rm -f $(DOCS_SRC_DIR)/cdist-types.rst
 	rm -f $(DOCS_SRC_DIR)/cdist.cfg.skeleton
 
 	find "$(DOCS_SRC_DIR)" -mindepth 2 -type l \
diff --git a/docs/src/cdist-types.rst b/docs/src/cdist-types.rst
deleted file mode 100644
index d5104667..00000000
--- a/docs/src/cdist-types.rst
+++ /dev/null
@@ -1,8 +0,0 @@
-cdist types
-===========
-
-.. toctree::
-   :titlesonly:
-   :glob:
-
-   man7/*
diff --git a/docs/src/cdist-types.rst.sh b/docs/src/cdist-types.rst.sh
new file mode 100755
index 00000000..2eb66009
--- /dev/null
+++ b/docs/src/cdist-types.rst.sh
@@ -0,0 +1,55 @@
+#!/bin/sh
+#
+# 2019 Darko Poljak (darko.poljak at gmail.com) 
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Generate cdist-types.rst that lists available types.
+#
+
+__cdist_pwd="$(pwd -P)"
+__cdist_mydir="${0%/*}";
+__cdist_abs_mydir="$(cd "$__cdist_mydir" && pwd -P)"
+__cdist_myname=${0##*/};
+__cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname"
+
+filename="${__cdist_myname%.sh}"
+dest="$__cdist_abs_mydir/$filename"
+
+cd "$__cdist_abs_mydir"
+
+exec > "$dest"
+cat << eof 
+cdist types
+===========
+
+.. toctree::
+   :titlesonly:
+
+eof
+
+# If there is no such file then ls prints error to stderr,
+# so redirect stderr to /dev/null.
+for type in $(ls man7/cdist-type__*.rst 2>/dev/null | LC_ALL=C sort); do
+    no_dir="${type#man7/}";
+    no_type="${no_dir#cdist-type}";
+    name="${no_type%.rst}";
+    manref="${no_dir%.rst}"
+    man="${manref}(7)"
+
+    echo "   $name" "<man7/${manref}>"
+done

From f2cbc01e3f885c41962a1e474922127c49099203 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 6 Dec 2019 10:04:06 +0100
Subject: [PATCH 082/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index ca33d7bd..bf0398a9 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __package_update_index: Fix Alpine part (Dominique Roux)
 	* Documentation: Fix man pages for install types (Darko Poljak)
 	* Documentation: Embed config skeleton instead of rewriting it (Darko Poljak)
+	* Documentation: Remove cdist-type prefix and man page reference from type list in html (Darko Poljak)
 
 6.2.0: 2019-11-30
 	* Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak)

From 839e7a408e199cd5a0884c25e283c9bebe13e26d Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 6 Dec 2019 19:46:17 +0100
Subject: [PATCH 083/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index bf0398a9..f4e38578 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Documentation: Fix man pages for install types (Darko Poljak)
 	* Documentation: Embed config skeleton instead of rewriting it (Darko Poljak)
 	* Documentation: Remove cdist-type prefix and man page reference from type list in html (Darko Poljak)
+	* Documentation: PreOS english nitpicking (Evil Ham)
 
 6.2.0: 2019-11-30
 	* Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak)

From d5ac9ea348aa0ed640941232fbc2c783d95b8750 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sun, 8 Dec 2019 16:34:38 +0100
Subject: [PATCH 084/743] Docs: install from source, verify signature

Add chapter for installing from source with signature verification.
Resolves #795.
---
 docs/changelog             |  1 +
 docs/src/cdist-install.rst | 27 +++++++++++++++++++++++++--
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/docs/changelog b/docs/changelog
index f4e38578..33db10da 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Documentation: Embed config skeleton instead of rewriting it (Darko Poljak)
 	* Documentation: Remove cdist-type prefix and man page reference from type list in html (Darko Poljak)
 	* Documentation: PreOS english nitpicking (Evil Ham)
+	* Documentation: Add installing from source with signature verification (Darko Poljak)
 
 6.2.0: 2019-11-30
 	* Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak)
diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst
index a9b7d6b5..880b9f8e 100644
--- a/docs/src/cdist-install.rst
+++ b/docs/src/cdist-install.rst
@@ -23,8 +23,6 @@ Target Hosts
 Install cdist
 -------------
 
-You can install cdist either from git or as a python package.
-
 From git
 ~~~~~~~~
 
@@ -42,6 +40,7 @@ To install cdist, execute the following commands:
 
 From version 4.2.0 cdist tags and releases are signed.
 You can get GPG public key used for signing `here <_static/pgp-key-EFD2AE4EC36B6901.asc>`_.
+It is assumed that you are familiar with *git* ways of signing and verification.
 
 You can also get cdist from `github mirror <https://github.com/ungleich/cdist>`_.
 
@@ -157,3 +156,27 @@ Cdist is available as a python package at
 .. code-block:: sh
 
     pip install cdist
+
+Installing from source with signature verification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you want to install cdist from signed source and verify it, first you need to
+download cdist archive and its detached signature.
+
+Get both, *cdist-x.y.z.tar.gz* and *cdist-x.y.z.tar.gz.asc* from release
+notes of the desired tag *x.y.z* at
+`cdist git repository <https://code.ungleich.ch/ungleich-public/cdist/-/tags>`_.
+
+Get GPG public key used for signing `here <_static/pgp-key-EFD2AE4EC36B6901.asc>`_
+and import it into GPG.
+
+Now cdist source archive can be verified using `gpg`, e.g. to verify `cdist-6.2.0`:
+
+.. code-block:: sh
+
+    $ gpg --verify cdist-6.2.0.tar.gz.asc cdist-6.2.0.targ.gz
+    gpg: Signature made Sat Nov 30 23:14:19 2019 CET
+    gpg:                using RSA key 69767822F3ECC3C349C1EFFFEFD2AE4EC36B6901
+    gpg: Good signature from "ungleich GmbH (ungleich FOSS) <foss@ungleich.ch>" [ultimate]
+
+Further steps are the same as for `installing from git <cdist-install.html#from-git>`_.

From 4735df1bed2b23db8c16f568280ef1c5a8ed4c03 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 6 Dec 2019 19:40:05 +0100
Subject: [PATCH 085/743] Add plugins-dir preos option

---
 cdist/argparse.py        |  2 +-
 cdist/preos.py           | 72 +++++++++++++++++++++++++++++-----------
 docs/src/cdist-preos.rst | 30 ++++++-----------
 docs/src/man1/cdist.rst  | 52 +++++++++++++++++------------
 4 files changed, 95 insertions(+), 61 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index ed94d266..7dc683f3 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -103,7 +103,7 @@ def get_parsers():
                                    name="log level"),
             help=('Set the specified verbosity level. '
                   'The levels, in order from the lowest to the highest, are: '
-                  'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) '
+                  'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3), '
                   'TRACE (4 or higher). If used along with -v then -v '
                   'increases last set value and -l overwrites last set '
                   'value.'),
diff --git a/cdist/preos.py b/cdist/preos.py
index 46b45554..378071db 100644
--- a/cdist/preos.py
+++ b/cdist/preos.py
@@ -5,6 +5,8 @@ import inspect
 import argparse
 import cdist
 import logging
+import re
+import cdist.argparse
 
 
 _PREOS_CALL = "commandline"
@@ -12,15 +14,24 @@ _PREOS_NAME = "_preos_name"
 _PREOS_MARKER = "_cdist_preos"
 _PLUGINS_DIR = "preos"
 _PLUGINS_PATH = [os.path.join(os.path.dirname(__file__), _PLUGINS_DIR), ]
+log = logging.getLogger("PreOS")
+
+
+def extend_plugins_path(dirs):
+    for dir in dirs:
+        preos_dir = os.path.expanduser(os.path.join(dir, "preos"))
+        if os.path.isdir(preos_dir):
+            _PLUGINS_PATH.append(preos_dir)
+
+
 cdist_home = cdist.home_dir()
 if cdist_home:
-    cdist_home_preos = os.path.join(cdist_home, "preos")
-    if os.path.isdir(cdist_home_preos):
-        _PLUGINS_PATH.append(cdist_home_preos)
-sys.path.extend(_PLUGINS_PATH)
-
-
-log = logging.getLogger("PreOS")
+    extend_plugins_path((cdist_home, ))
+x = 'CDIST_PATH'
+if x in os.environ:
+    vals = re.split(r'(?<!\\):', os.environ[x])
+    vals = [x for x in vals if x]
+    extend_plugins_path(vals)
 
 
 def preos_plugin(obj):
@@ -71,31 +82,54 @@ def check_root():
         raise cdist.Error("Must be run with root privileges")
 
 
+def get_available_preoses_string(cls):
+    preoses = ['    - {}'.format(x) for x in sorted(set(cls.preoses))]
+    return "Available PreOS-es:\n{}".format("\n".join(preoses))
+
+
 class PreOS(object):
     preoses = None
 
     @classmethod
     def commandline(cls, argv):
-
-        if not cls.preoses:
-            cls.preoses = find_preoses()
-
+        cdist_parser = cdist.argparse.get_parsers()
         parser = argparse.ArgumentParser(
-            description="Create PreOS", prog="cdist preos")
-        parser.add_argument('preos', help='PreOS to create, one of: {}'.format(
-            set(cls.preoses)))
-        args = parser.parse_args(argv[1:2])
+            description="Create PreOS", prog="cdist preos",
+            parents=[cdist_parser['loglevel'], ])
+        parser.add_argument('preos', help='PreOS to create',
+                            nargs='?', default=None)
+        parser.add_argument('-c', '--conf-dir',
+                            help=('Add configuration directory (one that '
+                                  'contains "preos" subdirectory)'),
+                            action='append')
+        parser.add_argument('-L', '--list-preoses',
+                            help='List available PreOS-es',
+                            action='store_true', default=False)
+        parser.add_argument('remainder_args', nargs=argparse.REMAINDER)
+        args = parser.parse_args(argv[1:])
+        cdist.argparse.handle_loglevel(args)
+        log.debug("preos args : {}".format(args))
+
+        if args.conf_dir:
+            extend_plugins_path(args.conf_dir)
+        sys.path.extend(_PLUGINS_PATH)
+        cls.preoses = find_preoses()
+
+        if args.list_preoses or not args.preos:
+            print(get_available_preoses_string(cls))
+            sys.exit(0)
 
         preos_name = args.preos
         if preos_name in cls.preoses:
             preos = cls.preoses[preos_name]
             func = getattr(preos, _PREOS_CALL)
             if inspect.ismodule(preos):
-                func_args = [preos, argv[2:], ]
+                func_args = [preos, args.remainder_args, ]
             else:
-                func_args = [argv[2:], ]
+                func_args = [args.remainder_args, ]
             log.info("Running preos : {}".format(preos_name))
             func(*func_args)
         else:
-            log.error("Unknown preos: {}, available preoses: {}".format(
-                preos_name, set(cls.preoses.keys())))
+            raise cdist.Error(
+                "Invalid PreOS {}. {}".format(
+                    preos_name, get_available_preoses_string(cls)))
diff --git a/docs/src/cdist-preos.rst b/docs/src/cdist-preos.rst
index 2f102fef..9570bcfc 100644
--- a/docs/src/cdist-preos.rst
+++ b/docs/src/cdist-preos.rst
@@ -88,16 +88,12 @@ When you try to run this new preos you will get:
 
 .. code-block:: sh
 
-    $ cdist preos -h
-    usage: cdist preos [-h] preos
-
-    Create PreOS
-
-    positional arguments:
-      preos       PreOS to create, one of: {'netbsd', 'debian', 'ubuntu'}
-
-    optional arguments:
-      -h, --help  show this help message and exit
+    $ cdist preos -L
+    Available PreOS-es:
+        - debian
+        - devuan
+        - netbsd
+        - ubuntu
     $ cdist preos netbsd
     NetBSD PreOS: []
 
@@ -121,15 +117,11 @@ When you try to run this new preos you will get:
 .. code-block:: sh
 
     $ cdist preos -h
-    usage: cdist preos [-h] preos
-
-    Create PreOS
-
-    positional arguments:
-      preos       PreOS to create, one of: {'freebsd', 'debian', 'ubuntu'}
-
-    optional arguments:
-      -h, --help  show this help message and exit
+    Available PreOS-es:
+        - debian
+        - devuan
+        - freebsd
+        - ubuntu
     $ cdist preos freebsd
     FreeBSD dummy preos: []
 
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index f40491e9..d6937272 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -59,28 +59,28 @@ SYNOPSIS
                          [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t]
                          [host [host ...]]
 
-    cdist preos [-h] preos
+    cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-L] [preos] ...
 
-    cdist preos debian [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
-                       [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
-                       [-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
-                       [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
-                       [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
-                       target_dir
+    cdist preos [preos-options] debian [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
+                                       [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
+                                       [-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
+                                       [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
+                                       [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
+                                       target_dir
 
-    cdist preos devuan [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
-                       [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
-                       [-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
-                       [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
-                       [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
-                       target_dir
+    cdist preos [preos-options] devuan [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
+                                       [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
+                                       [-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
+                                       [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
+                                       [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
+                                       target_dir
 
-    cdist preos ubuntu [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
-                       [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
-                       [-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
-                       [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
-                       [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
-                       target_dir
+    cdist preos [preos-options] ubuntu [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
+                                       [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
+                                       [-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
+                                       [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
+                                       [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
+                                       target_dir
 
     cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL]
 
@@ -103,7 +103,7 @@ All commands accept the following options:
 **-l LOGLEVEL, --log-level LOGLEVEL**
     Set the specified verbosity level. The levels, in
     order from the lowest to the highest, are: ERROR (-1),
-    WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) TRACE (4
+    WARNING (0), INFO (1), VERBOSE (2), DEBUG (3), TRACE (4
     or higher). If used along with -v then -v increases
     last set value and -l overwrites last set value.
 
@@ -116,7 +116,7 @@ All commands accept the following options:
     value is 0 which includes ERROR and WARNING levels.
     The levels, in order from the lowest to the highest,
     are: ERROR (-1), WARNING (0), INFO (1), VERBOSE (2),
-    DEBUG (3) TRACE (4 or higher). If used along with -l
+    DEBUG (3), TRACE (4 or higher). If used along with -l
     then -l overwrites last set value and -v increases
     last set value.
 
@@ -457,7 +457,15 @@ List inventory database.
 
 PREOS
 -----
-Create PreOS. Currently, the following PreOS-es are supported:
+Create PreOS.
+
+**-c CONF_DIR, --conf-dir CONF_DIR**
+    Add configuration directory (one that contains "preos" subdirectory).
+
+**-L, --list-preoses**
+    List available PreOS-es.
+
+Currently, the following PreOS-es are supported:
 
 * debian
 * ubuntu

From 553cfc28e7193ade2678cd805a7f236e51e92972 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 9 Dec 2019 09:57:54 +0100
Subject: [PATCH 086/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 33db10da..04ffac62 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Documentation: Remove cdist-type prefix and man page reference from type list in html (Darko Poljak)
 	* Documentation: PreOS english nitpicking (Evil Ham)
 	* Documentation: Add installing from source with signature verification (Darko Poljak)
+	* Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak)
 
 6.2.0: 2019-11-30
 	* Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak)

From bee95efa4317eacbde98237f7952d020cdfb5e61 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 9 Dec 2019 19:13:10 +0100
Subject: [PATCH 087/743] [new-type] __openldap_server

Originally developed at ungleich and improved for Open Sourcing by Evilham.
---
 cdist/conf/type/__openldap_server/man.rst     | 148 +++++++++++
 cdist/conf/type/__openldap_server/manifest    | 235 ++++++++++++++++++
 .../type/__openldap_server/parameter/boolean  |   2 +
 .../type/__openldap_server/parameter/optional |   4 +
 .../parameter/optional_multiple               |   2 +
 .../type/__openldap_server/parameter/required |   4 +
 6 files changed, 395 insertions(+)
 create mode 100644 cdist/conf/type/__openldap_server/man.rst
 create mode 100644 cdist/conf/type/__openldap_server/manifest
 create mode 100644 cdist/conf/type/__openldap_server/parameter/boolean
 create mode 100644 cdist/conf/type/__openldap_server/parameter/optional
 create mode 100644 cdist/conf/type/__openldap_server/parameter/optional_multiple
 create mode 100644 cdist/conf/type/__openldap_server/parameter/required

diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
new file mode 100644
index 00000000..29bbc231
--- /dev/null
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -0,0 +1,148 @@
+cdist-type__openldap_server(7)
+==============================
+
+NAME
+----
+cdist-type__openldap_server - Setup an openldap(4) server instance
+
+
+DESCRIPTION
+-----------
+This type can be used to bootstrap an LDAP environment using openldap as slapd.
+
+
+REQUIRED PARAMETERS
+-------------------
+manager-dn
+    The rootdn to set up in the directory.
+    E.g. `cn=manager,dc=ungleich,dc=ch`. See `slapd.conf(5)`.
+
+manager-password-hash
+    The password for `manager-dn` in the directory.
+    This should be valid for `slapd.conf` like `{SSHA}qV+mCs3u8Q2sCmUXT4Ybw7MebHTASMyr`.
+    Generate e.g. with: `slappasswd -s weneedgoodsecurity`.
+    See `slappasswd(8C)`, `slapd.conf(5)`.
+    TODO: implement this: http://blog.adamsbros.org/2015/06/09/openldap-ssha-salted-hashes-by-hand/
+      to allow for a manager-password parameter and ensure idempotency (care with salts).
+      Such manager-password parameter should be mutually exclusive with this one.
+
+serverid
+    The server for the directory.
+    E.g. `dc=ungleich,dc=ch`. See `slapd.conf(5)`.
+
+suffix
+    The suffix for the directory.
+    E.g. `dc=ungleich,dc=ch`. See `slapd.conf(5)`.
+
+
+OPTIONAL PARAMETERS
+-------------------
+syncrepl-credentials
+    Only has an effect if `replicate` is set; required in that case.
+    This secret is shared amongst the hosts that will replicate the directory.
+    Note that each replication server needs this secret and it is saved in
+    plain text in the directory.
+
+syncrepl-searchbase
+    Only has an effect if `replicate` is set; required in that case.
+    The searchbase to use for replication.
+    E.g. `dc=ungleich,dc=ch`. See `slapd.conf(5)`.
+
+tls-cert
+    If defined, `__letsencrypt_cert` is not used and this must be the path in
+    the remote hosts to the PEM-encoded TLS certificate.
+    Requires: `tls-privkey` and `tls-ca`.
+    Permissions, existence and renewal of these files are left up to the
+    type's user.
+
+tls-privkey
+    Required if `tls-cert` is defined.
+    Path in the remote hosts to the PEM-encoded private key file.
+
+tls-ca
+    Required if `tls-cert` is defined.
+    Path in the remote hosts to the PEM-encoded CA certificate file.
+
+
+OPTIONAL MULTIPLE PARAMETERS
+----------------------------
+syncrepl-host
+    Only has an effect if `replicate` is set; required in that case.
+    Set once per host that will replicate the directory.
+
+module
+    LDAP module to load. See `slapd.conf(5)`.
+    Default value is OS-dependent, see manifest.
+
+
+BOOLEAN PARAMETERS
+------------------
+staging
+    Passed to `cdist-type__letsencrypt_cert`; has otherwise no use.
+    Obtain a test certificate from a staging server.
+
+replicate
+    Whether to setup replication or not.
+    If present `syncrepl-credentials` and `syncrepl-host` are also required.
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Modify the ruleset on $__target_host:
+    __pf_ruleset --state present --source /my/pf/ruleset.conf
+    require="__pf_ruleset" \
+       __pf_apply
+
+    # Remove the ruleset on $__target_host (implies disabling pf(4):
+    __pf_ruleset --state absent
+    require="__pf_ruleset" \
+       __pf_apply
+
+    root@ldap-for-guacamole:~# cat ldapbase.ldif
+    dn: dc=guaca-test,dc=ungleich,dc=ch
+    objectClass: top
+    objectClass: dcObject
+    objectClass: organization
+    o: Some description
+    dc: guaca-test
+
+
+    # Sample usage:
+    #
+    # id=1
+    # for host in ldap-test1.ungleich.ch ldap-test2.ungleich.ch; do
+    #     echo "__ungleich_ldap ${host} \
+    #         --manager-dn 'cn=manager,dc=ungleich,dc=ch' \
+    #         --manager-password '{SSHA}fooo' \
+    #         --serverid '${id}' \
+    #         --staging \
+    #         --suffix 'dc=ungleich,dc=ch' \
+    #         --searchbase 'dc=ungleich,dc=ch' \
+    #         --syncrepl-credentials 'fooo' \
+    #         --syncrepl-host 'ldap-test1.ungleich.ch' \
+    #         --syncrepl-host 'ldap-test2.ungleich.ch' \
+    #         --descriptiont 'Ungleich LDAP server'" \
+    #         | cdist config -i - -v ${host}
+    #     id=$((id + 1))
+    # done
+
+
+SEE ALSO
+--------
+:strong:`cdist-type__letsencrypt_cert`\ (7)
+
+
+AUTHORS
+-------
+ungleich <foss--@--ungleich.ch>
+Evilham <contact--@--evilham.com>
+
+
+COPYING
+-------
+Copyright \(C) 2020 ungleich glarus ag. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest
new file mode 100644
index 00000000..2acaaed5
--- /dev/null
+++ b/cdist/conf/type/__openldap_server/manifest
@@ -0,0 +1,235 @@
+#!/bin/sh
+
+name="${__target_host}"
+manager_dn=$(cat "${__object}/parameter/manager-dn")
+manager_password_hash=$(cat "${__object}/parameter/manager-password-hash")
+serverid=$(cat "${__object}/parameter/serverid")
+suffix=$(cat "${__object}/parameter/suffix")
+slapd_modules=$(cat "${__object}/parameter/module" || true)
+
+
+OS="$(cat "${__global}/explorer/os")"
+
+# Setup OS-dependent vars
+# TODO: treat other OS better, defaulting to Debian-like
+case "${OS}" in
+    freebsd)
+	      PKGS="openldap-server"
+	      ETC="/usr/local/etc"
+        SLAPD_DIR="/usr/local/etc/openldap"
+        SLAPD_DATA_DIR="/var/db/openldap-data"
+        SLAPD_RUN_DIR="/var/run/openldap"
+        SLAPD_MODULE_PATH="/usr/local/libexec/openldap"
+        if [ -z "${slapd_modules}" ]; then
+            # It looks like ppolicy and syncprov must be compiled
+            slapd_modules="back_mdb back_monitor"
+        fi
+        ;;
+    *)
+        PKGS="slapd ldap-utils"
+	      ETC="/etc"
+        SLAPD_DIR="/etc/ldap"
+        SLAPD_DATA_DIR="/var/lib/ldap"
+        SLAPD_RUN_DIR="/var/run/slapd"
+        SLAPD_MODULE_PATH="/usr/lib/ldap"
+        if [ -z "${slapd_modules}" ]; then
+            slapd_modules="back_mdb ppolicy syncprov back_monitor"
+        fi
+        ;;
+esac
+
+
+
+# Determine if __letsencrypt_cert is to be used and setup vars accordingly
+if [ -f "${__object}/parameter/tls-cert" ]; then
+    tls_cert=$(cat "${__object}/parameter/tls-cert")
+
+    if [ ! -f "${__object}/parameter/tls-privkey" ]; then
+        echo "When tls-cert is defined, tls-privkey is also required." >&2
+        exit 1
+    fi
+    tls_privkey=$(cat "${__object}/parameter/tls-privkey")
+
+    if [ ! -f "${__object}/parameter/tls-ca" ]; then
+        echo "When tls-cert is defined, tls-ca is also required." >&2
+        exit 1
+    fi
+    tls_ca=$(cat "${__object}/parameter/tls-ca")
+
+    _skip_letsencrypt_cert="YES"
+else
+    tls_cert="${SLAPD_DIR}/sasl2/cert.pem"
+    tls_privkey="${SLAPD_DIR}/sasl2/privkey.pem"
+    tls_ca="${SLAPD_DIR}/sasl2/chain.pem"
+fi
+
+mkdir "${__object}/files"
+ldapconf="${__object}/files/ldapconf"
+
+replication=""
+if [ -f "${__object}/parameter/replicate" ]; then
+    replication=yes
+
+    if [ ! -f "${__object}/parameter/syncrepl-searchbase" ]; then
+        echo "Requiring the searchbase for replication" >&2
+        exit 1
+    fi
+    syncrepl_searchbase=$(cat "${__object}/parameter/syncrepl-searchbase")
+
+    if [ ! -f "${__object}/parameter/syncrepl-credentials" ]; then
+        echo "Requiring credentials for replication" >&2
+        exit 1
+    fi
+
+    syncrepl_credentials=$(cat "${__object}/parameter/syncrepl-credentials")
+
+    if [ ! -f "${__object}/parameter/syncrepl-host" ]; then
+        echo "Requiring host(s) for replication" >&2
+        exit 1
+    fi
+    syncrepl_hosts=$(cat "${__object}/parameter/syncrepl-host")
+
+fi
+
+# Install required packages
+for pkg in ${PKGS}; do
+    __package ${pkg}
+done
+
+
+# TODO: Implement __start_on_boot for BSD
+require="__package/slapd" __start_on_boot slapd
+
+# TODO: treat other OS better. Defaulting to Debian-like.
+if [ "${OS}" != "freebsd" ]; then
+    require="__package/slapd" __line rm_slapd_conf \
+       --file ${ETC}/default/slapd \
+       --regex 'SLAPD_CONF=.*' \
+       --state absent
+
+    require="__package/slapd" __line rm_slapd_services \
+       --file ${ETC}/default/slapd \
+       --regex 'SLAPD_SERVICES=.*' \
+       --state absent
+
+    require="__line/rm_slapd_conf" __line add_slapd_conf \
+       --file ${ETC}/default/slapd \
+       --line 'SLAPD_CONF=${SLAPD_DIR}/slapd.conf' \
+       --state present
+
+    require="__line/rm_slapd_services" __line add_slapd_services \
+       --file ${ETC}/default/slapd \
+       --line "SLAPD_SERVICES=\"ldap://localhost/ ldap://${name}/\"" \
+       --state present
+fi
+
+
+if [ -z "${_skip_letsencrypt_cert}" ]; then
+    if [ -f "${__object}/parameter/staging" ]; then
+        staging="--staging"
+    else
+        staging=""
+    fi
+
+    __letsencrypt_cert "${name}" --admin-email technik@ungleich.ch \
+        --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R openldap:openldap ${SLAPD_DIR}/sasl2 && service slapd restart" \
+        --automatic-renewal ${staging}
+fi
+
+require="__package/slapd" __directory ${SLAPD_DIR}/slapd.d --state absent
+
+if [ -z "${_skip_letsencrypt_cert}" ]; then
+    require="__package/slapd __letsencrypt_cert/${name}" \
+           __file ${SLAPD_DIR}/slapd.conf --owner root --group root --mode 644 \
+           --source "${ldapconf}"
+else
+    require="__package/slapd" \
+           __file ${SLAPD_DIR}/slapd.conf --owner root --group root --mode 644 \
+           --source "${ldapconf}"
+fi
+
+# Start slapd.conf
+cat << EOF > "${ldapconf}"
+pidfile ${SLAPD_RUN_DIR}/slapd.pid
+argsfile ${SLAPD_RUN_DIR}/slapd.args
+
+TLSCipherSuite NORMAL
+TLSCertificateFile ${tls_cert}
+TLSCertificateKeyFile ${tls_privkey}
+TLSCACertificateFile ${tls_ca}
+
+disallow bind_anon
+require bind
+security tls=1
+
+include ${SLAPD_DIR}/schema/corba.schema
+include ${SLAPD_DIR}/schema/core.schema
+include ${SLAPD_DIR}/schema/cosine.schema
+include ${SLAPD_DIR}/schema/duaconf.schema
+include ${SLAPD_DIR}/schema/dyngroup.schema
+include ${SLAPD_DIR}/schema/inetorgperson.schema
+include ${SLAPD_DIR}/schema/java.schema
+include ${SLAPD_DIR}/schema/misc.schema
+include ${SLAPD_DIR}/schema/nis.schema
+include ${SLAPD_DIR}/schema/openldap.schema
+include ${SLAPD_DIR}/schema/ppolicy.schema
+include ${SLAPD_DIR}/schema/collective.schema
+
+modulepath ${SLAPD_MODULE_PATH}
+EOF
+
+# Add specified modules
+for module in ${slapd_modules}; do
+    echo "moduleload ${module}.la" >> "${ldapconf}"
+done
+
+# Rest of the config
+cat << EOF >> "${ldapconf}"
+loglevel 1024
+
+database mdb
+maxsize 1073741824
+
+suffix "${suffix}"
+directory ${SLAPD_DATA_DIR}
+rootdn "${manager_dn}"
+rootpw "${manager_password_hash}"
+
+index objectClass eq,pres
+index ou,cn,mail,surname,givenname eq,pres,sub
+index uidNumber,gidNumber,loginShell eq,pres
+index uid,memberUid eq,pres,sub
+index nisMapName,nisMapEntry eq,pres,sub
+index entryCSN,entryUUID eq
+
+serverid ${serverid}
+EOF
+
+# Setup replication
+if [ "${replication}" ]; then
+    rid=1;
+    for syncrepl in ${syncrepl_hosts}; do
+    cat <<EOF >> "${ldapconf}"
+syncrepl rid=${rid}
+ provider=ldap://${syncrepl}
+ bindmethod=simple
+ starttls=yes
+ binddn="${manager_dn}"
+ credentials=${syncrepl_credentials}
+ searchbase="${syncrepl_searchbase}"
+ type=refreshAndPersist
+ retry="5 + 5 +"
+ interval=00:00:00:05
+EOF
+    rid=$((rid + 1))
+    done
+    cat <<EOF >> "${ldapconf}"
+mirrormode true
+overlay syncprov
+syncprov-checkpoint 100 5
+syncprov-sessionlog 100
+
+database monitor
+limits dn.exact="${manager_dn}" time=unlimited size=unlimited
+EOF
+fi
diff --git a/cdist/conf/type/__openldap_server/parameter/boolean b/cdist/conf/type/__openldap_server/parameter/boolean
new file mode 100644
index 00000000..45056fe9
--- /dev/null
+++ b/cdist/conf/type/__openldap_server/parameter/boolean
@@ -0,0 +1,2 @@
+staging
+replicate
diff --git a/cdist/conf/type/__openldap_server/parameter/optional b/cdist/conf/type/__openldap_server/parameter/optional
new file mode 100644
index 00000000..a9a8ab2c
--- /dev/null
+++ b/cdist/conf/type/__openldap_server/parameter/optional
@@ -0,0 +1,4 @@
+description
+syncrepl-credentials
+syncrepl-searchbase
+tls-cert
diff --git a/cdist/conf/type/__openldap_server/parameter/optional_multiple b/cdist/conf/type/__openldap_server/parameter/optional_multiple
new file mode 100644
index 00000000..107c03d9
--- /dev/null
+++ b/cdist/conf/type/__openldap_server/parameter/optional_multiple
@@ -0,0 +1,2 @@
+syncrepl-host
+module
diff --git a/cdist/conf/type/__openldap_server/parameter/required b/cdist/conf/type/__openldap_server/parameter/required
new file mode 100644
index 00000000..1ee6f219
--- /dev/null
+++ b/cdist/conf/type/__openldap_server/parameter/required
@@ -0,0 +1,4 @@
+manager-dn
+manager-password-hash
+serverid
+suffix

From fd430eab622206b9a04780ca5fa9d7c807f16c93 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 9 Dec 2019 19:39:43 +0100
Subject: [PATCH 088/743] [new-type] __openldap_server: Add a "schema" optional
 parameter.

---
 cdist/conf/type/__openldap_server/man.rst     |  7 +++++
 cdist/conf/type/__openldap_server/manifest    | 29 +++++++------------
 .../parameter/default/schema                  | 12 ++++++++
 .../type/__openldap_server/parameter/optional |  3 +-
 4 files changed, 31 insertions(+), 20 deletions(-)
 create mode 100644 cdist/conf/type/__openldap_server/parameter/default/schema

diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
index 29bbc231..1fc24eaa 100644
--- a/cdist/conf/type/__openldap_server/man.rst
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -74,6 +74,13 @@ module
     LDAP module to load. See `slapd.conf(5)`.
     Default value is OS-dependent, see manifest.
 
+schema
+    Name of LDAP schema to load. Must be the name without extension of a
+    `.schema` file in slapd's schema directory (usually `/etc/slapd/schema` or
+    `/usr/local/etc/openldap/schema`).
+    Example value: `inetorgperson`
+    The type user must ensure that the schema file is deployed.
+    This defaults to a sensible subset, for details see the type definition.
 
 BOOLEAN PARAMETERS
 ------------------
diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest
index 2acaaed5..518edd86 100644
--- a/cdist/conf/type/__openldap_server/manifest
+++ b/cdist/conf/type/__openldap_server/manifest
@@ -6,6 +6,7 @@ manager_password_hash=$(cat "${__object}/parameter/manager-password-hash")
 serverid=$(cat "${__object}/parameter/serverid")
 suffix=$(cat "${__object}/parameter/suffix")
 slapd_modules=$(cat "${__object}/parameter/module" || true)
+schemas=$(cat "${__object}/parameter/schema")
 
 
 OS="$(cat "${__global}/explorer/os")"
@@ -14,8 +15,8 @@ OS="$(cat "${__global}/explorer/os")"
 # TODO: treat other OS better, defaulting to Debian-like
 case "${OS}" in
     freebsd)
-	      PKGS="openldap-server"
-	      ETC="/usr/local/etc"
+        PKGS="openldap-server"
+        ETC="/usr/local/etc"
         SLAPD_DIR="/usr/local/etc/openldap"
         SLAPD_DATA_DIR="/var/db/openldap-data"
         SLAPD_RUN_DIR="/var/run/openldap"
@@ -27,7 +28,7 @@ case "${OS}" in
         ;;
     *)
         PKGS="slapd ldap-utils"
-	      ETC="/etc"
+        ETC="/etc"
         SLAPD_DIR="/etc/ldap"
         SLAPD_DATA_DIR="/var/lib/ldap"
         SLAPD_RUN_DIR="/var/run/slapd"
@@ -39,7 +40,6 @@ case "${OS}" in
 esac
 
 
-
 # Determine if __letsencrypt_cert is to be used and setup vars accordingly
 if [ -f "${__object}/parameter/tls-cert" ]; then
     tls_cert=$(cat "${__object}/parameter/tls-cert")
@@ -161,24 +161,15 @@ TLSCACertificateFile ${tls_ca}
 disallow bind_anon
 require bind
 security tls=1
-
-include ${SLAPD_DIR}/schema/corba.schema
-include ${SLAPD_DIR}/schema/core.schema
-include ${SLAPD_DIR}/schema/cosine.schema
-include ${SLAPD_DIR}/schema/duaconf.schema
-include ${SLAPD_DIR}/schema/dyngroup.schema
-include ${SLAPD_DIR}/schema/inetorgperson.schema
-include ${SLAPD_DIR}/schema/java.schema
-include ${SLAPD_DIR}/schema/misc.schema
-include ${SLAPD_DIR}/schema/nis.schema
-include ${SLAPD_DIR}/schema/openldap.schema
-include ${SLAPD_DIR}/schema/ppolicy.schema
-include ${SLAPD_DIR}/schema/collective.schema
-
-modulepath ${SLAPD_MODULE_PATH}
 EOF
 
+# Add specified schemas
+for schema in ${schemas}; do
+    echo "include ${SLAPD_DIR}/schema/${schema}.schema" >> "${ldapconf}"
+done
+
 # Add specified modules
+echo "modulepath ${SLAPD_MODULE_PATH}" >> "${ldapconf}"
 for module in ${slapd_modules}; do
     echo "moduleload ${module}.la" >> "${ldapconf}"
 done
diff --git a/cdist/conf/type/__openldap_server/parameter/default/schema b/cdist/conf/type/__openldap_server/parameter/default/schema
new file mode 100644
index 00000000..825bdb15
--- /dev/null
+++ b/cdist/conf/type/__openldap_server/parameter/default/schema
@@ -0,0 +1,12 @@
+corba
+core
+cosine
+duaconf
+dyngroup
+inetorgperson
+java
+misc
+nis
+openldap
+ppolicy
+collective
diff --git a/cdist/conf/type/__openldap_server/parameter/optional b/cdist/conf/type/__openldap_server/parameter/optional
index a9a8ab2c..53587c4e 100644
--- a/cdist/conf/type/__openldap_server/parameter/optional
+++ b/cdist/conf/type/__openldap_server/parameter/optional
@@ -1,4 +1,5 @@
-description
 syncrepl-credentials
 syncrepl-searchbase
 tls-cert
+tls-privkey
+tls-ca

From 1ad605a509cfd534334111c1d6c1ea8682446b10 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 9 Dec 2019 19:49:05 +0100
Subject: [PATCH 089/743] [new-type] __openldap_server: Add admin-email
 parameter.

---
 cdist/conf/type/__openldap_server/man.rst            | 5 +++++
 cdist/conf/type/__openldap_server/manifest           | 8 +++++++-
 cdist/conf/type/__openldap_server/parameter/optional | 1 +
 3 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
index 1fc24eaa..4447971a 100644
--- a/cdist/conf/type/__openldap_server/man.rst
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -48,6 +48,11 @@ syncrepl-searchbase
     The searchbase to use for replication.
     E.g. `dc=ungleich,dc=ch`. See `slapd.conf(5)`.
 
+admin-email
+    Passed to `cdist-type__letsencrypt_cert`; has otherwise no use.
+    Required if using `__letsencrypt_cert`.
+    Where to send Let's Encrypt emails like "certificate needs renewal".
+
 tls-cert
     If defined, `__letsencrypt_cert` is not used and this must be the path in
     the remote hosts to the PEM-encoded TLS certificate.
diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest
index 518edd86..e10432d6 100644
--- a/cdist/conf/type/__openldap_server/manifest
+++ b/cdist/conf/type/__openldap_server/manifest
@@ -58,6 +58,12 @@ if [ -f "${__object}/parameter/tls-cert" ]; then
 
     _skip_letsencrypt_cert="YES"
 else
+    if [ ! -f "${__object}/parameter/admin-email" ]; then
+        echo "When using __letsencrypt_cert, admin-email is also required." >&2
+        exit 1
+    fi
+    admin_email=$(cat "${__object}/parameter/admin-email")
+
     tls_cert="${SLAPD_DIR}/sasl2/cert.pem"
     tls_privkey="${SLAPD_DIR}/sasl2/privkey.pem"
     tls_ca="${SLAPD_DIR}/sasl2/chain.pem"
@@ -131,7 +137,7 @@ if [ -z "${_skip_letsencrypt_cert}" ]; then
         staging=""
     fi
 
-    __letsencrypt_cert "${name}" --admin-email technik@ungleich.ch \
+    __letsencrypt_cert "${name}" --admin-email "${admin_email}" \
         --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R openldap:openldap ${SLAPD_DIR}/sasl2 && service slapd restart" \
         --automatic-renewal ${staging}
 fi
diff --git a/cdist/conf/type/__openldap_server/parameter/optional b/cdist/conf/type/__openldap_server/parameter/optional
index 53587c4e..f4254cb6 100644
--- a/cdist/conf/type/__openldap_server/parameter/optional
+++ b/cdist/conf/type/__openldap_server/parameter/optional
@@ -1,5 +1,6 @@
 syncrepl-credentials
 syncrepl-searchbase
+admin-email
 tls-cert
 tls-privkey
 tls-ca

From 42914d26c5addae027cc382ec035a687cd3aaf6d Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 9 Dec 2019 19:59:15 +0100
Subject: [PATCH 090/743] [new-type] __openldap_server: sudo make nico.happy.

---
 cdist/conf/type/__openldap_server/manifest | 53 ++++++++++++----------
 1 file changed, 30 insertions(+), 23 deletions(-)

diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest
index e10432d6..ffd92626 100644
--- a/cdist/conf/type/__openldap_server/manifest
+++ b/cdist/conf/type/__openldap_server/manifest
@@ -9,11 +9,10 @@ slapd_modules=$(cat "${__object}/parameter/module" || true)
 schemas=$(cat "${__object}/parameter/schema")
 
 
-OS="$(cat "${__global}/explorer/os")"
+os="$(cat "${__global}/explorer/os")"
 
 # Setup OS-dependent vars
-# TODO: treat other OS better, defaulting to Debian-like
-case "${OS}" in
+case "${os}" in
     freebsd)
         PKGS="openldap-server"
         ETC="/usr/local/etc"
@@ -26,7 +25,7 @@ case "${OS}" in
             slapd_modules="back_mdb back_monitor"
         fi
         ;;
-    *)
+    debian|ubuntu|devuan)
         PKGS="slapd ldap-utils"
         ETC="/etc"
         SLAPD_DIR="/etc/ldap"
@@ -37,6 +36,10 @@ case "${OS}" in
             slapd_modules="back_mdb ppolicy syncprov back_monitor"
         fi
         ;;
+    *)
+        echo "Don't know the openldap defaults for: $os" >&2
+        exit 1
+        ;;
 esac
 
 
@@ -106,28 +109,32 @@ done
 # TODO: Implement __start_on_boot for BSD
 require="__package/slapd" __start_on_boot slapd
 
-# TODO: treat other OS better. Defaulting to Debian-like.
-if [ "${OS}" != "freebsd" ]; then
-    require="__package/slapd" __line rm_slapd_conf \
-       --file ${ETC}/default/slapd \
-       --regex 'SLAPD_CONF=.*' \
-       --state absent
+case "${os}" in
+    debian|ubuntu|devuan)
+        require="__package/slapd" __line rm_slapd_conf \
+               --file ${ETC}/default/slapd \
+               --regex 'SLAPD_CONF=.*' \
+               --state absent
 
-    require="__package/slapd" __line rm_slapd_services \
-       --file ${ETC}/default/slapd \
-       --regex 'SLAPD_SERVICES=.*' \
-       --state absent
+        require="__package/slapd" __line rm_slapd_services \
+               --file ${ETC}/default/slapd \
+               --regex 'SLAPD_SERVICES=.*' \
+               --state absent
 
-    require="__line/rm_slapd_conf" __line add_slapd_conf \
-       --file ${ETC}/default/slapd \
-       --line 'SLAPD_CONF=${SLAPD_DIR}/slapd.conf' \
-       --state present
+        require="__line/rm_slapd_conf" __line add_slapd_conf \
+               --file ${ETC}/default/slapd \
+               --line 'SLAPD_CONF=${SLAPD_DIR}/slapd.conf' \
+               --state present
 
-    require="__line/rm_slapd_services" __line add_slapd_services \
-       --file ${ETC}/default/slapd \
-       --line "SLAPD_SERVICES=\"ldap://localhost/ ldap://${name}/\"" \
-       --state present
-fi
+        require="__line/rm_slapd_services" __line add_slapd_services \
+               --file ${ETC}/default/slapd \
+               --line "SLAPD_SERVICES=\"ldap://localhost/ ldap://${name}/\"" \
+               --state present
+        ;;
+    *)
+        # Nothing to do here, move on.
+        ;;
+esac
 
 
 if [ -z "${_skip_letsencrypt_cert}" ]; then

From 22c5cd550bb1069c2bb8106bbe907f016b9b821a Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 10 Dec 2019 12:49:07 +0100
Subject: [PATCH 091/743] [new-type] __openldap_server: first public version.

This already takes care of setting up the base DN and managing it as well as
allowing for settings for the listener URLS.

The type was also made a singleton as it doesn't make much sense to setup
multiple LDAP servers on the same machine.
---
 .../type/__openldap_server/gencode-remote     |  44 ++++++
 cdist/conf/type/__openldap_server/man.rst     | 126 +++++++++++++-----
 cdist/conf/type/__openldap_server/manifest    |  48 +++++--
 .../parameter/default/description             |   1 +
 .../type/__openldap_server/parameter/optional |   4 +-
 .../parameter/optional_multiple               |   1 +
 .../type/__openldap_server/parameter/required |   1 +
 .../parameter/required_multiple               |   1 +
 cdist/conf/type/__openldap_server/singleton   |   0
 9 files changed, 176 insertions(+), 50 deletions(-)
 create mode 100644 cdist/conf/type/__openldap_server/gencode-remote
 create mode 100644 cdist/conf/type/__openldap_server/parameter/default/description
 create mode 100644 cdist/conf/type/__openldap_server/parameter/required_multiple
 create mode 100644 cdist/conf/type/__openldap_server/singleton

diff --git a/cdist/conf/type/__openldap_server/gencode-remote b/cdist/conf/type/__openldap_server/gencode-remote
new file mode 100644
index 00000000..0ac434af
--- /dev/null
+++ b/cdist/conf/type/__openldap_server/gencode-remote
@@ -0,0 +1,44 @@
+#!/bin/sh
+
+manager_dn=$(cat "${__object}/parameter/manager-dn")
+manager_password=$(cat "${__object}/parameter/manager-password")
+description=$(cat "${__object}/parameter/description")
+suffix=$(cat "${__object}/parameter/suffix")
+suffix_dc=$(echo -n ${suffix} | awk -F',' '{print $1}' | awk -F'=' '{print $2}')
+
+SLAPD_IPC=$(cat "${__object}/parameter/slapd-url" | tr '\n' ' ' | awk '{ print $1}')
+
+cat <<DONE # | tee /dev/stderr
+# Restart service
+service slapd restart
+# It can sometimes take a tiny bit to bind
+sleep 1
+# Create or update base object
+if ldapsearch -xZ -D "${manager_dn}" -w "${manager_password}" -H '${SLAPD_IPC}' -b '${suffix}' -s base 2>&1 > /dev/null; then
+    # Already exists, use ldapmodify
+    ldapmodify -xZ -D "${manager_dn}" -w "${manager_password}" -H '${SLAPD_IPC}' <<EOF
+dn: ${suffix}
+changetype: modify
+replace: objectClass
+objectClass: top
+objectClass: dcObject
+objectClass: organization
+-
+replace: o
+o: ${description}
+-
+replace: dc
+dc: ${suffix_dc}
+EOF
+else
+    # Does not exist, use ldapadd
+    ldapadd -xZ -D "${manager_dn}" -w "${manager_password}" -H '${SLAPD_IPC}' <<EOF
+dn: ${suffix}
+objectClass: top
+objectClass: dcObject
+objectClass: organization
+o: ${description}
+dc: ${suffix_dc}
+EOF
+fi
+DONE
diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
index 4447971a..84b534bd 100644
--- a/cdist/conf/type/__openldap_server/man.rst
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -10,6 +10,9 @@ DESCRIPTION
 -----------
 This type can be used to bootstrap an LDAP environment using openldap as slapd.
 
+It bootstraps the LDAP server with sane defaults and creates and manages the
+base DN defined by `suffix`.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -17,14 +20,19 @@ manager-dn
     The rootdn to set up in the directory.
     E.g. `cn=manager,dc=ungleich,dc=ch`. See `slapd.conf(5)`.
 
+manager-password
+    The password for `manager-dn` in the directory.
+    This will be used to connect to the LDAP server on the first `slapd-url`
+    with the given `manager-dn`.
+
 manager-password-hash
     The password for `manager-dn` in the directory.
     This should be valid for `slapd.conf` like `{SSHA}qV+mCs3u8Q2sCmUXT4Ybw7MebHTASMyr`.
     Generate e.g. with: `slappasswd -s weneedgoodsecurity`.
     See `slappasswd(8C)`, `slapd.conf(5)`.
     TODO: implement this: http://blog.adamsbros.org/2015/06/09/openldap-ssha-salted-hashes-by-hand/
-      to allow for a manager-password parameter and ensure idempotency (care with salts).
-      Such manager-password parameter should be mutually exclusive with this one.
+      to derive from the manager-password parameter and ensure idempotency (care with salts).
+      At that point, manager-password-hash should be deprecated and ignored.
 
 serverid
     The server for the directory.
@@ -35,6 +43,17 @@ suffix
     E.g. `dc=ungleich,dc=ch`. See `slapd.conf(5)`.
 
 
+REQUIRED MULTIPLE PARAMETERS
+----------------------------
+slapd-url
+    A URL for slapd to listen on.
+    Pass once for each URL you want to support,
+    e.g.: `--slapd-url ldaps://my.fqdn/ --slapd-url ldap://my.fqdn/`.
+    The first instance that is passed will be used as the main URL to
+    connect to this LDAP server
+    See the `-h` flag in `slapd(8C)`.
+
+
 OPTIONAL PARAMETERS
 -------------------
 syncrepl-credentials
@@ -53,6 +72,11 @@ admin-email
     Required if using `__letsencrypt_cert`.
     Where to send Let's Encrypt emails like "certificate needs renewal".
 
+tls-cipher-suite
+    Setting for TLSCipherSuite.
+    Defaults to `NORMAL` in a Debian-like OS and `HIGH:MEDIUM:+SSLv2` on FreeBSD.
+    See `slapd.conf(5)`.
+
 tls-cert
     If defined, `__letsencrypt_cert` is not used and this must be the path in
     the remote hosts to the PEM-encoded TLS certificate.
@@ -87,6 +111,11 @@ schema
     The type user must ensure that the schema file is deployed.
     This defaults to a sensible subset, for details see the type definition.
 
+description
+    The description of the base DN passed in the `suffix` parameter.
+    Defaults to `Managed by cdist, do not edit manually.`
+
+
 BOOLEAN PARAMETERS
 ------------------
 staging
@@ -97,48 +126,71 @@ replicate
     Whether to setup replication or not.
     If present `syncrepl-credentials` and `syncrepl-host` are also required.
 
+
 EXAMPLES
 --------
 
 .. code-block:: sh
 
-    # Modify the ruleset on $__target_host:
-    __pf_ruleset --state present --source /my/pf/ruleset.conf
-    require="__pf_ruleset" \
-       __pf_apply
+    # Example of a simple server with manual certificate management.
+    pki_prefix="/usr/local/etc/pki/realms/ldap.camilion.cloud" \
+      __openldap_server \
+        --manager-dn 'cn=manager,dc=camilion,dc=cloud' \
+        --manager-password "foo" \
+        --manager-password-hash '{SSHA}foo' \
+        --serverid 0 \
+        --suffix 'dc=camilion,dc=cloud' \
+        --slapd-url 'ldaps://ldap.camilion.cloud' \
+        --tls-cert '${pki_prefix}/default.crt' \
+        --tls-privkey '${pki_prefix}/default.key' \
+        --tls-ca '${pki_prefix}/CA.crt'
 
-    # Remove the ruleset on $__target_host (implies disabling pf(4):
-    __pf_ruleset --state absent
-    require="__pf_ruleset" \
-       __pf_apply
-
-    root@ldap-for-guacamole:~# cat ldapbase.ldif
-    dn: dc=guaca-test,dc=ungleich,dc=ch
-    objectClass: top
-    objectClass: dcObject
-    objectClass: organization
-    o: Some description
-    dc: guaca-test
-
-
-    # Sample usage:
+    # The created basedn looks as follows:
     #
-    # id=1
-    # for host in ldap-test1.ungleich.ch ldap-test2.ungleich.ch; do
-    #     echo "__ungleich_ldap ${host} \
-    #         --manager-dn 'cn=manager,dc=ungleich,dc=ch' \
-    #         --manager-password '{SSHA}fooo' \
-    #         --serverid '${id}' \
-    #         --staging \
-    #         --suffix 'dc=ungleich,dc=ch' \
-    #         --searchbase 'dc=ungleich,dc=ch' \
-    #         --syncrepl-credentials 'fooo' \
-    #         --syncrepl-host 'ldap-test1.ungleich.ch' \
-    #         --syncrepl-host 'ldap-test2.ungleich.ch' \
-    #         --descriptiont 'Ungleich LDAP server'" \
-    #         | cdist config -i - -v ${host}
-    #     id=$((id + 1))
-    # done
+    # dn: dc=camilion,dc=cloud
+    # objectClass: top
+    # objectClass: dcObject
+    # objectClass: organization
+    # o: Managed by cdist, do not edit manually.
+    # dc: camilion
+    #
+    # Do not change it manually, the type will overwrite your changes.
+
+
+    #
+    # Changing to a replicated setup is a simple change to something like:
+    #
+    # Example for multiple servers with replication and automatic
+    # Let's Encrypt certificate management through certbot.
+    id=1
+    for host in ldap-test1.ungleich.ch ldap-test2.ungleich.ch; do
+        echo "__ungleich_ldap \
+            --manager-dn 'cn=manager,dc=ungleich,dc=ch' \
+            --manager-psasword 'foo' \
+            --manager-password-hash '{SSHA}fooo' \
+            --serverid '${id}' \
+            --suffix 'dc=ungleich,dc=ch' \
+            --slapd-url ldap://${host} \
+            --searchbase 'dc=ungleich,dc=ch' \
+            --syncrepl-credentials 'fooo' \
+            --syncrepl-host 'ldap-test1.ungleich.ch' \
+            --syncrepl-host 'ldap-test2.ungleich.ch' \
+            --description 'Ungleich LDAP server'" \
+            --staging \
+            | cdist config -i - -v ${host}
+        id=$((id + 1))
+    done
+
+    # The created basedn looks as follows:
+    #
+    # dn: dc=ungleich,dc=ch
+    # objectClass: top
+    # objectClass: dcObject
+    # objectClass: organization
+    # o: Ungleich LDAP server
+    # dc: ungleich
+    #
+    # Do not change it manually, the type will overwrite your changes.
 
 
 SEE ALSO
diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest
index ffd92626..2d1df32f 100644
--- a/cdist/conf/type/__openldap_server/manifest
+++ b/cdist/conf/type/__openldap_server/manifest
@@ -5,13 +5,17 @@ manager_dn=$(cat "${__object}/parameter/manager-dn")
 manager_password_hash=$(cat "${__object}/parameter/manager-password-hash")
 serverid=$(cat "${__object}/parameter/serverid")
 suffix=$(cat "${__object}/parameter/suffix")
-slapd_modules=$(cat "${__object}/parameter/module" || true)
+slapd_modules=$(cat "${__object}/parameter/module" 2>/dev/null || true)
 schemas=$(cat "${__object}/parameter/schema")
+slapd_urls=$(cat "${__object}/parameter/slapd-url" | tr '\n' ' ')
+tls_cipher_suite=$(cat "${__object}/parameter/tls-cipher-suite" 2>/dev/null || true)
 
 
 os="$(cat "${__global}/explorer/os")"
 
 # Setup OS-dependent vars
+CONF_OWNER="root"
+CONF_GROUP="root"
 case "${os}" in
     freebsd)
         PKGS="openldap-server"
@@ -24,6 +28,12 @@ case "${os}" in
             # It looks like ppolicy and syncprov must be compiled
             slapd_modules="back_mdb back_monitor"
         fi
+        CONF_OWNER="ldap"
+        CONF_GROUP="ldap"
+        if [ -z "${tls_cipher_suite}" ]; then
+            # TODO: research default for FreeBSD. 'NORMAL' appears to not work
+            tls_cipher_suite="HIGH:MEDIUM:+SSLv2"
+        fi
         ;;
     debian|ubuntu|devuan)
         PKGS="slapd ldap-utils"
@@ -35,6 +45,9 @@ case "${os}" in
         if [ -z "${slapd_modules}" ]; then
             slapd_modules="back_mdb ppolicy syncprov back_monitor"
         fi
+        if [ -z "${tls_cipher_suite}" ]; then
+            tls_cipher_suite="NORMAL"
+        fi
         ;;
     *)
         echo "Don't know the openldap defaults for: $os" >&2
@@ -42,6 +55,8 @@ case "${os}" in
         ;;
 esac
 
+PKG_MAIN=$(echo ${PKGS} | awk '{print $1;}')
+
 
 # Determine if __letsencrypt_cert is to be used and setup vars accordingly
 if [ -f "${__object}/parameter/tls-cert" ]; then
@@ -106,17 +121,26 @@ for pkg in ${PKGS}; do
 done
 
 
-# TODO: Implement __start_on_boot for BSD
-require="__package/slapd" __start_on_boot slapd
+require="__package/${PKG_MAIN}" __start_on_boot slapd
 
+# Setup -h flag for the listeners. See man slapd (-h flag).
 case "${os}" in
+    freebsd)
+        require="__package/${PKG_MAIN}" __key_value \
+               --file "/etc/rc.conf" \
+               --key "slapd_flags" \
+               --value "\"-h '${slapd_urls}'\"" \
+               --delimiter "=" \
+               --comment "# LDAP Listener URLs" \
+               "${__target_host}__slapd_flags"
+        ;;
     debian|ubuntu|devuan)
-        require="__package/slapd" __line rm_slapd_conf \
+        require="__package/${PKG_MAIN}" __line rm_slapd_conf \
                --file ${ETC}/default/slapd \
                --regex 'SLAPD_CONF=.*' \
                --state absent
 
-        require="__package/slapd" __line rm_slapd_services \
+        require="__package/${PKG_MAIN}" __line rm_slapd_services \
                --file ${ETC}/default/slapd \
                --regex 'SLAPD_SERVICES=.*' \
                --state absent
@@ -128,7 +152,7 @@ case "${os}" in
 
         require="__line/rm_slapd_services" __line add_slapd_services \
                --file ${ETC}/default/slapd \
-               --line "SLAPD_SERVICES=\"ldap://localhost/ ldap://${name}/\"" \
+               --line "SLAPD_SERVICES=\"${slapd_urls}\"" \
                --state present
         ;;
     *)
@@ -149,15 +173,15 @@ if [ -z "${_skip_letsencrypt_cert}" ]; then
         --automatic-renewal ${staging}
 fi
 
-require="__package/slapd" __directory ${SLAPD_DIR}/slapd.d --state absent
+require="__package/${PKG_MAIN}" __directory ${SLAPD_DIR}/slapd.d --state absent
 
 if [ -z "${_skip_letsencrypt_cert}" ]; then
-    require="__package/slapd __letsencrypt_cert/${name}" \
-           __file ${SLAPD_DIR}/slapd.conf --owner root --group root --mode 644 \
+    require="__package/${PKG_MAIN} __letsencrypt_cert/${name}" \
+           __file ${SLAPD_DIR}/slapd.conf --owner ${CONF_OWNER} --group ${CONF_GROUP} --mode 644 \
            --source "${ldapconf}"
 else
-    require="__package/slapd" \
-           __file ${SLAPD_DIR}/slapd.conf --owner root --group root --mode 644 \
+    require="__package/${PKG_MAIN}" \
+           __file ${SLAPD_DIR}/slapd.conf --owner ${CONF_OWNER} --group ${CONF_GROUP} --mode 644 \
            --source "${ldapconf}"
 fi
 
@@ -166,7 +190,7 @@ cat << EOF > "${ldapconf}"
 pidfile ${SLAPD_RUN_DIR}/slapd.pid
 argsfile ${SLAPD_RUN_DIR}/slapd.args
 
-TLSCipherSuite NORMAL
+TLSCipherSuite ${tls_cipher_suite}
 TLSCertificateFile ${tls_cert}
 TLSCertificateKeyFile ${tls_privkey}
 TLSCACertificateFile ${tls_ca}
diff --git a/cdist/conf/type/__openldap_server/parameter/default/description b/cdist/conf/type/__openldap_server/parameter/default/description
new file mode 100644
index 00000000..6d8e37e1
--- /dev/null
+++ b/cdist/conf/type/__openldap_server/parameter/default/description
@@ -0,0 +1 @@
+Managed by cdist, do not edit manually.
diff --git a/cdist/conf/type/__openldap_server/parameter/optional b/cdist/conf/type/__openldap_server/parameter/optional
index f4254cb6..a92b9c6e 100644
--- a/cdist/conf/type/__openldap_server/parameter/optional
+++ b/cdist/conf/type/__openldap_server/parameter/optional
@@ -1,6 +1,8 @@
+description
 syncrepl-credentials
 syncrepl-searchbase
 admin-email
+tls-cipher-suite
 tls-cert
 tls-privkey
-tls-ca
+tls-ca
\ No newline at end of file
diff --git a/cdist/conf/type/__openldap_server/parameter/optional_multiple b/cdist/conf/type/__openldap_server/parameter/optional_multiple
index 107c03d9..52a83d5c 100644
--- a/cdist/conf/type/__openldap_server/parameter/optional_multiple
+++ b/cdist/conf/type/__openldap_server/parameter/optional_multiple
@@ -1,2 +1,3 @@
 syncrepl-host
 module
+schema
diff --git a/cdist/conf/type/__openldap_server/parameter/required b/cdist/conf/type/__openldap_server/parameter/required
index 1ee6f219..ff58158d 100644
--- a/cdist/conf/type/__openldap_server/parameter/required
+++ b/cdist/conf/type/__openldap_server/parameter/required
@@ -1,4 +1,5 @@
 manager-dn
+manager-password
 manager-password-hash
 serverid
 suffix
diff --git a/cdist/conf/type/__openldap_server/parameter/required_multiple b/cdist/conf/type/__openldap_server/parameter/required_multiple
new file mode 100644
index 00000000..848b8dc2
--- /dev/null
+++ b/cdist/conf/type/__openldap_server/parameter/required_multiple
@@ -0,0 +1 @@
+slapd-url
\ No newline at end of file
diff --git a/cdist/conf/type/__openldap_server/singleton b/cdist/conf/type/__openldap_server/singleton
new file mode 100644
index 00000000..e69de29b

From 79d58f0813fd08c77d826ac552dfdb9e0f40dbbd Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 10 Dec 2019 13:04:24 +0100
Subject: [PATCH 092/743] [new-type] __openldap_server: fix docs / dependency.

It's nicer to have slapd_flags after slapd_enable in rc.conf on freebsd.
---
 cdist/conf/type/__openldap_server/man.rst  | 6 +++---
 cdist/conf/type/__openldap_server/manifest | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
index 84b534bd..1c2c9b70 100644
--- a/cdist/conf/type/__openldap_server/man.rst
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -141,9 +141,9 @@ EXAMPLES
         --serverid 0 \
         --suffix 'dc=camilion,dc=cloud' \
         --slapd-url 'ldaps://ldap.camilion.cloud' \
-        --tls-cert '${pki_prefix}/default.crt' \
-        --tls-privkey '${pki_prefix}/default.key' \
-        --tls-ca '${pki_prefix}/CA.crt'
+        --tls-cert "${pki_prefix}/default.crt" \
+        --tls-privkey "${pki_prefix}/default.key" \
+        --tls-ca "${pki_prefix}/CA.crt"
 
     # The created basedn looks as follows:
     #
diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest
index 2d1df32f..070a31d5 100644
--- a/cdist/conf/type/__openldap_server/manifest
+++ b/cdist/conf/type/__openldap_server/manifest
@@ -126,7 +126,7 @@ require="__package/${PKG_MAIN}" __start_on_boot slapd
 # Setup -h flag for the listeners. See man slapd (-h flag).
 case "${os}" in
     freebsd)
-        require="__package/${PKG_MAIN}" __key_value \
+        require="__start_on_boot/slapd" __key_value \
                --file "/etc/rc.conf" \
                --key "slapd_flags" \
                --value "\"-h '${slapd_urls}'\"" \

From 35f0d4dbfa9307923fa013ff806d30087a8cf291 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 10 Dec 2019 13:10:19 +0100
Subject: [PATCH 093/743] [new-type] __openldap_server: another documentation
 typo.

---
 cdist/conf/type/__openldap_server/man.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
index 1c2c9b70..d20101d1 100644
--- a/cdist/conf/type/__openldap_server/man.rst
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -133,8 +133,8 @@ EXAMPLES
 .. code-block:: sh
 
     # Example of a simple server with manual certificate management.
-    pki_prefix="/usr/local/etc/pki/realms/ldap.camilion.cloud" \
-      __openldap_server \
+    pki_prefix="/usr/local/etc/pki/realms/ldap.camilion.cloud"
+    __openldap_server \
         --manager-dn 'cn=manager,dc=camilion,dc=cloud' \
         --manager-password "foo" \
         --manager-password-hash '{SSHA}foo' \

From 3ba230c10db98cf2cddb0aeeb256392223fb2558 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 10 Dec 2019 13:12:54 +0100
Subject: [PATCH 094/743] [type-docs] __start_on_boot: remove unspported *BSD
 claim.

The type appears to support {Open,Free}BSD properly.
---
 cdist/conf/type/__start_on_boot/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__start_on_boot/man.rst b/cdist/conf/type/__start_on_boot/man.rst
index b7c73ab1..f8afe94b 100644
--- a/cdist/conf/type/__start_on_boot/man.rst
+++ b/cdist/conf/type/__start_on_boot/man.rst
@@ -12,7 +12,7 @@ This cdist type allows you to enable or disable stuff to be started
 at boot of your operating system.
 
 Warning: This type has not been tested intensively and is not fully
-supported (i.e. \*BSD are not implemented).
+supported.
 
 
 REQUIRED PARAMETERS

From 3c62a88ca7907b15dc501ddb3395c66b6196cb13 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 10 Dec 2019 19:33:29 +0100
Subject: [PATCH 095/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 04ffac62..186dce7a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,8 @@ next:
 	* Documentation: PreOS english nitpicking (Evil Ham)
 	* Documentation: Add installing from source with signature verification (Darko Poljak)
 	* Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak)
+	* Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham)
+	* New type: __openldap_server (Evil Ham)
 
 6.2.0: 2019-11-30
 	* Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak)

From 2c7b4ddc55a9a2f76ae5b51a6020c0cd14900705 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 12 Dec 2019 06:18:04 +0100
Subject: [PATCH 096/743] Update cdist man page

---
 docs/src/man1/cdist.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index d6937272..55db82ed 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -915,5 +915,5 @@ such case and display a warning message. An example of such a case:
 
 COPYING
 -------
-Copyright \(C) 2011-2017 Nico Schottelius. Free use of this software is
+Copyright \(C) 2011-2019 Nico Schottelius. Free use of this software is
 granted under the terms of the GNU General Public License v3 or later (GPLv3+).

From bd4eee7925356c8e557a75688694feff94cca297 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 12 Dec 2019 07:00:23 +0100
Subject: [PATCH 097/743] Fix shellcheck reported issues

---
 cdist/conf/type/__openldap_server/gencode-remote | 4 ++--
 cdist/conf/type/__openldap_server/manifest       | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__openldap_server/gencode-remote b/cdist/conf/type/__openldap_server/gencode-remote
index 0ac434af..b1e98f8c 100644
--- a/cdist/conf/type/__openldap_server/gencode-remote
+++ b/cdist/conf/type/__openldap_server/gencode-remote
@@ -4,9 +4,9 @@ manager_dn=$(cat "${__object}/parameter/manager-dn")
 manager_password=$(cat "${__object}/parameter/manager-password")
 description=$(cat "${__object}/parameter/description")
 suffix=$(cat "${__object}/parameter/suffix")
-suffix_dc=$(echo -n ${suffix} | awk -F',' '{print $1}' | awk -F'=' '{print $2}')
+suffix_dc=$(printf "%s" "${suffix}" | awk -F',' '{print $1}' | awk -F'=' '{print $2}')
 
-SLAPD_IPC=$(cat "${__object}/parameter/slapd-url" | tr '\n' ' ' | awk '{ print $1}')
+SLAPD_IPC=$(tr '\n' ' ' < "${__object}/parameter/slapd-url" | awk '{ print $1}')
 
 cat <<DONE # | tee /dev/stderr
 # Restart service
diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest
index 070a31d5..dadc9f20 100644
--- a/cdist/conf/type/__openldap_server/manifest
+++ b/cdist/conf/type/__openldap_server/manifest
@@ -7,7 +7,7 @@ serverid=$(cat "${__object}/parameter/serverid")
 suffix=$(cat "${__object}/parameter/suffix")
 slapd_modules=$(cat "${__object}/parameter/module" 2>/dev/null || true)
 schemas=$(cat "${__object}/parameter/schema")
-slapd_urls=$(cat "${__object}/parameter/slapd-url" | tr '\n' ' ')
+slapd_urls=$(tr '\n' ' ' < "${__object}/parameter/slapd-url")
 tls_cipher_suite=$(cat "${__object}/parameter/tls-cipher-suite" 2>/dev/null || true)
 
 
@@ -55,7 +55,7 @@ case "${os}" in
         ;;
 esac
 
-PKG_MAIN=$(echo ${PKGS} | awk '{print $1;}')
+PKG_MAIN=$(echo "${PKGS}" | awk '{print $1;}')
 
 
 # Determine if __letsencrypt_cert is to be used and setup vars accordingly
@@ -117,7 +117,7 @@ fi
 
 # Install required packages
 for pkg in ${PKGS}; do
-    __package ${pkg}
+    __package "${pkg}"
 done
 
 
@@ -147,7 +147,7 @@ case "${os}" in
 
         require="__line/rm_slapd_conf" __line add_slapd_conf \
                --file ${ETC}/default/slapd \
-               --line 'SLAPD_CONF=${SLAPD_DIR}/slapd.conf' \
+               --line "SLAPD_CONF=${SLAPD_DIR}/slapd.conf" \
                --state present
 
         require="__line/rm_slapd_services" __line add_slapd_services \

From 8562871da9a51752fea47308e496e139210edf75 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 12 Dec 2019 07:35:53 +0100
Subject: [PATCH 098/743] Fix shellcheck exit status

shellcheck* targets were always reporting exit status 0.
With this fix, if shellcheck fails, then build-helper script
exits with 1.
---
 bin/build-helper | 50 ++++++++++++++++++++++++++++++++----------------
 1 file changed, 33 insertions(+), 17 deletions(-)

diff --git a/bin/build-helper b/bin/build-helper
index 7cfc4d55..69dee4c7 100755
--- a/bin/build-helper
+++ b/bin/build-helper
@@ -74,6 +74,7 @@ SHELLCHECKCMD="shellcheck -s sh -f gcc -x"
 # Skip SC2154 for variables starting with __ since such variables are cdist
 # environment variables.
 SHELLCHECK_SKIP=': __.*is referenced but not assigned.*\[SC2154\]'
+SHELLCHECKTMP=".shellcheck.tmp"
 
 # Change to checkout directory
 basedir="${0%/*}/../"
@@ -431,53 +432,67 @@ eof
     ;;
 
     shellcheck-global-explorers)
-        find cdist/conf/explorer -type f -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+        # shellcheck disable=SC2086
+        find cdist/conf/explorer -type f -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}"
+        test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
     shellcheck-type-explorers)
-        find cdist/conf/type -type f -path "*/explorer/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+        # shellcheck disable=SC2086
+        find cdist/conf/type -type f -path "*/explorer/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}"
+        test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
     shellcheck-manifests)
-        find cdist/conf/type -type f -name manifest -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+        # shellcheck disable=SC2086
+        find cdist/conf/type -type f -name manifest -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}"
+        test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
     shellcheck-local-gencodes)
-        find cdist/conf/type -type f -name gencode-local -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+        # shellcheck disable=SC2086
+        find cdist/conf/type -type f -name gencode-local -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}"
+        test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
     shellcheck-remote-gencodes)
-        find cdist/conf/type -type f -name gencode-remote -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+        # shellcheck disable=SC2086
+        find cdist/conf/type -type f -name gencode-remote -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}"
+        test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
     shellcheck-scripts)
-        ${SHELLCHECKCMD} scripts/cdist-dump scripts/cdist-new-type || exit 0
+        # shellcheck disable=SC2086
+        ${SHELLCHECKCMD} scripts/cdist-dump scripts/cdist-new-type > "${SHELLCHECKTMP}"
+        test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
     shellcheck-gencodes)
-        "$0" shellcheck-local-gencodes
-        "$0" shellcheck-remote-gencodes
+        "$0" shellcheck-local-gencodes || exit 1
+        "$0" shellcheck-remote-gencodes || exit 1
     ;;
 
     shellcheck-types)
-        "$0" shellcheck-type-explorers
-        "$0" shellcheck-manifests
-        "$0" shellcheck-gencodes
+        "$0" shellcheck-type-explorers || exit 1
+        "$0" shellcheck-manifests || exit 1
+        "$0" shellcheck-gencodes || exit 1
     ;;
 
     shellcheck)
-        "$0" shellcheck-global-explorers
-        "$0" shellcheck-types
-        "$0" shellcheck-scripts
+        "$0" shellcheck-global-explorers || exit 1
+        "$0" shellcheck-types || exit 1
+        "$0" shellcheck-scripts || exit 1
     ;;
 
     shellcheck-type-files)
-        find cdist/conf/type -type f -path "*/files/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
+        # shellcheck disable=SC2086
+        find cdist/conf/type -type f -path "*/files/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}"
+        test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
     shellcheck-with-files)
-        "$0" shellcheck
-        "$0" shellcheck-type-files
+        "$0" shellcheck || exit 1
+        "$0" shellcheck-type-files || exit 1
     ;;
 
     shellcheck-build-helper)
@@ -535,6 +550,7 @@ eof
 
         # Temp files
         rm -f ./*.tmp
+        rm -f ./.*.tmp
     ;;
 
     distclean)

From 9859080217a9dc067250a0fc0f686a2242ef1083 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 12 Dec 2019 07:43:07 +0100
Subject: [PATCH 099/743] Release 6.3.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 186dce7a..674e14c7 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.3.0: 2019-12-12
 	* Type __package_update_index: Fix Alpine part (Dominique Roux)
 	* Documentation: Fix man pages for install types (Darko Poljak)
 	* Documentation: Embed config skeleton instead of rewriting it (Darko Poljak)

From f407e8825fc2eb70fa489dc218b46f1f2d92d747 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Sun, 15 Dec 2019 22:56:16 +0100
Subject: [PATCH 100/743] [consul/alpine] no init script required to be
 deployed anymore

---
 cdist/conf/type/__consul_agent/manifest | 2 +-
 docs/changelog                          | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest
index a88d26ed..ee682d72 100755
--- a/cdist/conf/type/__consul_agent/manifest
+++ b/cdist/conf/type/__consul_agent/manifest
@@ -181,7 +181,7 @@ init_upstart()
 
 # Install init script to start on boot
 case "$os" in
-    alpine|devuan)
+    devuan)
         init_sysvinit debian
         ;;
     centos|redhat)
diff --git a/docs/changelog b/docs/changelog
index 674e14c7..415af0af 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* Type __consul_agent: don't deploy init script on Alpine anymore,
+	it ships with one itself (Nico Schottelius)
+
 6.3.0: 2019-12-12
 	* Type __package_update_index: Fix Alpine part (Dominique Roux)
 	* Documentation: Fix man pages for install types (Darko Poljak)

From df05abd15b6676d6594b3e8aebff112c2f5c0ce7 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Thu, 19 Dec 2019 12:33:47 +0100
Subject: [PATCH 101/743] bugfix: __install_chroot_umount was not using
 __chroot_umount/manifest

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/conf/type/__install_chroot_umount/manifest | 1 +
 1 file changed, 1 insertion(+)
 create mode 120000 cdist/conf/type/__install_chroot_umount/manifest

diff --git a/cdist/conf/type/__install_chroot_umount/manifest b/cdist/conf/type/__install_chroot_umount/manifest
new file mode 120000
index 00000000..f17af67a
--- /dev/null
+++ b/cdist/conf/type/__install_chroot_umount/manifest
@@ -0,0 +1 @@
+../__chroot_umount/manifest
\ No newline at end of file

From beb930c0dc6e80d0718029fa8fc5539f373154a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 31 Dec 2019 11:05:40 +0100
Subject: [PATCH 102/743] __postgres_*: use delimited identifiers (double
 quoted) in generated SQL

---
 cdist/conf/type/__postgres_database/gencode-remote | 6 +++---
 cdist/conf/type/__postgres_role/gencode-remote     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote
index 61cfa50d..9f12d215 100755
--- a/cdist/conf/type/__postgres_database/gencode-remote
+++ b/cdist/conf/type/__postgres_database/gencode-remote
@@ -41,12 +41,12 @@ if [ "$state_should" != "$state_is" ]; then
       present)
          owner=""
          if [ -f "$__object/parameter/owner" ]; then
-            owner="-O '$(cat "$__object/parameter/owner")'"
+            owner="-O \"$(cat "$__object/parameter/owner")\""
          fi
-         echo "su - '$postgres_user' -c \"createdb $owner '$name'\""
+         echo "su - '$postgres_user' -c \"createdb $owner \"$name\"\""
       ;;
       absent)
-         echo "su - '$postgres_user' -c \"dropdb '$name'\""
+         echo "su - '$postgres_user' -c \"dropdb \"$name\"\""
       ;;
    esac
 fi
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index fd56e85d..0b8852f4 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -54,7 +54,7 @@ case "$state_should" in
 
         [ -n "$password" ] && password="PASSWORD '$password'"
 
-        cmd="CREATE ROLE $name WITH $password $booleans"
+        cmd="CREATE ROLE \"$name\" WITH $password $booleans"
         echo "su - '$postgres_user' -c \"psql postgres -wc \\\"$cmd\\\"\""
     ;;
     absent)

From c32e4040b1b17f1e4c8173f23f1bdb2d5112f227 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 31 Dec 2019 19:16:49 +0200
Subject: [PATCH 103/743] __mysql_*: initial rewrite

---
 .../conf/type/__mysql_database/explorer/state | 15 ++++
 .../conf/type/__mysql_database/gencode-remote | 72 ++++++-------------
 cdist/conf/type/__mysql_database/man.rst      | 49 -------------
 cdist/conf/type/__mysql_database/manifest     | 26 +++++++
 .../__mysql_database/parameter/default/state  |  1 +
 .../type/__mysql_database/parameter/optional  |  1 +
 .../type/__mysql_privileges/explorer/state    | 22 ++++++
 .../type/__mysql_privileges/gencode-remote    | 31 ++++++++
 .../__mysql_privileges/parameter/default/host |  1 +
 .../parameter/default/privileges              |  1 +
 .../parameter/default/state                   |  1 +
 .../parameter/default/table                   |  1 +
 .../__mysql_privileges/parameter/optional     |  4 ++
 .../__mysql_privileges/parameter/required     |  2 +
 cdist/conf/type/__mysql_user/explorer/state   | 36 ++++++++++
 cdist/conf/type/__mysql_user/gencode-remote   | 50 +++++++++++++
 .../type/__mysql_user/parameter/default/host  |  1 +
 .../type/__mysql_user/parameter/default/state |  1 +
 .../conf/type/__mysql_user/parameter/optional |  4 ++
 19 files changed, 221 insertions(+), 98 deletions(-)
 create mode 100755 cdist/conf/type/__mysql_database/explorer/state
 delete mode 100644 cdist/conf/type/__mysql_database/man.rst
 create mode 100755 cdist/conf/type/__mysql_database/manifest
 create mode 100644 cdist/conf/type/__mysql_database/parameter/default/state
 create mode 100755 cdist/conf/type/__mysql_privileges/explorer/state
 create mode 100755 cdist/conf/type/__mysql_privileges/gencode-remote
 create mode 100644 cdist/conf/type/__mysql_privileges/parameter/default/host
 create mode 100644 cdist/conf/type/__mysql_privileges/parameter/default/privileges
 create mode 100644 cdist/conf/type/__mysql_privileges/parameter/default/state
 create mode 100644 cdist/conf/type/__mysql_privileges/parameter/default/table
 create mode 100644 cdist/conf/type/__mysql_privileges/parameter/optional
 create mode 100644 cdist/conf/type/__mysql_privileges/parameter/required
 create mode 100755 cdist/conf/type/__mysql_user/explorer/state
 create mode 100755 cdist/conf/type/__mysql_user/gencode-remote
 create mode 100644 cdist/conf/type/__mysql_user/parameter/default/host
 create mode 100644 cdist/conf/type/__mysql_user/parameter/default/state
 create mode 100644 cdist/conf/type/__mysql_user/parameter/optional

diff --git a/cdist/conf/type/__mysql_database/explorer/state b/cdist/conf/type/__mysql_database/explorer/state
new file mode 100755
index 00000000..16cc9ce5
--- /dev/null
+++ b/cdist/conf/type/__mysql_database/explorer/state
@@ -0,0 +1,15 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/name" ]
+then
+    name="$( cat "$__object/parameter/name" )"
+else
+    name="$__object_id"
+fi
+
+if [ -n "$( mysql -B -N -e "show databases like '$name'" )" ]
+then
+    echo 'present'
+else
+    echo 'absent'
+fi
diff --git a/cdist/conf/type/__mysql_database/gencode-remote b/cdist/conf/type/__mysql_database/gencode-remote
index 23e51b05..d3692572 100755
--- a/cdist/conf/type/__mysql_database/gencode-remote
+++ b/cdist/conf/type/__mysql_database/gencode-remote
@@ -1,54 +1,28 @@
 #!/bin/sh -e
-#
-# 2012 Benedikt Koeppel (code@benediktkoeppel.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
 
-# if --database was specified
-if [ -f "$__object/parameter/name" ]; then
-   database="$(cat "$__object/parameter/name")"
-else # otherwise use the object id as database name
-   database="$__object_id"
+state_is="$( cat "$__object/explorer/state" )"
+
+state_should="$( cat "$__object/parameter/state" )"
+
+if [ "$state_is" = "$state_should" ]
+then
+    exit 0
 fi
 
-cat <<-EOFF
-mysql -u root <<-EOF
-	CREATE DATABASE IF NOT EXISTS $database
-EOF
-EOFF
-
-# if --user was specified
-if [ -f "$__object/parameter/user" ]; then
-   user="$(cat "$__object/parameter/user")"
-
-   # if --password was specified
-   if [ -f "$__object/parameter/password" ]; then
-      password="$(cat "$__object/parameter/password")"
-      cat <<-EOFF
-      mysql -u root <<-EOF
-      	GRANT ALL PRIVILEGES ON $database.* to '$user'@'localhost' IDENTIFIED BY '$password';
-EOF
-EOFF
-   else
-      cat <<-EOFF
-      mysql -u root <<-EOF
-      	GRANT ALL PRIVILEGES ON $database.* to '$user'@'localhost';
-EOF
-EOFF
-   fi
+if [ -f "$__object/parameter/name" ]
+then
+    name="$( cat "$__object/parameter/name" )"
+else
+    name="$__object_id"
 fi
+
+case "$state_should" in
+    present)
+        echo "mysql -e 'create database \`$name\`'"
+        echo "create database $name" >> "$__messages_out"
+    ;;
+    absent)
+        echo "mysql -e 'drop database \`$name\`'"
+        echo "drop database $name" >> "$__messages_out"
+    ;;
+esac
diff --git a/cdist/conf/type/__mysql_database/man.rst b/cdist/conf/type/__mysql_database/man.rst
deleted file mode 100644
index 1e245a08..00000000
--- a/cdist/conf/type/__mysql_database/man.rst
+++ /dev/null
@@ -1,49 +0,0 @@
-cdist-type__mysql_database(7)
-=============================
-
-NAME
-----
-cdist-type__mysql_database - Manage a MySQL database
-
-
-DESCRIPTION
------------
-This cdist type allows you to install a MySQL database.
-
-
-REQUIRED PARAMETERS
--------------------
-None.
-
-OPTIONAL PARAMETERS
--------------------
-name
-   The name of the database to install
-   defaults to the object id
-
-user
-   A user that should have access to the database
-
-password
-   The password for the user who manages the database
-
-
-EXAMPLES
---------
-
-.. code-block:: sh
-
-    __mysql_database "cdist" --name "cdist" --user "myuser" --password "mypwd"
-
-
-AUTHORS
--------
-Benedikt Koeppel <code@benediktkoeppel.ch>
-
-
-COPYING
--------
-Copyright \(C) 2012 Benedikt Koeppel. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
diff --git a/cdist/conf/type/__mysql_database/manifest b/cdist/conf/type/__mysql_database/manifest
new file mode 100755
index 00000000..a57c31ce
--- /dev/null
+++ b/cdist/conf/type/__mysql_database/manifest
@@ -0,0 +1,26 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/user" ]
+then
+    user="$( cat "$__object/parameter/user" )"
+fi
+
+if [ -f "$__object/parameter/password" ]
+then
+    password="$( cat "$__object/parameter/password" )"
+fi
+
+if [ -n "$user" ] && [ -n "$password" ]
+then
+    if [ -f "$__object/parameter/name" ]
+    then
+        database="$( cat "$__object/parameter/name" )"
+    else
+        database="$__object_id"
+    fi
+
+    __mysql_user "$user" --password "$password"
+
+    require="__mysql_user/$user" \
+        __mysql_privileges "$database/$user" --database "$database" --user "$user"
+fi
diff --git a/cdist/conf/type/__mysql_database/parameter/default/state b/cdist/conf/type/__mysql_database/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__mysql_database/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__mysql_database/parameter/optional b/cdist/conf/type/__mysql_database/parameter/optional
index 756afee7..6c0b1e85 100644
--- a/cdist/conf/type/__mysql_database/parameter/optional
+++ b/cdist/conf/type/__mysql_database/parameter/optional
@@ -1,3 +1,4 @@
 name
 user
 password
+state
diff --git a/cdist/conf/type/__mysql_privileges/explorer/state b/cdist/conf/type/__mysql_privileges/explorer/state
new file mode 100755
index 00000000..97674479
--- /dev/null
+++ b/cdist/conf/type/__mysql_privileges/explorer/state
@@ -0,0 +1,22 @@
+#!/bin/sh -e
+
+privileges="$( cat "$__object/parameter/privileges" )"
+
+database="$( cat "$__object/parameter/database" )"
+
+table="$( cat "$__object/parameter/table" )"
+
+user="$( cat "$__object/parameter/user" )"
+
+host="$( cat "$__object/parameter/host" )"
+
+check_privileges="$( 
+    mysql -B -N -e "show grants for '$user'@'$host'" \
+        | grep -Ei "^grant $privileges on .$database.\..$table. to " || true )"
+
+if [ -n "$check_privileges" ]
+then
+    echo 'present'
+else
+    echo 'absent'
+fi
diff --git a/cdist/conf/type/__mysql_privileges/gencode-remote b/cdist/conf/type/__mysql_privileges/gencode-remote
new file mode 100755
index 00000000..6b2e0fc1
--- /dev/null
+++ b/cdist/conf/type/__mysql_privileges/gencode-remote
@@ -0,0 +1,31 @@
+#!/bin/sh -e
+
+state_is="$( cat "$__object/explorer/state" )"
+
+state_should="$( cat "$__object/parameter/state" )"
+
+if [ "$state_is" = "$state_should" ]
+then
+    exit 0
+fi
+
+privileges="$( cat "$__object/parameter/privileges" )"
+
+database="$( cat "$__object/parameter/database" )"
+
+table="$( cat "$__object/parameter/table" )"
+
+user="$( cat "$__object/parameter/user" )"
+
+host="$( cat "$__object/parameter/host" )"
+
+case "$state_should" in
+    present)
+        echo "mysql -e 'grant $privileges on \`$database\`.\`$table\` to \`$user\`@\`$host\`'"
+        echo "grant $privileges on $database.$table to $user@$host" >> "$__messages_out"
+    ;;
+    absent)
+        echo "mysql -e 'revoke $privileges on \`$database\`.\`$table\` from \`$user\`@\`$host\`'"
+        echo "revoke $privileges on $database.$table from $user@$host" >> "$__messages_out"
+    ;;
+esac
diff --git a/cdist/conf/type/__mysql_privileges/parameter/default/host b/cdist/conf/type/__mysql_privileges/parameter/default/host
new file mode 100644
index 00000000..2fbb50c4
--- /dev/null
+++ b/cdist/conf/type/__mysql_privileges/parameter/default/host
@@ -0,0 +1 @@
+localhost
diff --git a/cdist/conf/type/__mysql_privileges/parameter/default/privileges b/cdist/conf/type/__mysql_privileges/parameter/default/privileges
new file mode 100644
index 00000000..5472efad
--- /dev/null
+++ b/cdist/conf/type/__mysql_privileges/parameter/default/privileges
@@ -0,0 +1 @@
+all privileges
diff --git a/cdist/conf/type/__mysql_privileges/parameter/default/state b/cdist/conf/type/__mysql_privileges/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__mysql_privileges/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__mysql_privileges/parameter/default/table b/cdist/conf/type/__mysql_privileges/parameter/default/table
new file mode 100644
index 00000000..72e8ffc0
--- /dev/null
+++ b/cdist/conf/type/__mysql_privileges/parameter/default/table
@@ -0,0 +1 @@
+*
diff --git a/cdist/conf/type/__mysql_privileges/parameter/optional b/cdist/conf/type/__mysql_privileges/parameter/optional
new file mode 100644
index 00000000..d4ed5bc5
--- /dev/null
+++ b/cdist/conf/type/__mysql_privileges/parameter/optional
@@ -0,0 +1,4 @@
+privileges
+table
+host
+state
diff --git a/cdist/conf/type/__mysql_privileges/parameter/required b/cdist/conf/type/__mysql_privileges/parameter/required
new file mode 100644
index 00000000..152b4a1e
--- /dev/null
+++ b/cdist/conf/type/__mysql_privileges/parameter/required
@@ -0,0 +1,2 @@
+database
+user
diff --git a/cdist/conf/type/__mysql_user/explorer/state b/cdist/conf/type/__mysql_user/explorer/state
new file mode 100755
index 00000000..c91bb36a
--- /dev/null
+++ b/cdist/conf/type/__mysql_user/explorer/state
@@ -0,0 +1,36 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/name" ]
+then
+    name="$( cat "$__object/parameter/name" )"
+else
+    name="$__object_id"
+fi
+
+if [ -f "$__object/parameter/password" ]
+then
+    password="$( cat "$__object/parameter/password" )"
+else
+    password=''
+fi
+
+host="$( cat "$__object/parameter/host" )"
+
+check_user="$( mysql -B -N -e "select user from mysql.user where user = '$name' and host = '$host'" )"
+
+if [ -n "$check_user" ]
+then
+    if [ -n "$password" ]
+    then
+        check_password="$( mysql -B -N -e "select user from mysql.user where user = '$name' and host = '$host' and password = password( '$password' )" )"
+    fi
+
+    if [ -n "$password" ] && [ -z "$check_password" ]
+    then
+        echo 'change-password'
+    else
+        echo 'present'
+    fi
+else
+    echo 'absent'
+fi
diff --git a/cdist/conf/type/__mysql_user/gencode-remote b/cdist/conf/type/__mysql_user/gencode-remote
new file mode 100755
index 00000000..67500716
--- /dev/null
+++ b/cdist/conf/type/__mysql_user/gencode-remote
@@ -0,0 +1,50 @@
+#!/bin/sh -e
+
+state_is="$( cat "$__object/explorer/state" )"
+
+state_should="$( cat "$__object/parameter/state" )"
+
+if [ "$state_is" = "$state_should" ]
+then
+    exit 0
+fi
+
+if [ -f "$__object/parameter/name" ]
+then
+    name="$( cat "$__object/parameter/name" )"
+else
+    name="$__object_id"
+fi
+
+host="$( cat "$__object/parameter/host" )"
+
+if [ -f "$__object/parameter/password" ]
+then
+    password="$( cat "$__object/parameter/password" )"
+else
+    if [ "$state_should" = 'present' ]
+    then
+        echo '--password needed' >&2
+        exit 1
+    else
+        password=''
+    fi
+fi
+
+if [ "$state_is" = 'absent' ] && [ "$state_should" = 'present' ]
+then
+    echo "mysql -e 'create user \`$name\`@\`$host\` identified by \"$password\"'"
+    echo "create user $name@$host" >> "$__messages_out"
+
+elif [ "$state_is" != 'absent' ] && [ "$state_should" = 'absent' ]
+then
+    echo "mysql -e 'drop user \`$name\`@\`$host\`'"
+    echo "drop user $name@$host" >> "$__messages_out"
+
+elif [ "$state_is" = 'change-password' ]
+then
+    # this only works with MySQL 5.7.6 and later or MariaDB 10.1.20 and later
+    echo "mysql -e 'alter user \`$name\`@\`$host\` identified by \"$password\"'"
+    echo "mysql -e 'flush privileges'"
+    echo "change password $name@$host" >> "$__messages_out"
+fi
diff --git a/cdist/conf/type/__mysql_user/parameter/default/host b/cdist/conf/type/__mysql_user/parameter/default/host
new file mode 100644
index 00000000..2fbb50c4
--- /dev/null
+++ b/cdist/conf/type/__mysql_user/parameter/default/host
@@ -0,0 +1 @@
+localhost
diff --git a/cdist/conf/type/__mysql_user/parameter/default/state b/cdist/conf/type/__mysql_user/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__mysql_user/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__mysql_user/parameter/optional b/cdist/conf/type/__mysql_user/parameter/optional
new file mode 100644
index 00000000..a286266c
--- /dev/null
+++ b/cdist/conf/type/__mysql_user/parameter/optional
@@ -0,0 +1,4 @@
+name
+host
+password
+state

From 9a693537f4192e3cd133e14cd31ab1bdcc792608 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 1 Jan 2020 12:38:12 +0200
Subject: [PATCH 104/743] __mysql_*: add license headers

---
 .../conf/type/__mysql_database/explorer/state  | 18 ++++++++++++++++++
 .../conf/type/__mysql_database/gencode-remote  | 18 ++++++++++++++++++
 cdist/conf/type/__mysql_database/manifest      | 18 ++++++++++++++++++
 .../type/__mysql_privileges/explorer/state     | 18 ++++++++++++++++++
 .../type/__mysql_privileges/gencode-remote     | 18 ++++++++++++++++++
 cdist/conf/type/__mysql_user/explorer/state    | 18 ++++++++++++++++++
 cdist/conf/type/__mysql_user/gencode-remote    | 18 ++++++++++++++++++
 7 files changed, 126 insertions(+)

diff --git a/cdist/conf/type/__mysql_database/explorer/state b/cdist/conf/type/__mysql_database/explorer/state
index 16cc9ce5..79858695 100755
--- a/cdist/conf/type/__mysql_database/explorer/state
+++ b/cdist/conf/type/__mysql_database/explorer/state
@@ -1,4 +1,22 @@
 #!/bin/sh -e
+#
+# 2020 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 if [ -f "$__object/parameter/name" ]
 then
diff --git a/cdist/conf/type/__mysql_database/gencode-remote b/cdist/conf/type/__mysql_database/gencode-remote
index d3692572..1bdb2b11 100755
--- a/cdist/conf/type/__mysql_database/gencode-remote
+++ b/cdist/conf/type/__mysql_database/gencode-remote
@@ -1,4 +1,22 @@
 #!/bin/sh -e
+#
+# 2020 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 state_is="$( cat "$__object/explorer/state" )"
 
diff --git a/cdist/conf/type/__mysql_database/manifest b/cdist/conf/type/__mysql_database/manifest
index a57c31ce..628b543c 100755
--- a/cdist/conf/type/__mysql_database/manifest
+++ b/cdist/conf/type/__mysql_database/manifest
@@ -1,4 +1,22 @@
 #!/bin/sh -e
+#
+# 2020 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 if [ -f "$__object/parameter/user" ]
 then
diff --git a/cdist/conf/type/__mysql_privileges/explorer/state b/cdist/conf/type/__mysql_privileges/explorer/state
index 97674479..0cfbaacd 100755
--- a/cdist/conf/type/__mysql_privileges/explorer/state
+++ b/cdist/conf/type/__mysql_privileges/explorer/state
@@ -1,4 +1,22 @@
 #!/bin/sh -e
+#
+# 2020 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 privileges="$( cat "$__object/parameter/privileges" )"
 
diff --git a/cdist/conf/type/__mysql_privileges/gencode-remote b/cdist/conf/type/__mysql_privileges/gencode-remote
index 6b2e0fc1..bcd362e6 100755
--- a/cdist/conf/type/__mysql_privileges/gencode-remote
+++ b/cdist/conf/type/__mysql_privileges/gencode-remote
@@ -1,4 +1,22 @@
 #!/bin/sh -e
+#
+# 2020 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 state_is="$( cat "$__object/explorer/state" )"
 
diff --git a/cdist/conf/type/__mysql_user/explorer/state b/cdist/conf/type/__mysql_user/explorer/state
index c91bb36a..6817ee9d 100755
--- a/cdist/conf/type/__mysql_user/explorer/state
+++ b/cdist/conf/type/__mysql_user/explorer/state
@@ -1,4 +1,22 @@
 #!/bin/sh -e
+#
+# 2020 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 if [ -f "$__object/parameter/name" ]
 then
diff --git a/cdist/conf/type/__mysql_user/gencode-remote b/cdist/conf/type/__mysql_user/gencode-remote
index 67500716..5f13bc87 100755
--- a/cdist/conf/type/__mysql_user/gencode-remote
+++ b/cdist/conf/type/__mysql_user/gencode-remote
@@ -1,4 +1,22 @@
 #!/bin/sh -e
+#
+# 2020 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 state_is="$( cat "$__object/explorer/state" )"
 

From 24862e0208705cd9081f5963d1f07e0fcceb23f2 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 3 Jan 2020 18:26:11 +0200
Subject: [PATCH 105/743] __mysql_database: carry over state

---
 cdist/conf/type/__mysql_database/manifest | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__mysql_database/manifest b/cdist/conf/type/__mysql_database/manifest
index 628b543c..a3c9ed5d 100755
--- a/cdist/conf/type/__mysql_database/manifest
+++ b/cdist/conf/type/__mysql_database/manifest
@@ -37,8 +37,16 @@ then
         database="$__object_id"
     fi
 
-    __mysql_user "$user" --password "$password"
+    state_should="$( cat "$__object/parameter/state" )"
 
+    __mysql_user "$user" \
+        --password "$password" \
+        --state "$state_should"
+
+    # removing user should remove all user's privileges
     require="__mysql_user/$user" \
-        __mysql_privileges "$database/$user" --database "$database" --user "$user"
+        __mysql_privileges "$database/$user" \
+            --database "$database" \
+            --user "$user" \
+            --state "$state_should"
 fi

From fcc774cb7b2b3f2128dc77d622d7801397d906b7 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 3 Jan 2020 18:33:23 +0200
Subject: [PATCH 106/743] __mysql_database: add manual

---
 cdist/conf/type/__mysql_database/man.rst | 55 ++++++++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 cdist/conf/type/__mysql_database/man.rst

diff --git a/cdist/conf/type/__mysql_database/man.rst b/cdist/conf/type/__mysql_database/man.rst
new file mode 100644
index 00000000..b3b56b5f
--- /dev/null
+++ b/cdist/conf/type/__mysql_database/man.rst
@@ -0,0 +1,55 @@
+cdist-type__mysql_database(7)
+=============================
+
+NAME
+----
+cdist-type__mysql_database - Manage a MySQL database
+
+
+DESCRIPTION
+-----------
+
+Create MySQL database and optionally user with all privileges.
+
+
+OPTIONAL PARAMETERS
+-------------------
+name
+   Name of database. Defaults to object id.
+
+user
+   Create user and give all privileges to database.
+
+password
+   Password for user.
+
+state
+   Defaults to present.
+   If absent and user is also set, both will be removed (with privileges).
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # just create database
+    __mysql_database foo
+
+    # create database with respective user with all privileges to database
+    __mysql_database bar \
+        --user name \
+        --password secret
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option) any
+later version.

From 5e8dc7122d764896247258ae4c9049e222f4d7ff Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 3 Jan 2020 18:48:11 +0200
Subject: [PATCH 107/743] __mysql_user: add manual

---
 cdist/conf/type/__mysql_user/man.rst | 48 ++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 cdist/conf/type/__mysql_user/man.rst

diff --git a/cdist/conf/type/__mysql_user/man.rst b/cdist/conf/type/__mysql_user/man.rst
new file mode 100644
index 00000000..c2b222d5
--- /dev/null
+++ b/cdist/conf/type/__mysql_user/man.rst
@@ -0,0 +1,48 @@
+cdist-type__mysql_user(7)
+=========================
+
+NAME
+----
+cdist-type__mysql_user - Manage a MySQL user
+
+
+DESCRIPTION
+-----------
+
+Create MySQL user or change password for the user.
+
+
+OPTIONAL PARAMETERS
+-------------------
+name
+   Name of user. Defaults to object id.
+
+host
+   Host of user. Defaults to localhost.
+
+password
+   Password of user.
+
+state
+   Defaults to present.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __mysql_user user --password secret
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option) any
+later version.

From 4329cced82930a336378765b294b9a3be9433991 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 3 Jan 2020 18:55:55 +0200
Subject: [PATCH 108/743] __mysql_privileges: add manual

---
 cdist/conf/type/__mysql_privileges/man.rst | 57 ++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 cdist/conf/type/__mysql_privileges/man.rst

diff --git a/cdist/conf/type/__mysql_privileges/man.rst b/cdist/conf/type/__mysql_privileges/man.rst
new file mode 100644
index 00000000..8208d7d4
--- /dev/null
+++ b/cdist/conf/type/__mysql_privileges/man.rst
@@ -0,0 +1,57 @@
+cdist-type__mysql_privileges(7)
+===============================
+
+NAME
+----
+cdist-type__mysql_privileges - Manage MySQL privileges
+
+
+DESCRIPTION
+-----------
+
+Grant and revoke privileges of MySQL user.
+
+
+REQUIRED PARAMETERS
+-------------------
+database
+   Name of database.
+
+User
+   Name of user.
+
+
+OPTIONAL PARAMETERS
+-------------------
+privileges
+   Defaults to "all".
+
+table
+   Defaults to "*".
+
+host
+   Defaults to localhost.
+
+state
+   "present" grants and "absent" revokes. Defaults to present.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __mysql_privileges user-to-db --database db --user user
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option) any
+later version.

From 6369bc1ae5e17dd9c0a13d57419107cbdb166245 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 4 Jan 2020 10:07:53 +0100
Subject: [PATCH 109/743] ++changelog

---
 docs/changelog | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/changelog b/docs/changelog
index 415af0af..8feb0b83 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -2,8 +2,9 @@ Changelog
 ---------
 
 next:
-	* Type __consul_agent: don't deploy init script on Alpine anymore,
-	it ships with one itself (Nico Schottelius)
+	* Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius)
+	* Type __install_chroot_umount: Bugfix: type was not using __chroot_umount/manifest (Steven Armstrong)
+	* Types __postgres_*: Use double quoted identifiers in generated SQL (fnux)
 
 6.3.0: 2019-12-12
 	* Type __package_update_index: Fix Alpine part (Dominique Roux)

From 72935e0a797461d39f64b9f85bac02ad06347b1d Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 4 Jan 2020 11:08:48 +0100
Subject: [PATCH 110/743] ++changelog

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 8feb0b83..d0a3a2de 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,7 +4,7 @@ Changelog
 next:
 	* Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius)
 	* Type __install_chroot_umount: Bugfix: type was not using __chroot_umount/manifest (Steven Armstrong)
-	* Types __postgres_*: Use double quoted identifiers in generated SQL (fnux)
+	* Types __postgres_*: Use double quoted identifiers in generated SQL (Timothée Floure)
 
 6.3.0: 2019-12-12
 	* Type __package_update_index: Fix Alpine part (Dominique Roux)

From e4596593c08f43284e1fa85d1e353b733bebdc57 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 30 Dec 2019 09:32:15 +0100
Subject: [PATCH 111/743] Add cdist info command

---
 cdist/argparse.py       |  32 +++++++
 cdist/exec/local.py     |  42 ++-------
 cdist/exec/util.py      |  25 ++++++
 cdist/info.py           | 183 ++++++++++++++++++++++++++++++++++++++++
 docs/src/man1/cdist.rst |  37 +++++++-
 5 files changed, 283 insertions(+), 36 deletions(-)
 create mode 100644 cdist/info.py

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 7dc683f3..611c484a 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -6,6 +6,7 @@ import collections
 import functools
 import cdist.configuration
 import cdist.preos
+import cdist.info
 
 
 # set of beta sub-commands
@@ -436,6 +437,37 @@ def get_parsers():
                   ' should be POSIX compatible shell.'))
     parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
 
+    # Info
+    parser['info'] = parser['sub'].add_parser('info')
+    parser['info'].add_argument(
+            '-a', '--all', help='Display all info. This is the default.',
+            action='store_true', default=False)
+    parser['info'].add_argument(
+            '-c', '--conf-dir',
+            help='Add configuration directory (can be repeated).',
+            action='append')
+    parser['info'].add_argument(
+            '-e', '--global-explorers',
+            help='Display info for global explorers.', action='store_true',
+            default=False)
+    parser['info'].add_argument(
+            '-F', '--fixed-string',
+            help='Interpret pattern as a fixed string.', action='store_true',
+            default=False)
+    parser['info'].add_argument(
+            '-f', '--full', help='Display full details.',
+            action='store_true', default=False)
+    parser['info'].add_argument(
+           '-g', '--config-file',
+           help='Use specified custom configuration file.',
+           dest="config_file", required=False)
+    parser['info'].add_argument(
+            '-t', '--types', help='Display info for types.',
+            action='store_true', default=False)
+    parser['info'].add_argument(
+            'pattern', nargs='?', help='Glob pattern.')
+    parser['info'].set_defaults(func=cdist.info.Info.commandline)
+
     for p in parser:
         parser[p].epilog = EPILOG
 
diff --git a/cdist/exec/local.py b/cdist/exec/local.py
index f83c85df..ad6c6e36 100644
--- a/cdist/exec/local.py
+++ b/cdist/exec/local.py
@@ -69,7 +69,6 @@ class Local(object):
 
         self.exec_path = exec_path
         self.custom_initial_manifest = initial_manifest
-        self._add_conf_dirs = add_conf_dirs
         self.cache_path_pattern = cache_path_pattern
         self.quiet_mode = quiet_mode
         if configuration:
@@ -84,16 +83,7 @@ class Local(object):
         self._init_cache_dir(None)
         self._init_paths()
         self._init_object_marker()
-        self._init_conf_dirs()
-
-    @property
-    def dist_conf_dir(self):
-        return os.path.abspath(os.path.join(os.path.dirname(cdist.__file__),
-                                            "conf"))
-
-    @property
-    def home_dir(self):
-        return cdist.home_dir()
+        self._init_conf_dirs(add_conf_dirs)
 
     def _init_log(self):
         self.log = logging.getLogger(self.target_host[0])
@@ -140,28 +130,9 @@ class Local(object):
         # Does not need to be secure - just randomly different from .cdist
         self.object_marker_name = tempfile.mktemp(prefix='.cdist-', dir='')
 
-    def _init_conf_dirs(self):
-        self.conf_dirs = []
-
-        self.conf_dirs.append(self.dist_conf_dir)
-
-        # Is the default place for user created explorer, type and manifest
-        if self.home_dir:
-            self.conf_dirs.append(self.home_dir)
-
-        # Add directories defined in the CDIST_PATH environment variable
-        # if 'CDIST_PATH' in os.environ:
-        #     cdist_path_dirs = re.split(r'(?<!\\):', os.environ['CDIST_PATH'])
-        #     cdist_path_dirs.reverse()
-        #     self.conf_dirs.extend(cdist_path_dirs)
-        if 'conf_dir' in self.configuration:
-            conf_dirs = self.configuration['conf_dir']
-            if conf_dirs:
-                self.conf_dirs.extend(conf_dirs)
-
-        # Add command line supplied directories
-        if self._add_conf_dirs:
-            self.conf_dirs.extend(self._add_conf_dirs)
+    def _init_conf_dirs(self, add_conf_dirs):
+        self.conf_dirs = util.resolve_conf_dirs(
+            self.configuration, add_conf_dirs=add_conf_dirs)
 
     def _init_directories(self):
         self.mkdir(self.conf_path)
@@ -187,10 +158,11 @@ class Local(object):
             self.object_marker_name, self.object_marker_file))
 
     def _init_cache_dir(self, cache_dir):
+        home_dir = cdist.home_dir()
         if cache_dir:
             self.cache_path = cache_dir
-        elif self.home_dir:
-            self.cache_path = os.path.join(self.home_dir, "cache")
+        elif home_dir:
+            self.cache_path = os.path.join(home_dir, "cache")
         else:
             raise cdist.Error(
                 "No homedir setup and no cache dir location given")
diff --git a/cdist/exec/util.py b/cdist/exec/util.py
index c96f757b..5513f01d 100644
--- a/cdist/exec/util.py
+++ b/cdist/exec/util.py
@@ -175,3 +175,28 @@ def log_std_fd(log, command, stdfd, prefix):
         stdfd.seek(0, 0)
         log.trace("Command: {}; {}: {}".format(
             command, prefix, stdfd.read().decode()))
+
+
+def dist_conf_dir():
+    return os.path.abspath(os.path.join(os.path.dirname(cdist.__file__),
+                                        "conf"))
+
+
+def resolve_conf_dirs(configuration, add_conf_dirs):
+    conf_dirs = []
+
+    conf_dirs.append(dist_conf_dir())
+
+    home_dir = cdist.home_dir()
+    if home_dir:
+        conf_dirs.append(home_dir)
+
+    if 'conf_dir' in configuration:
+        x = configuration['conf_dir']
+        if x:
+            conf_dirs.extend(x)
+
+    if add_conf_dirs:
+        conf_dirs.extend(add_conf_dirs)
+    conf_dirs = set(conf_dirs)
+    return conf_dirs
diff --git a/cdist/info.py b/cdist/info.py
new file mode 100644
index 00000000..4c1d3560
--- /dev/null
+++ b/cdist/info.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2019-2020 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import cdist
+import cdist.configuration
+import cdist.core
+import cdist.exec.util as util
+import os
+import glob
+import fnmatch
+
+
+class Info(object):
+
+    def __init__(self, conf_dirs, args):
+        self.conf_dirs = conf_dirs
+        self.all = args.all
+        self.display_global_explorers = args.global_explorers
+        self.display_types = args.types
+        if not self.display_global_explorers and not self.display_types:
+            self.all = True
+        self.fixed_string = args.fixed_string
+        self._setup_glob_pattern(args.pattern)
+        self.full = args.full
+
+    def _setup_glob_pattern(self, pattern):
+        if pattern is None:
+            self.glob_pattern = '*'
+        elif ('?' in pattern or '*' in pattern or '[' in pattern or
+              self.fixed_string):
+            self.glob_pattern = pattern
+        else:
+            self.glob_pattern = '*' + pattern + '*'
+
+    @classmethod
+    def commandline(cls, args):
+        cfg = cdist.configuration.Configuration(args)
+        configuration = cfg.get_config(section='GLOBAL')
+        conf_dirs = util.resolve_conf_dirs(configuration,
+                                           args.conf_dir)
+        c = cls(conf_dirs, args)
+        c.run()
+
+    def _get_global_explorers(self, conf_path):
+        rv = []
+        global_explorer_path = os.path.join(conf_path, "explorer",
+                                            self.glob_pattern)
+        if self.fixed_string:
+            if os.path.exists(global_explorer_path):
+                rv.append(global_explorer_path)
+        else:
+            for explorer in glob.glob(global_explorer_path):
+                rv.append(explorer)
+        return rv
+
+    def _should_display_type(self, dir_entry):
+        if not dir_entry.is_dir():
+            return False
+        if self.glob_pattern is None:
+            return True
+        if self.fixed_string:
+            return dir_entry.name == self.glob_pattern
+        else:
+            return fnmatch.fnmatch(dir_entry.name, self.glob_pattern)
+
+    def _get_types(self, conf_path):
+        rv = []
+        types_path = os.path.join(conf_path, "type")
+        if not os.path.exists(types_path):
+            return rv
+        with os.scandir(types_path) as it:
+            for entry in it:
+                if self._should_display_type(entry):
+                    rv.append(entry.path)
+        return rv
+
+    def _display_details(self, title, details, default_values=None,
+                         deprecated=None):
+        if not details:
+            return
+        if isinstance(details, bool):
+            print("\t{}: {}".format(title, 'yes' if details else 'no'))
+        elif isinstance(details, str):
+            print("\t{}: {}".format(title, details))
+        elif isinstance(details, list):
+            dv = dict(default_values) if default_values else {}
+            dp = dict(deprecated) if deprecated else {}
+
+            print("\t{}:".format(title))
+            for x in sorted(details):
+                print("\t\t{}".format(x), end='')
+                has_default = x in dv
+                is_deprecated = x in dp
+                need_comma = False
+                if has_default or is_deprecated:
+                    print(" (", end='')
+                if has_default:
+                    print("default: {}".format(dv[x]), end='')
+                    need_comma = True
+                if is_deprecated:
+                    print("{}deprecated".format(', ' if need_comma else ''),
+                          end='')
+                if has_default or is_deprecated:
+                    print(")", end='')
+                print()
+
+    def _display_type_parameters(self, cdist_type):
+        self._display_details("required parameters",
+                              cdist_type.required_parameters,
+                              default_values=cdist_type.parameter_defaults,
+                              deprecated=cdist_type.deprecated_parameters)
+        self._display_details("required multiple parameters",
+                              cdist_type.required_multiple_parameters,
+                              default_values=cdist_type.parameter_defaults,
+                              deprecated=cdist_type.deprecated_parameters)
+        self._display_details("optional parameters",
+                              cdist_type.optional_parameters,
+                              default_values=cdist_type.parameter_defaults,
+                              deprecated=cdist_type.deprecated_parameters)
+        self._display_details("optional multiple parameters",
+                              cdist_type.optional_multiple_parameters,
+                              default_values=cdist_type.parameter_defaults,
+                              deprecated=cdist_type.deprecated_parameters)
+        self._display_details("boolean parameters",
+                              cdist_type.boolean_parameters,
+                              default_values=cdist_type.parameter_defaults,
+                              deprecated=cdist_type.deprecated_parameters)
+
+    def _display_type_characteristics(self, cdist_type):
+        characteristics = []
+        if cdist_type.is_install:
+            characteristics.append('install')
+        else:
+            characteristics.append('config')
+        if cdist_type.is_singleton:
+            characteristics.append('singleton')
+        if cdist_type.is_nonparallel:
+            characteristics.append('nonparallel')
+        else:
+            characteristics.append('parallel')
+        if cdist_type.deprecated is not None:
+            characteristics.append('deprecated')
+        print("\t{}".format(', '.join(characteristics)))
+
+    def _display_type_details(self, type_path):
+        dirname, basename = os.path.split(type_path)
+        cdist_type = cdist.core.CdistType(dirname, basename)
+
+        self._display_type_characteristics(cdist_type)
+        self._display_type_parameters(cdist_type)
+
+    def run(self):
+        rv = []
+        for conf_path in self.conf_dirs:
+            if self.all or self.display_global_explorers:
+                rv.extend((x, 'E', ) for x in self._get_global_explorers(
+                    conf_path))
+            if self.all or self.display_types:
+                rv.extend((x, 'T', ) for x in self._get_types(conf_path))
+        rv = sorted(rv, key=lambda x: x[0])
+        for x, t in rv:
+            print(x)
+            if self.full and t == 'T':
+                self._display_type_details(x)
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 55db82ed..66c356ec 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -11,7 +11,7 @@ SYNOPSIS
 
 ::
 
-    cdist [-h] [-V] {banner,config,install,inventory,preos,shell} ...
+    cdist [-h] [-V] {banner,config,install,inventory,preos,shell,info} ...
 
     cdist banner [-h] [-l LOGLEVEL] [-q] [-v]
 
@@ -84,6 +84,8 @@ SYNOPSIS
 
     cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL]
 
+    cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-t] [pattern]
+
 
 DESCRIPTION
 -----------
@@ -604,6 +606,39 @@ usage. Its primary use is for debugging type parameters.
     be POSIX compatible shell.
 
 
+INFO
+----
+Display information for cdist (global explorers, types).
+
+**pattern**
+    Glob pattern. If it contains special characters('?', '*', '[') then it is
+    used as specified, otherwise it is translated to `*pattern*`.
+
+**-h, --help**
+    Show help message and exit.
+
+**-a, --all**
+    Display all info. This is the default.
+
+**-c CONF_DIR, --conf-dir CONF_DIR**
+    Add configuration directory (can be repeated).
+
+**-e, --global-explorers**
+    Display info for global explorers.
+
+**-F, --fixed-string**
+    Interpret pattern as a fixed string.
+
+**-f, --full**
+    Display full details.
+
+**-g CONFIG_FILE, --config-file CONFIG_FILE**
+    Use specified custom configuration file.
+
+**-t, --types**
+    Display info for types.
+
+
 CONFIGURATION
 -------------
 cdist obtains configuration data from the following sources in the following

From 7b1192257d8517cc838f84a8c57ac0964d104588 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 4 Jan 2020 13:17:54 +0100
Subject: [PATCH 112/743] Fix incomplete cdist info synopsis

---
 docs/src/man1/cdist.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 66c356ec..bc73a0b8 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -84,7 +84,7 @@ SYNOPSIS
 
     cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL]
 
-    cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-t] [pattern]
+    cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern]
 
 
 DESCRIPTION

From e2015367925c8a2716e6a79f6f2609f2877cd134 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 4 Jan 2020 13:18:13 +0100
Subject: [PATCH 113/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index d0a3a2de..7489489e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius)
 	* Type __install_chroot_umount: Bugfix: type was not using __chroot_umount/manifest (Steven Armstrong)
 	* Types __postgres_*: Use double quoted identifiers in generated SQL (Timothée Floure)
+	* Core: Add cdist info command (Darko Poljak)
 
 6.3.0: 2019-12-12
 	* Type __package_update_index: Fix Alpine part (Dominique Roux)

From bc1990c7c8244b2d00a91bc13dbf796d91c21041 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 4 Jan 2020 13:44:04 +0100
Subject: [PATCH 114/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 7489489e..fc4d69a8 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,8 @@ next:
 	* Type __install_chroot_umount: Bugfix: type was not using __chroot_umount/manifest (Steven Armstrong)
 	* Types __postgres_*: Use double quoted identifiers in generated SQL (Timothée Floure)
 	* Core: Add cdist info command (Darko Poljak)
+	* New types: __mysql_user, __mysql_privileges (Ander Punnar)
+	* Type __mysql_database: Rewrite (Ander Punnar)
 
 6.3.0: 2019-12-12
 	* Type __package_update_index: Fix Alpine part (Dominique Roux)

From 7c9dd3b03e31f39cbe758510e2aa1f542eae4825 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 4 Jan 2020 14:36:14 +0100
Subject: [PATCH 115/743] Release 6.4.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index fc4d69a8..706d76af 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.4.0: 2020-01-04
 	* Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius)
 	* Type __install_chroot_umount: Bugfix: type was not using __chroot_umount/manifest (Steven Armstrong)
 	* Types __postgres_*: Use double quoted identifiers in generated SQL (Timothée Floure)

From d1a64596fe73697d3e6a8e514991ffc173d04772 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 4 Jan 2020 14:56:05 +0100
Subject: [PATCH 116/743] Update build-helper

---
 bin/build-helper | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/build-helper b/bin/build-helper
index 69dee4c7..ed41e438 100755
--- a/bin/build-helper
+++ b/bin/build-helper
@@ -370,7 +370,7 @@ eof
         cat << eof
 Manual steps post release:
     - cdist-web
-    - send mail body generated in mailinglist.tmp and inform Dmitry for deb
+    - send generated mailinglist.tmp mail
     - twitter
 eof
     ;;

From 94e32dcd78d73ea5f09845cfdd4f488095f17145 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sat, 4 Jan 2020 16:36:44 +0200
Subject: [PATCH 117/743] __apt_unattended_upgrades: initial commit

---
 .../type/__apt_unattended_upgrades/manifest   | 74 +++++++++++++++++++
 .../parameter/boolean                         |  1 +
 .../parameter/optional                        |  1 +
 .../parameter/optional_multiple               |  1 +
 .../type/__apt_unattended_upgrades/singleton  |  0
 5 files changed, 77 insertions(+)
 create mode 100755 cdist/conf/type/__apt_unattended_upgrades/manifest
 create mode 100644 cdist/conf/type/__apt_unattended_upgrades/parameter/boolean
 create mode 100644 cdist/conf/type/__apt_unattended_upgrades/parameter/optional
 create mode 100644 cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple
 create mode 100644 cdist/conf/type/__apt_unattended_upgrades/singleton

diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest
new file mode 100755
index 00000000..bf4d825a
--- /dev/null
+++ b/cdist/conf/type/__apt_unattended_upgrades/manifest
@@ -0,0 +1,74 @@
+#!/bin/sh -e
+
+__package unattended-upgrades
+
+# in normal circumstances 20auto-upgrades is managed
+# by debconf and it can only contain these lines
+# https://wiki.debian.org/UnattendedUpgrades
+
+require='__package/unattended-upgrades' \
+    __file /etc/apt/apt.conf.d/20auto-upgrades \
+        --owner root \
+        --group root \
+        --mode 644 \
+        --source - << EOF
+APT::Periodic::Update-Package-Lists "1";
+APT::Periodic::Unattended-Upgrade "1";
+EOF
+
+# lets not write into upstream 50unattended-upgrades file,
+# but use our own config files to avoid clashes
+
+ml_conf='/etc/apt/apt.conf.d/51unattended-upgrades-mail'
+bl_conf='/etc/apt/apt.conf.d/51unattended-upgrades-blacklist'
+
+if [ -f "$__object/parameter/mail" ]
+then
+    mail="$( cat "$__object/parameter/mail" )"
+else
+    mail=''
+fi
+
+if [ -n "$mail" ]
+then
+    if [ -f "$__object/parameter/mail-on-error" ]
+    then
+        mail_on_error='true'
+    else
+        mail_on_error='false'
+    fi
+
+    __file "$ml_conf" \
+        --owner root \
+        --group root \
+        --mode 644 \
+        --source - << EOF
+Unattended-Upgrade::Mail "$mail";
+Unattended-Upgrade::MailOnlyOnError "$mail_on_error";
+EOF
+
+else
+    __file "$ml_conf" --state absent
+fi
+
+if [ -f "$__object/parameter/blacklist" ]
+then
+    bl='Unattended-Upgrade::Package-Blacklist {';
+
+    while read -r l
+    do
+        bl="$( printf '%s\n"%s";\n' "$bl" "$l" )"
+    done \
+        < "$__object/parameter/blacklist"
+
+    bl="$( printf '%s\n}' "$bl" )"
+
+    echo "$bl" \
+        | __file "$bl_conf" \
+            --owner root \
+            --group root \
+            --mode 644 \
+            --source -
+else
+    __file "$bl_conf" --state absent
+fi
diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean b/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean
new file mode 100644
index 00000000..edcaa12a
--- /dev/null
+++ b/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean
@@ -0,0 +1 @@
+mail-on-error
diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/optional b/cdist/conf/type/__apt_unattended_upgrades/parameter/optional
new file mode 100644
index 00000000..fa7963cc
--- /dev/null
+++ b/cdist/conf/type/__apt_unattended_upgrades/parameter/optional
@@ -0,0 +1 @@
+mail
diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple b/cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple
new file mode 100644
index 00000000..27b9ffc9
--- /dev/null
+++ b/cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple
@@ -0,0 +1 @@
+blacklist
diff --git a/cdist/conf/type/__apt_unattended_upgrades/singleton b/cdist/conf/type/__apt_unattended_upgrades/singleton
new file mode 100644
index 00000000..e69de29b

From a9d491f998c2b41ad97bfeace7f1d24a73ff7482 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sat, 4 Jan 2020 16:59:42 +0200
Subject: [PATCH 118/743] __apt_unattended_upgrades: add license header

---
 .../type/__apt_unattended_upgrades/manifest    | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest
index bf4d825a..88a5ccd8 100755
--- a/cdist/conf/type/__apt_unattended_upgrades/manifest
+++ b/cdist/conf/type/__apt_unattended_upgrades/manifest
@@ -1,4 +1,22 @@
 #!/bin/sh -e
+#
+# 2020 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 __package unattended-upgrades
 

From ec8d9571f971c0a0616acdccc9a32589f1b6f042 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sat, 4 Jan 2020 17:07:46 +0200
Subject: [PATCH 119/743] __apt_unattended_upgrades: add manual

---
 .../type/__apt_unattended_upgrades/man.rst    | 54 +++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 cdist/conf/type/__apt_unattended_upgrades/man.rst

diff --git a/cdist/conf/type/__apt_unattended_upgrades/man.rst b/cdist/conf/type/__apt_unattended_upgrades/man.rst
new file mode 100644
index 00000000..d64b2e9e
--- /dev/null
+++ b/cdist/conf/type/__apt_unattended_upgrades/man.rst
@@ -0,0 +1,54 @@
+cdist-type__apt_unattended_upgrades(7)
+======================================
+
+NAME
+----
+cdist-type__apt_unattended_upgrades - automatic installation of updates
+
+
+DESCRIPTION
+-----------
+
+Install and configure unattended-upgrades package.
+
+
+OPTIONAL PARAMETERS
+-------------------
+mail
+   Send email to this address for problems or packages upgrades.
+
+
+OPTIONAL MULTIPLE PARAMETERS
+----------------------------
+blacklist
+   Python regular expressions, matching packages to exclude from upgrading.
+
+
+BOOLEAN PARAMETERS
+------------------
+mail-on-error
+   Get emails only on errors.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __apt_unattended_upgrades \
+        --mail root \
+        --mail-on-error \
+        --blacklist multipath-tools \
+        --blacklist open-iscsi
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option) any
+later version.

From d4bd49bbb598dfd4e4a510a2bf9035dcb4686e4a Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sat, 4 Jan 2020 17:43:57 +0200
Subject: [PATCH 120/743] __acl: rename --acl to --entry for the sake of
 consistency, add compatibility

---
 cdist/conf/type/__acl/gencode-remote          |  5 +++-
 cdist/conf/type/__acl/man.rst                 | 28 +++++++++----------
 .../conf/type/__acl/parameter/deprecated/acl  |  1 +
 .../type/__acl/parameter/optional_multiple    |  1 +
 4 files changed, 20 insertions(+), 15 deletions(-)
 create mode 100644 cdist/conf/type/__acl/parameter/deprecated/acl

diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote
index 6dab4d09..f4f0d1e2 100755
--- a/cdist/conf/type/__acl/gencode-remote
+++ b/cdist/conf/type/__acl/gencode-remote
@@ -28,7 +28,10 @@ acl_path="/$__object_id"
 
 acl_is="$( cat "$__object/explorer/acl_is" )"
 
-if [ -f "$__object/parameter/acl" ]
+if [ -f "$__object/parameter/entry" ]
+then
+    acl_should="$( cat "$__object/parameter/entry" )"
+elif [ -f "$__object/parameter/acl" ]
 then
     acl_should="$( cat "$__object/parameter/acl" )"
 elif
diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst
index 85e946ce..c3493e49 100644
--- a/cdist/conf/type/__acl/man.rst
+++ b/cdist/conf/type/__acl/man.rst
@@ -15,7 +15,7 @@ See ``setfacl`` and ``acl`` manpages for more details.
 
 REQUIRED MULTIPLE PARAMETERS
 ----------------------------
-acl
+entry
    Set ACL entry following ``getfacl`` output syntax.
 
 
@@ -36,8 +36,8 @@ remove
 
 DEPRECATED PARAMETERS
 ---------------------
-Parameters ``user``, ``group``, ``mask`` and ``other`` are deprecated and they
-will be removed in future versions. Please use ``acl`` parameter instead.
+Parameters ``acl``, ``user``, ``group``, ``mask`` and ``other`` are deprecated and they
+will be removed in future versions. Please use ``entry`` parameter instead.
 
 
 EXAMPLES
@@ -49,27 +49,27 @@ EXAMPLES
         --default \
         --recursive \
         --remove \
-        --acl user:alice:rwx \
-        --acl user:bob:r-x \
-        --acl group:project-group:rwx \
-        --acl group:some-other-group:r-x \
-        --acl mask::r-x \
-        --acl other::r-x
+        --entry user:alice:rwx \
+        --entry user:bob:r-x \
+        --entry group:project-group:rwx \
+        --entry group:some-other-group:r-x \
+        --entry mask::r-x \
+        --entry other::r-x
 
     # give Alice read-only access to subdir,
     # but don't allow her to see parent content.
 
     __acl /srv/project2 \
         --remove \
-        --acl default:group:secret-project:rwx \
-        --acl group:secret-project:rwx \
-        --acl user:alice:--x
+        --entry default:group:secret-project:rwx \
+        --entry group:secret-project:rwx \
+        --entry user:alice:--x
 
     __acl /srv/project2/subdir \
         --default \
         --remove \
-        --acl group:secret-project:rwx \
-        --acl user:alice:r-x
+        --entry group:secret-project:rwx \
+        --entry user:alice:r-x
 
 
 AUTHORS
diff --git a/cdist/conf/type/__acl/parameter/deprecated/acl b/cdist/conf/type/__acl/parameter/deprecated/acl
new file mode 100644
index 00000000..94e14159
--- /dev/null
+++ b/cdist/conf/type/__acl/parameter/deprecated/acl
@@ -0,0 +1 @@
+see manual for details
diff --git a/cdist/conf/type/__acl/parameter/optional_multiple b/cdist/conf/type/__acl/parameter/optional_multiple
index 95c25d55..c615d507 100644
--- a/cdist/conf/type/__acl/parameter/optional_multiple
+++ b/cdist/conf/type/__acl/parameter/optional_multiple
@@ -1,3 +1,4 @@
+entry
 acl
 user
 group

From 51ba4a49d8ec79968f79563f994489c619f10bac Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 4 Jan 2020 18:21:23 +0100
Subject: [PATCH 121/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 706d76af..a7bcf9b1 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar)
+
 6.4.0: 2020-01-04
 	* Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius)
 	* Type __install_chroot_umount: Bugfix: type was not using __chroot_umount/manifest (Steven Armstrong)

From 11f569959d6e331d4d5052ca73fb5d83bf9df8e7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 11 Jan 2020 14:16:33 +0100
Subject: [PATCH 122/743] Fix missing configuration file usage, support -g

PreOS code did not use configuration support. This fix adds support
for using cdist configuration, which takes into account cdist
configuration file, environment variables and command line options,
especially conf_dir.

It also adds support for -g, --config-file option, for specifying
custom configuration file.
---
 cdist/preos.py          | 24 +++++++++++-------------
 docs/changelog          |  1 +
 docs/src/man1/cdist.rst |  7 ++++++-
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/cdist/preos.py b/cdist/preos.py
index 378071db..491338d2 100644
--- a/cdist/preos.py
+++ b/cdist/preos.py
@@ -5,8 +5,9 @@ import inspect
 import argparse
 import cdist
 import logging
-import re
 import cdist.argparse
+import cdist.configuration
+import cdist.exec.util as util
 
 
 _PREOS_CALL = "commandline"
@@ -24,16 +25,6 @@ def extend_plugins_path(dirs):
             _PLUGINS_PATH.append(preos_dir)
 
 
-cdist_home = cdist.home_dir()
-if cdist_home:
-    extend_plugins_path((cdist_home, ))
-x = 'CDIST_PATH'
-if x in os.environ:
-    vals = re.split(r'(?<!\\):', os.environ[x])
-    vals = [x for x in vals if x]
-    extend_plugins_path(vals)
-
-
 def preos_plugin(obj):
     """It is preos if _PREOS_MARKER is True and has _PREOS_CALL."""
     if hasattr(obj, _PREOS_MARKER):
@@ -102,6 +93,9 @@ class PreOS(object):
                             help=('Add configuration directory (one that '
                                   'contains "preos" subdirectory)'),
                             action='append')
+        parser.add_argument('-g', '--config-file',
+                            help='Use specified custom configuration file.',
+                            dest="config_file", required=False)
         parser.add_argument('-L', '--list-preoses',
                             help='List available PreOS-es',
                             action='store_true', default=False)
@@ -110,8 +104,12 @@ class PreOS(object):
         cdist.argparse.handle_loglevel(args)
         log.debug("preos args : {}".format(args))
 
-        if args.conf_dir:
-            extend_plugins_path(args.conf_dir)
+        cfg = cdist.configuration.Configuration(args)
+        configuration = cfg.get_config(section='GLOBAL')
+        conf_dirs = util.resolve_conf_dirs(configuration,
+                                           args.conf_dir)
+
+        extend_plugins_path(conf_dirs)
         sys.path.extend(_PLUGINS_PATH)
         cls.preoses = find_preoses()
 
diff --git a/docs/changelog b/docs/changelog
index a7bcf9b1..1b1a909e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar)
+	* Core: preos: Fix missing configuration file usage, support -g, --config-file option (Darko Poljak)
 
 6.4.0: 2020-01-04
 	* Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius)
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index bc73a0b8..38248821 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -59,7 +59,9 @@ SYNOPSIS
                          [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t]
                          [host [host ...]]
 
-    cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-L] [preos] ...
+    cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-g CONFIG_FILE]
+                [-L]
+                [preos] ...
 
     cdist preos [preos-options] debian [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
                                        [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
@@ -464,6 +466,9 @@ Create PreOS.
 **-c CONF_DIR, --conf-dir CONF_DIR**
     Add configuration directory (one that contains "preos" subdirectory).
 
+**-g CONFIG_FILE, --config-file CONFIG_FILE**
+    Use specified custom configuration file.
+
 **-L, --list-preoses**
     List available PreOS-es.
 

From 3f133dbc1752437617c4b1589259a85827d79ef9 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 11 Jan 2020 15:26:46 +0100
Subject: [PATCH 123/743] Info command: support tilde expansion

---
 cdist/exec/util.py | 7 +++++++
 cdist/info.py      | 8 +++-----
 cdist/preos.py     | 7 ++-----
 docs/changelog     | 1 +
 4 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/cdist/exec/util.py b/cdist/exec/util.py
index 5513f01d..9787f431 100644
--- a/cdist/exec/util.py
+++ b/cdist/exec/util.py
@@ -24,6 +24,7 @@ import os
 from tempfile import TemporaryFile
 
 import cdist
+import cdist.configuration
 
 
 # IMPORTANT:
@@ -200,3 +201,9 @@ def resolve_conf_dirs(configuration, add_conf_dirs):
         conf_dirs.extend(add_conf_dirs)
     conf_dirs = set(conf_dirs)
     return conf_dirs
+
+
+def resolve_conf_dirs_from_config_and_args(args):
+    cfg = cdist.configuration.Configuration(args)
+    configuration = cfg.get_config(section='GLOBAL')
+    return resolve_conf_dirs(configuration, args.conf_dir)
diff --git a/cdist/info.py b/cdist/info.py
index 4c1d3560..b896a3d1 100644
--- a/cdist/info.py
+++ b/cdist/info.py
@@ -53,10 +53,7 @@ class Info(object):
 
     @classmethod
     def commandline(cls, args):
-        cfg = cdist.configuration.Configuration(args)
-        configuration = cfg.get_config(section='GLOBAL')
-        conf_dirs = util.resolve_conf_dirs(configuration,
-                                           args.conf_dir)
+        conf_dirs = util.resolve_conf_dirs_from_config_and_args(args)
         c = cls(conf_dirs, args)
         c.run()
 
@@ -170,7 +167,8 @@ class Info(object):
 
     def run(self):
         rv = []
-        for conf_path in self.conf_dirs:
+        for cp in self.conf_dirs:
+            conf_path = os.path.expanduser(cp)
             if self.all or self.display_global_explorers:
                 rv.extend((x, 'E', ) for x in self._get_global_explorers(
                     conf_path))
diff --git a/cdist/preos.py b/cdist/preos.py
index 491338d2..e353fe3b 100644
--- a/cdist/preos.py
+++ b/cdist/preos.py
@@ -101,13 +101,10 @@ class PreOS(object):
                             action='store_true', default=False)
         parser.add_argument('remainder_args', nargs=argparse.REMAINDER)
         args = parser.parse_args(argv[1:])
-        cdist.argparse.handle_loglevel(args)
+        st.argparse.handle_loglevel(args)
         log.debug("preos args : {}".format(args))
 
-        cfg = cdist.configuration.Configuration(args)
-        configuration = cfg.get_config(section='GLOBAL')
-        conf_dirs = util.resolve_conf_dirs(configuration,
-                                           args.conf_dir)
+        conf_dirs = util.resolve_conf_dirs_from_config_and_args(args)
 
         extend_plugins_path(conf_dirs)
         sys.path.extend(_PLUGINS_PATH)
diff --git a/docs/changelog b/docs/changelog
index 1b1a909e..526fc320 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar)
 	* Core: preos: Fix missing configuration file usage, support -g, --config-file option (Darko Poljak)
+	* Core info command: Support tilde expansion of conf directories (Darko Poljak)
 
 6.4.0: 2020-01-04
 	* Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius)

From 3258fc98e15fedbd98e17f7d0b568a38b8da139c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sun, 12 Jan 2020 12:19:49 +0100
Subject: [PATCH 124/743] Fix typo

---
 cdist/preos.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/preos.py b/cdist/preos.py
index e353fe3b..bf2a8e60 100644
--- a/cdist/preos.py
+++ b/cdist/preos.py
@@ -101,7 +101,7 @@ class PreOS(object):
                             action='store_true', default=False)
         parser.add_argument('remainder_args', nargs=argparse.REMAINDER)
         args = parser.parse_args(argv[1:])
-        st.argparse.handle_loglevel(args)
+        cdist.argparse.handle_loglevel(args)
         log.debug("preos args : {}".format(args))
 
         conf_dirs = util.resolve_conf_dirs_from_config_and_args(args)

From 93ec4b46aab9cd61d66cb375fcf7a2599c5ef9bb Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 15 Jan 2020 17:23:13 +0100
Subject: [PATCH 125/743] [__line] Ensure the line is only added once

---
 cdist/conf/type/__line/explorer/state | 22 +++++++++++-----------
 cdist/conf/type/__line/gencode-remote |  3 +++
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state
index 2ef252c8..9c0dd1b2 100755
--- a/cdist/conf/type/__line/explorer/state
+++ b/cdist/conf/type/__line/explorer/state
@@ -18,6 +18,17 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+if [ -f "$__object/parameter/file" ]; then
+   file="$(cat "$__object/parameter/file")"
+else
+   file="/$__object_id"
+fi
+
+if [ ! -f "$file" ]; then
+  echo "file_missing"
+  exit 0
+fi
+
 if [ -f "$__object/parameter/before" ]; then
    position="before"
 elif [ -f "$__object/parameter/after" ]; then
@@ -33,17 +44,6 @@ else
    needle="line"
 fi
 
-if [ -f "$__object/parameter/file" ]; then
-   file="$(cat "$__object/parameter/file")"
-else
-   file="/$__object_id"
-fi
-
-if [ ! -f "$file" ]; then
-  echo "file_missing"
-  exit 0
-fi
-
 awk -v position="$position" -v needle="$needle" '
 function _find(_text, _pattern) {
    if (needle == "regex") {
diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote
index 03e90c1b..0dd8609a 100755
--- a/cdist/conf/type/__line/gencode-remote
+++ b/cdist/conf/type/__line/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2018 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -104,10 +105,12 @@ BEGIN {
       if (anchor && match(\$0, anchor)) {
          if (position == "before") {
             print line
+            add = 0
             print
          } else if (position == "after") {
             print
             print line
+            add = 0
          }
          next
       }

From 629d0795c80bdf8af83a71712c643275be0799f1 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 15 Jan 2020 17:23:26 +0100
Subject: [PATCH 126/743] [__line] Always add line to end if anchor is not
 found

---
 cdist/conf/type/__line/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote
index 0dd8609a..c8c90c38 100755
--- a/cdist/conf/type/__line/gencode-remote
+++ b/cdist/conf/type/__line/gencode-remote
@@ -118,7 +118,7 @@ BEGIN {
    print
 }
 END {
-   if (add && position == "end") {
+   if (add) {
       print line
    }
 }

From 4cdb8aaa03d07aa72de8dd5961844699ad5888bd Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 15 Jan 2020 17:39:21 +0100
Subject: [PATCH 127/743] [__line/state] Make sure the index match is at the
 beginning

Without the == 1 all lines which contain --line as a substring match.  e.g. if
--line is "line" and the file contains the line "wrong line" this was considered
a match.
---
 cdist/conf/type/__line/explorer/state | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state
index 9c0dd1b2..28ec35e2 100755
--- a/cdist/conf/type/__line/explorer/state
+++ b/cdist/conf/type/__line/explorer/state
@@ -49,7 +49,7 @@ function _find(_text, _pattern) {
    if (needle == "regex") {
       return match(_text, _pattern)
    } else {
-      return index(_text, _pattern)
+      return index(_text, _pattern) == 1
    }
 }
 BEGIN {

From 51b1b11cc21e257acbce420ecfcd48ec37e66705 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 15 Jan 2020 17:54:40 +0100
Subject: [PATCH 128/743] [__line/state] Logic fixes in explorer

This commit fixes the incorrectly reported state "wrongposition" if position is
"after" and anchor is present in the file but the line missing.
---
 cdist/conf/type/__line/explorer/state | 41 +++++++++++++++------------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state
index 28ec35e2..6ff0a798 100755
--- a/cdist/conf/type/__line/explorer/state
+++ b/cdist/conf/type/__line/explorer/state
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2018 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,7 +20,7 @@
 #
 
 if [ -f "$__object/parameter/file" ]; then
-   file="$(cat "$__object/parameter/file")"
+   file=$(cat "$__object/parameter/file")
 else
    file="/$__object_id"
 fi
@@ -55,41 +56,45 @@ function _find(_text, _pattern) {
 BEGIN {
    getline anchor < (ENVIRON["__object"] "/parameter/" position)
    getline pattern < (ENVIRON["__object"] "/parameter/" needle)
-   state = "absent"
+
+   found_line = 0
+   correct_pos = (position != "after" && position != "before")
 }
 {
    if (position == "after") {
       if (match($0, anchor)) {
          getline
          if (_find($0, pattern)) {
-            state = "present"
+            found_line++
+            correct_pos = 1
+            exit 0
          }
-         else {
-            state = "wrongposition"
-         }
-         exit 0
+      } else if (_find($0, pattern)) {
+         found_line++
       }
-   }
-   else if (position == "before") {
+   } else if (position == "before") {
       if (_find($0, pattern)) {
+         found_line++
          getline
          if (match($0, anchor)) {
-            state = "present"
+            correct_pos = 1
+            exit 0
          }
-         else {
-            state = "wrongposition"
-         }
-         exit 0
       }
-   }
-   else {
+   } else {
       if (_find($0, pattern)) {
-         state = "present"
+         found_line++
          exit 0
       }
    }
 }
 END {
-   print state
+   if (found_line && correct_pos) {
+      print "present"
+   } else if (found_line) {
+      print "wrongposition"
+   } else {
+      print "absent"
+   }
 }
 ' "$file"

From cd2d5b3f79d1e3ea8545b67096d67d6ceda763e6 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 15 Jan 2020 19:24:21 +0200
Subject: [PATCH 129/743] __apt_unattended_upgrades: all objects depend on
 package

---
 cdist/conf/type/__apt_unattended_upgrades/manifest | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest
index 88a5ccd8..811e79a6 100755
--- a/cdist/conf/type/__apt_unattended_upgrades/manifest
+++ b/cdist/conf/type/__apt_unattended_upgrades/manifest
@@ -20,16 +20,17 @@
 
 __package unattended-upgrades
 
+export require='__package/unattended-upgrades'
+
 # in normal circumstances 20auto-upgrades is managed
 # by debconf and it can only contain these lines
 # https://wiki.debian.org/UnattendedUpgrades
 
-require='__package/unattended-upgrades' \
-    __file /etc/apt/apt.conf.d/20auto-upgrades \
-        --owner root \
-        --group root \
-        --mode 644 \
-        --source - << EOF
+__file /etc/apt/apt.conf.d/20auto-upgrades \
+    --owner root \
+    --group root \
+    --mode 644 \
+    --source - << EOF
 APT::Periodic::Update-Package-Lists "1";
 APT::Periodic::Unattended-Upgrade "1";
 EOF

From 68e5502fce3d3ae043ae0792a14abd1970834d36 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 15 Jan 2020 19:27:02 +0200
Subject: [PATCH 130/743] __apt_unattended_upgrades: move debian wiki link to
 manual

---
 cdist/conf/type/__apt_unattended_upgrades/man.rst  | 2 ++
 cdist/conf/type/__apt_unattended_upgrades/manifest | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_unattended_upgrades/man.rst b/cdist/conf/type/__apt_unattended_upgrades/man.rst
index d64b2e9e..f75c6513 100644
--- a/cdist/conf/type/__apt_unattended_upgrades/man.rst
+++ b/cdist/conf/type/__apt_unattended_upgrades/man.rst
@@ -11,6 +11,8 @@ DESCRIPTION
 
 Install and configure unattended-upgrades package.
 
+For more information see https://wiki.debian.org/UnattendedUpgrades.
+
 
 OPTIONAL PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest
index 811e79a6..03cc2c50 100755
--- a/cdist/conf/type/__apt_unattended_upgrades/manifest
+++ b/cdist/conf/type/__apt_unattended_upgrades/manifest
@@ -24,7 +24,6 @@ export require='__package/unattended-upgrades'
 
 # in normal circumstances 20auto-upgrades is managed
 # by debconf and it can only contain these lines
-# https://wiki.debian.org/UnattendedUpgrades
 
 __file /etc/apt/apt.conf.d/20auto-upgrades \
     --owner root \

From 2cf44c66d46435e859fe0528d4b414dafd8b468a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 15 Jan 2020 19:00:30 +0100
Subject: [PATCH 131/743] [__directory] Add --state exists and --state
 pre-exists

---
 cdist/conf/type/__directory/gencode-remote | 17 ++++++++++++++---
 cdist/conf/type/__directory/man.rst        | 15 +++++++++++++--
 2 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote
index 374db47a..e1ab69d7 100755
--- a/cdist/conf/type/__directory/gencode-remote
+++ b/cdist/conf/type/__directory/gencode-remote
@@ -3,6 +3,7 @@
 # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
 # 2013 Steven Armstrong (steven-cdist armstrong.cc)
 # 2014 Daniel Heule (hda at sfs.biz)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -21,8 +22,8 @@
 #
 
 destination="/$__object_id"
-state_should="$(cat "$__object/parameter/state")"
-type="$(cat "$__object/explorer/type")"
+state_should=$(cat "$__object/parameter/state")
+type=$(cat "$__object/explorer/type")
 stat_file="$__object/explorer/stat"
 
 # variable to keep track if we have to set directory attributes
@@ -72,7 +73,7 @@ set_mode() {
 }
 
 case "$state_should" in
-   present)
+   present|exists)
       if [ "$type" != "directory" ]; then
          set_attributes=1
          if [ "$type" != "none" ]; then
@@ -83,6 +84,10 @@ case "$state_should" in
          fi
          echo "mkdir $mkdiropt '$destination'"
          echo "create" >> "$__messages_out"
+      elif [ "$state_should" = 'exists' ]; then
+         # The type is directory and --state exists. We are done and do not
+         # check or set the attributes.
+         exit 0
       fi
 
       # Note: Mode - needs to happen last as a chown/chgrp can alter mode by
@@ -103,6 +108,12 @@ case "$state_should" in
          fi
       done
    ;;
+   pre-exists)
+      if [ "$type" != "directory" ]; then
+         echo "Directory \"$destination\" does not exist" >&2
+         exit 1
+      fi
+   ;;
    absent)
         if [ "$type" = "directory" ]; then
             echo "rm -rf '$destination'"
diff --git a/cdist/conf/type/__directory/man.rst b/cdist/conf/type/__directory/man.rst
index 74b00afe..7755334c 100644
--- a/cdist/conf/type/__directory/man.rst
+++ b/cdist/conf/type/__directory/man.rst
@@ -19,7 +19,18 @@ None.
 OPTIONAL PARAMETERS
 -------------------
 state
-   'present' or 'absent', defaults to 'present'
+   'present', 'absent', 'exists' or 'pre-exists', defaults to 'present' where:
+
+   present
+      the directory exists and the given attributes are set.
+   absent
+      the directory does not exist.
+   exists
+      the directory exists, but its attributes are not altered if it already
+      existed.
+   pre-exists
+      check that the directory exists and is indeed a directory, but do not
+      create or modify it.
 
 group
    Group to chgrp to.
@@ -36,7 +47,7 @@ BOOLEAN PARAMETERS
 parents
    Whether to create parents as well (mkdir -p behaviour).
    Warning: all intermediate directory permissions default
-   to whatever mkdir -p does. 
+   to whatever mkdir -p does.
 
    Usually this means root:root, 0700.
 

From cd24a806e7be7b3300543943e8c7a5582a55d15e Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 15 Jan 2020 20:48:32 +0200
Subject: [PATCH 132/743] __apt_unattended_upgrades: rewrite manifest, add more
 parameters

---
 .../type/__apt_unattended_upgrades/manifest   | 88 +++++++++++++------
 .../parameter/boolean                         |  4 +
 2 files changed, 63 insertions(+), 29 deletions(-)

diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest
index 03cc2c50..cb99b611 100755
--- a/cdist/conf/type/__apt_unattended_upgrades/manifest
+++ b/cdist/conf/type/__apt_unattended_upgrades/manifest
@@ -34,39 +34,61 @@ APT::Periodic::Update-Package-Lists "1";
 APT::Periodic::Unattended-Upgrade "1";
 EOF
 
-# lets not write into upstream 50unattended-upgrades file,
-# but use our own config files to avoid clashes
-
-ml_conf='/etc/apt/apt.conf.d/51unattended-upgrades-mail'
-bl_conf='/etc/apt/apt.conf.d/51unattended-upgrades-blacklist'
+conf='# this file is managed by cdist'
 
 if [ -f "$__object/parameter/mail" ]
 then
-    mail="$( cat "$__object/parameter/mail" )"
-else
-    mail=''
+    conf="$(
+        printf \
+            '%s\nUnattended-Upgrade::Mail "%s";\n' \
+            "$conf" \
+            "$( cat "$__object/parameter/mail" )"
+        )"
 fi
 
-if [ -n "$mail" ]
+if [ -f "$__object/parameter/mail-on-error" ]
 then
-    if [ -f "$__object/parameter/mail-on-error" ]
-    then
-        mail_on_error='true'
-    else
-        mail_on_error='false'
-    fi
+    conf="$(
+        printf \
+            '%s\nUnattended-Upgrade::MailOnlyOnError "true";\n' \
+            "$conf"
+        )"
+fi
 
-    __file "$ml_conf" \
-        --owner root \
-        --group root \
-        --mode 644 \
-        --source - << EOF
-Unattended-Upgrade::Mail "$mail";
-Unattended-Upgrade::MailOnlyOnError "$mail_on_error";
-EOF
+if [ -f "$__object/parameter/no-auto-fix" ]
+then
+    conf="$(
+        printf \
+            '%s\nUnattended-Upgrade::AutoFixInterruptedDpkg "false";\n' \
+            "$conf"
+        )"
+fi
 
-else
-    __file "$ml_conf" --state absent
+if [ -f "$__object/parameter/no-minimal-steps" ]
+then
+    conf="$(
+        printf \
+            '%s\nUnattended-Upgrade::MinimalSteps "false";\n' \
+            "$conf"
+        )"
+fi
+
+if [ -f "$__object/parameter/on-shutdown" ]
+then
+    conf="$(
+        printf \
+            '%s\nUnattended-Upgrade::InstallOnShutdown "true";\n' \
+            "$conf"
+        )"
+fi
+
+if [ -f "$__object/parameter/reboot" ]
+then
+    conf="$(
+        printf \
+            '%s\nUnattended-Upgrade::Automatic-Reboot "true";\n' \
+            "$conf"
+        )"
 fi
 
 if [ -f "$__object/parameter/blacklist" ]
@@ -79,14 +101,22 @@ then
     done \
         < "$__object/parameter/blacklist"
 
-    bl="$( printf '%s\n}' "$bl" )"
+    conf="$( printf '%s\n%s\n}\n' "$conf" "$bl" )"
+fi
 
-    echo "$bl" \
-        | __file "$bl_conf" \
+# lets not write into upstream 50unattended-upgrades file,
+# but use our own config files to avoid clashes
+
+conf_file='/etc/apt/apt.conf.d/51unattended-upgrades-cdist'
+
+if [ "$( echo "$conf" | wc -l )" -gt 1 ]
+then
+    echo "$conf" \
+        | __file "$conf_file" \
             --owner root \
             --group root \
             --mode 644 \
             --source -
 else
-    __file "$bl_conf" --state absent
+    __file "$conf_file" --state absent
 fi
diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean b/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean
index edcaa12a..6ad9790f 100644
--- a/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean
+++ b/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean
@@ -1 +1,5 @@
 mail-on-error
+no-auto-fix
+no-minimal-steps
+on-shutdown
+reboot

From b3f36dbe5bc7d1e21892f765ee08dfc5d8c6a6e3 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 15 Jan 2020 20:51:47 +0200
Subject: [PATCH 133/743] __apt_unattended_upgrades: fix typo

---
 cdist/conf/type/__apt_unattended_upgrades/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest
index cb99b611..04225a9b 100755
--- a/cdist/conf/type/__apt_unattended_upgrades/manifest
+++ b/cdist/conf/type/__apt_unattended_upgrades/manifest
@@ -105,7 +105,7 @@ then
 fi
 
 # lets not write into upstream 50unattended-upgrades file,
-# but use our own config files to avoid clashes
+# but use our own config file to avoid clashes
 
 conf_file='/etc/apt/apt.conf.d/51unattended-upgrades-cdist'
 

From 5a9a1ba57fc45df63cff123a42905360c3a7bddb Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 15 Jan 2020 22:00:56 +0100
Subject: [PATCH 134/743] [__line] Produce error when file does not exist

---
 cdist/conf/type/__line/explorer/state |  5 +----
 cdist/conf/type/__line/gencode-remote | 17 +++++++++++------
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state
index 6ff0a798..e8fc3630 100755
--- a/cdist/conf/type/__line/explorer/state
+++ b/cdist/conf/type/__line/explorer/state
@@ -25,10 +25,7 @@ else
    file="/$__object_id"
 fi
 
-if [ ! -f "$file" ]; then
-  echo "file_missing"
-  exit 0
-fi
+[ -f "$file" ] || exit 0
 
 if [ -f "$__object/parameter/before" ]; then
    position="before"
diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote
index c8c90c38..88cae68b 100755
--- a/cdist/conf/type/__line/gencode-remote
+++ b/cdist/conf/type/__line/gencode-remote
@@ -24,9 +24,20 @@ if [ -f "$__object/parameter/before" ] && [ -f "$__object/parameter/after" ]; th
    exit 1
 fi
 
+if [ -f "$__object/parameter/file" ]; then
+   file="$(cat "$__object/parameter/file")"
+else
+   file="/$__object_id"
+fi
+
 state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
 
+if [ -z "$state_is" ]; then
+   printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2
+   exit 1
+fi
+
 if [ "$state_should" = "$state_is" ]; then
    # nothing to do
    exit 0
@@ -47,12 +58,6 @@ else
    needle="line"
 fi
 
-if [ -f "$__object/parameter/file" ]; then
-   file="$(cat "$__object/parameter/file")"
-else
-   file="/$__object_id"
-fi
-
 add=0
 remove=0
 case "$state_should" in

From f5f70671cb3e4ca8b2d5e2a0e66a1847f05ef4bf Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 16 Jan 2020 00:59:03 +0200
Subject: [PATCH 135/743] __apt_unattended_upgrades: make parameters more
 speaking

---
 cdist/conf/type/__apt_unattended_upgrades/manifest        | 8 ++++----
 .../conf/type/__apt_unattended_upgrades/parameter/boolean | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest
index 04225a9b..094dfa1c 100755
--- a/cdist/conf/type/__apt_unattended_upgrades/manifest
+++ b/cdist/conf/type/__apt_unattended_upgrades/manifest
@@ -46,7 +46,7 @@ then
         )"
 fi
 
-if [ -f "$__object/parameter/mail-on-error" ]
+if [ -f "$__object/parameter/mail-only-on-error" ]
 then
     conf="$(
         printf \
@@ -55,7 +55,7 @@ then
         )"
 fi
 
-if [ -f "$__object/parameter/no-auto-fix" ]
+if [ -f "$__object/parameter/no-auto-fix-interrupted-dpkg" ]
 then
     conf="$(
         printf \
@@ -73,7 +73,7 @@ then
         )"
 fi
 
-if [ -f "$__object/parameter/on-shutdown" ]
+if [ -f "$__object/parameter/install-on-shutdown" ]
 then
     conf="$(
         printf \
@@ -82,7 +82,7 @@ then
         )"
 fi
 
-if [ -f "$__object/parameter/reboot" ]
+if [ -f "$__object/parameter/automatic-reboot" ]
 then
     conf="$(
         printf \
diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean b/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean
index 6ad9790f..831dc95c 100644
--- a/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean
+++ b/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean
@@ -1,5 +1,5 @@
-mail-on-error
-no-auto-fix
+mail-only-on-error
+no-auto-fix-interrupted-dpkg
 no-minimal-steps
-on-shutdown
-reboot
+install-on-shutdown
+automatic-reboot

From 6fa0d687f7bdf3ba1e6a032118553d90ceba0b22 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 16 Jan 2020 01:13:27 +0200
Subject: [PATCH 136/743] __apt_unattended_upgrades: alternative approach

---
 .../type/__apt_unattended_upgrades/manifest   | 100 ++++++------------
 .../parameter/boolean                         |   5 -
 .../parameter/optional                        |   1 -
 .../parameter/optional_multiple               |   1 +
 4 files changed, 31 insertions(+), 76 deletions(-)
 delete mode 100644 cdist/conf/type/__apt_unattended_upgrades/parameter/boolean
 delete mode 100644 cdist/conf/type/__apt_unattended_upgrades/parameter/optional

diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest
index 094dfa1c..2f8f6e76 100755
--- a/cdist/conf/type/__apt_unattended_upgrades/manifest
+++ b/cdist/conf/type/__apt_unattended_upgrades/manifest
@@ -34,81 +34,41 @@ APT::Periodic::Update-Package-Lists "1";
 APT::Periodic::Unattended-Upgrade "1";
 EOF
 
-conf='# this file is managed by cdist'
-
-if [ -f "$__object/parameter/mail" ]
-then
-    conf="$(
-        printf \
-            '%s\nUnattended-Upgrade::Mail "%s";\n' \
-            "$conf" \
-            "$( cat "$__object/parameter/mail" )"
-        )"
-fi
-
-if [ -f "$__object/parameter/mail-only-on-error" ]
-then
-    conf="$(
-        printf \
-            '%s\nUnattended-Upgrade::MailOnlyOnError "true";\n' \
-            "$conf"
-        )"
-fi
-
-if [ -f "$__object/parameter/no-auto-fix-interrupted-dpkg" ]
-then
-    conf="$(
-        printf \
-            '%s\nUnattended-Upgrade::AutoFixInterruptedDpkg "false";\n' \
-            "$conf"
-        )"
-fi
-
-if [ -f "$__object/parameter/no-minimal-steps" ]
-then
-    conf="$(
-        printf \
-            '%s\nUnattended-Upgrade::MinimalSteps "false";\n' \
-            "$conf"
-        )"
-fi
-
-if [ -f "$__object/parameter/install-on-shutdown" ]
-then
-    conf="$(
-        printf \
-            '%s\nUnattended-Upgrade::InstallOnShutdown "true";\n' \
-            "$conf"
-        )"
-fi
-
-if [ -f "$__object/parameter/automatic-reboot" ]
-then
-    conf="$(
-        printf \
-            '%s\nUnattended-Upgrade::Automatic-Reboot "true";\n' \
-            "$conf"
-        )"
-fi
-
-if [ -f "$__object/parameter/blacklist" ]
-then
-    bl='Unattended-Upgrade::Package-Blacklist {';
-
-    while read -r l
-    do
-        bl="$( printf '%s\n"%s";\n' "$bl" "$l" )"
-    done \
-        < "$__object/parameter/blacklist"
-
-    conf="$( printf '%s\n%s\n}\n' "$conf" "$bl" )"
-fi
-
 # lets not write into upstream 50unattended-upgrades file,
 # but use our own config file to avoid clashes
 
 conf_file='/etc/apt/apt.conf.d/51unattended-upgrades-cdist'
 
+conf='# this file is managed by cdist'
+
+if [ -f "$__object/parameter/option" ]
+then
+    o=''
+
+    while read -r l
+    do
+        k="$( echo "$l" | awk -F= '{print $1}' )"
+        v="$( echo "$l" | awk -F= '{print $2}' )"
+        o="$( printf '%s\nUnattended-Upgrade::%s "%s";\n' "$o" "$k" "$v" )"
+    done \
+        < "$__object/parameter/option"
+
+    conf="$( printf '%s\n%s\n' "$conf" "$o" )"
+fi
+
+if [ -f "$__object/parameter/blacklist" ]
+then
+    b='Unattended-Upgrade::Package-Blacklist {'
+
+    while read -r l
+    do
+        b="$( printf '%s\n"%s";\n' "$b" "$l" )"
+    done \
+        < "$__object/parameter/blacklist"
+
+    conf="$( printf '%s\n%s\n}\n' "$conf" "$b" )"
+fi
+
 if [ "$( echo "$conf" | wc -l )" -gt 1 ]
 then
     echo "$conf" \
diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean b/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean
deleted file mode 100644
index 831dc95c..00000000
--- a/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean
+++ /dev/null
@@ -1,5 +0,0 @@
-mail-only-on-error
-no-auto-fix-interrupted-dpkg
-no-minimal-steps
-install-on-shutdown
-automatic-reboot
diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/optional b/cdist/conf/type/__apt_unattended_upgrades/parameter/optional
deleted file mode 100644
index fa7963cc..00000000
--- a/cdist/conf/type/__apt_unattended_upgrades/parameter/optional
+++ /dev/null
@@ -1 +0,0 @@
-mail
diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple b/cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple
index 27b9ffc9..ea4fba2b 100644
--- a/cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple
+++ b/cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple
@@ -1 +1,2 @@
+option
 blacklist

From ef2f4b9a004369cc390f1271721bc22d6a44f402 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Fri, 17 Jan 2020 11:21:28 +0100
Subject: [PATCH 137/743] __postgres_*: fix forgotten edge cases in delimited
 identifier escape

---
 cdist/conf/type/__postgres_database/gencode-remote |  8 ++++++--
 cdist/conf/type/__postgres_role/gencode-remote     | 10 ++++++----
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote
index 9f12d215..47e6b97c 100755
--- a/cdist/conf/type/__postgres_database/gencode-remote
+++ b/cdist/conf/type/__postgres_database/gencode-remote
@@ -43,10 +43,14 @@ if [ "$state_should" != "$state_is" ]; then
          if [ -f "$__object/parameter/owner" ]; then
             owner="-O \"$(cat "$__object/parameter/owner")\""
          fi
-         echo "su - '$postgres_user' -c \"createdb $owner \"$name\"\""
+         cat << EOF
+su - '$postgres_user' -c "createdb $owner \"$name\""
+EOF
       ;;
       absent)
-         echo "su - '$postgres_user' -c \"dropdb \"$name\"\""
+         cat << EOF
+su - '$postgres_user' -c "dropdb \"$name\""
+EOF
       ;;
    esac
 fi
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 0b8852f4..977832c9 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -53,11 +53,13 @@ case "$state_should" in
         done
 
         [ -n "$password" ] && password="PASSWORD '$password'"
-
-        cmd="CREATE ROLE \"$name\" WITH $password $booleans"
-        echo "su - '$postgres_user' -c \"psql postgres -wc \\\"$cmd\\\"\""
+        cat << EOF
+su - '$postgres_user' -c "psql postgres -wc 'CREATE ROLE \"$name\" WITH $password $booleans;'"
+EOF
     ;;
     absent)
-        echo "su - '$postgres_user' -c \"dropuser \\\"$name\\\"\""
+        cat << EOF
+su - '$postgres_user' -c "dropuser \"$name\""
+EOF
     ;;
 esac

From 6097cd9e90341146948886a9480cd0b70ee9e60f Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 17 Jan 2020 14:41:00 +0200
Subject: [PATCH 138/743] __apt_unattended_upgrades: use word expansions

---
 cdist/conf/type/__apt_unattended_upgrades/manifest | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest
index 2f8f6e76..3c00e2f4 100755
--- a/cdist/conf/type/__apt_unattended_upgrades/manifest
+++ b/cdist/conf/type/__apt_unattended_upgrades/manifest
@@ -47,9 +47,7 @@ then
 
     while read -r l
     do
-        k="$( echo "$l" | awk -F= '{print $1}' )"
-        v="$( echo "$l" | awk -F= '{print $2}' )"
-        o="$( printf '%s\nUnattended-Upgrade::%s "%s";\n' "$o" "$k" "$v" )"
+        o="$( printf '%s\nUnattended-Upgrade::%s "%s";\n' "$o" "${l%%=*}" "${l#*=}" )"
     done \
         < "$__object/parameter/option"
 

From 65c5b592a590f3f4f8b31a1b7c32b2a98354d054 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 17 Jan 2020 14:48:54 +0200
Subject: [PATCH 139/743] __apt_unattended_upgrades: update manual

---
 .../type/__apt_unattended_upgrades/man.rst    | 40 ++++++++++++-------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/cdist/conf/type/__apt_unattended_upgrades/man.rst b/cdist/conf/type/__apt_unattended_upgrades/man.rst
index f75c6513..2231b5f9 100644
--- a/cdist/conf/type/__apt_unattended_upgrades/man.rst
+++ b/cdist/conf/type/__apt_unattended_upgrades/man.rst
@@ -14,35 +14,47 @@ Install and configure unattended-upgrades package.
 For more information see https://wiki.debian.org/UnattendedUpgrades.
 
 
-OPTIONAL PARAMETERS
--------------------
-mail
-   Send email to this address for problems or packages upgrades.
-
-
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
+option
+   Set options for unattended-upgrades. See examples.
+
+   Supported options with default values (as of 2020-01-17) are:
+
+   - AutoFixInterruptedDpkg, default is "true"
+   - MinimalSteps, default is "true"
+   - InstallOnShutdown, default is "false"
+   - Mail, default is "" (empty)
+   - MailOnlyOnError, default is "false"
+   - Remove-Unused-Kernel-Packages, default is "true"
+   - Remove-New-Unused-Dependencies, default is "true"
+   - Remove-Unused-Dependencies, default is "false"
+   - Automatic-Reboot, default is "false"
+   - Automatic-Reboot-WithUsers, default is "true"
+   - Automatic-Reboot-Time, default is "02:00"
+   - SyslogEnable, default is "false"
+   - SyslogFacility, default is "daemon"
+   - OnlyOnACPower, default is "true"
+   - Skip-Updates-On-Metered-Connections, default is "true"
+   - Verbose, default is "false"
+   - Debug, default is "false"
+
 blacklist
    Python regular expressions, matching packages to exclude from upgrading.
 
 
-BOOLEAN PARAMETERS
-------------------
-mail-on-error
-   Get emails only on errors.
-
-
 EXAMPLES
 --------
 
 .. code-block:: sh
 
     __apt_unattended_upgrades \
-        --mail root \
-        --mail-on-error \
+        --option Mail=root \
+        --option MailOnlyOnError=true \
         --blacklist multipath-tools \
         --blacklist open-iscsi
 
+
 AUTHORS
 -------
 Ander Punnar <ander-at-kvlt-dot-ee>

From e292c9f1245a59cda5c721d4ce4e201ce9799349 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 17 Jan 2020 15:48:59 +0100
Subject: [PATCH 140/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 526fc320..bc251d64 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,8 @@ next:
 	* Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar)
 	* Core: preos: Fix missing configuration file usage, support -g, --config-file option (Darko Poljak)
 	* Core info command: Support tilde expansion of conf directories (Darko Poljak)
+	* Types __postgres_*: Fix edge cases in quoted identifiers (Timothée Floure)
+	* New type: __apt_unattended_upgrades (Ander Punnar)
 
 6.4.0: 2020-01-04
 	* Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius)

From 45890cc7e43577252e8560ca70611a88e9936ffa Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 17 Jan 2020 18:14:49 +0100
Subject: [PATCH 141/743] ++changelog

---
 docs/changelog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index bc251d64..ccaf848f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,12 @@ next:
 	* Core info command: Support tilde expansion of conf directories (Darko Poljak)
 	* Types __postgres_*: Fix edge cases in quoted identifiers (Timothée Floure)
 	* New type: __apt_unattended_upgrades (Ander Punnar)
+	* Type __line: Bugfixes: (Dennis Camera)
+		- ensure the line is only added once
+		- always add line to end if anchor is not found
+		- match line at the beginning when not regex
+		- fix incorrect 'wrongposition' in state explorer
+		- produce error when file does not exist
 
 6.4.0: 2020-01-04
 	* Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius)

From e1ac97b6a5b10d29b7b9bd70c5adea7e9c33b2e9 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 19 Jan 2020 14:02:00 +0200
Subject: [PATCH 142/743] __acl: add --source

---
 cdist/conf/type/__acl/gencode-remote     | 12 +++++++++++-
 cdist/conf/type/__acl/man.rst            | 12 ++++++++++++
 cdist/conf/type/__acl/parameter/optional |  1 +
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote
index f4f0d1e2..70a46af5 100755
--- a/cdist/conf/type/__acl/gencode-remote
+++ b/cdist/conf/type/__acl/gencode-remote
@@ -28,7 +28,17 @@ acl_path="/$__object_id"
 
 acl_is="$( cat "$__object/explorer/acl_is" )"
 
-if [ -f "$__object/parameter/entry" ]
+if [ -f "$__object/parameter/source" ]
+then
+    acl_source="$( cat "$__object/parameter/source" )"
+
+    if [ "$acl_source" = '-' ]
+    then
+        acl_should="$( cat "$__object/stdin" )"
+    else
+        acl_should="$( grep -Ev '^#|^$' "$acl_source" )"
+    fi
+elif [ -f "$__object/parameter/entry" ]
 then
     acl_should="$( cat "$__object/parameter/entry" )"
 elif [ -f "$__object/parameter/acl" ]
diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst
index c3493e49..e7ef9579 100644
--- a/cdist/conf/type/__acl/man.rst
+++ b/cdist/conf/type/__acl/man.rst
@@ -19,6 +19,14 @@ entry
    Set ACL entry following ``getfacl`` output syntax.
 
 
+OPTIONAL PARAMETERS
+-------------------
+source
+   Read ACL entries from stdin or file.
+   Ordering of entries is not important.
+   When reading from file, comments and empty lines are ignored.
+
+
 BOOLEAN PARAMETERS
 ------------------
 default
@@ -71,6 +79,10 @@ EXAMPLES
         --entry group:secret-project:rwx \
         --entry user:alice:r-x
 
+    # read acl from stdin
+    echo 'user:alice:rwx' \
+        | __acl /path/to/directory --source -
+
 
 AUTHORS
 -------
diff --git a/cdist/conf/type/__acl/parameter/optional b/cdist/conf/type/__acl/parameter/optional
index 4b32086b..12edcccb 100644
--- a/cdist/conf/type/__acl/parameter/optional
+++ b/cdist/conf/type/__acl/parameter/optional
@@ -1,2 +1,3 @@
 mask
 other
+source

From 5089f9055ecb0b826942c8f947eb26f9ef397aaa Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sun, 19 Jan 2020 19:41:26 +0100
Subject: [PATCH 143/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index ccaf848f..3f240e5a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -13,6 +13,7 @@ next:
 		- match line at the beginning when not regex
 		- fix incorrect 'wrongposition' in state explorer
 		- produce error when file does not exist
+	* Type __acl: Add --source parameter (Ander Punnar)
 
 6.4.0: 2020-01-04
 	* Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius)

From 1fbd2fc2bdf76697dd50808cd3927d15f6b8bc5c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 19 Jan 2020 22:51:40 +0200
Subject: [PATCH 144/743] __acl: add --file and --directory for convenience

---
 cdist/conf/type/__acl/gencode-remote     |  8 +++++++-
 cdist/conf/type/__acl/man.rst            | 13 +++++++++++++
 cdist/conf/type/__acl/manifest           | 11 +++++++++++
 cdist/conf/type/__acl/parameter/optional |  2 ++
 4 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100755 cdist/conf/type/__acl/manifest

diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote
index 70a46af5..e5404a9d 100755
--- a/cdist/conf/type/__acl/gencode-remote
+++ b/cdist/conf/type/__acl/gencode-remote
@@ -20,7 +20,13 @@
 
 file_is="$( cat "$__object/explorer/file_is" )"
 
-[ "$file_is" = 'missing' ] && [ -z "$__cdist_dry_run" ] && exit 0
+if [ "$file_is" = 'missing' ] \
+    && [ -z "$__cdist_dry_run" ] \
+    && \( [ ! -f "$__object/parameter/file" ] \
+        || [ ! -f "$__object/parameter/directory" ] \)
+then
+    exit 0
+fi
 
 os="$( cat "$__global/explorer/os" )"
 
diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst
index e7ef9579..28412871 100644
--- a/cdist/conf/type/__acl/man.rst
+++ b/cdist/conf/type/__acl/man.rst
@@ -26,6 +26,12 @@ source
    Ordering of entries is not important.
    When reading from file, comments and empty lines are ignored.
 
+file
+   Create/change file with ``__file`` using ``user:group:mode`` pattern.
+
+directory
+   Create/change directory with ``__directory`` using ``user:group:mode`` pattern.
+
 
 BOOLEAN PARAMETERS
 ------------------
@@ -83,6 +89,13 @@ EXAMPLES
     echo 'user:alice:rwx' \
         | __acl /path/to/directory --source -
 
+    # create/change directory too
+    __acl /path/to/directory \
+        --default \
+        --remove \
+        --directory root:root:770 \
+        --entry user:nobody:rwx
+
 
 AUTHORS
 -------
diff --git a/cdist/conf/type/__acl/manifest b/cdist/conf/type/__acl/manifest
new file mode 100755
index 00000000..5fd23110
--- /dev/null
+++ b/cdist/conf/type/__acl/manifest
@@ -0,0 +1,11 @@
+#!/bin/sh -e
+
+for p in file directory
+do
+    [ ! -f "$__object/parameter/$p" ] && continue
+
+    "__$p" "/$__object_id" \
+        --owner "$( awk -F: '{print $1}' "$__object/parameter/$p" )" \
+        --group "$( awk -F: '{print $2}' "$__object/parameter/$p" )" \
+        --mode  "$( awk -F: '{print $3}' "$__object/parameter/$p" )"
+done
diff --git a/cdist/conf/type/__acl/parameter/optional b/cdist/conf/type/__acl/parameter/optional
index 12edcccb..cdcbc0b8 100644
--- a/cdist/conf/type/__acl/parameter/optional
+++ b/cdist/conf/type/__acl/parameter/optional
@@ -1,3 +1,5 @@
 mask
 other
 source
+file
+directory

From 4e8ccd9b788c11403f586bf0c413d57546013c1b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 20 Jan 2020 09:08:55 +0100
Subject: [PATCH 145/743] ++changelog

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 3f240e5a..083cf5fb 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -13,7 +13,7 @@ next:
 		- match line at the beginning when not regex
 		- fix incorrect 'wrongposition' in state explorer
 		- produce error when file does not exist
-	* Type __acl: Add --source parameter (Ander Punnar)
+	* Type __acl: Add --source, --file and --directory parameters (Ander Punnar)
 
 6.4.0: 2020-01-04
 	* Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius)

From 8f12a4c505812d640cd926567ee8c6300a6341fb Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 23 Jan 2020 07:10:48 +0100
Subject: [PATCH 146/743] Release 6.5.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 083cf5fb..467c0f22 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.5.0: 2020-01-23
 	* Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar)
 	* Core: preos: Fix missing configuration file usage, support -g, --config-file option (Darko Poljak)
 	* Core info command: Support tilde expansion of conf directories (Darko Poljak)

From 87f30b6053315f11013bbf92ce1281bca2d54f43 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 23 Jan 2020 14:40:07 +0100
Subject: [PATCH 147/743] Update sphinx docs copyright year

---
 docs/src/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/conf.py b/docs/src/conf.py
index 78f9842c..47765413 100644
--- a/docs/src/conf.py
+++ b/docs/src/conf.py
@@ -56,7 +56,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'cdist'
-copyright = 'ungleich GmbH 2019'
+copyright = 'ungleich GmbH 2020'
 # author = 'Darko Poljak'
 
 # The version info for the project you're documenting, acts as replacement for

From de1a421b68edeb2bf3660f2ed98e109db93e8d88 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 2 Oct 2019 15:39:30 +0200
Subject: [PATCH 148/743] [explorer/init] Support for Darwin and more BusyBox
 combinations

---
 cdist/conf/explorer/init | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init
index a8a7857e..829d6ab8 100755
--- a/cdist/conf/explorer/init
+++ b/cdist/conf/explorer/init
@@ -23,14 +23,22 @@
 # for example at linux this value is "init" or "systemd" in most cases
 #
 
-uname_s="$(uname -s)"
-
-case "$uname_s" in
+case $(uname -s) in
     Linux)
-        (pgrep -P0 -l | awk '/^1[ \t]/ {print $2;}') || true
+        if command -v pgrep >/dev/null
+        then
+            # BusyBox's version of ps does not support some options.
+            # On Linux systems, we prefer pgrep to get the name of PID1.
+            (pgrep -P0 -l | awk '/^1[ \t]/ {print $2;}') || true
+        else
+            ps -o comm= -p 1 2>/dev/null || cat /proc/1/comm
+        fi
     ;;
     FreeBSD|OpenBSD)
-        ps -o comm= -p 1 || true
+        ps -o comm= -p 1 2>/dev/null || true
+    ;;
+    Darwin)
+        basename "$(ps -o comm= -p 1 2>/dev/null)"
     ;;
     *)
         # return a empty string as unknown value

From 4fe2dcba891ae52cccbe02553976be2bb7291fd1 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 30 Jan 2020 18:35:50 +0100
Subject: [PATCH 149/743] [explorer/init] Linux is a mess...

---
 cdist/conf/explorer/init | 44 +++++++++++++++++++++++++++++++++++-----
 1 file changed, 39 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init
index 829d6ab8..ceae2e9f 100755
--- a/cdist/conf/explorer/init
+++ b/cdist/conf/explorer/init
@@ -25,14 +25,48 @@
 
 case $(uname -s) in
     Linux)
-        if command -v pgrep >/dev/null
+        if test -d /proc/1/
         then
-            # BusyBox's version of ps does not support some options.
-            # On Linux systems, we prefer pgrep to get the name of PID1.
-            (pgrep -P0 -l | awk '/^1[ \t]/ {print $2;}') || true
+            comm_name=$(cat /proc/1/comm)
         else
-            ps -o comm= -p 1 2>/dev/null || cat /proc/1/comm
+            # BusyBox's versions of ps and pgrep do not support some options
+            # depending on which compile-time options have been used.
+            # Both pgrep and ps are tried to get the command name
+            comm_name=$(
+                pgrep -P0 -l 2>/dev/null | awk '/^1[ \t]/ { print $2 }'
+                || ps -o comm= -p 1 2>/dev/null)
         fi
+
+        case $comm_name
+        in
+            systemd)
+                echo systemd
+            ;;
+            init)
+                # It could be anything...
+
+                if test -h /proc/1/exe
+                then
+                    init_exe=/proc/1/exe
+                else
+                    init_exe=$(command -v "$comm_name")
+                fi
+
+                test -x "$comm_exe" || exit 1
+
+                case $("$comm_exe" --version | head -n 1)
+                in
+                    *SysV*)
+                        echo init
+                    ;;
+                    *upstart*)
+                        echo upstart
+                    ;;
+                    *)
+                        echo ""
+                    ;;
+                esac
+        esac
     ;;
     FreeBSD|OpenBSD)
         ps -o comm= -p 1 2>/dev/null || true

From e5d86ffc9360767502469d00a0db1aca72d4f483 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 1 Feb 2020 17:08:25 +0100
Subject: [PATCH 150/743] Fix to remove cronjobs when the cronjob expression
 did not match.

---
 cdist/conf/type/__cron/gencode-remote | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__cron/gencode-remote b/cdist/conf/type/__cron/gencode-remote
index 59398058..8f3c92dc 100755
--- a/cdist/conf/type/__cron/gencode-remote
+++ b/cdist/conf/type/__cron/gencode-remote
@@ -42,8 +42,12 @@ fi
 mkdir "$__object/files"
 echo "$entry" > "$__object/files/entry"
 
-if diff -q "$__object/files/entry" "$__object/explorer/entry" >/dev/null; then
-    state_is=present
+if [ -s "$__object/explorer/entry" ]; then
+    if diff -q "$__object/files/entry" "$__object/explorer/entry" >/dev/null; then
+        state_is=present
+    else
+        state_is=modified
+    fi
 else
     state_is=absent
 fi

From f3237e5d76c995d138bdf287c00a03d48f4ac742 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Mon, 3 Feb 2020 17:44:47 +0100
Subject: [PATCH 151/743] [consul agent] add support for Debian 10

---
 cdist/conf/type/__consul_agent/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest
index ee682d72..e00f29ec 100755
--- a/cdist/conf/type/__consul_agent/manifest
+++ b/cdist/conf/type/__consul_agent/manifest
@@ -209,7 +209,7 @@ case "$os" in
             [567])
                 init_sysvinit debian
             ;;
-            [89])
+            [89]|10)
                 init_systemd
             ;;
             *)

From 0f420993e1befd2692aca6cae43efb751a4a78de Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Mon, 3 Feb 2020 17:45:34 +0100
Subject: [PATCH 152/743] ++ changelog update

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 467c0f22..9f5e73bd 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)
+
 6.5.0: 2020-01-23
 	* Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar)
 	* Core: preos: Fix missing configuration file usage, support -g, --config-file option (Darko Poljak)

From e82dd35869e00b2fb00b6907b5fc3a960981bc6e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 30 Sep 2019 18:50:20 +0200
Subject: [PATCH 153/743] [__file] stat explorer patch for systems without
 stat(1)

Some embedded systems (like OpenWrt) do not ship a stat(1) binary.
This workaround parses the output of ls(1) and /etc/passwd, /etc/group to
gather the information needed.
---
 cdist/conf/type/__file/explorer/stat | 64 ++++++++++++++++++++--------
 1 file changed, 46 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat
index 13c1c208..91c8cc84 100755
--- a/cdist/conf/type/__file/explorer/stat
+++ b/cdist/conf/type/__file/explorer/stat
@@ -2,6 +2,7 @@
 #
 # 2013 Steven Armstrong (steven-cdist armstrong.cc)
 # 2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -21,29 +22,54 @@
 
 destination="/$__object_id"
 
+fallback() {
+   # Fallback: Patch the output together, manually.
+
+   ls_line=$(ls -ldn "$destination")
+
+   uid=$(echo "$ls_line" | awk '{ print $3 }')
+   gid=$(echo "$ls_line" | awk '{ print $4 }')
+
+   owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd)
+   group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
+
+   mode_text=$(echo "$ls_line" | awk '{ print $1 }')
+   mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }')
+
+   size=$(echo "$ls_line" | awk '{ print $5 }')
+   links=$(echo "$ls_line" | awk '{ print $2 }')
+
+   printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\nsize: %d\nlinks: %d\n' \
+      "$("$__type_explorer/type")" \
+      "$uid" "$owner" \
+      "$gid" "$group" \
+      "$mode" "$mode_text" \
+      "$size" \
+      "$links"
+}
+
+
 # nothing to work with, nothing we could do
 [ -e "$destination" ] || exit 0
 
-os=$("$__explorer/os")
-case "$os" in
-   "freebsd"|"netbsd"|"openbsd"|"macosx")
+
+if ! command -v stat >/dev/null
+then
+   fallback
+   exit
+fi
+
+
+case $("$__explorer/os")
+in
+   freebsd|netbsd|openbsd|macosx)
       stat -f "type: %HT
 owner: %Du %Su
 group: %Dg %Sg
 mode: %Lp %Sp
 size: %Dz
 links: %Dl
-" "$destination" | awk '/^type/ { print tolower($0); next; } { print; }'
-      ;;
-   alpine)
-       # busybox stat
-      stat -c "type: %F
-owner: %u %U
-group: %g %G
-mode: %a %A
-size: %s
-links: %h
-" "$destination"
+" "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
     solaris)
         ls1="$( ls -ld "$destination" )"
@@ -77,12 +103,14 @@ links: %h
         echo "links: $( echo "$ls1" | awk '{print $2}' )"
     ;;
    *)
-      stat --printf="type: %F
+      # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
+      # NOTE: BusyBox's stat might not support the "-c" option, in which case
+      #       we fall through to the shell fallback.
+      stat -c "type: %F
 owner: %u %U
 group: %g %G
 mode: %a %A
 size: %s
-links: %h
-" "$destination"
-      ;;
+links: %h" "$destination" 2>/dev/null || fallback
+         ;;
 esac

From 3b5433d63af27f06f3c44b121b21d7e0520af7bf Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 3 Feb 2020 18:12:27 +0100
Subject: [PATCH 154/743] [__directory] stat explorer patch for systems without
 stat(1)

Some embedded systems (like OpenWrt) do not ship a stat(1) binary.
This workaround parses the output of ls(1) and /etc/passwd, /etc/group to
gather the information needed.
---
 cdist/conf/type/__directory/explorer/stat | 48 +++++++++++++++++------
 1 file changed, 35 insertions(+), 13 deletions(-)

diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat
index 03d466ba..105d894f 100755
--- a/cdist/conf/type/__directory/explorer/stat
+++ b/cdist/conf/type/__directory/explorer/stat
@@ -1,6 +1,7 @@
 #!/bin/sh
 #
 # 2013 Steven Armstrong (steven-cdist armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -20,24 +21,43 @@
 
 destination="/$__object_id"
 
+fallback() {
+   # Patch the output together, manually
+
+   ls_line=$(ls -ldn "$destination")
+
+   uid=$(echo "$ls_line" | awk '{ print $3 }')
+   gid=$(echo "$ls_line" | awk '{ print $4 }')
+
+   owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd)
+   group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
+
+   mode_text=$(echo "$ls_line" | awk '{ print $1 }')
+   mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }')
+
+   printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\n' \
+      "$("$__type_explorer/type")" \
+      "$uid" "$owner" \
+      "$gid" "$group" \
+      "$mode" "$mode_text"
+}
+
 # nothing to work with, nothing we could do
 [ -e "$destination" ] || exit 0
 
-os=$("$__explorer/os")
-case "$os" in
+if ! command -v stat >/dev/null
+then
+   fallback
+   exit
+fi
+
+case $("$__explorer/os") in
    "freebsd"|"netbsd"|"openbsd"|"macosx")
       stat -f "type: %HT
 owner: %Du %Su
 group: %Dg %Sg
 mode: %Lp %Sp
-" "$destination" | awk '/^type/ { print tolower($0); next; } { print; }'
-      ;;
-   alpine)
-      stat -c "type: %F
-owner: %u %U
-group: %g %G
-mode: %a %A
-" "$destination"
+" "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
     solaris)
         ls1="$( ls -ld "$destination" )"
@@ -69,10 +89,12 @@ mode: %a %A
         echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )"
     ;;
    *)
-       stat --printf="type: %F
+      # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
+      # NOTE: BusyBox's stat might not support the "-c" option, in which case
+      #       we fall through to the shell fallback.
+       stat -c "type: %F
 owner: %u %U
 group: %g %G
-mode: %a %A
-" "$destination"
+mode: %a %A" "$destination" 2>/dev/null || fallback
    ;;
 esac

From 984e0dc8c4975904057a7ab0a8c9d357ed0b4fdf Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 3 Feb 2020 19:29:04 +0100
Subject: [PATCH 155/743] [explorer/os_release] Add fallbacks to
 /usr/lib/os-release and /var/run/os-release

---
 cdist/conf/explorer/os_release | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/explorer/os_release b/cdist/conf/explorer/os_release
index cfc01004..6489446b 100644
--- a/cdist/conf/explorer/os_release
+++ b/cdist/conf/explorer/os_release
@@ -1,6 +1,7 @@
 #!/bin/sh
 #
 # 2018 Adam Dej (dejko.a at gmail.com)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -21,6 +22,17 @@
 
 # See os-release(5) and http://0pointer.de/blog/projects/os-release
 
-set +e
+if test -f /etc/os-release
+then
+	# Linux and FreeBSD (usually a symlink)
+	cat /etc/os-release
+elif test -f /usr/lib/os-release
+then
+	# systemd
+	cat /usr/lib/os-release
+elif test -f /var/run/os-release
+then
+	# FreeBSD (created by os-release service)
+	cat /var/run/os-release
+fi
 
-cat /etc/os-release || cat /usr/lib/os-release || true

From 7a5d6d5a7d602863cd7d74bfb4e13a8bb59c5944 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 4 Feb 2020 07:45:04 +0100
Subject: [PATCH 156/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 9f5e73bd..bf0a8777 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,8 @@ Changelog
 
 next:
 	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)
+	* Explorer os_release: Add fallbacks (Dennis Camera)
+	* Types __file, __directory: Add fallback for systems without stat (Dennis Camera)
 
 6.5.0: 2020-01-23
 	* Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar)

From 08d5814e2cc8ea6a204afca91bec7b831116112b Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 9 Feb 2020 20:37:06 +0200
Subject: [PATCH 157/743] __package_apt: update index cache, on installation,
 when it is older than one day

---
 cdist/conf/type/__package_apt/gencode-remote | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote
index 699eb0c9..b3184a9c 100755
--- a/cdist/conf/type/__package_apt/gencode-remote
+++ b/cdist/conf/type/__package_apt/gencode-remote
@@ -74,6 +74,11 @@ fi
 
 case "$state_should" in
     present)
+        cat << EOF
+if [ -f /var/cache/apt/pkgcache.bin ] && [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ]
+then echo apt-get update > /dev/null 2>&1 || true
+fi
+EOF
         if [ -n "$version" ]; then
             name="${name}=${version}"
         fi

From b3bad9468de78cf90e8ce92a59c8bb3d285883fe Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 9 Feb 2020 20:43:09 +0200
Subject: [PATCH 158/743] __package_apt: pkgcache.bin may not exist

---
 cdist/conf/type/__package_apt/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote
index b3184a9c..cb79e886 100755
--- a/cdist/conf/type/__package_apt/gencode-remote
+++ b/cdist/conf/type/__package_apt/gencode-remote
@@ -75,7 +75,7 @@ fi
 case "$state_should" in
     present)
         cat << EOF
-if [ -f /var/cache/apt/pkgcache.bin ] && [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ]
+if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ]
 then echo apt-get update > /dev/null 2>&1 || true
 fi
 EOF

From bcefeb240c81fd263763c794acc7f6ce0b8f6db0 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Mon, 10 Feb 2020 15:51:15 +0100
Subject: [PATCH 159/743] add note about IPv6, because people ask

---
 docs/src/index.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/src/index.rst b/docs/src/index.rst
index 5e54d8fc..31c044dc 100644
--- a/docs/src/index.rst
+++ b/docs/src/index.rst
@@ -2,8 +2,9 @@ cdist - usable configuration management
 =======================================
 
 cdist is a usable configuration management system.
-It adheres to the KISS principle and 
+It adheres to the KISS principle and
 is being used in small up to enterprise grade environments.
+It natively supports IPv6 since the first release.
 
 
 .. toctree::

From 710e99f240ef616c6c1978167749b877b3d1e8dc Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 13 Feb 2020 17:29:44 +0200
Subject: [PATCH 160/743] __mysql_privileges: fix quoting

---
 cdist/conf/type/__mysql_privileges/explorer/state | 2 +-
 cdist/conf/type/__mysql_privileges/gencode-remote | 9 +++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__mysql_privileges/explorer/state b/cdist/conf/type/__mysql_privileges/explorer/state
index 0cfbaacd..4f13a70c 100755
--- a/cdist/conf/type/__mysql_privileges/explorer/state
+++ b/cdist/conf/type/__mysql_privileges/explorer/state
@@ -30,7 +30,7 @@ host="$( cat "$__object/parameter/host" )"
 
 check_privileges="$( 
     mysql -B -N -e "show grants for '$user'@'$host'" \
-        | grep -Ei "^grant $privileges on .$database.\..$table. to " || true )"
+        | grep -Ei "^grant $privileges on .$database.\..?$table.? to " || true )"
 
 if [ -n "$check_privileges" ]
 then
diff --git a/cdist/conf/type/__mysql_privileges/gencode-remote b/cdist/conf/type/__mysql_privileges/gencode-remote
index bcd362e6..20975c50 100755
--- a/cdist/conf/type/__mysql_privileges/gencode-remote
+++ b/cdist/conf/type/__mysql_privileges/gencode-remote
@@ -37,13 +37,18 @@ user="$( cat "$__object/parameter/user" )"
 
 host="$( cat "$__object/parameter/host" )"
 
+if [ "$table" != '*' ]
+then
+    table="$( printf '`%s`' "$table" )"
+fi
+
 case "$state_should" in
     present)
-        echo "mysql -e 'grant $privileges on \`$database\`.\`$table\` to \`$user\`@\`$host\`'"
+        echo "mysql -e 'grant $privileges on \`$database\`.$table to \`$user\`@\`$host\`'"
         echo "grant $privileges on $database.$table to $user@$host" >> "$__messages_out"
     ;;
     absent)
-        echo "mysql -e 'revoke $privileges on \`$database\`.\`$table\` from \`$user\`@\`$host\`'"
+        echo "mysql -e 'revoke $privileges on \`$database\`.$table from \`$user\`@\`$host\`'"
         echo "revoke $privileges on $database.$table from $user@$host" >> "$__messages_out"
     ;;
 esac

From b7483d73ebc5a95c96adc4a40173ce5015173eeb Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 13 Feb 2020 17:38:19 +0200
Subject: [PATCH 161/743] __mysql_privileges: fix shellcheck

---
 cdist/conf/type/__mysql_privileges/gencode-remote | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__mysql_privileges/gencode-remote b/cdist/conf/type/__mysql_privileges/gencode-remote
index 20975c50..0656699f 100755
--- a/cdist/conf/type/__mysql_privileges/gencode-remote
+++ b/cdist/conf/type/__mysql_privileges/gencode-remote
@@ -39,6 +39,7 @@ host="$( cat "$__object/parameter/host" )"
 
 if [ "$table" != '*' ]
 then
+    # shellcheck disable=SC2016
     table="$( printf '`%s`' "$table" )"
 fi
 

From f771840178ed2586a4abc47f5e513da2309d5128 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 13 Feb 2020 17:38:34 +0200
Subject: [PATCH 162/743] __mysql_privileges: fix typo

---
 cdist/conf/type/__mysql_privileges/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__mysql_privileges/man.rst b/cdist/conf/type/__mysql_privileges/man.rst
index 8208d7d4..b72c9eba 100644
--- a/cdist/conf/type/__mysql_privileges/man.rst
+++ b/cdist/conf/type/__mysql_privileges/man.rst
@@ -17,7 +17,7 @@ REQUIRED PARAMETERS
 database
    Name of database.
 
-User
+user
    Name of user.
 
 

From e5f9d320049dea5acef6e505cc7a4966ecbd72c6 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 14 Feb 2020 16:07:06 +0100
Subject: [PATCH 163/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index bf0a8777..3f69899a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)
 	* Explorer os_release: Add fallbacks (Dennis Camera)
 	* Types __file, __directory: Add fallback for systems without stat (Dennis Camera)
+	* Type __mysql_privileges: Fix quoting (Ander Punnar)
 
 6.5.0: 2020-01-23
 	* Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar)

From 1be5a9d3163fc5b8aa107764ea5aab41f46e860a Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 14 Feb 2020 19:19:12 +0200
Subject: [PATCH 164/743] __package_apt: add note about updating index

---
 cdist/conf/type/__package_apt/man.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cdist/conf/type/__package_apt/man.rst b/cdist/conf/type/__package_apt/man.rst
index a3a70d91..a1691eac 100644
--- a/cdist/conf/type/__package_apt/man.rst
+++ b/cdist/conf/type/__package_apt/man.rst
@@ -11,6 +11,9 @@ DESCRIPTION
 apt-get is usually used on Debian and variants (like Ubuntu) to
 manage packages.
 
+This type will also update package index, if it is older
+than one day, to avoid missing package error messages.
+
 
 REQUIRED PARAMETERS
 -------------------

From 869a38676f97b7863997b7d2c257447a63344368 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 14 Feb 2020 19:22:06 +0200
Subject: [PATCH 165/743] __package_apt: add comment about package index update

---
 cdist/conf/type/__package_apt/gencode-remote | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote
index cb79e886..e02564a2 100755
--- a/cdist/conf/type/__package_apt/gencode-remote
+++ b/cdist/conf/type/__package_apt/gencode-remote
@@ -74,6 +74,9 @@ fi
 
 case "$state_should" in
     present)
+        # following is bit ugly, but important hack.
+        # due to how cdist config run works, there isn't
+        # currently better way to do it :(
         cat << EOF
 if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ]
 then echo apt-get update > /dev/null 2>&1 || true

From 48bff6656322eecd940a1e0a2c272fe2303268f6 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 14 Feb 2020 21:55:35 +0100
Subject: [PATCH 166/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 3f69899a..734db7ea 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Explorer os_release: Add fallbacks (Dennis Camera)
 	* Types __file, __directory: Add fallback for systems without stat (Dennis Camera)
 	* Type __mysql_privileges: Fix quoting (Ander Punnar)
+	* Type __package_apt: Update package index if it is older than one day (Ander Punnar)
 
 6.5.0: 2020-01-23
 	* Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar)

From a761d4842ca36688fd6355a63185da985528366a Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 15 Feb 2020 08:21:24 +0100
Subject: [PATCH 167/743] Add check for the state parameter in manifest.

This only allow following cases:
  - present
  - absent
else, it will abort with an error message.
---
 cdist/conf/type/__cron/manifest | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/cdist/conf/type/__cron/manifest b/cdist/conf/type/__cron/manifest
index 53973e07..3017e2fd 100755
--- a/cdist/conf/type/__cron/manifest
+++ b/cdist/conf/type/__cron/manifest
@@ -22,3 +22,12 @@ if [ -f "$__object/parameter/raw" ] && [ -f "$__object/parameter/raw_command" ];
     echo "ERROR: both raw and raw_command specified" >&2
     exit 1
 fi
+
+case "$(cat "$__object/parameter/state" 2>/dev/null || echo "present")" in
+    present) ;;
+    absent) ;;
+
+    *)
+        echo "ERROR: unkown cron state" >&2
+        exit 2
+esac

From 5e6e17b3e55ea5b53a3ad9fe28f1d1c422121369 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 15 Feb 2020 09:46:23 +0100
Subject: [PATCH 168/743] Moved default parameter values from scripts to cdist
 parameter handling.

For more generalisation, the default parameter values are now handled by
cdist instead of trying to get a value and use a default if parameter is
not given.

It handles the default values in a more general way, instead of write
one default in (possibly) multiple places.
Problem occurred when checking the 'state' parameter, which required to
set a default value in two places.
---
 cdist/conf/type/__cron/gencode-remote                | 12 ++++++------
 cdist/conf/type/__cron/manifest                      |  2 +-
 .../conf/type/__cron/parameter/default/day_of_month  |  1 +
 cdist/conf/type/__cron/parameter/default/day_of_week |  1 +
 cdist/conf/type/__cron/parameter/default/hour        |  1 +
 cdist/conf/type/__cron/parameter/default/minute      |  1 +
 cdist/conf/type/__cron/parameter/default/month       |  1 +
 cdist/conf/type/__cron/parameter/default/state       |  1 +
 8 files changed, 13 insertions(+), 7 deletions(-)
 create mode 100644 cdist/conf/type/__cron/parameter/default/day_of_month
 create mode 100644 cdist/conf/type/__cron/parameter/default/day_of_week
 create mode 100644 cdist/conf/type/__cron/parameter/default/hour
 create mode 100644 cdist/conf/type/__cron/parameter/default/minute
 create mode 100644 cdist/conf/type/__cron/parameter/default/month
 create mode 100644 cdist/conf/type/__cron/parameter/default/state

diff --git a/cdist/conf/type/__cron/gencode-remote b/cdist/conf/type/__cron/gencode-remote
index 8f3c92dc..9debbc47 100755
--- a/cdist/conf/type/__cron/gencode-remote
+++ b/cdist/conf/type/__cron/gencode-remote
@@ -31,11 +31,11 @@ if [ -f "$__object/parameter/raw" ]; then
 elif [ -f "$__object/parameter/raw_command" ]; then
    entry="$command"
 else
-   minute="$(cat "$__object/parameter/minute" 2>/dev/null || echo "*")"
-   hour="$(cat "$__object/parameter/hour" 2>/dev/null || echo "*")"
-   day_of_month="$(cat "$__object/parameter/day_of_month" 2>/dev/null || echo "*")"
-   month="$(cat "$__object/parameter/month" 2>/dev/null || echo "*")"
-   day_of_week="$(cat "$__object/parameter/day_of_week" 2>/dev/null || echo "*")"
+   minute="$(cat "$__object/parameter/minute")"
+   hour="$(cat "$__object/parameter/hour")"
+   day_of_month="$(cat "$__object/parameter/day_of_month")"
+   month="$(cat "$__object/parameter/month")"
+   day_of_week="$(cat "$__object/parameter/day_of_week")"
    entry="$minute $hour $day_of_month $month $day_of_week $command # $name"
 fi
 
@@ -52,7 +52,7 @@ else
     state_is=absent
 fi
 
-state_should="$(cat "$__object/parameter/state" 2>/dev/null || echo "present")"
+state_should="$(cat "$__object/parameter/state")"
 
 [ "$state_is" = "$state_should" ] && exit 0
 
diff --git a/cdist/conf/type/__cron/manifest b/cdist/conf/type/__cron/manifest
index 3017e2fd..e7b51863 100755
--- a/cdist/conf/type/__cron/manifest
+++ b/cdist/conf/type/__cron/manifest
@@ -23,7 +23,7 @@ if [ -f "$__object/parameter/raw" ] && [ -f "$__object/parameter/raw_command" ];
     exit 1
 fi
 
-case "$(cat "$__object/parameter/state" 2>/dev/null || echo "present")" in
+case "$(cat "$__object/parameter/state")" in
     present) ;;
     absent) ;;
 
diff --git a/cdist/conf/type/__cron/parameter/default/day_of_month b/cdist/conf/type/__cron/parameter/default/day_of_month
new file mode 100644
index 00000000..72e8ffc0
--- /dev/null
+++ b/cdist/conf/type/__cron/parameter/default/day_of_month
@@ -0,0 +1 @@
+*
diff --git a/cdist/conf/type/__cron/parameter/default/day_of_week b/cdist/conf/type/__cron/parameter/default/day_of_week
new file mode 100644
index 00000000..72e8ffc0
--- /dev/null
+++ b/cdist/conf/type/__cron/parameter/default/day_of_week
@@ -0,0 +1 @@
+*
diff --git a/cdist/conf/type/__cron/parameter/default/hour b/cdist/conf/type/__cron/parameter/default/hour
new file mode 100644
index 00000000..72e8ffc0
--- /dev/null
+++ b/cdist/conf/type/__cron/parameter/default/hour
@@ -0,0 +1 @@
+*
diff --git a/cdist/conf/type/__cron/parameter/default/minute b/cdist/conf/type/__cron/parameter/default/minute
new file mode 100644
index 00000000..72e8ffc0
--- /dev/null
+++ b/cdist/conf/type/__cron/parameter/default/minute
@@ -0,0 +1 @@
+*
diff --git a/cdist/conf/type/__cron/parameter/default/month b/cdist/conf/type/__cron/parameter/default/month
new file mode 100644
index 00000000..72e8ffc0
--- /dev/null
+++ b/cdist/conf/type/__cron/parameter/default/month
@@ -0,0 +1 @@
+*
diff --git a/cdist/conf/type/__cron/parameter/default/state b/cdist/conf/type/__cron/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__cron/parameter/default/state
@@ -0,0 +1 @@
+present

From ef089d1c6114e840c4b8022d43322bdb18b96e8f Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 6 Jan 2020 10:42:12 +0100
Subject: [PATCH 169/743] [__systemd_service] new type to manage the state of
 systemd services

---
 .../type/__systemd_service/explorer/state     |  43 +++++++
 .../type/__systemd_service/gencode-remote     |  98 ++++++++++++++++
 cdist/conf/type/__systemd_service/man.rst     | 110 ++++++++++++++++++
 .../type/__systemd_service/parameter/boolean  |   1 +
 .../__systemd_service/parameter/default/state |   1 +
 .../type/__systemd_service/parameter/optional |   3 +
 6 files changed, 256 insertions(+)
 create mode 100755 cdist/conf/type/__systemd_service/explorer/state
 create mode 100755 cdist/conf/type/__systemd_service/gencode-remote
 create mode 100644 cdist/conf/type/__systemd_service/man.rst
 create mode 100644 cdist/conf/type/__systemd_service/parameter/boolean
 create mode 100644 cdist/conf/type/__systemd_service/parameter/default/state
 create mode 100644 cdist/conf/type/__systemd_service/parameter/optional

diff --git a/cdist/conf/type/__systemd_service/explorer/state b/cdist/conf/type/__systemd_service/explorer/state
new file mode 100755
index 00000000..f5f751d4
--- /dev/null
+++ b/cdist/conf/type/__systemd_service/explorer/state
@@ -0,0 +1,43 @@
+#!/bin/sh -e
+# explorer/state
+#
+# 2020 Matthias Stecher <matthiasstecher at gmx.de>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# Check if the service is running or stopped.
+#
+# The explorer must check before if the service exist, because 'systemctl is-active'
+# will return "inactive" even if there is no service there:
+#   systemctl cat foo        # does not exist
+#   systemctl is-active foo  # is "inactive"
+
+
+# get name of the service
+if [ -f "$__object/parameter/name" ]; then
+    name="$(cat "$__object/parameter/name")"
+else
+    name="$__object_id"
+fi
+
+
+# check if the service exist, else exit without output (also if systemd doesn't exist)
+# do not exit here with an error code, will be done in the gencode-remote script
+systemctl cat "$name" > /dev/null 2>&1 || exit 0
+
+# print if the service is running or not
+systemctl is-active -q "$name" && printf "running" || printf "stopped"
diff --git a/cdist/conf/type/__systemd_service/gencode-remote b/cdist/conf/type/__systemd_service/gencode-remote
new file mode 100755
index 00000000..c867ff22
--- /dev/null
+++ b/cdist/conf/type/__systemd_service/gencode-remote
@@ -0,0 +1,98 @@
+#!/bin/sh -e
+# gencode-remote
+#
+# 2020 Matthias Stecher <matthiasstecher at gmx.de>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# Checks the given state of the service and set it to the given
+# state. Optionally, it executes the action if service running.
+
+
+# get name of the service
+name="$__object/parameter/name"
+if [ -f "$name" ]; then
+    name="$(cat "$name")"
+else
+    name="$__object_id"
+fi
+
+
+# read current status and parameters
+state="$(cat "$__object/explorer/state")"
+should="$(cat "$__object/parameter/state")"
+
+# if systemd/service does not exist
+if [ -z "$state" ]; then
+    printf "systemd or service '%s' does not exist!\n" "$name" >&2
+    exit 1
+fi
+
+
+# save the action required
+required_action=""
+
+# check the state of the service that should be
+if [ "$state" != "$should" ]; then
+    # select what to do to get the $should state
+    case "$should" in
+        running)
+            if [ "$state" = "stopped" ]; then required_action="start"; fi
+            ;;
+
+        stopped)
+            if [ "$state" = "running" ]; then required_action="stop"; fi
+            ;;
+    esac
+fi
+
+# check if the action can be achieved if given
+if [ -f "$__object/parameter/action" ] \
+    && [ -z "$required_action" ] && [ "$state" = "running" ]; then
+
+    # there must be an action
+    action="$(cat "$__object/parameter/action")"
+
+    # select the action to the required element
+    case "$action" in
+        restart)
+            required_action="restart"
+            ;;
+
+        reload)
+            required_action="reload"
+            ;;
+
+        *)
+            printf "action '%s' does not exist!" "$action" >&2
+            exit 2
+    esac
+
+    # Make a special check: only do this action if a dependency did something
+    # it is required that the dependencies write there action to $__messages_in
+    if [ -f "$__object/parameter/if-required" ]; then
+        # exit here if there are no changes from the dependencies affected (nothing to do)
+        if ! grep -q -f "$__object/require" "$__messages_in"; then exit 0; fi
+    fi
+fi
+
+# print the execution command if a action given
+if [ -n "$required_action" ]; then
+    # also print it as message
+    echo "$required_action" >> "$__messages_out"
+    echo "systemctl $required_action '$name'"
+fi
diff --git a/cdist/conf/type/__systemd_service/man.rst b/cdist/conf/type/__systemd_service/man.rst
new file mode 100644
index 00000000..678cee29
--- /dev/null
+++ b/cdist/conf/type/__systemd_service/man.rst
@@ -0,0 +1,110 @@
+cdist-type__systemd-service(7)
+==============================
+
+NAME
+----
+cdist-type__systemd-service - Controls a systemd service state
+
+DESCRIPTION
+-----------
+This type controls systemd services to define a state of the service,
+or an action like reloading or restarting. It is useful to reload a
+service after configuration applied or shutdown one service.
+
+The activation or deactivation is out of scope. Look for the
+:strong:`cdist-type__systemd_util`\ (7) type instead.
+
+REQUIRED PARAMETERS
+-------------------
+
+None.
+
+OPTIONAL PARAMETERS
+-------------------
+
+name
+    String which will used as name instead of the object id.
+
+state
+    The state which the service should be in:
+
+    running
+        Service should run (default)
+
+    stoppend
+        Service should stopped
+
+action
+    Executes an action on on the service. It will only execute it if the
+    service keeps the state **running**. There are following actions, where:
+
+    reload
+        Reloads the service
+
+    restart
+        Restarts the service
+
+BOOLEAN PARAMETERS
+-----------------
+
+if-required
+    Only execute the action if minimum one required type outputs a message to
+    **$__messages_out**. Through this, the action should only executed if a
+    dependency did something. The action will not executed if no dependencies
+    given.
+
+MESSAGES
+--------
+
+start
+    Started the service
+
+stop
+    Stopped the service
+
+restart
+    Restarted the service
+
+reload
+    Reloaded the service
+
+ABORTS
+------
+Aborts in following cases:
+
+systemd or the service does not exist
+
+EXAMPLES
+--------
+.. code-block:: sh
+
+    # service must run
+    __systemd_service nginx
+
+    # service must stopped
+    __systemd_service sshd \
+        --state stopped
+
+    # restart the service
+    __systemd_service apache2 \
+        --action restart
+
+    # makes sure the service exist with an alternative name
+    __systemd_service foo \
+        --name sshd
+
+    # reload the service for a modified configuration file
+    # only reloads the service if the file really changed
+    require="__config_file/etc/foo.conf" __systemd_service foo \
+        --action reload --if-required
+
+AUTHORS
+-------
+Matthias Stecher <matthiasstecher at gmx.de>
+
+COPYRIGHT
+---------
+Copyright \(C) 2020 Matthias Stecher. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__systemd_service/parameter/boolean b/cdist/conf/type/__systemd_service/parameter/boolean
new file mode 100644
index 00000000..a4bccb66
--- /dev/null
+++ b/cdist/conf/type/__systemd_service/parameter/boolean
@@ -0,0 +1 @@
+if-required
diff --git a/cdist/conf/type/__systemd_service/parameter/default/state b/cdist/conf/type/__systemd_service/parameter/default/state
new file mode 100644
index 00000000..a2ae71b3
--- /dev/null
+++ b/cdist/conf/type/__systemd_service/parameter/default/state
@@ -0,0 +1 @@
+running
diff --git a/cdist/conf/type/__systemd_service/parameter/optional b/cdist/conf/type/__systemd_service/parameter/optional
new file mode 100644
index 00000000..fc78265f
--- /dev/null
+++ b/cdist/conf/type/__systemd_service/parameter/optional
@@ -0,0 +1,3 @@
+name
+state
+action

From 58841fc4bfd8dbeb82717fe3638b3bb4d2ed017b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 15 Feb 2020 12:20:41 +0100
Subject: [PATCH 170/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 734db7ea..f687e5ed 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,8 @@ next:
 	* Types __file, __directory: Add fallback for systems without stat (Dennis Camera)
 	* Type __mysql_privileges: Fix quoting (Ander Punnar)
 	* Type __package_apt: Update package index if it is older than one day (Ander Punnar)
+	* Type __cron: Fix job removal if 'is' and 'should' don't match (Matthias Stecher)
+	* New type: __systemd_service (Matthias Stecher)
 
 6.5.0: 2020-01-23
 	* Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar)

From a1536933ab066671e09aaf47a3c6101a68a6b60f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Sat, 15 Feb 2020 08:51:29 +0100
Subject: [PATCH 171/743] Fix password command synthax in __postgres_role

---
 cdist/conf/type/__postgres_role/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 977832c9..282294c9 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -54,7 +54,7 @@ case "$state_should" in
 
         [ -n "$password" ] && password="PASSWORD '$password'"
         cat << EOF
-su - '$postgres_user' -c "psql postgres -wc 'CREATE ROLE \"$name\" WITH $password $booleans;'"
+su - '$postgres_user' -c "psql postgres -wc \"CREATE ROLE \\\\\"$name\\\\\" WITH $password $booleans;\""
 EOF
     ;;
     absent)

From cda77e5e8a44eabde7e39396c39fd34daba1c702 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 15 Feb 2020 14:35:29 +0100
Subject: [PATCH 172/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index f687e5ed..d5ed1b06 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,7 @@ next:
 	* Type __package_apt: Update package index if it is older than one day (Ander Punnar)
 	* Type __cron: Fix job removal if 'is' and 'should' don't match (Matthias Stecher)
 	* New type: __systemd_service (Matthias Stecher)
+	* Type __postgres_role: Fix password command synthax (Timothée Floure)
 
 6.5.0: 2020-01-23
 	* Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar)

From 9f3747cf3f664ba5cc2f74ec356d833e7eda2621 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 15 Feb 2020 14:36:26 +0100
Subject: [PATCH 173/743] Release 6.5.1

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index d5ed1b06..9f1e6245 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.5.1: 2020-02-15
 	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)
 	* Explorer os_release: Add fallbacks (Dennis Camera)
 	* Types __file, __directory: Add fallback for systems without stat (Dennis Camera)

From 0640b02f90d92e11f6a4f8b7c0110d9b5a9bfb89 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 15 Feb 2020 14:59:12 +0100
Subject: [PATCH 174/743] Fix too short title underline

---
 cdist/conf/type/__systemd_service/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__systemd_service/man.rst b/cdist/conf/type/__systemd_service/man.rst
index 678cee29..7eca398b 100644
--- a/cdist/conf/type/__systemd_service/man.rst
+++ b/cdist/conf/type/__systemd_service/man.rst
@@ -45,7 +45,7 @@ action
         Restarts the service
 
 BOOLEAN PARAMETERS
------------------
+------------------
 
 if-required
     Only execute the action if minimum one required type outputs a message to

From c09165d1228e887e2fb4dad456c6b1409211a499 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 15 Feb 2020 21:05:24 +0100
Subject: [PATCH 175/743] Fix spelling

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 9f1e6245..bd767b9d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,7 +9,7 @@ Changelog
 	* Type __package_apt: Update package index if it is older than one day (Ander Punnar)
 	* Type __cron: Fix job removal if 'is' and 'should' don't match (Matthias Stecher)
 	* New type: __systemd_service (Matthias Stecher)
-	* Type __postgres_role: Fix password command synthax (Timothée Floure)
+	* Type __postgres_role: Fix password command syntax (Timothée Floure)
 
 6.5.0: 2020-01-23
 	* Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar)

From 70200cd28fe9b35670f574e77d49e747f23072f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 21 Jan 2020 10:06:09 +0100
Subject: [PATCH 176/743] Refactor __consul_agent type to support distribution
 packages

---
 cdist/conf/type/__consul_agent/man.rst        |   3 +
 cdist/conf/type/__consul_agent/manifest       | 373 +++++++++++-------
 .../type/__consul_agent/parameter/boolean     |   1 +
 3 files changed, 225 insertions(+), 152 deletions(-)

diff --git a/cdist/conf/type/__consul_agent/man.rst b/cdist/conf/type/__consul_agent/man.rst
index 966abc60..62ee70bb 100644
--- a/cdist/conf/type/__consul_agent/man.rst
+++ b/cdist/conf/type/__consul_agent/man.rst
@@ -116,6 +116,9 @@ verify-incoming
 verify-outgoing
    enforce the use of TLS and verify the peers authenticity on outgoing connections
 
+use-distribution-package
+   uses distribution package instead of upstream binary
+
 
 EXAMPLES
 --------
diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest
index e00f29ec..599f15b4 100755
--- a/cdist/conf/type/__consul_agent/manifest
+++ b/cdist/conf/type/__consul_agent/manifest
@@ -2,6 +2,7 @@
 #
 # 2015 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2015-2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2019 Timothée Floure (timothee.floure at ungleich.ch)
 #
 # This file is part of cdist.
 #
@@ -19,133 +20,64 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-
 os=$(cat "$__global/explorer/os")
 
-case "$os" in
-   alpine|scientific|centos|debian|devuan|redhat|ubuntu)
-      # whitelist safeguard
-      :
-   ;;
-   *)
-      echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
-      echo "Please contribute an implementation for it if you can." >&2
-      exit 1
-   ;;
-esac
+###
+# Type parameters.
 
 state="$(cat "$__object/parameter/state")"
 user="$(cat "$__object/parameter/user")"
 group="$(cat "$__object/parameter/group")"
+release=$(cat "$__global/explorer/lsb_release")
+if [ -f "$__object/parameter/use-distribution-package" ]; then
+  use_distribution_package=1
+fi
+
+###
+# Those are default that might be overriden by os-specific logic.
+
 data_dir="/var/lib/consul"
 conf_dir="/etc/consul/conf.d"
 conf_file="config.json"
+tls_dir="$conf_dir/tls"
 
-# FIXME: there has got to be a better way to handle the dependencies in this case
-case "$state" in
-   present)
-      __group "$group" --system --state "$state"
-      require="__group/$group" \
-         __user "$user" --system --gid "$group" \
-            --home "$data_dir" --state "$state"
-      export require="__user/consul"
-   ;;
-   absent)
-      echo "Sorry, state=absent currently not supported :-(" >&2
-      exit 1
-      require="$__object_name" \
-         __user "$user" --system --gid "$group" --state "$state"
-      require="__user/$user" \
-         __group "$group" --system --state "$state"
-   ;;
-esac
+###
+# Sane deployment, based on distribution package when available.
 
-__directory /etc/consul \
-   --owner root --group "$group" --mode 750 --state "$state"
-require="__directory/etc/consul" \
-   __directory "$conf_dir" \
-      --owner root --group "$group" --mode 750 --state "$state"
+distribution_setup () {
+  case "$os" in
+     debian)
+       # consul is only available starting Debian 10 (buster).
+       # See https://packages.debian.org/buster/consul
+       if [ $release -lt 10 ]; then
+         echo "Consul is not available for your debian release." >&2
+         echo "Please use the 'manual' (i.e. non-package) installation or \
+           upgrade the target system." >&2
+         exit 1
+       fi
 
-if [ -f "$__object/parameter/ca-file-source" ] || [ -f "$__object/parameter/cert-file-source" ] || [ -f "$__object/parameter/key-file-source" ]; then
-   # create directory for ssl certs
-   require="__directory/etc/consul" \
-      __directory /etc/consul/ssl \
-         --owner root --group "$group" --mode 750 --state "$state"
-fi
+       # Override previously defined environment to match debian packaging.
+       conf_dir='/etc/consul.d'
+       user='consul'
+       grou='consul'
+     ;;
+     *)
+        echo "Your operating system ($os) is currently not supported with the \
+          --use-distribution-package flag (${__type##*/})." >&2
+        echo "Please use non-package installation or contribute an \
+          implementation for if you can." >&2
+        exit 1
+     ;;
+  esac
 
-__directory "$data_dir" \
-   --owner "$user" --group "$group" --mode 770 --state "$state"
+  # Install consul package.
+  __package consul --state $state
 
+  export config_deployment_requires="__package/consul"
+}
 
-# Generate json config file
-(
-echo "{"
-
-# parameters we define ourself
-printf '   "data_dir": "%s"\n' "$data_dir"
-
-cd "$__object/parameter/"
-for param in *; do
-   case "$param" in
-      state|user|group|json-config) continue ;;
-      ca-file-source|cert-file-source|key-file-source)
-         source="$(cat "$__object/parameter/$param")"
-         destination="/etc/consul/ssl/${source##*/}"
-         require="__directory/etc/consul/ssl" \
-            __file "$destination" \
-               --owner root --group consul --mode 640 \
-               --source "$source" \
-               --state "$state"
-         key="$(echo "${param%-*}" | tr '-' '_')"
-         printf '   ,"%s": "%s"\n' "$key" "$destination"
-      ;;
-      disable-remote-exec|disable-update-check|leave-on-terminate|rejoin-after-leave|server|enable-syslog|verify-incoming|verify-outgoing)
-         # handle boolean parameters
-         key="$(echo "$param" | tr '-' '_')"
-         printf '   ,"%s": true\n' "$key"
-      ;;
-      retry-join)
-         # join multiple parameters into json array
-         retry_join="$(awk '{printf "\""$1"\","}' "$__object/parameter/retry-join")"
-         # remove trailing ,
-         printf '   ,"retry_join": [%s]\n' "${retry_join%*,}"
-      ;;
-      retry-join-wan)
-         # join multiple parameters into json array over wan
-         retry_join_wan="$(awk '{printf "\""$1"\","}' "$__object/parameter/retry-join-wan")"
-         # remove trailing ,
-         printf '   ,"retry_join_wan": [%s]\n' "${retry_join_wan%*,}"
-      ;;
-      bootstrap-expect)
-         # integer key=value parameters
-         key="$(echo "$param" | tr '-' '_')"
-         printf '   ,"%s": %s\n' "$key" "$(cat "$__object/parameter/$param")"
-      ;;
-      *)
-         # string key=value parameters
-         key="$(echo "$param" | tr '-' '_')"
-         printf '   ,"%s": "%s"\n' "$key" "$(cat "$__object/parameter/$param")"
-      ;;
-   esac
-done
-if [ -f "$__object/parameter/json-config" ]; then
-   json_config="$(cat "$__object/parameter/json-config")"
-   if [ "$json_config" = "-" ]; then
-      json_config="$__object/stdin"
-   fi
-   # remove leading and trailing whitespace and commas from first and last line
-   # indent each line with 3 spaces for consistency
-   json=$(sed -e 's/^[ \t]*/   /' -e '1s/^[ \t,]*//' -e '$s/[ \t,]*$//' "$json_config")
-   printf '   ,%s\n' "$json"
-fi
-echo "}"
-) | \
-require="__directory${conf_dir}" \
-   __config_file "${conf_dir}/${conf_file}" \
-      --owner root --group "$group" --mode 640 \
-      --state "$state" \
-      --onchange 'service consul status >/dev/null && service consul reload || true' \
-      --source -
+###
+# LEGACY manual deployment, kept for compatibility reasons.
 
 init_sysvinit()
 {
@@ -179,47 +111,184 @@ init_upstart()
     require="__file/etc/init/consul.conf" __start_on_boot consul
 }
 
-# Install init script to start on boot
-case "$os" in
-    devuan)
-        init_sysvinit debian
-        ;;
-    centos|redhat)
-        os_version="$(sed 's/[^0-9.]//g' "$__global/explorer/os_version")"
-        major_version="${os_version%%.*}"
-        case "$major_version" in
-            [456])
-                init_sysvinit redhat
-                ;;
-            7)
-                init_systemd
-                ;;
-            *)
-                echo "Unsupported CentOS/Redhat version: $os_version" >&2
-                exit 1
-                ;;
-        esac
-        ;;
+manual_setup () {
+  case "$os" in
+     alpine|scientific|centos|debian|devuan|redhat|ubuntu)
+        # whitelist safeguard
+        :
+     ;;
+     *)
+        echo "Your operating system ($os) is currently not supported by this \
+          type (${__type##*/})." >&2
+        echo "Please contribute an implementation for it if you can." >&2
+        exit 1
+     ;;
+  esac
 
-    debian)
-        os_version=$(cat "$__global/explorer/os_version")
-        major_version="${os_version%%.*}"
+  # FIXME: there has got to be a better way to handle the dependencies in this case
+  case "$state" in
+     present)
+        __group "$group" --system --state "$state"
+        require="__group/$group" __user "$user" \
+          --system --gid "$group" --home "$data_dir" --state "$state"
+     ;;
+     *)
+        echo "The $state state is not (yet?) supported by this type." >&2
+        exit 1
+     ;;
+  esac
 
-        case "$major_version" in
-            [567])
-                init_sysvinit debian
-            ;;
-            [89]|10)
-                init_systemd
-            ;;
-            *)
-                echo "Unsupported Debian version $os_version" >&2
-                exit 1
-            ;;
-        esac
-        ;;
+  # Create data directory.
+  require="__user/consul"__directory "$data_dir" \
+    --owner "$user" --group "$group" --mode 770 --state "$state"
 
-    ubuntu)
-        init_upstart
+  # Create config directory.
+  require="__user/consul" __directory "$conf_dir" \
+    --parents --owner root --group "$group" --mode 750 --state "$state"
+
+  # Install init script to start on boot
+  case "$os" in
+      devuan)
+          init_sysvinit debian
+          ;;
+      centos|redhat)
+          os_version="$(sed 's/[^0-9.]//g' "$__global/explorer/os_version")"
+          major_version="${os_version%%.*}"
+          case "$major_version" in
+              [456])
+                  init_sysvinit redhat
+                  ;;
+              7)
+                  init_systemd
+                  ;;
+              *)
+                  echo "Unsupported CentOS/Redhat version: $os_version" >&2
+                  exit 1
+                  ;;
+          esac
+          ;;
+
+      debian)
+          os_version=$(cat "$__global/explorer/os_version")
+          major_version="${os_version%%.*}"
+
+          case "$major_version" in
+              [567])
+                  init_sysvinit debian
+              ;;
+              [89]|10)
+                  init_systemd
+              ;;
+              *)
+                  echo "Unsupported Debian version $os_version" >&2
+                  exit 1
+              ;;
+          esac
+          ;;
+
+      ubuntu)
+          init_upstart
+          ;;
+  esac
+
+  config_deployment_requires="__user/consul __directory/$conf_dir"
+}
+
+###
+# Trigger requested installation method.
+if [ $use_distribution_package ]; then
+  distribution_setup
+else
+  manual_setup
+fi
+
+###
+# Generate and deploy configuration.
+json_configuration=$(
+  echo "{"
+
+  # parameters we define ourself
+  printf '   "data_dir": "%s"\n' "$data_dir"
+
+  cd "$__object/parameter/"
+  for param in *; do
+     case "$param" in
+        state|user|group|json-config|use-distribution-package) continue ;;
+        ca-file-source|cert-file-source|key-file-source)
+           source="$(cat "$__object/parameter/$param")"
+           destination="/etc/consul/ssl/${source##*/}"
+           require="__directory/etc/consul/ssl" \
+              __file "$destination" \
+                 --owner root --group consul --mode 640 \
+                 --source "$source" \
+                 --state "$state"
+           key="$(echo "${param%-*}" | tr '-' '_')"
+           printf '   ,"%s": "%s"\n' "$key" "$destination"
         ;;
-esac
+        disable-remote-exec|disable-update-check|leave-on-terminate\
+          |rejoin-after-leave|server|enable-syslog|verify-incoming|verify-outgoing)
+           # handle boolean parameters
+           key="$(echo "$param" | tr '-' '_')"
+           printf '   ,"%s": true\n' "$key"
+        ;;
+        retry-join)
+           # join multiple parameters into json array
+           retry_join="$(awk '{printf "\""$1"\","}' "$__object/parameter/retry-join")"
+           # remove trailing ,
+           printf '   ,"retry_join": [%s]\n' "${retry_join%*,}"
+        ;;
+        retry-join-wan)
+           # join multiple parameters into json array over wan
+           retry_join_wan="$(awk '{printf "\""$1"\","}' "$__object/parameter/retry-join-wan")"
+           # remove trailing ,
+           printf '   ,"retry_join_wan": [%s]\n' "${retry_join_wan%*,}"
+        ;;
+        bootstrap-expect)
+           # integer key=value parameters
+           key="$(echo "$param" | tr '-' '_')"
+           printf '   ,"%s": %s\n' "$key" "$(cat "$__object/parameter/$param")"
+        ;;
+        *)
+           # string key=value parameters
+           key="$(echo "$param" | tr '-' '_')"
+           printf '   ,"%s": "%s"\n' "$key" "$(cat "$__object/parameter/$param")"
+        ;;
+     esac
+  done
+  if [ -f "$__object/parameter/json-config" ]; then
+     json_config="$(cat "$__object/parameter/json-config")"
+     if [ "$json_config" = "-" ]; then
+        json_config="$__object/stdin"
+     fi
+     # remove leading and trailing whitespace and commas from first and last line
+     # indent each line with 3 spaces for consistency
+     json=$(sed -e 's/^[ \t]*/   /' -e '1s/^[ \t,]*//' -e '$s/[ \t,]*$//' "$json_config")
+     printf '   ,%s\n' "$json"
+  fi
+  echo "}"
+)
+echo "$json_configuration" | require="$config_deployment_requires" \
+  __file "$conf_dir/$conf_file" \
+      --owner root --group "$group" --mode 640 \
+      --state "$state" \
+      --source -
+
+# Set configuration deployment as requirement for service restart.
+restart_requires="__file/$conf_dir/$conf_file"
+
+###
+# Install TLS certificates.
+if [ -f "$__object/parameter/ca-file-source" ] || \
+   [ -f "$__object/parameter/cert-file-source" ] || \
+   [ -f "$__object/parameter/key-file-source" ]; then
+
+   requires="__file/$conf_dir/$conf_file" __directory $conf_dir/tls \
+     --owner root --group "$group" --mode 750 --state "$state"
+
+   # Append to service restart requirements.
+   restart_requires="$restart_requires __directory/$conf_dir/tls"
+fi
+
+###
+# Restart consul agent after everything else.
+require="$restart_requires" __service consul --action restart
diff --git a/cdist/conf/type/__consul_agent/parameter/boolean b/cdist/conf/type/__consul_agent/parameter/boolean
index 91f7f17e..c86853c3 100644
--- a/cdist/conf/type/__consul_agent/parameter/boolean
+++ b/cdist/conf/type/__consul_agent/parameter/boolean
@@ -6,3 +6,4 @@ server
 enable-syslog
 verify-incoming
 verify-outgoing
+use-distribution-package

From f595664924b3ae2ad190f9469fdf0be3b38d8a47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 21 Jan 2020 10:07:21 +0100
Subject: [PATCH 177/743] Patch __consul_* to discover remote consul
 configuration dir

---
 cdist/conf/type/__consul_check/explorer/conf-dir  |  1 +
 cdist/conf/type/__consul_check/manifest           |  2 +-
 .../conf/type/__consul_service/explorer/conf-dir  | 15 +++++++++++++++
 cdist/conf/type/__consul_service/manifest         |  5 ++---
 .../type/__consul_watch_checks/explorer/conf-dir  |  1 +
 cdist/conf/type/__consul_watch_checks/manifest    |  2 +-
 .../type/__consul_watch_event/explorer/conf-dir   |  1 +
 cdist/conf/type/__consul_watch_event/manifest     |  2 +-
 .../type/__consul_watch_key/explorer/conf-dir     |  1 +
 cdist/conf/type/__consul_watch_key/manifest       |  2 +-
 .../__consul_watch_keyprefix/explorer/conf-dir    |  1 +
 cdist/conf/type/__consul_watch_keyprefix/manifest |  2 +-
 .../type/__consul_watch_nodes/explorer/conf-dir   |  1 +
 cdist/conf/type/__consul_watch_nodes/manifest     |  2 +-
 .../type/__consul_watch_service/explorer/conf-dir |  1 +
 cdist/conf/type/__consul_watch_service/manifest   |  2 +-
 .../__consul_watch_services/explorer/conf-dir     |  1 +
 cdist/conf/type/__consul_watch_services/manifest  |  2 +-
 18 files changed, 33 insertions(+), 11 deletions(-)
 create mode 120000 cdist/conf/type/__consul_check/explorer/conf-dir
 create mode 100644 cdist/conf/type/__consul_service/explorer/conf-dir
 create mode 120000 cdist/conf/type/__consul_watch_checks/explorer/conf-dir
 create mode 120000 cdist/conf/type/__consul_watch_event/explorer/conf-dir
 create mode 120000 cdist/conf/type/__consul_watch_key/explorer/conf-dir
 create mode 120000 cdist/conf/type/__consul_watch_keyprefix/explorer/conf-dir
 create mode 120000 cdist/conf/type/__consul_watch_nodes/explorer/conf-dir
 create mode 120000 cdist/conf/type/__consul_watch_service/explorer/conf-dir
 create mode 120000 cdist/conf/type/__consul_watch_services/explorer/conf-dir

diff --git a/cdist/conf/type/__consul_check/explorer/conf-dir b/cdist/conf/type/__consul_check/explorer/conf-dir
new file mode 120000
index 00000000..daa712c3
--- /dev/null
+++ b/cdist/conf/type/__consul_check/explorer/conf-dir
@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir
\ No newline at end of file
diff --git a/cdist/conf/type/__consul_check/manifest b/cdist/conf/type/__consul_check/manifest
index c9f7add9..522aa1a9 100755
--- a/cdist/conf/type/__consul_check/manifest
+++ b/cdist/conf/type/__consul_check/manifest
@@ -19,7 +19,7 @@
 #
 
 name="$(cat "$__object/parameter/name" 2>/dev/null || echo "$__object_id")"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="check_${name}.json"
 state="$(cat "$__object/parameter/state")"
 
diff --git a/cdist/conf/type/__consul_service/explorer/conf-dir b/cdist/conf/type/__consul_service/explorer/conf-dir
new file mode 100644
index 00000000..6e94f781
--- /dev/null
+++ b/cdist/conf/type/__consul_service/explorer/conf-dir
@@ -0,0 +1,15 @@
+# Determine the configuration directory used by consul.
+
+check_dir () {
+  if [ -d "$1" ]; then
+    echo -n "$1"
+    exit
+  fi
+}
+
+check_dir '/etc/consul/conf.d'
+check_dir '/etc/consul.d'
+check_dir '/etc/consul'
+
+echo 'Could not determine consul configuration dir. Exiting.' >&2
+exit 1
diff --git a/cdist/conf/type/__consul_service/manifest b/cdist/conf/type/__consul_service/manifest
index 60397db7..d16f18e0 100755
--- a/cdist/conf/type/__consul_service/manifest
+++ b/cdist/conf/type/__consul_service/manifest
@@ -19,7 +19,7 @@
 #
 
 name="$(cat "$__object/parameter/name" 2>/dev/null || echo "$__object_id")"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="service_${name}.json"
 state="$(cat "$__object/parameter/state")"
 
@@ -45,7 +45,7 @@ printf '      "name": "%s"\n' "$name"
 cd "$__object/parameter/"
 for param in *; do
    case "$param" in
-      state|name|check-interval) continue ;;
+      state|name|check-interval|conf-dir) continue ;;
       check-script)
          printf '     ,"check": {\n'
          printf '         "script": "%s"\n' "$(cat "$__object/parameter/check-script")"
@@ -86,7 +86,6 @@ echo "   }"
 # end json file
 echo "}"
 ) | \
-require="__directory${conf_dir}" \
    __config_file "${conf_dir}/${conf_file}" \
       --owner root --group consul --mode 640 \
       --state "$state" \
diff --git a/cdist/conf/type/__consul_watch_checks/explorer/conf-dir b/cdist/conf/type/__consul_watch_checks/explorer/conf-dir
new file mode 120000
index 00000000..daa712c3
--- /dev/null
+++ b/cdist/conf/type/__consul_watch_checks/explorer/conf-dir
@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir
\ No newline at end of file
diff --git a/cdist/conf/type/__consul_watch_checks/manifest b/cdist/conf/type/__consul_watch_checks/manifest
index 5fdd7a74..4976b25a 100755
--- a/cdist/conf/type/__consul_watch_checks/manifest
+++ b/cdist/conf/type/__consul_watch_checks/manifest
@@ -20,7 +20,7 @@
 
 cdist_type="${__type##*/}"
 watch_type="${cdist_type##*_}"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="watch_${watch_type}_${__object_id}.json"
 state="$(cat "$__object/parameter/state")"
 
diff --git a/cdist/conf/type/__consul_watch_event/explorer/conf-dir b/cdist/conf/type/__consul_watch_event/explorer/conf-dir
new file mode 120000
index 00000000..daa712c3
--- /dev/null
+++ b/cdist/conf/type/__consul_watch_event/explorer/conf-dir
@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir
\ No newline at end of file
diff --git a/cdist/conf/type/__consul_watch_event/manifest b/cdist/conf/type/__consul_watch_event/manifest
index 61934656..b17680c1 100755
--- a/cdist/conf/type/__consul_watch_event/manifest
+++ b/cdist/conf/type/__consul_watch_event/manifest
@@ -20,7 +20,7 @@
 
 cdist_type="${__type##*/}"
 watch_type="${cdist_type##*_}"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="watch_${watch_type}_${__object_id}.json"
 state="$(cat "$__object/parameter/state")"
 
diff --git a/cdist/conf/type/__consul_watch_key/explorer/conf-dir b/cdist/conf/type/__consul_watch_key/explorer/conf-dir
new file mode 120000
index 00000000..daa712c3
--- /dev/null
+++ b/cdist/conf/type/__consul_watch_key/explorer/conf-dir
@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir
\ No newline at end of file
diff --git a/cdist/conf/type/__consul_watch_key/manifest b/cdist/conf/type/__consul_watch_key/manifest
index 61934656..b17680c1 100755
--- a/cdist/conf/type/__consul_watch_key/manifest
+++ b/cdist/conf/type/__consul_watch_key/manifest
@@ -20,7 +20,7 @@
 
 cdist_type="${__type##*/}"
 watch_type="${cdist_type##*_}"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="watch_${watch_type}_${__object_id}.json"
 state="$(cat "$__object/parameter/state")"
 
diff --git a/cdist/conf/type/__consul_watch_keyprefix/explorer/conf-dir b/cdist/conf/type/__consul_watch_keyprefix/explorer/conf-dir
new file mode 120000
index 00000000..daa712c3
--- /dev/null
+++ b/cdist/conf/type/__consul_watch_keyprefix/explorer/conf-dir
@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir
\ No newline at end of file
diff --git a/cdist/conf/type/__consul_watch_keyprefix/manifest b/cdist/conf/type/__consul_watch_keyprefix/manifest
index 61934656..b17680c1 100755
--- a/cdist/conf/type/__consul_watch_keyprefix/manifest
+++ b/cdist/conf/type/__consul_watch_keyprefix/manifest
@@ -20,7 +20,7 @@
 
 cdist_type="${__type##*/}"
 watch_type="${cdist_type##*_}"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="watch_${watch_type}_${__object_id}.json"
 state="$(cat "$__object/parameter/state")"
 
diff --git a/cdist/conf/type/__consul_watch_nodes/explorer/conf-dir b/cdist/conf/type/__consul_watch_nodes/explorer/conf-dir
new file mode 120000
index 00000000..daa712c3
--- /dev/null
+++ b/cdist/conf/type/__consul_watch_nodes/explorer/conf-dir
@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir
\ No newline at end of file
diff --git a/cdist/conf/type/__consul_watch_nodes/manifest b/cdist/conf/type/__consul_watch_nodes/manifest
index 61934656..b17680c1 100755
--- a/cdist/conf/type/__consul_watch_nodes/manifest
+++ b/cdist/conf/type/__consul_watch_nodes/manifest
@@ -20,7 +20,7 @@
 
 cdist_type="${__type##*/}"
 watch_type="${cdist_type##*_}"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="watch_${watch_type}_${__object_id}.json"
 state="$(cat "$__object/parameter/state")"
 
diff --git a/cdist/conf/type/__consul_watch_service/explorer/conf-dir b/cdist/conf/type/__consul_watch_service/explorer/conf-dir
new file mode 120000
index 00000000..daa712c3
--- /dev/null
+++ b/cdist/conf/type/__consul_watch_service/explorer/conf-dir
@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir
\ No newline at end of file
diff --git a/cdist/conf/type/__consul_watch_service/manifest b/cdist/conf/type/__consul_watch_service/manifest
index db38eb18..e8d18328 100755
--- a/cdist/conf/type/__consul_watch_service/manifest
+++ b/cdist/conf/type/__consul_watch_service/manifest
@@ -20,7 +20,7 @@
 
 cdist_type="${__type##*/}"
 watch_type="${cdist_type##*_}"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="watch_${watch_type}_${__object_id}.json"
 state="$(cat "$__object/parameter/state")"
 
diff --git a/cdist/conf/type/__consul_watch_services/explorer/conf-dir b/cdist/conf/type/__consul_watch_services/explorer/conf-dir
new file mode 120000
index 00000000..daa712c3
--- /dev/null
+++ b/cdist/conf/type/__consul_watch_services/explorer/conf-dir
@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir
\ No newline at end of file
diff --git a/cdist/conf/type/__consul_watch_services/manifest b/cdist/conf/type/__consul_watch_services/manifest
index 61934656..b17680c1 100755
--- a/cdist/conf/type/__consul_watch_services/manifest
+++ b/cdist/conf/type/__consul_watch_services/manifest
@@ -20,7 +20,7 @@
 
 cdist_type="${__type##*/}"
 watch_type="${cdist_type##*_}"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="watch_${watch_type}_${__object_id}.json"
 state="$(cat "$__object/parameter/state")"
 

From 31ad1bdaad701720303054d668cf9635987edf1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Sun, 26 Jan 2020 15:13:12 +0100
Subject: [PATCH 178/743] Fix various typos and styling errors in
 __consul_agent, conf-dir explorer

---
 cdist/conf/type/__consul_agent/manifest       | 40 ++++++++++---------
 .../type/__consul_service/explorer/conf-dir   |  2 +-
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest
index 599f15b4..40667002 100755
--- a/cdist/conf/type/__consul_agent/manifest
+++ b/cdist/conf/type/__consul_agent/manifest
@@ -49,7 +49,7 @@ distribution_setup () {
      debian)
        # consul is only available starting Debian 10 (buster).
        # See https://packages.debian.org/buster/consul
-       if [ $release -lt 10 ]; then
+       if [ "$release" -lt 10 ]; then
          echo "Consul is not available for your debian release." >&2
          echo "Please use the 'manual' (i.e. non-package) installation or \
            upgrade the target system." >&2
@@ -59,7 +59,7 @@ distribution_setup () {
        # Override previously defined environment to match debian packaging.
        conf_dir='/etc/consul.d'
        user='consul'
-       grou='consul'
+       group='consul'
      ;;
      *)
         echo "Your operating system ($os) is currently not supported with the \
@@ -71,7 +71,7 @@ distribution_setup () {
   esac
 
   # Install consul package.
-  __package consul --state $state
+  __package consul --state "$state"
 
   export config_deployment_requires="__package/consul"
 }
@@ -139,7 +139,7 @@ manual_setup () {
   esac
 
   # Create data directory.
-  require="__user/consul"__directory "$data_dir" \
+  require="__user/consul" __directory "$data_dir" \
     --owner "$user" --group "$group" --mode 770 --state "$state"
 
   # Create config directory.
@@ -202,8 +202,23 @@ else
   manual_setup
 fi
 
+###
+# Install TLS certificates.
+
+if [ -f "$__object/parameter/ca-file-source" ] || \
+   [ -f "$__object/parameter/cert-file-source" ] || \
+   [ -f "$__object/parameter/key-file-source" ]; then
+
+   requires="$config_deployment_requires" __directory $tls_dir \
+     --owner root --group "$group" --mode 750 --state "$state"
+
+   # Append to service restart requirements.
+   restart_requires="$restart_requires __directory/$conf_dir/tls"
+fi
+
 ###
 # Generate and deploy configuration.
+
 json_configuration=$(
   echo "{"
 
@@ -216,8 +231,8 @@ json_configuration=$(
         state|user|group|json-config|use-distribution-package) continue ;;
         ca-file-source|cert-file-source|key-file-source)
            source="$(cat "$__object/parameter/$param")"
-           destination="/etc/consul/ssl/${source##*/}"
-           require="__directory/etc/consul/ssl" \
+           destination="$tls_dir/${source##*/}"
+           require="__directory/$tls_dir" \
               __file "$destination" \
                  --owner root --group consul --mode 640 \
                  --source "$source" \
@@ -276,19 +291,6 @@ echo "$json_configuration" | require="$config_deployment_requires" \
 # Set configuration deployment as requirement for service restart.
 restart_requires="__file/$conf_dir/$conf_file"
 
-###
-# Install TLS certificates.
-if [ -f "$__object/parameter/ca-file-source" ] || \
-   [ -f "$__object/parameter/cert-file-source" ] || \
-   [ -f "$__object/parameter/key-file-source" ]; then
-
-   requires="__file/$conf_dir/$conf_file" __directory $conf_dir/tls \
-     --owner root --group "$group" --mode 750 --state "$state"
-
-   # Append to service restart requirements.
-   restart_requires="$restart_requires __directory/$conf_dir/tls"
-fi
-
 ###
 # Restart consul agent after everything else.
 require="$restart_requires" __service consul --action restart
diff --git a/cdist/conf/type/__consul_service/explorer/conf-dir b/cdist/conf/type/__consul_service/explorer/conf-dir
index 6e94f781..0fc9ef84 100644
--- a/cdist/conf/type/__consul_service/explorer/conf-dir
+++ b/cdist/conf/type/__consul_service/explorer/conf-dir
@@ -2,7 +2,7 @@
 
 check_dir () {
   if [ -d "$1" ]; then
-    echo -n "$1"
+    printf '%s' "$1"
     exit
   fi
 }

From 09540dc6bd63cb338ed8dc27bf69e2cc3547f02f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 17 Feb 2020 11:18:36 +0100
Subject: [PATCH 179/743] Add simple __service type

---
 .../type/__service/explorer/service-manager   |  8 +++
 cdist/conf/type/__service/gencode-remote      |  9 ++++
 cdist/conf/type/__service/man.rst             | 51 +++++++++++++++++++
 cdist/conf/type/__service/manifest            | 15 ++++++
 cdist/conf/type/__service/parameter/required  |  1 +
 5 files changed, 84 insertions(+)
 create mode 100755 cdist/conf/type/__service/explorer/service-manager
 create mode 100755 cdist/conf/type/__service/gencode-remote
 create mode 100644 cdist/conf/type/__service/man.rst
 create mode 100644 cdist/conf/type/__service/manifest
 create mode 100644 cdist/conf/type/__service/parameter/required

diff --git a/cdist/conf/type/__service/explorer/service-manager b/cdist/conf/type/__service/explorer/service-manager
new file mode 100755
index 00000000..55a873fa
--- /dev/null
+++ b/cdist/conf/type/__service/explorer/service-manager
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Assume systemd if systemctl is in PATH.
+if [ "$(command -v systemctl)" ]; then
+	printf "systemd"
+else
+	printf "unknown"
+fi
diff --git a/cdist/conf/type/__service/gencode-remote b/cdist/conf/type/__service/gencode-remote
new file mode 100755
index 00000000..ac62e05f
--- /dev/null
+++ b/cdist/conf/type/__service/gencode-remote
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+manager="$(cat "$__object/explorer/service-manager")"
+name=$__object_id
+action="$(cat "$__object/parameter/action")"
+
+if [ "$manager" = "unknown" ]; then
+	echo "service '$name' '$action'"
+fi
diff --git a/cdist/conf/type/__service/man.rst b/cdist/conf/type/__service/man.rst
new file mode 100644
index 00000000..f9b23d5b
--- /dev/null
+++ b/cdist/conf/type/__service/man.rst
@@ -0,0 +1,51 @@
+cdist-type__service(7)
+======================
+
+NAME
+----
+cdist-type__service - Run action on a system service
+
+
+DESCRIPTION
+-----------
+This type allows you to run an action against a system service.
+
+
+REQUIRED PARAMETERS
+-------------------
+action
+  Arbitrary parameter passed as action. Usually 'start', 'stop', 'reload' or 'restart'.
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Restart nginx service.
+    __service nginx --action restart
+
+    # Stop postfix service.
+    __service postfix --action stop
+
+
+AUTHORS
+-------
+Timothée Floure <timothee.floure@ungleich.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2019 Timothée Floure. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__service/manifest b/cdist/conf/type/__service/manifest
new file mode 100644
index 00000000..cb5af234
--- /dev/null
+++ b/cdist/conf/type/__service/manifest
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+manager="$(cat "$__object/explorer/service-manager")"
+
+name=$__object_id
+action="$(cat "$__object/parameter/action")"
+
+case "$manager" in
+	systemd)
+		__systemd_service "$name" --action "$action"
+	;;
+	*)
+		# Unknown: handled by `service $NAME $action` in gencode-remote.
+	;;
+esac
diff --git a/cdist/conf/type/__service/parameter/required b/cdist/conf/type/__service/parameter/required
new file mode 100644
index 00000000..a9f84d41
--- /dev/null
+++ b/cdist/conf/type/__service/parameter/required
@@ -0,0 +1 @@
+action

From b891bb05d5751fe0f80829d7fe54cd56468d5e51 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 19 Feb 2020 12:58:22 +0200
Subject: [PATCH 180/743] __update_alternatives: add state explorer

---
 cdist/conf/type/__update_alternatives/explorer/state | 8 ++++++++
 cdist/conf/type/__update_alternatives/gencode-remote | 4 ++++
 2 files changed, 12 insertions(+)
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/state

diff --git a/cdist/conf/type/__update_alternatives/explorer/state b/cdist/conf/type/__update_alternatives/explorer/state
new file mode 100755
index 00000000..04a78aaa
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/state
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+path="$(cat "$__object/parameter/path")"
+name="$__object_id"
+link="$(readlink "/etc/alternatives/$name")"
+if [ "$path" = "$link" ]
+then echo present
+else echo absent
+fi
diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index 0e7b0d89..b632deb2 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -21,6 +21,10 @@
 # Setup alternative - no standard way to create, always set
 #
 
+if [ "$(cat "$__object/explorer/state")" = 'present' ]
+then exit 0
+fi
+
 path="$(cat "$__object/parameter/path")"
 name="$__object_id"
 echo "update-alternatives --quiet --set '$name' '$path'"

From a3bc8f94075e23d8c0032fc8322aba5eebebd41e Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 19 Feb 2020 12:59:14 +0200
Subject: [PATCH 181/743] __update_alternatives: remove expired comment

---
 cdist/conf/type/__update_alternatives/gencode-remote | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index b632deb2..c0b49814 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -17,9 +17,6 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# Setup alternative - no standard way to create, always set
-#
 
 if [ "$(cat "$__object/explorer/state")" = 'present' ]
 then exit 0

From 1b0caeda13ddfc3d941332ea912983aba01b4819 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 19 Feb 2020 14:49:35 +0100
Subject: [PATCH 182/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index bd767b9d..6410d36d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __update_alternatives: Add state explorer (Ander Punnar)
+
 6.5.1: 2020-02-15
 	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)
 	* Explorer os_release: Add fallbacks (Dennis Camera)

From 28d3760e29aaf8cdb30390f7e78c2334d7af8bc4 Mon Sep 17 00:00:00 2001
From: Jin-Guk Kwon <jinguk.kwon@ungleich.ch>
Date: Thu, 20 Feb 2020 09:40:55 +0100
Subject: [PATCH 183/743] [cdist]Update os_version for alpine

---
 cdist/conf/explorer/os_version | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 4c41695b..1d54ea60 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -70,4 +70,7 @@ case "$("$__explorer/os")" in
    ubuntu)
       lsb_release -sr
    ;;
-esac
+   alpine)
+       cat /etc/alpine-release
+   ;;
+esac
\ No newline at end of file

From ceddbd15a04327489916b61e58fdf2d7a7728427 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 20 Feb 2020 10:15:34 +0100
Subject: [PATCH 184/743] ++changes

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 6410d36d..b0ebf789 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __update_alternatives: Add state explorer (Ander Punnar)
+	* Explorer os_version: Add support for Alpine Linux (Jin-Guk Kwon)
 
 6.5.1: 2020-02-15
 	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)

From 21c9e3db1852eebb88be5454137ceb23159f7dc0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 3 Feb 2020 22:12:21 +0100
Subject: [PATCH 185/743] [explorer/init] Support more init systems

---
 cdist/conf/explorer/init | 225 ++++++++++++++++++++++++++++++---------
 1 file changed, 174 insertions(+), 51 deletions(-)

diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init
index ceae2e9f..bf1736cd 100755
--- a/cdist/conf/explorer/init
+++ b/cdist/conf/explorer/init
@@ -1,7 +1,8 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2016 Daniel Heule (hda at sfs.biz)
 # Copyright 2017, Philippe Gregoire <pg@pgregoire.xyz>
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -23,59 +24,181 @@
 # for example at linux this value is "init" or "systemd" in most cases
 #
 
+set -e
+
+# Expected values:
+# Linux:
+#  Gentoo:
+#	 sysvinit, openrc-init
+
+# GNU:
+#	Debian:
+#	  hurd-init, sysvinit
+
+# [root@fedora-12 ~]# readlink /proc/1/exe
+# /sbin/init (deleted)
+# [root@fedora-12 ~]# ls -l /proc/1/exe
+# lrwxrwxrwx. 1 root root 0 2020-01-30 23:00 /proc/1/exe -> /sbin/init (deleted)
+
+# inspired by https://stackoverflow.com/a/33266819
+shreadlink() (
+	CDPATH=
+	target=$1 fname= targetDir=
+
+	# Resolve potential symlinks until the ultimate target is found.
+	while :
+	do
+		if ! test -e "$target"
+		then
+			printf 'ERROR: %s does not exist.\n' "'$target'" >&2
+			return 1
+		fi
+
+		# Change to target dir; necessary for correct resolution of target path.
+		cd "$(dirname -- "$target")"
+
+		fname=$(basename -- "$target") # Extract filename.
+		[ "$fname" = '/' ] && fname='' # !! curiously, `basename /` returns '/'
+
+		[ -L "$fname" ] || break
+
+		# Extract [next] target path, which may be defined
+		# *relative* to the symlink's own directory.
+		# Note: We parse `ls -l` output to find the symlink target
+		#		which is the only POSIX-compliant, albeit somewhat fragile, way.
+		# FIXME: Will break if one of the filenames contain ’ -> ’
+		target=$(ls -l "$fname" | sed -e 's/^.* -> //')
+	done
+
+	# Get canonical dir. path
+	targetDir=$(pwd -P)
+
+	# Output the ultimate target's canonical path.
+	# Note that we manually resolve paths ending in /. and /.. to make sure we have a normalized path.
+	if test "$fname" = '.'
+	then
+		printf '%s\n' "${targetDir%/}"
+	elif test "$fname" = '..'
+	then
+		# Caveat: something like /var/.. will resolve to /private (assuming /var@ -> /private/var), i.e. the '..' is applied
+		# AFTER canonicalization.
+		printf '%s\n' "$(dirname -- "${targetDir}")"
+	else
+		printf '%s/%s\n' "${targetDir%/}" "$fname"
+	fi
+)
+
+
 case $(uname -s) in
-    Linux)
-        if test -d /proc/1/
-        then
-            comm_name=$(cat /proc/1/comm)
-        else
-            # BusyBox's versions of ps and pgrep do not support some options
-            # depending on which compile-time options have been used.
-            # Both pgrep and ps are tried to get the command name
-            comm_name=$(
-                pgrep -P0 -l 2>/dev/null | awk '/^1[ \t]/ { print $2 }'
-                || ps -o comm= -p 1 2>/dev/null)
-        fi
+	Linux|GNU)
+		# if test -f /proc/1/comm
+		# then
+		#	  comm_name=$(cat /proc/1/comm)
+		# else
+			# BusyBox's versions of ps and pgrep do not support some options
+			# depending on which compile-time options have been used.
+			# Both pgrep and ps are tried to get the command name
+		#	  comm_name=$(
+		#		  pgrep -P0 -l 2>/dev/null | awk '/^1[ \t]/ { print $2 }'
+		#		  || ps -o comm= -p 1 2>/dev/null)
+		# fi
 
-        case $comm_name
-        in
-            systemd)
-                echo systemd
-            ;;
-            init)
-                # It could be anything...
+		init_exe=$(shreadlink /proc/1/exe)
 
-                if test -h /proc/1/exe
-                then
-                    init_exe=/proc/1/exe
-                else
-                    init_exe=$(command -v "$comm_name")
-                fi
+		if ! test -x "$init_exe"
+		then
+			# On some rare occasions it can happen that the
+			# running init's binary has been replaced. In this
+			# case Linux adjusts the symlink to "X (deleted)"
+			case $init_exe
+			in
+				*' (deleted)')
+					init_exe=${init_exe% (deleted)}
+					test -x "$init_exe" || exit 1
+					;;
+				*)
+					exit 1
+					;;
+			esac
+		fi
 
-                test -x "$comm_exe" || exit 1
+		if test "$init_exe" = '/hurd/init'
+		then
+			# XXX: Could maybe be removed
+			echo hurd-init
+			exit 0
+		fi
 
-                case $("$comm_exe" --version | head -n 1)
-                in
-                    *SysV*)
-                        echo init
-                    ;;
-                    *upstart*)
-                        echo upstart
-                    ;;
-                    *)
-                        echo ""
-                    ;;
-                esac
-        esac
-    ;;
-    FreeBSD|OpenBSD)
-        ps -o comm= -p 1 2>/dev/null || true
-    ;;
-    Darwin)
-        basename "$(ps -o comm= -p 1 2>/dev/null)"
-    ;;
-    *)
-        # return a empty string as unknown value
-        echo ""
-    ;;
+		comm_name=$(basename "$init_exe")
+		case $comm_name
+		in
+			init)
+				:  # handled below
+				;;
+			systemd)
+				# NOTE: sd_booted(3)
+				if test -d /run/systemd/system/
+				then
+					echo systemd
+					exit 0
+				fi
+				# otherwise: treat like "init"
+				;;
+			*)
+				echo "$comm_name"
+				exit 0
+				;;
+		esac
+
+		# init: it could be anything...
+		case $("$init_exe" --version 2>/dev/null | head -n 1)
+		in
+			SysV*)
+				# This is a little bit more specific than init
+				echo sysvinit
+				exit 0
+				;;
+			*'GNU Hurd'*)
+				echo hurd-init
+				;;
+			*upstart*)
+				echo upstart
+				exit 0
+				;;
+		esac
+		case $("$init_exe" --help 2>/dev/null | head -n 1)
+		in
+			BusyBox*)
+				echo busybox
+				exit 0
+				;;
+		esac
+
+		echo init
+		;;
+	FreeBSD|OpenBSD)
+		ps -o comm= -p 1 2>/dev/null || true
+		;;
+	Darwin)
+		basename "$(ps -o comm= -p 1 2>/dev/null)"
+		;;
+	SunOS)
+		comm_name=$(ps -o comm= -p 1 2>/dev/null)
+		if test "$(basename "$comm_name")" != 'init'
+		then
+			echo "${comm_name}"
+			exit 0
+		fi
+
+		# XXX: Is this the correct way??
+		if test -f /etc/svc/volatile/svc_nonpersist.db
+		then
+			echo smf
+			exit 0
+		fi
+		;;
+	*)
+		# return a empty string as unknown value
+		echo ""
+		;;
 esac

From d895bb0e87f524e8a64a802e6ff1922e52860ffa Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 18 Feb 2020 01:24:41 +0100
Subject: [PATCH 186/743] [explorer/init] Clean up

---
 cdist/conf/explorer/init | 399 ++++++++++++++++++++++++---------------
 1 file changed, 246 insertions(+), 153 deletions(-)

diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init
index bf1736cd..2d4f07c1 100755
--- a/cdist/conf/explorer/init
+++ b/cdist/conf/explorer/init
@@ -20,185 +20,278 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Returns the process name of pid 1 ( normaly the init system )
-# for example at linux this value is "init" or "systemd" in most cases
+# Returns the name of the init system (PID 1)
 #
-
-set -e
-
 # Expected values:
 # Linux:
+#  Adélie Linux:
+#    sysvinit+openrc
+#  Alpine Linux:
+#    busybox-init+openrc
+#  ArchLinux:
+#    systemd, sysvinit
+#  CRUX:
+#    sysvinit
+#  Debian:
+#    systemd, upstart, sysvinit, openrc, ???
+#  Devuan:
+#    sysvinit, ???
 #  Gentoo:
-#	 sysvinit, openrc-init
-
+#    sysvinit+openrc, openrc-init, systemd
+#  OpenBMC:
+#    systemd
+#  OpenWrt:
+#    procd, init??
+#  RedHat (RHEL, CentOS, Fedora, RedHat Linux, ...):
+#    systemd, upstart, sysvinit
+#  Slackware:
+#    sysvinit
+#  SuSE:
+#    systemd, sysvinit
+#  Ubuntu:
+#    systemd, upstart, sysvinit
+#
 # GNU:
-#	Debian:
-#	  hurd-init, sysvinit
+#   Debian:
+#     hurd-init, sysvinit
+#
+# BSD:
+#  {Free,Open,Net}BSD:
+#    init
+#
+# Mac OS X:
+#   launchd, init
+#
+# Solaris/Illumos:
+#   smf, init
+
 
 # [root@fedora-12 ~]# readlink /proc/1/exe
 # /sbin/init (deleted)
 # [root@fedora-12 ~]# ls -l /proc/1/exe
 # lrwxrwxrwx. 1 root root 0 2020-01-30 23:00 /proc/1/exe -> /sbin/init (deleted)
 
-# inspired by https://stackoverflow.com/a/33266819
-shreadlink() (
-	CDPATH=
-	target=$1 fname= targetDir=
+set -e
+#set -x  # DEBUG
 
-	# Resolve potential symlinks until the ultimate target is found.
-	while :
-	do
-		if ! test -e "$target"
-		then
-			printf 'ERROR: %s does not exist.\n' "'$target'" >&2
-			return 1
-		fi
-
-		# Change to target dir; necessary for correct resolution of target path.
-		cd "$(dirname -- "$target")"
-
-		fname=$(basename -- "$target") # Extract filename.
-		[ "$fname" = '/' ] && fname='' # !! curiously, `basename /` returns '/'
-
-		[ -L "$fname" ] || break
-
-		# Extract [next] target path, which may be defined
-		# *relative* to the symlink's own directory.
-		# Note: We parse `ls -l` output to find the symlink target
-		#		which is the only POSIX-compliant, albeit somewhat fragile, way.
-		# FIXME: Will break if one of the filenames contain ’ -> ’
-		target=$(ls -l "$fname" | sed -e 's/^.* -> //')
-	done
-
-	# Get canonical dir. path
-	targetDir=$(pwd -P)
-
-	# Output the ultimate target's canonical path.
-	# Note that we manually resolve paths ending in /. and /.. to make sure we have a normalized path.
-	if test "$fname" = '.'
+validate_busybox_init() {
+	# It is quite common to use SysVinit to stack other init systemd
+	# (like OpenRC) on top of it. So we check for that, too.
+	if stacked=$(validate_openrc)
 	then
-		printf '%s\n' "${targetDir%/}"
-	elif test "$fname" = '..'
-	then
-		# Caveat: something like /var/.. will resolve to /private (assuming /var@ -> /private/var), i.e. the '..' is applied
-		# AFTER canonicalization.
-		printf '%s\n' "$(dirname -- "${targetDir}")"
+		echo "busybox-init+${stacked}"
 	else
-		printf '%s/%s\n' "${targetDir%/}" "$fname"
+		echo busybox-init
 	fi
-)
+}
 
+validate_hurd_init() {
+	# FIXME: Test me!
+	test -x /hurd/init || return 1
+	grep -q 'GNU Hurd' /hurd/init || return 1
+	echo hurd-init
+}
 
-case $(uname -s) in
-	Linux|GNU)
-		# if test -f /proc/1/comm
-		# then
-		#	  comm_name=$(cat /proc/1/comm)
-		# else
-			# BusyBox's versions of ps and pgrep do not support some options
-			# depending on which compile-time options have been used.
-			# Both pgrep and ps are tried to get the command name
-		#	  comm_name=$(
-		#		  pgrep -P0 -l 2>/dev/null | awk '/^1[ \t]/ { print $2 }'
-		#		  || ps -o comm= -p 1 2>/dev/null)
-		# fi
+validate_openrc() {
+	test -f /run/openrc/softlevel || return 1
+	echo openrc
+}
 
-		init_exe=$(shreadlink /proc/1/exe)
+validate_procd() {
+	grep -q 'procd' /sbin/procd || return 1
+	echo procd
+}
 
-		if ! test -x "$init_exe"
-		then
-			# On some rare occasions it can happen that the
-			# running init's binary has been replaced. In this
-			# case Linux adjusts the symlink to "X (deleted)"
-			case $init_exe
-			in
-				*' (deleted)')
-					init_exe=${init_exe% (deleted)}
-					test -x "$init_exe" || exit 1
-					;;
-				*)
-					exit 1
-					;;
-			esac
-		fi
+validate_runit() {
+	test -d /run/runit || return 1
+	echo runit
+}
 
-		if test "$init_exe" = '/hurd/init'
-		then
-			# XXX: Could maybe be removed
-			echo hurd-init
-			exit 0
-		fi
+validate_smf() {
+	# XXX: Is this the correct way??
+	test -f /etc/svc/volatile/svc_nonpersist.db || return 1
+	echo smf
+}
 
-		comm_name=$(basename "$init_exe")
-		case $comm_name
+validate_systemd() {
+	# NOTE: sd_booted(3)
+	test -d /run/systemd/system/ || return 1
+	# systemctl --version | sed -e '/^systemd/!d;s/^systemd //'
+	echo systemd
+}
+
+validate_sysvinit() {
+	test -x /sbin/init \
+	 && grep -q 'INIT_VERSION=sysvinit-[0-9.]*' /sbin/init \
+	 || return 1
+
+	# It is quite common to use SysVinit to stack other init systemd
+	# (like OpenRC) on top of it. So we check for that, too.
+	if stacked=$(validate_openrc)
+	then
+		echo "sysvinit+${stacked}"
+	else
+		echo sysvinit
+	fi
+	unset stacked
+}
+
+validate_upstart() {
+	test -x "$(command -v initctl)" || return 1
+	case $(initctl version)
+	in
+		*'(upstart '*')')
+			# if type -d /etc/init
+			# then
+			# 	# modern (DBus-based?) upstart >= 0.5
+			# 	:
+			# elif type -d /etc/events.d
+			# then
+			# 	# ancient upstart
+			# 	:
+			# fi
+			echo upstart
+			;;
+		*)
+			return 1
+			;;
+	esac
+}
+
+find_init_procfs() (
+	# First, check if the required file in procfs exists...
+	test -h /proc/1/exe || return 1
+
+	# Find init executable
+	init_exe=$(ls -l /proc/1/exe 2>/dev/null)
+	init_exe=${init_exe#* -> }
+
+	if ! test -x "$init_exe"
+	then
+		# On some rare occasions it can happen that the
+		# running init's binary has been replaced. In this
+		# case Linux adjusts the symlink to "X (deleted)"
+		case $init_exe
 		in
-			init)
-				:  # handled below
-				;;
-			systemd)
-				# NOTE: sd_booted(3)
-				if test -d /run/systemd/system/
-				then
-					echo systemd
-					exit 0
-				fi
-				# otherwise: treat like "init"
+			*' (deleted)')
+				init_exe=${init_exe% (deleted)}
+				test -x "$init_exe" || exit 1
 				;;
 			*)
-				echo "$comm_name"
-				exit 0
+				exit 1
 				;;
 		esac
+	fi
 
-		# init: it could be anything...
-		case $("$init_exe" --version 2>/dev/null | head -n 1)
-		in
-			SysV*)
-				# This is a little bit more specific than init
-				echo sysvinit
-				exit 0
-				;;
-			*'GNU Hurd'*)
-				echo hurd-init
-				;;
-			*upstart*)
-				echo upstart
-				exit 0
-				;;
-		esac
-		case $("$init_exe" --help 2>/dev/null | head -n 1)
-		in
-			BusyBox*)
-				echo busybox
-				exit 0
-				;;
-		esac
+	echo "${init_exe}"
+)
 
-		echo init
-		;;
-	FreeBSD|OpenBSD)
-		ps -o comm= -p 1 2>/dev/null || true
-		;;
-	Darwin)
-		basename "$(ps -o comm= -p 1 2>/dev/null)"
-		;;
-	SunOS)
-		comm_name=$(ps -o comm= -p 1 2>/dev/null)
-		if test "$(basename "$comm_name")" != 'init'
-		then
-			echo "${comm_name}"
-			exit 0
-		fi
+# BusyBox's versions of ps and pgrep do not support some options
+# depending on which compile-time options have been used.
 
-		# XXX: Is this the correct way??
-		if test -f /etc/svc/volatile/svc_nonpersist.db
-		then
-			echo smf
-			exit 0
-		fi
-		;;
-	*)
-		# return a empty string as unknown value
-		echo ""
-		;;
-esac
+find_init_pgrep() {
+	pgrep -P0 -fl 2>/dev/null | awk -F '[[:blank:]]' '$1 == 1 { print $2 }'
+}
+
+find_init_ps() {
+	case $(uname -s)
+	in
+		Darwin|NetBSD)
+			ps -o ucomm= -p 1 2>/dev/null
+			;;
+		FreeBSD)
+			ps -o command= -p 1 2>/dev/null | cut -d ' ' -f 1
+			;;
+		OpenBSD)
+			ps -o command -p 1 2>/dev/null | tail -n +2 | cut -d ' ' -f 1
+			;;
+		*)
+			ps -o comm= -p 1 2>/dev/null
+			;;
+	esac
+}
+
+find_init() {
+	case $(uname -s)
+	in
+		Linux|GNU|NetBSD)
+			find_init_procfs || find_init_pgrep || find_init_ps
+			;;
+		FreeBSD)
+			find_init_procfs || find_init_ps
+			;;
+		OpenBSD)
+			find_init_pgrep || find_init_ps
+			;;
+		Darwin|FreeBSD|SunOS)
+			find_init_ps
+			;;
+		*)
+			echo "Don't know how to determine init." >&2
+			echo 'Please send a patch.' >&2
+			exit 1
+	esac
+}
+
+validate_by_comm_name() {
+	case $1
+	in
+		busybox)
+			validate_busybox_init
+			;;
+		init)
+			# FIXME: Do some more magic here!
+			echo init
+			;;
+		openrc-init)
+			validate_openrc >/dev/null && echo openrc-init
+			;;
+		runit)
+			validate_runit
+			;;
+		systemd)
+			validate_systemd
+			;;
+		*)
+			# Run validate function by comm name if available.
+			# Fall back to comm name if either it does not exist or
+			# returns non-zero.
+			type "validate_$1" >/dev/null && "validate_$1" || echo $1
+	esac
+}
+
+try_all() {
+	# init: it could be anything...
+	# We try some approaches to gather more information about init without
+	# calling it! On some init systemd this triggers a reinitialisation of
+	# the system which we don't want (e.g. embedded systems).
+
+	validate_sysvinit || \
+	validate_openrc || \
+	validate_runit || \
+	validate_smf || \
+	validate_upstart || \
+	validate_hurd_init || \
+	echo init  # fallback
+}
+
+init=$(find_init)
+
+if test -x "${init}"
+then
+	case $init
+	in
+		/hurd/init)
+			# FIXME: Create validate function
+			echo hurd-init
+			;;
+		*/init)
+			try_all
+			;;
+		*)
+			validate_by_comm_name "$(basename "${init}")"
+			;;
+	esac
+else
+	validate_by_comm_name "${init}"
+fi

From 364340c8d5a1bcb6492d6c8cdfbbd808b13b5024 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 20 Feb 2020 21:34:21 +0100
Subject: [PATCH 187/743] [explorer/init] Refactor and testing

---
 cdist/conf/explorer/init | 392 ++++++++++++++++++++++++++-------------
 1 file changed, 267 insertions(+), 125 deletions(-)

diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init
index 2d4f07c1..db417a14 100755
--- a/cdist/conf/explorer/init
+++ b/cdist/conf/explorer/init
@@ -21,7 +21,7 @@
 #
 #
 # Returns the name of the init system (PID 1)
-#
+
 # Expected values:
 # Linux:
 #  Adélie Linux:
@@ -35,122 +35,221 @@
 #  Debian:
 #    systemd, upstart, sysvinit, openrc, ???
 #  Devuan:
-#    sysvinit, ???
+#    sysvinit, sysvinit+openrc
 #  Gentoo:
 #    sysvinit+openrc, openrc-init, systemd
 #  OpenBMC:
 #    systemd
 #  OpenWrt:
-#    procd, init??
+#    procd, init???
 #  RedHat (RHEL, CentOS, Fedora, RedHat Linux, ...):
-#    systemd, upstart, sysvinit
+#    systemd, upstart, upstart-legacy, sysvinit
 #  Slackware:
 #    sysvinit
 #  SuSE:
 #    systemd, sysvinit
 #  Ubuntu:
-#    systemd, upstart, sysvinit
+#    systemd, upstart, upstart-legacy, sysvinit
+#  VoidLinux:
+#    runit
 #
 # GNU:
 #   Debian:
-#     hurd-init, sysvinit
+#     sysvinit, hurd-init
 #
 # BSD:
 #  {Free,Open,Net}BSD:
 #    init
 #
 # Mac OS X:
-#   launchd, init
+#   launchd, init+SystemStarter
 #
 # Solaris/Illumos:
-#   smf, init
+#   smf, init???
 
+# NOTE: init systems can be stacked. This is popular to run OpenRC on top of
+# sysvinit (Gentoo) or busybox-init (Alpine), but can also be used to run runit
+# as a systemd service.  This makes init system detection very complicated
+# (which result is expected?)  This script tries to untangle some combinations,
+# OpenRC on top of sysv or busybox (X+openrc), but will ignore others (runit as
+# a systemd service)
+
+# NOTE: When we have no idea, nothing will be printed!
+
+# NOTE:
+# When trying to gather information about the init system make sure to do so
+# without calling the binary!   On some systems this triggers a reinitialisation
+# of the system which we don't want (e.g. embedded systems).
 
-# [root@fedora-12 ~]# readlink /proc/1/exe
-# /sbin/init (deleted)
-# [root@fedora-12 ~]# ls -l /proc/1/exe
-# lrwxrwxrwx. 1 root root 0 2020-01-30 23:00 /proc/1/exe -> /sbin/init (deleted)
 
 set -e
-#set -x  # DEBUG
 
-validate_busybox_init() {
-	# It is quite common to use SysVinit to stack other init systemd
+KERNEL_NAME=$(uname -s)
+
+KNOWN_INIT_SYSTEMS=$(cat <<EOF
+systemd
+sysvinit
+upstart
+runit
+procd
+smf
+launchd
+init
+hurd_init
+systemstarter
+EOF
+)
+
+
+common_candidates_by_kernel() {
+	case $KERNEL_NAME
+	in
+		FreeBSD|NetBSD|OpenBSD)
+			echo init
+			;;
+		Linux)
+			echo systemd
+			echo sysvinit
+			echo upstart
+			;;
+		GNU)
+			echo sysvinit
+			echo hurd-init
+			;;
+		Darwin)
+			echo launchd
+			echo systemstarter
+			;;
+		SunOS)
+			echo smf
+			;;
+	esac
+}
+
+
+## Helpers
+
+trim() {
+	sed -e 's/^[[:blank:]]*//' -e 's/[[:blank:]]*$//' -e '/^[[:blank:]]*$/d'
+}
+
+unique() {
+	# Delete duplicate lines (keeping input order)
+	awk '!x[$0]++'
+}
+
+
+## Check functions
+# These functions are used to verify if a guess is correct by checking some
+# common property of a running system (presence of a directory in /run etc.)
+
+check_busybox_init() (
+	busybox_path=${1:-/bin/busybox}
+	test -x "${busybox_path}" || return 1
+	grep -q 'BusyBox v[0-9]' "${busybox_path}" || return 1
+
+	# It is quite common to use Busybox init to stack other init systemd
 	# (like OpenRC) on top of it. So we check for that, too.
-	if stacked=$(validate_openrc)
+	if stacked=$(check_openrc)
 	then
 		echo "busybox-init+${stacked}"
 	else
 		echo busybox-init
 	fi
-}
+)
 
-validate_hurd_init() {
-	# FIXME: Test me!
-	test -x /hurd/init || return 1
-	grep -q 'GNU Hurd' /hurd/init || return 1
+check_hurd_init() (
+	init_exe=${1:-/hurd/init}
+	test -x "${init_exe}" || return 1
+	grep -q 'GNU Hurd' "${init_exe}" || return 1
 	echo hurd-init
+)
+
+check_init() {
+	# Checks for various BSD inits...
+	test -x /sbin/init || return 1
+
+	if grep -q -E '(Free|Net|Open)BSD' /sbin/init
+	then
+		echo init
+		return 0
+	fi
 }
 
-validate_openrc() {
+check_launchd() {
+	command -v launchctl >/dev/null 2>&1 || return 1
+	launchctl getenv PATH >/dev/null || return 1
+	echo launchd
+}
+
+check_openrc() {
 	test -f /run/openrc/softlevel || return 1
 	echo openrc
 }
 
-validate_procd() {
-	grep -q 'procd' /sbin/procd || return 1
+check_procd() (
+	procd_path=${1:-/sbin/procd}
+	test -x "${procd_path}" || return 1
+	grep -q 'procd' "${procd_path}" || return 1
 	echo procd
-}
+)
 
-validate_runit() {
+check_runit() {
 	test -d /run/runit || return 1
 	echo runit
 }
 
-validate_smf() {
+check_smf() {
 	# XXX: Is this the correct way??
 	test -f /etc/svc/volatile/svc_nonpersist.db || return 1
 	echo smf
 }
 
-validate_systemd() {
+check_systemd() {
 	# NOTE: sd_booted(3)
 	test -d /run/systemd/system/ || return 1
 	# systemctl --version | sed -e '/^systemd/!d;s/^systemd //'
 	echo systemd
 }
 
-validate_sysvinit() {
-	test -x /sbin/init \
-	 && grep -q 'INIT_VERSION=sysvinit-[0-9.]*' /sbin/init \
-	 || return 1
+check_systemstarter() {
+	test -d /System/Library/StartupItems/ || return 1
+	test -f /System/Library/StartupItems/LoginWindow/StartupParameters.plist || return 1
+	echo init+SystemStarter
+}
+
+check_sysvinit() (
+	init_path=${1:-/sbin/init}
+	grep -q 'INIT_VERSION=sysvinit-[0-9.]*' "${init_path}" || return 1
 
 	# It is quite common to use SysVinit to stack other init systemd
 	# (like OpenRC) on top of it. So we check for that, too.
-	if stacked=$(validate_openrc)
+	if stacked=$(check_openrc)
 	then
 		echo "sysvinit+${stacked}"
 	else
 		echo sysvinit
 	fi
 	unset stacked
-}
+)
 
-validate_upstart() {
+check_upstart() {
 	test -x "$(command -v initctl)" || return 1
 	case $(initctl version)
 	in
 		*'(upstart '*')')
-			# if type -d /etc/init
-			# then
-			# 	# modern (DBus-based?) upstart >= 0.5
-			# 	:
-			# elif type -d /etc/events.d
-			# then
-			# 	# ancient upstart
-			# 	:
-			# fi
-			echo upstart
+			if test -d /etc/init
+			then
+				# modern (DBus-based?) upstart >= 0.5
+				echo upstart
+			elif test -d /etc/event.d
+			then
+				# ancient upstart
+				echo upstart-legacy
+			else
+				# whatever...
+				echo upstart
+			fi
 			;;
 		*)
 			return 1
@@ -163,7 +262,7 @@ find_init_procfs() (
 	test -h /proc/1/exe || return 1
 
 	# Find init executable
-	init_exe=$(ls -l /proc/1/exe 2>/dev/null)
+	init_exe=$(ls -l /proc/1/exe 2>/dev/null) || return 1
 	init_exe=${init_exe#* -> }
 
 	if ! test -x "$init_exe"
@@ -171,21 +270,100 @@ find_init_procfs() (
 		# On some rare occasions it can happen that the
 		# running init's binary has been replaced. In this
 		# case Linux adjusts the symlink to "X (deleted)"
-		case $init_exe
-		in
-			*' (deleted)')
-				init_exe=${init_exe% (deleted)}
-				test -x "$init_exe" || exit 1
-				;;
-			*)
-				exit 1
-				;;
-		esac
+
+		# [root@fedora-12 ~]# readlink /proc/1/exe
+		# /sbin/init (deleted)
+		# [root@fedora-12 ~]# ls -l /proc/1/exe
+		# lrwxrwxrwx. 1 root root 0 2020-01-30 23:00 /proc/1/exe -> /sbin/init (deleted)
+
+		init_exe=${init_exe% (deleted)}
+		test -x "$init_exe" || return 1
 	fi
 
 	echo "${init_exe}"
 )
 
+guess_by_path() {
+	case $1
+	in
+		/bin/busybox)
+			check_busybox_init "$1" && return
+			;;
+		/lib/systemd/systemd)
+			check_systemd "$1" && return
+			;;
+		/hurd/init)
+			check_hurd_init "$1" && return
+			;;
+		/sbin/launchd)
+			check_launchd "$1" && return
+			;;
+		/usr/bin/runit|/sbin/runit)
+			check_runit "$1" && return
+			;;
+		/sbin/openrc-init)
+			if check_openrc "$1" >/dev/null
+			then
+				echo openrc-init
+				return
+			fi
+			;;
+		/sbin/procd)
+			check_procd && return
+			;;
+		/sbin/init|*/init)
+			# init: it could be anything -> (explicit) no match
+			return 1
+			;;
+	esac
+
+	# No match
+	return 1
+}
+
+guess_by_comm_name() {
+	case $1
+	in
+		busybox)
+			check_busybox_init && return
+			;;
+		openrc-init)
+			if check_openrc >/dev/null
+			then
+				echo openrc-init
+				return 0
+			fi
+			;;
+		init)
+			# init could be anything -> no match
+			return 1
+			;;
+		*)
+			# Run check function by comm name if available.
+			# Fall back to comm name if either it does not exist or
+			# returns non-zero.
+			if type "check_$1" >/dev/null
+			then
+				"check_$1" && return
+			else
+				echo "$1" ; return 0
+			fi
+	esac
+
+	return 1
+}
+
+check_list() (
+	# List must be a multi-line input on stdin (one name per line)
+	while read init
+	do
+		"check_${init}" || continue
+		return 0
+	done
+	return 1
+)
+
+
 # BusyBox's versions of ps and pgrep do not support some options
 # depending on which compile-time options have been used.
 
@@ -194,25 +372,31 @@ find_init_pgrep() {
 }
 
 find_init_ps() {
-	case $(uname -s)
+	case $KERNEL_NAME
 	in
-		Darwin|NetBSD)
-			ps -o ucomm= -p 1 2>/dev/null
+		Darwin)
+			ps -o command -p 1 2>/dev/null | tail -n +2
 			;;
 		FreeBSD)
-			ps -o command= -p 1 2>/dev/null | cut -d ' ' -f 1
+			ps -o args= -p 1 2>/dev/null | cut -d ' ' -f 1
 			;;
-		OpenBSD)
-			ps -o command -p 1 2>/dev/null | tail -n +2 | cut -d ' ' -f 1
-			;;
-		*)
+		Linux)
 			ps -o comm= -p 1 2>/dev/null
 			;;
-	esac
+		NetBSD)
+			ps -o comm= -p 1 2>/dev/null
+			;;
+		OpenBSD)
+			ps -o args -p 1 2>/dev/null | tail -n +2 | cut -d ' ' -f 1
+			;;
+		*)
+			ps -o args= -p 1 2>/dev/null
+			;;
+	esac | trim  # trim trailing whitespace (some ps like Darwin add it)
 }
 
 find_init() {
-	case $(uname -s)
+	case $KERNEL_NAME
 	in
 		Linux|GNU|NetBSD)
 			find_init_procfs || find_init_pgrep || find_init_ps
@@ -233,65 +417,23 @@ find_init() {
 	esac
 }
 
-validate_by_comm_name() {
-	case $1
-	in
-		busybox)
-			validate_busybox_init
-			;;
-		init)
-			# FIXME: Do some more magic here!
-			echo init
-			;;
-		openrc-init)
-			validate_openrc >/dev/null && echo openrc-init
-			;;
-		runit)
-			validate_runit
-			;;
-		systemd)
-			validate_systemd
-			;;
-		*)
-			# Run validate function by comm name if available.
-			# Fall back to comm name if either it does not exist or
-			# returns non-zero.
-			type "validate_$1" >/dev/null && "validate_$1" || echo $1
-	esac
-}
-
-try_all() {
-	# init: it could be anything...
-	# We try some approaches to gather more information about init without
-	# calling it! On some init systemd this triggers a reinitialisation of
-	# the system which we don't want (e.g. embedded systems).
-
-	validate_sysvinit || \
-	validate_openrc || \
-	validate_runit || \
-	validate_smf || \
-	validate_upstart || \
-	validate_hurd_init || \
-	echo init  # fallback
-}
+# -----
 
 init=$(find_init)
 
-if test -x "${init}"
-then
-	case $init
-	in
-		/hurd/init)
-			# FIXME: Create validate function
-			echo hurd-init
-			;;
-		*/init)
-			try_all
-			;;
-		*)
-			validate_by_comm_name "$(basename "${init}")"
-			;;
-	esac
-else
-	validate_by_comm_name "${init}"
-fi
+# If we got a path, guess by the path first (fall back to file name if no match)
+# else guess by file name directly.
+{
+	test -x "${init}" \
+		&& guess_by_path "${init}" \
+		|| guess_by_comm_name "$(basename "${init}")"
+} && exit 0 || true
+
+
+# Guessing based on the file path and name didn’t lead to a definitive result.
+#
+# We go through all of the checks until we find a match. To speed up the
+# process, common cases will be checked first based on the underlying kernel.
+
+{ common_candidates_by_kernel; echo "${KNOWN_INIT_SYSTEMS}"; } \
+	| unique | check_list

From 0d84c91b4047d3da0571d0262b4b5d9a8f9796b9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 20 Feb 2020 22:55:46 +0100
Subject: [PATCH 188/743] [explorer/init] Fix unique() for Solaris

---
 cdist/conf/explorer/init | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init
index db417a14..0f04a0ee 100755
--- a/cdist/conf/explorer/init
+++ b/cdist/conf/explorer/init
@@ -135,7 +135,8 @@ trim() {
 
 unique() {
 	# Delete duplicate lines (keeping input order)
-	awk '!x[$0]++'
+	# NOTE: Solaris AWK breaks without if/print construct.
+	awk '{ if (!x[$0]++) print }'
 }
 
 

From 0d6bc8e8f8166a3f61dd4da4a0e499499d3702c4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 20 Feb 2020 23:29:21 +0100
Subject: [PATCH 189/743] [explorer/init] Make shellcheck happy

---
 cdist/conf/explorer/init | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init
index 0f04a0ee..1b921c68 100755
--- a/cdist/conf/explorer/init
+++ b/cdist/conf/explorer/init
@@ -310,7 +310,7 @@ guess_by_path() {
 			fi
 			;;
 		/sbin/procd)
-			check_procd && return
+			check_procd "$1" && return
 			;;
 		/sbin/init|*/init)
 			# init: it could be anything -> (explicit) no match
@@ -356,7 +356,7 @@ guess_by_comm_name() {
 
 check_list() (
 	# List must be a multi-line input on stdin (one name per line)
-	while read init
+	while read -r init
 	do
 		"check_${init}" || continue
 		return 0
@@ -408,7 +408,7 @@ find_init() {
 		OpenBSD)
 			find_init_pgrep || find_init_ps
 			;;
-		Darwin|FreeBSD|SunOS)
+		Darwin|SunOS)
 			find_init_ps
 			;;
 		*)
@@ -424,6 +424,7 @@ init=$(find_init)
 
 # If we got a path, guess by the path first (fall back to file name if no match)
 # else guess by file name directly.
+# shellcheck disable=SC2015
 {
 	test -x "${init}" \
 		&& guess_by_path "${init}" \

From 49fc21ec47d665155edac73bdbae4fed9258f382 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sun, 23 Feb 2020 09:32:03 +0100
Subject: [PATCH 190/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index b0ebf789..8463bb89 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,9 @@ Changelog
 next:
 	* Type __update_alternatives: Add state explorer (Ander Punnar)
 	* Explorer os_version: Add support for Alpine Linux (Jin-Guk Kwon)
+	* Explorer init: Rewrite and support more init systems (Dennis Camera)
+	* New type: __service (Timothée Floure)
+	* Types __consul_*: Add optional parameter for using distribution packages (Timothée Floure)
 
 6.5.1: 2020-02-15
 	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)

From c6aba8d189c2efc7c88f9c595acba7ceae5a4e00 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 23 Feb 2020 22:59:41 +0100
Subject: [PATCH 191/743] [explorer/disks] Fix for NetBSD

When connecting over SSH and running /bin/sh, the PATH is missing sbin locations.
sysctl is located at /sbin/sysctl on NetBSD.
---
 cdist/conf/explorer/disks | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks
index 87a6b5c6..08290bc7 100755
--- a/cdist/conf/explorer/disks
+++ b/cdist/conf/explorer/disks
@@ -1,14 +1,20 @@
-#!/bin/sh
+#!/bin/sh -e
 
 uname_s="$(uname -s)"
 
-case "${uname_s}" in
+case $uname_s in
     FreeBSD)
         sysctl -n kern.disks
     ;;
-    OpenBSD|NetBSD)
+    OpenBSD)
         sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+' | xargs
     ;;
+    NetBSD)
+        PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin"
+        sysctl -n hw.disknames \
+        | awk 'BEGIN { RS = " " } /^[lsw]d[0-9]+/' \
+        | xargs
+    ;;
     Linux)
         if command -v lsblk > /dev/null
         then
@@ -23,5 +29,3 @@ case "${uname_s}" in
         printf "Don't know how to list disks for %s operating system, if you can please submit a patch\n" "${uname_s}" >&2
     ;;
 esac
-
-exit 0

From e6f683b88633fa0722b3787e3daa39db640ce0f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 24 Feb 2020 09:20:49 +0100
Subject: [PATCH 192/743] Add support for alpine (edge) package to
 __consul_agent

---
 cdist/conf/type/__consul_agent/manifest | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest
index 40667002..0d819d45 100755
--- a/cdist/conf/type/__consul_agent/manifest
+++ b/cdist/conf/type/__consul_agent/manifest
@@ -61,6 +61,17 @@ distribution_setup () {
        user='consul'
        group='consul'
      ;;
+     alpine)
+       # consul is only available starting Alpine 3.12 (= edge during the 3.11 cycle).
+       # See https://pkgs.alpinelinux.org/packages?name=consul&branch=edge
+
+       # Override previously defined environment to match alpine packaging.
+       conf_dir='/etc/consul'
+       conf_file='server.json'
+       data_dir='/var/consul'
+       user='consul'
+       group='consul'
+     ;;
      *)
         echo "Your operating system ($os) is currently not supported with the \
           --use-distribution-package flag (${__type##*/})." >&2

From d3bd2669ec49fb861016e614893dac280ed5fd35 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 23 Feb 2020 23:07:40 +0100
Subject: [PATCH 193/743] [explorer/disks] Support Linux without lsblk
 (fallback to sysfs)

---
 cdist/conf/explorer/disks | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks
index 08290bc7..0fabc95f 100755
--- a/cdist/conf/explorer/disks
+++ b/cdist/conf/explorer/disks
@@ -16,16 +16,33 @@ case $uname_s in
         | xargs
     ;;
     Linux)
-        if command -v lsblk > /dev/null
+        # list of major device numbers toexclude:
+        #  ram disks, floppies, cdroms
+        # https://www.kernel.org/doc/Documentation/admin-guide/devices.txt
+        ign_majors='1 2 11'
+
+        if command -v lsblk >/dev/null 2>&1
         then
-            # exclude ram disks, floppies and cdroms
-            # https://www.kernel.org/doc/Documentation/admin-guide/devices.txt
-            lsblk -e 1,2,11 -dno name | xargs
+            lsblk -e "$(echo "$ign_majors" | tr ' ' ',')" -dno name | xargs
+        elif test -d /sys/block/
+        then
+            # shellcheck disable=SC2012
+            ls -1 /sys/block/ \
+            | awk -v ign_majors="$(echo "$ign_majors" | tr ' ' '|')" '
+                {
+                  devfile = "/sys/block/" $0 "/dev"
+                  getline devno < devfile
+                  close(devfile)
+                  if (devno !~ "^(" ign_majors "):") print
+                }' \
+            | xargs
         else
-            printf "Don't know how to list disks for %s operating system without lsblk, if you can please submit a patch\n" "${uname_s}" >&2
+            echo "Don't know how to list disks on Linux without lsblk and sysfs." >&2
+            echo 'If you can, please submit a patch.'>&2
         fi
     ;;
     *)
-        printf "Don't know how to list disks for %s operating system, if you can please submit a patch\n" "${uname_s}" >&2
+        printf "Don't know how to list disks for %s operating system.\n" "${uname_s}" >&2
+        printf 'If you can please submit a patch\n' >&2
     ;;
 esac

From 1ef126e16f95e822562978abd895a3c036f7d5c4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 23 Feb 2020 23:08:40 +0100
Subject: [PATCH 194/743] [explorer/disks] Move xargs call to the bottom

---
 cdist/conf/explorer/disks | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks
index 0fabc95f..ed1afce4 100755
--- a/cdist/conf/explorer/disks
+++ b/cdist/conf/explorer/disks
@@ -7,13 +7,12 @@ case $uname_s in
         sysctl -n kern.disks
     ;;
     OpenBSD)
-        sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+' | xargs
+        sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+'
     ;;
     NetBSD)
         PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin"
         sysctl -n hw.disknames \
-        | awk 'BEGIN { RS = " " } /^[lsw]d[0-9]+/' \
-        | xargs
+        | awk 'BEGIN { RS = " " } /^[lsw]d[0-9]+/'
     ;;
     Linux)
         # list of major device numbers toexclude:
@@ -23,7 +22,7 @@ case $uname_s in
 
         if command -v lsblk >/dev/null 2>&1
         then
-            lsblk -e "$(echo "$ign_majors" | tr ' ' ',')" -dno name | xargs
+            lsblk -e "$(echo "$ign_majors" | tr ' ' ',')" -dno name
         elif test -d /sys/block/
         then
             # shellcheck disable=SC2012
@@ -34,8 +33,7 @@ case $uname_s in
                   getline devno < devfile
                   close(devfile)
                   if (devno !~ "^(" ign_majors "):") print
-                }' \
-            | xargs
+                }'
         else
             echo "Don't know how to list disks on Linux without lsblk and sysfs." >&2
             echo 'If you can, please submit a patch.'>&2
@@ -45,4 +43,5 @@ case $uname_s in
         printf "Don't know how to list disks for %s operating system.\n" "${uname_s}" >&2
         printf 'If you can please submit a patch\n' >&2
     ;;
-esac
+esac \
+| xargs

From 6db6dc4ac0950579ce13252dcca6d0f61f5533c6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 23 Feb 2020 23:14:14 +0100
Subject: [PATCH 195/743] [explorer/disks] Add license header

---
 cdist/conf/explorer/disks | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks
index ed1afce4..24540601 100755
--- a/cdist/conf/explorer/disks
+++ b/cdist/conf/explorer/disks
@@ -1,4 +1,24 @@
 #!/bin/sh -e
+#
+# based on previous work by other people, modified by:
+# 2020 Dennis Camera <dennis.camera at ssrq-sds-fds.ch>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Finds disks of the system (excl. ram disks, floppy, cdrom)
 
 uname_s="$(uname -s)"
 

From b2db864eaf95a50e7e0d31be52f86ab1d0bea480 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 26 Feb 2020 13:01:29 +0100
Subject: [PATCH 196/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8463bb89..35590655 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Explorer init: Rewrite and support more init systems (Dennis Camera)
 	* New type: __service (Timothée Floure)
 	* Types __consul_*: Add optional parameter for using distribution packages (Timothée Floure)
+	* Explorer disks: Fix NetBSD, support Linux w/o lsblk (Dennis Camera)
 
 6.5.1: 2020-02-15
 	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)

From 6b4b6534a1187e2533f627ff485873aeac7c4a32 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 26 Feb 2020 16:12:21 +0100
Subject: [PATCH 197/743] [__directory] Give more precise error message when
 --state pre-exists

---
 cdist/conf/type/__directory/gencode-remote | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote
index e1ab69d7..a1a32ea2 100755
--- a/cdist/conf/type/__directory/gencode-remote
+++ b/cdist/conf/type/__directory/gencode-remote
@@ -109,10 +109,24 @@ case "$state_should" in
       done
    ;;
    pre-exists)
-      if [ "$type" != "directory" ]; then
-         echo "Directory \"$destination\" does not exist" >&2
-         exit 1
-      fi
+      case $type in
+         directory)
+            # all good
+            exit 0
+         ;;
+         none)
+            printf 'Directory "%s" does not exist\n' "$destination" >&2
+            exit 1
+         ;;
+         file|symlink)
+            printf 'File "%s" exists and is a %s, but should be a directory\n' "$destination" "$type" >&2
+            exit 1
+         ;;
+         *)
+            printf 'File or directory "%s" is in an unknown state\n' "$destination" >&2
+            exit 1
+         ;;
+      esac
    ;;
    absent)
         if [ "$type" = "directory" ]; then

From 046f7d0663fb6e117f76747bf71c7976abc90962 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 26 Feb 2020 16:32:03 +0100
Subject: [PATCH 198/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 35590655..4fdfcd68 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* New type: __service (Timothée Floure)
 	* Types __consul_*: Add optional parameter for using distribution packages (Timothée Floure)
 	* Explorer disks: Fix NetBSD, support Linux w/o lsblk (Dennis Camera)
+	* Type __directory: Add 'exists' and 'pre-exists' states (Dennis Camera)
 
 6.5.1: 2020-02-15
 	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)

From 46d2487f08e04d3f6f3cbd434404dda006e83276 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 26 Feb 2020 16:40:53 +0100
Subject: [PATCH 199/743] [__file] Give more precise error message when --state
 pre-exists

---
 cdist/conf/type/__file/gencode-local | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local
index fb9f9a92..231b6927 100755
--- a/cdist/conf/type/__file/gencode-local
+++ b/cdist/conf/type/__file/gencode-local
@@ -31,12 +31,24 @@ if [ "$state_should" = "pre-exists" ]; then
       exit 1
    fi
 
-   if [ "$type" = "file" ]; then
-      exit 0 # nothing to do
-   else
-      echo "File \"$destination\" does not exist"
-      exit 1
-   fi
+   case $type in
+      file)
+         # nothing to do
+         exit 0
+      ;;
+      none)
+         printf 'File "%s" does not exist\n' "$destination" >&2
+         exit 1
+      ;;
+      directory|symlink)
+         printf 'File "%s" exists and is a %s, but should be a regular file\n' "$destination" "$type" >&2
+         exit 1
+      ;;
+      *)
+         printf 'File or directory "%s" is in an unknown state\n' "$destination" >&2
+         exit 1
+      ;;
+   esac
 fi
 
 upload_file=

From 9eacba06bb218bd0627ef633b363495331ac6c5b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 26 Feb 2020 16:48:14 +0100
Subject: [PATCH 200/743] [__file] Treat pre-exists as a special case in
 gencode-remote as it should never get there

---
 cdist/conf/type/__file/gencode-remote | 56 ++++++++++++++-------------
 1 file changed, 30 insertions(+), 26 deletions(-)

diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index b04c471e..815593bd 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -55,37 +55,41 @@ set_owner() {
 }
 
 set_mode() {
-   echo "chmod '$1' '$destination'"
-   echo "chmod '$1'" >> "$__messages_out"
-   fire_onchange=1
+    echo "chmod '$1' '$destination'"
+    echo "chmod '$1'" >> "$__messages_out"
+    fire_onchange=1
 }
 
 case "$state_should" in
-    present|exists|pre-exists)
-    # Note: Mode - needs to happen last as a chown/chgrp can alter mode by
-    #  clearing S_ISUID and S_ISGID bits (see chown(2))
-    for attribute in group owner mode; do
-        if [ -f "$__object/parameter/$attribute" ]; then
-            value_should="$(cat "$__object/parameter/$attribute")"
+    present|exists)
+        # Note: Mode - needs to happen last as a chown/chgrp can alter mode by
+        #  clearing S_ISUID and S_ISGID bits (see chown(2))
+        for attribute in group owner mode; do
+            if [ -f "$__object/parameter/$attribute" ]; then
+                value_should="$(cat "$__object/parameter/$attribute")"
 
-            # change 0xxx format to xxx format => same as stat returns
-            if [ "$attribute" = mode ]; then
-                value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')"
-            fi
-            
-            value_is="$(get_current_value "$attribute" "$value_should")"
-            if [ -f "$__object/files/set-attributes" ] || [ "$value_should" != "$value_is" ]; then
-                "set_$attribute" "$value_should"
+                # change 0xxx format to xxx format => same as stat returns
+                if [ "$attribute" = mode ]; then
+                    value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')"
+                fi
+
+                value_is="$(get_current_value "$attribute" "$value_should")"
+                if [ -f "$__object/files/set-attributes" ] || [ "$value_should" != "$value_is" ]; then
+                    "set_$attribute" "$value_should"
+                fi
             fi
+        done
+        if [ -f "$__object/files/set-attributes" ]; then
+            # set-attributes is created if file is created or uploaded in gencode-local
+            fire_onchange=1
         fi
-    done
-    if [ -f "$__object/files/set-attributes" ]; then
-        # set-attributes is created if file is created or uploaded in gencode-local
-        fire_onchange=1
-    fi
-
     ;;
 
+    pre-exists)
+        # pre-exists should never reach gencode-remote…
+        exit 1
+   ;;
+
     absent)
         if [ "$type" = "file" ]; then
             echo "rm -f '$destination'"
@@ -101,7 +105,7 @@ case "$state_should" in
 esac
 
 if [ -f "$__object/parameter/onchange" ]; then
-   if [ -n "$fire_onchange" ]; then
-      cat "$__object/parameter/onchange"
-   fi
+    if [ -n "$fire_onchange" ]; then
+        cat "$__object/parameter/onchange"
+    fi
 fi

From da6ccf808ef4417b90e0882abd042f0a40f28a8e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 26 Feb 2020 21:48:08 +0100
Subject: [PATCH 201/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 4fdfcd68..9075cf3f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,7 @@ next:
 	* Types __consul_*: Add optional parameter for using distribution packages (Timothée Floure)
 	* Explorer disks: Fix NetBSD, support Linux w/o lsblk (Dennis Camera)
 	* Type __directory: Add 'exists' and 'pre-exists' states (Dennis Camera)
+	* Type __file: Improve error messages for pre-exists state (Dennis Camera)
 
 6.5.1: 2020-02-15
 	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)

From 213f1b049c55205ffe233de22e68c4a4738c8b82 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 27 Feb 2020 20:23:04 +0100
Subject: [PATCH 202/743] Release 6.5.2

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 9075cf3f..64b124e8 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.5.2: 2020-02-27
 	* Type __update_alternatives: Add state explorer (Ander Punnar)
 	* Explorer os_version: Add support for Alpine Linux (Jin-Guk Kwon)
 	* Explorer init: Rewrite and support more init systems (Dennis Camera)

From fb32d6ed3f9421d515b0ec2eabe9ab4fcc6ccacb Mon Sep 17 00:00:00 2001
From: llnu <llnu@protonmail.ch>
Date: Sun, 8 Mar 2020 16:04:02 +0100
Subject: [PATCH 203/743] alpine uses a different getent lib which doesnt
 support: getent shadow

---
 cdist/conf/type/__user/explorer/shadow | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__user/explorer/shadow b/cdist/conf/type/__user/explorer/shadow
index 73ce0e29..63d38f0d 100755
--- a/cdist/conf/type/__user/explorer/shadow
+++ b/cdist/conf/type/__user/explorer/shadow
@@ -24,7 +24,7 @@
 name=$__object_id
 
 case $("$__explorer/os") in
-  'freebsd'|'netbsd'|'openbsd')
+  'freebsd'|'netbsd'|'openbsd'|'alpine')
     database='passwd'
     ;;
   # Default to using shadow passwords

From f7d5f5bc974fff858c9999752badce3c6e8ba72e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 9 Mar 2020 08:02:18 +0100
Subject: [PATCH 204/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 64b124e8..93df32a2 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __user: Fix missing shadow for alpine (llnu)
+
 6.5.2: 2020-02-27
 	* Type __update_alternatives: Add state explorer (Ander Punnar)
 	* Explorer os_version: Add support for Alpine Linux (Jin-Guk Kwon)

From 358e04b2afa380b63843869f1f57967e0ef8de22 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 14 Mar 2020 09:58:38 +0100
Subject: [PATCH 205/743] Handle specially if no time about the last index
 update found.

The explorer 'currage' now returns -1 if he can not find any value about
this. The gencode-remote script handle this value special to not exit if
-1 given as value.

This fixes the bug
  https://code.ungleich.ch/ungleich-public/cdist/issues/803
---
 cdist/conf/type/__package_update_index/explorer/currage | 6 +++---
 cdist/conf/type/__package_update_index/gencode-remote   | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__package_update_index/explorer/currage b/cdist/conf/type/__package_update_index/explorer/currage
index cfb778d5..bd51ee86 100644
--- a/cdist/conf/type/__package_update_index/explorer/currage
+++ b/cdist/conf/type/__package_update_index/explorer/currage
@@ -24,18 +24,18 @@ case "$type" in
         if [ -f "/var/cache/apt/pkgcache.bin" ]; then
             echo $(($(date +"%s")-$(stat --format '%Y' /var/cache/apt/pkgcache.bin)))
         else
-            echo 0
+            echo -1
         fi
         ;;
     pacman)
         if [ -d "/var/lib/pacman/sync" ]; then
             echo $(($(date +"%s")-$(stat --format '%Y' /var/lib/pacman/sync)))
         else
-            echo 0
+            echo -1
         fi
         ;;
     alpine)
-        echo 0
+        echo -1
         ;;
     *)  echo "Your specified type ($type) is currently not supported." >&2
         echo "Please contribute an implementation for it if you can." >&2
diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote
index 6c51cbed..803468b5 100755
--- a/cdist/conf/type/__package_update_index/gencode-remote
+++ b/cdist/conf/type/__package_update_index/gencode-remote
@@ -31,7 +31,8 @@ if [ -n "$maxage" ]; then
     if [ "$type" != "apt" ] && [ "$type" != "pacman" ]; then
         echo "ERROR: \"--maxage\" only supported for \"apt\" or \"pacman\" pkg-manager." >&2
         exit 1
-    elif [ "$currage" -lt "$maxage" ]; then
+    # do not exit if no value found (represented as -1)
+    elif [ "$currage" -ne -1 ] && [ "$currage" -lt "$maxage" ]; then
         exit 0 # no need to update
     fi
 fi

From f00e4af5f0f1f49ab93be001468327ea90df6bd0 Mon Sep 17 00:00:00 2001
From: Andrew Schleifer <me@andrewschleifer.name>
Date: Thu, 26 Mar 2020 21:17:32 +0800
Subject: [PATCH 206/743] fix typo

---
 cdist/conf/type/__letsencrypt_cert/man.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__letsencrypt_cert/man.rst b/cdist/conf/type/__letsencrypt_cert/man.rst
index c4ffc6bc..85eb88ea 100644
--- a/cdist/conf/type/__letsencrypt_cert/man.rst
+++ b/cdist/conf/type/__letsencrypt_cert/man.rst
@@ -59,13 +59,13 @@ MESSAGES
 --------
 
 change
-    Certificte was changed.
+    Certificate was changed.
 
 create
-    Certificte was created.
+    Certificate was created.
 
 remove
-    Certificte was removed.
+    Certificate was removed.
 
 EXAMPLES
 --------

From 66d990502987aae97695a4b8678e7d160f1066f7 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 26 Mar 2020 21:48:17 +0100
Subject: [PATCH 207/743] [__consul_agent] make conf_dir depent on the OS

---
 cdist/conf/type/__consul_agent/manifest | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest
index 0d819d45..a7fe0bc5 100755
--- a/cdist/conf/type/__consul_agent/manifest
+++ b/cdist/conf/type/__consul_agent/manifest
@@ -1,7 +1,7 @@
 #!/bin/sh -e
 #
 # 2015 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2015-2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2015-2020 Nico Schottelius (nico-cdist at schottelius.org)
 # 2019 Timothée Floure (timothee.floure at ungleich.ch)
 #
 # This file is part of cdist.
@@ -37,10 +37,22 @@ fi
 # Those are default that might be overriden by os-specific logic.
 
 data_dir="/var/lib/consul"
-conf_dir="/etc/consul/conf.d"
-conf_file="config.json"
+
+
+
 tls_dir="$conf_dir/tls"
 
+case "$os" in
+    alpine)
+        conf_dir="/etc/consul"
+        conf_file="server.json"
+        ;;
+    *)
+        conf_dir="/etc/consul/conf.d"
+        conf_file="config.json"
+        ;;
+esac
+
 ###
 # Sane deployment, based on distribution package when available.
 

From ec11f04ab83504035271483a61549a4e0c51053c Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 26 Mar 2020 21:49:43 +0100
Subject: [PATCH 208/743] ++changes

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 93df32a2..76f47679 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,8 @@ Changelog
 
 next:
 	* Type __user: Fix missing shadow for alpine (llnu)
+	* Type __consule_agent: Make conf_dir dependent on OS - fixes
+	Alpine (Nico Schottelius)
 
 6.5.2: 2020-02-27
 	* Type __update_alternatives: Add state explorer (Ander Punnar)

From df63cfe0884d460942a34b31d9296a31aa8edb2f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 27 Mar 2020 08:50:27 +0100
Subject: [PATCH 209/743] ++changelog

---
 docs/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/changelog b/docs/changelog
index 76f47679..8e51b949 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,8 +3,8 @@ Changelog
 
 next:
 	* Type __user: Fix missing shadow for alpine (llnu)
-	* Type __consule_agent: Make conf_dir dependent on OS - fixes
-	Alpine (Nico Schottelius)
+	* Type __consule_agent: Make conf_dir dependent on OS - fixes Alpine (Nico Schottelius)
+	* Type __letsencrypt_cert: Fix typo (Andrew Schleifer)
 
 6.5.2: 2020-02-27
 	* Type __update_alternatives: Add state explorer (Ander Punnar)

From b25939cdd64fdab7d71ab9285c64ad5bf14797f5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 27 Mar 2020 13:30:52 +0100
Subject: [PATCH 210/743] Fix shellcheck

---
 cdist/conf/type/__consul_agent/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest
index a7fe0bc5..7b54529c 100755
--- a/cdist/conf/type/__consul_agent/manifest
+++ b/cdist/conf/type/__consul_agent/manifest
@@ -232,7 +232,7 @@ if [ -f "$__object/parameter/ca-file-source" ] || \
    [ -f "$__object/parameter/cert-file-source" ] || \
    [ -f "$__object/parameter/key-file-source" ]; then
 
-   requires="$config_deployment_requires" __directory $tls_dir \
+   requires="$config_deployment_requires" __directory "$tls_dir" \
      --owner root --group "$group" --mode 750 --state "$state"
 
    # Append to service restart requirements.

From da30afe791e1eca11d68fedc88d7a943ecf089e5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sun, 29 Mar 2020 12:54:02 +0200
Subject: [PATCH 211/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8e51b949..62ef28be 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __user: Fix missing shadow for alpine (llnu)
 	* Type __consule_agent: Make conf_dir dependent on OS - fixes Alpine (Nico Schottelius)
 	* Type __letsencrypt_cert: Fix typo (Andrew Schleifer)
+	* Type __package_update_index: Fix maxage false positives (Matthias Stecher)
 
 6.5.2: 2020-02-27
 	* Type __update_alternatives: Add state explorer (Ander Punnar)

From 0f639a9278e32b395bbc0f20b53b89ce1bb324a8 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 2 Apr 2020 18:14:09 +0200
Subject: [PATCH 212/743] Make __cron nonparallel

It fails due to removal / edit of crontab at the same time

VERBOSE: [18331]: uncloud.ungleich.ch: Running object __cron/pg_dump_cleanup
VERBOSE: [18332]: uncloud.ungleich.ch: Running object __ungleich_packages/server
VERBOSE: [18335]: uncloud.ungleich.ch: Running object __postgres_role/app
INFO: [18335]: uncloud.ungleich.ch: Processing __postgres_role/app
INFO: [18331]: uncloud.ungleich.ch: Processing __cron/pg_dump_cleanup
ERROR: [16451]: uncloud.ungleich.ch: ssh -o User=root -o ControlPath=/tmp/tmp_8eu4oep/s -o ControlMaster=auto -o ControlPersist=2h uncloud.ungleich.ch /bin/sh -c ' export __object=/var/lib/cdist/object/__cron/pg_dumpall_under_day/.cdist-mfd00npk;  export __object_id=pg_dumpall_under_day;/bin/sh -e /var/lib/cdist/object/__cron/pg_dumpall_under_day/.cdist-mfd00npk/code-remote': ['ssh', '-o', 'User=root', '-o', 'ControlPath=/tmp/tmp_8eu4oep/s', '-o', 'ControlMaster=auto', '-o', 'ControlPersist=2h', 'uncloud.ungleich.ch', "/bin/sh -c ' export __object=/var/lib/cdist/object/__cron/pg_dumpall_under_day/.cdist-mfd00npk;  export __object_id=pg_dumpall_under_day;/bin/sh -e /var/lib/cdist/object/__cron/pg_dumpall_under_day/.cdist-mfd00npk/code-remote'"]

Error processing object '__cron/pg_dumpall_under_day'
=====================================================
name: __cron/pg_dumpall_under_day
path: /tmp/tmplaq9cwdh/6318c251013a449595327745daacf3ee/data/object/__cron/pg_dumpall_under_day/.cdist-mfd00npk
source: /tmp/tmplaq9cwdh/6318c251013a449595327745daacf3ee/data/conf/type/__ungleich_postgresql/manifest
type: /home/nico/vcs/cdist/cdist/conf/type/__cron

code-remote:stderr
------------------
crontab: can't move 'postgres.new' to 'postgres': No such file or directory

VERBOSE: [16451]: config: Total processing time for 1 host(s): 13.98031210899353
[18:09] line:~%
---
 cdist/conf/type/__cron/nonparallel | 0
 docs/changelog                     | 1 +
 2 files changed, 1 insertion(+)
 create mode 100644 cdist/conf/type/__cron/nonparallel

diff --git a/cdist/conf/type/__cron/nonparallel b/cdist/conf/type/__cron/nonparallel
new file mode 100644
index 00000000..e69de29b
diff --git a/docs/changelog b/docs/changelog
index 62ef28be..b608b35f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -2,6 +2,7 @@ Changelog
 ---------
 
 next:
+	* Type __cron: Make non parallel due to race condition (Nico Schottelius)
 	* Type __user: Fix missing shadow for alpine (llnu)
 	* Type __consule_agent: Make conf_dir dependent on OS - fixes Alpine (Nico Schottelius)
 	* Type __letsencrypt_cert: Fix typo (Andrew Schleifer)

From d034fe9369f1f7aa00a7b0136ffd940ecbceb588 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 2 Apr 2020 18:24:13 +0200
Subject: [PATCH 213/743] [__pyvenv] use python3 -m venv on alpine

Until python4 comes, this will work
---
 cdist/conf/type/__pyvenv/gencode-remote | 12 +++++++++++-
 docs/changelog                          |  1 +
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__pyvenv/gencode-remote b/cdist/conf/type/__pyvenv/gencode-remote
index 04700683..05ec3b29 100755
--- a/cdist/conf/type/__pyvenv/gencode-remote
+++ b/cdist/conf/type/__pyvenv/gencode-remote
@@ -37,11 +37,21 @@ mode="$(cat "$__object/parameter/mode")"
 destination="/$__object_id"
 venvparams="$(cat "$__object/parameter/venvparams")"
 pyvenvparam="$__object/parameter/pyvenv"
+
+os=$(cat $__global/explorer/os)
+
 if [ -f "$pyvenvparam" ]
 then
     pyvenv=$(cat "$pyvenvparam")
 else
-    pyvenv="pyvenv"
+    case "$os" in
+        alpine) # no pyvenv on alpine - I assume others will follow
+            pyvenv="python3 -m venv"
+            ;;
+        *)
+            pyvenv="pyvenv"
+            ;;
+    esac
 fi
 
 case $state_should in
diff --git a/docs/changelog b/docs/changelog
index b608b35f..f5846925 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __cron: Make non parallel due to race condition (Nico Schottelius)
+	* Type __pyvenv: Use python3 -m venv on Alpine (Nico Schottelius)
 	* Type __user: Fix missing shadow for alpine (llnu)
 	* Type __consule_agent: Make conf_dir dependent on OS - fixes Alpine (Nico Schottelius)
 	* Type __letsencrypt_cert: Fix typo (Andrew Schleifer)

From 890c73f6bdd027e60cedd868323f9d0438408b7a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 2 Apr 2020 20:18:04 +0200
Subject: [PATCH 214/743] Fix shellcheck issues

---
 cdist/conf/type/__package_update_index/explorer/currage | 6 +++---
 cdist/conf/type/__pyvenv/gencode-remote                 | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__package_update_index/explorer/currage b/cdist/conf/type/__package_update_index/explorer/currage
index bd51ee86..8eadaf53 100644
--- a/cdist/conf/type/__package_update_index/explorer/currage
+++ b/cdist/conf/type/__package_update_index/explorer/currage
@@ -24,18 +24,18 @@ case "$type" in
         if [ -f "/var/cache/apt/pkgcache.bin" ]; then
             echo $(($(date +"%s")-$(stat --format '%Y' /var/cache/apt/pkgcache.bin)))
         else
-            echo -1
+            echo -- -1
         fi
         ;;
     pacman)
         if [ -d "/var/lib/pacman/sync" ]; then
             echo $(($(date +"%s")-$(stat --format '%Y' /var/lib/pacman/sync)))
         else
-            echo -1
+            echo -- -1
         fi
         ;;
     alpine)
-        echo -1
+        echo -- -1
         ;;
     *)  echo "Your specified type ($type) is currently not supported." >&2
         echo "Please contribute an implementation for it if you can." >&2
diff --git a/cdist/conf/type/__pyvenv/gencode-remote b/cdist/conf/type/__pyvenv/gencode-remote
index 05ec3b29..9c7b7fab 100755
--- a/cdist/conf/type/__pyvenv/gencode-remote
+++ b/cdist/conf/type/__pyvenv/gencode-remote
@@ -38,7 +38,7 @@ destination="/$__object_id"
 venvparams="$(cat "$__object/parameter/venvparams")"
 pyvenvparam="$__object/parameter/pyvenv"
 
-os=$(cat $__global/explorer/os)
+os=$(cat "$__global/explorer/os")
 
 if [ -f "$pyvenvparam" ]
 then

From ceb6b597ef66dc21c901b5b52f251f4c8c61cd54 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 3 Apr 2020 19:45:44 +0200
Subject: [PATCH 215/743] Release 6.5.3

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index f5846925..aa5439d5 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.5.3: 2020-04-03
 	* Type __cron: Make non parallel due to race condition (Nico Schottelius)
 	* Type __pyvenv: Use python3 -m venv on Alpine (Nico Schottelius)
 	* Type __user: Fix missing shadow for alpine (llnu)

From d53077f4e8fbca26c6f1105efa1060df749900b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 6 Apr 2020 09:26:52 +0200
Subject: [PATCH 216/743] Add --encoding, --lc-collate, --lc-support to
 __postgres_database type

---
 .../type/__postgres_database/gencode-remote    | 18 +++++++++++++++++-
 cdist/conf/type/__postgres_database/man.rst    | 12 ++++++++++--
 .../__postgres_database/parameter/optional     |  3 +++
 3 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote
index 47e6b97c..b5f52f50 100755
--- a/cdist/conf/type/__postgres_database/gencode-remote
+++ b/cdist/conf/type/__postgres_database/gencode-remote
@@ -43,8 +43,24 @@ if [ "$state_should" != "$state_is" ]; then
          if [ -f "$__object/parameter/owner" ]; then
             owner="-O \"$(cat "$__object/parameter/owner")\""
          fi
+
+         encoding=""
+         if [ -f "$__object/parameter/encoding" ]; then
+            encoding="--encoding \"$(cat "$__object/parameter/encoding")\""
+         fi
+
+         lc_collate=""
+         if [ -f "$__object/parameter/lc-collate" ]; then
+            lc_collate="--lc-collate \"$(cat "$__object/parameter/lc-collate")\""
+         fi
+
+         lc_ctype=""
+         if [ -f "$__object/parameter/lc-ctype" ]; then
+            lc_ctype="--lc-ctype \"$(cat "$__object/parameter/lc-ctype")\""
+         fi
+
          cat << EOF
-su - '$postgres_user' -c "createdb $owner \"$name\""
+su - '$postgres_user' -c "createdb $owner \"$name\" $encoding $lc_collate $lc_ctype"
 EOF
       ;;
       absent)
diff --git a/cdist/conf/type/__postgres_database/man.rst b/cdist/conf/type/__postgres_database/man.rst
index acceec9b..97ac95eb 100644
--- a/cdist/conf/type/__postgres_database/man.rst
+++ b/cdist/conf/type/__postgres_database/man.rst
@@ -14,11 +14,19 @@ This cdist type allows you to create or drop postgres databases.
 OPTIONAL PARAMETERS
 -------------------
 state
-   either 'present' or 'absent', defaults to 'present'.
+   Either 'present' or 'absent', defaults to 'present'.
 
 owner
-   the role owning this database
+   Specifies the database user who will own the new database.
 
+encoding
+   Specifies the character encoding scheme to be used in this database.
+
+lc-collate
+   Specifies the LC_COLLATE setting to be used in this database.
+
+lc-ctype
+   Specifies the LC_CTYPE setting to be used in this database.
 
 EXAMPLES
 --------
diff --git a/cdist/conf/type/__postgres_database/parameter/optional b/cdist/conf/type/__postgres_database/parameter/optional
index d86b6469..fed2581e 100644
--- a/cdist/conf/type/__postgres_database/parameter/optional
+++ b/cdist/conf/type/__postgres_database/parameter/optional
@@ -1,2 +1,5 @@
 state
 owner
+encoding
+lc-collate
+lc-ctype

From 76d978d3d85051fe943334771c30dfff907f6e21 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Fri, 10 Apr 2020 10:51:17 +0200
Subject: [PATCH 217/743] explorer/init: do not grep on non-existent init

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/conf/explorer/init | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init
index 1b921c68..f27c77ef 100755
--- a/cdist/conf/explorer/init
+++ b/cdist/conf/explorer/init
@@ -221,6 +221,7 @@ check_systemstarter() {
 
 check_sysvinit() (
 	init_path=${1:-/sbin/init}
+	test -x "${init_path}" || return 1
 	grep -q 'INIT_VERSION=sysvinit-[0-9.]*' "${init_path}" || return 1
 
 	# It is quite common to use SysVinit to stack other init systemd

From e19c1bb1e0072a4bfd6ec64ada2d3c4fab1ac064 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Fri, 10 Apr 2020 21:50:39 +0200
Subject: [PATCH 218/743] remove duplicates from conf dirs while preserving
 order

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/exec/util.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cdist/exec/util.py b/cdist/exec/util.py
index 9787f431..e3be2235 100644
--- a/cdist/exec/util.py
+++ b/cdist/exec/util.py
@@ -199,7 +199,9 @@ def resolve_conf_dirs(configuration, add_conf_dirs):
 
     if add_conf_dirs:
         conf_dirs.extend(add_conf_dirs)
-    conf_dirs = set(conf_dirs)
+
+    # Remove duplicates.
+    conf_dirs = list(dict.fromkeys(conf_dirs))
     return conf_dirs
 
 

From 1ebcc219c26f91bcc634a9526171871f15210bd9 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 11 Apr 2020 09:54:57 +0200
Subject: [PATCH 219/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index aa5439d5..a927a08d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Explorer init: Do not grep on non-existent init (Steven Armstrong)
+
 6.5.3: 2020-04-03
 	* Type __cron: Make non parallel due to race condition (Nico Schottelius)
 	* Type __pyvenv: Use python3 -m venv on Alpine (Nico Schottelius)

From 704e78322ed5794fd9baf324f64363858fda2b26 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 11 Apr 2020 20:26:20 +0200
Subject: [PATCH 220/743] Use OrderedDict to guarantee order

Note:
> Changed in version 3.7: Dictionary order is guaranteed to be
> insertion order. This behavior was an implementation detail of
> CPython from 3.6.
---
 cdist/exec/util.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/exec/util.py b/cdist/exec/util.py
index e3be2235..90a26ad3 100644
--- a/cdist/exec/util.py
+++ b/cdist/exec/util.py
@@ -22,6 +22,7 @@
 import subprocess
 import os
 from tempfile import TemporaryFile
+from collections import OrderedDict
 
 import cdist
 import cdist.configuration
@@ -201,7 +202,7 @@ def resolve_conf_dirs(configuration, add_conf_dirs):
         conf_dirs.extend(add_conf_dirs)
 
     # Remove duplicates.
-    conf_dirs = list(dict.fromkeys(conf_dirs))
+    conf_dirs = list(OrderedDict.fromkeys(conf_dirs))
     return conf_dirs
 
 

From d1eecb93eeef11da057052fe14818ff442a0d0c8 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 11 Apr 2020 20:30:56 +0200
Subject: [PATCH 221/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index a927a08d..8c7d497b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Explorer init: Do not grep on non-existent init (Steven Armstrong)
+	* Core: Bugfix to preserve conf dirs order (Steven Armstrong)
 
 6.5.3: 2020-04-03
 	* Type __cron: Make non parallel due to race condition (Nico Schottelius)

From 0805fac7e91b9ea3d0dd92fb00e36bc33d8cf779 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 11 Apr 2020 20:35:29 +0200
Subject: [PATCH 222/743] Release 6.5.4

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 8c7d497b..b6b1bb39 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.5.4: 2020-04-11
 	* Explorer init: Do not grep on non-existent init (Steven Armstrong)
 	* Core: Bugfix to preserve conf dirs order (Steven Armstrong)
 

From c3f924d350772d7c3af1ce8bbbe29f1b8010d4f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 14 Apr 2020 10:23:08 +0200
Subject: [PATCH 223/743] Add --template flag to __postgres_database type

---
 cdist/conf/type/__postgres_database/gencode-remote     | 7 ++++++-
 cdist/conf/type/__postgres_database/man.rst            | 4 ++++
 cdist/conf/type/__postgres_database/parameter/optional | 1 +
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote
index b5f52f50..0f11cff4 100755
--- a/cdist/conf/type/__postgres_database/gencode-remote
+++ b/cdist/conf/type/__postgres_database/gencode-remote
@@ -44,6 +44,11 @@ if [ "$state_should" != "$state_is" ]; then
             owner="-O \"$(cat "$__object/parameter/owner")\""
          fi
 
+         template=""
+         if [ -f "$__object/parameter/template" ]; then
+            template="--template \"$(cat "$__object/parameter/template")\""
+         fi
+
          encoding=""
          if [ -f "$__object/parameter/encoding" ]; then
             encoding="--encoding \"$(cat "$__object/parameter/encoding")\""
@@ -60,7 +65,7 @@ if [ "$state_should" != "$state_is" ]; then
          fi
 
          cat << EOF
-su - '$postgres_user' -c "createdb $owner \"$name\" $encoding $lc_collate $lc_ctype"
+su - '$postgres_user' -c "createdb $owner \"$name\" $template $encoding $lc_collate $lc_ctype"
 EOF
       ;;
       absent)
diff --git a/cdist/conf/type/__postgres_database/man.rst b/cdist/conf/type/__postgres_database/man.rst
index 97ac95eb..870b4917 100644
--- a/cdist/conf/type/__postgres_database/man.rst
+++ b/cdist/conf/type/__postgres_database/man.rst
@@ -28,6 +28,10 @@ lc-collate
 lc-ctype
    Specifies the LC_CTYPE setting to be used in this database.
 
+template
+   Specifies the template database from which to build this database.
+
+
 EXAMPLES
 --------
 
diff --git a/cdist/conf/type/__postgres_database/parameter/optional b/cdist/conf/type/__postgres_database/parameter/optional
index fed2581e..877fbf32 100644
--- a/cdist/conf/type/__postgres_database/parameter/optional
+++ b/cdist/conf/type/__postgres_database/parameter/optional
@@ -3,3 +3,4 @@ owner
 encoding
 lc-collate
 lc-ctype
+template

From 742163e38c2a8124390cecee02a9c865fe18fdff Mon Sep 17 00:00:00 2001
From: Joachim Desroches <joachim.desroches@epfl.ch>
Date: Wed, 15 Apr 2020 17:10:33 +0200
Subject: [PATCH 224/743] Fix configuration file location

---
 cdist/configuration.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/configuration.py b/cdist/configuration.py
index f05a5963..1011a382 100644
--- a/cdist/configuration.py
+++ b/cdist/configuration.py
@@ -274,7 +274,8 @@ class Configuration(metaclass=Singleton):
              os.path.isfile(_local_config_file))):
         _local_config_file = os.path.join(
             os.environ.get('XDG_CONFIG_HOME',
-                           os.path.expanduser('~/.config/cdist')),
+                           os.path.expanduser('~/.config/')),
+            'cdist',
             _config_basename)
     _dist_config_file = os.path.join(
         os.path.abspath(os.path.join(os.path.dirname(cdist.__file__), "conf")),

From c9c1e7d79050afe2bc69b875096dbd1379d70eea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 21 Apr 2020 08:13:32 +0200
Subject: [PATCH 225/743] Import CI image definition

---
 other/ci/Dockerfile   | 8 ++++++++
 other/ci/README.md    | 3 +++
 other/ci/repositories | 3 +++
 3 files changed, 14 insertions(+)
 create mode 100644 other/ci/Dockerfile
 create mode 100644 other/ci/README.md
 create mode 100644 other/ci/repositories

diff --git a/other/ci/Dockerfile b/other/ci/Dockerfile
new file mode 100644
index 00000000..03d6b546
--- /dev/null
+++ b/other/ci/Dockerfile
@@ -0,0 +1,8 @@
+FROM alpine:latest
+
+COPY ./repositories /etc/apk/
+
+RUN apk update
+RUN apk upgrade
+RUN apk add python3 py3-pycodestyle rsync make shellcheck git
+RUN apk fix
diff --git a/other/ci/README.md b/other/ci/README.md
new file mode 100644
index 00000000..0bd64613
--- /dev/null
+++ b/other/ci/README.md
@@ -0,0 +1,3 @@
+This container is used for cdist's CI pipeline, and deployed in ungleich's docker registry at:
+
+	code.ungleich.ch:5050/ungleich-public/cdist-ci:latest
diff --git a/other/ci/repositories b/other/ci/repositories
new file mode 100644
index 00000000..46cabcc3
--- /dev/null
+++ b/other/ci/repositories
@@ -0,0 +1,3 @@
+https://mirror.ungleich.ch/mirror/packages/alpine/edge/main
+https://mirror.ungleich.ch/mirror/packages/alpine/edge/community
+https://mirror.ungleich.ch/mirror/packages/alpine/edge/testing

From 7d576554708115cef78eac364b20265386998882 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 21 Apr 2020 08:15:51 +0200
Subject: [PATCH 226/743] Fix typo in cdist-ci image README

---
 other/ci/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/other/ci/README.md b/other/ci/README.md
index 0bd64613..6a9b0ac5 100644
--- a/other/ci/README.md
+++ b/other/ci/README.md
@@ -1,3 +1,3 @@
 This container is used for cdist's CI pipeline, and deployed in ungleich's docker registry at:
 
-	code.ungleich.ch:5050/ungleich-public/cdist-ci:latest
+	code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest

From e2b26aa233fe702f19a39fa1378a6c531d36720f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 21 Apr 2020 08:16:46 +0200
Subject: [PATCH 227/743] Patch CI configuration to use cdist-ci image

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1cc17995..e215652c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,8 @@
 stages:
     - test
 
+image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
+
 unit_tests:
     stage: test
     script:

From 38ccdfda321e6bdb683842d227b379feea5e752c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 22 Apr 2020 23:21:34 +0200
Subject: [PATCH 228/743] Fix newly found shellcheck issues

---
 cdist/conf/type/__cdist/manifest           | 1 +
 cdist/conf/type/__openldap_server/manifest | 1 +
 2 files changed, 2 insertions(+)

diff --git a/cdist/conf/type/__cdist/manifest b/cdist/conf/type/__cdist/manifest
index a97cf288..0b0f1263 100755
--- a/cdist/conf/type/__cdist/manifest
+++ b/cdist/conf/type/__cdist/manifest
@@ -37,6 +37,7 @@ source="$(cat "$__object/parameter/source")"
 # out of it
 home=/home/$username
 
+# shellcheck disable=SC2086
 __user "$username" --home "$home" $shell
 
 require="__user/$username" __directory "$home" \
diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest
index dadc9f20..d35603c4 100644
--- a/cdist/conf/type/__openldap_server/manifest
+++ b/cdist/conf/type/__openldap_server/manifest
@@ -168,6 +168,7 @@ if [ -z "${_skip_letsencrypt_cert}" ]; then
         staging=""
     fi
 
+    # shellcheck disable=SC2086
     __letsencrypt_cert "${name}" --admin-email "${admin_email}" \
         --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R openldap:openldap ${SLAPD_DIR}/sasl2 && service slapd restart" \
         --automatic-renewal ${staging}

From 41e59a748dc45aaec88edc7a7eaed96c6086b1b1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 22 Apr 2020 23:24:34 +0200
Subject: [PATCH 229/743] Fix newly found shellcheck issues

---
 scripts/cdist-dump | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/cdist-dump b/scripts/cdist-dump
index 83b09eb8..d29e5985 100755
--- a/scripts/cdist-dump
+++ b/scripts/cdist-dump
@@ -224,6 +224,7 @@ hor_line()
 if [ "${do_global_explorer}" ]
 then
     print_verbose 2 "Dumping global explorers"
+    # shellcheck disable=SC2086
     set -- "$@" ${or} \( \
                 -path "*/explorer/*" -a \
                 ! -path "*/conf/*" -a \

From de37b0ce450109306e31affa67dd2ff3653b6d68 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 24 Apr 2020 20:26:44 +0200
Subject: [PATCH 230/743] [__motd] Improve documentation and support for
 FreeBSD

This makes it easier to use the type just by reading the man page and also
treats FreeBSD's MOTD better.
---
 cdist/conf/type/__motd/gencode-remote | 11 +++++++++++
 cdist/conf/type/__motd/man.rst        | 19 ++++++++++++++++++-
 cdist/conf/type/__motd/manifest       |  8 ++++++++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__motd/gencode-remote b/cdist/conf/type/__motd/gencode-remote
index bc842cc8..738ea834 100755
--- a/cdist/conf/type/__motd/gencode-remote
+++ b/cdist/conf/type/__motd/gencode-remote
@@ -29,7 +29,18 @@ case "$os" in
         echo "uname -snrvm > /var/run/motd"
         echo "cat /etc/motd.tail >> /var/run/motd"
     ;;
+    freebsd)
+        # FreeBSD only updates /etc/motd on boot,
+	# as seen in /etc/rc.d/motd
+	echo "uname -sri > /etc/motd"
+	echo "cat /etc/motd.template >> /etc/motd"
+	# FreeBSD 13 starts treating motd slightly different from previous
+	# versions this ensures hosts have the expected config.
+	echo "rm /etc/motd.template || true"
+	echo "service motd start"
+    ;;
     *)
+        # Other OS tend to treat /etc/motd statically
         exit 0
     ;;
 esac
diff --git a/cdist/conf/type/__motd/man.rst b/cdist/conf/type/__motd/man.rst
index 17369684..a567dc80 100644
--- a/cdist/conf/type/__motd/man.rst
+++ b/cdist/conf/type/__motd/man.rst
@@ -10,6 +10,13 @@ DESCRIPTION
 -----------
 This cdist type allows you to easily setup /etc/motd.
 
+.. note::
+      In some OS, motd is a bit special, check `motd(5)`.
+      Currently Debian, Devuan, Ubuntu and FreeBSD are taken into account.
+      If your OS of choice does something besides /etc/motd, check the source
+      and contribute support for it.
+      Otherwise it will likely just work.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -20,6 +27,7 @@ OPTIONAL PARAMETERS
 -------------------
 source
    If supplied, copy this file from the host running cdist to the target.
+   If source is '-' (dash), take what was written to stdin as the file content.
    If not supplied, a default message will be placed onto the target.
 
 
@@ -34,6 +42,15 @@ EXAMPLES
     # Supply source file from a different type
     __motd --source "$__type/files/my-motd"
 
+    # Supply source from stdin
+    __motd --source "-" <<EOF
+    Take this kiss upon the brow!
+    And, in parting from you now,
+    Thus much let me avow-
+    You are not wrong, who deem
+    That my days have been a dream
+    EOF
+
 
 AUTHORS
 -------
@@ -42,7 +59,7 @@ Nico Schottelius <nico-cdist--@--schottelius.org>
 
 COPYING
 -------
-Copyright \(C) 2011 Nico Schottelius. You can redistribute it
+Copyright \(C) 2020 Nico Schottelius. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__motd/manifest b/cdist/conf/type/__motd/manifest
index cd741cf4..ded734d7 100755
--- a/cdist/conf/type/__motd/manifest
+++ b/cdist/conf/type/__motd/manifest
@@ -34,9 +34,17 @@ os=$(cat "$__global/explorer/os")
 
 case "$os" in
    debian|ubuntu|devuan)
+      # Debian-based systems use /etc/motd.tail as a template
       destination=/etc/motd.tail
    ;;
+   freebsd)
+      # FreeBSD uses motd.template to prepend system information on boot
+      # (this actually only applies starting with version 13,
+      #  but we fix that for whatever version in gencode-remote)
+      destination=/etc/motd.template
+   ;;
    *)
+      # Most UNIX systems, including other Linux and OpenBSD just use /etc/motd
       destination=/etc/motd
    ;;
 esac

From 056c7c5400e7beeb15de800be6b62661998259ff Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 25 Apr 2020 00:12:24 +0200
Subject: [PATCH 231/743] [__openldap_server] Support extra config parameter.

This allows the user to, e.g. manually define ACLs, while this type does not
support that.
---
 cdist/conf/type/__openldap_server/man.rst            | 3 +++
 cdist/conf/type/__openldap_server/manifest           | 3 +++
 cdist/conf/type/__openldap_server/parameter/optional | 3 ++-
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
index d20101d1..fbad21d8 100644
--- a/cdist/conf/type/__openldap_server/man.rst
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -92,6 +92,9 @@ tls-ca
     Required if `tls-cert` is defined.
     Path in the remote hosts to the PEM-encoded CA certificate file.
 
+extra-config
+    Custom settings to be added in `slapd.conf(5)`.
+
 
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest
index d35603c4..84ba176f 100644
--- a/cdist/conf/type/__openldap_server/manifest
+++ b/cdist/conf/type/__openldap_server/manifest
@@ -9,6 +9,7 @@ slapd_modules=$(cat "${__object}/parameter/module" 2>/dev/null || true)
 schemas=$(cat "${__object}/parameter/schema")
 slapd_urls=$(tr '\n' ' ' < "${__object}/parameter/slapd-url")
 tls_cipher_suite=$(cat "${__object}/parameter/tls-cipher-suite" 2>/dev/null || true)
+extra_config=$(cat "${__object}/parameter/extra-config" || true)
 
 
 os="$(cat "${__global}/explorer/os")"
@@ -231,6 +232,8 @@ index uid,memberUid eq,pres,sub
 index nisMapName,nisMapEntry eq,pres,sub
 index entryCSN,entryUUID eq
 
+${extra_config}
+
 serverid ${serverid}
 EOF
 
diff --git a/cdist/conf/type/__openldap_server/parameter/optional b/cdist/conf/type/__openldap_server/parameter/optional
index a92b9c6e..71c64659 100644
--- a/cdist/conf/type/__openldap_server/parameter/optional
+++ b/cdist/conf/type/__openldap_server/parameter/optional
@@ -5,4 +5,5 @@ admin-email
 tls-cipher-suite
 tls-cert
 tls-privkey
-tls-ca
\ No newline at end of file
+tls-ca
+extra-config

From 5981d0a5f1deb2e60ea3c95ad3b535f1425060f6 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 25 Apr 2020 00:22:28 +0200
Subject: [PATCH 232/743] [__postfix] Automagically support more OSs by not
 checking too much.

It is quite likely that the package is going to be called postfix, rather
than trying to have an exhaustive "allow list" for this package, we can
just add special cases for OSs where that is not the case (not aware of any
atm).
---
 cdist/conf/type/__postfix/manifest | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/cdist/conf/type/__postfix/manifest b/cdist/conf/type/__postfix/manifest
index f3616979..121bba96 100755
--- a/cdist/conf/type/__postfix/manifest
+++ b/cdist/conf/type/__postfix/manifest
@@ -19,16 +19,4 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-
-os=$(cat "$__global/explorer/os")
-
-case "$os" in
-   alpine|ubuntu|debian|archlinux|suse|scientific|centos|devuan)
-      __package postfix --state present
-   ;;
-   *)
-      echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
-      echo "Please contribute an implementation for it if you can." >&2
-      exit 1
-   ;;
-esac
+__package postfix --state present

From fefe90e9c9bde2ad00c142c01d5a588eb99751e5 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 25 Apr 2020 12:23:27 +0200
Subject: [PATCH 233/743] [__pf*] (~) __pf_ruleset (+)__pf_apply_anchor,
 deprecate __pf_apply
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

__pf_apply the way it exists on cdist is not really useful and __pf_ruleset does
not take advantage of other types as it should, being instead overly complex and
not as reliable.

The new __pf_ruleset is compatible with the previous one, and __pf_apply_anchors
allows for a simple and powerful way of managing pf anchors.
The functionality previously provided by __pf_apply is still possible out of the
box in __pf_ruleset.

These patches were mostly contributed by Kamila Součková and made fit for
upstreaming by Evilham.
---
 cdist/conf/type/__pf_apply/deprecated         |  1 +
 .../gencode-remote}                           | 28 +++----
 cdist/conf/type/__pf_apply_anchor/man.rst     | 62 ++++++++++++++
 .../type/__pf_apply_anchor/parameter/optional |  1 +
 cdist/conf/type/__pf_ruleset/gencode-local    | 81 -------------------
 cdist/conf/type/__pf_ruleset/man.rst          | 14 ++--
 .../__pf_ruleset/{gencode-remote => manifest} | 47 +++++------
 7 files changed, 105 insertions(+), 129 deletions(-)
 create mode 100644 cdist/conf/type/__pf_apply/deprecated
 rename cdist/conf/type/{__pf_ruleset/explorer/cksum => __pf_apply_anchor/gencode-remote} (55%)
 create mode 100644 cdist/conf/type/__pf_apply_anchor/man.rst
 create mode 100644 cdist/conf/type/__pf_apply_anchor/parameter/optional
 delete mode 100755 cdist/conf/type/__pf_ruleset/gencode-local
 rename cdist/conf/type/__pf_ruleset/{gencode-remote => manifest} (51%)

diff --git a/cdist/conf/type/__pf_apply/deprecated b/cdist/conf/type/__pf_apply/deprecated
new file mode 100644
index 00000000..36cfed90
--- /dev/null
+++ b/cdist/conf/type/__pf_apply/deprecated
@@ -0,0 +1 @@
+Consider moving to __pf_apply_anchor. Get in touch if you need __pf_apply.
diff --git a/cdist/conf/type/__pf_ruleset/explorer/cksum b/cdist/conf/type/__pf_apply_anchor/gencode-remote
similarity index 55%
rename from cdist/conf/type/__pf_ruleset/explorer/cksum
rename to cdist/conf/type/__pf_apply_anchor/gencode-remote
index 9be6c901..36c26521 100755
--- a/cdist/conf/type/__pf_ruleset/explorer/cksum
+++ b/cdist/conf/type/__pf_apply_anchor/gencode-remote
@@ -1,6 +1,6 @@
-#!/bin/sh
+#!/bin/sh -e
 #
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
+# 2016 Kamila Součková (coding at kamila.is)
 #
 # This file is part of cdist.
 #
@@ -18,24 +18,16 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Get the 256 bit SHA2 checksum of the pf ruleset on the target host.
+# Apply pf(4) ruleset on *BSD
 #
 
-# Debug
-#exec >&2
-#set -x
+ANCHORS_DIR="/etc/pf.d"
 
-# Check /etc/rc.conf for pf's configuration file name. Default to /etc/pf.conf
-# See if file exists and if so, get checksum
-
-RC="/etc/rc.conf"
-TMP="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')"
-PFCONF="${TMP:-"/etc/pf.conf"}"
-
-if [ -f "${PFCONF}" ]; then	# The pf config file exists, find its cksum.
-	cksum -o 1 "${PFCONF}" | cut -d= -f2 | awk '{print $1}'
+if [ -f "${__object}/parameter/anchor_name" ]; then
+    anchor_name="$(cat "${__object}/parameter/anchor_name")"
+else
+    anchor_name="${__object_id}"
 fi
+anchor_file="${ANCHORS_DIR}/${anchor_name}"
 
-# Debug
-#set +x
-
+echo "pfctl -a \"${anchor_name}\" -f \"${anchor_file}\""
diff --git a/cdist/conf/type/__pf_apply_anchor/man.rst b/cdist/conf/type/__pf_apply_anchor/man.rst
new file mode 100644
index 00000000..aef6cdf4
--- /dev/null
+++ b/cdist/conf/type/__pf_apply_anchor/man.rst
@@ -0,0 +1,62 @@
+cdist-type__pf_apply_anchor(7)
+==============================
+
+NAME
+----
+cdist-type__pf_apply_anchor - Apply a pf(4) anchor on $__target_host
+
+
+DESCRIPTION
+-----------
+This type is used on \*BSD systems to manage anchors for the pf firewall.
+
+Notice this type does not take care of copying the ruleset, that must be
+done by the user with, e.g. `__file`.
+
+
+OPTIONAL PARAMETERS
+-------------------
+anchor_name
+   The name of the anchor to apply. If not set, `${__object_id}` is used.
+   This type requires `/etc/pf.d/${anchor_name}` to exist on
+   `$__target_host`.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Copy anchor file to ${__target_host}
+    __file "/etc/pf.d/80_dns" --source - <<EOF
+    # Managed remotely, changes will be lost
+
+    pass quick proto {tcp,udp} from any to any port domain
+    EOF
+
+    # Apply the anchor
+    require="__file/etc/pf.d/80_dns" __pf_apply_anchor 80_dns
+    # This is roughly equivalent to:
+    #   pfctl -a "${anchor_name}" -f "/etc/pf.d/${anchor_name}"
+
+
+SEE ALSO
+--------
+:strong:`pf`\ (4)
+
+
+AUTHORS
+-------
+Evilham <contact--@--evilham.com>
+Kamila Součková <coding--@--kamila.is>
+Jake Guffey <jake.guffey--@--eprotex.com>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Evilham.
+Copyright \(C) 2016 Kamila Součková.
+Copyright \(C) 2012 Jake Guffey. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__pf_apply_anchor/parameter/optional b/cdist/conf/type/__pf_apply_anchor/parameter/optional
new file mode 100644
index 00000000..b9f61e28
--- /dev/null
+++ b/cdist/conf/type/__pf_apply_anchor/parameter/optional
@@ -0,0 +1 @@
+anchor_name
diff --git a/cdist/conf/type/__pf_ruleset/gencode-local b/cdist/conf/type/__pf_ruleset/gencode-local
deleted file mode 100755
index 11bfb0b1..00000000
--- a/cdist/conf/type/__pf_ruleset/gencode-local
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/bin/sh -e
-#
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Manage pf(4) on *BSD
-#
-
-# Debug
-#exec >&2
-#set -x
-
-# Send files to $__target_host via $__remote_copy
-
-uname=$(uname)   # Need to know what the cdist host is running so we know how to compute the ruleset's checksum
-state=$(cat "$__object/parameter/state")
-
-if [ "$state" = "absent" ]; then   # There is nothing more for a *local* script to do
-   exit 0
-fi
-
-if [ -f "$__object/parameter/source" ]; then
-   source=$(cat "$__object/parameter/source")
-fi
-
-rcvar=$(cat "$__object/explorer/rcvar")
-cksum=$(cat "$__object/explorer/cksum")
-
-
-cat <<EOF
-case $uname in
-   Darwin)
-      currentSum=\$(cksum -o 1 ${source} | cut '-d ' -f1)
-      ;;
-   Linux)
-      currentSum=\$(cksum ${source} | cut '-d ' -f1)
-      ;;
-   FreeBSD)
-      currentSum=\$(cksum -o 1 ${source} | cut -d= -f2 | sed 's/ //g')
-      ;;
-   *)
-      echo "Sorry, I do not know how to find a cksum on ${uname}." >&2
-      exit 1
-      ;;
-esac
-
-# IPv6 fix
-if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$')
-then
-    my_target_host="[${__target_host}]"
-else
-    my_target_host="${__target_host}"
-fi
-
-if [ -n "${cksum}" ]; then
-   if [ ! "\${currentSum}" = "${cksum}" ]; then
-      $__remote_copy "${source}" "\${my_target_host}:${rcvar}.new"
-   fi
-else # File just doesn't exist yet
-   $__remote_copy "${source}" "\${my_target_host}:${rcvar}.new"
-fi
-EOF
-
-# Debug
-#exec +x
-
diff --git a/cdist/conf/type/__pf_ruleset/man.rst b/cdist/conf/type/__pf_ruleset/man.rst
index 5719e94e..db8873ac 100644
--- a/cdist/conf/type/__pf_ruleset/man.rst
+++ b/cdist/conf/type/__pf_ruleset/man.rst
@@ -10,6 +10,9 @@ DESCRIPTION
 -----------
 This type is used on \*BSD systems to manage the pf firewall's ruleset.
 
+It will also enable and disable the pf firewall as requested in the `state`
+parameter.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -20,9 +23,8 @@ state
 OPTIONAL PARAMETERS
 -------------------
 source
-   If supplied, use to define the ruleset to load onto the $__target_host for pf(4).
-   Note that this type is almost useless without a ruleset defined, but it's technically not
-   needed, e.g. for the case of disabling the firewall temporarily.
+   Required when state is "present".
+   Defines the ruleset to load onto the $__target_host for `pf(4)`.
 
 
 EXAMPLES
@@ -30,10 +32,10 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # Remove the current ruleset in place
+    # Remove the current ruleset in place and disable pf
     __pf_ruleset --state absent
 
-    # Enable the firewall with the ruleset defined in $__manifest/files/pf.conf
+    # Enable pf with the ruleset defined in $__manifest/files/pf.conf
     __pf_ruleset --state present --source $__manifest/files/pf.conf
 
 
@@ -44,11 +46,13 @@ SEE ALSO
 
 AUTHORS
 -------
+Kamila Součková <coding--@--kamila.is>
 Jake Guffey <jake.guffey--@--eprotex.com>
 
 
 COPYING
 -------
+Copyright \(C) 2016 Kamila Součková.
 Copyright \(C) 2012 Jake Guffey. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
diff --git a/cdist/conf/type/__pf_ruleset/gencode-remote b/cdist/conf/type/__pf_ruleset/manifest
similarity index 51%
rename from cdist/conf/type/__pf_ruleset/gencode-remote
rename to cdist/conf/type/__pf_ruleset/manifest
index 12760fdf..34fee5dd 100755
--- a/cdist/conf/type/__pf_ruleset/gencode-remote
+++ b/cdist/conf/type/__pf_ruleset/manifest
@@ -1,6 +1,6 @@
 #!/bin/sh -e
 #
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
+# 2016 Kamila Součková (coding at kamila.is)
 #
 # This file is part of cdist.
 #
@@ -21,29 +21,26 @@
 # Manage pf(4) on *BSD
 #
 
-# Debug
-#exec >&2
-#set -x
-
-# Remove ${rcvar} in the case of --state absent
-
-state=$(cat "$__object/parameter/state")
-rcvar=$(cat "$__object/explorer/rcvar")
-
-if [ "$state" = "present" ]; then   # There is nothing more for a *remote* script to do
-   exit 0
-elif [ "$state" = "absent" ]; then
-   # --state absent, so ensure that .new doesn't exist and that conf is renamed to .old
-   cat <<EOF
-   if [ -f "${rcvar}.new" ]; then
-      rm "${rcvar}.new"
-   fi
-   if [ -f "${rcvar}" ]; then
-      mv "${rcvar}" "${rcvar}.old"
-   fi
-EOF
-else
-   echo "Unknown state ${state}!" >&2
-   exit 1
+rcvar="$(cat "${__object}/explorer/rcvar")"
+state="$(cat "${__object}/parameter/state")"
+if [ -f "${__object}/parameter/source" ]; then
+    source="$(cat "${__object}/parameter/source")"
 fi
 
+if [ "${state}" = "absent" ]; then
+    action="/etc/rc.d/pf stop"
+else
+    action="/etc/rc.d/pf reload || /etc/rc.d/pf start"
+fi
+
+__key_value __pf_ruleset/rcvar \
+    --state "${state}" \
+    --file /etc/rc.conf \
+    --delimiter "=" \
+    --key "pf_enable" \
+    --value "YES"
+
+require="__key_value/__pf_ruleset/rcvar" __config_file ${rcvar} \
+    --source "${source}" \
+    --state "${state}" \
+    --onchange "${action}"

From 292879544154f11f4cd719ebf0c67c25000a25fd Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 25 Apr 2020 14:54:29 +0200
Subject: [PATCH 234/743] [__pf_ruleset] Fix shellcheck issue.

---
 cdist/conf/type/__pf_ruleset/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__pf_ruleset/manifest b/cdist/conf/type/__pf_ruleset/manifest
index 34fee5dd..27b35328 100755
--- a/cdist/conf/type/__pf_ruleset/manifest
+++ b/cdist/conf/type/__pf_ruleset/manifest
@@ -40,7 +40,7 @@ __key_value __pf_ruleset/rcvar \
     --key "pf_enable" \
     --value "YES"
 
-require="__key_value/__pf_ruleset/rcvar" __config_file ${rcvar} \
+require="__key_value/__pf_ruleset/rcvar" __config_file "${rcvar}" \
     --source "${source}" \
     --state "${state}" \
     --onchange "${action}"

From 04b7f240ebc1ae5b0f23279d75b65b50d062ccd1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sun, 26 Apr 2020 10:01:18 +0200
Subject: [PATCH 235/743] ++changelog

---
 docs/changelog | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index b6b1bb39..7cacf3cf 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,16 @@
 Changelog
 ---------
 
+next:
+	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)
+	* Type __postgres_database: Add encoding, lc-collate, lc-ctype, template parameters (Timothée Floure)
+	* Type __motd: Improve documentation and support for FreeBSD (Evil Ham)
+	* Type __openldap_server: Support extra config parameter (Evil Ham)
+	* Type __postfix: Automagically support more OSs by not checking too much (Evil Ham)
+	* New type: __pf_apply_anchor (Kamila Součková, Evil Ham)
+	* Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham)
+	* Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham)
+
 6.5.4: 2020-04-11
 	* Explorer init: Do not grep on non-existent init (Steven Armstrong)
 	* Core: Bugfix to preserve conf dirs order (Steven Armstrong)

From fefc828780bc76c4245fa96ef517a66ec3eda3fa Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sun, 26 Apr 2020 19:06:42 +0200
Subject: [PATCH 236/743] [docs] Improve cdist.cfg.skeleton

---
 configuration/cdist.cfg.skeleton | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton
index 22c1ccaf..bfac9f5c 100644
--- a/configuration/cdist.cfg.skeleton
+++ b/configuration/cdist.cfg.skeleton
@@ -19,6 +19,9 @@
 #     such as ':' for POSIX or ';' for Windows.
 #     If also specified at command line then values from command line are
 #     appended to this value.
+#     Notice that this works in a "last one wins" fashion, so if a type is redefined
+#     in multiple conf_dirs, the last one in which it is defined will be used.
+#     Consider using a unique prefix for your own roles if this can be an issue.
 # conf_dir = <dir1>:<dir2>
 #
 # init_manifest

From 678df1ec8a24b4c3884554b1931445435e098787 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 27 Apr 2020 01:23:48 +0200
Subject: [PATCH 237/743] [explorers] Improve *BSD support.

cpu_cores and memory did lacked support for other BSDs.
---
 cdist/conf/explorer/cpu_cores | 4 ++++
 cdist/conf/explorer/memory    | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/cpu_cores b/cdist/conf/explorer/cpu_cores
index a52bddac..c6744142 100755
--- a/cdist/conf/explorer/cpu_cores
+++ b/cdist/conf/explorer/cpu_cores
@@ -32,6 +32,10 @@ case "$os" in
         sysctl -n hw.ncpuonline
     ;;
 
+    "freebsd"|"netbsd")
+        sysctl -n hw.ncpu
+    ;;
+
     *)
         if [ -r /proc/cpuinfo ]; then
             cores="$(grep "core id" /proc/cpuinfo | sort | uniq | wc -l)"
diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory
index 4e3efff8..302b4cda 100755
--- a/cdist/conf/explorer/memory
+++ b/cdist/conf/explorer/memory
@@ -29,7 +29,7 @@ case "$os" in
         echo "$(sysctl -n hw.memsize)/1024" | bc
     ;;
 
-    "openbsd")
+    *"bsd")
         echo "$(sysctl -n hw.physmem) / 1048576" | bc
     ;;
 

From 0b3c417aef13eceb51195bb42e9a17205a9afc6c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 27 Apr 2020 14:00:39 +0300
Subject: [PATCH 238/743] update README

---
 README    |  7 -------
 README.md | 20 ++++++++++++++++++++
 2 files changed, 20 insertions(+), 7 deletions(-)
 delete mode 100644 README
 create mode 100644 README.md

diff --git a/README b/README
deleted file mode 100644
index caf2dac8..00000000
--- a/README
+++ /dev/null
@@ -1,7 +0,0 @@
-cdist
------
-
-cdist is a usable configuration management system.
-
-For the web documentation have a look at https://www.cdi.st/
-or at docs/src for reStructuredText manual.
diff --git a/README.md b/README.md
new file mode 100644
index 00000000..9e49b053
--- /dev/null
+++ b/README.md
@@ -0,0 +1,20 @@
+# cdist
+
+**cdist** is a usable configuration management system.
+
+It adheres to the [**KISS principle**](https://en.wikipedia.org/wiki/KISS_principle)
+and is being used in small up to enterprise grade environments.
+
+For more information have a look at [**homepage**](https://cdi.st)
+or at **``docs/src``** for manual in **reStructuredText** format.
+
+## Contributing
+
+Merge/Pull requests can be made in both
+[upstream **GitLab**](https://code.ungleich.ch/ungleich-public/cdist/merge_requests)
+(managed by [**ungleich**](https://ungleich.ch))
+and [**GitHub** project](https://github.com/ungleich/cdist/pulls).
+
+Issues can be made and other project management activites happen
+[**only in GitLab**](https://code.ungleich.ch/ungleich-public/cdist)
+(needs [**ungleich** account](https://account.ungleich.ch)).

From 56a65518ab6171d0f19152642fccb94038a3c5ad Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 27 Apr 2020 15:25:43 +0300
Subject: [PATCH 239/743] README: add participating section

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index 9e49b053..0a0d6e6d 100644
--- a/README.md
+++ b/README.md
@@ -18,3 +18,11 @@ and [**GitHub** project](https://github.com/ungleich/cdist/pulls).
 Issues can be made and other project management activites happen
 [**only in GitLab**](https://code.ungleich.ch/ungleich-public/cdist)
 (needs [**ungleich** account](https://account.ungleich.ch)).
+
+## Participating
+
+IRC: ``#cdist`` @ freenode
+
+Matrix: ``#cdist:ungleich.ch``
+
+Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist

From b31e13eacf2cb23e6a7bdadc33741026ae88553a Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 27 Apr 2020 16:30:52 +0300
Subject: [PATCH 240/743] README: add bits about cdist-contrib

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index 0a0d6e6d..de6901c7 100644
--- a/README.md
+++ b/README.md
@@ -19,6 +19,9 @@ Issues can be made and other project management activites happen
 [**only in GitLab**](https://code.ungleich.ch/ungleich-public/cdist)
 (needs [**ungleich** account](https://account.ungleich.ch)).
 
+For community-maintained types there is
+[**cdist-contrib** project](https://code.ungleich.ch/ungleich-public/cdist-contrib).
+
 ## Participating
 
 IRC: ``#cdist`` @ freenode

From 515992249de513492a725dbf4072a6c3f376668a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 27 Apr 2020 22:55:57 +0200
Subject: [PATCH 241/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 7cacf3cf..1e213187 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,6 +10,8 @@ next:
 	* New type: __pf_apply_anchor (Kamila Součková, Evil Ham)
 	* Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham)
 	* Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham)
+	* Configuration: Add notes to cdist.cfg.skeleton (Evil Ham)
+	* Explorers cpu_cores, memory: Improve *BSD support (Evil Ham)
 
 6.5.4: 2020-04-11
 	* Explorer init: Do not grep on non-existent init (Steven Armstrong)

From ea3bd14d8b377818a16578bd5032a853188baeec Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 28 Apr 2020 14:54:51 +0200
Subject: [PATCH 242/743] [logging] Mute warning on return_output=True when
 running scripts.

This fixes #806 which contains more information about the issue.

The TL;DR: this warning is not being useful and hinders debugging types because
it creates an innecessary line for each explorer.

An alternative proposal was #807 but was abandoned in favour of just dropping
the warning.
---
 cdist/exec/remote.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py
index e0ef66ec..f72bf3bf 100644
--- a/cdist/exec/remote.py
+++ b/cdist/exec/remote.py
@@ -280,9 +280,6 @@ class Remote(object):
         assert isinstance(command, (list, tuple)), (
                 "list or tuple argument expected, got: %s" % command)
 
-        if return_output and stdout is not subprocess.PIPE:
-            self.log.debug("return_output is True, ignoring stdout")
-
         close_stdout = False
         close_stderr = False
         if self.save_output_streams:

From 250161e42d1fe32bcb7b6945014aa4786d3b46ed Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 28 Apr 2020 23:08:03 +0200
Subject: [PATCH 243/743] ++

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 1e213187..c29f70de 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,7 @@ next:
 	* Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham)
 	* Configuration: Add notes to cdist.cfg.skeleton (Evil Ham)
 	* Explorers cpu_cores, memory: Improve *BSD support (Evil Ham)
+	* Core: Remove debug logging noise (Evil Ham)
 
 6.5.4: 2020-04-11
 	* Explorer init: Do not grep on non-existent init (Steven Armstrong)

From 310045d9fb5a5a63a337867a90009f89e0ded5ed Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 1 May 2020 13:02:00 +0200
Subject: [PATCH 244/743] Release 6.5.5

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index c29f70de..e6a7927e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)
 	* Type __postgres_database: Add encoding, lc-collate, lc-ctype, template parameters (Timothée Floure)
 	* Type __motd: Improve documentation and support for FreeBSD (Evil Ham)

From f58d662b32fb9c1c226b5340ccd06c6dd3e2a3f9 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 1 May 2020 15:28:01 +0200
Subject: [PATCH 245/743] [__pyvenv] Switch to python3 -m venv for ubuntu

---
 cdist/conf/type/__pyvenv/gencode-remote | 3 ++-
 cdist/conf/type/__pyvenv/man.rst        | 5 ++---
 docs/changelog                          | 4 ++++
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__pyvenv/gencode-remote b/cdist/conf/type/__pyvenv/gencode-remote
index 9c7b7fab..c5b64eff 100755
--- a/cdist/conf/type/__pyvenv/gencode-remote
+++ b/cdist/conf/type/__pyvenv/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2016 Darko Poljak (darko.poljak at gmail.com)
+# 2020 Nico Schotetlius (nico.schottelius at ungleich.ch)
 #
 # This file is part of cdist.
 #
@@ -45,7 +46,7 @@ then
     pyvenv=$(cat "$pyvenvparam")
 else
     case "$os" in
-        alpine) # no pyvenv on alpine - I assume others will follow
+        alpine|ubuntu) # no pyvenv on alpine - I assume others will follow
             pyvenv="python3 -m venv"
             ;;
         *)
diff --git a/cdist/conf/type/__pyvenv/man.rst b/cdist/conf/type/__pyvenv/man.rst
index d7de92fa..8085ff12 100644
--- a/cdist/conf/type/__pyvenv/man.rst
+++ b/cdist/conf/type/__pyvenv/man.rst
@@ -9,7 +9,7 @@ cdist-type__pyvenv - Create or remove python virtual environment
 DESCRIPTION
 -----------
 This cdist type allows you to create or remove python virtual
-environment using pyvenv.
+environment using pyvenv on python3 -m venv.
 It assumes pyvenv is already installed. Concrete package depends
 on concrete OS and/or OS version/distribution.
 Ensure this for e.g. in your init manifest as in the following example:
@@ -57,7 +57,7 @@ EXAMPLES
 
     __pyvenv /home/services/djangoenv
 
-    # Use specific pyvenv 
+    # Use specific pyvenv
     __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4
 
     # Create python virtualenv for user foo.
@@ -76,4 +76,3 @@ COPYING
 -------
 Copyright \(C) 2016 Darko Poljak. Free use of this software is
 granted under the terms of the GNU General Public License v3 or later (GPLv3+).
-
diff --git a/docs/changelog b/docs/changelog
index e6a7927e..4ee47d37 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)
+
+
 6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)
 	* Type __postgres_database: Add encoding, lc-collate, lc-ctype, template parameters (Timothée Floure)

From d4059fd29ecb93e094c411bb8ca09a234904b763 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 1 May 2020 15:31:23 +0200
Subject: [PATCH 246/743] [__letsencrypt_cert] whitelist Ubuntu

---
 cdist/conf/type/__letsencrypt_cert/manifest | 3 +++
 docs/changelog                              | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 68ecf9d4..b4464366 100755
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -91,6 +91,9 @@ if [ -z "${certbot_fullpath}" ]; then
 
 			certbot_fullpath=/usr/local/bin/certbot
 		;;
+		ubuntu)
+            __package certbot
+            ;;
 		*)
 			echo "Unsupported os: $os" >&2
 			exit 1
diff --git a/docs/changelog b/docs/changelog
index 4ee47d37..e61933ab 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,7 +3,7 @@ Changelog
 
 next:
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)
-
+	* Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius)
 
 6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)

From 6f4649efc69593fb4178fb6c2c5c95bbd93054d1 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 8 May 2020 16:08:21 +0200
Subject: [PATCH 247/743] Reference the new cdist chat on matrix

---
 docs/src/cdist-support.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/cdist-support.rst b/docs/src/cdist-support.rst
index 19afde2f..f9f61f01 100644
--- a/docs/src/cdist-support.rst
+++ b/docs/src/cdist-support.rst
@@ -3,7 +3,7 @@ Support
 
 Chat
 ~~~~
-Chat with us: `ungleich chat <https://chat.ungleich.ch/ungleich/channels/cdist>`_.
+Chat with us on `#cdist:ungleich.ch <https://ungleich.ch/u/projects/open-chat/>`_.
 
 Mailing list
 ~~~~~~~~~~~~

From 42f2dceeb12fee55616202bf085e6823d18ee714 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 9 May 2020 18:40:44 +0200
Subject: [PATCH 248/743] [__link] fix typo in the manual

---
 cdist/conf/type/__link/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__link/man.rst b/cdist/conf/type/__link/man.rst
index fe0ce425..2e81aea9 100644
--- a/cdist/conf/type/__link/man.rst
+++ b/cdist/conf/type/__link/man.rst
@@ -18,7 +18,7 @@ source
    Specifies the link source.
 
 type
-   Specifies the link type: Either hard or symoblic.
+   Specifies the link type: Either hard or symbolic.
 
 
 OPTIONAL PARAMETERS

From dcfabf9268865794602d3e59af2f312a1b0392f0 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 16 May 2020 14:00:02 +0200
Subject: [PATCH 249/743] [__cron] add hint for default values of time-related
 parameters

This should resolve some misunderstanding when leave a time-related
parameter with it's default value.
---
 cdist/conf/type/__cron/man.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__cron/man.rst b/cdist/conf/type/__cron/man.rst
index d0694738..e39bfb5c 100644
--- a/cdist/conf/type/__cron/man.rst
+++ b/cdist/conf/type/__cron/man.rst
@@ -21,6 +21,11 @@ command
 
 OPTIONAL PARAMETERS
 -------------------
+**NOTE**: All time-related parameters (``--minute``, ``--hour``, ``--day_of_month``
+``--month`` and ``--day_of_week``) defaults to ``*``, which means to execute it
+**always**. If you set ``--hour 0`` to execute the cronjob only at midnight, it
+will execute **every** minute in the first hour of the morning all days.
+
 state
    Either present or absent. Defaults to present.
 minute

From 086e683c997729817300c600307592c6fc02ffd8 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 16 May 2020 16:33:38 +0200
Subject: [PATCH 250/743] [__file] add "default values" in manual for group,
 mode and owner

Because at least the --mode default value may not be expected, the
manual lists the "default values". As they are not explicitly set in the
`__file` type, it is a bit more difficult, but should resolve to
following:
  - the mode comes from the umask set in the cdist code to protect file
    creation and uses the strongest umask possible.
  - the owner and group comes from the ssh user, which should always be
    the root user. (I think this can be swaped, too, but who will?)

At the owner and group parameter, it could be replaced with something
like "the ssh user and group", which would be more correct, but less
understandable.
---
 cdist/conf/type/__file/man.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__file/man.rst b/cdist/conf/type/__file/man.rst
index 7a0603bb..2f3b9e69 100644
--- a/cdist/conf/type/__file/man.rst
+++ b/cdist/conf/type/__file/man.rst
@@ -50,13 +50,13 @@ state
       create or modify it
 
 group
-   Group to chgrp to.
+   Group to chgrp to. Defaults to ``root``.
 
 mode
-   Unix permissions, suitable for chmod.
+   Unix permissions, suitable for chmod. Defaults to a very secure ``0600``.
 
 owner
-   User to chown to.
+   User to chown to. Defaults to ``root``.
 
 source
    If supplied, copy this file from the host running cdist to the target.

From 7ca2bfc14a8ed0cb73b3a01c85a562060264d02b Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 18 May 2020 16:00:23 +0200
Subject: [PATCH 251/743] [explorer/machine_type] Add support for FreeBSD.

More research is needed for {Net,Open}BSD support.

Indentation is left as-is for the linux code as I intend to simplify it in a
future MR, this way the diff is minimal.
---
 cdist/conf/explorer/machine_type | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index bb21f69c..fe0ae7d5 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -2,6 +2,7 @@
 #
 # 2014 Daniel Heule  (hda at sfs.biz)
 # 2014 Thomas Oettli (otho at sfs.biz)
+# 2020 Evilham (contact at evilham.com)
 #
 # This file is part of cdist.
 #
@@ -18,9 +19,27 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
 
-# FIXME: other system types (not linux ...)
+os=$("$__explorer/os")
+
+case "$os" in
+    "freebsd")
+        # FreeBSD does not have /proc/cpuinfo even when procfs is used.
+        # Instead there is a sysctl kern.vm_guest.
+        # Which is 'none' if physical, else the virtualisation.
+        vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)"
+        if [ -n "${vm_guest}" ]; then
+            if [ "${vm_guest}" = "none" ]; then
+                echo "physical"
+                exit
+            fi
+            echo "virtual_by_${vm_guest}"
+            exit
+        fi
+    ;;
+
+    *)
+        # Defaulting to linux for compatibility with previous cdist behaviour
 
 if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then
     echo openvz
@@ -72,9 +91,13 @@ if [ -r /proc/cpuinfo ]; then
             fi                        
         fi
         echo "virtual_by_unknown"
+        exit
     else
         echo "physical"
+        exit
     fi
-else
-    echo "unknown"
 fi
+    ;;
+esac
+
+echo "unknown"

From cf44c4a01bcfba95e11ba8d6aa64156c5470adc1 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 18 May 2020 16:35:50 +0200
Subject: [PATCH 252/743] [explorer/machine_type] Simplify Linux + basic
 OpenBSD support.

By abstracting away vendor-dependent pattern matching for the linux code, we can
re-use that and be reasonably sure about OpenBSD machines being virtualised when
we can identify the undelrying virtualisation technology.
It remains to be solved how to tell if an OpenBSD machine is physical; in that
case previous cdist behaviour ("unknown") remains.

For NetBSD something similar to OpenBSD could be done, with different sysctls:
hw.machine and hw.model wary of adding those without testing though, so for
NetBSD previous cdist behaviour ("unknown") remains.
https://netbsd.gw.com/cgi-bin/man-cgi?sysctl+7.i386+NetBSD-9.0
---
 cdist/conf/explorer/machine_type | 116 ++++++++++++++++---------------
 1 file changed, 61 insertions(+), 55 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index fe0ae7d5..6800234d 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -22,6 +22,18 @@
 
 os=$("$__explorer/os")
 
+vendor_string_to_machine_type() {
+    for vendor in vmware bochs kvm qemu virtualbox bhyve; do
+        if echo "${1}" | grep -q -i "${vendor}"; then
+            if [ "${vendor}" = "bochs" -o "${vendor}" = "qemu" ]; then
+                vendor="kvm"
+            fi
+            echo "virtual_by_${vendor}"
+            exit
+        fi
+    done
+}
+
 case "$os" in
     "freebsd")
         # FreeBSD does not have /proc/cpuinfo even when procfs is used.
@@ -38,65 +50,59 @@ case "$os" in
         fi
     ;;
 
+    "openbsd")
+        # OpenBSD can also use the sysctl's: hw.vendor or hw.product.
+        # Note we can be reasonably sure about a machine being virtualised
+        # as long as we can identify the virtualisation technology.
+        # But not so much about it being physical...
+        # Patches are welcome / reach out if you have better ideas.
+        for sysctl in hw.vendor hw.product; do
+            # This exits if we can make a reasonable judgement
+            vendor_string_to_machine_type "$(sysctl -n "${sysctl}")"
+        done
+    ;;
+
     *)
         # Defaulting to linux for compatibility with previous cdist behaviour
 
-if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then
-    echo openvz
-    exit
-fi
-
-if [ -e "/proc/1/environ" ] &&
-    tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then
-    echo lxc
-    exit
-fi
-
-if [ -r /proc/cpuinfo ]; then
-    # this should only exist on virtual guest machines,
-    # tested on vmware, xen, kvm
-    if grep -q "hypervisor" /proc/cpuinfo; then
-        # this file is aviable in xen guest systems
-        if [ -r /sys/hypervisor/type ]; then
-            if grep -q -i "xen" /sys/hypervisor/type; then
-                echo virtual_by_xen
-                exit 
-            fi
-        else
-            if [ -r /sys/class/dmi/id/product_name ]; then
-                if grep -q -i 'vmware' /sys/class/dmi/id/product_name; then
-                    echo "virtual_by_vmware"
-                    exit
-                elif grep -q -i 'bochs' /sys/class/dmi/id/product_name; then
-                    echo "virtual_by_kvm"
-                    exit 
-                elif grep -q -i 'virtualbox' /sys/class/dmi/id/product_name; then
-                    echo "virtual_by_virtualbox"
-                    exit 
-                fi
-            fi
-
-            if [ -r /sys/class/dmi/id/sys_vendor ]; then
-                if grep -q -i 'qemu' /sys/class/dmi/id/sys_vendor; then
-                    echo "virtual_by_kvm"
-                    exit
-                fi
-            fi
-
-            if [ -r /sys/class/dmi/id/chassis_vendor ]; then
-                if grep -q -i 'qemu' /sys/class/dmi/id/chassis_vendor; then
-                    echo "virtual_by_kvm"
-                    exit
-                fi
-            fi                        
+        if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then
+            echo openvz
+            exit
+        fi
+
+        if [ -e "/proc/1/environ" ] &&
+            tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then
+            echo lxc
+            exit
+        fi
+
+        if [ -r /proc/cpuinfo ]; then
+            # this should only exist on virtual guest machines,
+            # tested on vmware, xen, kvm, bhyve
+            if grep -q "hypervisor" /proc/cpuinfo; then
+                # this file is aviable in xen guest systems
+                if [ -r /sys/hypervisor/type ]; then
+                    if grep -q -i "xen" /sys/hypervisor/type; then
+                        echo virtual_by_xen
+                        exit
+                    fi
+                else
+                    for vendor_file in /sys/class/dmi/id/product_name \
+                                       /sys/class/dmi/id/sys_vendor \
+                                       /sys/class/dmi/id/chasis_vendor; do
+                        if [ -r ${vendor_file} ]; then
+                            # This exits if we can make a reasonable judgement
+                            vendor_string_to_machine_type "$(cat "${vendor_file}")"
+                        fi
+                    done
+                fi
+                echo "virtual_by_unknown"
+                exit
+            else
+                echo "physical"
+                exit
+            fi
         fi
-        echo "virtual_by_unknown"
-        exit
-    else
-        echo "physical"
-        exit
-    fi
-fi
     ;;
 esac
 

From 8b790b0a54aecf93298c97948c48be80b67ca7b5 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 18 May 2020 16:47:20 +0200
Subject: [PATCH 253/743] [explorer/machine_type] Make shellcheck happy!

---
 cdist/conf/explorer/machine_type | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 6800234d..1c84f4d7 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -25,7 +25,7 @@ os=$("$__explorer/os")
 vendor_string_to_machine_type() {
     for vendor in vmware bochs kvm qemu virtualbox bhyve; do
         if echo "${1}" | grep -q -i "${vendor}"; then
-            if [ "${vendor}" = "bochs" -o "${vendor}" = "qemu" ]; then
+            if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then
                 vendor="kvm"
             fi
             echo "virtual_by_${vendor}"

From 6d502f737aac8a836949d2b4b8a55c8ba20161fa Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 18 May 2020 18:37:16 +0200
Subject: [PATCH 254/743] [__ssh_authorizedkey{,s}] Improve documentation.

---
 cdist/conf/type/__ssh_authorized_key/man.rst  | 16 +++++-----
 cdist/conf/type/__ssh_authorized_keys/man.rst | 29 ++++++++++---------
 2 files changed, 25 insertions(+), 20 deletions(-)

diff --git a/cdist/conf/type/__ssh_authorized_key/man.rst b/cdist/conf/type/__ssh_authorized_key/man.rst
index 087a3dae..5bae02aa 100644
--- a/cdist/conf/type/__ssh_authorized_key/man.rst
+++ b/cdist/conf/type/__ssh_authorized_key/man.rst
@@ -15,25 +15,27 @@ This type was created to be used by the __ssh_authorized_keys type.
 REQUIRED PARAMETERS
 -------------------
 file
-   the authorized_keys file to which the given key should be added
+   The authorized_keys file where the given key should be managed.
 
 key
-   a string containing the ssh keytype, base 64 encoded key and optional
-   trailing comment which shall be added to the given authorized_keys file.
+   The ssh key which shall be managed in this authorized_keys file.
+   Must be a string containing the ssh keytype, base 64 encoded key and
+   optional trailing comment which shall be added to the given
+   authorized_keys file.
 
 
 OPTIONAL PARAMETERS
 -------------------
 comment
-   explicit comment instead of the one which may be trailing the given key
+   Use this comment instead of the one which may be trailing in the key.
 
 option
-   an option to set for this authorized_key entry.
+   An option to set for this authorized_key entry.
    Can be specified multiple times.
    See sshd(8) for available options.
 
 state
-   if the given keys should be 'present' or 'absent', defaults to 'present'.
+   If the managed key should be 'present' or 'absent', defaults to 'present'.
 
 
 MESSAGES
@@ -64,7 +66,7 @@ EXAMPLES
 
 SEE ALSO
 --------
-:strong:`cdist__ssh_authorized_keys`\ (7), :strong:`sshd`\ (8)
+:strong:`cdist-type__ssh_authorized_keys`\ (7), :strong:`sshd`\ (8)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__ssh_authorized_keys/man.rst b/cdist/conf/type/__ssh_authorized_keys/man.rst
index ba310ff9..93357b1d 100644
--- a/cdist/conf/type/__ssh_authorized_keys/man.rst
+++ b/cdist/conf/type/__ssh_authorized_keys/man.rst
@@ -20,42 +20,45 @@ then left to the user to ensure that the file exists and that ownership and
 permissions work with ssh.
 
 
-REQUIRED PARAMETERS
--------------------
+REQUIRED MULTIPLE PARAMETERS
+----------------------------
 key
-   the ssh key which shall be added to this authorized_keys file.
-   Must be a string and can be specified multiple times.
+   An ssh key which shall be managed in this authorized_keys file.
+   Must be a string containing the ssh keytype, base 64 encoded key and
+   optional trailing comment which shall be added to the given
+   authorized_keys file.
+   Can be specified multiple times.
 
 
 OPTIONAL PARAMETERS
 -------------------
 comment
-   explicit comment instead of the one which may be trailing the given key
+   Use this comment instead of the one which may be trailing in each key.
 
 file
-   an alternative destination file, defaults to ~$owner/.ssh/authorized_keys
+   An alternative destination file, defaults to ~$owner/.ssh/authorized_keys.
 
 option
-   an option to set for all created authorized_key entries.
+   An option to set for all authorized_key entries in the key parameter.
    Can be specified multiple times.
    See sshd(8) for available options.
 
 owner
-   the user owning the authorized_keys file, defaults to object_id.
+   The user owning the authorized_keys file, defaults to object_id.
 
 state
-   if the given keys should be 'present' or 'absent', defaults to 'present'.
+   If the given keys should be 'present' or 'absent', defaults to 'present'.
 
 
 BOOLEAN PARAMETERS
 ------------------
 noparent
-   don't create or change ownership and permissions of the directory containing
-   the authorized_keys file
+   Don't create or change ownership and permissions of the directory containing
+   the authorized_keys file.
 
 nofile
-   don't manage existence, ownership and permissions of the the authorized_keys
-   file
+   Don't manage existence, ownership and permissions of the the authorized_keys
+   file.
 
 
 EXAMPLES

From f9afac4dd617c5ee7934cdb03477e6a30ae70af7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 18 May 2020 19:51:08 +0200
Subject: [PATCH 255/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index e61933ab..013e0083 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,9 @@ Changelog
 next:
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)
 	* Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius)
+	* Types __cron, __file, __link: Improve manpages (Matthias Stecher)
+	* Explorer machine_type: Add support for FreeBSD and OpenBSD, and simplify Linux code (Evil Ham)
+	* Type __ssh_authorized_key, __ssh_authorized_keys: Improve manpages (Evil Ham)
 
 6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)

From ad58ea79c2cd3edd11a828d26a28975aa332bdd0 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 18 May 2020 20:01:01 +0200
Subject: [PATCH 256/743] [__ssh_authorized_keys] Fix bug where --option was
 not multiple

This went against both documentation and intent.
---
 cdist/conf/type/__ssh_authorized_keys/parameter/optional         | 1 -
 .../conf/type/__ssh_authorized_keys/parameter/optional_multiple  | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)
 create mode 100644 cdist/conf/type/__ssh_authorized_keys/parameter/optional_multiple

diff --git a/cdist/conf/type/__ssh_authorized_keys/parameter/optional b/cdist/conf/type/__ssh_authorized_keys/parameter/optional
index 21f9bc29..fa64fc43 100644
--- a/cdist/conf/type/__ssh_authorized_keys/parameter/optional
+++ b/cdist/conf/type/__ssh_authorized_keys/parameter/optional
@@ -1,5 +1,4 @@
 comment
 file
-option
 owner
 state
diff --git a/cdist/conf/type/__ssh_authorized_keys/parameter/optional_multiple b/cdist/conf/type/__ssh_authorized_keys/parameter/optional_multiple
new file mode 100644
index 00000000..01925a15
--- /dev/null
+++ b/cdist/conf/type/__ssh_authorized_keys/parameter/optional_multiple
@@ -0,0 +1 @@
+option

From f354d80308b1f048416b2d52a397035a93a8ee03 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 18 May 2020 21:03:29 +0200
Subject: [PATCH 257/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 013e0083..782d9bd7 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Types __cron, __file, __link: Improve manpages (Matthias Stecher)
 	* Explorer machine_type: Add support for FreeBSD and OpenBSD, and simplify Linux code (Evil Ham)
 	* Type __ssh_authorized_key, __ssh_authorized_keys: Improve manpages (Evil Ham)
+	* Type __ssh_authorized_keys: Fix bug where --option was not multiple (Evil Ham)
 
 6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)

From bf25a18a048ae3e3d1fd4a450d77fe516c18c861 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 22 May 2020 02:31:38 +0300
Subject: [PATCH 258/743] [__group] fix --gid on freebsd

---
 cdist/conf/type/__group/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__group/gencode-remote b/cdist/conf/type/__group/gencode-remote
index 6091c548..ff63e218 100755
--- a/cdist/conf/type/__group/gencode-remote
+++ b/cdist/conf/type/__group/gencode-remote
@@ -88,7 +88,7 @@ if [ "$state" = "present" ]; then
          fi
       done
       if [ "$os" = "freebsd" ]; then
-         echo pw groupadd "$@" "$name"
+         echo pw groupadd "$name" "$@"
       else
          echo groupadd "$@" "$name"
       fi

From 3bcbd952698cd5a4b8333551660ba308e4f30d7b Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 22 May 2020 02:36:49 +0300
Subject: [PATCH 259/743] [__motd] debian|ubuntu|devuan use /etc/motd

---
 cdist/conf/type/__motd/gencode-remote | 7 -------
 cdist/conf/type/__motd/manifest       | 4 ----
 2 files changed, 11 deletions(-)

diff --git a/cdist/conf/type/__motd/gencode-remote b/cdist/conf/type/__motd/gencode-remote
index 738ea834..cb7bfc84 100755
--- a/cdist/conf/type/__motd/gencode-remote
+++ b/cdist/conf/type/__motd/gencode-remote
@@ -22,13 +22,6 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-    debian|ubuntu|devuan)
-
-        # Debian and Ubuntu need to be updated,
-        # as seen in /etc/init.d/bootlogs
-        echo "uname -snrvm > /var/run/motd"
-        echo "cat /etc/motd.tail >> /var/run/motd"
-    ;;
     freebsd)
         # FreeBSD only updates /etc/motd on boot,
 	# as seen in /etc/rc.d/motd
diff --git a/cdist/conf/type/__motd/manifest b/cdist/conf/type/__motd/manifest
index ded734d7..b8f74ebf 100755
--- a/cdist/conf/type/__motd/manifest
+++ b/cdist/conf/type/__motd/manifest
@@ -33,10 +33,6 @@ os=$(cat "$__global/explorer/os")
 
 
 case "$os" in
-   debian|ubuntu|devuan)
-      # Debian-based systems use /etc/motd.tail as a template
-      destination=/etc/motd.tail
-   ;;
    freebsd)
       # FreeBSD uses motd.template to prepend system information on boot
       # (this actually only applies starting with version 13,

From 226ed02c1c2556ed30e4f8aaf1946099bd590725 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 22 May 2020 10:09:05 +0200
Subject: [PATCH 260/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 782d9bd7..f0aa3aa7 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,8 @@ next:
 	* Explorer machine_type: Add support for FreeBSD and OpenBSD, and simplify Linux code (Evil Ham)
 	* Type __ssh_authorized_key, __ssh_authorized_keys: Improve manpages (Evil Ham)
 	* Type __ssh_authorized_keys: Fix bug where --option was not multiple (Evil Ham)
+	* Type __motd: Debian/Ubuntu/Devuan use /etc/motd (Ander Punnar)
+	* Type __group: Fix --gid on FreeBSD (Ander Punnar)
 
 6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)

From 66f4421089660fc0a083abe3d2c93470eacd55ce Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Fri, 22 May 2020 17:14:29 +0300
Subject: [PATCH 261/743] [docs] Fixed two typos in cdist.cfg.skeleton

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 configuration/cdist.cfg.skeleton | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton
index bfac9f5c..91c5ab02 100644
--- a/configuration/cdist.cfg.skeleton
+++ b/configuration/cdist.cfg.skeleton
@@ -26,7 +26,7 @@
 #
 # init_manifest
 #     Specify default initial manifest.
-# init_mainfest = <path-to-init-manifst>
+# init_manifest = <path-to-init-manifest>
 #
 # inventory_dir
 #     Specify inventory directory.

From 716d3554f3dd9a7cf83cb0fd4f706b6458cb13d8 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sat, 23 May 2020 00:48:35 +0300
Subject: [PATCH 262/743] [__user] fix user delete on freebsd

---
 cdist/conf/type/__user/gencode-remote | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__user/gencode-remote b/cdist/conf/type/__user/gencode-remote
index ee18c18f..41c3a57b 100755
--- a/cdist/conf/type/__user/gencode-remote
+++ b/cdist/conf/type/__user/gencode-remote
@@ -135,11 +135,19 @@ elif [ "$state" = "absent" ]; then
     if grep -q "^${name}:" "$__object/explorer/passwd"; then
         #user exists, but state != present, so delete it
         if [ -f "$__object/parameter/remove-home" ]; then
-            printf "userdel -r '%s' >/dev/null 2>&1\\n" "${name}"
-	    echo "userdel -r" >> "$__messages_out"
+            if [ "$os" = "freebsd" ]; then
+                printf "pw userdel '%s' -r >/dev/null 2>&1\\n" "${name}"
+            else
+                printf "userdel -r '%s' >/dev/null 2>&1\\n" "${name}"
+            fi
+            echo "userdel -r" >> "$__messages_out"
         else
-            printf "userdel '%s' >/dev/null 2>&1\\n" "${name}"
-	    echo "userdel" >> "$__messages_out"
+            if [ "$os" = "freebsd" ]; then
+                printf "pw userdel '%s' >/dev/null 2>&1\\n" "${name}"
+            else
+                printf "userdel '%s' >/dev/null 2>&1\\n" "${name}"
+            fi
+            echo "userdel" >> "$__messages_out"
         fi
     fi
 else

From 6ba73c4be6a57007c035f748c3273dd7a7041992 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 23 May 2020 19:03:13 +0200
Subject: [PATCH 263/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index f0aa3aa7..6746bec6 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,6 +10,8 @@ next:
 	* Type __ssh_authorized_keys: Fix bug where --option was not multiple (Evil Ham)
 	* Type __motd: Debian/Ubuntu/Devuan use /etc/motd (Ander Punnar)
 	* Type __group: Fix --gid on FreeBSD (Ander Punnar)
+	* Configuration: Fix typos in cdist.cfg.skeleton (Jaak Ristioja)
+	* Type __user: Fix user deletion on FreeBSD (Ander Punnar)
 
 6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)

From f4e1bbc87e9127ed4ac1789c700e1cbee02ff61c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sun, 24 May 2020 01:10:56 +0200
Subject: [PATCH 264/743] Fix multiple log lines

Fixes #813.
---
 cdist/log.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/log.py b/cdist/log.py
index 5d431130..2d0bef0b 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -64,6 +64,7 @@ class DefaultLog(logging.Logger):
 
     def __init__(self, name):
         super().__init__(name)
+        self.propagate = False
 
         formatter = logging.Formatter(self.FORMAT)
 

From ba64971a567722c29cfe5a93c7ccc7e03bbf7940 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sun, 24 May 2020 17:05:01 +0200
Subject: [PATCH 265/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 6746bec6..5a2f905a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,7 @@ next:
 	* Type __group: Fix --gid on FreeBSD (Ander Punnar)
 	* Configuration: Fix typos in cdist.cfg.skeleton (Jaak Ristioja)
 	* Type __user: Fix user deletion on FreeBSD (Ander Punnar)
+	* Core: Fix double log lines (Darko Poljak)
 
 6.5.5: 2020-05-01
 	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)

From abac79d4a50857c81e12660aaa13424d371f2137 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 25 May 2020 11:16:48 +0200
Subject: [PATCH 266/743] Release 6.5.6

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 5a2f905a..ad008cb2 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)
 	* Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius)
 	* Types __cron, __file, __link: Improve manpages (Matthias Stecher)

From 29c0180204beced2813f2d1ec2b50ee069d205c6 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 28 May 2020 23:31:13 +0300
Subject: [PATCH 267/743] [__ssh_authorized_keys] add --remove-unknown
 parameter

---
 .../type/__ssh_authorized_keys/explorer/keys  |  8 ++++++
 cdist/conf/type/__ssh_authorized_keys/man.rst |  9 +++++++
 .../conf/type/__ssh_authorized_keys/manifest  | 27 ++++++++++++++++++-
 .../__ssh_authorized_keys/parameter/boolean   |  1 +
 4 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100755 cdist/conf/type/__ssh_authorized_keys/explorer/keys

diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
new file mode 100755
index 00000000..b515c630
--- /dev/null
+++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+file="$( . "$__type_explorer/file" )"
+
+if [ -f "$file" ]
+then
+    cat "$file"
+fi
diff --git a/cdist/conf/type/__ssh_authorized_keys/man.rst b/cdist/conf/type/__ssh_authorized_keys/man.rst
index 93357b1d..dac6adeb 100644
--- a/cdist/conf/type/__ssh_authorized_keys/man.rst
+++ b/cdist/conf/type/__ssh_authorized_keys/man.rst
@@ -60,6 +60,9 @@ nofile
    Don't manage existence, ownership and permissions of the the authorized_keys
    file.
 
+remove-unknown
+   Remove undefined keys.
+
 
 EXAMPLES
 --------
@@ -70,6 +73,12 @@ EXAMPLES
     __ssh_authorized_keys root \
        --key "$(cat ~/.ssh/id_rsa.pub)"
 
+    # same as above, but make sure your key is only key in
+    # root's authorized_keys file
+    __ssh_authorized_keys root \
+       --key "$(cat ~/.ssh/id_rsa.pub)" \
+       --remove-unknown
+
     # allow key to login as user-name
     __ssh_authorized_keys user-name \
        --key "ssh-rsa AXYZAAB3NzaC1yc2..."
diff --git a/cdist/conf/type/__ssh_authorized_keys/manifest b/cdist/conf/type/__ssh_authorized_keys/manifest
index b9f0582e..b319316b 100755
--- a/cdist/conf/type/__ssh_authorized_keys/manifest
+++ b/cdist/conf/type/__ssh_authorized_keys/manifest
@@ -55,8 +55,12 @@ _cksum() {
    echo "$1" | cksum | cut -d' ' -f 1
 }
 
+_type_and_key() {
+   echo "$1" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }'
+}
+
 while read -r key; do
-   type_and_key="$(echo "$key" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }')"
+   type_and_key="$( _type_and_key "$key" )"
    object_id="$(_cksum "$file")-$(_cksum "$type_and_key")"
    set -- "$object_id"
    set -- "$@" --file "$file"
@@ -72,3 +76,24 @@ while read -r key; do
    # Ensure __ssh_authorized_key does not read stdin
    __ssh_authorized_key "$@" < /dev/null
 done < "$__object/parameter/key"
+
+if [ -f "$__object/parameter/remove-unknown" ] &&
+    [ -s "$__object/explorer/keys" ]
+then
+    while read -r key
+    do
+        type_and_key="$( _type_and_key "$key" )"
+
+        if grep -Fq "$type_and_key" "$__object/parameter/key"
+        then
+            continue
+        fi
+
+        __ssh_authorized_key "remove-$( _cksum "$file$key" )" \
+            --file "$file" \
+            --key "$key" \
+            --state absent \
+                < /dev/null
+    done \
+        < "$__object/explorer/keys"
+fi
diff --git a/cdist/conf/type/__ssh_authorized_keys/parameter/boolean b/cdist/conf/type/__ssh_authorized_keys/parameter/boolean
index 4bb126fe..7388fed5 100644
--- a/cdist/conf/type/__ssh_authorized_keys/parameter/boolean
+++ b/cdist/conf/type/__ssh_authorized_keys/parameter/boolean
@@ -1,2 +1,3 @@
 noparent
 nofile
+remove-unknown

From b354ea6e9430816bd62168c4c8e7b936ad4a4885 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 30 May 2020 10:49:13 +0200
Subject: [PATCH 268/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index ad008cb2..76b5edfa 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)
+
 6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)
 	* Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius)

From 988190363aeff5fba9be80355c318e1ff372eda9 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 30 May 2020 15:10:13 +0200
Subject: [PATCH 269/743] Resolve shellcheck SC1090

---
 cdist/conf/type/__ssh_authorized_keys/explorer/keys | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
index b515c630..cec25746 100755
--- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys
+++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
@@ -1,5 +1,6 @@
 #!/bin/sh -e
 
+# shellcheck disable=SC1090
 file="$( . "$__type_explorer/file" )"
 
 if [ -f "$file" ]

From ba77ea9edcf2d99c6edd76adaa83017821babae3 Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Mon, 1 Jun 2020 19:11:58 +0200
Subject: [PATCH 270/743] [UX] Add option to enable LogLevel-based coloured
 output.

This makes it easier for new and experienced users to run cdist with higher
verbosity levels, both to know that things are working as expected and to debug
issues.

Documentation has been modified accordingly and default behaviour is not
changed.
---
 cdist/argparse.py                    | 31 ++++++++++++++++--
 cdist/configuration.py               | 11 +++++++
 cdist/core/manifest.py               |  1 +
 cdist/emulator.py                    |  3 ++
 cdist/log.py                         | 26 ++++++++++++++-
 cdist/test/configuration/__init__.py | 28 +++++++++++++++++
 configuration/cdist.cfg.skeleton     |  6 ++++
 docs/src/cdist-reference.rst.sh      |  5 +++
 docs/src/man1/cdist.rst              | 47 +++++++++++++++++++---------
 9 files changed, 140 insertions(+), 18 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 611c484a..c30e2030 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -5,6 +5,7 @@ import logging
 import collections
 import functools
 import cdist.configuration
+import cdist.log
 import cdist.preos
 import cdist.info
 
@@ -88,6 +89,13 @@ def check_lower_bounded_int(value, lower_bound, name):
     return val
 
 
+def colored_output_type(val):
+    boolean_states = cdist.configuration.ColoredOutputOption.BOOLEAN_STATES
+    if val not in boolean_states.keys():
+        raise argparse.ArgumentError()
+    return boolean_states[val]
+
+
 def get_parsers():
     global parser
 
@@ -125,6 +133,15 @@ def get_parsers():
                   'value.'),
             action='count', default=None)
 
+    parser['colored_output'] = argparse.ArgumentParser(add_help=False)
+    parser['colored_output'].add_argument(
+            '--colors',
+            help='Use a colored output for different log levels.'
+                 'It can be a boolean or "auto" (default) which enables this '
+                 'feature if stdout is a tty and disables it otherwise.',
+            action='store', dest='colored_output', required=False,
+            type=colored_output_type)
+
     parser['beta'] = argparse.ArgumentParser(add_help=False)
     parser['beta'].add_argument(
            '-b', '--beta',
@@ -283,6 +300,7 @@ def get_parsers():
             'host', nargs='*', help='Host(s) to operate on.')
     parser['config'] = parser['sub'].add_parser(
             'config', parents=[parser['loglevel'], parser['beta'],
+                               parser['colored_output'],
                                parser['common'],
                                parser['config_main'],
                                parser['inventory_common'],
@@ -301,6 +319,7 @@ def get_parsers():
 
     parser['add-host'] = parser['invsub'].add_parser(
             'add-host', parents=[parser['loglevel'], parser['beta'],
+                                 parser['colored_output'],
                                  parser['common'],
                                  parser['inventory_common']])
     parser['add-host'].add_argument(
@@ -315,6 +334,7 @@ def get_parsers():
 
     parser['add-tag'] = parser['invsub'].add_parser(
             'add-tag', parents=[parser['loglevel'], parser['beta'],
+                                parser['colored_output'],
                                 parser['common'],
                                 parser['inventory_common']])
     parser['add-tag'].add_argument(
@@ -346,6 +366,7 @@ def get_parsers():
 
     parser['del-host'] = parser['invsub'].add_parser(
             'del-host', parents=[parser['loglevel'], parser['beta'],
+                                 parser['colored_output'],
                                  parser['common'],
                                  parser['inventory_common']])
     parser['del-host'].add_argument(
@@ -363,6 +384,7 @@ def get_parsers():
 
     parser['del-tag'] = parser['invsub'].add_parser(
             'del-tag', parents=[parser['loglevel'], parser['beta'],
+                                parser['colored_output'],
                                 parser['common'],
                                 parser['inventory_common']])
     parser['del-tag'].add_argument(
@@ -398,6 +420,7 @@ def get_parsers():
 
     parser['list'] = parser['invsub'].add_parser(
             'list', parents=[parser['loglevel'], parser['beta'],
+                             parser['colored_output'],
                              parser['common'],
                              parser['inventory_common']])
     parser['list'].add_argument(
@@ -430,7 +453,7 @@ def get_parsers():
 
     # Shell
     parser['shell'] = parser['sub'].add_parser(
-            'shell', parents=[parser['loglevel']])
+            'shell', parents=[parser['loglevel'], parser['colored_output']])
     parser['shell'].add_argument(
             '-s', '--shell',
             help=('Select shell to use, defaults to current shell. Used shell'
@@ -495,9 +518,13 @@ def parse_and_configure(argv, singleton=True):
 
     log = logging.getLogger("cdist")
 
+    config = cfg.get_config()
+    if config.get('GLOBAL', {}).get('colored_output', False):
+        cdist.log.ColorFormatter.USE_COLORS = True
+
     log.verbose("version %s" % cdist.VERSION)
     log.trace('command line args: {}'.format(cfg.command_line_args))
-    log.trace('configuration: {}'.format(cfg.get_config()))
+    log.trace('configuration: {}'.format(config))
     log.trace('configured args: {}'.format(args))
 
     check_beta(vars(args))
diff --git a/cdist/configuration.py b/cdist/configuration.py
index 1011a382..6f07c27f 100644
--- a/cdist/configuration.py
+++ b/cdist/configuration.py
@@ -27,6 +27,7 @@ import cdist.argparse
 import re
 import multiprocessing
 import logging
+import sys
 
 
 class Singleton(type):
@@ -47,6 +48,7 @@ _VERBOSITY_VALUES = (
 _ARCHIVING_VALUES = (
     'tar', 'tgz', 'tbz2', 'txz', 'none',
 )
+_COLORED_OUTPUT_DEFAULT = sys.stdout.isatty()
 
 
 class OptionBase:
@@ -246,9 +248,15 @@ class LogLevelOption(OptionBase):
         return VerbosityOption().translate(val)
 
 
+class ColoredOutputOption(BooleanOption):
+    BOOLEAN_STATES = dict(configparser.ConfigParser.BOOLEAN_STATES,
+                          auto=_COLORED_OUTPUT_DEFAULT)
+
+
 _ARG_OPTION_MAPPING = {
     'beta': 'beta',
     'cache_path_pattern': 'cache_path_pattern',
+    'colored_output': 'colored_output',
     'conf_dir': 'conf_dir',
     'manifest': 'init_manifest',
     'out_path': 'out_path',
@@ -294,6 +302,7 @@ class Configuration(metaclass=Singleton):
             'remote_shell': StringOption('remote_shell'),
             'cache_path_pattern': StringOption('cache_path_pattern'),
             'conf_dir': ConfDirOption(),
+            'colored_output': ColoredOutputOption('colored_output'),
             'init_manifest': StringOption('init_manifest'),
             'out_path': StringOption('out_path'),
             'remote_out_path': StringOption('remote_out_path'),
@@ -319,6 +328,7 @@ class Configuration(metaclass=Singleton):
         'CDIST_REMOTE_COPY': 'remote_copy',
         'CDIST_INVENTORY_DIR': 'inventory_dir',
         'CDIST_CACHE_PATH_PATTERN': 'cache_path_pattern',
+        'CDIST_COLORED_OUTPUT': 'colored_output',
         '__cdist_log_level': 'verbosity',
     }
     ENV_VAR_BOOLEAN_OPTIONS = ('CDIST_BETA', )
@@ -332,6 +342,7 @@ class Configuration(metaclass=Singleton):
     }
     REQUIRED_DEFAULT_CONFIG_VALUES = {
         'GLOBAL': {
+            'colored_output': _COLORED_OUTPUT_DEFAULT,
             'verbosity': 0,
         },
     }
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 8aeaf860..32520e49 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -119,6 +119,7 @@ class Manifest(object):
             '__cdist_log_level': util.log_level_env_var_val(self.log),
             '__cdist_log_level_name': util.log_level_name_env_var_val(
                 self.log),
+            '__cdist_colored_log': str(cdist.log.ColorFormatter.USE_COLORS),
         }
 
         if dry_run:
diff --git a/cdist/emulator.py b/cdist/emulator.py
index 4800e2a3..87c9fe12 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -129,6 +129,9 @@ class Emulator(object):
             # if invalid __cdist_log_level value
             logging.root.setLevel(logging.WARNING)
 
+        colored_log = self.env.get('__cdist_colored_log', 'False')
+        cdist.log.ColorFormatter.USE_COLORS = colored_log == 'True'
+
         self.log = logging.getLogger(self.target_host[0])
 
     def commandline(self):
diff --git a/cdist/log.py b/cdist/log.py
index 2d0bef0b..5f2d8f53 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -50,6 +50,30 @@ def _trace(msg, *args, **kwargs):
 logging.trace = _trace
 
 
+class ColorFormatter(logging.Formatter):
+    USE_COLORS = False
+    RESET = '\033[0m'
+    COLOR_MAP = {
+        'ERROR': '\033[0;31m',
+        'WARNING': '\033[0;33m',
+        'INFO': '\033[0;94m',
+        'VERBOSE': '\033[0;34m',
+        'DEBUG': '\033[0;90m',
+        'TRACE': '\033[0;37m',
+    }
+
+    def __init__(self, msg):
+        super().__init__(msg)
+
+    def format(self, record):
+        msg = super().format(record)
+        if self.USE_COLORS:
+            color = self.COLOR_MAP.get(record.levelname)
+            if color:
+                msg = color + msg + self.RESET
+        return msg
+
+
 class DefaultLog(logging.Logger):
 
     FORMAT = '%(levelname)s: %(message)s'
@@ -66,7 +90,7 @@ class DefaultLog(logging.Logger):
         super().__init__(name)
         self.propagate = False
 
-        formatter = logging.Formatter(self.FORMAT)
+        formatter = ColorFormatter(self.FORMAT)
 
         self.addFilter(self)
 
diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py
index 182868a6..07a73bda 100644
--- a/cdist/test/configuration/__init__.py
+++ b/cdist/test/configuration/__init__.py
@@ -28,10 +28,12 @@ import argparse
 from cdist import test
 import cdist.argparse as cap
 import logging
+import sys
 
 my_dir = op.abspath(op.dirname(__file__))
 fixtures = op.join(my_dir, 'fixtures')
 interpolation_config_file = op.join(fixtures, "interpolation-test.cfg")
+colored_output_default = sys.stdout.isatty()
 
 
 def newConfigParser():
@@ -153,6 +155,7 @@ class ConfigurationTestCase(test.CdistTestCase):
             'remote_shell': '/bin/sh',
             'inventory_dir': '',
             'cache_path_pattern': '',
+            'colored_output': colored_output_default,
             'conf_dir': '',
             'init_manifest': '',
             'out_path': '',
@@ -184,6 +187,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/bin/sh',
                 'inventory_dir': None,
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': None,
                 'init_manifest': None,
                 'out_path': None,
@@ -390,6 +394,7 @@ class ConfigurationTestCase(test.CdistTestCase):
         args = argparse.Namespace()
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'verbosity': 0,
             },
         }
@@ -440,6 +445,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/bin/sh',
                 'inventory_dir': None,
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': None,
                 'init_manifest': None,
                 'out_path': None,
@@ -515,6 +521,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': '/var/db/cdist/inventory',
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': ['/opt/cdist', ],
                 'init_manifest': None,
                 'out_path': None,
@@ -556,6 +563,7 @@ class ConfigurationTestCase(test.CdistTestCase):
             'remote_shell': '/bin/sh',
             'inventory_dir': '',
             'cache_path_pattern': '',
+            'colored_output': colored_output_default,
             'conf_dir': '',
             'init_manifest': '',
             'out_path': '',
@@ -579,6 +587,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': None,
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': [
                     '/opt/cdist/conf',
                     '/usr/local/share/cdist/conf',
@@ -623,6 +632,7 @@ class ConfigurationTestCase(test.CdistTestCase):
             'remote_shell': '/bin/sh',
             'inventory_dir': '',
             'cache_path_pattern': '',
+            'colored_output': colored_output_default,
             'conf_dir': '',
             'init_manifest': '',
             'out_path': '',
@@ -645,6 +655,7 @@ class ConfigurationTestCase(test.CdistTestCase):
             'local_shell': '/usr/bin/sh',
             'remote_shell': '/usr/bin/sh',
             'inventory_dir': '/var/db/cdist/inventory',
+            'colored_output': colored_output_default,
             'conf_dir': '/opt/cdist',
             'remote_copy': 'myscp',
             'remote_exec': 'myexec',
@@ -663,6 +674,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': '/var/db/cdist/inventory',
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': [
                     '/opt/cdist/conf',
                     '/usr/local/share/cdist/conf',
@@ -694,6 +706,7 @@ class ConfigurationTestCase(test.CdistTestCase):
         }
         expected_config = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'verbosity': 0,
             },
         }
@@ -767,6 +780,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': '/opt/sysadmin/cdist/inventory',
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': [
                     '/opt/cdist/conf',
                     '/usr/local/share/cdist/conf',
@@ -865,6 +879,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': '/var/db/cdist/inventory',
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': [
                     '/opt/conf/cdist',
                 ],
@@ -964,6 +979,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': '/var/db/cdist/inventory',
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': [
                     '/opt/conf/cdist',
                 ],
@@ -1063,6 +1079,7 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': '/var/db/cdist/inventory',
                 'cache_path_pattern': None,
+                'colored_output': colored_output_default,
                 'conf_dir': [
                     '/opt/conf/cdist',
                 ],
@@ -1095,6 +1112,7 @@ class ConfigurationTestCase(test.CdistTestCase):
             'beta': True,
             'inventory_dir': '/var/db/cdist/inventory',
             'cache_path_pattern': None,
+            'colored_output': colored_output_default,
             'conf_dir': [
                 '/opt/conf/cdist',
             ],
@@ -1125,6 +1143,7 @@ class ConfigurationTestCase(test.CdistTestCase):
         expected_config_dict = {
             'GLOBAL': {
                 'inventory_dir': None,
+                'colored_output': colored_output_default,
                 'conf_dir': None,
                 'verbosity': 0,
             },
@@ -1148,6 +1167,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'verbosity': cap.VERBOSE_DEBUG,
             },
         }
@@ -1185,6 +1205,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'save_output_streams': True,
                 'verbosity': 0,
             },
@@ -1213,6 +1234,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'save_output_streams': False,
                 'verbosity': 0,
             },
@@ -1241,6 +1263,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'save_output_streams': False,
                 'verbosity': 0,
             },
@@ -1269,6 +1292,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'save_output_streams': False,
                 'verbosity': 0,
             },
@@ -1308,6 +1332,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'timestamp': True,
                 'verbosity': 0,
             },
@@ -1336,6 +1361,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'timestamp': True,
                 'verbosity': 0,
             },
@@ -1364,6 +1390,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'timestamp': False,
                 'verbosity': 0,
             },
@@ -1392,6 +1419,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 
         expected_config_dict = {
             'GLOBAL': {
+                'colored_output': colored_output_default,
                 'timestamp': False,
                 'verbosity': 0,
             },
diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton
index 91c5ab02..f2a09064 100644
--- a/configuration/cdist.cfg.skeleton
+++ b/configuration/cdist.cfg.skeleton
@@ -13,6 +13,12 @@
 #     Specify cache path pattern.
 # cache_path_pattern = %h
 #
+# colored_output
+#     Use a colored output for different log levels.
+#     It can be a boolean or 'auto' (default) which enables this feature if
+#     stdout is a tty and disables it otherwise.
+# colored_output = auto
+#
 # conf_dir
 #     List of configuration directories separated with the character conventionally
 #     used by the operating system to separate search path components (as in PATH),
diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh
index e77d98f6..3b997f63 100755
--- a/docs/src/cdist-reference.rst.sh
+++ b/docs/src/cdist-reference.rst.sh
@@ -344,6 +344,11 @@ CDIST_INVENTORY_DIR
 CDIST_BETA
     Enable beta functionalities.
 
+CDIST_COLORED_OUTPUT
+    Use a colored output for different log levels.
+    It can be a boolean or 'auto' (default) which enables this feature if
+    stdout is a tty and disables it otherwise.
+
 CDIST_CACHE_PATH_PATTERN
     Custom cache path pattern.
 eof
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 38248821..4c34c4b7 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -15,8 +15,9 @@ SYNOPSIS
 
     cdist banner [-h] [-l LOGLEVEL] [-q] [-v]
 
-    cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4]
-                 [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
+    cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b]
+                 [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6]
+                 [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
                  [-j [JOBS]] [-n] [-o OUT_PATH] [-P]
                  [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
                  [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
@@ -24,8 +25,9 @@ SYNOPSIS
                  [-p [HOST_MAX]] [-s] [-t]
                  [host [host ...]]
 
-    cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4]
-                  [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
+    cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b]
+                  [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6]
+                  [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
                   [-j [JOBS]] [-n] [-o OUT_PATH] [-P]
                   [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
                   [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
@@ -36,26 +38,29 @@ SYNOPSIS
     cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ...
 
     cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                             [-g CONFIG_FILE] [-I INVENTORY_DIR]
-                             [-f HOSTFILE]
+                             [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
+                             [-I INVENTORY_DIR] [-f HOSTFILE]
                              [host [host ...]]
 
     cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                            [-g CONFIG_FILE] [-I INVENTORY_DIR]
-                            [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
+                            [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
+                            [-I INVENTORY_DIR] [-f HOSTFILE] [-T TAGFILE]
+                            [-t TAGLIST]
                             [host [host ...]]
 
     cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                             [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a]
-                             [-f HOSTFILE]
+                             [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
+                             [-I INVENTORY_DIR] [-a] [-f HOSTFILE]
                              [host [host ...]]
 
     cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                            [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a]
-                            [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
+                            [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
+                            [-I INVENTORY_DIR] [-a] [-f HOSTFILE]
+                            [-T TAGFILE] [-t TAGLIST]
                             [host [host ...]]
 
-    cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE]
+    cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b]
+                         [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
                          [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t]
                          [host [host ...]]
 
@@ -84,9 +89,11 @@ SYNOPSIS
                                        [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
                                        target_dir
 
-    cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL]
+    cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [--colors COLORED_OUTPUT]
+                [-s SHELL]
 
-    cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern]
+    cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t]
+               [pattern]
 
 
 DESCRIPTION
@@ -104,6 +111,11 @@ All commands accept the following options:
 **-h, --help**
     Show the help screen.
 
+**--colors COLORED_OUTPUT**
+    Use a colored output for different log levels.It can
+    be a boolean or "auto" (default) which enables this
+    feature if stdout is a tty and disables it otherwise.
+
 **-l LOGLEVEL, --log-level LOGLEVEL**
     Set the specified verbosity level. The levels, in
     order from the lowest to the highest, are: ERROR (-1),
@@ -893,6 +905,11 @@ CDIST_BETA
 CDIST_CACHE_PATH_PATTERN
     Custom cache path pattern.
 
+CDIST_COLORED_OUTPUT
+    Use a colored output for different log levels.
+    It can be a boolean or 'auto' (default) which enables this feature if
+    stdout is a tty and disables it otherwise.
+
 CDIST_CONFIG_FILE
     Custom configuration file.
 

From 9a4e3488c2104ad5562b5ab9aad8120969df73a8 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 1 Jun 2020 19:17:02 +0200
Subject: [PATCH 271/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 76b5edfa..5408c54f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)
+	* Core: Support colored log output (Evil Ham)
 
 6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)

From 55ebd1a4c56e788b18911f273d931f950a0c5271 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 1 Jun 2020 20:22:40 +0200
Subject: [PATCH 272/743] Fix man build failure in newer sphinx versions

---
 cdist/sphinxext/manpage.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/cdist/sphinxext/manpage.py b/cdist/sphinxext/manpage.py
index 135fe22e..1f8ac4f6 100644
--- a/cdist/sphinxext/manpage.py
+++ b/cdist/sphinxext/manpage.py
@@ -7,6 +7,7 @@ from docutils.io import FileOutput
 from os import path
 from sphinx.util.nodes import inline_all_toctrees
 from sphinx import addnodes
+from sphinx.util import logging
 
 """
     Extension based on sphinx builtin manpage.
@@ -15,6 +16,9 @@ from sphinx import addnodes
 """
 
 
+logger = logging.getLogger(__name__)
+
+
 class ManualPageTranslator(sphinx.writers.manpage.ManualPageTranslator):
 
     def header(self):
@@ -28,7 +32,7 @@ class ManualPageWriter(sphinx.writers.manpage.ManualPageWriter):
     def __init__(self, builder):
         super().__init__(builder)
         self.translator_class = (
-            self.builder.translator_class or ManualPageTranslator)
+            self.builder.get_translator_class() or ManualPageTranslator)
 
 
 class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
@@ -43,7 +47,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
             components=(docwriter,),
             read_config_files=True).get_default_values()
 
-        self.info(bold('writing... '), nonl=True)
+        logger.info(bold('writing... '), nonl=True)
 
         for info in self.config.man_pages:
             docname, name, description, authors, section = info
@@ -54,7 +58,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
                     authors = []
 
             targetname = '%s.%s' % (name, section)
-            self.info(darkgreen(targetname) + ' { ', nonl=True)
+            logger.info(darkgreen(targetname) + ' { ', nonl=True)
             destination = FileOutput(
                 destination_path=path.join(self.outdir, targetname),
                 encoding='utf-8')
@@ -63,7 +67,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
             docnames = set()
             largetree = inline_all_toctrees(self, docnames, docname, tree,
                                             darkgreen, [docname])
-            self.info('} ', nonl=True)
+            logger.info('} ', nonl=True)
             self.env.resolve_references(largetree, docname, self)
             # remove pending_xref nodes
             for pendingnode in largetree.traverse(addnodes.pending_xref):
@@ -76,7 +80,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
             largetree.settings.section = section
 
             docwriter.write(largetree, destination)
-        self.info()
+        logger.info("")
 
 
 def setup(app):

From 48d66b014351982e70aa3e065ffc1ed344660bab Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 1 Jun 2020 22:25:15 +0200
Subject: [PATCH 273/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 5408c54f..5b57e98b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)
 	* Core: Support colored log output (Evil Ham)
+	* Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak)
 
 6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)

From 6a611e556a54a60952a833989a5a0fca79011217 Mon Sep 17 00:00:00 2001
From: Daniel Heule <nerve@ipwar.ch>
Date: Tue, 2 Jun 2020 13:40:21 +0200
Subject: [PATCH 274/743] fix os explorer for sles15

---
 cdist/conf/explorer/os | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os
index 563fa4cf..2d2aede6 100755
--- a/cdist/conf/explorer/os
+++ b/cdist/conf/explorer/os
@@ -143,6 +143,11 @@ case "$uname_s" in
 esac
 
 if [ -f /etc/os-release ]; then
+   # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse
+   if grep -q ^ID_LIKE=\"suse\" /etc/os-release 2>/dev/null; then
+      echo suse
+      exit 0
+   fi
    # already lowercase, according to:
    # https://www.freedesktop.org/software/systemd/man/os-release.html
    awk -F= '/^ID=/ { if ($2 ~ /^'"'"'(.*)'"'"'$/ || $2 ~ /^"(.*)"$/) { print substr($2, 2, length($2) - 2) } else { print $2 } }' /etc/os-release

From 747c6b10762e18bbda24fbb876ac2e88f29dc70b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 21:45:04 +0200
Subject: [PATCH 275/743] Respect NO_COLOR environment variable

---
 cdist/argparse.py                    | 23 +++++++++--------------
 cdist/config.py                      |  1 +
 cdist/configuration.py               | 28 +++++++++++++++++++---------
 cdist/core/manifest.py               |  2 +-
 cdist/emulator.py                    |  4 ++--
 cdist/log.py                         | 19 ++++++++++---------
 cdist/test/configuration/__init__.py |  2 +-
 configuration/cdist.cfg.skeleton     |  6 ++++--
 8 files changed, 47 insertions(+), 38 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index c30e2030..0782654f 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -89,13 +89,6 @@ def check_lower_bounded_int(value, lower_bound, name):
     return val
 
 
-def colored_output_type(val):
-    boolean_states = cdist.configuration.ColoredOutputOption.BOOLEAN_STATES
-    if val not in boolean_states.keys():
-        raise argparse.ArgumentError()
-    return boolean_states[val]
-
-
 def get_parsers():
     global parser
 
@@ -140,7 +133,7 @@ def get_parsers():
                  'It can be a boolean or "auto" (default) which enables this '
                  'feature if stdout is a tty and disables it otherwise.',
             action='store', dest='colored_output', required=False,
-            type=colored_output_type)
+            choices=cdist.configuration.ColoredOutputOption.CHOICES)
 
     parser['beta'] = argparse.ArgumentParser(add_help=False)
     parser['beta'].add_argument(
@@ -501,7 +494,12 @@ def handle_loglevel(args):
     if hasattr(args, 'quiet') and args.quiet:
         args.verbose = _verbosity_level_off
 
-    logging.root.setLevel(_verbosity_level[args.verbose])
+    logging.getLogger().setLevel(_verbosity_level[args.verbose])
+
+
+def handle_log_colors(args):
+    if cdist.configuration.ColoredOutputOption.translate(args.colored_output):
+        cdist.log.DefaultLog.USE_COLORS = True
 
 
 def parse_and_configure(argv, singleton=True):
@@ -515,16 +513,13 @@ def parse_and_configure(argv, singleton=True):
         raise cdist.Error(str(e))
     # Loglevels are handled globally in here
     handle_loglevel(args)
+    handle_log_colors(args)
 
     log = logging.getLogger("cdist")
 
-    config = cfg.get_config()
-    if config.get('GLOBAL', {}).get('colored_output', False):
-        cdist.log.ColorFormatter.USE_COLORS = True
-
     log.verbose("version %s" % cdist.VERSION)
     log.trace('command line args: {}'.format(cfg.command_line_args))
-    log.trace('configuration: {}'.format(config))
+    log.trace('configuration: {}'.format(cfg.get_config()))
     log.trace('configured args: {}'.format(args))
 
     check_beta(vars(args))
diff --git a/cdist/config.py b/cdist/config.py
index 97cc1da6..b2d72f05 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -203,6 +203,7 @@ class Config(object):
                 cdist.log.setupParallelLogging()
         elif args.timestamp:
             cdist.log.setupTimestampingLogging()
+
         log = logging.getLogger("config")
 
         # No new child process if only one host at a time.
diff --git a/cdist/configuration.py b/cdist/configuration.py
index 6f07c27f..c0fbc063 100644
--- a/cdist/configuration.py
+++ b/cdist/configuration.py
@@ -48,7 +48,6 @@ _VERBOSITY_VALUES = (
 _ARCHIVING_VALUES = (
     'tar', 'tgz', 'tbz2', 'txz', 'none',
 )
-_COLORED_OUTPUT_DEFAULT = sys.stdout.isatty()
 
 
 class OptionBase:
@@ -249,8 +248,22 @@ class LogLevelOption(OptionBase):
 
 
 class ColoredOutputOption(BooleanOption):
-    BOOLEAN_STATES = dict(configparser.ConfigParser.BOOLEAN_STATES,
-                          auto=_COLORED_OUTPUT_DEFAULT)
+    CHOICES = tuple(configparser.ConfigParser.BOOLEAN_STATES) + ('auto',)
+    DEFAULT = 'auto'
+
+    def get_converter(self):
+        return self.translate
+
+    @staticmethod
+    def translate(val):
+        if 'NO_COLOR' in os.environ:
+            return False
+        elif isinstance(val, bool):
+            return val
+        elif val == 'auto':
+            return sys.stdout.isatty()
+        else:
+            return configparser.ConfigParser.BOOLEAN_STATES[val]
 
 
 _ARG_OPTION_MAPPING = {
@@ -337,12 +350,10 @@ class Configuration(metaclass=Singleton):
     }
 
     ARG_OPTION_MAPPING = _ARG_OPTION_MAPPING
-    ADJUST_ARG_OPTION_MAPPING = {
-       _ARG_OPTION_MAPPING[key]: key for key in _ARG_OPTION_MAPPING
-    }
+    ADJUST_ARG_OPTION_MAPPING = {v: k for k, v in _ARG_OPTION_MAPPING.items()}
     REQUIRED_DEFAULT_CONFIG_VALUES = {
         'GLOBAL': {
-            'colored_output': _COLORED_OUTPUT_DEFAULT,
+            'colored_output': 'auto',
             'verbosity': 0,
         },
     }
@@ -495,8 +506,7 @@ class Configuration(metaclass=Singleton):
             newconfig = self._read_config_file(config_file)
             self._update_config_dict(config, newconfig)
         # command line config file
-        if (self.args and 'config_file' in self.args and
-                self.args['config_file']):
+        if (self.args and self.args.get('config_file', None)):
             newconfig = self._read_config_file(self.args['config_file'])
             self._update_config_dict(config, newconfig)
         # command line
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 32520e49..8b833ff2 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -119,7 +119,7 @@ class Manifest(object):
             '__cdist_log_level': util.log_level_env_var_val(self.log),
             '__cdist_log_level_name': util.log_level_name_env_var_val(
                 self.log),
-            '__cdist_colored_log': str(cdist.log.ColorFormatter.USE_COLORS),
+            '__cdist_colored_log': str(self.log.USE_COLORS).lower(),
         }
 
         if dry_run:
diff --git a/cdist/emulator.py b/cdist/emulator.py
index 87c9fe12..4eaf2c93 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -129,8 +129,8 @@ class Emulator(object):
             # if invalid __cdist_log_level value
             logging.root.setLevel(logging.WARNING)
 
-        colored_log = self.env.get('__cdist_colored_log', 'False')
-        cdist.log.ColorFormatter.USE_COLORS = colored_log == 'True'
+        colored_log = self.env.get('__cdist_colored_log', 'false')
+        cdist.log.ColorFormatter.USE_COLORS = colored_log == 'true'
 
         self.log = logging.getLogger(self.target_host[0])
 
diff --git a/cdist/log.py b/cdist/log.py
index 5f2d8f53..19efebdb 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -51,7 +51,6 @@ logging.trace = _trace
 
 
 class ColorFormatter(logging.Formatter):
-    USE_COLORS = False
     RESET = '\033[0m'
     COLOR_MAP = {
         'ERROR': '\033[0;31m',
@@ -62,20 +61,19 @@ class ColorFormatter(logging.Formatter):
         'TRACE': '\033[0;37m',
     }
 
-    def __init__(self, msg):
-        super().__init__(msg)
+    def __init__(self, fmt):
+        super().__init__(fmt=fmt)
 
     def format(self, record):
         msg = super().format(record)
-        if self.USE_COLORS:
-            color = self.COLOR_MAP.get(record.levelname)
-            if color:
-                msg = color + msg + self.RESET
+        color = self.COLOR_MAP.get(record.levelname)
+        if color:
+            msg = color + msg + self.RESET
         return msg
 
 
 class DefaultLog(logging.Logger):
-
+    USE_COLORS = False
     FORMAT = '%(levelname)s: %(message)s'
 
     class StdoutFilter(logging.Filter):
@@ -90,7 +88,10 @@ class DefaultLog(logging.Logger):
         super().__init__(name)
         self.propagate = False
 
-        formatter = ColorFormatter(self.FORMAT)
+        if self.USE_COLORS:
+            formatter = ColorFormatter(self.FORMAT)
+        else:
+            formatter = logging.Formatter(self.FORMAT)
 
         self.addFilter(self)
 
diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py
index 07a73bda..5305b6d3 100644
--- a/cdist/test/configuration/__init__.py
+++ b/cdist/test/configuration/__init__.py
@@ -33,7 +33,7 @@ import sys
 my_dir = op.abspath(op.dirname(__file__))
 fixtures = op.join(my_dir, 'fixtures')
 interpolation_config_file = op.join(fixtures, "interpolation-test.cfg")
-colored_output_default = sys.stdout.isatty()
+colored_output_default = 'auto'
 
 
 def newConfigParser():
diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton
index f2a09064..0730201d 100644
--- a/configuration/cdist.cfg.skeleton
+++ b/configuration/cdist.cfg.skeleton
@@ -17,6 +17,8 @@
 #     Use a colored output for different log levels.
 #     It can be a boolean or 'auto' (default) which enables this feature if
 #     stdout is a tty and disables it otherwise.
+#     Colored output is always disabled if the NO_COLOR environment variable is
+#     defined (https://no-color.org/).
 # colored_output = auto
 #
 # conf_dir
@@ -51,7 +53,7 @@
 #
 # out_path
 #     Directory to save cdist output in.
-# out_path = 
+# out_path =
 #
 # parallel
 #     Process hosts in parallel. If -1 then the default, number of CPU's in
@@ -77,6 +79,6 @@
 # remote_shell = /bin/sh
 #
 # verbosity
-#     Set verbosity level. Valid values are: 
+#     Set verbosity level. Valid values are:
 #     ERROR, WARNING, INFO, VERBOSE, DEBUG, TRACE and OFF.
 # verbosity = INFO

From cdb0d2be413114a237a0f56c948cc5cd6cfc2b0b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 23:21:50 +0200
Subject: [PATCH 276/743] Patch tests

---
 cdist/configuration.py               | 4 ++++
 cdist/test/configuration/__init__.py | 9 ++++++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/cdist/configuration.py b/cdist/configuration.py
index c0fbc063..f68a5e09 100644
--- a/cdist/configuration.py
+++ b/cdist/configuration.py
@@ -266,6 +266,10 @@ class ColoredOutputOption(BooleanOption):
             return configparser.ConfigParser.BOOLEAN_STATES[val]
 
 
+ColoredOutputOption.DEFAULT = ColoredOutputOption.translate(
+    ColoredOutputOption.DEFAULT)
+
+
 _ARG_OPTION_MAPPING = {
     'beta': 'beta',
     'cache_path_pattern': 'cache_path_pattern',
diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py
index 5305b6d3..3fd24ca5 100644
--- a/cdist/test/configuration/__init__.py
+++ b/cdist/test/configuration/__init__.py
@@ -187,7 +187,8 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/bin/sh',
                 'inventory_dir': None,
                 'cache_path_pattern': None,
-                'colored_output': colored_output_default,
+                'colored_output': cc.ColoredOutputOption.translate(
+                    colored_output_default),
                 'conf_dir': None,
                 'init_manifest': None,
                 'out_path': None,
@@ -587,7 +588,8 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': None,
                 'cache_path_pattern': None,
-                'colored_output': colored_output_default,
+                'colored_output': cc.ColoredOutputOption.translate(
+                    colored_output_default),
                 'conf_dir': [
                     '/opt/cdist/conf',
                     '/usr/local/share/cdist/conf',
@@ -674,7 +676,8 @@ class ConfigurationTestCase(test.CdistTestCase):
                 'remote_shell': '/usr/bin/sh',
                 'inventory_dir': '/var/db/cdist/inventory',
                 'cache_path_pattern': None,
-                'colored_output': colored_output_default,
+                'colored_output': cc.ColoredOutputOption.translate(
+                    colored_output_default),
                 'conf_dir': [
                     '/opt/cdist/conf',
                     '/usr/local/share/cdist/conf',

From 89e48734bf13446024e9009bce7e77cbd4255840 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 5 Jun 2020 12:22:49 +0200
Subject: [PATCH 277/743] Let config file and command line override NO_COLOR
 envvar

---
 cdist/configuration.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/cdist/configuration.py b/cdist/configuration.py
index f68a5e09..a12940b0 100644
--- a/cdist/configuration.py
+++ b/cdist/configuration.py
@@ -256,14 +256,12 @@ class ColoredOutputOption(BooleanOption):
 
     @staticmethod
     def translate(val):
-        if 'NO_COLOR' in os.environ:
-            return False
-        elif isinstance(val, bool):
+        if isinstance(val, bool):
             return val
-        elif val == 'auto':
-            return sys.stdout.isatty()
-        else:
+        elif val in configparser.ConfigParser.BOOLEAN_STATES:
             return configparser.ConfigParser.BOOLEAN_STATES[val]
+        elif val == 'auto':
+            return 'NO_COLOR' not in os.environ and sys.stdout.isatty()
 
 
 ColoredOutputOption.DEFAULT = ColoredOutputOption.translate(

From 790c6efae9e379cb140e59469f07d2c3732f536c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 5 Jun 2020 13:56:30 +0200
Subject: [PATCH 278/743] Update colored output documentation

---
 docs/src/cdist-reference.rst.sh | 38 +++++++++++++++++++++++++++------
 docs/src/man1/cdist.rst         | 35 +++++++++++++++++-------------
 2 files changed, 51 insertions(+), 22 deletions(-)

diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh
index 3b997f63..a841862e 100755
--- a/docs/src/cdist-reference.rst.sh
+++ b/docs/src/cdist-reference.rst.sh
@@ -34,7 +34,7 @@ dest="$__cdist_abs_mydir/$filename"
 cd "$__cdist_abs_mydir"
 
 exec > "$dest"
-cat << eof 
+cat << eof
 Reference
 =========
 Variable, path and type reference for cdist
@@ -51,7 +51,7 @@ eof
     done
 )
 
-cat << eof 
+cat << eof
 
 Paths
 -----
@@ -187,13 +187,13 @@ usable within a object directory:
 
 files
     This directory is reserved for user data and will not be used
-    by cdist at any time. It can be used freely by the type 
+    by cdist at any time. It can be used freely by the type
     (for instance to store template results).
 changed
     This empty file exists in an object directory, if the object has
     code to be executed (either remote or local).
 stdin
-    This file exists and contains data, if data was provided on stdin 
+    This file exists and contains data, if data was provided on stdin
     when the type was called.
 
 
@@ -222,65 +222,89 @@ __cdist_log_level, __cdist_log_level_name
     | TRACE          | 5               |
     +----------------+-----------------+
 
+    Available for: initial manifest, explorer, type manifest, type explorer,
+    type gencode.
+__cdist_colored_log
+    whether or not cdist's log has colors enabled.
+    Is set to the string ``true`` if cdist's output is using colors,
+    otherwise the variable contains the string ``false``.
+
     Available for: initial manifest, explorer, type manifest, type explorer,
     type gencode.
 __cdist_dry_run
     Is set only when doing dry run (``-n`` flag).
+
     Available for: initial manifest, explorer, type manifest, type explorer,
     type gencode.
 __explorer
     Directory that contains all global explorers.
+
     Available for: initial manifest, explorer, type explorer, shell.
 __files
     Directory that contains content from the "files" subdirectories
     from the configuration directories.
+
     Available for: initial manifest, type manifest, type gencode, shell.
 __manifest
     Directory that contains the initial manifest.
+
     Available for: initial manifest, type manifest, shell.
 __global
     Directory that contains generic output like explorer.
+
     Available for: initial manifest, type manifest, type gencode, shell.
 __messages_in
     File to read messages from.
+
     Available for: initial manifest, type manifest, type gencode.
 __messages_out
     File to write messages.
+
     Available for: initial manifest, type manifest, type gencode.
 __object
     Directory that contains the current object.
+
     Available for: type manifest, type explorer, type gencode and code scripts.
 __object_id
     The type unique object id.
+
     Available for: type manifest, type explorer, type gencode and code scripts.
-    Note: The leading and the trailing "/" will always be stripped (caused by
-    the filesystem database and ensured by the core).
-    Note: Double slashes ("//") will not be fixed and result in an error.
+
+    | Note: The leading and the trailing "/" will always be stripped (caused by
+      the filesystem database and ensured by the core).
+    | Note: Double slashes ("//") will not be fixed and result in an error.
 __object_name
     The full qualified name of the current object.
+
     Available for: type manifest, type explorer, type gencode.
 __target_host
     The host we are deploying to. This is primary variable. It's content is
     literally the one user passed in.
+
     Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
 __target_hostname
     The hostname of host we are deploying to. This variable is derived from
     **__target_host** (using **socket.getaddrinfo(__target_host)** and then
     **socket.gethostbyaddr()**).
+
     Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
 __target_fqdn
     The fully qualified domain name of the host we are deploying to.
     This variable is derived from **__target_host**
     (using **socket.getfqdn()**).
+
     Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
 __target_host_tags
     Comma separated list of target host tags.
+
     Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
 __type
     Path to the current type.
+
     Available for: type manifest, type gencode.
 __type_explorer
     Directory that contains the type explorers.
+
     Available for: type explorer.
 
 Environment variables (for writing)
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 4c34c4b7..4def97b5 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -112,9 +112,13 @@ All commands accept the following options:
     Show the help screen.
 
 **--colors COLORED_OUTPUT**
-    Use a colored output for different log levels.It can
-    be a boolean or "auto" (default) which enables this
-    feature if stdout is a tty and disables it otherwise.
+    Colorize cdist's output. If enabled, cdist will use different colors for
+    different log levels.
+    COLORED_OUTPUT recognizes the boolean values 'yes'/'no', 'on'/'off',
+    'true'/'false', '1'/'0', and 'auto' (the default).
+
+    If the value is 'auto', colored output is enabled if stdout is a TTY
+    unless the NO_COLOR (https://no-color.org/) environment variable is defined.
 
 **-l LOGLEVEL, --log-level LOGLEVEL**
     Set the specified verbosity level. The levels, in
@@ -168,7 +172,7 @@ Install command is currently in beta.
 
 **-b, --beta**
     Enable beta functionality.
-    
+
 **-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN**
     Specify custom cache path pattern. If it is not set then
     default hostdir is used. For more info on format see
@@ -191,7 +195,7 @@ Install command is currently in beta.
 
 **-I INVENTORY_DIR, --inventory INVENTORY_DIR**
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if cdist 
+    directory is set up by the following rules: if cdist
     configuration resolves this value then specified
     directory is used, if HOME env var is set then
     ~/.cdit/inventory is used, otherwise distribution
@@ -306,7 +310,7 @@ Add host(s) to inventory database.
 
 **-I INVENTORY_DIR, --inventory INVENTORY_DIR**
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if cdist 
+    directory is set up by the following rules: if cdist
     configuration resolves this value then specified
     directory is used, if HOME env var is set then
     ~/.cdit/inventory is used, otherwise distribution
@@ -336,7 +340,7 @@ Add tag(s) to inventory database.
 
 **-I INVENTORY_DIR, --inventory INVENTORY_DIR**
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if cdist 
+    directory is set up by the following rules: if cdist
     configuration resolves this value then specified
     directory is used, if HOME env var is set then
     ~/.cdit/inventory is used, otherwise distribution
@@ -379,7 +383,7 @@ Delete host(s) from inventory database.
 
 **-I INVENTORY_DIR, --inventory INVENTORY_DIR**
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if cdist 
+    directory is set up by the following rules: if cdist
     configuration resolves this value then specified
     directory is used, if HOME env var is set then
     ~/.cdit/inventory is used, otherwise distribution
@@ -413,7 +417,7 @@ Delete tag(s) from inventory database.
 
 **-I INVENTORY_DIR, --inventory INVENTORY_DIR**
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if cdist 
+    directory is set up by the following rules: if cdist
     configuration resolves this value then specified
     directory is used, if HOME env var is set then
     ~/.cdit/inventory is used, otherwise distribution
@@ -460,7 +464,7 @@ List inventory database.
 
 **-I INVENTORY_DIR, --inventory INVENTORY_DIR**
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if cdist 
+    directory is set up by the following rules: if cdist
     configuration resolves this value then specified
     directory is used, if HOME env var is set then
     ~/.cdit/inventory is used, otherwise distribution
@@ -685,6 +689,9 @@ The possible keywords and their meanings are as follows:
 :strong:`cache_path_pattern`
     Specify cache path pattern.
 
+:strong:`colored_output`
+    Colorize cdist's output. cf. the :code:`--colors` option.
+
 :strong:`conf_dir`
     List of configuration directories separated with the character conventionally
     used by the operating system to separate search path components (as in PATH),
@@ -738,7 +745,7 @@ The possible keywords and their meanings are as follows:
     in the format: YYYYMMDDHHMMSS.us.
 
 :strong:`verbosity`
-    Set verbosity level. Valid values are: 
+    Set verbosity level. Valid values are:
     'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'.
 
 
@@ -770,7 +777,7 @@ cdist/preos
 NOTES
 -----
 cdist detects if host is specified by IPv6 address. If so then remote_copy
-command is executed with host address enclosed in square brackets 
+command is executed with host address enclosed in square brackets
 (see :strong:`scp`\ (1)).
 
 EXAMPLES
@@ -906,9 +913,7 @@ CDIST_CACHE_PATH_PATTERN
     Custom cache path pattern.
 
 CDIST_COLORED_OUTPUT
-    Use a colored output for different log levels.
-    It can be a boolean or 'auto' (default) which enables this feature if
-    stdout is a tty and disables it otherwise.
+    Colorize cdist's output. cf. the :code:`--colors` option.
 
 CDIST_CONFIG_FILE
     Custom configuration file.

From 7a570f8692a727662a6eda2c13f2b578b55864e4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 5 Jun 2020 13:59:17 +0200
Subject: [PATCH 279/743] [cdist.cfg.skeleton] Update colored_output
 documentation based on cdist(1)

---
 configuration/cdist.cfg.skeleton | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton
index 0730201d..9c66b51b 100644
--- a/configuration/cdist.cfg.skeleton
+++ b/configuration/cdist.cfg.skeleton
@@ -14,11 +14,12 @@
 # cache_path_pattern = %h
 #
 # colored_output
-#     Use a colored output for different log levels.
-#     It can be a boolean or 'auto' (default) which enables this feature if
-#     stdout is a tty and disables it otherwise.
-#     Colored output is always disabled if the NO_COLOR environment variable is
-#     defined (https://no-color.org/).
+#     Colorize cdist's output. If enabled, cdist will use different colors for
+#     different log levels.
+#     Recognized values are 'yes'/'no', 'on'/'off', 'true'/'false', '1'/'0',
+#     and 'auto'
+#     If the value is 'auto', colored output is enabled if stdout is a TTY
+#     unless the NO_COLOR (https://no-color.org/) environment variable is defined.
 # colored_output = auto
 #
 # conf_dir

From 23e66e08fab14f254c426c88271f461fe8499ba4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 6 Jun 2020 13:39:29 +0200
Subject: [PATCH 280/743] Restrict colored_output value to always/never/auto.

---
 cdist/argparse.py                | 7 +++----
 cdist/configuration.py           | 8 +++++---
 configuration/cdist.cfg.skeleton | 7 +++----
 docs/src/cdist-reference.rst.sh  | 6 +++---
 docs/src/man1/cdist.rst          | 6 +++---
 5 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 0782654f..85dba246 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -128,10 +128,9 @@ def get_parsers():
 
     parser['colored_output'] = argparse.ArgumentParser(add_help=False)
     parser['colored_output'].add_argument(
-            '--colors',
-            help='Use a colored output for different log levels.'
-                 'It can be a boolean or "auto" (default) which enables this '
-                 'feature if stdout is a tty and disables it otherwise.',
+            '--colors', metavar='WHEN',
+            help="Colorize cdist's output based on log level; "
+                 "WHEN is 'always', 'never', or 'auto'.",
             action='store', dest='colored_output', required=False,
             choices=cdist.configuration.ColoredOutputOption.CHOICES)
 
diff --git a/cdist/configuration.py b/cdist/configuration.py
index a12940b0..4ba43a7a 100644
--- a/cdist/configuration.py
+++ b/cdist/configuration.py
@@ -248,7 +248,7 @@ class LogLevelOption(OptionBase):
 
 
 class ColoredOutputOption(BooleanOption):
-    CHOICES = tuple(configparser.ConfigParser.BOOLEAN_STATES) + ('auto',)
+    CHOICES = ('always', 'never', 'auto')
     DEFAULT = 'auto'
 
     def get_converter(self):
@@ -258,8 +258,10 @@ class ColoredOutputOption(BooleanOption):
     def translate(val):
         if isinstance(val, bool):
             return val
-        elif val in configparser.ConfigParser.BOOLEAN_STATES:
-            return configparser.ConfigParser.BOOLEAN_STATES[val]
+        elif val == 'always':
+            return True
+        elif val == 'never':
+            return False
         elif val == 'auto':
             return 'NO_COLOR' not in os.environ and sys.stdout.isatty()
 
diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton
index 9c66b51b..14861592 100644
--- a/configuration/cdist.cfg.skeleton
+++ b/configuration/cdist.cfg.skeleton
@@ -16,10 +16,9 @@
 # colored_output
 #     Colorize cdist's output. If enabled, cdist will use different colors for
 #     different log levels.
-#     Recognized values are 'yes'/'no', 'on'/'off', 'true'/'false', '1'/'0',
-#     and 'auto'
-#     If the value is 'auto', colored output is enabled if stdout is a TTY
-#     unless the NO_COLOR (https://no-color.org/) environment variable is defined.
+#     Recognized values are 'always', 'never', and 'auto'.
+#     If the value is 'auto', colors are enabled if stdout is a TTY unless
+#     the NO_COLOR (https://no-color.org/) environment variable is defined.
 # colored_output = auto
 #
 # conf_dir
diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh
index a841862e..c0ac2c3e 100755
--- a/docs/src/cdist-reference.rst.sh
+++ b/docs/src/cdist-reference.rst.sh
@@ -369,9 +369,9 @@ CDIST_BETA
     Enable beta functionalities.
 
 CDIST_COLORED_OUTPUT
-    Use a colored output for different log levels.
-    It can be a boolean or 'auto' (default) which enables this feature if
-    stdout is a tty and disables it otherwise.
+    Colorize cdist's output. If enabled, cdist will use different colors for
+    different log levels.
+    Recognized values are 'always', 'never', and 'auto' (the default).
 
 CDIST_CACHE_PATH_PATTERN
     Custom cache path pattern.
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 4def97b5..760ff8f6 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -111,11 +111,11 @@ All commands accept the following options:
 **-h, --help**
     Show the help screen.
 
-**--colors COLORED_OUTPUT**
+**--colors WHEN**
     Colorize cdist's output. If enabled, cdist will use different colors for
     different log levels.
-    COLORED_OUTPUT recognizes the boolean values 'yes'/'no', 'on'/'off',
-    'true'/'false', '1'/'0', and 'auto' (the default).
+    COLORED_OUTPUT recognizes the values 'always', 'never',
+    and 'auto' (the default).
 
     If the value is 'auto', colored output is enabled if stdout is a TTY
     unless the NO_COLOR (https://no-color.org/) environment variable is defined.

From 89ebd7a4f79bef894cb3f31931571cf577778b26 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 7 Jun 2020 16:45:48 +0200
Subject: [PATCH 281/743] cdist man page: update --colors metavar name

---
 docs/src/man1/cdist.rst | 62 ++++++++++++++++++-----------------------
 1 file changed, 27 insertions(+), 35 deletions(-)

diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 760ff8f6..aae98538 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -15,21 +15,19 @@ SYNOPSIS
 
     cdist banner [-h] [-l LOGLEVEL] [-q] [-v]
 
-    cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                 [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6]
-                 [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
-                 [-j [JOBS]] [-n] [-o OUT_PATH] [-P]
+    cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
+                 [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN]
+                 [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-P]
                  [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
                  [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
                  [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE]
                  [-p [HOST_MAX]] [-s] [-t]
                  [host [host ...]]
 
-    cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                  [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6]
-                  [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
-                  [-j [JOBS]] [-n] [-o OUT_PATH] [-P]
-                  [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
+    cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
+                  [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN]
+                  [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
+                  [-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
                   [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
                   [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE]
                   [-p [HOST_MAX]] [-s] [-t]
@@ -37,35 +35,31 @@ SYNOPSIS
 
     cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ...
 
-    cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                             [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
-                             [-I INVENTORY_DIR] [-f HOSTFILE]
+    cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
+                             [-g CONFIG_FILE] [-I INVENTORY_DIR] [-f HOSTFILE]
                              [host [host ...]]
 
-    cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                            [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
-                            [-I INVENTORY_DIR] [-f HOSTFILE] [-T TAGFILE]
-                            [-t TAGLIST]
-                            [host [host ...]]
-
-    cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                             [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
-                             [-I INVENTORY_DIR] [-a] [-f HOSTFILE]
-                             [host [host ...]]
-
-    cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                            [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
-                            [-I INVENTORY_DIR] [-a] [-f HOSTFILE]
+    cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
+                            [-g CONFIG_FILE] [-I INVENTORY_DIR] [-f HOSTFILE]
                             [-T TAGFILE] [-t TAGLIST]
                             [host [host ...]]
 
-    cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b]
-                         [--colors COLORED_OUTPUT] [-g CONFIG_FILE]
-                         [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t]
+    cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
+                             [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a]
+                             [-f HOSTFILE]
+                             [host [host ...]]
+
+    cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
+                            [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a]
+                            [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
+                            [host [host ...]]
+
+    cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
+                         [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] [-f HOSTFILE]
+                         [-H] [-t]
                          [host [host ...]]
 
-    cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-g CONFIG_FILE]
-                [-L]
+    cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-g CONFIG_FILE] [-L]
                 [preos] ...
 
     cdist preos [preos-options] debian [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
@@ -89,8 +83,7 @@ SYNOPSIS
                                        [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
                                        target_dir
 
-    cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [--colors COLORED_OUTPUT]
-                [-s SHELL]
+    cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [--colors WHEN] [-s SHELL]
 
     cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t]
                [pattern]
@@ -114,8 +107,7 @@ All commands accept the following options:
 **--colors WHEN**
     Colorize cdist's output. If enabled, cdist will use different colors for
     different log levels.
-    COLORED_OUTPUT recognizes the values 'always', 'never',
-    and 'auto' (the default).
+    WHEN recognizes the values 'always', 'never', and 'auto' (the default).
 
     If the value is 'auto', colored output is enabled if stdout is a TTY
     unless the NO_COLOR (https://no-color.org/) environment variable is defined.

From be47619b1ed7a87090f7615c19a9504f7cf2b590 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 8 Jun 2020 09:11:51 +0200
Subject: [PATCH 282/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 5b57e98b..94ef55d8 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)
 	* Core: Support colored log output (Evil Ham)
+	* Core: Tune colored log output and respect NO_COLOR env var (Dennis Camera)
 	* Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak)
 
 6.5.6: 2020-05-25

From f5630297bd58fc92d95ab174ed6c8a8e14abf0b9 Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Wed, 3 Jun 2020 23:06:49 +0300
Subject: [PATCH 283/743] [docs] Fixed typo in cdist-configuration.rst

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 docs/src/cdist-configuration.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/cdist-configuration.rst b/docs/src/cdist-configuration.rst
index 706ed761..a2024584 100644
--- a/docs/src/cdist-configuration.rst
+++ b/docs/src/cdist-configuration.rst
@@ -21,7 +21,7 @@ precedence. Configuration option value read from source with higher
 precedence will overwrite option value, if exists, read from source with
 lower precedence. That means that command-line option wins them all.
 
-Users can decide on the local conifguration file location. It can be either
+Users can decide on the local configuration file location. It can be either
 ~/.cdist.cfg or $XDG_CONFIG_HOME/cdist/cdist.cfg. Note that, if both exist,
 then ~/.cdist.cfg is used.
 

From 3d725f12dace8d2e977d408b7c01fa28b8b17bde Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Wed, 3 Jun 2020 23:10:27 +0300
Subject: [PATCH 284/743] [docs] Fixed a typo and s/posix/POSIX/ in
 cdist-why.rst

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 docs/src/cdist-why.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/src/cdist-why.rst b/docs/src/cdist-why.rst
index 1123a1de..0e2cd34d 100644
--- a/docs/src/cdist-why.rst
+++ b/docs/src/cdist-why.rst
@@ -21,7 +21,7 @@ Not only is shell scripting widely known by system engineers,
 but it is also a very powerful language. Here are some features
 which make daily work easy:
 
- * Configuration can react dynamicly on explored values
+ * Configuration can react dynamically on explored values
  * High level string manipulation (using sed, awk, grep)
  * Conditional support (**if, case**)
  * Loop support (**for, while**)
@@ -44,7 +44,7 @@ Cdist requires very little on a target system. Even better,
 in almost all cases all dependencies are usually fulfilled.
 Cdist does not require an agent or high level programming
 languages on the target host: it will run on any host that
-has a **ssh server running** and a posix compatible shell
+has a **ssh server running** and a POSIX compatible shell
 (**/bin/sh**). Compared to other configuration management systems,
 it does not require to open up an additional port.
 

From 46574fc5779b939db5d90e8c11f28e808be458be Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Wed, 3 Jun 2020 23:12:10 +0300
Subject: [PATCH 285/743] [docs] Fixed three typos in cdist-upgrade.rst

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 docs/src/cdist-upgrade.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/src/cdist-upgrade.rst b/docs/src/cdist-upgrade.rst
index 67fd4934..f745b212 100644
--- a/docs/src/cdist-upgrade.rst
+++ b/docs/src/cdist-upgrade.rst
@@ -14,7 +14,7 @@ To upgrade cdist in the current branch use
     make man
     export MANPATH=$MANPATH:$(pwd -P)/doc/man
 
-If you stay on a version branche (i.e. 1.0, 1.1., ...), nothing should break.
+If you stay on a version branch (i.e. 1.0, 1.1., ...), nothing should break.
 The master branch on the other hand is the development branch and may not be
 working, break your setup or eat the tree in your garden.
 
@@ -61,7 +61,7 @@ After that, you can go back and continue the upgrade:
 Update the python package
 -------------------------
 
-To upgrade to the lastet version do
+To upgrade to the latest version do
 
 .. code-block:: sh
 
@@ -158,7 +158,7 @@ Updating from 1.5 to 1.6
 * If you used **\_\_package_apt --preseed**, you need to use the new
   type **\_\_debconf_set_selections** instead.
 * The **\_\_package** types accepted either --state deinstalled or
-  --state uninstaaled. Starting with 1.6, it was made consistently
+  --state uninstalled. Starting with 1.6, it was made consistently
   to --state removed.
 
 Updating from 1.3 to 1.5

From e1ff1bfdffd9a188a045669c4168dcd1fb561b0b Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Wed, 3 Jun 2020 23:15:38 +0300
Subject: [PATCH 286/743] [docs] Fixed two typos in cdist-real-world.rst

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 docs/src/cdist-real-world.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/src/cdist-real-world.rst b/docs/src/cdist-real-world.rst
index ba118d63..973dcd6d 100644
--- a/docs/src/cdist-real-world.rst
+++ b/docs/src/cdist-real-world.rst
@@ -198,7 +198,7 @@ We require package uWSGI present in order to create **/etc/uwsgi/apps-enabled/$u
 Installation of uWSGI also creates configuration layout: **/etc/uwsgi/apps-enabled**.
 If this directory does not exist then **__file** type would error.
 We also use stdin as file content source. For details see `Input from stdin <cdist-type.html#input-from-stdin>`_.
-For feading stdin we use here-document (**<<** operator). It allows redirection of subsequent
+For feeding stdin we use here-document (**<<** operator). It allows redirection of subsequent
 lines read by the shell to the input of a command until a line containing only the delimiter
 and a newline, with no blank characters in between (EOF in our case).
 
@@ -546,7 +546,7 @@ we have changed our **wsgi.py** file uWSGI reloads the application.
 
 Our application selects and lists items from **items** table.
 
-Openning application
+Opening application
 ~~~~~~~~~~~~~~~~~~~~
 
 Finally try the application::

From 978e24904336e6af074a7de757282456ff0314a4 Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Wed, 3 Jun 2020 23:23:49 +0300
Subject: [PATCH 287/743] [docs] Fixed capitalization of POSIX in
 cdist-install.rst

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 docs/src/cdist-install.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst
index 880b9f8e..0b65cad6 100644
--- a/docs/src/cdist-install.rst
+++ b/docs/src/cdist-install.rst
@@ -9,7 +9,7 @@ Source Host
 
 This is the machine from which you will configure target hosts.
 
- * /bin/sh: A posix like shell (for instance bash, dash, zsh)
+ * /bin/sh: A POSIX like shell (for instance bash, dash, zsh)
  * Python >= 3.2
  * SSH client
  * sphinx (for building html docs and/or the man pages)
@@ -17,7 +17,7 @@ This is the machine from which you will configure target hosts.
 Target Hosts
 ~~~~~~~~~~~~
 
- * /bin/sh: A posix like shell (for instance bash, dash, zsh)
+ * /bin/sh: A POSIX like shell (for instance bash, dash, zsh)
  * SSH server
 
 Install cdist

From dc018fdb16ba8b6044bbb55efd44e975fc5247ae Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Wed, 3 Jun 2020 23:25:46 +0300
Subject: [PATCH 288/743] [docs] Fixed typo in cdist-cache.rst

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 docs/src/cdist-cache.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/cdist-cache.rst b/docs/src/cdist-cache.rst
index 0e5361ee..d2d2d56c 100644
--- a/docs/src/cdist-cache.rst
+++ b/docs/src/cdist-cache.rst
@@ -59,7 +59,7 @@ typeorder
 
 Object cache overview
 ~~~~~~~~~~~~~~~~~~~~~
-Each object under :strong:`object` directory has its own structurue.
+Each object under :strong:`object` directory has its own structure.
 
 code-local
     code generated from gencode-local, present only if something is

From 7b262c0cec2c3c42822cef103220965424500bcb Mon Sep 17 00:00:00 2001
From: Jaak Ristioja <jaak@ristioja.ee>
Date: Wed, 3 Jun 2020 23:27:16 +0300
Subject: [PATCH 289/743] [docs] Fixed capitalization of URLs in
 cdist-bootstrap.rst

Signed-off-by: Jaak Ristioja <jaak@ristioja.ee>
---
 docs/src/cdist-bootstrap.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/cdist-bootstrap.rst b/docs/src/cdist-bootstrap.rst
index c9972a99..10f86c5a 100644
--- a/docs/src/cdist-bootstrap.rst
+++ b/docs/src/cdist-bootstrap.rst
@@ -25,7 +25,7 @@ people, have a look at `cdist best practice <cdist-best-practice.html>`_.
 Setup working directory and branch
 ----------------------------------
 I assume you have a fresh copy of the cdist tree in ~/cdist, cloned from
-one of the official urls (see `cdist quickstart <cdist-quickstart.html>`_ if you don't).
+one of the official URLs (see `cdist quickstart <cdist-quickstart.html>`_ if you don't).
 Entering the command "git branch" should show you "* master", which indicates
 you are on the **master** branch.
 

From 191f45eb7f8ab8ce97f996f3bb8375da5142d39a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 8 Jun 2020 13:48:37 +0200
Subject: [PATCH 290/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 94ef55d8..e73cfa6f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,8 @@ next:
 	* Core: Support colored log output (Evil Ham)
 	* Core: Tune colored log output and respect NO_COLOR env var (Dennis Camera)
 	* Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak)
+	* Documentation: Fix trivial grammatical mistakes (Jaak Ristioja)
+	* Explorer os: Fix for sles15 (Daniel Heule)
 
 6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)

From 955243a93b28d3524f93eb6d69af1f0eec5ff36d Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 9 Jun 2020 12:51:19 +0200
Subject: [PATCH 291/743] Update cdist man page copyright years

---
 docs/src/man1/cdist.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index aae98538..68a9a4ee 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -969,5 +969,5 @@ such case and display a warning message. An example of such a case:
 
 COPYING
 -------
-Copyright \(C) 2011-2019 Nico Schottelius. Free use of this software is
+Copyright \(C) 2011-2020 Nico Schottelius. Free use of this software is
 granted under the terms of the GNU General Public License v3 or later (GPLv3+).

From 7c490a703d0203747e481a9fc3405bf7eb5ba01a Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 10 Jun 2020 11:38:14 +0300
Subject: [PATCH 292/743] [__clean_path] add --path parameter

---
 cdist/conf/type/__clean_path/explorer/list      |  7 ++++++-
 cdist/conf/type/__clean_path/gencode-remote     |  7 ++++++-
 cdist/conf/type/__clean_path/man.rst            | 10 +++++++++-
 cdist/conf/type/__clean_path/parameter/optional |  1 +
 4 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__clean_path/explorer/list b/cdist/conf/type/__clean_path/explorer/list
index 07d38127..2bdc63a5 100755
--- a/cdist/conf/type/__clean_path/explorer/list
+++ b/cdist/conf/type/__clean_path/explorer/list
@@ -18,7 +18,12 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-path="/$__object_id"
+if [ -f "$__object/parameter/path" ]
+then
+    path="$( cat "$__object/parameter/path" )"
+else
+    path="/$__object_id"
+fi
 
 [ ! -d "$path" ] && exit 0
 
diff --git a/cdist/conf/type/__clean_path/gencode-remote b/cdist/conf/type/__clean_path/gencode-remote
index 998a70d8..2899c4a5 100755
--- a/cdist/conf/type/__clean_path/gencode-remote
+++ b/cdist/conf/type/__clean_path/gencode-remote
@@ -20,7 +20,12 @@
 
 [ ! -s "$__object/explorer/list" ] && exit 0
 
-path="/$__object_id"
+if [ -f "$__object/parameter/path" ]
+then
+    path="$( cat "$__object/parameter/path" )"
+else
+    path="/$__object_id"
+fi
 
 pattern="$( cat "$__object/parameter/pattern" )"
 
diff --git a/cdist/conf/type/__clean_path/man.rst b/cdist/conf/type/__clean_path/man.rst
index 826f4589..31d90701 100644
--- a/cdist/conf/type/__clean_path/man.rst
+++ b/cdist/conf/type/__clean_path/man.rst
@@ -10,7 +10,7 @@ DESCRIPTION
 -----------
 Remove files and directories which match the pattern.
 
-Provided path (as __object_id) must be a directory.
+Provided path must be a directory.
 
 Patterns are passed to ``find``'s ``-regex`` - see ``find(1)`` for more details.
 
@@ -29,6 +29,9 @@ pattern
 
 OPTIONAL PARAMETERS
 -------------------
+path
+   Path which will be cleaned. Defaults to ``$__object_id``.
+
 exclude
    Pattern of files which are excluded from removal.
 
@@ -46,6 +49,11 @@ EXAMPLES
         --exclude '.+\(charset\.conf\|security\.conf\)' \
         --onchange 'service apache2 restart'
 
+    __clean_path apache2-conf-enabled \
+        --path /etc/apache2/conf-enabled \
+        --pattern '.+' \
+        --exclude '.+\(charset\.conf\|security\.conf\)' \
+        --onchange 'service apache2 restart'
 
 AUTHORS
 -------
diff --git a/cdist/conf/type/__clean_path/parameter/optional b/cdist/conf/type/__clean_path/parameter/optional
index 6f313474..3b97f71c 100644
--- a/cdist/conf/type/__clean_path/parameter/optional
+++ b/cdist/conf/type/__clean_path/parameter/optional
@@ -1,2 +1,3 @@
 exclude
 onchange
+path

From 74e5d7182a466cd02b79137f1d73500c0cc50808 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 10 Jun 2020 10:45:20 +0200
Subject: [PATCH 293/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index e73cfa6f..bcde9268 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak)
 	* Documentation: Fix trivial grammatical mistakes (Jaak Ristioja)
 	* Explorer os: Fix for sles15 (Daniel Heule)
+	* Type __clean_path: Add --path parameter (Ander Punnar)
 
 6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)

From 840e417eb79c7135762d3353f34a3f8330cb0e84 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 11 Jun 2020 14:16:37 +0200
Subject: [PATCH 294/743] Fix emulator colored logging

---
 cdist/argparse.py      |  2 +-
 cdist/core/manifest.py |  3 ++-
 cdist/emulator.py      |  2 +-
 cdist/flock.py         |  1 +
 cdist/log.py           | 16 +++++++---------
 5 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 85dba246..1c83237c 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -498,7 +498,7 @@ def handle_loglevel(args):
 
 def handle_log_colors(args):
     if cdist.configuration.ColoredOutputOption.translate(args.colored_output):
-        cdist.log.DefaultLog.USE_COLORS = True
+        cdist.log.CdistFormatter.USE_COLORS = True
 
 
 def parse_and_configure(argv, singleton=True):
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 8b833ff2..2e7b873e 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -119,7 +119,8 @@ class Manifest(object):
             '__cdist_log_level': util.log_level_env_var_val(self.log),
             '__cdist_log_level_name': util.log_level_name_env_var_val(
                 self.log),
-            '__cdist_colored_log': str(self.log.USE_COLORS).lower(),
+            '__cdist_colored_log': str(
+                cdist.log.CdistFormatter.USE_COLORS).lower(),
         }
 
         if dry_run:
diff --git a/cdist/emulator.py b/cdist/emulator.py
index 4eaf2c93..08e15a78 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -130,7 +130,7 @@ class Emulator(object):
             logging.root.setLevel(logging.WARNING)
 
         colored_log = self.env.get('__cdist_colored_log', 'false')
-        cdist.log.ColorFormatter.USE_COLORS = colored_log == 'true'
+        cdist.log.CdistFormatter.USE_COLORS = colored_log == 'true'
 
         self.log = logging.getLogger(self.target_host[0])
 
diff --git a/cdist/flock.py b/cdist/flock.py
index d8bac916..3afacfd9 100644
--- a/cdist/flock.py
+++ b/cdist/flock.py
@@ -22,6 +22,7 @@
 import fcntl
 import logging
 import os
+import cdist.log
 
 
 log = logging.getLogger('cdist-flock')
diff --git a/cdist/log.py b/cdist/log.py
index 19efebdb..efae2cd3 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -50,7 +50,8 @@ def _trace(msg, *args, **kwargs):
 logging.trace = _trace
 
 
-class ColorFormatter(logging.Formatter):
+class CdistFormatter(logging.Formatter):
+    USE_COLORS = False
     RESET = '\033[0m'
     COLOR_MAP = {
         'ERROR': '\033[0;31m',
@@ -66,14 +67,14 @@ class ColorFormatter(logging.Formatter):
 
     def format(self, record):
         msg = super().format(record)
-        color = self.COLOR_MAP.get(record.levelname)
-        if color:
-            msg = color + msg + self.RESET
+        if self.USE_COLORS:
+            color = self.COLOR_MAP.get(record.levelname)
+            if color:
+                msg = color + msg + self.RESET
         return msg
 
 
 class DefaultLog(logging.Logger):
-    USE_COLORS = False
     FORMAT = '%(levelname)s: %(message)s'
 
     class StdoutFilter(logging.Filter):
@@ -88,10 +89,7 @@ class DefaultLog(logging.Logger):
         super().__init__(name)
         self.propagate = False
 
-        if self.USE_COLORS:
-            formatter = ColorFormatter(self.FORMAT)
-        else:
-            formatter = logging.Formatter(self.FORMAT)
+        formatter = CdistFormatter(self.FORMAT)
 
         self.addFilter(self)
 

From 4167f9f60c30a4587b492c2fa3dce2f02ded463a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 11 Jun 2020 14:22:54 +0200
Subject: [PATCH 295/743] Use proper format string with name

---
 cdist/log.py | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/cdist/log.py b/cdist/log.py
index efae2cd3..790059df 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -75,7 +75,7 @@ class CdistFormatter(logging.Formatter):
 
 
 class DefaultLog(logging.Logger):
-    FORMAT = '%(levelname)s: %(message)s'
+    FORMAT = '%(levelname)s: %(name)s: %(message)s'
 
     class StdoutFilter(logging.Filter):
         def filter(self, rec):
@@ -91,8 +91,6 @@ class DefaultLog(logging.Logger):
 
         formatter = CdistFormatter(self.FORMAT)
 
-        self.addFilter(self)
-
         stdout_handler = logging.StreamHandler(sys.stdout)
         stdout_handler.addFilter(self.StdoutFilter())
         stdout_handler.setLevel(logging.TRACE)
@@ -106,13 +104,6 @@ class DefaultLog(logging.Logger):
         self.addHandler(stdout_handler)
         self.addHandler(stderr_handler)
 
-    def filter(self, record):
-        """Prefix messages with logger name"""
-
-        record.msg = self.name + ": " + str(record.msg)
-
-        return True
-
     def verbose(self, msg, *args, **kwargs):
         self.log(logging.VERBOSE, msg, *args, **kwargs)
 
@@ -134,7 +125,7 @@ class TimestampingLog(DefaultLog):
 
 
 class ParallelLog(DefaultLog):
-    FORMAT = '%(levelname)s: [%(process)d]: %(message)s'
+    FORMAT = '%(levelname)s: [%(process)d]: %(name)s: %(message)s'
 
 
 class TimestampingParallelLog(TimestampingLog, ParallelLog):

From eec7ab8e4577008c6a48a473e295a9d1e31f2358 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 10 Jun 2020 16:59:18 +0200
Subject: [PATCH 296/743] Increase minimum supported Python version to 3.5

---
 docs/src/cdist-install.rst | 2 +-
 docs/src/man1/cdist.rst    | 2 +-
 scripts/cdist              | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst
index 0b65cad6..6f4f14d7 100644
--- a/docs/src/cdist-install.rst
+++ b/docs/src/cdist-install.rst
@@ -10,7 +10,7 @@ Source Host
 This is the machine from which you will configure target hosts.
 
  * /bin/sh: A POSIX like shell (for instance bash, dash, zsh)
- * Python >= 3.2
+ * Python >= 3.5
  * SSH client
  * sphinx (for building html docs and/or the man pages)
 
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 68a9a4ee..9bf8fc9b 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -95,7 +95,7 @@ cdist is the frontend executable to the cdist configuration management.
 It supports different subcommands as explained below.
 
 It is written in Python so it requires :strong:`python`\ (1) to be installed.
-It requires a minimal Python version 3.2.
+It requires a minimal Python version 3.5.
 
 GENERAL
 -------
diff --git a/scripts/cdist b/scripts/cdist
index 7bf12c01..664504a0 100755
--- a/scripts/cdist
+++ b/scripts/cdist
@@ -60,7 +60,7 @@ def commandline():
 
 
 if __name__ == "__main__":
-    cdistpythonversion = '3.2'
+    cdistpythonversion = '3.5'
     if sys.version < cdistpythonversion:
         print('Python >= {} is required on the source host.'.format(
                 cdistpythonversion), file=sys.stderr)

From eba3d0505bba4ee0472df3452142cc63c8fe2f5c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 12 Jun 2020 06:29:39 +0200
Subject: [PATCH 297/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index bcde9268..0cc2a714 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,7 @@ next:
 	* Documentation: Fix trivial grammatical mistakes (Jaak Ristioja)
 	* Explorer os: Fix for sles15 (Daniel Heule)
 	* Type __clean_path: Add --path parameter (Ander Punnar)
+	* Core: Increase minimal supported Python version to 3.5 (Darko Poljak)
 
 6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)

From 6e9e9ad557479354938257f04c8fab283b8251e8 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Tue, 12 Nov 2019 00:40:58 +0100
Subject: [PATCH 298/743] implement log server to capture nested logging output

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/core/code.py |  4 +++
 cdist/install.py   | 23 +++++++++++++-
 cdist/log.py       | 77 ++++++++++++++++++++++++++++++++++++++--------
 3 files changed, 90 insertions(+), 14 deletions(-)

diff --git a/cdist/core/code.py b/cdist/core/code.py
index 1550880a..a7d9b7ca 100644
--- a/cdist/core/code.py
+++ b/cdist/core/code.py
@@ -116,6 +116,10 @@ class Code(object):
         if dry_run:
             self.env['__cdist_dry_run'] = '1'
 
+        if '__cdist_log_server_socket_to_export' in os.environ:
+            self.env['__cdist_log_server_socket'] = os.environ['__cdist_log_server_socket_to_export']
+
+
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.cdist_type
         script = os.path.join(self.local.type_path,
diff --git a/cdist/install.py b/cdist/install.py
index b88ad016..3f94ca68 100644
--- a/cdist/install.py
+++ b/cdist/install.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
-# 2013 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2013-2019 Steven Armstrong (steven-cdist at armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -20,11 +20,32 @@
 #
 #
 
+import os
+import logging
+import tempfile
+
 import cdist.config
 import cdist.core
 
 
 class Install(cdist.config.Config):
+
+    @classmethod
+    def onehost(cls, host, host_tags, host_base_path, host_dir_name, args,
+                parallel, configuration, remove_remote_files_dirs=False):
+        # Start a log server so nested `cdist config` runs have a place to
+        # send their logs to.
+        log_server_socket_dir = tempfile.mkdtemp()
+        log_server_socket = os.path.join(log_server_socket_dir, 'log-server')
+        cls._register_path_for_removal(log_server_socket_dir)
+        log = logging.getLogger(host)
+        log.debug('Starting logging server on: %s', log_server_socket)
+        os.environ['__cdist_log_server_socket_to_export'] = log_server_socket
+        cdist.log.setupLogServer(log_server_socket)
+
+        super().onehost(host, host_tags, host_base_path, host_dir_name, args,
+                parallel, configuration, remove_remote_files_dirs=False)
+
     def object_list(self):
         """Short name for object list retrieval.
         In install mode, we only care about install objects.
diff --git a/cdist/log.py b/cdist/log.py
index 790059df..94dd11e8 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 #
 # 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2019-2020 Steven Armstrong
 #
 # This file is part of cdist.
 #
@@ -20,9 +21,17 @@
 #
 #
 
-import logging
-import sys
+import asyncio
+import contextlib
 import datetime
+import logging
+import logging.handlers
+import os
+import pickle
+import struct
+import sys
+import threading
+import time
 
 
 # Define additional cdist logging levels.
@@ -89,20 +98,25 @@ class DefaultLog(logging.Logger):
         super().__init__(name)
         self.propagate = False
 
-        formatter = CdistFormatter(self.FORMAT)
+        if '__cdist_log_server_socket' in os.environ:
+            log_server_socket = os.environ['__cdist_log_server_socket']
+            socket_handler = logging.handlers.SocketHandler(log_server_socket, None)
+            self.addHandler(socket_handler)
+        else:
+            formatter = CdistFormatter(self.FORMAT)
 
-        stdout_handler = logging.StreamHandler(sys.stdout)
-        stdout_handler.addFilter(self.StdoutFilter())
-        stdout_handler.setLevel(logging.TRACE)
-        stdout_handler.setFormatter(formatter)
+            stdout_handler = logging.StreamHandler(sys.stdout)
+            stdout_handler.addFilter(self.StdoutFilter())
+            stdout_handler.setLevel(logging.TRACE)
+            stdout_handler.setFormatter(formatter)
 
-        stderr_handler = logging.StreamHandler(sys.stderr)
-        stderr_handler.addFilter(self.StderrFilter())
-        stderr_handler.setLevel(logging.ERROR)
-        stderr_handler.setFormatter(formatter)
+            stderr_handler = logging.StreamHandler(sys.stderr)
+            stderr_handler.addFilter(self.StderrFilter())
+            stderr_handler.setLevel(logging.ERROR)
+            stderr_handler.setFormatter(formatter)
 
-        self.addHandler(stdout_handler)
-        self.addHandler(stderr_handler)
+            self.addHandler(stdout_handler)
+            self.addHandler(stderr_handler)
 
     def verbose(self, msg, *args, **kwargs):
         self.log(logging.VERBOSE, msg, *args, **kwargs)
@@ -152,4 +166,41 @@ def setupParallelLogging():
     logging.setLoggerClass(ParallelLog)
 
 
+async def handle_log_client(reader, writer):
+    while True:
+        chunk = await reader.read(4)
+        if len(chunk) < 4:
+            return
+
+        data_size = struct.unpack('>L', chunk)[0]
+        data = bytearray(data_size)
+        view = memoryview(data)
+        data_pending = data_size
+        data = await reader.read(data_size)
+
+        obj = pickle.loads(data)
+        record = logging.makeLogRecord(obj)
+        logger = logging.getLogger(record.name)
+        logger.handle(record)
+
+
+def run_log_server(server_address):
+    # Get a new loop inside the current thread to run the log server.
+    loop = asyncio.new_event_loop()
+    loop.create_task(asyncio.start_unix_server(handle_log_client, server_address))
+    loop.run_forever()
+
+
+def setupLogServer(log_server_socket):
+    """Run a asyncio based unix socket log server in a background thread.
+    """
+    with contextlib.suppress(FileNotFoundError):
+        os.remove(log_server_socket)
+    t = threading.Thread(target=run_log_server, args=(log_server_socket,))
+    # Deamonizing the thread means we don't have to care about stoping it.
+    # It will die together with the main process.
+    t.daemon = True
+    t.start()
+
+
 setupDefaultLogging()

From 831bfc822b61a5e5cf8186a32ea592622693efdb Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Tue, 12 Nov 2019 09:22:28 +0100
Subject: [PATCH 299/743] remove unused code

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/log.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/cdist/log.py b/cdist/log.py
index 94dd11e8..c1376a58 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -173,9 +173,6 @@ async def handle_log_client(reader, writer):
             return
 
         data_size = struct.unpack('>L', chunk)[0]
-        data = bytearray(data_size)
-        view = memoryview(data)
-        data_pending = data_size
         data = await reader.read(data_size)
 
         obj = pickle.loads(data)

From 57e352cd1e7b400ca1daf7e1e680874bd9a6cc01 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Sat, 11 Jan 2020 01:59:18 +0100
Subject: [PATCH 300/743] log server is also usefull for cdist config

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/argparse.py |  6 ++++++
 cdist/config.py   | 10 ++++++++++
 cdist/install.py  | 13 +++----------
 3 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 1c83237c..15b9ca7a 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -250,6 +250,12 @@ def get_parsers():
            '-S', '--disable-saving-output-streams',
            help='Disable saving output streams.',
            action='store_false', dest='save_output_streams', default=True)
+    parser['config_main'].add_argument(
+           '--log-server',
+           action='store_true',
+           help=('Start a log server for sub processes to use.'
+            'This is mainly usefull when running cdist nested'
+            'from a code-local script.'))
 
     # Config
     parser['config_args'] = argparse.ArgumentParser(add_help=False)
diff --git a/cdist/config.py b/cdist/config.py
index b2d72f05..82e8559b 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -385,6 +385,16 @@ class Config(object):
 
         log = logging.getLogger(host)
 
+        if args.log_server:
+            # Start a log server so that nested `cdist config` runs have a place
+            # to send their logs to.
+            log_server_socket_dir = tempfile.mkdtemp()
+            log_server_socket = os.path.join(log_server_socket_dir, 'log-server')
+            cls._register_path_for_removal(log_server_socket_dir)
+            log.debug('Starting logging server on: %s', log_server_socket)
+            os.environ['__cdist_log_server_socket_to_export'] = log_server_socket
+            cdist.log.setupLogServer(log_server_socket)
+
         try:
             remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds(
                 args)
diff --git a/cdist/install.py b/cdist/install.py
index 3f94ca68..ec0a7270 100644
--- a/cdist/install.py
+++ b/cdist/install.py
@@ -33,16 +33,9 @@ class Install(cdist.config.Config):
     @classmethod
     def onehost(cls, host, host_tags, host_base_path, host_dir_name, args,
                 parallel, configuration, remove_remote_files_dirs=False):
-        # Start a log server so nested `cdist config` runs have a place to
-        # send their logs to.
-        log_server_socket_dir = tempfile.mkdtemp()
-        log_server_socket = os.path.join(log_server_socket_dir, 'log-server')
-        cls._register_path_for_removal(log_server_socket_dir)
-        log = logging.getLogger(host)
-        log.debug('Starting logging server on: %s', log_server_socket)
-        os.environ['__cdist_log_server_socket_to_export'] = log_server_socket
-        cdist.log.setupLogServer(log_server_socket)
-
+        # Always start log server during cdist install so that nested
+        # `cdist config` runs have a place to send their logs to.
+        args.log_server = True
         super().onehost(host, host_tags, host_base_path, host_dir_name, args,
                 parallel, configuration, remove_remote_files_dirs=False)
 

From 59b98091d78c5195d2aa4a2ea3ef79f93e5ba67c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 9 Jun 2020 12:47:50 +0200
Subject: [PATCH 301/743] Adapt; update docs and code style

---
 cdist/__init__.py       |  4 ++++
 cdist/argparse.py       | 13 +++++++------
 cdist/config.py         | 27 ++++++++++++---------------
 cdist/core/code.py      |  6 +++---
 cdist/install.py        |  8 +++-----
 cdist/log.py            | 18 +++++++++++-------
 docs/src/man1/cdist.rst | 26 ++++++++++++++++----------
 scripts/cdist           |  5 ++---
 8 files changed, 58 insertions(+), 49 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index c673b3ba..be573170 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -26,6 +26,7 @@ import hashlib
 import cdist.log
 import cdist.version
 
+
 VERSION = cdist.version.VERSION
 
 BANNER = """
@@ -48,6 +49,9 @@ REMOTE_EXEC = "ssh -o User=root"
 REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
 
 
+MIN_SUPPORTED_PYTHON_VERSION = '3.5'
+
+
 class Error(Exception):
     """Base exception class for this project"""
     pass
diff --git a/cdist/argparse.py b/cdist/argparse.py
index 15b9ca7a..77303591 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -206,6 +206,13 @@ def get_parsers():
                  'supported. Without argument CPU count is used by default. '),
            action='store', dest='jobs',
            const=multiprocessing.cpu_count())
+    parser['config_main'].add_argument(
+           '--log-server',
+           action='store_true',
+           help=('Start a log server for sub processes to use. '
+                 'This is mainly useful when running cdist nested '
+                 'from a code-local script. Log server is alwasy '
+                 'implicitly started for \'install\' command.'))
     parser['config_main'].add_argument(
            '-n', '--dry-run',
            help='Do not execute code.', action='store_true')
@@ -250,12 +257,6 @@ def get_parsers():
            '-S', '--disable-saving-output-streams',
            help='Disable saving output streams.',
            action='store_false', dest='save_output_streams', default=True)
-    parser['config_main'].add_argument(
-           '--log-server',
-           action='store_true',
-           help=('Start a log server for sub processes to use.'
-            'This is mainly usefull when running cdist nested'
-            'from a code-local script.'))
 
     # Config
     parser['config_args'] = argparse.ArgumentParser(add_help=False)
diff --git a/cdist/config.py b/cdist/config.py
index 82e8559b..9fe9b676 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -29,18 +29,20 @@ import time
 import itertools
 import tempfile
 import multiprocessing
-from cdist.mputil import mp_pool_run, mp_sig_handler
 import atexit
 import shutil
 import socket
+
+from cdist.mputil import mp_pool_run, mp_sig_handler
+from cdist import core, inventory
+from cdist.util.remoteutil import inspect_ssh_mux_opts
+
 import cdist
 import cdist.hostsource
 import cdist.exec.local
 import cdist.exec.remote
 import cdist.util.ipaddr as ipaddr
 import cdist.configuration
-from cdist import core, inventory
-from cdist.util.remoteutil import inspect_ssh_mux_opts
 
 
 def graph_check_cycle(graph):
@@ -195,7 +197,6 @@ class Config(object):
     @classmethod
     def commandline(cls, args):
         """Configure remote system"""
-
         if (args.parallel and args.parallel != 1) or args.jobs:
             if args.timestamp:
                 cdist.log.setupTimestampingParallelLogging()
@@ -382,20 +383,16 @@ class Config(object):
            If operating in parallel then return tuple (host, True|False, )
            so that main process knows for which host function was successful.
         """
-
         log = logging.getLogger(host)
 
-        if args.log_server:
-            # Start a log server so that nested `cdist config` runs have a place
-            # to send their logs to.
-            log_server_socket_dir = tempfile.mkdtemp()
-            log_server_socket = os.path.join(log_server_socket_dir, 'log-server')
-            cls._register_path_for_removal(log_server_socket_dir)
-            log.debug('Starting logging server on: %s', log_server_socket)
-            os.environ['__cdist_log_server_socket_to_export'] = log_server_socket
-            cdist.log.setupLogServer(log_server_socket)
-
         try:
+            if args.log_server:
+                # Start a log server so that nested `cdist config` runs
+                # have a place to send their logs to.
+                log_server_socket_dir = tempfile.mkdtemp()
+                cls._register_path_for_removal(log_server_socket_dir)
+                cdist.log.setupLogServer(log_server_socket_dir, log)
+
             remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds(
                 args)
             log.debug("remote_exec for host \"{}\": {}".format(
diff --git a/cdist/core/code.py b/cdist/core/code.py
index a7d9b7ca..2a30908e 100644
--- a/cdist/core/code.py
+++ b/cdist/core/code.py
@@ -116,9 +116,9 @@ class Code(object):
         if dry_run:
             self.env['__cdist_dry_run'] = '1'
 
-        if '__cdist_log_server_socket_to_export' in os.environ:
-            self.env['__cdist_log_server_socket'] = os.environ['__cdist_log_server_socket_to_export']
-
+        if '__cdist_log_server_socket_export' in os.environ:
+            self.env['__cdist_log_server_socket'] = os.environ[
+                '__cdist_log_server_socket_export']
 
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.cdist_type
diff --git a/cdist/install.py b/cdist/install.py
index ec0a7270..561b2fa6 100644
--- a/cdist/install.py
+++ b/cdist/install.py
@@ -20,12 +20,9 @@
 #
 #
 
-import os
-import logging
-import tempfile
-
 import cdist.config
 import cdist.core
+import cdist.log
 
 
 class Install(cdist.config.Config):
@@ -36,8 +33,9 @@ class Install(cdist.config.Config):
         # Always start log server during cdist install so that nested
         # `cdist config` runs have a place to send their logs to.
         args.log_server = True
+
         super().onehost(host, host_tags, host_base_path, host_dir_name, args,
-                parallel, configuration, remove_remote_files_dirs=False)
+                        parallel, configuration, remove_remote_files_dirs)
 
     def object_list(self):
         """Short name for object list retrieval.
diff --git a/cdist/log.py b/cdist/log.py
index c1376a58..bee99fac 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -21,17 +21,16 @@
 #
 #
 
-import asyncio
-import contextlib
 import datetime
 import logging
 import logging.handlers
+import sys
 import os
+import asyncio
+import contextlib
 import pickle
 import struct
-import sys
 import threading
-import time
 
 
 # Define additional cdist logging levels.
@@ -100,7 +99,8 @@ class DefaultLog(logging.Logger):
 
         if '__cdist_log_server_socket' in os.environ:
             log_server_socket = os.environ['__cdist_log_server_socket']
-            socket_handler = logging.handlers.SocketHandler(log_server_socket, None)
+            socket_handler = logging.handlers.SocketHandler(log_server_socket,
+                                                            None)
             self.addHandler(socket_handler)
         else:
             formatter = CdistFormatter(self.FORMAT)
@@ -184,13 +184,17 @@ async def handle_log_client(reader, writer):
 def run_log_server(server_address):
     # Get a new loop inside the current thread to run the log server.
     loop = asyncio.new_event_loop()
-    loop.create_task(asyncio.start_unix_server(handle_log_client, server_address))
+    loop.create_task(asyncio.start_unix_server(handle_log_client,
+                                               server_address))
     loop.run_forever()
 
 
-def setupLogServer(log_server_socket):
+def setupLogServer(socket_dir, log=logging.getLogger(__name__)):
     """Run a asyncio based unix socket log server in a background thread.
     """
+    log_server_socket = os.path.join(socket_dir, 'log-server')
+    log.debug('Starting logging server on: %s', log_server_socket)
+    os.environ['__cdist_log_server_socket_export'] = log_server_socket
     with contextlib.suppress(FileNotFoundError):
         os.remove(log_server_socket)
     t = threading.Thread(target=run_log_server, args=(log_server_socket,))
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 9bf8fc9b..aa2607f8 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -17,20 +17,20 @@ SYNOPSIS
 
     cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
                  [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN]
-                 [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-P]
-                 [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
-                 [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
-                 [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE]
-                 [-p [HOST_MAX]] [-s] [-t]
+                 [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [--log-server]
+                 [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]]
+                 [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY]
+                 [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A]
+                 [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
                  [host [host ...]]
 
     cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
                   [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN]
-                  [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
-                  [-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
-                  [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
-                  [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE]
-                  [-p [HOST_MAX]] [-s] [-t]
+                  [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [--log-server]
+                  [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]]
+                  [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY]
+                  [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A]
+                  [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
                   [host [host ...]]
 
     cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ...
@@ -202,6 +202,12 @@ Install command is currently in beta.
     are supported. Without argument CPU count is used by
     default.
 
+**--log-server**
+    Start a log server for sub processes to use. This is
+    mainly useful when running cdist nested from a code-
+    local script. Log server is always implicitly started
+    for 'install' command.
+
 **-n, --dry-run**
     Do not execute code.
 
diff --git a/scripts/cdist b/scripts/cdist
index 664504a0..b1d782ab 100755
--- a/scripts/cdist
+++ b/scripts/cdist
@@ -60,10 +60,9 @@ def commandline():
 
 
 if __name__ == "__main__":
-    cdistpythonversion = '3.5'
-    if sys.version < cdistpythonversion:
+    if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION:
         print('Python >= {} is required on the source host.'.format(
-                cdistpythonversion), file=sys.stderr)
+                cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr)
         sys.exit(1)
 
     exit_code = 0

From 5be8437a602907009d7603fa41ae0a4e6528ffdf Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 13 Jun 2020 13:48:34 +0200
Subject: [PATCH 302/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 0cc2a714..fa5f7405 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,6 +10,7 @@ next:
 	* Explorer os: Fix for sles15 (Daniel Heule)
 	* Type __clean_path: Add --path parameter (Ander Punnar)
 	* Core: Increase minimal supported Python version to 3.5 (Darko Poljak)
+	* Core: Add log server for nested logging (Steven Armstrong)
 
 6.5.6: 2020-05-25
 	* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)

From 201050a9e5144d0f7da2a8019e5822fedaa1d1b1 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 14 Jun 2020 11:07:18 +0300
Subject: [PATCH 303/743] new type: __download

---
 cdist/conf/type/__download/explorer/state     | 20 ++++++
 cdist/conf/type/__download/gencode-local      | 35 ++++++++++
 cdist/conf/type/__download/man.rst            | 66 +++++++++++++++++++
 .../type/__download/parameter/default/cmd-get |  1 +
 .../type/__download/parameter/default/cmd-sum |  1 +
 cdist/conf/type/__download/parameter/optional |  2 +
 cdist/conf/type/__download/parameter/required |  2 +
 7 files changed, 127 insertions(+)
 create mode 100755 cdist/conf/type/__download/explorer/state
 create mode 100755 cdist/conf/type/__download/gencode-local
 create mode 100644 cdist/conf/type/__download/man.rst
 create mode 100644 cdist/conf/type/__download/parameter/default/cmd-get
 create mode 100644 cdist/conf/type/__download/parameter/default/cmd-sum
 create mode 100644 cdist/conf/type/__download/parameter/optional
 create mode 100644 cdist/conf/type/__download/parameter/required

diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
new file mode 100755
index 00000000..6a50f5a5
--- /dev/null
+++ b/cdist/conf/type/__download/explorer/state
@@ -0,0 +1,20 @@
+#!/bin/sh -e
+
+dst="/$__object_id"
+
+# shellcheck disable=SC2059
+cmd="$( printf "$( cat "$__object/parameter/cmd-sum" )" "$dst" )"
+
+sum="$( cat "$__object/parameter/sum" )"
+
+if [ -f "$dst" ]
+then
+    if [ "$( eval "$cmd" )" = "$sum" ]
+    then
+        echo 'present'
+    else
+        echo 'mismatch'
+    fi
+else
+    echo 'absent'
+fi
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
new file mode 100755
index 00000000..49e9c699
--- /dev/null
+++ b/cdist/conf/type/__download/gencode-local
@@ -0,0 +1,35 @@
+#!/bin/sh -e
+
+state_is="$( cat "$__object/explorer/state" )"
+
+if [ "$state_is" = 'present' ]
+then
+    exit 0
+fi
+
+url="$( cat "$__object/parameter/url" )"
+
+cmd="$( cat "$__object/parameter/cmd-get" )"
+
+tmp="$( mktemp )"
+
+dst="/$__object_id"
+
+printf "$cmd > %s\n" \
+    "$url" \
+    "$tmp"
+
+if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$'
+then
+    target_host="[$__target_host]"
+else
+    target_host="$__target_host"
+fi
+
+printf '%s %s %s:%s\n' \
+    "$__remote_copy" \
+    "$tmp" \
+    "$target_host" \
+    "$dst"
+
+echo "rm -f '$tmp'"
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
new file mode 100644
index 00000000..c973448f
--- /dev/null
+++ b/cdist/conf/type/__download/man.rst
@@ -0,0 +1,66 @@
+cdist-type__download(7)
+=======================
+
+NAME
+----
+cdist-type__download - Download file to local storage and copy it to target host
+
+
+DESCRIPTION
+-----------
+You must use persistent storage in target host for destination file
+(``$__object_id``) because it will be used for checksum calculation
+in order to decide if file must be downloaded.
+
+
+REQUIRED PARAMETERS
+-------------------
+url
+   URL from which to download the file.
+
+sum
+   Checksum of downloaded file.
+
+
+OPTIONAL PARAMETERS
+-------------------
+cmd-get
+   Command used for downloading.
+   Default is ``wget -O- '%s'``.
+   Command must output to ``stdout``.
+
+cmd-sum
+   Command used for checksum calculation.
+   Default is ``md5sum '%s' | awk '{print $1}'``.
+   Command output and ``--sum`` parameter must match.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __directory /opt/cpma
+
+    require='__directory/opt/cpma' \
+        __download /opt/cpma/cnq3.zip \
+            --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
+            --sum 46da3021ca9eace277115ec9106c5b46
+
+    require='__download/opt/cpma/cnq3.zip' \
+        __unpack /opt/cpma/cnq3.zip \
+            --move-existing-destination \
+            --destination /opt/cpma/server
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__download/parameter/default/cmd-get b/cdist/conf/type/__download/parameter/default/cmd-get
new file mode 100644
index 00000000..2daa38a1
--- /dev/null
+++ b/cdist/conf/type/__download/parameter/default/cmd-get
@@ -0,0 +1 @@
+wget -O- '%s'
diff --git a/cdist/conf/type/__download/parameter/default/cmd-sum b/cdist/conf/type/__download/parameter/default/cmd-sum
new file mode 100644
index 00000000..3e8a9295
--- /dev/null
+++ b/cdist/conf/type/__download/parameter/default/cmd-sum
@@ -0,0 +1 @@
+md5sum '%s' | awk '{print $1}'
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
new file mode 100644
index 00000000..22783e02
--- /dev/null
+++ b/cdist/conf/type/__download/parameter/optional
@@ -0,0 +1,2 @@
+cmd-get
+cmd-sum
diff --git a/cdist/conf/type/__download/parameter/required b/cdist/conf/type/__download/parameter/required
new file mode 100644
index 00000000..6ea4c38f
--- /dev/null
+++ b/cdist/conf/type/__download/parameter/required
@@ -0,0 +1,2 @@
+url
+sum

From cdb998398d25f1ba99f01364bbc42b900be944d5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 17 Jun 2020 12:10:58 +0200
Subject: [PATCH 304/743] Release 6.6.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index fa5f7405..c8388b93 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)
 	* Core: Support colored log output (Evil Ham)
 	* Core: Tune colored log output and respect NO_COLOR env var (Dennis Camera)

From a6543a72ade5a3b64cab1b0164d5df0b4388e88f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 17 Jun 2020 13:40:31 +0200
Subject: [PATCH 305/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index c8388b93..a0de09fa 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* New type: __download (Ander Punnar)
+
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)
 	* Core: Support colored log output (Evil Ham)

From 97e48be39e80ffe50946606b912ca28d3612cc61 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 21:11:28 +0200
Subject: [PATCH 306/743] [type/__package_opkg] Fix explorer running in
 parallel

---
 .../type/__package_opkg/explorer/pkg_status   | 57 ++++++++++++++-----
 1 file changed, 44 insertions(+), 13 deletions(-)

diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status
index 5da4f742..1ba88e81 100755
--- a/cdist/conf/type/__package_opkg/explorer/pkg_status
+++ b/cdist/conf/type/__package_opkg/explorer/pkg_status
@@ -1,7 +1,8 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011 Nico Schottelius (nico-cdist at schottelius.org)
 # 2012 Giel van Schijndel (giel plus cdist at mortis dot eu)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,21 +20,51 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Retrieve the status of a package - parsed opkg output
+# Retrieve the status of a package - parses opkg output
 #
 
-if [ -f "$__object/parameter/name" ]; then
-   name="$(cat "$__object/parameter/name")"
+LOCKFILE="${__type_explorer}/cdist_opkg.lock"
+_lock() (
+	set -o noclobber
+	until echo $$>"${LOCKFILE}"
+	do
+		while test -f "${LOCKFILE}"; do sleep 1; done
+	done
+
+) 2>/dev/null
+_unlock() {
+	if test -s "${LOCKFILE}" && test "$(cat "${LOCKFILE}")" = $$
+	then
+		rm "${LOCKFILE}"
+	fi
+}
+
+
+if test -f "${__object}/parameter/name"
+then
+	pkg_name=$(cat "${__object}/parameter/name")
 else
-   name="$__object_id"
+	pkg_name=$__object_id
 fi
 
-# Except dpkg failing, if package is not known / installed
-if opkg status "$name" 2>/dev/null | grep -q "^Status: install user installed$"; then
-   echo "present"
-   exit 0
-elif [ "$(opkg info "$name" 2> /dev/null | wc -l)" -eq 0 ]; then
-   echo "absent notpresent"
-   exit 0
+
+# NOTE: We need to lock parallel execution of this explorer because opkg will
+# try to acquire the OPKG lock (usually /var/lock/opkg.lock) using lockf(2) for
+# every operation. It will not wait for the lock but terminate with an error.
+# This leads to incorrect 'absent notpresent' statuses when parallel execution
+# is enabled.
+trap _unlock EXIT
+_lock
+
+
+# Except opkg failing, if package is not known / installed
+if opkg status "${pkg_name}" 2>/dev/null \
+	| grep -q -e '^Status: [^ ][^ ]* [^ ][^ ]* installed$'
+then
+	echo 'present'
+elif opkg info "${pkg_name}" 2>/dev/null | grep -q .
+then
+	echo 'absent notpresent'
+else
+	echo 'absent'
 fi
-echo "absent"

From e79b26a61f790108b90c3625d554f6df4086d616 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 21 Jun 2020 13:15:38 +0200
Subject: [PATCH 307/743] [type/__package_opkg] Also lock execution of
 code-remote

---
 .../type/__package_opkg/explorer/pkg_status   | 11 ++-
 cdist/conf/type/__package_opkg/gencode-remote | 94 +++++++++++++------
 2 files changed, 73 insertions(+), 32 deletions(-)

diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status
index 1ba88e81..f5a6f098 100755
--- a/cdist/conf/type/__package_opkg/explorer/pkg_status
+++ b/cdist/conf/type/__package_opkg/explorer/pkg_status
@@ -23,7 +23,9 @@
 # Retrieve the status of a package - parses opkg output
 #
 
-LOCKFILE="${__type_explorer}/cdist_opkg.lock"
+__type_path=${__object%%${__object_id}*}
+
+LOCKFILE="${__type_path}/cdist_opkg.lock"
 _lock() (
 	set -o noclobber
 	until echo $$>"${LOCKFILE}"
@@ -48,9 +50,10 @@ else
 fi
 
 
-# NOTE: We need to lock parallel execution of this explorer because opkg will
-# try to acquire the OPKG lock (usually /var/lock/opkg.lock) using lockf(2) for
-# every operation. It will not wait for the lock but terminate with an error.
+# NOTE: We need to lock parallel execution of type explorers and code-remote
+# because opkg will try to acquire the OPKG lock (usually /var/lock/opkg.lock)
+# using lockf(2) for every operation.
+# It will not wait for the lock but terminate with an error.
 # This leads to incorrect 'absent notpresent' statuses when parallel execution
 # is enabled.
 trap _unlock EXIT
diff --git a/cdist/conf/type/__package_opkg/gencode-remote b/cdist/conf/type/__package_opkg/gencode-remote
index 269d5f49..ad90dc24 100755
--- a/cdist/conf/type/__package_opkg/gencode-remote
+++ b/cdist/conf/type/__package_opkg/gencode-remote
@@ -2,6 +2,7 @@
 #
 # 2011,2013 Nico Schottelius (nico-cdist at schottelius.org)
 # 2012 Giel van Schijndel (giel plus cdist at mortis dot eu)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,41 +20,78 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Manage packages on OpenWRT and co.
+# Manage packages on OpenWrt, optware, and co.
 #
 
-if [ -f "$__object/parameter/name" ]; then
-   name="$(cat "$__object/parameter/name")"
+if test -f "${__object}/parameter/name"
+then
+   name=$(cat "${__object}/parameter/name")
 else
-   name="$__object_id"
+   name=$__object_id
 fi
 
-state_should="$(cat "$__object/parameter/state")"
+state_should=$(cat "${__object}/parameter/state")
+state_is=$(cat "${__object}/explorer/pkg_status")
 
-state_is="$(cat "$__object/explorer/pkg_status")"
-case "$state_is" in
-    absent*)
-       present="$(echo "$state_is" | cut -d ' ' -f 2)"
-       state_is="absent"
-    ;;
+case $state_is
+in
+	(absent*)
+		presence=$(echo "${state_is}" | cut -d ' ' -f 2)
+		state_is='absent'
+		;;
 esac
 
-[ "$state_is" = "$state_should" ] && exit 0
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
 
-case "$state_should" in
-    present)
-        if [ "$present" = "notpresent" ]; then
-            echo "opkg --verbosity=0 update"
-        fi
-        echo "opkg --verbosity=0 install '$name'"
-        echo "installed" >> "$__messages_out"
-    ;;
-    absent)
-        echo "opkg --verbosity=0 remove '$name'"
-        echo "removed" >> "$__messages_out"
-    ;;
-    *)
-        echo "Unknown state: ${state_should}" >&2
-        exit 1
-    ;;
+cat <<'EOF'
+__type_path=${__object%%${__object_id}*}
+
+LOCKFILE="${__type_path}/cdist_opkg.lock"
+_lock() (
+	set -o noclobber
+	until echo $$>"${LOCKFILE}"
+	do
+		while test -f "${LOCKFILE}"; do sleep 1; done
+	done
+
+) 2>/dev/null
+_unlock() {
+	if test -s "${LOCKFILE}" && test "$(cat "${LOCKFILE}")" = $$
+	then
+		rm "${LOCKFILE}"
+	fi
+}
+EOF
+
+# NOTE: We need to lock parallel execution of code-remote to ensure that it is
+# not executed concurrently with a type explorer.
+# opkg will try to acquire the OPKG lock (usually /var/lock/opkg.lock) using
+# lockf(2) for every operation.
+# It will not wait for the lock but terminate with an error leading to an
+# incorrect outcome.
+echo 'trap _unlock EXIT'
+echo '_lock'
+
+case $state_should
+in
+	(present)
+		if test "${presence}" = 'notpresent'
+		then
+			echo 'opkg --verbosity=0 update'
+		fi
+
+		printf "opkg --verbosity=0 install '%s'\n" "${name}"
+		echo 'installed' >>"${__messages_out}"
+		;;
+	(absent)
+		printf "opkg --verbosity=0 remove '%s'" "${name}"
+		echo 'removed' >>"${__messages_out}"
+		;;
+	(*)
+		printf 'Unknown state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
 esac

From a6a3fb40bfcbaa5d6506a1ac5ebe2e40376dcb6f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 22:22:29 +0200
Subject: [PATCH 308/743] Remove unnecessary Python shebangs

---
 cdist/config.py                        | 1 -
 cdist/info.py                          | 1 -
 cdist/install.py                       | 1 -
 cdist/integration.py                   | 1 -
 cdist/inventory.py                     | 1 -
 cdist/log.py                           | 1 -
 cdist/preos/debootstrap/debootstrap.py | 1 -
 7 files changed, 7 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index b2d72f05..b71536a8 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
diff --git a/cdist/info.py b/cdist/info.py
index b896a3d1..9e29f5d9 100644
--- a/cdist/info.py
+++ b/cdist/info.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2019-2020 Darko Poljak (darko.poljak at gmail.com)
diff --git a/cdist/install.py b/cdist/install.py
index b88ad016..a9c8119a 100644
--- a/cdist/install.py
+++ b/cdist/install.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2013 Steven Armstrong (steven-cdist at armstrong.cc)
diff --git a/cdist/integration.py b/cdist/integration.py
index ee742cc5..03e4167d 100644
--- a/cdist/integration.py
+++ b/cdist/integration.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2017 Darko Poljak (darko.poljak at gmail.com)
diff --git a/cdist/inventory.py b/cdist/inventory.py
index 138a2034..fb5ab960 100644
--- a/cdist/inventory.py
+++ b/cdist/inventory.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2016 Darko Poljak (darko.poljak at gmail.com)
diff --git a/cdist/log.py b/cdist/log.py
index 19efebdb..c77ba8ec 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py
index f53dd4a7..d3e590f2 100644
--- a/cdist/preos/debootstrap/debootstrap.py
+++ b/cdist/preos/debootstrap/debootstrap.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2016 Darko Poljak (darko.poljak at ungleich.ch)

From ce07021580219d130c3968c547d03f451f2dea12 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 21:16:23 +0200
Subject: [PATCH 309/743] Do not subclass object

---
 cdist/config.py                        | 2 +-
 cdist/core/cdist_object.py             | 2 +-
 cdist/core/cdist_type.py               | 2 +-
 cdist/core/code.py                     | 2 +-
 cdist/core/explorer.py                 | 2 +-
 cdist/core/manifest.py                 | 2 +-
 cdist/emulator.py                      | 2 +-
 cdist/exec/local.py                    | 2 +-
 cdist/exec/remote.py                   | 2 +-
 cdist/hostsource.py                    | 2 +-
 cdist/info.py                          | 3 +--
 cdist/inventory.py                     | 2 +-
 cdist/message.py                       | 2 +-
 cdist/preos.py                         | 2 +-
 cdist/preos/debootstrap/debootstrap.py | 2 +-
 cdist/shell.py                         | 4 +---
 cdist/test/config/__init__.py          | 2 +-
 cdist/util/fsproperty.py               | 4 ++--
 18 files changed, 19 insertions(+), 22 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index b2d72f05..982d8b75 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -70,7 +70,7 @@ def _graph_dfs_cycle(graph, node, path):
     return False
 
 
-class Config(object):
+class Config:
     """Cdist main class to hold arbitrary data"""
 
     # list of paths (files and/or directories) that will be removed on finish
diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index 114a47e0..51d61e04 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -47,7 +47,7 @@ class MissingObjectIdError(cdist.Error):
         return '%s' % (self.message)
 
 
-class CdistObject(object):
+class CdistObject:
     """Represents a cdist object.
 
     All interaction with objects in cdist should be done through this class.
diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py
index 4500f50d..c0329c8a 100644
--- a/cdist/core/cdist_type.py
+++ b/cdist/core/cdist_type.py
@@ -38,7 +38,7 @@ class InvalidTypeError(cdist.Error):
                 self.type_path, self.type_absolute_path, self.source_path)
 
 
-class CdistType(object):
+class CdistType:
     """Represents a cdist type.
 
     All interaction with types in cdist should be done through this class.
diff --git a/cdist/core/code.py b/cdist/core/code.py
index 1550880a..226bc63d 100644
--- a/cdist/core/code.py
+++ b/cdist/core/code.py
@@ -92,7 +92,7 @@ code-remote
 '''
 
 
-class Code(object):
+class Code:
     """Generates and executes cdist code scripts.
 
     """
diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py
index 353d7681..a3baa959 100644
--- a/cdist/core/explorer.py
+++ b/cdist/core/explorer.py
@@ -63,7 +63,7 @@ type explorer is:
 '''
 
 
-class Explorer(object):
+class Explorer:
     """Executes cdist explorers.
 
     """
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 8b833ff2..2a0b2189 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -92,7 +92,7 @@ class NoInitialManifestError(cdist.Error):
         return repr(self.message)
 
 
-class Manifest(object):
+class Manifest:
     """Executes cdist manifests.
 
     """
diff --git a/cdist/emulator.py b/cdist/emulator.py
index 4eaf2c93..24d239fa 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -56,7 +56,7 @@ class DefaultList(list):
             return cls(initial.split('\n'))
 
 
-class Emulator(object):
+class Emulator:
     def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ):
         self.argv = argv
         self.stdin = stdin
diff --git a/cdist/exec/local.py b/cdist/exec/local.py
index ad6c6e36..e0aab190 100644
--- a/cdist/exec/local.py
+++ b/cdist/exec/local.py
@@ -39,7 +39,7 @@ import cdist.exec.util as util
 CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ]
 
 
-class Local(object):
+class Local:
     """Execute commands locally.
 
     All interaction with the local side should be done through this class.
diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py
index f72bf3bf..e5af2f34 100644
--- a/cdist/exec/remote.py
+++ b/cdist/exec/remote.py
@@ -49,7 +49,7 @@ class DecodeError(cdist.Error):
         return "Cannot decode output of " + " ".join(self.command)
 
 
-class Remote(object):
+class Remote:
     """Execute commands remotely.
 
     All interaction with the remote side should be done through this class.
diff --git a/cdist/hostsource.py b/cdist/hostsource.py
index a7b8f0b4..5f927b36 100644
--- a/cdist/hostsource.py
+++ b/cdist/hostsource.py
@@ -41,7 +41,7 @@ def hostfile_process_line(line, strip_func=str.strip):
         return None
 
 
-class HostSource(object):
+class HostSource:
     """
     Host source object.
     Source can be a sequence or filename (stdin if \'-\').
diff --git a/cdist/info.py b/cdist/info.py
index b896a3d1..a1fad237 100644
--- a/cdist/info.py
+++ b/cdist/info.py
@@ -29,8 +29,7 @@ import glob
 import fnmatch
 
 
-class Info(object):
-
+class Info:
     def __init__(self, conf_dirs, args):
         self.conf_dirs = conf_dirs
         self.all = args.all
diff --git a/cdist/inventory.py b/cdist/inventory.py
index 138a2034..c06efff0 100644
--- a/cdist/inventory.py
+++ b/cdist/inventory.py
@@ -80,7 +80,7 @@ def rstrip_nl(s):
     return str.rstrip(s, "\n")
 
 
-class Inventory(object):
+class Inventory:
     """Inventory main class"""
 
     def __init__(self, db_basedir=dist_inventory_db, configuration=None):
diff --git a/cdist/message.py b/cdist/message.py
index 450fc3c3..ffa8c2bb 100644
--- a/cdist/message.py
+++ b/cdist/message.py
@@ -27,7 +27,7 @@ import tempfile
 log = logging.getLogger(__name__)
 
 
-class Message(object):
+class Message:
     """Support messaging between types
 
     """
diff --git a/cdist/preos.py b/cdist/preos.py
index bf2a8e60..f8a5dd67 100644
--- a/cdist/preos.py
+++ b/cdist/preos.py
@@ -78,7 +78,7 @@ def get_available_preoses_string(cls):
     return "Available PreOS-es:\n{}".format("\n".join(preoses))
 
 
-class PreOS(object):
+class PreOS:
     preoses = None
 
     @classmethod
diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py
index f53dd4a7..ac01e08b 100644
--- a/cdist/preos/debootstrap/debootstrap.py
+++ b/cdist/preos/debootstrap/debootstrap.py
@@ -31,7 +31,7 @@ import os
 import subprocess
 
 
-class Debian(object):
+class Debian:
     _preos_name = 'debian'
     _cdist_preos = True
 
diff --git a/cdist/shell.py b/cdist/shell.py
index 60b6a9f0..04a68937 100644
--- a/cdist/shell.py
+++ b/cdist/shell.py
@@ -32,10 +32,8 @@ import cdist.config
 log = logging.getLogger(__name__)
 
 
-class Shell(object):
-
+class Shell:
     def __init__(self, shell=None):
-
         self.shell = shell
 
         self.target_host = (
diff --git a/cdist/test/config/__init__.py b/cdist/test/config/__init__.py
index 499593e3..0ed614b1 100644
--- a/cdist/test/config/__init__.py
+++ b/cdist/test/config/__init__.py
@@ -44,7 +44,7 @@ expected_object_names = sorted([
     '__third/moon'])
 
 
-class CdistObjectErrorContext(object):
+class CdistObjectErrorContext:
     def __init__(self, original_error):
         self.original_error = original_error
 
diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py
index 5a27c9d7..1d76fd76 100644
--- a/cdist/util/fsproperty.py
+++ b/cdist/util/fsproperty.py
@@ -177,7 +177,7 @@ class DirectoryDict(collections.MutableMapping):
             raise cdist.Error(str(e))
 
 
-class FileBasedProperty(object):
+class FileBasedProperty:
     attribute_class = None
 
     def __init__(self, path):
@@ -189,7 +189,7 @@ class FileBasedProperty(object):
 
         Usage with a sublcass:
 
-        class Foo(object):
+        class Foo:
             # note that the actual DirectoryDict is stored as __parameters
             # on the instance
             parameters = DirectoryDictProperty(

From 6aae58dea77002dea0b57d20cbb01b5e2a1066b5 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 21 Jun 2020 17:35:28 +0200
Subject: [PATCH 310/743] [type/__package_opkg] Mark lock variables readonly

---
 cdist/conf/type/__package_opkg/explorer/pkg_status | 4 ++--
 cdist/conf/type/__package_opkg/gencode-remote      | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status
index f5a6f098..9d993055 100755
--- a/cdist/conf/type/__package_opkg/explorer/pkg_status
+++ b/cdist/conf/type/__package_opkg/explorer/pkg_status
@@ -23,9 +23,9 @@
 # Retrieve the status of a package - parses opkg output
 #
 
-__type_path=${__object%%${__object_id}*}
+readonly __type_path=${__object%%${__object_id}*}
+readonly LOCKFILE="${__type_path}/cdist_opkg.lock"
 
-LOCKFILE="${__type_path}/cdist_opkg.lock"
 _lock() (
 	set -o noclobber
 	until echo $$>"${LOCKFILE}"
diff --git a/cdist/conf/type/__package_opkg/gencode-remote b/cdist/conf/type/__package_opkg/gencode-remote
index ad90dc24..c7f21751 100755
--- a/cdist/conf/type/__package_opkg/gencode-remote
+++ b/cdist/conf/type/__package_opkg/gencode-remote
@@ -47,9 +47,9 @@ then
 fi
 
 cat <<'EOF'
-__type_path=${__object%%${__object_id}*}
+readonly __type_path=${__object%%${__object_id}*}
+readonly LOCKFILE="${__type_path}/cdist_opkg.lock"
 
-LOCKFILE="${__type_path}/cdist_opkg.lock"
 _lock() (
 	set -o noclobber
 	until echo $$>"${LOCKFILE}"

From 26dfdf37c21ce3c469ae0d6a717d390f422f9fbf Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 21 Jun 2020 01:13:30 +0300
Subject: [PATCH 311/743] [__download] support multiple checksum formats and
 download utilities, add --onchange and other minor changes

---
 cdist/conf/type/__download/explorer/state     | 78 +++++++++++++++----
 cdist/conf/type/__download/gencode-local      | 25 +++++-
 cdist/conf/type/__download/gencode-remote     |  7 ++
 cdist/conf/type/__download/man.rst            | 25 ++++--
 .../type/__download/parameter/default/cmd-get |  1 -
 .../type/__download/parameter/default/cmd-sum |  1 -
 cdist/conf/type/__download/parameter/optional |  1 +
 7 files changed, 116 insertions(+), 22 deletions(-)
 create mode 100755 cdist/conf/type/__download/gencode-remote
 delete mode 100644 cdist/conf/type/__download/parameter/default/cmd-get
 delete mode 100644 cdist/conf/type/__download/parameter/default/cmd-sum

diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 6a50f5a5..00362545 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -2,19 +2,71 @@
 
 dst="/$__object_id"
 
-# shellcheck disable=SC2059
-cmd="$( printf "$( cat "$__object/parameter/cmd-sum" )" "$dst" )"
-
-sum="$( cat "$__object/parameter/sum" )"
-
-if [ -f "$dst" ]
+if [ ! -f "$dst" ]
 then
-    if [ "$( eval "$cmd" )" = "$sum" ]
-    then
-        echo 'present'
-    else
-        echo 'mismatch'
-    fi
-else
     echo 'absent'
+    exit 0
+fi
+
+sum_should="$( cat "$__object/parameter/sum" )"
+
+if [ -f "$__object/parameter/cmd-sum" ]
+then
+    # shellcheck disable=SC2059
+    sum_is="$( eval "$( printf \
+        "$( cat "$__object/parameter/cmd-sum" )" \
+        "$dst" )" )"
+else
+    os="$( "$__explorer/os" )"
+
+    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+    then
+        sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
+
+    elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="md5:$( md5 -q "$dst" )"
+            ;;
+            *)
+                sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+
+    elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="sha1:$( sha1 -q "$dst" )"
+            ;;
+            *)
+                sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+
+    elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="sha256:$( sha256 -q "$dst" )"
+            ;;
+            *)
+                sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+    fi
+fi
+
+if [ -z "$sum_is" ]
+then
+    echo 'no checksum from target' >&2
+    exit 1
+fi
+
+if [ "$sum_is" = "$sum_should" ]
+then
+    echo 'present'
+else
+    echo 'mismatch'
 fi
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 49e9c699..85ef3a60 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -9,12 +9,31 @@ fi
 
 url="$( cat "$__object/parameter/url" )"
 
-cmd="$( cat "$__object/parameter/cmd-get" )"
-
 tmp="$( mktemp )"
 
 dst="/$__object_id"
 
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cmd="$( cat "$__object/parameter/cmd-get" )"
+
+elif command -v wget > /dev/null
+then
+    cmd="wget -O - '%s'"
+
+elif command -v curl > /dev/null
+then
+    cmd="curl -o - '%s'"
+
+elif command -v fetch > /dev/null
+then
+    cmd="fetch -o - '%s'"
+
+else
+    echo 'no usable locally installed utility for downloading' >&2
+    exit 1
+fi
+
 printf "$cmd > %s\n" \
     "$url" \
     "$tmp"
@@ -33,3 +52,5 @@ printf '%s %s %s:%s\n' \
     "$dst"
 
 echo "rm -f '$tmp'"
+
+echo 'downloaded' > "$__messages_out"
diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
new file mode 100755
index 00000000..b08d0050
--- /dev/null
+++ b/cdist/conf/type/__download/gencode-remote
@@ -0,0 +1,7 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/onchange" ] \
+    && grep -Fq "$__object_id:downloaded" "$__messages_in"
+then
+    cat "$__object/parameter/onchange"
+fi
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index c973448f..63a41bc4 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -10,7 +10,13 @@ DESCRIPTION
 -----------
 You must use persistent storage in target host for destination file
 (``$__object_id``) because it will be used for checksum calculation
-in order to decide if file must be downloaded.
+in order to decide if file must be (re-)downloaded.
+
+By default type will try to use following locally installed utilities
+for downloading (in order): ``wget``, ``curl`` or ``fetch``.
+
+Environment variables like ``{http,https,ftp}_proxy`` etc can be used on
+cdist execution (``http_proxy=foo cdist config ...``).
 
 
 REQUIRED PARAMETERS
@@ -19,20 +25,29 @@ url
    URL from which to download the file.
 
 sum
-   Checksum of downloaded file.
+   Checksum of file going to be downloaded.
+   By default output of ``cksum`` without filename is expected.
+   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
+
+onchange
+   Execute this command after download.
 
 
 OPTIONAL PARAMETERS
 -------------------
 cmd-get
    Command used for downloading.
-   Default is ``wget -O- '%s'``.
    Command must output to ``stdout``.
+   Parameter will be used for ``printf`` and must include only one
+   variable ``%s`` which will become URL.
+   For example: ``wget -O - '%s'``.
 
 cmd-sum
    Command used for checksum calculation.
-   Default is ``md5sum '%s' | awk '{print $1}'``.
    Command output and ``--sum`` parameter must match.
+   Parameter will be used for ``printf`` and must include only one
+   variable ``%s`` which will become destination.
+   For example: ``md5sum '%s' | awk '{print $1}'``.
 
 
 EXAMPLES
@@ -45,7 +60,7 @@ EXAMPLES
     require='__directory/opt/cpma' \
         __download /opt/cpma/cnq3.zip \
             --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
-            --sum 46da3021ca9eace277115ec9106c5b46
+            --sum md5:46da3021ca9eace277115ec9106c5b46
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
diff --git a/cdist/conf/type/__download/parameter/default/cmd-get b/cdist/conf/type/__download/parameter/default/cmd-get
deleted file mode 100644
index 2daa38a1..00000000
--- a/cdist/conf/type/__download/parameter/default/cmd-get
+++ /dev/null
@@ -1 +0,0 @@
-wget -O- '%s'
diff --git a/cdist/conf/type/__download/parameter/default/cmd-sum b/cdist/conf/type/__download/parameter/default/cmd-sum
deleted file mode 100644
index 3e8a9295..00000000
--- a/cdist/conf/type/__download/parameter/default/cmd-sum
+++ /dev/null
@@ -1 +0,0 @@
-md5sum '%s' | awk '{print $1}'
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index 22783e02..38c0ce4d 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,2 +1,3 @@
 cmd-get
 cmd-sum
+onchange

From 3649555f3522ea80f1999ca2cb7f8ddd112cbb33 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 22 Jun 2020 09:31:59 +0200
Subject: [PATCH 312/743] [type/__package_opkg] Do not lock execution of
 code-remote (revert)

Instead, rely on `nonparallel`.
In any case cdist should never run explorer and code concurrently even if the
dependency graph would allow to do so as it would result in many more
synchronization issues than this one.
---
 cdist/conf/type/__package_opkg/gencode-remote | 32 ++-----------------
 1 file changed, 2 insertions(+), 30 deletions(-)

diff --git a/cdist/conf/type/__package_opkg/gencode-remote b/cdist/conf/type/__package_opkg/gencode-remote
index c7f21751..28caff71 100755
--- a/cdist/conf/type/__package_opkg/gencode-remote
+++ b/cdist/conf/type/__package_opkg/gencode-remote
@@ -25,9 +25,9 @@
 
 if test -f "${__object}/parameter/name"
 then
-   name=$(cat "${__object}/parameter/name")
+	name=$(cat "${__object}/parameter/name")
 else
-   name=$__object_id
+	name=$__object_id
 fi
 
 state_should=$(cat "${__object}/parameter/state")
@@ -46,34 +46,6 @@ then
 	exit 0
 fi
 
-cat <<'EOF'
-readonly __type_path=${__object%%${__object_id}*}
-readonly LOCKFILE="${__type_path}/cdist_opkg.lock"
-
-_lock() (
-	set -o noclobber
-	until echo $$>"${LOCKFILE}"
-	do
-		while test -f "${LOCKFILE}"; do sleep 1; done
-	done
-
-) 2>/dev/null
-_unlock() {
-	if test -s "${LOCKFILE}" && test "$(cat "${LOCKFILE}")" = $$
-	then
-		rm "${LOCKFILE}"
-	fi
-}
-EOF
-
-# NOTE: We need to lock parallel execution of code-remote to ensure that it is
-# not executed concurrently with a type explorer.
-# opkg will try to acquire the OPKG lock (usually /var/lock/opkg.lock) using
-# lockf(2) for every operation.
-# It will not wait for the lock but terminate with an error leading to an
-# incorrect outcome.
-echo 'trap _unlock EXIT'
-echo '_lock'
 
 case $state_should
 in

From 49dde11def71cec121e5f698d0fb6ed3a539f97a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 24 Jun 2020 07:04:06 +0200
Subject: [PATCH 313/743] Remove deprecated __pf_apply

---
 cdist/conf/type/__pf_apply/deprecated     |  1 -
 cdist/conf/type/__pf_apply/explorer/rcvar | 36 ---------------
 cdist/conf/type/__pf_apply/gencode-remote | 51 ---------------------
 cdist/conf/type/__pf_apply/man.rst        | 55 -----------------------
 cdist/conf/type/__pf_apply/singleton      |  0
 5 files changed, 143 deletions(-)
 delete mode 100644 cdist/conf/type/__pf_apply/deprecated
 delete mode 100755 cdist/conf/type/__pf_apply/explorer/rcvar
 delete mode 100755 cdist/conf/type/__pf_apply/gencode-remote
 delete mode 100644 cdist/conf/type/__pf_apply/man.rst
 delete mode 100644 cdist/conf/type/__pf_apply/singleton

diff --git a/cdist/conf/type/__pf_apply/deprecated b/cdist/conf/type/__pf_apply/deprecated
deleted file mode 100644
index 36cfed90..00000000
--- a/cdist/conf/type/__pf_apply/deprecated
+++ /dev/null
@@ -1 +0,0 @@
-Consider moving to __pf_apply_anchor. Get in touch if you need __pf_apply.
diff --git a/cdist/conf/type/__pf_apply/explorer/rcvar b/cdist/conf/type/__pf_apply/explorer/rcvar
deleted file mode 100755
index 7c8d535f..00000000
--- a/cdist/conf/type/__pf_apply/explorer/rcvar
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/sh
-#
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Get the location of the pf ruleset on the target host.
-#
-
-# Debug
-#exec >&2
-#set -x
-
-# Check /etc/rc.conf for pf's configuration file name. Default to /etc/pf.conf
-
-RC="/etc/rc.conf"
-PFCONF="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')"
-echo "${PFCONF:-"/etc/pf.conf"}"
-
-# Debug
-#set +x
-
diff --git a/cdist/conf/type/__pf_apply/gencode-remote b/cdist/conf/type/__pf_apply/gencode-remote
deleted file mode 100755
index c8f7a25a..00000000
--- a/cdist/conf/type/__pf_apply/gencode-remote
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/sh -e
-#
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Apply pf(4) ruleset on *BSD
-#
-
-# Debug
-#exec >&2
-#set -x
-
-rcvar=$(cat "$__object/explorer/rcvar")
-
-cat <<EOF
-if [ -f "${rcvar}.old" ]; then # rcvar.old exists, we must need to disable pf
-   # Disable pf
-   # If it already is disabled, pfctl -d returns 1, go on with life
-   pfctl -d || true
-   # Cleanup
-   rm -f "${rcvar}.old"
-elif [ -f "${rcvar}.new" ]; then # rcvar.new exists, we must need to apply it
-   # Ensure that pf is enabled in the first place
-   # If it already is enabled, pfctl -e returns 1, go on with life
-   mv "${rcvar}.new" "${rcvar}"
-   pfctl -e || true
-   pfctl -f "${rcvar}"
-   if [ "\$?" -ne "0" ]; then # failed to configure new ruleset
-      echo "Failed to configure the new ruleset on ${__target_host}!" >&2
-   fi
-fi
-EOF
-
-# Debug
-#set +x
-
diff --git a/cdist/conf/type/__pf_apply/man.rst b/cdist/conf/type/__pf_apply/man.rst
deleted file mode 100644
index eee345e7..00000000
--- a/cdist/conf/type/__pf_apply/man.rst
+++ /dev/null
@@ -1,55 +0,0 @@
-cdist-type__pf_apply(7)
-=======================
-
-NAME
-----
-cdist-type__pf_apply - Apply pf(4) ruleset on \*BSD
-
-
-DESCRIPTION
------------
-This type is used on \*BSD systems to manage the pf firewall's active ruleset.
-
-
-REQUIRED PARAMETERS
--------------------
-NONE
-
-
-OPTIONAL PARAMETERS
--------------------
-NONE
-
-
-EXAMPLES
---------
-
-.. code-block:: sh
-
-    # Modify the ruleset on $__target_host:
-    __pf_ruleset --state present --source /my/pf/ruleset.conf
-    require="__pf_ruleset" \
-       __pf_apply
-
-    # Remove the ruleset on $__target_host (implies disabling pf(4):
-    __pf_ruleset --state absent
-    require="__pf_ruleset" \
-       __pf_apply
-
-
-SEE ALSO
---------
-:strong:`pf`\ (4), :strong:`cdist-type__pf_ruleset`\ (7)
-
-
-AUTHORS
--------
-Jake Guffey <jake.guffey--@--eprotex.com>
-
-
-COPYING
--------
-Copyright \(C) 2012 Jake Guffey. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
diff --git a/cdist/conf/type/__pf_apply/singleton b/cdist/conf/type/__pf_apply/singleton
deleted file mode 100644
index e69de29b..00000000

From a9778965be7154c0f42cf0170142ee2ec571d7ec Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 24 Jun 2020 08:47:22 +0200
Subject: [PATCH 314/743] [type/__package_opkg] Use mkdir(1) to lock instead of
 noclobber

noclobber is potentially unsafe, because it relies on the underlying shell to
implement noclobber in a safe way that avoids race conditions between multiple
processes.
mkdir is safer because it is mandated by POSIX to "fail" if the target already
exists.
---
 .../type/__package_opkg/explorer/pkg_status   | 25 ++++++++++++-------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status
index 9d993055..09c550e5 100755
--- a/cdist/conf/type/__package_opkg/explorer/pkg_status
+++ b/cdist/conf/type/__package_opkg/explorer/pkg_status
@@ -24,21 +24,28 @@
 #
 
 readonly __type_path=${__object%%${__object_id}*}
-readonly LOCKFILE="${__type_path}/cdist_opkg.lock"
+test -d "${__type_path}" || { echo 'Cannot determine __type_path' >&2; exit 1; }
+readonly LOCKDIR="${__type_path:?}/.cdist_opkg.lock.dir"
 
-_lock() (
-	set -o noclobber
-	until echo $$>"${LOCKFILE}"
+_lock() {
+	until mkdir "${LOCKDIR:?}" 2>/dev/null
 	do
-		while test -f "${LOCKFILE}"; do sleep 1; done
+		while test -d "${LOCKDIR}"
+		do
+			# DEBUG: printf 'Locked by PID: %u\n' "$(cat "${LOCKDIR}/pid")"
+			sleep 1
+		done
 	done
-
-) 2>/dev/null
+	echo $$ >"${LOCKDIR:?}/pid"
+}
 _unlock() {
-	if test -s "${LOCKFILE}" && test "$(cat "${LOCKFILE}")" = $$
+	test -d "${LOCKDIR}" || return 0
+	if test -s "${LOCKDIR}/pid"
 	then
-		rm "${LOCKFILE}"
+		test "$(cat "${LOCKDIR}/pid")" = $$ || return 1
+		rm "${LOCKDIR:?}/pid"
 	fi
+	rmdir "${LOCKDIR:?}"
 }
 
 

From 5364d3bc9037969c415140523c5cac2f8891b967 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 24 Jun 2020 20:49:48 +0200
Subject: [PATCH 315/743] [type/__package_opkg] Implement flock locking if
 available

---
 .../type/__package_opkg/explorer/pkg_status   | 55 ++++++++++++-------
 1 file changed, 36 insertions(+), 19 deletions(-)

diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status
index 09c550e5..de7b896b 100755
--- a/cdist/conf/type/__package_opkg/explorer/pkg_status
+++ b/cdist/conf/type/__package_opkg/explorer/pkg_status
@@ -25,28 +25,45 @@
 
 readonly __type_path=${__object%%${__object_id}*}
 test -d "${__type_path}" || { echo 'Cannot determine __type_path' >&2; exit 1; }
-readonly LOCKDIR="${__type_path:?}/.cdist_opkg.lock.dir"
+readonly LOCKFILE="${__type_path:?}/.cdist_opkg.lock"
 
-_lock() {
-	until mkdir "${LOCKDIR:?}" 2>/dev/null
-	do
-		while test -d "${LOCKDIR}"
+if command -v flock >/dev/null 2>&1
+then
+	# use flock (if available) on FD 9
+	_lock() {
+		exec 9<>"${LOCKFILE:?}"
+		flock -x 9
+		echo $$>&9
+	}
+	_unlock() {
+		:>"${LOCKFILE:?}"
+		flock -u 9
+		exec 9<&-
+	}
+else
+	# fallback to mkdir if flock is missing
+	_lock() {
+		until mkdir "${LOCKFILE:?}.dir" 2>/dev/null
 		do
-			# DEBUG: printf 'Locked by PID: %u\n' "$(cat "${LOCKDIR}/pid")"
-			sleep 1
+			while test -d "${LOCKFILE}.dir"
+			do
+				# DEBUG:
+				# printf 'Locked by PID: %u\n' "$(cat "${LOCKFILE}.dir/pid")"
+				sleep 1
+			done
 		done
-	done
-	echo $$ >"${LOCKDIR:?}/pid"
-}
-_unlock() {
-	test -d "${LOCKDIR}" || return 0
-	if test -s "${LOCKDIR}/pid"
-	then
-		test "$(cat "${LOCKDIR}/pid")" = $$ || return 1
-		rm "${LOCKDIR:?}/pid"
-	fi
-	rmdir "${LOCKDIR:?}"
-}
+		echo $$ >"${LOCKFILE:?}.dir/pid"
+	}
+	_unlock() {
+		test -d "${LOCKFILE}.dir" || return 0
+		if test -s "${LOCKFILE}.dir/pid"
+		then
+			test "$(cat "${LOCKFILE}.dir/pid")" = $$ || return 1
+			rm "${LOCKFILE:?}.dir/pid"
+		fi
+		rmdir "${LOCKFILE:?}.dir"
+	}
+fi
 
 
 if test -f "${__object}/parameter/name"

From 7074f9c395d3ab6f763682f860744ac8d12a8c26 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 25 Jun 2020 06:32:10 +0200
Subject: [PATCH 316/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index a0de09fa..f1713d3b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,8 @@ Changelog
 
 next:
 	* New type: __download (Ander Punnar)
+	* Type __locale_system: Add devuan support (Dennis Camera)
+	* Type __package_opkg: Add locking (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 077989e8fd749339da9a6ca8458b2c73ac71672c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 27 Jun 2020 15:55:04 +0200
Subject: [PATCH 317/743] Remove annoying warnings

Those warnings don't have any specail meaning and usage.
Resolve #825.
---
 cdist/util/ipaddr.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py
index 9b730225..95ca74ee 100644
--- a/cdist/util/ipaddr.py
+++ b/cdist/util/ipaddr.py
@@ -45,8 +45,6 @@ def resolve_target_host_name(host, family=0):
         log.debug("derived host_name for host \"{}\": {}".format(
             host, host_name))
     except (socket.gaierror, socket.herror) as e:
-        log.warning("Could not derive host_name for {}"
-                    ", $host_name will be empty. Error is: {}".format(host, e))
         # in case of error provide empty value
         host_name = ''
     return host_name
@@ -59,8 +57,6 @@ def resolve_target_fqdn(host):
         log.debug("derived host_fqdn for host \"{}\": {}".format(
             host, host_fqdn))
     except socket.herror as e:
-        log.warning("Could not derive host_fqdn for {}"
-                    ", $host_fqdn will be empty. Error is: {}".format(host, e))
         # in case of error provide empty value
         host_fqdn = ''
     return host_fqdn

From 85614aabd6f3abef5af6d362203db956c9d72aa9 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 28 Jun 2020 16:38:15 +0300
Subject: [PATCH 318/743] [__download] add --download (local|remote), update
 manual

---
 .../conf/type/__download/explorer/remote_cmd  | 19 +++++++++++++++
 cdist/conf/type/__download/gencode-local      |  4 +++-
 cdist/conf/type/__download/gencode-remote     | 22 +++++++++++++++--
 cdist/conf/type/__download/man.rst            | 24 ++++++++++++-------
 cdist/conf/type/__download/manifest           |  6 +++++
 .../__download/parameter/default/download     |  1 +
 cdist/conf/type/__download/parameter/optional |  1 +
 7 files changed, 65 insertions(+), 12 deletions(-)
 create mode 100755 cdist/conf/type/__download/explorer/remote_cmd
 create mode 100755 cdist/conf/type/__download/manifest
 create mode 100644 cdist/conf/type/__download/parameter/default/download

diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd
new file mode 100755
index 00000000..fbd4d84c
--- /dev/null
+++ b/cdist/conf/type/__download/explorer/remote_cmd
@@ -0,0 +1,19 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cmd="$( cat "$__object/parameter/cmd-get" )"
+
+elif command -v curl > /dev/null
+then
+    cmd="curl -o - '%s'"
+
+elif command -v fetch > /dev/null
+then
+    cmd="fetch -o - '%s'"
+
+else
+    cmd="wget -O - '%s'"
+fi
+
+echo "$cmd"
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 85ef3a60..339827c2 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -1,8 +1,10 @@
 #!/bin/sh -e
 
+download="$( cat "$__object/parameter/download" )"
+
 state_is="$( cat "$__object/explorer/state" )"
 
-if [ "$state_is" = 'present' ]
+if [ "$download" != 'local' ] || [ "$state_is" = 'present' ]
 then
     exit 0
 fi
diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
index b08d0050..89ba72af 100755
--- a/cdist/conf/type/__download/gencode-remote
+++ b/cdist/conf/type/__download/gencode-remote
@@ -1,7 +1,25 @@
 #!/bin/sh -e
 
-if [ -f "$__object/parameter/onchange" ] \
-    && grep -Fq "$__object_id:downloaded" "$__messages_in"
+download="$( cat "$__object/parameter/download" )"
+
+state_is="$( cat "$__object/explorer/state" )"
+
+if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
+then
+    cmd="$( cat "$__object/explorer/remote_cmd" )"
+
+    url="$( cat "$__object/parameter/url" )"
+
+    dst="/$__object_id"
+
+    printf "$cmd > %s\n" \
+        "$url" \
+        "$dst"
+
+    echo 'downloaded' > "$__messages_out"
+fi
+
+if [ -f "$__object/parameter/onchange" ] && [ "$state" != "present" ]
 then
     cat "$__object/parameter/onchange"
 fi
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index 63a41bc4..c161f4e4 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -3,26 +3,28 @@ cdist-type__download(7)
 
 NAME
 ----
-cdist-type__download - Download file to local storage and copy it to target host
+cdist-type__download - Download a file
 
 
 DESCRIPTION
 -----------
-You must use persistent storage in target host for destination file
-(``$__object_id``) because it will be used for checksum calculation
-in order to decide if file must be (re-)downloaded.
+Persistent storage for destination file in target host must be used
+(``$__object_id``) because it will be used for checksum calculation in
+order to decide if file must be (re-)downloaded.
 
-By default type will try to use following locally installed utilities
-for downloading (in order): ``wget``, ``curl`` or ``fetch``.
+By default type will try to use ``wget``, ``curl`` or ``fetch`` for
+downloading.  If ``--download remote`` type will fallback to (and
+install) ``wget``.
 
-Environment variables like ``{http,https,ftp}_proxy`` etc can be used on
-cdist execution (``http_proxy=foo cdist config ...``).
+If ``--download local`` (default), then environment variables like
+``{http,https,ftp}_proxy`` etc can be used on cdist execution
+(``http_proxy=foo cdist config ...``).
 
 
 REQUIRED PARAMETERS
 -------------------
 url
-   URL from which to download the file.
+   File's URL.
 
 sum
    Checksum of file going to be downloaded.
@@ -35,6 +37,10 @@ onchange
 
 OPTIONAL PARAMETERS
 -------------------
+download
+   If ``local`` (default), then download file to local storage and copy
+   it to target host. If ``remote``, then download happens in target.
+
 cmd-get
    Command used for downloading.
    Command must output to ``stdout``.
diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest
new file mode 100755
index 00000000..7ec8d86d
--- /dev/null
+++ b/cdist/conf/type/__download/manifest
@@ -0,0 +1,6 @@
+#!/bin/sh -e
+
+if grep -Eq '^wget' "$__object/explorer/remote_cmd"
+then
+    __package wget
+fi
diff --git a/cdist/conf/type/__download/parameter/default/download b/cdist/conf/type/__download/parameter/default/download
new file mode 100644
index 00000000..40830374
--- /dev/null
+++ b/cdist/conf/type/__download/parameter/default/download
@@ -0,0 +1 @@
+local
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index 38c0ce4d..838e2fbf 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,3 +1,4 @@
 cmd-get
 cmd-sum
+download
 onchange

From b6bf90e3f1a775ec1bd27a99015107d24a397787 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 28 Jun 2020 16:43:45 +0300
Subject: [PATCH 319/743] [__download] update manual

---
 cdist/conf/type/__download/man.rst | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index c161f4e4..eafa9dfc 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -8,15 +8,14 @@ cdist-type__download - Download a file
 
 DESCRIPTION
 -----------
-Persistent storage for destination file in target host must be used
-(``$__object_id``) because it will be used for checksum calculation in
-order to decide if file must be (re-)downloaded.
+Destination (``$__object_id``) in target host must be persistent storage
+in order to calculate checksum and decide if file must be (re-)downloaded.
 
-By default type will try to use ``wget``, ``curl`` or ``fetch`` for
-downloading.  If ``--download remote`` type will fallback to (and
-install) ``wget``.
+By default type will try to use ``wget``, ``curl`` or ``fetch``.
+If download happens in target (see ``--download``) then type will
+fallback to (and install) ``wget``.
 
-If ``--download local`` (default), then environment variables like
+If download happens in local machine, then environment variables like
 ``{http,https,ftp}_proxy`` etc can be used on cdist execution
 (``http_proxy=foo cdist config ...``).
 

From a263fdfe584bbba467086a3b2da3dfdd58ccc7fe Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 2 Oct 2019 17:52:20 +0200
Subject: [PATCH 320/743] [__hosts] Add --alias parameter

The --alias parameter allows to specify a hostname and multiple aliases on a
single /etc/hosts line.
---
 cdist/conf/type/__hosts/man.rst               | 17 +++++++---
 cdist/conf/type/__hosts/manifest              | 34 ++++++++++++-------
 .../type/__hosts/parameter/optional_multiple  |  1 +
 3 files changed, 35 insertions(+), 17 deletions(-)
 create mode 100644 cdist/conf/type/__hosts/parameter/optional_multiple

diff --git a/cdist/conf/type/__hosts/man.rst b/cdist/conf/type/__hosts/man.rst
index bece7967..1ac706cb 100644
--- a/cdist/conf/type/__hosts/man.rst
+++ b/cdist/conf/type/__hosts/man.rst
@@ -25,6 +25,10 @@ ip
     state is ``present``, this parameter is mandatory, if state is
     ``absent``, this parameter is silently ignored.
 
+alias
+    An alias for the hostname.
+    This parameter can be specified multiple times (once per alias).
+
 EXAMPLES
 --------
 
@@ -36,6 +40,8 @@ EXAMPLES
     # previously configured via __hosts.
     __hosts happy --state absent
 
+    __hosts srv1.example.com --ip 192.168.0.42 --alias srv1
+
 SEE ALSO
 --------
 
@@ -43,13 +49,14 @@ SEE ALSO
 
 AUTHORS
 -------
-
-Dmitry Bogatov <KAction@gnu.org>
+| Dmitry Bogatov <KAction@gnu.org>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
 
-Copyright (C) 2015,2016 Dmitry Bogatov. Free use of this software is granted
-under the terms of the GNU General Public License version 3 or later
-(GPLv3+).
+Copyright \(C) 2015-2016 Dmitry Bogatov, 2019 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__hosts/manifest b/cdist/conf/type/__hosts/manifest
index c536b83b..0d9e61f8 100755
--- a/cdist/conf/type/__hosts/manifest
+++ b/cdist/conf/type/__hosts/manifest
@@ -1,29 +1,39 @@
 #!/bin/sh -e
-# Copyright (C) 2015  Bogatov Dmitry <KAction@gnu.org>
 #
-# This program is free software: you can redistribute it and/or modify
+# Copyright (C) 2015  Bogatov Dmitry <KAction@gnu.org>
+# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-set -ue
+#
 
-hostname="$__object_id"
-state="$(cat "$__object/parameter/state")"
+set -e -u
+
+hostname=$__object_id
+state=$(cat "$__object/parameter/state")
 marker="# __hosts/$hostname"
 
-set -- "__hosts/$hostname" --file /etc/hosts --state "$state"
-
-if [ "$state" = absent ] ; then
-	__line "$@" --regex "$marker"
+if [ "$state" = 'absent' ]
+then
+	set -- --regex "$marker"
 else
-	ip="$(cat "$__object/parameter/ip")"
-	__line "$@" --line "$ip $hostname $marker"
+	ip=$(cat "$__object/parameter/ip")
+	aliases=$(while read -r a; do printf '\t%s' "$a"; done <"$__object/parameter/alias")
+
+	set -- --line "$(printf '%s\t%s%s  %s' \
+		"$ip" "$hostname" "$aliases" "$marker")"
 fi
+
+__line "__hosts/$hostname" --file /etc/hosts --state "$state" "$@"
diff --git a/cdist/conf/type/__hosts/parameter/optional_multiple b/cdist/conf/type/__hosts/parameter/optional_multiple
new file mode 100644
index 00000000..d077ed80
--- /dev/null
+++ b/cdist/conf/type/__hosts/parameter/optional_multiple
@@ -0,0 +1 @@
+alias

From 999e7b013449c51b848bb9365205810fc1a80fbd Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 28 Jan 2020 09:15:56 +0100
Subject: [PATCH 321/743] [type/__user] Fix shadow explorer for OpenBSD

---
 cdist/conf/type/__user/explorer/shadow | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__user/explorer/shadow b/cdist/conf/type/__user/explorer/shadow
index 73ce0e29..d516ed10 100755
--- a/cdist/conf/type/__user/explorer/shadow
+++ b/cdist/conf/type/__user/explorer/shadow
@@ -23,18 +23,24 @@
 
 name=$__object_id
 
-case $("$__explorer/os") in
-  'freebsd'|'netbsd'|'openbsd')
+case $("${__explorer}/os") in
+  freebsd|netbsd)
     database='passwd'
     ;;
-  # Default to using shadow passwords
+  openbsd)
+    database='master.passwd'
+    ;;
   *)
+    # Default to using shadow passwords
     database='shadow'
     ;;
 esac
 
-if command -v getent >/dev/null; then
-  getent "$database" "$name" || true
-elif [ -f /etc/shadow ]; then
-  grep "^${name}:" /etc/shadow || true
+if command -v getent >/dev/null 2>&1
+then
+  getent "${database}" "${name}" 2>/dev/null && exit || true  # fallback to file
+fi
+if test -n "${database}" -a -f "/etc/${database}"
+then
+  grep -e "^${name}:" "/etc/${database}" && exit || true  # ignore failure
 fi

From 6467ccbdccd380c11e37e747216bfc3aabc1b776 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 30 Jun 2020 14:31:11 +0200
Subject: [PATCH 322/743] [type/__user] Make shellcheck happy

---
 cdist/conf/type/__user/explorer/shadow | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__user/explorer/shadow b/cdist/conf/type/__user/explorer/shadow
index d516ed10..32496ee7 100755
--- a/cdist/conf/type/__user/explorer/shadow
+++ b/cdist/conf/type/__user/explorer/shadow
@@ -38,9 +38,10 @@ esac
 
 if command -v getent >/dev/null 2>&1
 then
+  # shellcheck disable=SC2015
   getent "${database}" "${name}" 2>/dev/null && exit || true  # fallback to file
 fi
 if test -n "${database}" -a -f "/etc/${database}"
 then
-  grep -e "^${name}:" "/etc/${database}" && exit || true  # ignore failure
+  grep -e "^${name}:" "/etc/${database}" || true  # ignore failure
 fi

From 3860f1feeaa2844191eeea84444963818bb42c4b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 26 Jun 2020 14:33:16 +0200
Subject: [PATCH 323/743] [type/{__file/__directory}] Support
 setuid,setguid,sticky bits

---
 cdist/conf/type/__directory/explorer/stat  | 17 +++++++++--------
 cdist/conf/type/__directory/gencode-remote |  4 ++--
 cdist/conf/type/__file/explorer/stat       | 14 +++++++-------
 cdist/conf/type/__file/gencode-remote      |  4 ++--
 4 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat
index 105d894f..a7dc8431 100755
--- a/cdist/conf/type/__directory/explorer/stat
+++ b/cdist/conf/type/__directory/explorer/stat
@@ -33,7 +33,7 @@ fallback() {
    group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
-   mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }')
+   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
 
    printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\n' \
       "$("$__type_explorer/type")" \
@@ -51,13 +51,14 @@ then
    exit
 fi
 
-case $("$__explorer/os") in
-   "freebsd"|"netbsd"|"openbsd"|"macosx")
-      stat -f "type: %HT
+case $("$__explorer/os")
+in
+   freebsd|netbsd|openbsd|macosx)
+      stat -f 'type: %HT
 owner: %Du %Su
 group: %Dg %Sg
-mode: %Lp %Sp
-" "$destination" | awk '/^type/ { print tolower($0); next } { print }'
+mode: %Mp%03Lp %Sp
+' "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
     solaris)
         ls1="$( ls -ld "$destination" )"
@@ -92,9 +93,9 @@ mode: %Lp %Sp
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
       #       we fall through to the shell fallback.
-       stat -c "type: %F
+       stat -c 'type: %F
 owner: %u %U
 group: %g %G
-mode: %a %A" "$destination" 2>/dev/null || fallback
+mode: %04a %A' "$destination" 2>/dev/null || fallback
    ;;
 esac
diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote
index a1a32ea2..2c2c56fd 100755
--- a/cdist/conf/type/__directory/gencode-remote
+++ b/cdist/conf/type/__directory/gencode-remote
@@ -97,9 +97,9 @@ case "$state_should" in
             value_should="$(cat "$__object/parameter/$attribute")"
             value_is="$(get_current_value "$attribute" "$value_should")"
 
-            # change 0xxx format to xxx format => same as stat returns
+            # format mode in four digits => same as stat returns
             if [ "$attribute" = mode ]; then
-                value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')"
+                value_should=$(printf '%04u' "${value_should}")
             fi
 
             if [ "$set_attributes" = 1 ] || [ "$value_should" != "$value_is" ]; then
diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat
index 91c8cc84..231768f6 100755
--- a/cdist/conf/type/__file/explorer/stat
+++ b/cdist/conf/type/__file/explorer/stat
@@ -34,7 +34,7 @@ fallback() {
    group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
-   mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }')
+   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
 
    size=$(echo "$ls_line" | awk '{ print $5 }')
    links=$(echo "$ls_line" | awk '{ print $2 }')
@@ -63,13 +63,13 @@ fi
 case $("$__explorer/os")
 in
    freebsd|netbsd|openbsd|macosx)
-      stat -f "type: %HT
+      stat -f 'type: %HT
 owner: %Du %Su
 group: %Dg %Sg
-mode: %Lp %Sp
+mode: %Mp%03Lp %Sp
 size: %Dz
 links: %Dl
-" "$destination" | awk '/^type/ { print tolower($0); next } { print }'
+' "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
     solaris)
         ls1="$( ls -ld "$destination" )"
@@ -106,11 +106,11 @@ links: %Dl
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
       #       we fall through to the shell fallback.
-      stat -c "type: %F
+      stat -c 'type: %F
 owner: %u %U
 group: %g %G
-mode: %a %A
+mode: %04a %A
 size: %s
-links: %h" "$destination" 2>/dev/null || fallback
+links: %h' "$destination" 2>/dev/null || fallback
          ;;
 esac
diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index 815593bd..a69154df 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -68,9 +68,9 @@ case "$state_should" in
             if [ -f "$__object/parameter/$attribute" ]; then
                 value_should="$(cat "$__object/parameter/$attribute")"
 
-                # change 0xxx format to xxx format => same as stat returns
+                # format mode in four digits => same as stat returns
                 if [ "$attribute" = mode ]; then
-                    value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')"
+                    value_should=$(printf '%04u' "${value_should}")
                 fi
 
                 value_is="$(get_current_value "$attribute" "$value_should")"

From 88400551f9490a4f8dc9b918e962e5ab3bba1286 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 30 Jun 2020 23:59:45 +0200
Subject: [PATCH 324/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index f1713d3b..b823421b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,8 @@ next:
 	* New type: __download (Ander Punnar)
 	* Type __locale_system: Add devuan support (Dennis Camera)
 	* Type __package_opkg: Add locking (Dennis Camera)
+	* Type __hosts: Add --alias parameter (Dennis Camera)
+	* Type __user: Fix shadow explorer for OpenBSD (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From fe193ecab8addf9b1c5b6d3c06c11ae9f9803378 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 1 Jul 2020 14:05:45 +0200
Subject: [PATCH 325/743] Make code consistent

* Remove supreflous checking and warning message.
* Fix cache recording.
---
 cdist/emulator.py | 65 +++++++++++++++--------------------------------
 1 file changed, 21 insertions(+), 44 deletions(-)

diff --git a/cdist/emulator.py b/cdist/emulator.py
index adbcbd9d..9fe84056 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -91,10 +91,6 @@ class Emulator:
         self.type_name = os.path.basename(argv[0])
         self.cdist_type = core.CdistType(self.type_base_path, self.type_name)
 
-        # If set then object alreay exists and this var holds existing
-        # requirements.
-        self._existing_reqs = None
-
         self.__init_log()
 
     def run(self):
@@ -230,9 +226,6 @@ class Emulator:
                 self.parameters[key] = value
 
         if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env:
-            # Make existing requirements a set so that we can compare it
-            # later with new requirements.
-            self._existing_reqs = set(self.cdist_object.requirements)
             obj_params = self._object_params_in_context()
             if obj_params != self.parameters:
                 errmsg = ("Object %s already exists with conflicting "
@@ -251,23 +244,26 @@ class Emulator:
             else:
                 self.cdist_object.create()
             self.cdist_object.parameters = self.parameters
-            # record the created object in typeorder file
-            with open(self.typeorder_path, 'a') as typeorderfile:
-                print(self.cdist_object.name, file=typeorderfile)
-            # record the created object in parent object typeorder file
-            __object_name = self.env.get('__object_name', None)
-            depname = self.cdist_object.name
-            if __object_name:
-                parent = self.cdist_object.object_from_name(__object_name)
-                parent.typeorder.append(self.cdist_object.name)
-                if self._order_dep_on():
-                    self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
-                                    ' for %s'), depname, parent.name)
-                    parent.typeorder_dep.append(depname)
-            elif self._order_dep_on():
-                self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
-                               depname)
-                self._add_typeorder_dep(depname)
+        # Do the following recording even if object exists, but with
+        # different requirements.
+
+        # record the created object in typeorder file
+        with open(self.typeorder_path, 'a') as typeorderfile:
+            print(self.cdist_object.name, file=typeorderfile)
+        # record the created object in parent object typeorder file
+        __object_name = self.env.get('__object_name', None)
+        depname = self.cdist_object.name
+        if __object_name:
+            parent = self.cdist_object.object_from_name(__object_name)
+            parent.typeorder.append(self.cdist_object.name)
+            if self._order_dep_on():
+                self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
+                                ' for %s'), depname, parent.name)
+                parent.typeorder_dep.append(depname)
+        elif self._order_dep_on():
+            self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
+                           depname)
+            self._add_typeorder_dep(depname)
 
         # Record / Append source
         self.cdist_object.source.append(self.object_source)
@@ -322,8 +318,6 @@ class Emulator:
         # This ensures pattern matching is done against sanitised list
         self.cdist_object.requirements.append(cdist_object.name)
 
-        return cdist_object.name
-
     def _order_dep_on(self):
         return os.path.exists(self.order_dep_state_path)
 
@@ -392,7 +386,6 @@ class Emulator:
                 # so do not set a requirement
                 pass
 
-        reqs = set()
         if "require" in self.env:
             requirements = self.env['require']
             self.log.debug("reqs = " + requirements)
@@ -400,23 +393,7 @@ class Emulator:
                 # Ignore empty fields - probably the only field anyway
                 if len(requirement) == 0:
                     continue
-                object_name = self.record_requirement(requirement)
-                reqs.add(object_name)
-        if self._existing_reqs is not None:
-            # If object exists then compare existing and new requirements.
-            if self._existing_reqs != reqs:
-                warnmsg = ("Object {} already exists with requirements:\n"
-                           "{}: {}\n"
-                           "{}: {}\n"
-                           "Dependency resolver could not handle dependencies "
-                           "as expected.".format(
-                               self.cdist_object.name,
-                               " ".join(self.cdist_object.source),
-                               self._existing_reqs,
-                               self.object_source,
-                               reqs
-                           ))
-                self.log.warning(warnmsg)
+                self.record_requirement(requirement)
 
     def record_auto_requirements(self):
         """An object shall automatically depend on all objects that it

From 93506d2113e1d05202527639efe883a5b44b220d Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 8 Jul 2020 00:17:12 +0300
Subject: [PATCH 326/743] [__download] curl follow redirects

---
 cdist/conf/type/__download/explorer/remote_cmd | 2 +-
 cdist/conf/type/__download/gencode-local       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd
index fbd4d84c..e3e35b45 100755
--- a/cdist/conf/type/__download/explorer/remote_cmd
+++ b/cdist/conf/type/__download/explorer/remote_cmd
@@ -6,7 +6,7 @@ then
 
 elif command -v curl > /dev/null
 then
-    cmd="curl -o - '%s'"
+    cmd="curl -L -o - '%s'"
 
 elif command -v fetch > /dev/null
 then
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 339827c2..571d2c3c 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -25,7 +25,7 @@ then
 
 elif command -v curl > /dev/null
 then
-    cmd="curl -o - '%s'"
+    cmd="curl -L -o - '%s'"
 
 elif command -v fetch > /dev/null
 then

From e9062662868d7677a235889ba39a6091085388f0 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 8 Jul 2020 00:20:55 +0300
Subject: [PATCH 327/743] [__download] s/variable/format specification/

---
 cdist/conf/type/__download/man.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index eafa9dfc..6ec0b19a 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -44,14 +44,14 @@ cmd-get
    Command used for downloading.
    Command must output to ``stdout``.
    Parameter will be used for ``printf`` and must include only one
-   variable ``%s`` which will become URL.
+   format specification ``%s`` which will become URL.
    For example: ``wget -O - '%s'``.
 
 cmd-sum
    Command used for checksum calculation.
    Command output and ``--sum`` parameter must match.
    Parameter will be used for ``printf`` and must include only one
-   variable ``%s`` which will become destination.
+   format specification ``%s`` which will become destination.
    For example: ``md5sum '%s' | awk '{print $1}'``.
 
 

From cb9933b4a0d343a059899101810b6ee86fc87dbc Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 8 Jul 2020 12:43:55 +0200
Subject: [PATCH 328/743] Fix state -> state_is

---
 cdist/conf/type/__download/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
index 89ba72af..029a0801 100755
--- a/cdist/conf/type/__download/gencode-remote
+++ b/cdist/conf/type/__download/gencode-remote
@@ -19,7 +19,7 @@ then
     echo 'downloaded' > "$__messages_out"
 fi
 
-if [ -f "$__object/parameter/onchange" ] && [ "$state" != "present" ]
+if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ]
 then
     cat "$__object/parameter/onchange"
 fi

From b8752e9ee390e983c1ce6467761aca8101b6548d Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 10 Jul 2020 21:03:35 +0200
Subject: [PATCH 329/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index b823421b..54c42bde 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Type __package_opkg: Add locking (Dennis Camera)
 	* Type __hosts: Add --alias parameter (Dennis Camera)
 	* Type __user: Fix shadow explorer for OpenBSD (Dennis Camera)
+	* Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From a5ae26116bd611fcab76c0383998fb4b05f1ae1d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 11 Jul 2020 18:57:47 +0200
Subject: [PATCH 330/743] [type/__hosts] Fix when used without --alias

---
 cdist/conf/type/__hosts/manifest | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/type/__hosts/manifest b/cdist/conf/type/__hosts/manifest
index 0d9e61f8..8103ebd5 100755
--- a/cdist/conf/type/__hosts/manifest
+++ b/cdist/conf/type/__hosts/manifest
@@ -19,21 +19,24 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-set -e -u
+set -e
 
 hostname=$__object_id
-state=$(cat "$__object/parameter/state")
-marker="# __hosts/$hostname"
+state=$(cat "${__object}/parameter/state")
+marker="# __hosts/${hostname}"
 
-if [ "$state" = 'absent' ]
+if test "${state}" != 'absent'
 then
-	set -- --regex "$marker"
-else
-	ip=$(cat "$__object/parameter/ip")
-	aliases=$(while read -r a; do printf '\t%s' "$a"; done <"$__object/parameter/alias")
+	ip=$(cat "${__object}/parameter/ip")
+	if test -s "${__object}/parameter/alias"
+	then
+		aliases=$(while read -r a; do printf '\t%s' "$a"; done <"$__object/parameter/alias")
+	fi
 
 	set -- --line "$(printf '%s\t%s%s  %s' \
-		"$ip" "$hostname" "$aliases" "$marker")"
+		"${ip}" "${hostname}" "${aliases}" "${marker}")"
+else
+	set -- --regex "$(echo "${marker}" | sed -e 's/\./\\./')$"
 fi
 
-__line "__hosts/$hostname" --file /etc/hosts --state "$state" "$@"
+__line "/etc/hosts:${hostname}" --file /etc/hosts --state "${state}" "$@"

From 19514662b022c79c4e6a56573c32dad45055310c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 12 Jul 2020 12:24:00 +0200
Subject: [PATCH 331/743] [type/{__file/__directory}] Fix typo

---
 cdist/conf/type/__directory/explorer/stat | 2 +-
 cdist/conf/type/__file/explorer/stat      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat
index a7dc8431..422c5819 100755
--- a/cdist/conf/type/__directory/explorer/stat
+++ b/cdist/conf/type/__directory/explorer/stat
@@ -30,7 +30,7 @@ fallback() {
    gid=$(echo "$ls_line" | awk '{ print $4 }')
 
    owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd)
-   group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
+   group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
    mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat
index 231768f6..3f971488 100755
--- a/cdist/conf/type/__file/explorer/stat
+++ b/cdist/conf/type/__file/explorer/stat
@@ -31,7 +31,7 @@ fallback() {
    gid=$(echo "$ls_line" | awk '{ print $4 }')
 
    owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd)
-   group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
+   group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
    mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')

From 9fb7e151b889214576cb891595a68ca8c6196d45 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 12 Jul 2020 12:31:55 +0200
Subject: [PATCH 332/743] [type/{__file/__directory}] Remove special Solaris
 blocks

Solaris 11 has GNU stat (handled by *)
Solaris 10 (and older?) does not have stat (handled by failing command -v stat)

On Solaris 10 (at least on UFS), setgid cannot be set on directories.
Unlike on other systems `chmod 2400` is not `-r----S---`, but `-r----l---`.
---
 cdist/conf/type/__directory/explorer/stat | 40 +++--------------------
 cdist/conf/type/__file/explorer/stat      | 40 +++--------------------
 2 files changed, 9 insertions(+), 71 deletions(-)

diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat
index 422c5819..f817cb02 100755
--- a/cdist/conf/type/__directory/explorer/stat
+++ b/cdist/conf/type/__directory/explorer/stat
@@ -33,7 +33,7 @@ fallback() {
    group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
-   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
+   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[lst]/)*2^(9+i/3)}printf("%04o",k)}')
 
    printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\n' \
       "$("$__type_explorer/type")" \
@@ -45,11 +45,10 @@ fallback() {
 # nothing to work with, nothing we could do
 [ -e "$destination" ] || exit 0
 
-if ! command -v stat >/dev/null
-then
+command -v stat >/dev/null 2>&1 || {
    fallback
    exit
-fi
+}
 
 case $("$__explorer/os")
 in
@@ -60,42 +59,13 @@ group: %Dg %Sg
 mode: %Mp%03Lp %Sp
 ' "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
-    solaris)
-        ls1="$( ls -ld "$destination" )"
-        ls2="$( ls -ldn "$destination" )"
-
-        if [ -f "$__object/parameter/mode" ]
-        then mode_should="$( cat "$__object/parameter/mode" )"
-        fi
-
-        # yes, it is ugly hack, but if you know better way...
-        if [ -z "$( find "$destination" -perm "$mode_should" )" ]
-        then octets=888
-        else octets="$( echo "$mode_should" | sed 's/^0//' )"
-        fi
-
-        case "$( echo "$ls1" | cut -c1-1 )" in
-            -) echo 'type: regular file' ;;
-            d) echo 'type: directory' ;;
-        esac
-
-        echo "owner: $( echo "$ls2" \
-            | awk '{print $3}' ) $( echo "$ls1" \
-                | awk '{print $3}' )"
-
-        echo "group: $( echo "$ls2" \
-            | awk '{print $4}' ) $( echo "$ls1" \
-                | awk '{print $4}' )"
-
-        echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )"
-    ;;
    *)
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
       #       we fall through to the shell fallback.
-       stat -c 'type: %F
+      stat -c 'type: %F
 owner: %u %U
 group: %g %G
 mode: %04a %A' "$destination" 2>/dev/null || fallback
-   ;;
+      ;;
 esac
diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat
index 3f971488..29b3c8a3 100755
--- a/cdist/conf/type/__file/explorer/stat
+++ b/cdist/conf/type/__file/explorer/stat
@@ -34,7 +34,7 @@ fallback() {
    group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
-   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}')
+   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[lst]/)*2^(9+i/3)}printf("%04o",k)}')
 
    size=$(echo "$ls_line" | awk '{ print $5 }')
    links=$(echo "$ls_line" | awk '{ print $2 }')
@@ -53,11 +53,10 @@ fallback() {
 [ -e "$destination" ] || exit 0
 
 
-if ! command -v stat >/dev/null
-then
+command -v stat >/dev/null 2>&1 || {
    fallback
    exit
-fi
+}
 
 
 case $("$__explorer/os")
@@ -71,37 +70,6 @@ size: %Dz
 links: %Dl
 ' "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
-    solaris)
-        ls1="$( ls -ld "$destination" )"
-        ls2="$( ls -ldn "$destination" )"
-
-        if [ -f "$__object/parameter/mode" ]
-        then mode_should="$( cat "$__object/parameter/mode" )"
-        fi
-
-        # yes, it is ugly hack, but if you know better way...
-        if [ -z "$( find "$destination" -perm "$mode_should" )" ]
-        then octets=888
-        else octets="$( echo "$mode_should" | sed 's/^0//' )"
-        fi
-
-        case "$( echo "$ls1" | cut -c1-1 )" in
-            -) echo 'type: regular file' ;;
-            d) echo 'type: directory' ;;
-        esac
-
-        echo "owner: $( echo "$ls2" \
-            | awk '{print $3}' ) $( echo "$ls1" \
-                | awk '{print $3}' )"
-
-        echo "group: $( echo "$ls2" \
-            | awk '{print $4}' ) $( echo "$ls1" \
-                | awk '{print $4}' )"
-
-        echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )"
-        echo "size: $( echo "$ls1" | awk '{print $5}' )"
-        echo "links: $( echo "$ls1" | awk '{print $2}' )"
-    ;;
    *)
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
@@ -112,5 +80,5 @@ group: %g %G
 mode: %04a %A
 size: %s
 links: %h' "$destination" 2>/dev/null || fallback
-         ;;
+      ;;
 esac

From bc970731316a59c21f8aa784fb27f1fa1b13ab89 Mon Sep 17 00:00:00 2001
From: Darko Poljak <darko.poljak@ungleich.ch>
Date: Mon, 13 Jul 2020 05:44:26 +0000
Subject: [PATCH 333/743] Merge branch 'bugfix/postfix-master-option' into
 '6.6'

Fix broken --option parameter in __postfix_master type

See merge request ungleich-public/cdist!905

(cherry picked from commit 2f433a1458f3a1f7f8859e9ae165178a0ec5b7a0)

9496b234 The option parameter is actually multi-valued
4009bbd7 Protect postfix variables in options
---
 cdist/conf/type/__postfix_master/gencode-remote              | 2 +-
 cdist/conf/type/__postfix_master/parameter/optional          | 1 -
 cdist/conf/type/__postfix_master/parameter/optional_multiple | 1 +
 3 files changed, 2 insertions(+), 2 deletions(-)
 create mode 100644 cdist/conf/type/__postfix_master/parameter/optional_multiple

diff --git a/cdist/conf/type/__postfix_master/gencode-remote b/cdist/conf/type/__postfix_master/gencode-remote
index 7c109a69..73de1088 100755
--- a/cdist/conf/type/__postfix_master/gencode-remote
+++ b/cdist/conf/type/__postfix_master/gencode-remote
@@ -67,7 +67,7 @@ case "$state_should" in
          remove_entry
       fi
       cat << DONE
-cat >> "$config" << ${__type##*/}_DONE
+cat >> "$config" << "${__type##*/}_DONE"
 $(cat "$entry")
 ${__type##*/}_DONE
 DONE
diff --git a/cdist/conf/type/__postfix_master/parameter/optional b/cdist/conf/type/__postfix_master/parameter/optional
index 792b42c5..410482b8 100644
--- a/cdist/conf/type/__postfix_master/parameter/optional
+++ b/cdist/conf/type/__postfix_master/parameter/optional
@@ -4,6 +4,5 @@ unpriv
 chroot
 wakeup
 maxproc
-option
 comment
 state
diff --git a/cdist/conf/type/__postfix_master/parameter/optional_multiple b/cdist/conf/type/__postfix_master/parameter/optional_multiple
new file mode 100644
index 00000000..01925a15
--- /dev/null
+++ b/cdist/conf/type/__postfix_master/parameter/optional_multiple
@@ -0,0 +1 @@
+option

From 8903540e9106db93e490515c07929a207326be77 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 13 Jul 2020 07:54:12 +0200
Subject: [PATCH 334/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 54c42bde..dac9cbce 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,8 @@ next:
 	* Type __hosts: Add --alias parameter (Dennis Camera)
 	* Type __user: Fix shadow explorer for OpenBSD (Dennis Camera)
 	* Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak)
+	* Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera)
+	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 3965c7f73844873aabb1d839b2298eed70a6b35c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 21 Jul 2020 19:42:40 +0200
Subject: [PATCH 335/743] [type/__user] Install user{add,mod,del} packages on
 OpenWrt

---
 cdist/conf/type/__user/manifest | 42 +++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/type/__user/manifest b/cdist/conf/type/__user/manifest
index 8f10b38c..b9fad65b 100644
--- a/cdist/conf/type/__user/manifest
+++ b/cdist/conf/type/__user/manifest
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,16 +18,37 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
 # Manage users.
+#
 
-os=$(cat "$__global/explorer/os")
-
-case "$os" in
-    alpine)
-        __package shadow
-        ;;
-    *)
-        :
-        ;;
+case $(cat "${__global}/explorer/os")
+in
+	(alpine)
+		__package shadow
+		;;
+	(openwrt)
+		case $(cat "${__object}/parameter/state")
+		in
+			(present)
+				if test -s "${__object}/explorer/passwd"
+				then
+					# NOTE: The package might not be required if no changes
+					# are required, but determining if changes are required is
+					# out of scope here, and 40k should be okay, I hope.
+					__package shadow-usermod
+				else
+					__package shadow-useradd
+				fi
+				;;
+			(absent)
+				if test -s "${__object}/explorer/passwd"
+				then
+					__package shadow-userdel
+				fi
+				;;
+		esac
+		;;
+	(*)
+		:
+		;;
 esac

From d8b5c733f6cec871ceb1933807f4877a4ee77ffa Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 22 Jul 2020 06:36:27 +0200
Subject: [PATCH 336/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index dac9cbce..040de9d8 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,6 +10,7 @@ next:
 	* Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak)
 	* Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera)
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
+	* Type __user: Install user packages on OpenWRT (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From fdef468f1ab6f2163da22a477df154a2082b57b4 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 22 Jul 2020 18:28:41 +0200
Subject: [PATCH 337/743] Fix OpenWrt spelling

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 040de9d8..4c17ee58 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,7 +10,7 @@ next:
 	* Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak)
 	* Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera)
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
-	* Type __user: Install user packages on OpenWRT (Dennis Camera)
+	* Type __user: Install user packages on OpenWrt (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 595e43b8d5f06641179c13de8ac04be46c1b2820 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 23 Jul 2020 09:41:53 +0200
Subject: [PATCH 338/743] [type/{__file,__directory}] Fix incorrect
 interpretation of strings with leading 0s as octal

---
 cdist/conf/type/__directory/gencode-remote | 4 +++-
 cdist/conf/type/__file/gencode-remote      | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote
index 2c2c56fd..d9c00b56 100755
--- a/cdist/conf/type/__directory/gencode-remote
+++ b/cdist/conf/type/__directory/gencode-remote
@@ -99,7 +99,9 @@ case "$state_should" in
 
             # format mode in four digits => same as stat returns
             if [ "$attribute" = mode ]; then
-                value_should=$(printf '%04u' "${value_should}")
+                # Convert to four-digit octal number (printf interprets
+                # strings with leading 0s as octal!)
+                value_should=$(printf '%04o' "0${value_should}")
             fi
 
             if [ "$set_attributes" = 1 ] || [ "$value_should" != "$value_is" ]; then
diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index a69154df..35356b13 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -70,7 +70,9 @@ case "$state_should" in
 
                 # format mode in four digits => same as stat returns
                 if [ "$attribute" = mode ]; then
-                    value_should=$(printf '%04u' "${value_should}")
+                    # Convert to four-digit octal number (printf interprets
+                    # strings with leading 0s as octal!)
+                    value_should=$(printf '%04o' "0${value_should}")
                 fi
 
                 value_is="$(get_current_value "$attribute" "$value_should")"

From ae5f0bba0b103aabcfbad0f105c69c0b403697ac Mon Sep 17 00:00:00 2001
From: fnux <timothee.floure@ungleich.ch>
Date: Fri, 24 Jul 2020 12:26:35 +0200
Subject: [PATCH 339/743] Add Alpine support to __openldap_server

---
 cdist/conf/type/__openldap_server/man.rst  |  4 +--
 cdist/conf/type/__openldap_server/manifest | 42 ++++++++++++++++++----
 2 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
index fbad21d8..a96c7dad 100644
--- a/cdist/conf/type/__openldap_server/man.rst
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -103,8 +103,8 @@ syncrepl-host
     Set once per host that will replicate the directory.
 
 module
-    LDAP module to load. See `slapd.conf(5)`.
-    Default value is OS-dependent, see manifest.
+    LDAP module to load. See `slapd.conf(5)`. Some dependencies might have to
+    be installed beforehand. Default value is OS-dependent, see manifest.
 
 schema
     Name of LDAP schema to load. Must be the name without extension of a
diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest
index 84ba176f..2aeece26 100644
--- a/cdist/conf/type/__openldap_server/manifest
+++ b/cdist/conf/type/__openldap_server/manifest
@@ -25,6 +25,7 @@ case "${os}" in
         SLAPD_DATA_DIR="/var/db/openldap-data"
         SLAPD_RUN_DIR="/var/run/openldap"
         SLAPD_MODULE_PATH="/usr/local/libexec/openldap"
+        SLAPD_MODULE_TYPE="la"
         if [ -z "${slapd_modules}" ]; then
             # It looks like ppolicy and syncprov must be compiled
             slapd_modules="back_mdb back_monitor"
@@ -43,13 +44,34 @@ case "${os}" in
         SLAPD_DATA_DIR="/var/lib/ldap"
         SLAPD_RUN_DIR="/var/run/slapd"
         SLAPD_MODULE_PATH="/usr/lib/ldap"
+        SLAPD_MODULE_TYPE="la"
         if [ -z "${slapd_modules}" ]; then
             slapd_modules="back_mdb ppolicy syncprov back_monitor"
         fi
+        CONF_OWNER="openldap"
+        CONF_GROUP="openldap"
         if [ -z "${tls_cipher_suite}" ]; then
             tls_cipher_suite="NORMAL"
         fi
         ;;
+    alpine)
+        PKGS="openldap openldap-clients"
+        ETC="/etc"
+        SLAPD_DIR="/etc/openldap"
+        SLAPD_DATA_DIR="/var/lib/openldap"
+        SLAPD_RUN_DIR="/var/run/openldap"
+        SLAPD_MODULE_PATH="/usr/lib/openldap"
+        SLAPD_MODULE_TYPE="so"
+        if [ -z "${slapd_modules}" ]; then
+            slapd_modules="back_mdb ppolicy syncprov back_monitor"
+            PKGS="$PKGS openldap-back-mdb openldap-back-monitor openldap-overlay-all"
+        fi
+        CONF_OWNER="ldap"
+        CONF_GROUP="$SLAPD_USER"
+        if [ -z "${tls_cipher_suite}" ]; then
+            tls_cipher_suite="DEFAULT"
+        fi
+        ;;
     *)
         echo "Don't know the openldap defaults for: $os" >&2
         exit 1
@@ -156,6 +178,12 @@ case "${os}" in
                --line "SLAPD_SERVICES=\"${slapd_urls}\"" \
                --state present
         ;;
+    alpine)
+        require="__package/${PKG_MAIN}" __line add_slapd_services \
+               --file ${ETC}/conf.d/slapd \
+               --line "command_args=\"-h '${slapd_urls}'\"" \
+               --state present
+        ;;
     *)
         # Nothing to do here, move on.
         ;;
@@ -170,20 +198,22 @@ if [ -z "${_skip_letsencrypt_cert}" ]; then
     fi
 
     # shellcheck disable=SC2086
-    __letsencrypt_cert "${name}" --admin-email "${admin_email}" \
-        --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R openldap:openldap ${SLAPD_DIR}/sasl2 && service slapd restart" \
-        --automatic-renewal ${staging}
+    __directory ${SLAPD_DIR}/sasl2
+    require="__directory/${SLAPD_DIR}/sasl2" __letsencrypt_cert "${name}" \
+        --admin-email "${admin_email}" \
+        --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R ${CONF_OWNER}:${CONF_GROUP} ${SLAPD_DIR}/sasl2 && service slapd restart" \
+        --automatic-renewal "${staging}"
 fi
 
 require="__package/${PKG_MAIN}" __directory ${SLAPD_DIR}/slapd.d --state absent
 
 if [ -z "${_skip_letsencrypt_cert}" ]; then
     require="__package/${PKG_MAIN} __letsencrypt_cert/${name}" \
-           __file ${SLAPD_DIR}/slapd.conf --owner ${CONF_OWNER} --group ${CONF_GROUP} --mode 644 \
+           __file "${SLAPD_DIR}/slapd.conf" --owner "${CONF_OWNER}" --group "${CONF_GROUP}" --mode 644 \
            --source "${ldapconf}"
 else
     require="__package/${PKG_MAIN}" \
-           __file ${SLAPD_DIR}/slapd.conf --owner ${CONF_OWNER} --group ${CONF_GROUP} --mode 644 \
+           __file "${SLAPD_DIR}/slapd.conf" --owner "${CONF_OWNER}" --group "${CONF_GROUP}" --mode 644 \
            --source "${ldapconf}"
 fi
 
@@ -210,7 +240,7 @@ done
 # Add specified modules
 echo "modulepath ${SLAPD_MODULE_PATH}" >> "${ldapconf}"
 for module in ${slapd_modules}; do
-    echo "moduleload ${module}.la" >> "${ldapconf}"
+    echo "moduleload ${module}.${SLAPD_MODULE_TYPE}" >> "${ldapconf}"
 done
 
 # Rest of the config

From 8654cbe4661762f25cd75c484f28344ce91ec55b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 24 Jul 2020 12:29:02 +0200
Subject: [PATCH 340/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 4c17ee58..0b9b8308 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,7 @@ next:
 	* Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera)
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
 	* Type __user: Install user packages on OpenWrt (Dennis Camera)
+	* Type __openldap_server: Add Alpine support (Timothée Floure)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 8b53f35ffab4e7b74d3fd019e7602c8a05338173 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 24 Jul 2020 12:33:16 +0200
Subject: [PATCH 341/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 0b9b8308..2457e90b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,7 @@ next:
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
 	* Type __user: Install user packages on OpenWrt (Dennis Camera)
 	* Type __openldap_server: Add Alpine support (Timothée Floure)
+	* Type __pf_apply: Remove deprecated type (Darko Poljak)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From ee71cad047ba0c86343fe8ee9ea2bc79bf950256 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 25 Jul 2020 19:19:38 +0200
Subject: [PATCH 342/743] [type/__package_apt] Fix type for legacy APT versions

--no-install-recommends was introduced with Debian 5.
The APT::Install-Recommends option gets ignored by old versions and
produces no error.
---
 cdist/conf/type/__package_apt/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote
index e02564a2..f3d91566 100755
--- a/cdist/conf/type/__package_apt/gencode-remote
+++ b/cdist/conf/type/__package_apt/gencode-remote
@@ -64,7 +64,7 @@ esac
 
 # Hint if we need to avoid questions at some point:
 # DEBIAN_PRIORITY=critical can reduce the number of questions
-aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes --no-install-recommends -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
+aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o APT::Install-Recommends=0 -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
 
 if [ "$state_is" = "$state_should" ]; then
     if [ -z "$version" ] || [ "$version" = "$version_is" ]; then

From 46d09392f08b0659557048c02e109f189ffd85f3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 19:36:34 +0200
Subject: [PATCH 343/743] [type/__key_value] Get AWK from POSIX PATH

This is required here, because Solaris /usr/bin/awk does not support the
sub() function.
So xpg4 AWK needs to be used.
---
 cdist/conf/type/__key_value/explorer/state         | 4 +++-
 cdist/conf/type/__key_value/files/remote_script.sh | 5 ++++-
 cdist/conf/type/__key_value/gencode-remote         | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__key_value/explorer/state b/cdist/conf/type/__key_value/explorer/state
index 7b2de1df..d24600af 100755
--- a/cdist/conf/type/__key_value/explorer/state
+++ b/cdist/conf/type/__key_value/explorer/state
@@ -40,7 +40,9 @@ else
 fi
 export key state delimiter value exact_delimiter
 
-awk -f - "$file" <<"AWK_EOF"
+awk_bin=$(PATH=$(getconf PATH 2>/dev/null) && command -v awk || echo awk)
+
+"${awk_bin}" -f - "$file" <<"AWK_EOF"
 BEGIN {
     state=ENVIRON["state"]
     key=ENVIRON["key"]
diff --git a/cdist/conf/type/__key_value/files/remote_script.sh b/cdist/conf/type/__key_value/files/remote_script.sh
index f7a1add5..faf080cb 100644
--- a/cdist/conf/type/__key_value/files/remote_script.sh
+++ b/cdist/conf/type/__key_value/files/remote_script.sh
@@ -24,7 +24,10 @@ if [ -f "$file" ]; then
 else
     touch "$file"
 fi
-awk -f - "$file" >"$tmpfile" <<"AWK_EOF"
+
+awk_bin=$(PATH=$(getconf PATH 2>/dev/null) && command -v awk || echo awk)
+
+"${awk_bin}" -f - "$file" >"$tmpfile" <<"AWK_EOF"
 BEGIN {
     # import variables in a secure way ..
     state=ENVIRON["state"]
diff --git a/cdist/conf/type/__key_value/gencode-remote b/cdist/conf/type/__key_value/gencode-remote
index 13cc27c7..1174400e 100755
--- a/cdist/conf/type/__key_value/gencode-remote
+++ b/cdist/conf/type/__key_value/gencode-remote
@@ -25,7 +25,7 @@ state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
 fire_onchange=''
 
-if [  "$state_is" = "$state_should" ]; then
+if [ "$state_is" = "$state_should" ]; then
     exit 0
 fi
 

From a5905044365e0f28e793f01e0be0cac8e6b2b379 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 24 Jul 2020 10:19:10 +0200
Subject: [PATCH 344/743] [type/__locale_system] RedHat systems on systemd use
 /etc/locale.conf

---
 cdist/conf/type/__locale_system/manifest | 26 ++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 4a1fdeed..22531a40 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -3,6 +3,7 @@
 # 2012-2016 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2016 Carlos Ortigoza (carlos.ortigoza at ungleich.ch)
 # 2016 Nico Schottelius (nico.schottelius at ungleich.ch)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -32,8 +33,25 @@ case "$os" in
     archlinux)
         locale_conf="/etc/locale.conf"
         ;;
-    redhat|centos)
-        locale_conf="/etc/sysconfig/i18n"
+    centos|redhat|scientific)
+        # shellcheck source=/dev/null
+        version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
+        if expr "${version_id}" '>=' 7 >/dev/null
+        then
+            locale_conf="/etc/locale.conf"
+        else
+            locale_conf="/etc/sysconfig/i18n"
+        fi
+        ;;
+    fedora)
+        # shellcheck source=/dev/null
+        version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
+        if expr "${version_id}" '>=' 18 >/dev/null
+        then
+            locale_conf="/etc/locale.conf"
+        else
+            locale_conf="/etc/sysconfig/i18n"
+        fi
         ;;
     *)
         echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
@@ -47,9 +65,9 @@ __file "$locale_conf" \
        --state exists
 
 require="__file/$locale_conf" \
-       __key_value "$locale_conf:$__object_id" \
+__key_value "$locale_conf:$__object_id" \
        --file "$locale_conf" \
        --key "$__object_id" \
-       --delimiter = \
+       --delimiter '=' --exact_delimiter \
        --state "$(cat "$__object/parameter/state")" \
        --value "$(cat "$__object/parameter/value")"

From 47e28fc441187b5deb869f53396eac1ee0f8cbe7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 12:07:38 +0200
Subject: [PATCH 345/743] [type/__locale_system] Support old Debian derivatives

---
 cdist/conf/type/__locale_system/manifest | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 22531a40..b9991fa3 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -27,9 +27,29 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-    debian|devuan|ubuntu)
+    debian)
+        os_version=$(cat "${__global}/explorer/os_version")
+        if expr "${os_version}" '>=' 4 >/dev/null
+        then
+            # Debian 4 (etch) and later
+            locale_conf="/etc/default/locale"
+        else
+            locale_conf="/etc/environment"
+        fi
+        ;;
+    devuan)
         locale_conf="/etc/default/locale"
         ;;
+    ubuntu)
+        os_version=$(cat "${__global}/explorer/os_version")
+        if expr "${os_version}" '>=' 6.10 >/dev/null
+        then
+            # Ubuntu 6.10 (edgy) and later
+            locale_conf="/etc/default/locale"
+        else
+            locale_conf="/etc/environment"
+        fi
+        ;;
     archlinux)
         locale_conf="/etc/locale.conf"
         ;;
@@ -61,7 +81,7 @@ case "$os" in
 esac
 
 __file "$locale_conf" \
-       --owner root --group root --mode 644 \
+       --owner root --group root --mode 0644 \
        --state exists
 
 require="__file/$locale_conf" \

From 0ef54a721d6f540f70959df2b7f1321a5688a594 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 12:17:30 +0200
Subject: [PATCH 346/743] [type/__locale_system] Add support for Gentoo Linux

---
 cdist/conf/type/__locale_system/manifest | 45 ++++++++++++++++++------
 1 file changed, 34 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index b9991fa3..0049ed5f 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -24,9 +24,19 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
+onchange_cmd=  # none, by default
+quote_value=false
+
+catval() {
+    # shellcheck disable=SC2059
+    printf "$($quote_value && echo '"%s"' || echo '%s')" "$(cat "$1")"
+}
+
+
 os=$(cat "$__global/explorer/os")
 
-case "$os" in
+case $os
+in
     debian)
         os_version=$(cat "${__global}/explorer/os_version")
         if expr "${os_version}" '>=' 4 >/dev/null
@@ -69,10 +79,24 @@ case "$os" in
         if expr "${version_id}" '>=' 18 >/dev/null
         then
             locale_conf="/etc/locale.conf"
+            quote_value=false
         else
             locale_conf="/etc/sysconfig/i18n"
         fi
         ;;
+    gentoo)
+        case $(cat "${__global}/explorer/init")
+        in
+            (*openrc*)
+                locale_conf="/etc/env.d/02locale"
+                onchange_cmd="env-update --no-ldconfig"
+                quote_value=true
+                ;;
+            (systemd)
+                locale_conf="/etc/locale.conf"
+                ;;
+        esac
+        ;;
     *)
         echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
         echo "Please contribute an implementation for it if you can." >&2
@@ -80,14 +104,13 @@ case "$os" in
         ;;
 esac
 
-__file "$locale_conf" \
-       --owner root --group root --mode 0644 \
-       --state exists
+__file "${locale_conf}" --state exists --owner root --group root --mode 0644
 
-require="__file/$locale_conf" \
-__key_value "$locale_conf:$__object_id" \
-       --file "$locale_conf" \
-       --key "$__object_id" \
-       --delimiter '=' --exact_delimiter \
-       --state "$(cat "$__object/parameter/state")" \
-       --value "$(cat "$__object/parameter/value")"
+require="__file/${locale_conf}" \
+__key_value "${locale_conf}:${__object_id}" \
+    --file "${locale_conf}" \
+    --key "${__object_id}" \
+    --delimiter '=' --exact_delimiter \
+    --state "$(cat "${__object}/parameter/state")" \
+    --value "$(catval "${__object}/parameter/value")" \
+    --onchange "${onchange_cmd}"

From 630d987d5f5a40ae783fd9d4ff0a10653fcab04c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 12:28:20 +0200
Subject: [PATCH 347/743] [type/__locale_system] Add support for Void Linux

---
 cdist/conf/type/__locale_system/manifest | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 0049ed5f..ff0a8c23 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -97,6 +97,9 @@ in
                 ;;
         esac
         ;;
+    voidlinux)
+        locale_conf="/etc/locale.conf"
+        ;;
     *)
         echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
         echo "Please contribute an implementation for it if you can." >&2

From 0ae0935afafd6b10516e9689c19000ef729d1b9b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 14:05:36 +0200
Subject: [PATCH 348/743] [type/__locale_system] Add support for SuSE

---
 cdist/conf/type/__locale_system/manifest | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index ff0a8c23..26d4f7b5 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -24,6 +24,7 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
+key=$__object_id
 onchange_cmd=  # none, by default
 quote_value=false
 
@@ -97,6 +98,25 @@ in
                 ;;
         esac
         ;;
+    suse)
+        os_version=$(cat "${__global}/explorer/os_version")
+        os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
+
+        # https://documentation.suse.com/sles/15-SP2/html/SLES-all/cha-suse.html#sec-suse-l10n
+        if expr "${os_major}" '>=' 15 \& "${os_major}" != 42
+        then
+            # It seems that starting with SuSE 15 the systemd /etc/locale.conf
+            # is the preferred way to set locales, although
+            # /etc/sysconfig/language is still available.
+            # Older documentation doesn't mention /etc/locale.conf, even though
+            # is it created when localectl is used.
+            locale_conf="/etc/locale.conf"
+        else
+            locale_conf="/etc/sysconfig/language"
+            quote_value=true
+            key="RC_${__object_id}"
+        fi
+        ;;
     voidlinux)
         locale_conf="/etc/locale.conf"
         ;;
@@ -110,9 +130,9 @@ esac
 __file "${locale_conf}" --state exists --owner root --group root --mode 0644
 
 require="__file/${locale_conf}" \
-__key_value "${locale_conf}:${__object_id}" \
+__key_value "${locale_conf}:${key}" \
     --file "${locale_conf}" \
-    --key "${__object_id}" \
+    --key "${key}" \
     --delimiter '=' --exact_delimiter \
     --state "$(cat "${__object}/parameter/state")" \
     --value "$(catval "${__object}/parameter/value")" \

From cbf22f3b2c8b9d0fef8f6b7b0bf16508f96675b3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 15:23:55 +0200
Subject: [PATCH 349/743] [type/__locale_system] Add support for Solaris

---
 cdist/conf/type/__locale_system/manifest | 41 ++++++++++++++++++++++--
 1 file changed, 39 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 26d4f7b5..71491fe5 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -33,6 +33,7 @@ catval() {
     printf "$($quote_value && echo '"%s"' || echo '%s')" "$(cat "$1")"
 }
 
+state_should=$(cat "${__object}/parameter/state")
 
 os=$(cat "$__global/explorer/os")
 
@@ -98,6 +99,39 @@ in
                 ;;
         esac
         ;;
+    solaris)
+        locale_conf="/etc/default/init"
+        locale_conf_group="sys"
+
+        if expr "$(cat "${__global}/explorer/os_version")" '>=' 5.11 >/dev/null
+        then
+            # mode on Oracle Solaris 11 is actually 0444,
+            # but the write bit makes sense, IMO
+            locale_conf_mode=0644
+
+            # Oracle Solaris 11.2 and later uses SMF to store environment info.
+            # This is a hack, but I didn't feel like modifying the whole type
+            # just for some Oracle nonsense.
+            # 11.3 apparently added nlsadm(1m), but it is missing from 11.2.
+            # Illumos continues to use /etc/default/init
+            # NOTE: Remember not to use "cool" POSIX features like -q or -e with
+            # Solaris grep.
+            release_regex='Oracle Solaris 11.[2-9][0-9]*'
+            case $state_should
+            in
+                (present)
+                    svccfg_cmd="svccfg -s svc:/system/environment:init setprop environment/${key} = astring: '$(cat "${__object}/parameter/value")'"
+                    ;;
+                (absent)
+                    svccfg_cmd="svccfg -s svc:/system/environment:init delprop environment/${key}"
+                    ;;
+            esac
+            refresh_cmd='svcadm refresh svc:/system/environment'
+            onchange_cmd="grep '${release_regex}' /etc/release >&- || exit 0; ${svccfg_cmd:-:} && ${refresh_cmd}"
+        else
+            locale_conf_mode=0555
+        fi
+        ;;
     suse)
         os_version=$(cat "${__global}/explorer/os_version")
         os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
@@ -127,13 +161,16 @@ in
         ;;
 esac
 
-__file "${locale_conf}" --state exists --owner root --group root --mode 0644
+__file "${locale_conf}" --state exists \
+    --owner "${locale_conf_owner:-0}" \
+    --group "${locale_conf_group:-0}" \
+    --mode "${locale_conf_mode:-0644}"
 
 require="__file/${locale_conf}" \
 __key_value "${locale_conf}:${key}" \
     --file "${locale_conf}" \
     --key "${key}" \
     --delimiter '=' --exact_delimiter \
-    --state "$(cat "${__object}/parameter/state")" \
+    --state "${state_should}" \
     --value "$(catval "${__object}/parameter/value")" \
     --onchange "${onchange_cmd}"

From a923e75d9b0054212031507986d3b935d74d52bf Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 18:29:14 +0200
Subject: [PATCH 350/743] [type/__locale_system] Add support for NetBSD

---
 cdist/conf/type/__locale_system/manifest | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 71491fe5..92af852f 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -99,6 +99,15 @@ in
                 ;;
         esac
         ;;
+    netbsd)
+        # NetBSD doesn't  have a separate configuration file to set locales.
+        # So the shell login file will have to do.
+        # "Non-POSIX" shells like csh will not be updated here.
+
+        locale_conf="/etc/profile"
+        quote_value=true
+        value="$(catval "${__object}/parameter/value"); export ${key}"
+        ;;
     solaris)
         locale_conf="/etc/default/init"
         locale_conf_group="sys"
@@ -172,5 +181,5 @@ __key_value "${locale_conf}:${key}" \
     --key "${key}" \
     --delimiter '=' --exact_delimiter \
     --state "${state_should}" \
-    --value "$(catval "${__object}/parameter/value")" \
+    --value "${value:-$(catval "${__object}/parameter/value")}" \
     --onchange "${onchange_cmd}"

From 511d8c96aa9201049eaafc459a2de9694fe2a4e4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 19:23:17 +0200
Subject: [PATCH 351/743] [type/__locale_system] Add support for Slackware

---
 cdist/conf/type/__locale_system/manifest | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 92af852f..180788e6 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -141,6 +141,12 @@ in
             locale_conf_mode=0555
         fi
         ;;
+    slackware)
+        # NOTE: lang.csh (csh config) is ignored here.
+        locale_conf="/etc/profile.d/lang.sh"
+        locale_conf_mode=0755
+        key="export ${__object_id}"
+        ;;
     suse)
         os_version=$(cat "${__global}/explorer/os_version")
         os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
@@ -176,7 +182,7 @@ __file "${locale_conf}" --state exists \
     --mode "${locale_conf_mode:-0644}"
 
 require="__file/${locale_conf}" \
-__key_value "${locale_conf}:${key}" \
+__key_value "${locale_conf}:${key#export }" \
     --file "${locale_conf}" \
     --key "${key}" \
     --delimiter '=' --exact_delimiter \

From 70d1228dc0c6a3af105f0864b46d7dea806aedc2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 19:26:45 +0200
Subject: [PATCH 352/743] [type/__locale_system] Add support for FreeBSD

---
 cdist/conf/type/__locale_system/manifest | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 180788e6..e4286ef6 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -99,9 +99,10 @@ in
                 ;;
         esac
         ;;
-    netbsd)
-        # NetBSD doesn't  have a separate configuration file to set locales.
-        # So the shell login file will have to do.
+    freebsd|netbsd)
+        # NetBSD doesn't have a separate configuration file to set locales.
+        # In FreeBSD locales could be configured via /etc/login.conf but parsing
+        # that would be annoying, so the shell login file will have to do.
         # "Non-POSIX" shells like csh will not be updated here.
 
         locale_conf="/etc/profile"

From 73f1937636bfa59eea0432e3625c0a27cc8171b5 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 27 Jul 2020 06:20:21 +0200
Subject: [PATCH 353/743] [__unpack] no mkdir by default, because destination
 can be file, but tar needs mkdir andrar needs slash at the end

---
 cdist/conf/type/__unpack/explorer/state       | 37 +++++++++
 cdist/conf/type/__unpack/gencode-remote       | 75 ++++++++++++++++++
 cdist/conf/type/__unpack/man.rst              | 79 +++++++++++++++++++
 cdist/conf/type/__unpack/manifest             | 41 ++++++++++
 cdist/conf/type/__unpack/parameter/boolean    |  2 +
 cdist/conf/type/__unpack/parameter/optional   |  2 +
 cdist/conf/type/__unpack/parameter/required   |  1 +
 cdist/conf/type/__unpack/test/README          |  3 +
 .../type/__unpack/test/make-init-manifest.sh  | 22 ++++++
 .../type/__unpack/test/make-test-files.sh     | 44 +++++++++++
 10 files changed, 306 insertions(+)
 create mode 100755 cdist/conf/type/__unpack/explorer/state
 create mode 100755 cdist/conf/type/__unpack/gencode-remote
 create mode 100644 cdist/conf/type/__unpack/man.rst
 create mode 100755 cdist/conf/type/__unpack/manifest
 create mode 100644 cdist/conf/type/__unpack/parameter/boolean
 create mode 100644 cdist/conf/type/__unpack/parameter/optional
 create mode 100644 cdist/conf/type/__unpack/parameter/required
 create mode 100644 cdist/conf/type/__unpack/test/README
 create mode 100755 cdist/conf/type/__unpack/test/make-init-manifest.sh
 create mode 100755 cdist/conf/type/__unpack/test/make-test-files.sh

diff --git a/cdist/conf/type/__unpack/explorer/state b/cdist/conf/type/__unpack/explorer/state
new file mode 100755
index 00000000..38bc0978
--- /dev/null
+++ b/cdist/conf/type/__unpack/explorer/state
@@ -0,0 +1,37 @@
+#!/bin/sh -e
+
+src="/$__object_id"
+
+if [ -f "$__object/parameter/sum-file" ]
+then
+    src_sum_was_file="$( cat "$__object/parameter/sum-file" )"
+else
+    src_sum_was_file="$src.cdist__unpack_sum"
+fi
+
+if [ ! -f "$src" ]
+then
+    if [ -n "$__cdist_dry_run" ]
+    then
+        echo 'mismatch'
+    else
+        echo 'missing'
+    fi
+else
+    if [ ! -f "$src_sum_was_file" ]
+    then
+        echo 'mismatch'
+        exit 0
+    fi
+
+    src_sum_was="$( cat "$src_sum_was_file" )"
+
+    src_sum_is="$( cksum "$src" | awk '{ print $1$2 }' )"
+
+    if [ "$src_sum_was" = "$src_sum_is" ]
+    then
+        echo 'match'
+    else
+        echo 'mismatch'
+    fi
+fi
diff --git a/cdist/conf/type/__unpack/gencode-remote b/cdist/conf/type/__unpack/gencode-remote
new file mode 100755
index 00000000..45c7173a
--- /dev/null
+++ b/cdist/conf/type/__unpack/gencode-remote
@@ -0,0 +1,75 @@
+#!/bin/sh -e
+
+if grep -Eq '^(missing|match)$' "$__object/explorer/state"
+then
+    exit 0
+fi
+
+os="$( cat "$__global/explorer/os" )"
+
+src="/$__object_id"
+
+dst="$( sed 's/\/$//' "$__object/parameter/destination" )"
+
+cmd=''
+
+case "$src" in
+    *.tar|*.tgz|*.tar.*)
+        cmd="mkdir -p '$dst' && tar --directory='$dst' --extract --file='$src'"
+
+        if [ -f "$__object/parameter/tar-strip" ]
+        then
+            tar_strip="$( cat "$__object/parameter/tar-strip" )"
+
+            cmd="$cmd --strip-components=$tar_strip"
+        fi
+    ;;
+    *.7z)
+        case "$os" in
+            centos|fedora|redhat)
+                cmd='7za'
+            ;;
+            *)
+                cmd='7zr'
+            ;;
+        esac
+
+        cmd="$cmd e -aoa -o'$dst' '$src'"
+    ;;
+    *.bz2)
+        cmd="bunzip2 --stdout '$src' > '$dst'"
+    ;;
+    *.gz)
+        cmd="gunzip --stdout '$src' > '$dst'"
+    ;;
+    *.lzma|*.xz)
+        cmd="xz --uncompress --stdout '$src' > '$dst'"
+    ;;
+    *.rar)
+        cmd="unrar x -o+ '$src' '$dst/'"
+    ;;
+    *.zip)
+        cmd="unzip -o '$src' -d '$dst'"
+    ;;
+esac
+
+if [ -f "$__object/parameter/backup-destination" ]
+then
+    echo "if [ -e '$dst' ]; then mv '$dst' '$dst.cdist__unpack_backup_$( date +%s )'; fi"
+fi
+
+echo "$cmd"
+
+if [ -f "$__object/parameter/sum-file" ]
+then
+    sum_file="$( cat "$__object/parameter/sum-file" )"
+else
+    sum_file="$src.cdist__unpack_sum"
+fi
+
+echo "cksum '$src' | awk '{ print \$1\$2 }' > '$sum_file'"
+
+if [ ! -f "$__object/parameter/preserve-archive" ]
+then
+    echo "rm -f '$src'"
+fi
diff --git a/cdist/conf/type/__unpack/man.rst b/cdist/conf/type/__unpack/man.rst
new file mode 100644
index 00000000..8fe96e43
--- /dev/null
+++ b/cdist/conf/type/__unpack/man.rst
@@ -0,0 +1,79 @@
+cdist-type__unpack(7)
+=====================
+
+NAME
+----
+cdist-type__unpack - Unpack archives
+
+
+DESCRIPTION
+-----------
+Unpack ``.tar``, ``.tgz``, ``.tar.*``, ``.7z``, ``.bz2``, ``.gz``,
+``.lzma``, ``.xz``, ``.rar`` and ``.zip`` archives. Archive type is
+detected by extension.
+
+To achieve idempotency, checksum file will be created in target. See
+``--sum-file`` parameter for details.
+
+
+REQUIRED PARAMETERS
+-------------------
+destination
+   Depending on archive format file or directory to where archive
+   contents will be written.
+
+
+OPTIONAL PARAMETERS
+-------------------
+sum-file
+    Override archive's checksum file in target. By default
+    ``XXX.cdist__unpack_sum`` will be used, where ``XXX`` is source
+    archive path. This file must be kept in target's persistent storage.
+
+tar-strip
+    Tarball specific. See ``man tar`` for ``--strip-components``.
+
+
+OPTIONAL BOOLEAN PARAMETERS
+---------------------------
+backup-destination
+    By default destination file will be overwritten. In case destination
+    is directory, files from archive will be added to or overwritten in
+    directory. This parameter moves existing destination to
+    ``XXX.cdist__unpack_backup_YYY``, where ``XXX`` is destination and
+    ``YYY`` current UNIX timestamp.
+
+preserve-archive
+    Don't delete archive after unpacking.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __directory /opt/cpma
+
+    require='__directory/opt/cpma' \
+        __download /opt/cpma/cnq3.zip \
+            --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
+            --sum md5:46da3021ca9eace277115ec9106c5b46
+
+    require='__download/opt/cpma/cnq3.zip' \
+        __unpack /opt/cpma/cnq3.zip \
+            --backup-destination \
+            --preserve-archive \
+            --destination /opt/cpma/server
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__unpack/manifest b/cdist/conf/type/__unpack/manifest
new file mode 100755
index 00000000..6bdf5a10
--- /dev/null
+++ b/cdist/conf/type/__unpack/manifest
@@ -0,0 +1,41 @@
+#!/bin/sh -e
+
+os="$( cat "$__global/explorer/os" )"
+
+src="/$__object_id"
+
+case "$src" in
+    *.7z)
+        __package p7zip
+    ;;
+    *.bz2)
+        case "$os" in
+            freebsd)
+                # bzip2 is part of freebsd base system
+            ;;
+            *)
+                __package bzip2
+            ;;
+        esac
+    ;;
+    *.lzma|*.xz|*.txz)
+        case "$os" in
+            debian|ubuntu|devuan)
+                __package xz-utils
+            ;;
+            alpine|centos)
+                __package xz
+            ;;
+        esac
+    ;;
+    *.rar)
+        case "$os" in
+            debian|ubuntu|devuan|alpine|freebsd)
+                __package unrar
+            ;;
+        esac
+    ;;
+    *.zip)
+        __package unzip
+    ;;
+esac
diff --git a/cdist/conf/type/__unpack/parameter/boolean b/cdist/conf/type/__unpack/parameter/boolean
new file mode 100644
index 00000000..99dca934
--- /dev/null
+++ b/cdist/conf/type/__unpack/parameter/boolean
@@ -0,0 +1,2 @@
+backup-destination
+preserve-archive
diff --git a/cdist/conf/type/__unpack/parameter/optional b/cdist/conf/type/__unpack/parameter/optional
new file mode 100644
index 00000000..d136dd0c
--- /dev/null
+++ b/cdist/conf/type/__unpack/parameter/optional
@@ -0,0 +1,2 @@
+sum-file
+tar-strip
diff --git a/cdist/conf/type/__unpack/parameter/required b/cdist/conf/type/__unpack/parameter/required
new file mode 100644
index 00000000..ac459b09
--- /dev/null
+++ b/cdist/conf/type/__unpack/parameter/required
@@ -0,0 +1 @@
+destination
diff --git a/cdist/conf/type/__unpack/test/README b/cdist/conf/type/__unpack/test/README
new file mode 100644
index 00000000..54f3972a
--- /dev/null
+++ b/cdist/conf/type/__unpack/test/README
@@ -0,0 +1,3 @@
+./make-test-files.sh
+./make-init-manifest.sh | cdist config -i - localhost
+sudo find /tmp/cdist__unpack_test/ -type f -exec cat {} \; | sort
diff --git a/cdist/conf/type/__unpack/test/make-init-manifest.sh b/cdist/conf/type/__unpack/test/make-init-manifest.sh
new file mode 100755
index 00000000..404bc106
--- /dev/null
+++ b/cdist/conf/type/__unpack/test/make-init-manifest.sh
@@ -0,0 +1,22 @@
+#!/bin/sh -e
+
+p="$( pwd )"
+d=/tmp/cdist__unpack_test
+
+echo 'export CDIST_ORDER_DEPENDENCY=1'
+
+echo "__directory $d"
+
+find "$p" -name 'test.*' -and -not -name '*.cdist__unpack_sum' \
+    | sort \
+    | while read -r l
+do
+    n="$( basename "$l" )"
+
+    printf '__unpack %s --destination %s/%s\n' \
+        "$l" \
+        "$d" \
+        "$n"
+done
+
+echo "__clean_path $p --pattern '.+/test\..+'"
diff --git a/cdist/conf/type/__unpack/test/make-test-files.sh b/cdist/conf/type/__unpack/test/make-test-files.sh
new file mode 100755
index 00000000..d18e9e9f
--- /dev/null
+++ b/cdist/conf/type/__unpack/test/make-test-files.sh
@@ -0,0 +1,44 @@
+#!/bin/sh -ex
+
+echo test.7z > test
+7z a test.7z test > /dev/null
+
+echo test.bz2 > test
+bzip2 test
+
+echo test.gz > test
+gzip test
+
+echo test.lzma > test
+lzma test
+
+echo test.rar > test
+rar a test.rar test > /dev/null
+
+echo test.tar.bz2 > test
+tar cf test.tar test
+bzip2 test.tar
+
+echo test.tar.xz > test
+tar cf test.tar test
+xz test.tar
+
+echo test.tgz > test
+tar cf test.tar test
+gzip test.tar
+mv test.tar.gz test.tgz
+
+echo test.tar.gz > test
+tar cf test.tar test
+gzip test.tar
+
+echo test.tar > test
+tar cf test.tar test
+
+echo test.xz > test
+xz test
+
+echo test.zip > test
+zip test.zip test > /dev/null
+
+rm test

From 463b6cd6b52da8918d2803721164748c2a632696 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 27 Jul 2020 06:22:25 +0200
Subject: [PATCH 354/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 2457e90b..8d26e767 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -13,6 +13,9 @@ next:
 	* Type __user: Install user packages on OpenWrt (Dennis Camera)
 	* Type __openldap_server: Add Alpine support (Timothée Floure)
 	* Type __pf_apply: Remove deprecated type (Darko Poljak)
+	* Type __package_apt: Fix for legacy APT versions that do not support --no-install-recommends (Dennis Camera)
+	* Type __key_value: Get awk from POSIX PATH (Dennis Camera)
+	* New type: __unpack (Ander Punnar)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From d26c36914a4fa9889af57f549cb2854968663a0b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 27 Jul 2020 10:55:28 +0200
Subject: [PATCH 355/743] [__timezone] Make type singleton

---
 cdist/conf/type/__timezone/gencode-remote     |  2 +-
 cdist/conf/type/__timezone/man.rst            | 22 ++++++++++++-------
 cdist/conf/type/__timezone/manifest           |  2 +-
 cdist/conf/type/__timezone/parameter/required |  1 +
 cdist/conf/type/__timezone/singleton          |  0
 5 files changed, 17 insertions(+), 10 deletions(-)
 create mode 100644 cdist/conf/type/__timezone/parameter/required
 create mode 100644 cdist/conf/type/__timezone/singleton

diff --git a/cdist/conf/type/__timezone/gencode-remote b/cdist/conf/type/__timezone/gencode-remote
index 5299f548..b685c990 100755
--- a/cdist/conf/type/__timezone/gencode-remote
+++ b/cdist/conf/type/__timezone/gencode-remote
@@ -22,7 +22,7 @@
 # This type allows to configure the desired localtime timezone.
 
 timezone_is=$(cat "$__object/explorer/timezone_is")
-timezone_should="$__object_id"
+timezone_should=$(cat "$__object/parameter/tz")
 os=$(cat "$__global/explorer/os")
 
 if [ "$timezone_is" = "$timezone_should" ]; then
diff --git a/cdist/conf/type/__timezone/man.rst b/cdist/conf/type/__timezone/man.rst
index 8a945c16..6012c552 100644
--- a/cdist/conf/type/__timezone/man.rst
+++ b/cdist/conf/type/__timezone/man.rst
@@ -14,7 +14,8 @@ This type creates a symlink (/etc/localtime) to the selected timezone
 
 REQUIRED PARAMETERS
 -------------------
-None.
+tz
+    The name of timezone to set.
 
 
 OPTIONAL PARAMETERS
@@ -27,19 +28,24 @@ EXAMPLES
 
 .. code-block:: sh
 
-    #Set up Europe/Andorra as our timezone.
-    __timezone Europe/Andorra
+    # Set up Europe/Andorra as our timezone.
+    __timezone --tz Europe/Andorra
 
-    #Set up US/Central as our timezone.
-    __timezone US/Central
+    # Set up US/Central as our timezone.
+    __timezone --tz US/Central
 
 
 AUTHORS
 -------
-Ramon Salvadó <rsalvado--@--gnuine--dot--com>
+| Steven Armstrong <steven-cdist--@--armstrong.cc>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Ramon Salvadó <rsalvado--@--gnuine--dot--com>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Free use of this software is
-granted under the terms of the GNU General Public License version 3 (GPLv3).
+Copyright \(C) 2012-2020 the `AUTHORS`_. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__timezone/manifest b/cdist/conf/type/__timezone/manifest
index 3d28ccba..0eb7fb9c 100755
--- a/cdist/conf/type/__timezone/manifest
+++ b/cdist/conf/type/__timezone/manifest
@@ -22,7 +22,7 @@
 #
 # This type allows to configure the desired localtime timezone.
 
-timezone="$__object_id"
+timezone=$(cat "$__object/parameter/tz")
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
diff --git a/cdist/conf/type/__timezone/parameter/required b/cdist/conf/type/__timezone/parameter/required
new file mode 100644
index 00000000..975445e4
--- /dev/null
+++ b/cdist/conf/type/__timezone/parameter/required
@@ -0,0 +1 @@
+tz
diff --git a/cdist/conf/type/__timezone/singleton b/cdist/conf/type/__timezone/singleton
new file mode 100644
index 00000000..e69de29b

From 627d215b637f1893e48a4a04c7e60686b960a698 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 27 Jul 2020 13:09:53 +0200
Subject: [PATCH 356/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8d26e767..e3711be2 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -16,6 +16,7 @@ next:
 	* Type __package_apt: Fix for legacy APT versions that do not support --no-install-recommends (Dennis Camera)
 	* Type __key_value: Get awk from POSIX PATH (Dennis Camera)
 	* New type: __unpack (Ander Punnar)
+	* Type __locale_system: Support more OSes (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From 5dfc996febd085e4592fe590c4837bd4802ef4c3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 22:53:34 +0200
Subject: [PATCH 357/743] Fix global explorers for NetBSD

On NetBSD sysctl is at /sbin/sysctl, but the default PATH does not
contain /sbin.
---
 cdist/conf/explorer/cpu_cores | 1 +
 cdist/conf/explorer/disks     | 5 ++---
 cdist/conf/explorer/memory    | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/explorer/cpu_cores b/cdist/conf/explorer/cpu_cores
index c6744142..81e5294e 100755
--- a/cdist/conf/explorer/cpu_cores
+++ b/cdist/conf/explorer/cpu_cores
@@ -33,6 +33,7 @@ case "$os" in
     ;;
 
     "freebsd"|"netbsd")
+        PATH=$(getconf PATH)
         sysctl -n hw.ncpu
     ;;
 
diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks
index 24540601..56d62d10 100755
--- a/cdist/conf/explorer/disks
+++ b/cdist/conf/explorer/disks
@@ -30,9 +30,8 @@ case $uname_s in
         sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+'
     ;;
     NetBSD)
-        PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin"
-        sysctl -n hw.disknames \
-        | awk 'BEGIN { RS = " " } /^[lsw]d[0-9]+/'
+        PATH=$(getconf PATH)
+        sysctl -n hw.disknames | awk -v RS=' ' '/^[lsw]d[0-9]+/'
     ;;
     Linux)
         # list of major device numbers toexclude:
diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory
index 302b4cda..5ea15ada 100755
--- a/cdist/conf/explorer/memory
+++ b/cdist/conf/explorer/memory
@@ -30,6 +30,7 @@ case "$os" in
     ;;
 
     *"bsd")
+        PATH=$(getconf PATH)
         echo "$(sysctl -n hw.physmem) / 1048576" | bc
     ;;
 

From 3a87a447d00ef12eb6ace7c240926568aec2a042 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 26 Jul 2020 23:37:48 +0200
Subject: [PATCH 358/743] [type/__sysctl] Fix on NetBSD

---
 cdist/conf/type/__sysctl/explorer/value | 9 +++++++--
 cdist/conf/type/__sysctl/gencode-remote | 2 ++
 cdist/conf/type/__sysctl/man.rst        | 7 +++++++
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__sysctl/explorer/value b/cdist/conf/type/__sysctl/explorer/value
index fc85b3d8..3e93c151 100755
--- a/cdist/conf/type/__sysctl/explorer/value
+++ b/cdist/conf/type/__sysctl/explorer/value
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -18,5 +18,10 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+if test "$(uname -s)" = NetBSD
+then
+	PATH=$(getconf PATH)
+fi
+
 # get the current runtime value
-sysctl -n "$__object_id" || true
+sysctl -n "${__object_id}" || true
diff --git a/cdist/conf/type/__sysctl/gencode-remote b/cdist/conf/type/__sysctl/gencode-remote
index 711d54e5..f0f6deef 100755
--- a/cdist/conf/type/__sysctl/gencode-remote
+++ b/cdist/conf/type/__sysctl/gencode-remote
@@ -44,6 +44,8 @@ case "$os" in
       flag='-w'
       ;;
    netbsd)
+      # shellcheck disable=SC2016
+      echo 'PATH=$(getconf PATH)'
       flag='-w'
       ;;
    freebsd|openbsd)
diff --git a/cdist/conf/type/__sysctl/man.rst b/cdist/conf/type/__sysctl/man.rst
index 6873003e..dbb9a1ac 100644
--- a/cdist/conf/type/__sysctl/man.rst
+++ b/cdist/conf/type/__sysctl/man.rst
@@ -26,6 +26,13 @@ EXAMPLES
 
     __sysctl net.ipv4.ip_forward --value 1
 
+    # On some operating systems, e.g. NetBSD, to prevent an error if the
+    # MIB style name does not exist (e.g. optional kernel components),
+    # name and value can be separated by `?=`. The same effect can be achieved
+    # in cdist by appending a `?` to the key:
+
+    __sysctl ddb.onpanic? --value -1
+
 
 AUTHORS
 -------

From 76bb214b5300707426030649cf03f35808e43256 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 27 Jul 2020 15:31:38 +0200
Subject: [PATCH 359/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index e3711be2..8fef4653 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -17,6 +17,8 @@ next:
 	* Type __key_value: Get awk from POSIX PATH (Dennis Camera)
 	* New type: __unpack (Ander Punnar)
 	* Type __locale_system: Support more OSes (Dennis Camera)
+	* Explorers cpu_cores, disks, memory: Fix for NetBSD (Dennis Camera)
+	* Type __sysctl: Fix for NetBSD (Dennis Camera)
 
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)

From f5b367dfdbf60e75cfb2be038a05b6f226a1c010 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 28 Jul 2020 07:14:26 +0200
Subject: [PATCH 360/743] Release 6.7.0

---
 docs/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/changelog b/docs/changelog
index 8fef4653..269a2049 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,8 @@
 Changelog
 ---------
 
-next:
+6.7.0: 2020-07-28
+	* Delete deprecated type: __pf_apply (Darko Poljak)
 	* New type: __download (Ander Punnar)
 	* Type __locale_system: Add devuan support (Dennis Camera)
 	* Type __package_opkg: Add locking (Dennis Camera)
@@ -12,7 +13,6 @@ next:
 	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
 	* Type __user: Install user packages on OpenWrt (Dennis Camera)
 	* Type __openldap_server: Add Alpine support (Timothée Floure)
-	* Type __pf_apply: Remove deprecated type (Darko Poljak)
 	* Type __package_apt: Fix for legacy APT versions that do not support --no-install-recommends (Dennis Camera)
 	* Type __key_value: Get awk from POSIX PATH (Dennis Camera)
 	* New type: __unpack (Ander Punnar)

From c053a2c4a000b0c15df186ef2e067ea68ec9499c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 29 Jul 2020 11:31:12 +0200
Subject: [PATCH 361/743] Fix building man pages

Resolves #830.
---
 Makefile                                  | 4 ++--
 cdist/conf/type/__openldap_server/man.rst | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index f89ac1e7..3712511c 100644
--- a/Makefile
+++ b/Makefile
@@ -81,7 +81,7 @@ version:
 	}
 
 # Manpages #3: generic part
-man: version $(MANTYPES) $(DOCSREF)
+man: version configskel $(MANTYPES) $(DOCSREF) $(DOCSTYPESREF)
 	$(SPHINXM)
 
 html: version configskel $(MANTYPES) $(DOCSREF) $(DOCSTYPESREF)
@@ -104,7 +104,7 @@ DOTMANTYPES=$(subst /man.rst,.rst,$(DOTMANTYPEPREFIX))
 $(DOTMAN7DSTDIR)/cdist-type%.rst: $(DOTTYPEDIR)/%/man.rst
 	ln -sf "$^" $@
 
-dotman: version $(DOTMANTYPES)
+dotman: version configskel $(DOTMANTYPES) $(DOCSREF) $(DOCSTYPESREF)
 	$(SPHINXM)
 
 ################################################################################
diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
index a96c7dad..fa714ec0 100644
--- a/cdist/conf/type/__openldap_server/man.rst
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -31,8 +31,8 @@ manager-password-hash
     Generate e.g. with: `slappasswd -s weneedgoodsecurity`.
     See `slappasswd(8C)`, `slapd.conf(5)`.
     TODO: implement this: http://blog.adamsbros.org/2015/06/09/openldap-ssha-salted-hashes-by-hand/
-      to derive from the manager-password parameter and ensure idempotency (care with salts).
-      At that point, manager-password-hash should be deprecated and ignored.
+    to derive from the manager-password parameter and ensure idempotency (care with salts).
+    At that point, manager-password-hash should be deprecated and ignored.
 
 serverid
     The server for the directory.

From d37d2dc307d066f4d5f1bbbb93e0480965dfe3b8 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 2 Aug 2020 13:53:38 +0300
Subject: [PATCH 362/743] [__unpack] add parameter --tar-extra-args

---
 cdist/conf/type/__unpack/gencode-remote     |  7 +++++++
 cdist/conf/type/__unpack/man.rst            | 11 +++++++++++
 cdist/conf/type/__unpack/parameter/optional |  1 +
 3 files changed, 19 insertions(+)

diff --git a/cdist/conf/type/__unpack/gencode-remote b/cdist/conf/type/__unpack/gencode-remote
index 45c7173a..3b7a19a7 100755
--- a/cdist/conf/type/__unpack/gencode-remote
+++ b/cdist/conf/type/__unpack/gencode-remote
@@ -23,6 +23,13 @@ case "$src" in
 
             cmd="$cmd --strip-components=$tar_strip"
         fi
+
+        if [ -f "$__object/parameter/tar-extra-args" ]
+        then
+            tar_extra_args="$( cat "$__object/parameter/tar-extra-args" )"
+
+            cmd="$cmd $tar_extra_args"
+        fi
     ;;
     *.7z)
         case "$os" in
diff --git a/cdist/conf/type/__unpack/man.rst b/cdist/conf/type/__unpack/man.rst
index 8fe96e43..bd0603cf 100644
--- a/cdist/conf/type/__unpack/man.rst
+++ b/cdist/conf/type/__unpack/man.rst
@@ -33,6 +33,10 @@ sum-file
 tar-strip
     Tarball specific. See ``man tar`` for ``--strip-components``.
 
+tar-extra-args
+    Tarball sepcific. Append additional arguments to ``tar`` command.
+    See ``man tar`` for possible arguments.
+
 
 OPTIONAL BOOLEAN PARAMETERS
 ---------------------------
@@ -65,6 +69,13 @@ EXAMPLES
             --preserve-archive \
             --destination /opt/cpma/server
 
+    # example usecase for --tar-* args
+    __unpack /root/strelaysrv.tar.gz \
+        --preserve-archive \
+        --destination /usr/local/bin \
+        --tar-strip 1 \
+        --tar-extra-args '--wildcards "*/strelaysrv"'
+
 
 AUTHORS
 -------
diff --git a/cdist/conf/type/__unpack/parameter/optional b/cdist/conf/type/__unpack/parameter/optional
index d136dd0c..da26ca63 100644
--- a/cdist/conf/type/__unpack/parameter/optional
+++ b/cdist/conf/type/__unpack/parameter/optional
@@ -1,2 +1,3 @@
 sum-file
 tar-strip
+tar-extra-args

From 935f2395bc8f02d373f06d1e22acad813403ead2 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 2 Aug 2020 13:54:30 +0300
Subject: [PATCH 363/743] [__locale_system] fix for debian and ubuntu

ubuntu 6.10 and debian etch are 10+ years old and EOL. rather than
preserving compatibility I'll just remove it. while /etc/environment
works too, correct place is /etc/default/locale (as it was before
breaking change). also /etc/debian_version (os_version explorer) may
contain minor version with dot (10.5) or string (bullseye/sid).
---
 cdist/conf/type/__locale_system/manifest | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index e4286ef6..f7d75387 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -40,27 +40,13 @@ os=$(cat "$__global/explorer/os")
 case $os
 in
     debian)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 4 >/dev/null
-        then
-            # Debian 4 (etch) and later
-            locale_conf="/etc/default/locale"
-        else
-            locale_conf="/etc/environment"
-        fi
+        locale_conf="/etc/default/locale"
         ;;
     devuan)
         locale_conf="/etc/default/locale"
         ;;
     ubuntu)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 6.10 >/dev/null
-        then
-            # Ubuntu 6.10 (edgy) and later
-            locale_conf="/etc/default/locale"
-        else
-            locale_conf="/etc/environment"
-        fi
+        locale_conf="/etc/default/locale"
         ;;
     archlinux)
         locale_conf="/etc/locale.conf"

From 885d5a58f40e5adfd6aa616712e4c7246eca7880 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 16:49:01 +0200
Subject: [PATCH 364/743] [type/__locale_system] Fix floating point version
 comparison

---
 cdist/conf/type/__locale_system/manifest | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index e4286ef6..521f1007 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -24,6 +24,9 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
+bccmp() { test "$(bc)" -gt 0; }
+
+
 key=$__object_id
 onchange_cmd=  # none, by default
 quote_value=false
@@ -41,7 +44,7 @@ case $os
 in
     debian)
         os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 4 >/dev/null
+        if printf '%f >= 4\n' "${os_version}" | bccmp
         then
             # Debian 4 (etch) and later
             locale_conf="/etc/default/locale"
@@ -54,7 +57,7 @@ in
         ;;
     ubuntu)
         os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 6.10 >/dev/null
+        if printf '%f >= 6.10\n' "${os_version}" | bccmp
         then
             # Ubuntu 6.10 (edgy) and later
             locale_conf="/etc/default/locale"
@@ -68,7 +71,7 @@ in
     centos|redhat|scientific)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if expr "${version_id}" '>=' 7 >/dev/null
+        if printf '%f >= 7\n' "${version_id}" | bccmp
         then
             locale_conf="/etc/locale.conf"
         else
@@ -78,7 +81,7 @@ in
     fedora)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if expr "${version_id}" '>=' 18 >/dev/null
+        if printf '%f >= 18\n' "${version_id}" | bccmp
         then
             locale_conf="/etc/locale.conf"
             quote_value=false
@@ -113,7 +116,8 @@ in
         locale_conf="/etc/default/init"
         locale_conf_group="sys"
 
-        if expr "$(cat "${__global}/explorer/os_version")" '>=' 5.11 >/dev/null
+        os_version=$(cat "${__global}/explorer/os_version")
+        if printf '%f >= 5.11\n' "${os_version}" | bccmp
         then
             # mode on Oracle Solaris 11 is actually 0444,
             # but the write bit makes sense, IMO

From 71710fa00a86e3dcda6be82b959956cb88ba9902 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 20:17:00 +0200
Subject: [PATCH 365/743] [type/__locale_system] Implement "proper" version
 comparison

Proper in the sense that it can handle all numeric version numbers even if they
are not floating point (e.g. 16.04.6).
---
 cdist/conf/type/__locale_system/manifest | 28 ++++++++++++++++--------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 521f1007..88eadb31 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -24,7 +24,20 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
-bccmp() { test "$(bc)" -gt 0; }
+version_ge() {
+    awk -F '[^0-9.]' -v target="${1:?}" '
+    function max(x, y) { return x > y ? x : y }
+    BEGIN {
+        getline
+        nx = split($1, x, ".")
+        ny = split(target, y, ".")
+        for (i = 1; i <= max(nx, ny); ++i) {
+            diff = int(x[i]) - int(y[i])
+            if (diff == 0) continue
+            exit (diff < 0)
+        }
+    }'
+}
 
 
 key=$__object_id
@@ -43,8 +56,7 @@ os=$(cat "$__global/explorer/os")
 case $os
 in
     debian)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if printf '%f >= 4\n' "${os_version}" | bccmp
+        if version_ge 4 <"${__global}/explorer/os_version"
         then
             # Debian 4 (etch) and later
             locale_conf="/etc/default/locale"
@@ -56,8 +68,7 @@ in
         locale_conf="/etc/default/locale"
         ;;
     ubuntu)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if printf '%f >= 6.10\n' "${os_version}" | bccmp
+        if version_ge 6.10 <"${__global}/explorer/os_version"
         then
             # Ubuntu 6.10 (edgy) and later
             locale_conf="/etc/default/locale"
@@ -71,7 +82,7 @@ in
     centos|redhat|scientific)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if printf '%f >= 7\n' "${version_id}" | bccmp
+        if echo "${version_id}" | version_ge 7
         then
             locale_conf="/etc/locale.conf"
         else
@@ -81,7 +92,7 @@ in
     fedora)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if printf '%f >= 18\n' "${version_id}" | bccmp
+        if echo "${version_id}" | version_ge 18
         then
             locale_conf="/etc/locale.conf"
             quote_value=false
@@ -116,8 +127,7 @@ in
         locale_conf="/etc/default/init"
         locale_conf_group="sys"
 
-        os_version=$(cat "${__global}/explorer/os_version")
-        if printf '%f >= 5.11\n' "${os_version}" | bccmp
+        if version_ge 5.11 <"${__global}/explorer/os_version"
         then
             # mode on Oracle Solaris 11 is actually 0444,
             # but the write bit makes sense, IMO

From 7b480f4293b84a97850430a9436e28c66ccf5736 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 22:08:14 +0200
Subject: [PATCH 366/743] [type/__locale_system] Fix version extraction for
 SuSE

---
 cdist/conf/type/__locale_system/manifest | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 88eadb31..4b996ebc 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -163,7 +163,13 @@ in
         key="export ${__object_id}"
         ;;
     suse)
-        os_version=$(cat "${__global}/explorer/os_version")
+        if test -s "${__global}/explorer/os_release"
+        then
+            # shellcheck source=/dev/null
+            os_version=$(. "${__global}/explorer/os_release" && echo "${VERSION}")
+        else
+            os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global}/explorer/os_version")
+        fi
         os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
 
         # https://documentation.suse.com/sles/15-SP2/html/SLES-all/cha-suse.html#sec-suse-l10n

From b370b70ff4ccc1c6efca97f61f220a5ce0b191da Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 22:13:41 +0200
Subject: [PATCH 367/743] [explorer/os] Fix OS detection for openSUSE

All distros with ID_LIKE suse should be treated as "suse".
My openSUSE Leap 15.1 installation has:
ID_LIKE="suse opensuse"

This patch doesn't require a strict "suse" value but only the word suse to be in
the list.
---
 cdist/conf/explorer/os | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os
index 2d2aede6..8f54d9c8 100755
--- a/cdist/conf/explorer/os
+++ b/cdist/conf/explorer/os
@@ -144,7 +144,8 @@ esac
 
 if [ -f /etc/os-release ]; then
    # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse
-   if grep -q ^ID_LIKE=\"suse\" /etc/os-release 2>/dev/null; then
+   if (. /etc/os-release && echo "${ID_LIKE}" | grep -q '\(^\|\ \)suse\($\|\ \)')
+   then
       echo suse
       exit 0
    fi

From 17ab4bd80c9c9dda2fa4f8ccde5ee454f7b5ba2d Mon Sep 17 00:00:00 2001
From: Joachim Desroches <joachim.desroches@epfl.ch>
Date: Thu, 6 Aug 2020 11:45:05 +0200
Subject: [PATCH 368/743] Add Alpine Linux as supported for __filesystem.

---
 cdist/conf/type/__filesystem/explorer/lsblk | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk
index 9ae544ac..9be3c575 100644
--- a/cdist/conf/type/__filesystem/explorer/lsblk
+++ b/cdist/conf/type/__filesystem/explorer/lsblk
@@ -18,16 +18,16 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$("$__explorer/os")
+os=$("${__explorer:?}/os")
 
-if [ -f "$__object/parameter/device" ]; then
+if [ -f "${__object:?}/parameter/device" ]; then
     blkdev="$(cat "$__object/parameter/device")"
 else
-    blkdev="$__object_id"
+    blkdev="${__object_id:?}"
 fi
 
 case "$os" in
-    centos|fedora|redhat|suse|gentoo)
+    alpine|centos|fedora|redhat|suse|gentoo)
         if [ ! -x "$(command -v lsblk)" ]; then
             echo "lsblk is required for __filesystem type" >&2
             exit 1

From 74dd47c8c352cc3de3b5684bbe90520f61d95f37 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 15 Aug 2020 21:11:43 +0200
Subject: [PATCH 369/743] ++changelog

---
 docs/changelog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 269a2049..2c09bfea 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,12 @@
 Changelog
 ---------
 
+next:
+	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)
+	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)
+	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)
+	* Type __filesystem: Support Alpine Linux (Joachim Desroches)
+
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)
 	* New type: __download (Ander Punnar)

From 8f94a226c7f0b875987e25e57682414204380fa2 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 15 Aug 2020 21:54:07 +0200
Subject: [PATCH 370/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 2c09bfea..9e44cc4d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)
 	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)
 	* Type __filesystem: Support Alpine Linux (Joachim Desroches)
+	* Type __locale_system: Fix version comparison (Dennis Camera)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From 6fed1785298eb506e7ea6cd177efe7d96658769f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 17 Aug 2020 09:27:48 +0200
Subject: [PATCH 371/743] [explorer/os_version] Convert Debian sid to version
 number.

Conversion of Debian sid to versions is done based on Debian codenames.
The version number is the version number of the final release - 0.01.

It is unknown if Debian < 4.0 has any sort of version information
available (apart from maybe checking base-files package version).
But I don't think any of these systems are still alive,
so I think going with 3.99 is fine for those.
---
 cdist/conf/explorer/os_version | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 1d54ea60..a7b1d3bc 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -31,7 +31,32 @@ case "$("$__explorer/os")" in
       cat /etc/arch-release
    ;;
    debian)
-      cat /etc/debian_version
+      debian_version=$(cat /etc/debian_version)
+      case $debian_version
+      in
+          testing/unstable)
+              # previous to Debian 4.0 testing/unstable was used
+              # cf. https://metadata.ftp-master.debian.org/changelogs/main/b/base-files/base-files_11_changelog
+              echo 3.99
+              ;;
+          */sid)
+              # sid versions don't have a number, so we decode by codename:
+              case $(expr "$debian_version" : '\([a-z]\{1,\}\)/')
+              in
+                  bullseye) echo 10.99 ;;
+                  buster) echo 9.99 ;;
+                  stretch) echo 8.99 ;;
+                  jessie) echo 7.99 ;;
+                  wheezy) echo 6.99 ;;
+                  squeeze) echo 5.99 ;;
+                  lenny) echo 4.99 ;;
+                  *) exit 1
+              esac
+              ;;
+          *)
+              echo "$debian_version"
+              ;;
+      esac
    ;;
    devuan)
       cat /etc/devuan_version
@@ -73,4 +98,4 @@ case "$("$__explorer/os")" in
    alpine)
        cat /etc/alpine-release
    ;;
-esac
\ No newline at end of file
+esac

From 502d75304707704d3ba73374701394b98d259506 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 18 Aug 2020 00:46:07 +0300
Subject: [PATCH 372/743] [__unpack] add --onchange

---
 cdist/conf/type/__unpack/gencode-remote     | 5 +++++
 cdist/conf/type/__unpack/man.rst            | 3 +++
 cdist/conf/type/__unpack/parameter/optional | 1 +
 3 files changed, 9 insertions(+)

diff --git a/cdist/conf/type/__unpack/gencode-remote b/cdist/conf/type/__unpack/gencode-remote
index 3b7a19a7..c4451f73 100755
--- a/cdist/conf/type/__unpack/gencode-remote
+++ b/cdist/conf/type/__unpack/gencode-remote
@@ -80,3 +80,8 @@ if [ ! -f "$__object/parameter/preserve-archive" ]
 then
     echo "rm -f '$src'"
 fi
+
+if [ -f "$__object/parameter/onchange" ]
+then
+    cat "$__object/parameter/onchange"
+fi
diff --git a/cdist/conf/type/__unpack/man.rst b/cdist/conf/type/__unpack/man.rst
index bd0603cf..daa03814 100644
--- a/cdist/conf/type/__unpack/man.rst
+++ b/cdist/conf/type/__unpack/man.rst
@@ -50,6 +50,9 @@ backup-destination
 preserve-archive
     Don't delete archive after unpacking.
 
+onchange
+    Execute this command after unpack.
+
 
 EXAMPLES
 --------
diff --git a/cdist/conf/type/__unpack/parameter/optional b/cdist/conf/type/__unpack/parameter/optional
index da26ca63..d846ac75 100644
--- a/cdist/conf/type/__unpack/parameter/optional
+++ b/cdist/conf/type/__unpack/parameter/optional
@@ -1,3 +1,4 @@
 sum-file
 tar-strip
 tar-extra-args
+onchange

From d239169c4f91a4ce33be170ed29b08299c51234f Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 18 Aug 2020 00:48:58 +0300
Subject: [PATCH 373/743] [__download] fix manual: onchange parameter in wrong
 section

---
 cdist/conf/type/__download/man.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index 6ec0b19a..eb3ac971 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -30,9 +30,6 @@ sum
    By default output of ``cksum`` without filename is expected.
    Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
 
-onchange
-   Execute this command after download.
-
 
 OPTIONAL PARAMETERS
 -------------------
@@ -54,6 +51,9 @@ cmd-sum
    format specification ``%s`` which will become destination.
    For example: ``md5sum '%s' | awk '{print $1}'``.
 
+onchange
+   Execute this command after download.
+
 
 EXAMPLES
 --------

From ba26a437be4d8034037e7a6eb42382aee01f829e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 18 Aug 2020 11:06:19 +0200
Subject: [PATCH 374/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 9e44cc4d..a3e4167a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,9 @@ next:
 	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)
 	* Type __filesystem: Support Alpine Linux (Joachim Desroches)
 	* Type __locale_system: Fix version comparison (Dennis Camera)
+	* Type __unpack: Add --onchange parameter (Ander Punnar)
+	* Type __download: Fix manual (Ander Punnar)
+	* Explorer os_version: Convert Debian sid to version number (Dennis Camera)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From c17541f24cf118b0989a5899cab3a439ed534a46 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 22 Aug 2020 19:53:42 +0200
Subject: [PATCH 375/743] Expand and split by consecutive require delimiters

Resolves #832.
---
 cdist/emulator.py               |  6 +++++-
 cdist/test/emulator/__init__.py | 10 ++++++++++
 docs/src/cdist-manifest.rst     |  3 ++-
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/cdist/emulator.py b/cdist/emulator.py
index 9fe84056..60b94880 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -25,6 +25,7 @@ import argparse
 import logging
 import os
 import sys
+import re
 
 import cdist
 from cdist import core
@@ -389,12 +390,15 @@ class Emulator:
         if "require" in self.env:
             requirements = self.env['require']
             self.log.debug("reqs = " + requirements)
-            for requirement in requirements.split(" "):
+            for requirement in self._parse_require(requirements):
                 # Ignore empty fields - probably the only field anyway
                 if len(requirement) == 0:
                     continue
                 self.record_requirement(requirement)
 
+    def _parse_require(self, require):
+        return re.split(r'[ \t\n]+', require)
+
     def record_auto_requirements(self):
         """An object shall automatically depend on all objects that it
            defined in it's type manifest.
diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py
index e375676c..befd7b57 100644
--- a/cdist/test/emulator/__init__.py
+++ b/cdist/test/emulator/__init__.py
@@ -685,6 +685,16 @@ class EmulatorAlreadyExistingRequirementsWarnTestCase(test.CdistTestCase):
         self.env['require'] = '__directory/spam'
         emu = emulator.Emulator(argv, env=self.env)
 
+    def test_parse_require(self):
+        require = " \t \n  \t\t\n\t\na\tb\nc d \te\t\nf\ng\t "
+        expected = ['', 'a', 'b', 'c', 'd', 'e', 'f', 'g', '', ]
+
+        argv = ['__directory', 'spam']
+        emu = emulator.Emulator(argv, env=self.env)
+        requirements = emu._parse_require(require)
+
+        self.assertEqual(expected, requirements)
+
 
 if __name__ == '__main__':
     import unittest
diff --git a/docs/src/cdist-manifest.rst b/docs/src/cdist-manifest.rst
index 5dbca479..2e49a721 100644
--- a/docs/src/cdist-manifest.rst
+++ b/docs/src/cdist-manifest.rst
@@ -95,7 +95,8 @@ Dependencies
 ------------
 If you want to describe that something requires something else, just
 setup the variable "require" to contain the requirements. Multiple
-requirements can be added white space separated.
+requirements can be added separated with (optionally consecutive)
+delimiters including space, tab and newline.
 
 ::
 

From b5a40eb0d1c8990f0feb8e918a95e54e4fd35d98 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 27 Aug 2020 12:25:11 +0200
Subject: [PATCH 376/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index a3e4167a..eda77a12 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,6 +10,7 @@ next:
 	* Type __unpack: Add --onchange parameter (Ander Punnar)
 	* Type __download: Fix manual (Ander Punnar)
 	* Explorer os_version: Convert Debian sid to version number (Dennis Camera)
+	* Core: Expand require delimiter characters, split by consecutive delimiters (Darko Poljak)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From b1375464cc7507b38dd02f2df52c75489837d129 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 6 Sep 2020 13:24:58 +0200
Subject: [PATCH 377/743] __systemd_service: fix manpage typos

---
 cdist/conf/type/__systemd_service/man.rst | 25 +++++++++++++++--------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__systemd_service/man.rst b/cdist/conf/type/__systemd_service/man.rst
index 7eca398b..cd14c985 100644
--- a/cdist/conf/type/__systemd_service/man.rst
+++ b/cdist/conf/type/__systemd_service/man.rst
@@ -1,9 +1,10 @@
-cdist-type__systemd-service(7)
+cdist-type__systemd_service(7)
 ==============================
 
 NAME
 ----
-cdist-type__systemd-service - Controls a systemd service state
+cdist-type__systemd_service - Controls a systemd service state
+
 
 DESCRIPTION
 -----------
@@ -14,11 +15,12 @@ service after configuration applied or shutdown one service.
 The activation or deactivation is out of scope. Look for the
 :strong:`cdist-type__systemd_util`\ (7) type instead.
 
+
 REQUIRED PARAMETERS
 -------------------
-
 None.
 
+
 OPTIONAL PARAMETERS
 -------------------
 
@@ -31,12 +33,12 @@ state
     running
         Service should run (default)
 
-    stoppend
-        Service should stopped
+    stopped
+        Service should be stopped
 
 action
     Executes an action on on the service. It will only execute it if the
-    service keeps the state **running**. There are following actions, where:
+    service keeps the state ``running``. There are following actions, where:
 
     reload
         Reloads the service
@@ -48,11 +50,12 @@ BOOLEAN PARAMETERS
 ------------------
 
 if-required
-    Only execute the action if minimum one required type outputs a message to
-    **$__messages_out**. Through this, the action should only executed if a
+    Only execute the action if at minimum one required type outputs a message
+    to ``$__messages_out``. Through this, the action should only executed if a
     dependency did something. The action will not executed if no dependencies
     given.
 
+
 MESSAGES
 --------
 
@@ -68,12 +71,14 @@ restart
 reload
     Reloaded the service
 
+
 ABORTS
 ------
 Aborts in following cases:
 
 systemd or the service does not exist
 
+
 EXAMPLES
 --------
 .. code-block:: sh
@@ -95,13 +100,15 @@ EXAMPLES
 
     # reload the service for a modified configuration file
     # only reloads the service if the file really changed
-    require="__config_file/etc/foo.conf" __systemd_service foo \
+    require="__file/etc/foo.conf" __systemd_service foo \
         --action reload --if-required
 
+
 AUTHORS
 -------
 Matthias Stecher <matthiasstecher at gmx.de>
 
+
 COPYRIGHT
 ---------
 Copyright \(C) 2020 Matthias Stecher. You can redistribute it

From 6b262a61c19234f6450cef11f715020dcd5f40d5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 10 Sep 2020 13:24:58 +0200
Subject: [PATCH 378/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index eda77a12..e4ec3fdd 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,8 @@ next:
 	* Type __download: Fix manual (Ander Punnar)
 	* Explorer os_version: Convert Debian sid to version number (Dennis Camera)
 	* Core: Expand require delimiter characters, split by consecutive delimiters (Darko Poljak)
+	* Type __timezone: Make singleton (Dennis Camera)
+	* Type __systemd_service: Fix manpage typos (Matthias Stecher)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From 53b91adbd8be4d858f1f4da3389b63babc85db0c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 11 Sep 2020 14:19:45 +0200
Subject: [PATCH 379/743] Fix shellcheck

---
 cdist/conf/explorer/os | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os
index 8f54d9c8..46d87f3e 100755
--- a/cdist/conf/explorer/os
+++ b/cdist/conf/explorer/os
@@ -144,6 +144,7 @@ esac
 
 if [ -f /etc/os-release ]; then
    # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse
+   # shellcheck disable=SC1091
    if (. /etc/os-release && echo "${ID_LIKE}" | grep -q '\(^\|\ \)suse\($\|\ \)')
    then
       echo suse

From 2885c6a24838956167ae6c33ebf87067e8790083 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 11 Sep 2020 14:15:26 +0200
Subject: [PATCH 380/743] Release 6.8.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index e4ec3fdd..7a0d050d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)
 	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)
 	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)

From decc0ad54dffbafcd7d3644c28ae4d149926e612 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 18 Sep 2020 19:33:36 +0300
Subject: [PATCH 381/743] [__package_pip] detect pip binary

---
 cdist/conf/type/__package_pip/explorer/pip   | 10 ++++++++++
 cdist/conf/type/__package_pip/explorer/state |  2 +-
 cdist/conf/type/__package_pip/gencode-remote |  7 ++++++-
 3 files changed, 17 insertions(+), 2 deletions(-)
 create mode 100755 cdist/conf/type/__package_pip/explorer/pip

diff --git a/cdist/conf/type/__package_pip/explorer/pip b/cdist/conf/type/__package_pip/explorer/pip
new file mode 100755
index 00000000..cf9fae89
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/pip
@@ -0,0 +1,10 @@
+#!/bin/sh -e
+
+for bin in pip3 pip
+do
+    if check="$( command -v "$bin" )"
+    then
+        echo "$check"
+        break
+    fi
+done
diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state
index 5be07280..3cc98ab9 100644
--- a/cdist/conf/type/__package_pip/explorer/state
+++ b/cdist/conf/type/__package_pip/explorer/state
@@ -32,7 +32,7 @@ pipparam="$__object/parameter/pip"
 if [ -f "$pipparam" ]; then
     pip=$(cat "$pipparam")
 else
-    pip="pip"
+    pip="$( "$__type_explorer/pip" )"
 fi
 
 # If there is no pip, it may get created from somebody else.
diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index dcc4fdf9..9c5eb0d1 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -38,7 +38,12 @@ pipparam="$__object/parameter/pip"
 if [ -f "$pipparam" ]; then
     pip=$(cat "$pipparam")
 else
-    pip="pip"
+    pip="$( cat "$__object/explorer/pip" )"
+    if [ -z "$pip" ]
+    then
+        echo 'pip not found in path' >&2
+        exit 1
+    fi
 fi
 
 runasparam="$__object/parameter/runas"

From 89b621511561105cfcfcf12bbf820b74ab03396d Mon Sep 17 00:00:00 2001
From: Darko Poljak <darko.poljak@ungleich.ch>
Date: Mon, 21 Sep 2020 09:04:05 +0200
Subject: [PATCH 382/743] Clarify stdin input

Resolve #836.
---
 cdist/argparse.py                | 36 ++++++--------------------
 cdist/config.py                  |  6 +++--
 cdist/inventory.py               |  4 +--
 cdist/test/inventory/__init__.py |  3 +--
 docs/src/man1/cdist.rst          | 44 ++++++++++----------------------
 5 files changed, 29 insertions(+), 64 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 77303591..1d16bb25 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -273,8 +273,7 @@ def get_parsers():
             '-f', '--file',
             help=('Read specified file for a list of additional hosts to '
                   'operate on or if \'-\' is given, read stdin (one host per '
-                  'line). If no host or host file is specified then, by '
-                  'default, read hosts from stdin.'),
+                  'line).'),
             dest='hostfile', required=False)
     parser['config_args'].add_argument(
            '-p', '--parallel', nargs='?', metavar='HOST_MAX',
@@ -326,9 +325,7 @@ def get_parsers():
     parser['add-host'].add_argument(
            '-f', '--file',
            help=('Read additional hosts to add from specified file '
-                 'or from stdin if \'-\' (each host on separate line). '
-                 'If no host or host file is specified then, by default, '
-                 'read from stdin.'),
+                 'or from stdin if \'-\' (each host on separate line). '),
            dest='hostfile', required=False)
 
     parser['add-tag'] = parser['invsub'].add_parser(
@@ -342,20 +339,12 @@ def get_parsers():
     parser['add-tag'].add_argument(
            '-f', '--file',
            help=('Read additional hosts to add tags from specified file '
-                 'or from stdin if \'-\' (each host on separate line). '
-                 'If no host or host file is specified then, by default, '
-                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                 ' are specified then tags are read from stdin and are'
-                 ' added to all hosts.'),
+                 'or from stdin if \'-\' (each host on separate line). '),
            dest='hostfile', required=False)
     parser['add-tag'].add_argument(
            '-T', '--tag-file',
            help=('Read additional tags to add from specified file '
-                 'or from stdin if \'-\' (each tag on separate line). '
-                 'If no tag or tag file is specified then, by default, '
-                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                 ' are specified then tags are read from stdin and are'
-                 ' added to all hosts.'),
+                 'or from stdin if \'-\' (each tag on separate line). '),
            dest='tagfile', required=False)
     parser['add-tag'].add_argument(
            '-t', '--taglist',
@@ -376,9 +365,7 @@ def get_parsers():
     parser['del-host'].add_argument(
             '-f', '--file',
             help=('Read additional hosts to delete from specified file '
-                  'or from stdin if \'-\' (each host on separate line). '
-                  'If no host or host file is specified then, by default, '
-                  'read from stdin.'),
+                  'or from stdin if \'-\' (each host on separate line). '),
             dest='hostfile', required=False)
 
     parser['del-tag'] = parser['invsub'].add_parser(
@@ -396,20 +383,13 @@ def get_parsers():
     parser['del-tag'].add_argument(
             '-f', '--file',
             help=('Read additional hosts to delete tags for from specified '
-                  'file or from stdin if \'-\' (each host on separate line). '
-                  'If no host or host file is specified then, by default, '
-                  'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                  ' are specified then tags are read from stdin and are'
-                  ' deleted from all hosts.'),
+                  'file or from stdin if \'-\' (each host on separate '
+                  'line). '),
             dest='hostfile', required=False)
     parser['del-tag'].add_argument(
             '-T', '--tag-file',
             help=('Read additional tags from specified file '
-                  'or from stdin if \'-\' (each tag on separate line). '
-                  'If no tag or tag file is specified then, by default, '
-                  'read from stdin. If no tags/tagfile nor'
-                  ' hosts/hostfile are specified then tags are read from'
-                  ' stdin and are added to all hosts.'),
+                  'or from stdin if \'-\' (each tag on separate line). '),
             dest='tagfile', required=False)
     parser['del-tag'].add_argument(
             '-t', '--taglist',
diff --git a/cdist/config.py b/cdist/config.py
index 30416008..e84f6f84 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -175,9 +175,11 @@ class Config:
             raise cdist.Error(("Cannot read both, manifest and host file, "
                                "from stdin"))
 
-        # if no host source is specified then read hosts from stdin
         if not (args.hostfile or args.host):
-            args.hostfile = '-'
+            if args.tag or args.all_tagged_hosts:
+                raise cdist.Error(("Target host tag(s) missing"))
+            else:
+                raise cdist.Error(("Target host(s) missing"))
 
         if args.manifest == '-':
             # read initial manifest from stdin
diff --git a/cdist/inventory.py b/cdist/inventory.py
index 6ab20fa7..0387f326 100644
--- a/cdist/inventory.py
+++ b/cdist/inventory.py
@@ -299,7 +299,7 @@ class InventoryHost(Inventory):
         self.all = all
 
         if not self.hosts and not self.hostfile:
-            self.hostfile = "-"
+            raise cdist.Error("Host(s) missing")
 
     def _new_hostpath(self, hostpath):
         # create empty file
@@ -355,7 +355,7 @@ class InventoryTag(Inventory):
         else:
             self.allhosts = False
         if not self.tags and not self.tagfile:
-            self.tagfile = "-"
+            raise cdist.Error("Tag(s) missing")
 
         if self.hostfile == "-" and self.tagfile == "-":
             raise cdist.Error("Cannot read both, hosts and tags, from stdin")
diff --git a/cdist/test/inventory/__init__.py b/cdist/test/inventory/__init__.py
index 287c855c..a8cd8bf8 100644
--- a/cdist/test/inventory/__init__.py
+++ b/cdist/test/inventory/__init__.py
@@ -307,11 +307,10 @@ class InventoryTestCase(test.CdistTestCase):
             raise e
 
     # InventoryTag
+    @unittest.expectedFailure
     def test_inventory_tag_init(self):
         invTag = inventory.InventoryTag(db_basedir=inventory_dir,
                                         action="add")
-        self.assertTrue(invTag.allhosts)
-        self.assertEqual(invTag.tagfile, "-")
 
     def test_inventory_tag_stdin_multiple_hosts(self):
         try:
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index aa2607f8..0ecb4a61 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -177,10 +177,8 @@ Install command is currently in beta.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read specified file for a list of additional hosts to operate on
-    or if '-' is given, read stdin (one host per line).
-    If no host or host file is specified then, by default,
-    read hosts from stdin. For the file format see
-    :strong:`HOSTFILE FORMAT` below.
+    or if '-' is given, read stdin (one host per line). For the file
+    format see :strong:`HOSTFILE FORMAT` below.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -299,9 +297,8 @@ Add host(s) to inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to add from specified file or
-    from stdin if '-' (each host on separate line). If no
-    host or host file is specified then, by default, read
-    from stdin. Hostfile format is the same as config hostfile format.
+    from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -327,11 +324,8 @@ Add tag(s) to inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to add tags from specified file
-    or from stdin if '-' (each host on separate line). If
-    no host or host file is specified then, by default,
-    read from stdin. If no tags/tagfile nor hosts/hostfile
-    are specified then tags are read from stdin and are
-    added to all hosts. Hostfile format is the same as config hostfile format.
+    or from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -346,11 +340,8 @@ Add tag(s) to inventory database.
 
 **-T TAGFILE, --tag-file TAGFILE**
     Read additional tags to add from specified file or
-    from stdin if '-' (each tag on separate line). If no
-    tag or tag file is specified then, by default, read
-    from stdin. If no tags/tagfile nor hosts/hostfile are
-    specified then tags are read from stdin and are added
-    to all hosts. Tagfile format is the same as config hostfile format.
+    from stdin if '-' (each tag on separate line).
+    Tagfile format is the same as config hostfile format.
 
 **-t TAGLIST, --taglist TAGLIST**
     Tag list to be added for specified host(s), comma
@@ -372,9 +363,8 @@ Delete host(s) from inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to delete from specified file or
-    from stdin if '-' (each host on separate line). If no
-    host or host file is specified then, by default, read
-    from stdin. Hostfile format is the same as config hostfile format.
+    from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -404,11 +394,8 @@ Delete tag(s) from inventory database.
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to delete tags for from
     specified file or from stdin if '-' (each host on
-    separate line). If no host or host file is specified
-    then, by default, read from stdin. If no tags/tagfile
-    nor hosts/hostfile are specified then tags are read
-    from stdin and are deleted from all hosts. Hostfile
-    format is the same as config hostfile format.
+    separate line). Hostfile format is the same as
+    config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -423,11 +410,8 @@ Delete tag(s) from inventory database.
 
 **-T TAGFILE, --tag-file TAGFILE**
     Read additional tags from specified file or from stdin
-    if '-' (each tag on separate line). If no tag or tag
-    file is specified then, by default, read from stdin.
-    If no tags/tagfile nor hosts/hostfile are specified
-    then tags are read from stdin and are added to all
-    hosts. Tagfile format is the same as config hostfile format.
+    if '-' (each tag on separate line).
+    Tagfile format is the same as config hostfile format.
 
 **-t TAGLIST, --taglist TAGLIST**
     Tag list to be deleted for specified host(s), comma

From 89a0080e133052b82ce19e7fdbaf045dcc833da7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:09:26 +0200
Subject: [PATCH 383/743] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 7a0d050d..260b4fe4 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* Core: Clarify stdin input (Darko Poljak)
+	* Type __package_pip: Detect pip binary (Ander Punnar)
+
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)
 	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)

From 0fc10749edc698d1f70f826603d009fba943d252 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:11:35 +0200
Subject: [PATCH 384/743] Fix shellcheck

---
 cdist/conf/type/__package_pip/gencode-remote | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 9c5eb0d1..a1375c2d 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -60,7 +60,7 @@ case "$state_should" in
         then
             echo "su -c '$pip install -q $name' $runas"
         else
-            echo $pip install -q "$name"
+            echo "$pip" install -q "$name"
         fi
         echo "installed" >> "$__messages_out"
     ;;
@@ -69,7 +69,7 @@ case "$state_should" in
         then
             echo "su -c '$pip uninstall -q -y $name' $runas"
         else
-            echo $pip uninstall -q -y "$name"
+            echo "$pip" uninstall -q -y "$name"
         fi
         echo "removed" >> "$__messages_out"
     ;;

From b6922508b9e07834009a6bfae2d3933b5ea62fbf Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:17:34 +0200
Subject: [PATCH 385/743] Update helper script

---
 bin/build-helper | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/build-helper b/bin/build-helper
index ed41e438..d4d603ed 100755
--- a/bin/build-helper
+++ b/bin/build-helper
@@ -371,7 +371,6 @@ eof
 Manual steps post release:
     - cdist-web
     - send generated mailinglist.tmp mail
-    - twitter
 eof
     ;;
 

From 84a7818121047cf6bb23d8bd3c055d09bb033d32 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 23 Sep 2020 20:29:47 +0200
Subject: [PATCH 386/743] docs: make varaibles environment-aware

There are all overwriting the environment, even the comment states
otherwise. Fixes it.
---
 docs/src/Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/src/Makefile b/docs/src/Makefile
index 2ecf7a32..ba87d170 100644
--- a/docs/src/Makefile
+++ b/docs/src/Makefile
@@ -2,10 +2,10 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-PAPER         =
-BUILDDIR      = ../dist
+SPHINXOPTS   ?=
+SPHINXBUILD  ?= sphinx-build
+PAPER        ?=
+BUILDDIR     ?= ../dist
 # for cache, etc.
 _BUILDDIR     = _build
 

From 73d6c9d469d1429999ea2e980b3ac4c63a3e0dd4 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 27 Sep 2020 10:17:35 +0200
Subject: [PATCH 387/743] Add custom remote copy/exec examples

---
 docs/src/cdist-remote-exec-copy.rst | 389 +++++++++++++++++++++++++++-
 1 file changed, 388 insertions(+), 1 deletion(-)

diff --git a/docs/src/cdist-remote-exec-copy.rst b/docs/src/cdist-remote-exec-copy.rst
index bb818310..e7b7b226 100644
--- a/docs/src/cdist-remote-exec-copy.rst
+++ b/docs/src/cdist-remote-exec-copy.rst
@@ -10,7 +10,7 @@ By default this is accomplished with ssh and scp respectively.
 The default implementations used by cdist are::
 
     __remote_exec: ssh -o User=root
-    __remote_copy: scp -o User=root
+    __remote_copy: scp -o User=root -q
 
 The user can override these defaults by providing custom implementations and
 passing them to cdist with the --remote-exec and/or --remote-copy arguments.
@@ -26,3 +26,390 @@ specified by enclosed in square brackets (see :strong:`ssh`\ (1) and
 With this simple interface the user can take total control of how cdist
 interacts with the target when required, while the default implementation 
 remains as simple as possible.
+
+
+Examples
+--------
+
+Here are examples of using alternative __remote_copy and __remote_exec scripts.
+
+All scripts from below are present in cdist sources in `other/examples/remote`
+directory.
+
+ssh
+~~~
+
+Same as cdist default.
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    #echo "$@" | logger -t "cdist-ssh-copy"
+    scp -o User=root -q $@
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    #echo "$@" | logger -t "cdist-ssh-exec"
+    ssh -o User=root $@
+
+local
+~~~~~
+
+This effectively turns remote calling into local calling. Probably most useful
+for the unit testing.
+
+**copy**
+
+.. code-block:: sh
+
+    code="$(echo "$@" | sed "s|\([[:space:]]\)$__target_host:|\1|g")"
+    cp -L $code
+
+**exec**
+
+.. code-block:: sh
+
+    target_host=$1; shift
+    echo "$@" | /bin/sh 
+
+chroot
+~~~~~~
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script /path/to/your/chroot" target-id
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-chroot-copy"
+       :
+    }
+
+    chroot="$1"; shift
+    target_host="$__target_host"
+
+    # replace target_host with chroot location
+    code="$(echo "$@" | sed "s|$target_host:|$chroot|g")"
+
+    log "target_host: $target_host"
+    log "chroot: $chroot"
+    log "$@"
+    log "$code"
+
+    # copy files into chroot
+    cp $code
+
+    log "-----"
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script /path/to/your/chroot" target-id
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-chroot-exec"
+       :
+    }
+
+    chroot="$1"; shift
+    target_host="$1"; shift
+
+    script=$(mktemp "${chroot}/tmp/chroot-${0##*/}.XXXXXXXXXX")
+    trap cleanup INT TERM EXIT
+    cleanup() {
+       [ $__cdist_debug ] || rm "$script"
+    }
+
+    log "target_host: $target_host"
+    log "script: $script"
+    log "@: $@"
+    echo "#!/bin/sh -l" > "$script"
+    echo "$@" >> "$script"
+    chmod +x "$script"
+
+    relative_script="${script#$chroot}"
+    log "relative_script: $relative_script"
+
+    # run in chroot
+    chroot "$chroot" "$relative_script"
+
+    log "-----"
+
+rsync
+~~~~~
+
+**copy**
+
+Usage: cdist config --remote-copy /path/to/this/script target_host
+
+.. code-block:: sh
+
+    # For rsync to do the right thing, the source has to end with "/" if it is
+    # a directory. The below preprocessor loop takes care of that.
+
+    # second last argument is the source
+    source_index=$(($#-1))
+    index=0
+    for arg in $@; do
+       if [ $index -eq 0 ]; then
+          # reset $@
+          set --
+       fi
+       index=$((index+=1))
+       if [ $index -eq $source_index -a -d "$arg" ]; then
+          arg="${arg%/}/"
+       fi
+       set -- "$@" "$arg"
+    done
+
+    rsync --backup --suffix=~cdist -e 'ssh -o User=root' $@
+
+schroot
+~~~~~~~
+
+__remote_copy and __remote_exec scripts to run cdist against a chroot on the
+target host over ssh.
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script schroot-chroot-name" target_host
+
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-schroot-copy"
+       :
+    }
+
+    chroot_name="$1"; shift
+    target_host="$__target_host"
+
+    # get directory for given chroot_name
+    chroot="$(ssh -o User=root -q $target_host schroot -c $chroot_name --config | awk -F = '/directory=/ {print $2}')"
+
+    # prefix destination with chroot
+    code="$(echo "$@" | sed "s|$target_host:|$target_host:$chroot|g")"
+
+    log "target_host: $target_host"
+    log "chroot_name: $chroot_name"
+    log "chroot: $chroot"
+    log "@: $@"
+    log "code: $code"
+
+    # copy files into remote chroot
+    scp -o User=root -q $code
+
+    log "-----"
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script schroot-chroot-name" target_host
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-schroot-exec"
+       :
+    }
+
+    chroot_name="$1"; shift
+    target_host="$1"; shift
+
+    code="ssh -o User=root -q $target_host schroot -c $chroot_name -- $@"
+
+    log "target_host: $target_host"
+    log "chroot_name: $chroot_name"
+    log "@: $@"
+    log "code: $code"
+
+    # run in remote chroot
+    $code
+
+    log "-----"
+
+schroot-uri
+~~~~~~~~~~~
+
+__remote_exec/__remote_copy script to run cdist against a schroot target URI.
+
+Usage::
+
+    cdist config \
+        --remote-exec "/path/to/this/script exec" \
+        --remote-copy "/path/to/this/script copy" \
+        target_uri
+
+    # target_uri examples:
+    schroot:///chroot-name
+    schroot://foo.ethz.ch/chroot-name
+    schroot://user-name@foo.ethz.ch/chroot-name
+
+    # and how to match them in .../manifest/init
+    case "$target_host" in
+    schroot://*)
+        # any schroot
+    ;;
+    schroot://foo.ethz.ch/*)
+        # any schroot on specific host
+    ;;
+    schroot://foo.ethz.ch/chroot-name)
+        # specific schroot on specific host
+    ;;
+    schroot:///chroot-name)
+        # specific schroot on localhost
+    ;;
+    esac
+
+**copy/exec**
+
+.. code-block:: sh
+
+    my_name="${0##*/}"
+    mode="$1"; shift
+
+    log() {
+       # uncomment me for debugging
+       #echo "$@" | logger -t "cdist-$my_name-$mode"
+       :
+    }
+
+    die() {
+       echo "$@" >&2
+       exit 1
+    }
+
+
+    uri="$__target_host"
+
+    scheme="${uri%%:*}"; rest="${uri#$scheme:}"; rest="${rest#//}"
+    authority="${rest%%/*}"; rest="${rest#$authority}"
+    path="${rest%\?*}"; rest="${rest#$path}"
+    schroot_name="${path#/}"
+
+    [ "$scheme" = "schroot" ] || die "Failed to parse scheme from __target_host ($__target_host). Expected 'schroot', got '$scheme'"
+    [ -n "$schroot_name" ] || die "Failed to parse schroot name from __target_host: $__target_host"
+
+    case "$authority" in
+       '')
+          # authority is empty, neither user nor host given
+          user=""
+          host=""
+       ;; 
+       *@*)
+          # authority contains @, take user from authority
+          user="${authority%@*}"
+          host="${authority#*@}"
+       ;; 
+       *) 
+          # no user in authority, default to root
+          user="root"
+          host="$authority"
+       ;;
+    esac
+
+    log "mode: $mode"
+    log "@: $@"
+    log "uri: $uri"
+    log "scheme: $scheme"
+    log "authority: $authority"
+    log "user: $user"
+    log "host: $host"
+    log "path: $path"
+    log "schroot_name: $schroot_name"
+
+    exec_prefix=""
+    copy_prefix=""
+    if [ -n "$host" ]; then
+       # we are working on a remote host
+       exec_prefix="ssh -o User=$user -q $host"
+       copy_prefix="scp -o User=$user -q"
+       copy_destination_prefix="$host:"
+    else
+       # working on local machine
+       copy_prefix="cp"
+       copy_destination_prefix=""
+    fi
+    log "exec_prefix: $exec_prefix"
+    log "copy_prefix: $copy_prefix"
+    log "copy_destination_prefix: $copy_destination_prefix"
+
+    case "$mode" in
+       exec)
+          # In exec mode the first argument is the __target_host which we already got from env. Get rid of it.
+          shift
+          code="$exec_prefix schroot -c $schroot_name -- sh -c '$@'"
+       ;;
+       copy)
+          # get directory for given chroot_name
+          schroot_directory="$($exec_prefix schroot -c $schroot_name --config | awk -F = '/directory=/ {print $2}')"
+          [ -n "$schroot_directory" ] || die "Failed to retreive schroot directory for schroot: $schroot_name"
+          log "schroot_directory: $schroot_directory"
+          # prefix destination with chroot
+          code="$copy_prefix $(echo "$@" | sed "s|$uri:|${copy_destination_prefix}${schroot_directory}|g")"
+       ;;
+       *) die "Unknown mode: $mode";;
+    esac
+
+    log "code: $code"
+
+    # Run the code
+    $code
+
+    log "-----"
+
+sudo
+~~~~
+
+**copy**
+
+Use rsync over ssh to copy files. Uses the "--rsync-path" option
+to run the remote rsync instance with sudo.
+
+This command assumes your ssh configuration is already set up in ~/.ssh/config.
+
+Usage: cdist config --remote-copy /path/to/this/script target_host
+
+.. code-block:: sh
+
+    # For rsync to do the right thing, the source has to end with "/" if it is
+    # a directory. The below preprocessor loop takes care of that.
+
+    # second last argument is the source
+    source_index=$(($#-1))
+    index=0
+    for arg in $@; do
+       if [ $index -eq 0 ]; then
+          # reset $@
+          set --
+       fi
+       index=$((index+=1))
+       if [ $index -eq $source_index -a -d "$arg" ]; then
+          arg="${arg%/}/"
+       fi
+       set -- "$@" "$arg"
+    done
+
+    rsync --copy-links --rsync-path="sudo rsync" -e 'ssh' "$@"
+
+**exec**
+
+Prefixes all remote commands with sudo.
+
+This command assumes your ssh configuration is already set up in ~/.ssh/config.
+
+Usage: cdist config --remote-exec "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    host="$1"; shift
+    ssh -q "$host" sudo sh -c \""$@"\"

From 652c89185816526e5165ff9d5686a0d245ec3e5f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 29 Sep 2020 05:57:54 +0200
Subject: [PATCH 388/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 260b4fe4..8d380524 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
+	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From f994226d0e23b667309fd75116c0c8a14d15f34d Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 29 Sep 2020 19:44:47 +0200
Subject: [PATCH 389/743] [__package_pkgng_freebsd] Bootstrap pkg if necessary

In a pristine FreeBSD base installation, pkg is really a bootstrapper utility,
in such cases the type used to fail instead of automatically bootstrapping pkg.
---
 .../type/__package_pkgng_freebsd/explorer/pkg_bootstrapped   | 4 ++++
 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version | 5 +++++
 cdist/conf/type/__package_pkgng_freebsd/gencode-remote       | 5 +++++
 3 files changed, 14 insertions(+)
 create mode 100755 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped

diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped
new file mode 100755
index 00000000..429f15d3
--- /dev/null
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+if pkg -N >/dev/null 2>&1; then
+	echo "YES"
+fi
diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
index 92ce0623..f0fb9127 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
@@ -21,6 +21,11 @@
 # Retrieve the status of a package - parsed dpkg output
 #
 
+if ! pkg -N >/dev/null 2>&1; then
+   # Nothing to do if pkg is not bootstrapped
+   exit
+fi
+
 if [ -f "$__object/parameter/name" ]; then
    name="$(cat "$__object/parameter/name")"
 else
diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
index dd36efda..b5944177 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
+++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
@@ -43,6 +43,7 @@ fi
 repo="$(cat "$__object/parameter/repo")"
 state="$(cat "$__object/parameter/state")"
 curr_version="$(cat "$__object/explorer/pkg_version")"
+pkg_bootstrapped="$(cat "$__object/explorer/pkg_bootstrapped")"
 add_cmd="pkg install -y"
 rm_cmd="pkg delete -y"
 upg_cmd="pkg upgrade -y"
@@ -73,6 +74,10 @@ execcmd(){
          ;;
    esac
 
+   if [ -z "${pkg_bootstrapped}" ]; then
+       echo "pkg bootstrap -y >/dev/null 2>&1"
+   fi
+
    echo "$_cmd >/dev/null 2>&1"   # Silence the output of the command
    echo "status=\$?"
    echo "if [ \"\$status\" -ne \"0\" ]; then"

From 52b5f05163416196403634cb9a52f746cdb96391 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 30 Sep 2020 08:56:31 +0200
Subject: [PATCH 390/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8d380524..34c97d67 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
+	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 3fa74b454a9512319e3559a53dd477f38724fd9b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 30 Sep 2020 15:43:32 +0200
Subject: [PATCH 391/743] Fix typo

---
 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
index f0fb9127..1c6ba5e5 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Retrieve the status of a package - parsed dpkg output
+# Retrieve the status of a package - parsed pkgng output
 #
 
 if ! pkg -N >/dev/null 2>&1; then

From 5aeed14b1b8b022f952680cceb12a5099a099558 Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Thu, 8 Oct 2020 16:15:20 +0200
Subject: [PATCH 392/743] Fixed calling of __systemd_service type with correct
 arguments.

---
 cdist/conf/type/__service/manifest | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__service/manifest b/cdist/conf/type/__service/manifest
index cb5af234..beb0713c 100644
--- a/cdist/conf/type/__service/manifest
+++ b/cdist/conf/type/__service/manifest
@@ -7,7 +7,9 @@ action="$(cat "$__object/parameter/action")"
 
 case "$manager" in
 	systemd)
-		__systemd_service "$name" --action "$action"
+		test "$action" = "start" && action="running"
+		test "$action" = "stop"  && action="stopped"
+		__systemd_service "$name" --state "$action"
 	;;
 	*)
 		# Unknown: handled by `service $NAME $action` in gencode-remote.

From c030deea3dbccfe73637b730f0d545db7f8d5f35 Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Fri, 9 Oct 2020 06:51:44 +0200
Subject: [PATCH 393/743] [__line] Add support for '--state replace'

It is currently counter-intuitive that something like:

    # File '/thing' contents
    #SomeSetting WrongValue

    # Manifest
    __line '/thing' \
           --line 'SomeSeting GoodValue' \
           --regex '^(#[[:space:]]*)?SomeSetting[[:space:]]'

Produces:

    # Resulting '/thing' contents
    #SomeSetting WrongValue

This makes sense given the implementation, but it masks a very common use-case.

Changing the default behaviour for such a base type is not really an option, so
instead we add a `replace` as a valid value for `--state`, which would result
in:

    # Resulting '/thing' contents with: --state replace
    SomeSetting GoodValue

For compatibility, if the regex is missing, `--state replace` behaves just as
`--state present`.
---
 cdist/conf/type/__line/explorer/state | 12 +++++++++++-
 cdist/conf/type/__line/gencode-remote | 10 +++++++---
 cdist/conf/type/__line/man.rst        | 13 +++++++++++--
 3 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state
index e8fc3630..9d480b19 100755
--- a/cdist/conf/type/__line/explorer/state
+++ b/cdist/conf/type/__line/explorer/state
@@ -53,8 +53,10 @@ function _find(_text, _pattern) {
 BEGIN {
    getline anchor < (ENVIRON["__object"] "/parameter/" position)
    getline pattern < (ENVIRON["__object"] "/parameter/" needle)
+   getline line < (ENVIRON["__object"] "/parameter/line")
 
    found_line = 0
+   correct_line = 0
    correct_pos = (position != "after" && position != "before")
 }
 {
@@ -63,15 +65,18 @@ BEGIN {
          getline
          if (_find($0, pattern)) {
             found_line++
+	    if (index($0, line) == 1) { correct_line++ }
             correct_pos = 1
             exit 0
          }
       } else if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
       }
    } else if (position == "before") {
       if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
          getline
          if (match($0, anchor)) {
             correct_pos = 1
@@ -81,13 +86,18 @@ BEGIN {
    } else {
       if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
          exit 0
       }
    }
 }
 END {
    if (found_line && correct_pos) {
-      print "present"
+      if (correct_line) {
+        print "present"
+      } else {
+        print "matching"
+      }
    } else if (found_line) {
       print "wrongposition"
    } else {
diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote
index 88cae68b..a89886da 100755
--- a/cdist/conf/type/__line/gencode-remote
+++ b/cdist/conf/type/__line/gencode-remote
@@ -38,7 +38,11 @@ if [ -z "$state_is" ]; then
    exit 1
 fi
 
-if [ "$state_should" = "$state_is" ]; then
+if [ "$state_should" = "$state_is" ] || \
+	{ [ "$state_should" = "present" ] && [ "$state_is" = "matching" ] ;} || \
+	{ [ "$state_should" = "replace" ] && [ "$state_is" = "present" ] ;} ; then
+   # If state matches already, or 'present' is used and regex matches
+   # or 'replace' is used and the exact line is present, then there is
    # nothing to do
    exit 0
 fi
@@ -61,8 +65,8 @@ fi
 add=0
 remove=0
 case "$state_should" in
-   present)
-      if [ "$state_is" = "wrongposition" ]; then
+   present|replace)
+      if [ "$state_is" = "wrongposition" ] || [ "$state_is" = "matching" ]; then
          echo updated >> "$__messages_out"
          remove=1
       else
diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst
index f76cab64..70490f68 100644
--- a/cdist/conf/type/__line/man.rst
+++ b/cdist/conf/type/__line/man.rst
@@ -31,7 +31,7 @@ file
 line
     Specifies the line which should be absent or present.
 
-    Must be present, if state is 'present'.
+    Must be present, if state is 'present' or 'replace'.
     Ignored if regex is given and state is 'absent'.
 
 regex
@@ -41,10 +41,13 @@ regex
     If state is 'absent', ensure all lines matching the regular expression
     are absent.
 
+    If state is 'replace', ensure all lines matching the regular expression
+    are exactly 'line'.
+
     The regular expression is interpreted by awk's match function.
 
 state
-    'present' or 'absent', defaults to 'present'
+    'present', 'absent' or 'replace', defaults to 'present'.
 
 onchange
     The code to run if line is added, removed or updated.
@@ -99,6 +102,12 @@ EXAMPLES
         --line '-session required pam_exec.so debug log=/tmp/classify.log /usr/local/libexec/classify' \
         --after '^session[[:space:]]+include[[:space:]]+password-auth-ac$'
 
+    # Uncomment as needed and set a value in a configuration file.
+    __line /etc/example.conf \
+        --line 'SomeSetting SomeValue' \
+        --regex '^(#[[:space:]]*)?SomeSetting[[:space:]]' \
+        --state replace
+
 
 SEE ALSO
 --------

From 4df5c91912edf9f31a6716d7cbe0adaedbc2b0e3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 9 Oct 2020 06:52:52 +0200
Subject: [PATCH 394/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 34c97d67..6d237449 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,8 @@ next:
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
 	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
+	* Type __service: Fix calling __systemd_service (Mark Verboom)
+	* Type __line: Add 'replace' state (Evil Ham)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 8ecae42199f27693eec771513174ce8143c8ca83 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:00:11 +0300
Subject: [PATCH 395/743] remove bin/cdist script

---
 bin/cdist | 33 ---------------------------------
 1 file changed, 33 deletions(-)
 delete mode 100755 bin/cdist

diff --git a/bin/cdist b/bin/cdist
deleted file mode 100755
index 645020a1..00000000
--- a/bin/cdist
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/sh
-# -*- coding: utf-8 -*-
-#
-# 2012 Nico Schottelius (nico-cdist at schottelius.org)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-# Wrapper for real script to allow execution from checkout
-dir=${0%/*}
-
-# Ensure version is present - the bundled/shipped version contains a static version,
-# the git version contains a dynamic version
-"$dir/build-helper" version
-
-libdir=$(cd "${dir}/../" && pwd -P)
-export PYTHONPATH="${libdir}"
-
-"$dir/../scripts/cdist" "$@"

From 45d51c0e1553ce12fcea09b1b9e0d2fb9c0304c5 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:00:44 +0300
Subject: [PATCH 396/743] rename build-helper -> cdist-build-helper

---
 bin/{build-helper => cdist-build-helper} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename bin/{build-helper => cdist-build-helper} (100%)

diff --git a/bin/build-helper b/bin/cdist-build-helper
similarity index 100%
rename from bin/build-helper
rename to bin/cdist-build-helper

From 3f1939716f4c4c2cdf538a22170a67fe181c675c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:02:45 +0300
Subject: [PATCH 397/743] enable running scripts/cdist directly and symlinked

---
 scripts/cdist | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scripts/cdist b/scripts/cdist
index b1d782ab..2ce40ae0 100755
--- a/scripts/cdist
+++ b/scripts/cdist
@@ -22,7 +22,15 @@
 #
 
 import logging
+import os
 import sys
+
+cdist_bin = os.path.abspath(__file__)
+if os.path.islink(cdist_bin):
+    cdist_bin = os.readlink(cdist_bin)
+cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
+sys.path.insert(0, cdist_dir)
+
 import cdist
 import cdist.argparse
 import cdist.banner

From fdc1ab93e968575b1d1f6f3b2a6cefa6ed6bb850 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:03:11 +0300
Subject: [PATCH 398/743] move scripts/* to bin/

---
 {scripts => bin}/cdist          | 0
 {scripts => bin}/cdist-dump     | 0
 {scripts => bin}/cdist-new-type | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 rename {scripts => bin}/cdist (100%)
 rename {scripts => bin}/cdist-dump (100%)
 rename {scripts => bin}/cdist-new-type (100%)

diff --git a/scripts/cdist b/bin/cdist
similarity index 100%
rename from scripts/cdist
rename to bin/cdist
diff --git a/scripts/cdist-dump b/bin/cdist-dump
similarity index 100%
rename from scripts/cdist-dump
rename to bin/cdist-dump
diff --git a/scripts/cdist-new-type b/bin/cdist-new-type
similarity index 100%
rename from scripts/cdist-new-type
rename to bin/cdist-new-type

From 86057cef19b10859cd66ceb4e3ebade5986aff89 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:05:17 +0300
Subject: [PATCH 399/743] don't die if there is no version.py

---
 cdist/__init__.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index be573170..26e7d071 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -24,10 +24,13 @@ import os
 import hashlib
 
 import cdist.log
-import cdist.version
 
 
-VERSION = cdist.version.VERSION
+try:
+    import cdist.version
+    VERSION = cdist.version.VERSION
+except ModuleNotFoundError:
+    VERSION = 'from git'
 
 BANNER = """
              ..          .       .x+=:.        s

From fd04c036139f150be79801e5fd3b49086e5d7263 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 13:42:16 +0300
Subject: [PATCH 400/743] add parent dir to module search path only when
 importing fails

---
 bin/cdist | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index 2ce40ae0..66ccbb5a 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -25,13 +25,19 @@ import logging
 import os
 import sys
 
-cdist_bin = os.path.abspath(__file__)
-if os.path.islink(cdist_bin):
-    cdist_bin = os.readlink(cdist_bin)
-cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
-sys.path.insert(0, cdist_dir)
+# try to import cdist and if that fails, then add this file's parent dir to
+# module search path and try again. additionally check if this file is
+# symlinked, so user can symlink this file to, for example, ~/.local/bin.
+try:
+    import cdist
+except ModuleNotFoundError:
+    cdist_bin = os.path.abspath(__file__)
+    if os.path.islink(cdist_bin):
+        cdist_bin = os.readlink(cdist_bin)
+    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
+    sys.path.insert(0, cdist_dir)
+    import cdist
 
-import cdist
 import cdist.argparse
 import cdist.banner
 import cdist.config

From 1614b62f702a82731b3b9c636894268b4310821b Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 13:48:28 +0300
Subject: [PATCH 401/743] fallback VERSION to "unknown version"

---
 cdist/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index 26e7d071..350a2765 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -30,7 +30,7 @@ try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
-    VERSION = 'from git'
+    VERSION = 'unknown version'
 
 BANNER = """
              ..          .       .x+=:.        s

From 174aa7728065da611dc8887d9f33ead7e27c473a Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:11:00 +0300
Subject: [PATCH 402/743] __file__ already is absolute

---
 bin/cdist | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/cdist b/bin/cdist
index 66ccbb5a..d7eae312 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -31,7 +31,7 @@ import sys
 try:
     import cdist
 except ModuleNotFoundError:
-    cdist_bin = os.path.abspath(__file__)
+    cdist_bin = __file__
     if os.path.islink(cdist_bin):
         cdist_bin = os.readlink(cdist_bin)
     cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))

From 65c8af4ba3751dddbfe07fd5077e5a99b5d240b8 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:11:12 +0300
Subject: [PATCH 403/743] overengineered version discovery

---
 cdist/__init__.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index 350a2765..f659506f 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -22,6 +22,7 @@
 
 import os
 import hashlib
+import subprocess
 
 import cdist.log
 
@@ -30,7 +31,19 @@ try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
-    VERSION = 'unknown version'
+    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+    if os.path.isdir(os.path.join(cdist_dir, '.git')):
+        run_git = subprocess.run(
+            ['git', 'describe', '--always'],
+            cwd=cdist_dir,
+            capture_output=True,
+            text=True)
+        if run_git.returncode == 0:
+            VERSION = str(run_git.stdout)
+        else:
+            VERSION = 'from git'
+    else:
+        VERSION = 'unknown version'
 
 BANNER = """
              ..          .       .x+=:.        s

From 42d5d6c3e29f6500c3f6321b01c3b268e6886ec6 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:12:39 +0300
Subject: [PATCH 404/743] redundant str()

---
 cdist/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index f659506f..c16562e7 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -39,7 +39,7 @@ except ModuleNotFoundError:
             capture_output=True,
             text=True)
         if run_git.returncode == 0:
-            VERSION = str(run_git.stdout)
+            VERSION = run_git.stdout
         else:
             VERSION = 'from git'
     else:

From b41d80075a616a1c7ac3a361079c748424409995 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:16:04 +0300
Subject: [PATCH 405/743] update paths in setup.py

---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 7b000041..002be19c 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ import subprocess
 
 
 # We have it only if it is a git cloned repo.
-build_helper = os.path.join('bin', 'build-helper')
+build_helper = os.path.join('bin', 'cdist-build-helper')
 # Version file path.
 version_file = os.path.join('cdist', 'version.py')
 # If we have build-helper we could be a git repo.
@@ -56,7 +56,7 @@ setup(
     name="cdist",
     packages=["cdist", "cdist.core", "cdist.exec", "cdist.util", ],
     package_data={'cdist': package_data},
-    scripts=["scripts/cdist", "scripts/cdist-dump", "scripts/cdist-new-type"],
+    scripts=["bin/cdist", "bin/cdist-dump", "bin/cdist-new-type"],
     version=cdist.version.VERSION,
     description="A Usable Configuration Management System",
     author="Nico Schottelius",

From e55db1b427fcc7f750adfcb3646a3eeb446113de Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 15:41:38 +0300
Subject: [PATCH 406/743] use check_output for git describe execution and
 define fallback VERSION earlier

---
 cdist/__init__.py | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index c16562e7..1c60ae0f 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -27,23 +27,21 @@ import subprocess
 import cdist.log
 
 
+VERSION = 'unknown version'
+
 try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
     cdist_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
     if os.path.isdir(os.path.join(cdist_dir, '.git')):
-        run_git = subprocess.run(
-            ['git', 'describe', '--always'],
-            cwd=cdist_dir,
-            capture_output=True,
-            text=True)
-        if run_git.returncode == 0:
-            VERSION = run_git.stdout
-        else:
-            VERSION = 'from git'
-    else:
-        VERSION = 'unknown version'
+        try:
+            VERSION = subprocess.check_output(
+                ['git', 'describe', '--always'],
+                cwd=cdist_dir,
+                universal_newlines=True)
+        except:
+            pass
 
 BANNER = """
              ..          .       .x+=:.        s

From 54d83a62118ed3f0c6328c4b25a3b9ee56adf27a Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 15:50:50 +0300
Subject: [PATCH 407/743] there is no single author anymore, also remove www.

---
 setup.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 002be19c..858c2c17 100644
--- a/setup.py
+++ b/setup.py
@@ -59,9 +59,8 @@ setup(
     scripts=["bin/cdist", "bin/cdist-dump", "bin/cdist-new-type"],
     version=cdist.version.VERSION,
     description="A Usable Configuration Management System",
-    author="Nico Schottelius",
-    author_email="nico-cdist-pypi@schottelius.org",
-    url="https://www.cdi.st/",
+    author="cdist contributors",
+    url="https://cdi.st",
     classifiers=[
         "Development Status :: 6 - Mature",
         "Environment :: Console",

From 507fa6fa93210959b52bb639a9394c8ac52b40e5 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 17 Oct 2020 17:05:09 +0200
Subject: [PATCH 408/743] __download: fix non-existent parameter of __unpack

Probably happened due to renaming .. guess it's correct now.
---
 cdist/conf/type/__download/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index eb3ac971..d8814683 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -69,7 +69,7 @@ EXAMPLES
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
-            --move-existing-destination \
+            --backup-destination \
             --destination /opt/cpma/server
 
 

From d20fb743243f0341820a5b41a5725fb705eb5aae Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sat, 17 Oct 2020 23:16:42 +0300
Subject: [PATCH 409/743] use os.path.realpath instead, because it eliminates
 any symbolic links encountered in the path

---
 bin/cdist | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index d7eae312..1f92f157 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -25,16 +25,16 @@ import logging
 import os
 import sys
 
-# try to import cdist and if that fails, then add this file's parent dir to
-# module search path and try again. additionally check if this file is
-# symlinked, so user can symlink this file to, for example, ~/.local/bin.
+# try to import cdist and if that fails,
+# then add this file's parent dir to
+# module search path and try again.
 try:
     import cdist
 except ModuleNotFoundError:
-    cdist_bin = __file__
-    if os.path.islink(cdist_bin):
-        cdist_bin = os.readlink(cdist_bin)
-    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
+    cdist_dir = os.path.realpath(
+        os.path.join(
+            os.path.dirname(os.path.realpath(__file__)),
+            os.pardir))
     sys.path.insert(0, cdist_dir)
     import cdist
 

From b2e6afb57e6799b934a2eaa387cea337d65d8aac Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 17 Oct 2020 23:01:36 +0200
Subject: [PATCH 410/743] __download: adapt download+unpack example in manpage

---
 cdist/conf/type/__download/man.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index d8814683..54503470 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -70,6 +70,7 @@ EXAMPLES
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
             --backup-destination \
+            --preserve-archive \
             --destination /opt/cpma/server
 
 

From 955b84727611caa5aa05e4521af326c94c649e89 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 18 Oct 2020 15:55:14 +0200
Subject: [PATCH 411/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 6d237449..8ad2c624 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
 	* Type __service: Fix calling __systemd_service (Mark Verboom)
 	* Type __line: Add 'replace' state (Evil Ham)
+	* Type __download: Fix man page (Matthias Stecher)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 6964070282a6c5d09a458017623012fc17e3008d Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 18 Oct 2020 17:13:22 +0300
Subject: [PATCH 412/743] s/build-helper/cdist-build-helper/

---
 .gitlab-ci.yml             | 8 ++++----
 README-maintainers         | 2 +-
 bin/cdist-build-helper     | 2 +-
 docs/src/cdist-install.rst | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e215652c..e48355ea 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,15 +6,15 @@ image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
 unit_tests:
     stage: test
     script:
-        - ./bin/build-helper version
-        - ./bin/build-helper test
+        - ./bin/cdist-build-helper version
+        - ./bin/cdist-build-helper test
 
 pycodestyle:
     stage: test
     script:
-        - ./bin/build-helper pycodestyle
+        - ./bin/cdist-build-helper pycodestyle
 
 shellcheck:
     stage: test
     script:
-        - ./bin/build-helper shellcheck
+        - ./bin/cdist-build-helper shellcheck
diff --git a/README-maintainers b/README-maintainers
index af57f475..5766dd7d 100644
--- a/README-maintainers
+++ b/README-maintainers
@@ -1,4 +1,4 @@
-Maintainers should use ./bin/build-helper script.
+Maintainers should use ./bin/cdist-build-helper script.
 
 Makefile is intended for end users. It can be used for non-maintaining
 targets that can be run from pure source (without git repository).
diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper
index d4d603ed..bdef0dbb 100755
--- a/bin/cdist-build-helper
+++ b/bin/cdist-build-helper
@@ -495,7 +495,7 @@ eof
     ;;
 
     shellcheck-build-helper)
-        ${SHELLCHECKCMD} ./bin/build-helper
+        ${SHELLCHECKCMD} ./bin/cdist-build-helper
     ;;
 
     check-shellcheck)
diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst
index 6f4f14d7..18863145 100644
--- a/docs/src/cdist-install.rst
+++ b/docs/src/cdist-install.rst
@@ -49,7 +49,7 @@ create version.py:
 
 .. code-block:: sh
 
-    ./bin/build-helper version
+    ./bin/cdist-build-helper version
 
 Then you install it with:
 
@@ -70,7 +70,7 @@ Or directly with distutils:
 
     python setup.py install
 
-Note that `bin/build-helper` script is intended for cdist maintainers.
+Note that `bin/cdist-build-helper` script is intended for cdist maintainers.
 
 
 Available versions in git

From e3d906a85fe5e0c9f254d3620e8568437c93d97c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 15 Sep 2020 00:55:26 +0300
Subject: [PATCH 413/743] [__acl] remove deprecated parameters, fix some bugs
 and improve manual

---
 cdist/conf/type/__acl/explorer/checks         | 39 -------------------
 cdist/conf/type/__acl/explorer/getent         |  4 ++
 cdist/conf/type/__acl/gencode-remote          | 36 ++++++++---------
 cdist/conf/type/__acl/man.rst                 | 12 +++---
 .../conf/type/__acl/parameter/deprecated/acl  |  1 -
 .../type/__acl/parameter/deprecated/group     |  1 -
 .../conf/type/__acl/parameter/deprecated/mask |  1 -
 .../type/__acl/parameter/deprecated/other     |  1 -
 .../conf/type/__acl/parameter/deprecated/user |  1 -
 cdist/conf/type/__acl/parameter/optional      |  2 -
 .../type/__acl/parameter/optional_multiple    |  3 --
 11 files changed, 26 insertions(+), 75 deletions(-)
 delete mode 100755 cdist/conf/type/__acl/explorer/checks
 create mode 100755 cdist/conf/type/__acl/explorer/getent
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/acl
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/group
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/mask
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/other
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/user

diff --git a/cdist/conf/type/__acl/explorer/checks b/cdist/conf/type/__acl/explorer/checks
deleted file mode 100755
index 70bb0412..00000000
--- a/cdist/conf/type/__acl/explorer/checks
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/sh -e
-#
-# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-# TODO check if filesystem has ACL turned on etc
-
-if [ -f "$__object/parameter/acl" ]
-then
-    grep -E '^(default:)?(user|group):' "$__object/parameter/acl" \
-    | while read -r acl
-    do
-        param="$( echo "$acl" | awk -F: '{print $(NF-2)}' )"
-        check="$( echo "$acl" | awk -F: '{print $(NF-1)}' )"
-
-        [ "$param" = 'user' ] && db=passwd || db="$param"
-
-        if ! getent "$db" "$check" > /dev/null
-        then
-            echo "missing $param '$check'" >&2
-            exit 1
-        fi
-    done
-fi
diff --git a/cdist/conf/type/__acl/explorer/getent b/cdist/conf/type/__acl/explorer/getent
new file mode 100755
index 00000000..7e6c2c30
--- /dev/null
+++ b/cdist/conf/type/__acl/explorer/getent
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+getent passwd | awk -F: '{print "user:"$1}'
+getent group | awk -F: '{print "group:"$1}'
diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote
index e5404a9d..32318e91 100755
--- a/cdist/conf/type/__acl/gencode-remote
+++ b/cdist/conf/type/__acl/gencode-remote
@@ -22,8 +22,8 @@ file_is="$( cat "$__object/explorer/file_is" )"
 
 if [ "$file_is" = 'missing' ] \
     && [ -z "$__cdist_dry_run" ] \
-    && \( [ ! -f "$__object/parameter/file" ] \
-        || [ ! -f "$__object/parameter/directory" ] \)
+    && [ ! -f "$__object/parameter/file" ] \
+    && [ ! -f "$__object/parameter/directory" ]
 then
     exit 0
 fi
@@ -47,28 +47,26 @@ then
 elif [ -f "$__object/parameter/entry" ]
 then
     acl_should="$( cat "$__object/parameter/entry" )"
-elif [ -f "$__object/parameter/acl" ]
-then
-    acl_should="$( cat "$__object/parameter/acl" )"
-elif
-    [ -f "$__object/parameter/user" ] \
-        || [ -f "$__object/parameter/group" ] \
-        || [ -f "$__object/parameter/mask" ] \
-        || [ -f "$__object/parameter/other" ]
-then
-    acl_should="$( for param in user group mask other
-    do
-        [ ! -f "$__object/parameter/$param" ] && continue
-
-        echo "$param" | grep -Eq 'mask|other' && sep=:: || sep=:
-
-        echo "$param$sep$( cat "$__object/parameter/$param" )"
-    done )"
 else
     echo 'no parameters set' >&2
     exit 1
 fi
 
+# instead of setfacl's non-helpful message "Option -m: Invalid argument near character X"
+# let's check if target has necessary users and groups, since mistyped or missing
+# users/groups in target is most common reason.
+echo "$acl_should" \
+    | grep -Po '(user|group):[^:]+' \
+    | sort -u \
+    | while read -r l
+    do
+        if ! grep "$l" -Fxq "$__object/explorer/getent"
+        then
+            echo "no $l' in target" | sed "s/:/ '/" >&2
+            exit 1
+        fi
+    done
+
 if [ -f "$__object/parameter/default" ]
 then
     acl_should="$( echo "$acl_should" \
diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst
index 28412871..307be72b 100644
--- a/cdist/conf/type/__acl/man.rst
+++ b/cdist/conf/type/__acl/man.rst
@@ -12,11 +12,14 @@ Fully supported and tested on Linux (ext4 filesystem), partial support for FreeB
 
 See ``setfacl`` and ``acl`` manpages for more details.
 
+One of ``--entry`` or ``--source`` must be used.
 
-REQUIRED MULTIPLE PARAMETERS
+
+OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
 entry
    Set ACL entry following ``getfacl`` output syntax.
+   Must be used if ``--source`` is not used.
 
 
 OPTIONAL PARAMETERS
@@ -25,6 +28,7 @@ source
    Read ACL entries from stdin or file.
    Ordering of entries is not important.
    When reading from file, comments and empty lines are ignored.
+   Must be used if ``--entry`` is not used.
 
 file
    Create/change file with ``__file`` using ``user:group:mode`` pattern.
@@ -48,12 +52,6 @@ remove
    ``mask`` and ``other`` entries can't be removed, but only changed.
 
 
-DEPRECATED PARAMETERS
----------------------
-Parameters ``acl``, ``user``, ``group``, ``mask`` and ``other`` are deprecated and they
-will be removed in future versions. Please use ``entry`` parameter instead.
-
-
 EXAMPLES
 --------
 
diff --git a/cdist/conf/type/__acl/parameter/deprecated/acl b/cdist/conf/type/__acl/parameter/deprecated/acl
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/acl
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/group b/cdist/conf/type/__acl/parameter/deprecated/group
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/group
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/mask b/cdist/conf/type/__acl/parameter/deprecated/mask
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/mask
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/other b/cdist/conf/type/__acl/parameter/deprecated/other
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/other
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/user b/cdist/conf/type/__acl/parameter/deprecated/user
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/user
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/optional b/cdist/conf/type/__acl/parameter/optional
index cdcbc0b8..5a0c29a3 100644
--- a/cdist/conf/type/__acl/parameter/optional
+++ b/cdist/conf/type/__acl/parameter/optional
@@ -1,5 +1,3 @@
-mask
-other
 source
 file
 directory
diff --git a/cdist/conf/type/__acl/parameter/optional_multiple b/cdist/conf/type/__acl/parameter/optional_multiple
index c615d507..4c884f03 100644
--- a/cdist/conf/type/__acl/parameter/optional_multiple
+++ b/cdist/conf/type/__acl/parameter/optional_multiple
@@ -1,4 +1 @@
 entry
-acl
-user
-group

From 716cd37281903aafd7d65df539265a6722e809b1 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 21 Sep 2020 11:18:39 +0300
Subject: [PATCH 414/743] [__update_alternatives] rewrite and support --install

---
 .../explorer/alternatives                     |  4 ++
 .../type/__update_alternatives/explorer/link  | 40 +++++++++++++++++++
 .../__update_alternatives/explorer/path_is    | 12 ++++++
 .../explorer/path_should_state                |  8 ++++
 .../type/__update_alternatives/explorer/state |  8 ----
 .../type/__update_alternatives/gencode-remote | 37 ++++++++++++++---
 cdist/conf/type/__update_alternatives/man.rst | 13 ++++--
 .../__update_alternatives/parameter/boolean   |  1 +
 8 files changed, 107 insertions(+), 16 deletions(-)
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/alternatives
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/link
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/path_is
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/path_should_state
 delete mode 100755 cdist/conf/type/__update_alternatives/explorer/state
 create mode 100644 cdist/conf/type/__update_alternatives/parameter/boolean

diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives
new file mode 100755
index 00000000..34aaca56
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/alternatives
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+update-alternatives --display "$__object_id" 2>/dev/null \
+    | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link
new file mode 100755
index 00000000..6519e7c2
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/link
@@ -0,0 +1,40 @@
+#!/bin/sh -e
+
+# fedora's (update-)alternatives --display output doesn't have
+# "link <name> is <path>" line, but debian does. so, let's find
+# out how they store this information.
+#
+# debian and friends:
+#   https://salsa.debian.org/dpkg-team/dpkg/-/blob/master/utils/update-alternatives.c
+#   see calls to altdb_print_line function
+#
+# fedora and friends:
+#   https://github.com/fedora-sysv/chkconfig/blob/master/alternatives.c
+#   see calls to parseLine function
+#
+# conclusion: it is safe to assume that (master) link is on second line
+
+for altdir in \
+    /var/lib/dpkg/alternatives \
+    /var/lib/alternatives
+do
+    if [ ! -f "$altdir/$__object_id" ]
+    then
+        continue
+    fi
+
+    link="$( awk 'NR==2' "$altdir/$__object_id" )"
+
+    if [ -n "$link" ]
+    then
+        break
+    fi
+done
+
+if [ -z "$link" ]
+then
+    echo "unable to get link for $__object_id" >&2
+    exit 1
+fi
+
+echo "$link"
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
new file mode 100755
index 00000000..fc304d5d
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -0,0 +1,12 @@
+#!/bin/sh -e
+
+path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \
+    | awk '/link currently points to/ {print $5}' )"
+
+if [ -z "$path_is" ]
+then
+    echo "unable to get current path for $__object_id" >&2
+    exit 1
+fi
+
+echo "$path_is"
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state
new file mode 100755
index 00000000..59e015c5
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+if [ -f "$( cat "$__object/parameter/path" )" ]
+then
+    echo 'present'
+else
+    echo 'absent'
+fi
diff --git a/cdist/conf/type/__update_alternatives/explorer/state b/cdist/conf/type/__update_alternatives/explorer/state
deleted file mode 100755
index 04a78aaa..00000000
--- a/cdist/conf/type/__update_alternatives/explorer/state
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh -e
-path="$(cat "$__object/parameter/path")"
-name="$__object_id"
-link="$(readlink "/etc/alternatives/$name")"
-if [ "$path" = "$link" ]
-then echo present
-else echo absent
-fi
diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index c0b49814..e393cdef 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Ander Punnar (ander@kvlt.ee)
 #
 # This file is part of cdist.
 #
@@ -16,12 +17,38 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
-if [ "$(cat "$__object/explorer/state")" = 'present' ]
-then exit 0
+path_is="$( cat "$__object/explorer/path_is" )"
+
+path_should="$( cat "$__object/parameter/path" )"
+
+if [ "$path_is" = "$path_should" ]
+then
+    exit 0
+fi
+
+if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ]
+then
+    echo "$path_should does not exist in target" >&2
+    exit 1
 fi
 
-path="$(cat "$__object/parameter/path")"
 name="$__object_id"
-echo "update-alternatives --quiet --set '$name' '$path'"
+
+alternatives="$( cat "$__object/explorer/alternatives" )"
+
+if ! echo "$alternatives" | grep -Fxq "$path_should"
+then
+    if [ ! -f "$__object/parameter/install" ]
+    then
+        echo "$path_should is not in $name alternatives." >&2
+        echo 'Please install missing packages or use --install to add path to alternatives.' >&2
+        exit 1
+    fi
+
+    link="$( cat "$__object/explorer/link" )"
+
+    echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
+fi
+
+echo "update-alternatives --set '$name' '$path_should'"
diff --git a/cdist/conf/type/__update_alternatives/man.rst b/cdist/conf/type/__update_alternatives/man.rst
index 73d82d11..0dc973f2 100644
--- a/cdist/conf/type/__update_alternatives/man.rst
+++ b/cdist/conf/type/__update_alternatives/man.rst
@@ -19,6 +19,12 @@ path
    Use this path for the given alternative
 
 
+BOOLEAN PARAMETERS
+------------------
+install
+   Add (``update-alternatives --install``) missing path to alternatives.
+
+
 EXAMPLES
 --------
 
@@ -36,11 +42,12 @@ SEE ALSO
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Ander Punnar <ander@kvlt.ee>
 
 
 COPYING
 -------
-Copyright \(C) 2013 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
+Copyright \(C) 2013 Nico Schottelius and 2020 Ander Punnar. You can
+redistribute it and/or modify it under the terms of the GNU General Public
+License as published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__update_alternatives/parameter/boolean b/cdist/conf/type/__update_alternatives/parameter/boolean
new file mode 100644
index 00000000..7c32f559
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/parameter/boolean
@@ -0,0 +1 @@
+install

From 687c1d2dd9bed4130a65885d19ce31c87c9d79de Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 19 Oct 2020 06:57:00 +0200
Subject: [PATCH 415/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 8ad2c624..a0f1ead2 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,8 @@ next:
 	* Type __service: Fix calling __systemd_service (Mark Verboom)
 	* Type __line: Add 'replace' state (Evil Ham)
 	* Type __download: Fix man page (Matthias Stecher)
+	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
+	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 367da4b77e79b82fe84c2ce1a7f75cfad92db81e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 28 Oct 2020 18:18:24 +0100
Subject: [PATCH 416/743] [type/__file] Fix --state pre-exists

---
 cdist/conf/type/__file/gencode-remote | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index 35356b13..2675b03a 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -87,11 +87,6 @@ case "$state_should" in
         fi
     ;;
 
-    pre-exists)
-        # pre-exists should never reach gencode-remote…
-        exit 1
-   ;;
-
     absent)
         if [ "$type" = "file" ]; then
             echo "rm -f '$destination'"

From 9277e0ba19c346189658712acf17a772967dfa32 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 29 Oct 2020 09:30:58 +0100
Subject: [PATCH 417/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index a0f1ead2..5977d80e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,7 @@ next:
 	* Type __download: Fix man page (Matthias Stecher)
 	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
+	* Type __file: Fix state pre-exists (Dennis Camera)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 82a9aa79020ff2369b1f4d530a3a6d5ca41915f8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 29 Oct 2020 10:39:42 +0100
Subject: [PATCH 418/743] [type/__apt_norecommends] Use 00InstallRecommends
 file as debian-installer does

debian-installer can be preseeded with `base-installer/install-recommends` to
disable installation of recommended packages already during OS installation.
d-i will then create the file `/etc/apt/apt.conf.d/00InstallRecommends`
(cf. https://salsa.debian.org/installer-team/base-installer/-/blob/master/library.sh).

__apt_norecommends should use the same file to avoid having two config files
effectively doing the same thing.
---
 cdist/conf/type/__apt_norecommends/man.rst  |  9 +++--
 cdist/conf/type/__apt_norecommends/manifest | 41 +++++++++++----------
 2 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/cdist/conf/type/__apt_norecommends/man.rst b/cdist/conf/type/__apt_norecommends/man.rst
index 001fffe4..9297b518 100644
--- a/cdist/conf/type/__apt_norecommends/man.rst
+++ b/cdist/conf/type/__apt_norecommends/man.rst
@@ -32,11 +32,12 @@ EXAMPLES
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
+Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Copyright \(C) 2014 Steven Armstrong. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2014 Steven Armstrong, 2020 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__apt_norecommends/manifest b/cdist/conf/type/__apt_norecommends/manifest
index e737df89..fc187784 100755
--- a/cdist/conf/type/__apt_norecommends/manifest
+++ b/cdist/conf/type/__apt_norecommends/manifest
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,26 +20,28 @@
 #
 
 
-os=$(cat "$__global/explorer/os")
+os=$(cat "${__global:?}/explorer/os")
 
-case "$os" in
-   ubuntu|debian|devuan)
-      # No stinking recommends thank you very much.
-      # If I want something installed I will do so myself.
-      __file /etc/apt/apt.conf.d/99-no-recommends \
-         --owner root --group root --mode 644 \
-         --source - << DONE
-APT::Install-Recommends "0";
-APT::Install-Suggests "0";
-APT::AutoRemove::RecommendsImportant "0";
-APT::AutoRemove::SuggestsImportant "0";
-DONE
-   ;;
-   *)
-      cat >&2 << DONE
+case ${os}
+in
+	(ubuntu|debian|devuan)
+		__file /etc/apt/apt.conf.d/00InstallRecommends --state present \
+			--owner root --group root --mode 0644 --source - <<-'EOF'
+		APT::Install-Recommends "false";
+		APT::Install-Suggests "false";
+		APT::AutoRemove::RecommendsImportant "false";
+		APT::AutoRemove::SuggestsImportant "false";
+		EOF
+
+		# TODO: Remove the following object after some time
+		require=__file/etc/apt/apt.conf.d/00InstallRecommends \
+		__file /etc/apt/apt.conf.d/99-no-recommends --state absent
+		;;
+	(*)
+		cat >&2 <<EOF
 The developer of this type (${__type##*/}) did not think your operating system
 ($os) would have any use for it. If you think otherwise please submit a patch.
-DONE
-      exit 1
-   ;;
+EOF
+		exit 1
+		;;
 esac

From b9ad22595fdb75c17e88e33dc028c33dbd5a7de9 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 18:03:27 +0100
Subject: [PATCH 419/743] [scanner] begin scanner implementation - non invasive

---
 cdist/scan.py | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 cdist/scan.py

diff --git a/cdist/scan.py b/cdist/scan.py
new file mode 100644
index 00000000..fa1bf5de
--- /dev/null
+++ b/cdist/scan.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+from scapy.all import *
+from scapy.data import ETHER_TYPES
+
+
+class Scanner(object):
+    def recv_msg_cpu(self, pkg):
+#        print(pkg.__repr__())
+        if ICMPv6EchoReply in pkg:
+            host = pkg['IPv6'].src
+            print(f"Host {host} is alive")
+
+
+    def scan(self):
+        sniff(iface="wlan0",
+              filter="icmp6",
+              prn=self.recv_msg_cpu)
+
+
+if __name__ == '__main__':
+    s = Scanner()
+    s.scan()

From 87b46a622411362f441b62199972e71d85d7d2d6 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 18:49:20 +0100
Subject: [PATCH 420/743] [scanner] finish prototype

ping @poljakowski - it's your turn now
---
 cdist/scan.py | 138 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 131 insertions(+), 7 deletions(-)

diff --git a/cdist/scan.py b/cdist/scan.py
index fa1bf5de..e2100499 100644
--- a/cdist/scan.py
+++ b/cdist/scan.py
@@ -19,24 +19,148 @@
 #
 #
 
-from scapy.all import *
-from scapy.data import ETHER_TYPES
+#
+# Interface to be implemented:
+# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated
+#   scan: scan / listen for icmp6 replies
+#   trigger: send trigger to multicast
+#   config: configure newly detected hosts
+#   install: install newly detected hosts
+#
+# Scanner logic
+#  - save results to configdir:
+#     basedir = ~/.cdist/scan/<ipv6-address>
+#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time or similar
+#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#
+#
+#
+#
+# cdist scan --list
+#       Show all known hosts including last seen flag
+#
+# Logic for reconfiguration:
+#
+#  - record when configured last time
+#  - introduce a parameter --reconfigure-after that takes time argument
+#  - reconfigure if a) host alive and b) reconfigure-after time passed
+#
 
 
+from multiprocessing import Process
+import os
+
+# FIXME: fail gracefully if non existent - i.e. "scapy required for scanner - please install python3-scapy"
+from scapy.all import *
+
+# Datetime overwrites scapy.all.datetime - needs to be imported AFTER
+import datetime
+
+class Trigger(object):
+    """
+    Trigger an ICMPv6EchoReply from all hosts that are alive
+    """
+
+    def __init__(self, interfaces=None, verbose=False):
+        self.interfaces = interfaces
+        self.verbose = verbose
+
+        # Wait 5 seconds before triggering again - FIXME: add parameter
+        self.sleeptime = 5
+
+    def start(self):
+        self.processes = []
+        for interface in self.interfaces:
+            p = Process(target=self.run_interface, args=(interface,))
+            self.processes.append(p)
+            p.start()
+
+    def join(self):
+        for process in self.processes:
+            process.join()
+
+    def run_interface(self, interface):
+        while True:
+            self.trigger(interface)
+            time.sleep(self.sleeptime)
+
+    def trigger(self, interface):
+        packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
+        send(packet, verbose=self.verbose)
+
 class Scanner(object):
-    def recv_msg_cpu(self, pkg):
-#        print(pkg.__repr__())
+    """
+    Scan for replies of hosts, maintain the up-to-date database
+    """
+
+    def __init__(self, interfaces=None, outdir=None):
+        self.interfaces = interfaces
+
+        if outdir:
+            self.outdir = outdir
+        else:
+            self.outdir = "."
+
+    def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
             host = pkg['IPv6'].src
             print(f"Host {host} is alive")
 
+            dir = os.path.join(self.outdir, host)
+            fname = os.path.join(dir, "last_seen")
+
+            now = datetime.datetime.now()
+
+            os.makedirs(dir, exist_ok=True)
+
+            # FIXME: maybe adjust the format so we can easily parse again
+            with open(fname, "w") as fd:
+                fd.write(f"{now}\n")
+
 
     def scan(self):
-        sniff(iface="wlan0",
+        sniff(iface=self.interfaces,
               filter="icmp6",
-              prn=self.recv_msg_cpu)
+              prn=self.handle_pkg)
 
 
 if __name__ == '__main__':
-    s = Scanner()
+    t = Trigger(interfaces=["wlan0"])
+    t.start()
+
+    # Scanner can listen on many interfaces at the same time
+    s = Scanner(interfaces=["wlan0"])
     s.scan()
+
+    # Join back the trigger processes
+    t.join()
+
+    # Test in my lan shows:
+    # [18:48] bridge:cdist% ls -1d fe80::*
+    # fe80::142d:f0a5:725b:1103
+    # fe80::20d:b9ff:fe49:ac11
+    # fe80::20d:b9ff:fe4c:547d
+    # fe80::219:d2ff:feb2:2e12
+    # fe80::21b:fcff:feee:f446
+    # fe80::21b:fcff:feee:f45c
+    # fe80::21b:fcff:feee:f4b1
+    # fe80::21b:fcff:feee:f4ba
+    # fe80::21b:fcff:feee:f4bc
+    # fe80::21b:fcff:feee:f4c1
+    # fe80::21d:72ff:fe86:46b
+    # fe80::42b0:34ff:fe6f:f6f0
+    # fe80::42b0:34ff:fe6f:f863
+    # fe80::42b0:34ff:fe6f:f9b2
+    # fe80::4a5d:60ff:fea1:e55f
+    # fe80::77a3:5e3f:82cc:f2e5
+    # fe80::9e93:4eff:fe6c:c1f4
+    # fe80::ba69:f4ff:fec5:6041
+    # fe80::ba69:f4ff:fec5:8db7
+    # fe80::bad8:12ff:fe65:313d
+    # fe80::bad8:12ff:fe65:d9b1
+    # fe80::ce2d:e0ff:fed4:2611
+    # fe80::ce32:e5ff:fe79:7ea7
+    # fe80::d66d:6dff:fe33:e00
+    # fe80::e2ff:f7ff:fe00:20e6
+    # fe80::f29f:c2ff:fe7c:275e

From 91d99bf08accb3bde0da420294fc9fb68d3d4068 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 21:22:36 +0100
Subject: [PATCH 421/743] [RFC] scanner documentation

---
 docs/dev/logs/2020-10-29.org | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 docs/dev/logs/2020-10-29.org

diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org
new file mode 100644
index 00000000..718fd68c
--- /dev/null
+++ b/docs/dev/logs/2020-10-29.org
@@ -0,0 +1,34 @@
+* The scanner, 2020-10-29, Hacking Villa Diesbach
+** Motivation
+   - The purpose of cdist is to ensure systems are in a configured state
+   - If systems reboot into a clean (think: netboot) state they are
+     stuck in an unconfigured mode
+   - We can either trigger *from* those machines
+     - this is what cdist trigger is for
+   - Or we can regulary *scan* for machines
+     - This method does not need any modification to standard OS
+** How it works
+   - cdist scan uses the all nodes multicast group ff02::1
+   - It sends a ping packet there in regular intervals
+   - This even works in non-IPv6 networks, as all operating systems
+     are IPv6 capable and usually IPv6 enabled by default
+     - Link local is always accessible!
+   - cdist scan receives an answer from all alive hosts
+     - These results are stored in ~/.cdist/scan/${hostip}
+     - We record the last_seen date  ~/.cdist/scan/${hostip}/last_seen
+   - After a host is detected, cdist *can* try to configure it
+     - It saves the result (+/- logging needs to be defined) in
+       ~/.cdist/scan/${hostip}/{config, install}_result
+     - If logging is saved: maybe in ~/.cdist/scan/${hostip}/{config, install}_log
+     - Final naming TBD
+** Benefits from the scanning approach
+   - We know when a host is alive/dead
+   - We can use standard OS w/o trigger customisation
+     - Only requirement: we can ssh into it
+     - Can make use f.i. of Alpine Linux w/ ssh keys feeding in
+   - We can trigger regular reconfiguration
+     - If alive && last_config_time > 1d -> reconfigure
+   - Data can be exported to f.i. prometheus
+     - Record when configured (successfully)
+     - Record when seen
+   - Enables configurations in stateless environments

From 09dfcfe81e0d9e6520d9bf98598037f86c84641e Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 23:16:08 +0100
Subject: [PATCH 422/743] [scanner] add to beta commands

---
 cdist/argparse.py            | 32 ++++++++++++++++++++-
 cdist/scan/commandline.py    | 55 ++++++++++++++++++++++++++++++++++++
 cdist/{ => scan}/scan.py     | 20 +++++++++----
 docs/dev/logs/2020-10-29.org | 23 +++++++++++++++
 4 files changed, 124 insertions(+), 6 deletions(-)
 create mode 100644 cdist/scan/commandline.py
 rename cdist/{ => scan}/scan.py (90%)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 1d16bb25..ff195e8c 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -8,10 +8,11 @@ import cdist.configuration
 import cdist.log
 import cdist.preos
 import cdist.info
+import cdist.scan.commandline
 
 
 # set of beta sub-commands
-BETA_COMMANDS = set(('install', 'inventory', ))
+BETA_COMMANDS = set(('install', 'inventory', 'scan', ))
 # set of beta arguments for sub-commands
 BETA_ARGS = {
     'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
@@ -470,6 +471,35 @@ def get_parsers():
             'pattern', nargs='?', help='Glob pattern.')
     parser['info'].set_defaults(func=cdist.info.Info.commandline)
 
+    # Scan = config + further
+    parser['scan'] = parser['sub'].add_parser('scan', add_help=False,
+                                                 parents=[parser['config']])
+
+    parser['scan'] = parser['sub'].add_parser(
+            'scan', parents=[parser['loglevel'],
+                             parser['beta'],
+                             parser['colored_output'],
+                             parser['common'],
+                             parser['config_main']])
+
+    parser['scan'].add_argument(
+        '-m', '--mode', help='Which modes should run',
+        action='append', default=[],
+        choices=['scan', 'trigger'])
+    parser['scan'].add_argument(
+        '--config',
+        action='store_true',
+        help='Try to configure detected hosts')
+    parser['scan'].add_argument(
+        '-I', '--interfaces',
+        action='append',  default=[],
+        help='On which interfaces to scan/trigger')
+    parser['scan'].add_argument(
+        '-d', '--delay',
+        action='store',  default=3600,
+        help='How long to wait before reconfiguring after last try')
+    parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
+
     for p in parser:
         parser[p].epilog = EPILOG
 
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
new file mode 100644
index 00000000..0fb718e5
--- /dev/null
+++ b/cdist/scan/commandline.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import logging
+
+log = logging.getLogger("scan")
+
+
+# define this outside of the class to not handle scapy import errors by default
+def commandline(args):
+    log.debug(args)
+
+    try:
+        import cdist.scan.scan as scan
+    except ModuleNotFoundError:
+        print('cdist scan requires scapy to be installed')
+
+    processes = []
+
+    if not args.mode:
+        # By default scan and trigger, but do not call any action
+        args.mode = ['scan', 'trigger' ]
+
+    if 'trigger' in args.mode:
+        t = scan.Trigger(interfaces=args.interfaces)
+        t.start()
+        processes.append(t)
+        log.debug("Trigger started")
+
+    if 'scan' in args.mode:
+        s = scan.Scanner(interfaces=args.interfaces, args=args)
+        s.start()
+        processes.append(s)
+        log.debug("Scanner started")
+
+    for process in processes:
+        process.join()
diff --git a/cdist/scan.py b/cdist/scan/scan.py
similarity index 90%
rename from cdist/scan.py
rename to cdist/scan/scan.py
index e2100499..fcbf1899 100644
--- a/cdist/scan.py
+++ b/cdist/scan/scan.py
@@ -50,13 +50,14 @@
 
 from multiprocessing import Process
 import os
-
-# FIXME: fail gracefully if non existent - i.e. "scapy required for scanner - please install python3-scapy"
+import logging
 from scapy.all import *
 
 # Datetime overwrites scapy.all.datetime - needs to be imported AFTER
 import datetime
 
+log = logging.getLogger("scan")
+
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
@@ -87,6 +88,7 @@ class Trigger(object):
 
     def trigger(self, interface):
         packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
+        log.debug(f"Sending request on {interface}")
         send(packet, verbose=self.verbose)
 
 class Scanner(object):
@@ -94,18 +96,18 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces=None, outdir=None):
+    def __init__(self, interfaces=None, args=None, outdir=None):
         self.interfaces = interfaces
 
         if outdir:
             self.outdir = outdir
         else:
-            self.outdir = "."
+            self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan')
 
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
             host = pkg['IPv6'].src
-            print(f"Host {host} is alive")
+            log.verbose(f"Host {host} is alive")
 
             dir = os.path.join(self.outdir, host)
             fname = os.path.join(dir, "last_seen")
@@ -118,13 +120,21 @@ class Scanner(object):
             with open(fname, "w") as fd:
                 fd.write(f"{now}\n")
 
+    def start(self):
+        self.process = Process(target=self.scan)
+        self.process.start()
+
+    def join(self):
+        self.process.join()
 
     def scan(self):
+        log.debug("Scanning - zzzzz")
         sniff(iface=self.interfaces,
               filter="icmp6",
               prn=self.handle_pkg)
 
 
+
 if __name__ == '__main__':
     t = Trigger(interfaces=["wlan0"])
     t.start()
diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org
index 718fd68c..4461be8c 100644
--- a/docs/dev/logs/2020-10-29.org
+++ b/docs/dev/logs/2020-10-29.org
@@ -32,3 +32,26 @@
      - Record when configured (successfully)
      - Record when seen
    - Enables configurations in stateless environments
+** Sample output v2020-10-29
+23:14] bridge:~% sudo  cdist scan -b -I wlan0 -vv
+VERBOSE: cdist: version 6.8.0-36-g91d99bf0
+VERBOSE: scan: Host fe80::21d:72ff:fe86:46b is alive
+VERBOSE: scan: Host fe80::ce2d:e0ff:fed4:2611 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4c1 is alive
+VERBOSE: scan: Host fe80::e2ff:f7ff:fe00:20e6 is alive
+VERBOSE: scan: Host fe80::20d:b9ff:fe49:ac11 is alive
+VERBOSE: scan: Host fe80::9e93:4eff:fe6c:c1f4 is alive
+VERBOSE: scan: Host fe80::ce32:e5ff:fe79:7ea7 is alive
+VERBOSE: scan: Host fe80::219:d2ff:feb2:2e12 is alive
+VERBOSE: scan: Host fe80::d66d:6dff:fe33:e00 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f446 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4b1 is alive
+VERBOSE: scan: Host fe80::20d:b9ff:fe4c:547d is alive
+VERBOSE: scan: Host fe80::bad8:12ff:fe65:313d is alive
+VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f6f0 is alive
+VERBOSE: scan: Host fe80::ba69:f4ff:fec5:6041 is alive
+VERBOSE: scan: Host fe80::f29f:c2ff:fe7c:275e is alive
+VERBOSE: scan: Host fe80::ba69:f4ff:fec5:8db7 is alive
+VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f863 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4bc is alive
+...

From e30ecdda535200ac75a6e774590d2e1b5e1b5b28 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 13:05:40 +0200
Subject: [PATCH 423/743] Add __uci and __uci_commit types

---
 cdist/conf/type/__uci/explorer/state          | 89 +++++++++++++++++++
 cdist/conf/type/__uci/gencode-remote          | 67 ++++++++++++++
 cdist/conf/type/__uci/man.rst                 | 77 ++++++++++++++++
 cdist/conf/type/__uci/manifest                | 40 +++++++++
 cdist/conf/type/__uci/parameter/default/state |  1 +
 .../type/__uci/parameter/default/transaction  |  1 +
 cdist/conf/type/__uci/parameter/optional      |  2 +
 .../type/__uci/parameter/required_multiple    |  1 +
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++
 cdist/conf/type/__uci_commit/man.rst          | 58 ++++++++++++
 cdist/conf/type/__uci_commit/nonparallel      |  0
 11 files changed, 357 insertions(+)
 create mode 100644 cdist/conf/type/__uci/explorer/state
 create mode 100755 cdist/conf/type/__uci/gencode-remote
 create mode 100644 cdist/conf/type/__uci/man.rst
 create mode 100755 cdist/conf/type/__uci/manifest
 create mode 100644 cdist/conf/type/__uci/parameter/default/state
 create mode 100644 cdist/conf/type/__uci/parameter/default/transaction
 create mode 100644 cdist/conf/type/__uci/parameter/optional
 create mode 100644 cdist/conf/type/__uci/parameter/required_multiple
 create mode 100755 cdist/conf/type/__uci_commit/gencode-remote
 create mode 100644 cdist/conf/type/__uci_commit/man.rst
 create mode 100644 cdist/conf/type/__uci_commit/nonparallel

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
new file mode 100644
index 00000000..2e98b606
--- /dev/null
+++ b/cdist/conf/type/__uci/explorer/state
@@ -0,0 +1,89 @@
+#!/bin/sh
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current state of the configuration option
+# The output of this explorer is one of these values:
+#   present
+#     The configuration option is present and has the value of the
+#     parameter --value.
+#   absent
+#     The configuration option is not defined.
+#   different
+#     The configuration option is present but has a different value than the
+#     parameter --value.
+#   rearranged
+#     The configuration option is present (a list) and has the same values as
+#     the parameter --value, but in a different order.
+
+RS=$(printf '\036')
+
+option=${__object_id:?}
+
+values_is=$(uci -s -N -d "${RS}" get "${option}" 2>/dev/null) || {
+	echo absent
+	exit 0
+}
+
+# strip off trailing newline
+printf '%s' "${values_is}" \
+| awk '
+BEGIN {
+	state = "present"  # assume all is fine
+}
+NR == FNR {
+	# memoize "should" state
+	should[FNR] = $0
+
+	# go to next line (important!)
+	next
+}
+
+# compare "is" state
+$0 == should[FNR] { next }
+
+FNR > length(should) {
+	# there are more "is" records than "should" -> definitely different
+	state = "different"
+	exit
+}
+
+{
+	# see if we can find the value somewhere in should
+	for (i in should) {
+		if ($0 == should[i]) {
+			# ... value found -> rearranged
+			# FIXME: Duplicate values are not properly handled here. Do they matter?
+			state = "rearranged"
+			next
+		}
+	}
+
+	state = "different"
+	exit
+}
+
+END {
+	if (FNR < length(should)) {
+		# "is" was shorter than "should" -> different
+		state = "different"
+	}
+
+	print state
+}
+' "${__object:?}/parameter/value" RS="${RS}" -
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
new file mode 100755
index 00000000..48f114fe
--- /dev/null
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -0,0 +1,67 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
+
+config=${__object_id:?}
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+case ${state_should}
+in
+	(present)
+		if in_list "${state_is}" 'present' 'rearranged'
+		then
+			# NOTE: order is ignored so rearranged is also fine.
+			exit 0
+		fi
+
+		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
+		then
+			# "should" is a list
+			if test "${state_is}" != 'absent'
+			then
+				printf "uci delete '%s'\n" "${config}"
+			fi
+
+			while read -r value
+			do
+				printf "uci add_list '%s'='%s'\n" "${config}" "${value}"
+			done <"${__object:?}/parameter/value"
+		else
+			# "should" is a scalar
+			value=$(cat "${__object:?}/parameter/value")
+			printf "uci set '%s'='%s'\n" "${config}" "${value}"
+		fi
+		;;
+	(absent)
+		if in_list "${state_is}" 'absent'
+		then
+			exit 0
+		fi
+
+		printf "uci delete '%s'\n" "${config}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
new file mode 100644
index 00000000..d23d8b2b
--- /dev/null
+++ b/cdist/conf/type/__uci/man.rst
@@ -0,0 +1,77 @@
+cdist-type__uci(7)
+==================
+
+NAME
+----
+cdist-type__uci - Manage configuration values in OpenWrt's
+Unified Configuration Interface (UCI)
+
+
+DESCRIPTION
+-----------
+This cdist type can be used to alter configuration options in OpenWrt's UCI
+system.
+
+Options can be applied in batches if the `--transaction` parameter is used.
+It is important to ensure that the `__uci_commit` object is executed before a
+new transaction is started.
+
+REQUIRED PARAMETERS
+-------------------
+value
+    The value to be set. Can be used multiple times.
+
+    Due to the way cdist handles arguments, values **must not** contain newline
+    characters.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    `present` or `absent`, defaults to `present`.
+transaction
+    The name of the transaction this option belongs to.
+    If none is given: "default" is used.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Set the system hostname
+    __uci system.@system[0].hostname --value 'OpenWrt'
+
+    # Enable NTP and NTPd (in one transaction)
+    __uci system.ntp.enabled --value 1 --transaction ntp
+    __uci system.ntp.enable_server --value 1 --transaction ntp
+    __uci system.ntp.server --transaction ntp \
+        --value '0.openwrt.pool.ntp.org' \
+        --value '1.openwrt.pool.ntp.org' \
+        --value '2.openwrt.pool.ntp.org' \
+        --value '3.openwrt.pool.ntp.org'
+    export require=__uci_commit/ntp
+
+
+SEE ALSO
+--------
+- https://openwrt.org/docs/guide-user/base-system/uci
+- :strong:`cdist-type__uci_commit`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
new file mode 100755
index 00000000..e5b0fb30
--- /dev/null
+++ b/cdist/conf/type/__uci/manifest
@@ -0,0 +1,40 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+os=$(cat "${__global:?}/explorer/os")
+
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+case ${os}
+in
+	(openwrt)
+		# okay
+		;;
+	(*)
+		printf "Your operating system (%s) is currently not supported by this type (%s)\n" "${os}" "${__type##*/}" >&2
+		printf "Please contribute an implementation for it if you can.\n" >&2
+		exit 1
+		;;
+esac
+
+
+# Make sure the changes are being commited
+require=${__object_name:?} __uci_commit "${transaction_name}"
diff --git a/cdist/conf/type/__uci/parameter/default/state b/cdist/conf/type/__uci/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__uci/parameter/default/transaction b/cdist/conf/type/__uci/parameter/default/transaction
new file mode 100644
index 00000000..4ad96d51
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/default/transaction
@@ -0,0 +1 @@
+default
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
new file mode 100644
index 00000000..ddbbba16
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -0,0 +1,2 @@
+state
+transaction
diff --git a/cdist/conf/type/__uci/parameter/required_multiple b/cdist/conf/type/__uci/parameter/required_multiple
new file mode 100644
index 00000000..6d4e1507
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/required_multiple
@@ -0,0 +1 @@
+value
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
new file mode 100755
index 00000000..bed0eefb
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -0,0 +1,21 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+echo 'uci commit'
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
new file mode 100644
index 00000000..c55d06e5
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/man.rst
@@ -0,0 +1,58 @@
+cdist-type__uci_commit(7)
+=========================
+
+NAME
+----
+cdist-type__uci_commit - Commit a UCI transaction.
+
+
+DESCRIPTION
+-----------
+This type executes the "uci commit" command on the target.
+It is usually not required to use this type. Use the `--transaction` parameter
+of `cdist-type__uci`\ (7) instead.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Commit the default transaction
+    __uci_commit default
+
+    # Commit another transaction
+    __uci_commit my_transaction
+
+
+SEE ALSO
+--------
+:strong:`cdist-type__uci`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_commit/nonparallel b/cdist/conf/type/__uci_commit/nonparallel
new file mode 100644
index 00000000..e69de29b

From 55e7b32449c9a86e34054c321b20a49795c6652f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 14:00:29 +0200
Subject: [PATCH 424/743] [type/__uci] Only generate __uci_commit if changes
 are required

---
 cdist/conf/type/__uci/manifest | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index e5b0fb30..74524513 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,9 +18,12 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; } }
 
 os=$(cat "${__global:?}/explorer/os")
 
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case ${os}
@@ -35,6 +38,25 @@ in
 		;;
 esac
 
+changes_required=false
 
-# Make sure the changes are being commited
-require=${__object_name:?} __uci_commit "${transaction_name}"
+case ${state_should}
+in
+	(present)
+		# NOTE: order is ignored so rearranged is also fine.
+		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
+		;;
+	(absent)
+		in_list "${state_is}" 'absent' || changes_required=true
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
+
+if ${changes_required}
+then
+	# Make sure the changes are being committed
+	require=${__object_name:?} __uci_commit "${transaction_name}"
+fi

From a09120977f231fe3dda1c7c07073fc7e03fc5ea0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 14:07:10 +0200
Subject: [PATCH 425/743] [type/__uci] Allow omission of --value parameter if
 --state absent

---
 cdist/conf/type/__uci/explorer/state                   | 10 +++++++++-
 cdist/conf/type/__uci/man.rst                          |  1 +
 cdist/conf/type/__uci/manifest                         |  5 +++++
 .../parameter/{required_multiple => optional_multiple} |  0
 4 files changed, 15 insertions(+), 1 deletion(-)
 rename cdist/conf/type/__uci/parameter/{required_multiple => optional_multiple} (100%)

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
index 2e98b606..6fb4b173 100644
--- a/cdist/conf/type/__uci/explorer/state
+++ b/cdist/conf/type/__uci/explorer/state
@@ -40,6 +40,14 @@ values_is=$(uci -s -N -d "${RS}" get "${option}" 2>/dev/null) || {
 	exit 0
 }
 
+if test -f "${__object:?}/parameter/value"
+then
+	should_file="${__object:?}/parameter/value"
+else
+	should_file='/dev/null'
+fi
+
+
 # strip off trailing newline
 printf '%s' "${values_is}" \
 | awk '
@@ -86,4 +94,4 @@ END {
 
 	print state
 }
-' "${__object:?}/parameter/value" RS="${RS}" -
+' "${should_file}" RS="${RS}" -
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index d23d8b2b..c6bb81ab 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -20,6 +20,7 @@ REQUIRED PARAMETERS
 -------------------
 value
     The value to be set. Can be used multiple times.
+    This parameter is allowed to be omitted if `--state` is `absent`.
 
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index 74524513..e56462e5 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -43,6 +43,11 @@ changes_required=false
 case ${state_should}
 in
 	(present)
+		test -s "${__object:?}/parameter/value" || {
+			echo 'The parameter --value is required.' >&2
+			exit 1
+		}
+
 		# NOTE: order is ignored so rearranged is also fine.
 		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
 		;;
diff --git a/cdist/conf/type/__uci/parameter/required_multiple b/cdist/conf/type/__uci/parameter/optional_multiple
similarity index 100%
rename from cdist/conf/type/__uci/parameter/required_multiple
rename to cdist/conf/type/__uci/parameter/optional_multiple

From d8f20a6a204142360e7fb7afd6603c69ae613e2d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 12 Jun 2020 19:39:19 +0200
Subject: [PATCH 426/743] [type/__uci] Implement "real" transactions using
 batch files

---
 .../__uci/{gencode-remote => gencode-local}   | 22 ++++++++++++++++---
 cdist/conf/type/__uci/man.rst                 |  3 +--
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++++++++++++++-
 3 files changed, 40 insertions(+), 6 deletions(-)
 rename cdist/conf/type/__uci/{gencode-remote => gencode-local} (79%)

diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-local
similarity index 79%
rename from cdist/conf/type/__uci/gencode-remote
rename to cdist/conf/type/__uci/gencode-local
index 48f114fe..bba5944d 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-local
@@ -20,11 +20,27 @@
 
 in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
 
+uci_cmd() {
+	printf 'printf "%s\n"' "$1"
+	shift
+	printf " '%s'" "$@"
+	printf " >>'%s'\n" "${tmpfile}"
+}
+
 config=${__object_id:?}
 
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+tmpdir="${__global:?}/tmp/__uci"
+# HACK
+mkdir -p "${tmpdir}"
+
+tmpfile="${tmpdir}/${transaction_name}.txt"
+
+
 case ${state_should}
 in
 	(present)
@@ -44,12 +60,12 @@ in
 
 			while read -r value
 			do
-				printf "uci add_list '%s'='%s'\n" "${config}" "${value}"
+				uci_cmd "add_list '%s'='%s'" "${config}" "${value}"
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
 			value=$(cat "${__object:?}/parameter/value")
-			printf "uci set '%s'='%s'\n" "${config}" "${value}"
+			uci_cmd "set '%s'='%s'" "${config}" "${value}"
 		fi
 		;;
 	(absent)
@@ -58,7 +74,7 @@ in
 			exit 0
 		fi
 
-		printf "uci delete '%s'\n" "${config}"
+		uci_cmd "delete '%s'" "${config}"
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index c6bb81ab..2f64355b 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -13,8 +13,7 @@ This cdist type can be used to alter configuration options in OpenWrt's UCI
 system.
 
 Options can be applied in batches if the `--transaction` parameter is used.
-It is important to ensure that the `__uci_commit` object is executed before a
-new transaction is started.
+
 
 REQUIRED PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index bed0eefb..ac74e5e4 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -18,4 +18,23 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-echo 'uci commit'
+transaction_name=${__object_id:?}
+batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
+
+test -s "${batchfile}" || exit 0
+
+cat <<'EOF'
+rollback() {
+	uci changes \
+	| sed -e 's/\..*$//' -e 's/^-//' \
+	| while read -r package
+	  do
+		  uci revert "${package}"
+	  done
+}
+
+EOF
+
+echo "uci batch <<'EOF' && uci commit || rollback"
+cat "${batchfile}"
+echo 'EOF'

From d3574b2d3e807ae436afd90aa1ed9d01b78ca6a3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 16:43:16 +0200
Subject: [PATCH 427/743] [type/__uci] Send messages when options are set to be
 altered

---
 cdist/conf/type/__uci/gencode-local | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index bba5944d..ba27dc5e 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -53,6 +53,8 @@ in
 		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
 		then
 			# "should" is a list
+			printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+
 			if test "${state_is}" != 'absent'
 			then
 				printf "uci delete '%s'\n" "${config}"
@@ -64,6 +66,8 @@ in
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
+			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
 			value=$(cat "${__object:?}/parameter/value")
 			uci_cmd "set '%s'='%s'" "${config}" "${value}"
 		fi
@@ -74,6 +78,7 @@ in
 			exit 0
 		fi
 
+		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
 		uci_cmd "delete '%s'" "${config}"
 		;;
 	(*)

From 3a3be3631010b1bf17d084c775e306f11429f5d9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 12 Jun 2020 19:53:26 +0200
Subject: [PATCH 428/743] [type/__uci_commit] Send message on commit of a
 transaction

---
 cdist/conf/type/__uci_commit/gencode-remote | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index ac74e5e4..0332d7e0 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,6 +23,8 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
+printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
+
 cat <<'EOF'
 rollback() {
 	uci changes \

From e7369a1f9957ac31830a69de8101c434666da5da Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 17 Jun 2020 13:00:24 +0200
Subject: [PATCH 429/743] [type/__uci_commit] Abort if uncommited changes are
 present on the target

---
 cdist/conf/type/__uci_commit/explorer/changes | 22 ++++++++++++++++
 cdist/conf/type/__uci_commit/gencode-remote   | 25 +++++++++++++------
 2 files changed, 39 insertions(+), 8 deletions(-)
 create mode 100644 cdist/conf/type/__uci_commit/explorer/changes

diff --git a/cdist/conf/type/__uci_commit/explorer/changes b/cdist/conf/type/__uci_commit/explorer/changes
new file mode 100644
index 00000000..2690def7
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/explorer/changes
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer outputs the uncommited UCI changes on the target.
+
+uci changes
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index 0332d7e0..dd1d839c 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,20 +23,29 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
+if test -s "${__object:?}/explorer/changes"
+then
+	echo 'Uncommited UCI changes were found on the target:'
+	cat "${__object:?}/explorer/changes"
+	echo
+	echo 'This can be caused by manual changes or due to a previous failed run.'
+	echo 'Please investigate the situation, revert or commit the changes, and try again.'
+	exit 1
+fi >&2
+
 printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
 
-cat <<'EOF'
+cat <<CODE
 rollback() {
-	uci changes \
-	| sed -e 's/\..*$//' -e 's/^-//' \
+	uci changes \\
+	| sed -e 's/\..*\$//' -e 's/^-//' \\
 	| while read -r package
 	  do
-		  uci revert "${package}"
+		  uci revert "\${package}"
 	  done
 }
 
+uci batch <<'EOF' && uci commit || rollback
+$(cat "${batchfile}")
 EOF
-
-echo "uci batch <<'EOF' && uci commit || rollback"
-cat "${batchfile}"
-echo 'EOF'
+CODE

From cc599dab15ed5c8761b3bcdfb113403fa6d4bc77 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 5 Jul 2020 10:25:36 +0200
Subject: [PATCH 430/743] [type/__uci_commit] Move uncommited changes check
 from explorer to code-remote

This is done to prevent false positives/negatives (see NOTE in code)
---
 cdist/conf/type/__uci_commit/explorer/changes | 22 -------------------
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++++++++++-----
 2 files changed, 15 insertions(+), 28 deletions(-)
 delete mode 100644 cdist/conf/type/__uci_commit/explorer/changes

diff --git a/cdist/conf/type/__uci_commit/explorer/changes b/cdist/conf/type/__uci_commit/explorer/changes
deleted file mode 100644
index 2690def7..00000000
--- a/cdist/conf/type/__uci_commit/explorer/changes
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/sh
-#
-# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-# This explorer outputs the uncommited UCI changes on the target.
-
-uci changes
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index dd1d839c..504f7c3b 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,19 +23,28 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
-if test -s "${__object:?}/explorer/changes"
+printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
+
+# NOTE: Uncommited changes are checked in code-remote instead of in an explorer
+#       because in cdist there is no interlocking between explorers and code
+#       execution.
+#       Checking for uncommited changes in an explorer leaves a time slot in
+#       which changes made are not detected by the code.
+#       Furthermore an explorer running concurrently with another transactions
+#       code-remote could lead to a false positive.
+
+cat <<CODE
+changes=\$(uci changes)
+
+if test -n "\${changes}"
 then
 	echo 'Uncommited UCI changes were found on the target:'
-	cat "${__object:?}/explorer/changes"
-	echo
+	printf '%s\n\n' "\${changes}"
 	echo 'This can be caused by manual changes or due to a previous failed run.'
 	echo 'Please investigate the situation, revert or commit the changes, and try again.'
 	exit 1
 fi >&2
 
-printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
-
-cat <<CODE
 rollback() {
 	uci changes \\
 	| sed -e 's/\..*\$//' -e 's/^-//' \\

From 3ef638a61123069953a9e24044b6c94eba0b3b6b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 20 Jul 2020 12:12:15 +0200
Subject: [PATCH 431/743] [type/__uci_commit] Fail when uci(1) reports errors

---
 cdist/conf/type/__uci_commit/gencode-remote | 37 ++++++++++++++++-----
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index 504f7c3b..e5c93966 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -45,16 +45,37 @@ then
 	exit 1
 fi >&2
 
-rollback() {
-	uci changes \\
-	| sed -e 's/\..*\$//' -e 's/^-//' \\
-	| while read -r package
-	  do
-		  uci revert "\${package}"
-	  done
+check_errors() {
+	# reads stdin and forwards non-empty lines to stderr.
+	# returns 0 if stdin is empty, else 1.
+	! grep -e . >&2
 }
 
-uci batch <<'EOF' && uci commit || rollback
+commit() {
+	uci commit
+}
+
+rollback() {
+	echo >&2
+	echo 'An error occurred when trying to commit transaction ${transaction_name}!' >&2
+
+	uci changes \\
+	| sed -e 's/^-//' -e 's/\..*\$//' \\
+	| sort -u \\
+	| while read -r _package
+	  do
+		  uci revert "\${_package}"
+		  echo "\${_package}"  # for logging
+	  done \\
+	| awk '
+	  BEGIN { printf "Reverted changes in: " }
+	  { printf "%s%s", (FNR > 1 ? ", " : ""), \$0 }
+	  END { printf "\\n" }' >&2
+
+	return 1
+}
+
+uci batch <<'EOF' 2>&1 | check_errors && commit || rollback
 $(cat "${batchfile}")
 EOF
 CODE

From 4da39681188493981ec3c55ea918787195655972 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 20:04:04 +0200
Subject: [PATCH 432/743] [type/__uci_section] Add type

---
 cdist/conf/type/__uci_section/explorer/match  |  68 ++++++++++
 .../conf/type/__uci_section/explorer/options  |  28 ++++
 cdist/conf/type/__uci_section/explorer/type   |  25 ++++
 cdist/conf/type/__uci_section/man.rst         |  76 +++++++++++
 cdist/conf/type/__uci_section/manifest        | 122 ++++++++++++++++++
 .../__uci_section/parameter/default/state     |   1 +
 .../parameter/default/transaction             |   1 +
 .../type/__uci_section/parameter/optional     |   4 +
 .../__uci_section/parameter/optional_multiple |   1 +
 9 files changed, 326 insertions(+)
 create mode 100644 cdist/conf/type/__uci_section/explorer/match
 create mode 100644 cdist/conf/type/__uci_section/explorer/options
 create mode 100644 cdist/conf/type/__uci_section/explorer/type
 create mode 100644 cdist/conf/type/__uci_section/man.rst
 create mode 100755 cdist/conf/type/__uci_section/manifest
 create mode 100644 cdist/conf/type/__uci_section/parameter/default/state
 create mode 100644 cdist/conf/type/__uci_section/parameter/default/transaction
 create mode 100644 cdist/conf/type/__uci_section/parameter/optional
 create mode 100644 cdist/conf/type/__uci_section/parameter/optional_multiple

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
new file mode 100644
index 00000000..9cf1ea3e
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -0,0 +1,68 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer the "prefix" of the section matching --match.
+
+squote_values() {
+	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
+	    -e "/='.*'$/"'!{s/=/='\''/;s/$/'\''/}'
+}
+
+RS=$(printf '\036')
+
+if ! test -e "${__object:?}/parameter/match"
+then
+	if echo "${__object_id:?}" | grep -qvE '^[^.]+\.[^.]+$'
+	then
+		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
+		exit 1
+	fi
+
+	# If no --match is given, we take the __object_id as the section identifier.
+	echo "${__object_id:?}"
+	exit 0
+fi
+
+test -s "${__object:?}/parameter/match" \
+&& test -s "${__object:?}/parameter/type" \
+|| {
+	echo 'Parameters --match and --type must be used together.' >&2
+	exit 1
+}
+
+# Find by match
+match=$(cat "${__object:?}/parameter/match")
+sect_type_filter=$(cat "${__object:?}/parameter/type")
+
+package_filter=${sect_type_filter%%.*}
+section_filter=${sect_type_filter##*.}
+regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
+
+matched_sections=$(
+	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
+	| grep -e "${regex}" \
+	| sed -e 's/\.[^.]*=.*$//')
+
+if test "$(echo "${matched_sections}" | wc -l)" -gt 1
+then
+	printf 'Found multiple matching sections:\n%s\n' "${matched_sections}" >&2
+	exit 1
+fi
+
+echo "${matched_sections}"
diff --git a/cdist/conf/type/__uci_section/explorer/options b/cdist/conf/type/__uci_section/explorer/options
new file mode 100644
index 00000000..67b4f83f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/options
@@ -0,0 +1,28 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current options of the configuration section.
+
+RS=$(printf '\036')
+
+section=$("${__type_explorer:?}/match")
+test -n "${section}" || exit 0
+
+uci -s -N -d "${RS}" show "${section}" 2>/dev/null \
+| grep -v -e "^${section}=" || true
diff --git a/cdist/conf/type/__uci_section/explorer/type b/cdist/conf/type/__uci_section/explorer/type
new file mode 100644
index 00000000..1675c2e0
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/type
@@ -0,0 +1,25 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current section type.
+
+section=$("${__type_explorer:?}/match")
+test -n "${section}" || exit 0
+
+uci -s -N get "${section}" 2>/dev/null || true
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
new file mode 100644
index 00000000..3468bfc3
--- /dev/null
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -0,0 +1,76 @@
+cdist-type__uci_section(7)
+==========================
+
+NAME
+----
+cdist-type__uci_section - Manage configuration sections in OpenWrt's
+Unified Configuration Interface (UCI)
+
+
+DESCRIPTION
+-----------
+This cdist type can be used to replace whole configuration sections in OpenWrt's
+UCI system.
+It can be thought of as syntactic sugar for `cdist-type__uci`\ (7), as this type
+will generate the required `__uci` objects to make the section contain exactly
+the options specified via ``--option``.
+
+Since many default UCI sections are unnamed, this type allows to find the
+matching section by one of its options using the ``--match`` parameter.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+match
+    Allows to find a section to "replace" through one of its parameters.
+    The value to this parameter is a ``<option>=<string>`` string.
+option
+    An option that should be present in the section.
+    This parameter can be used multiple times to specify multiple options.
+    The value to this parameter is a ``<option>=<string>`` string.
+
+    Lists can be expressed by repeatedly using the same key.
+state
+    `present` or `absent`, defaults to `present`.
+transaction
+    The name of the transaction this option belongs to.
+    The value will be forwarded to `cdist-type__uci`\ (7).
+type
+    The type of the section in the format: ``<config>.<section-type>``
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # TODO
+    __uci_section ...
+
+
+SEE ALSO
+--------
+:strong:`cdist-type__uci`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
new file mode 100755
index 00000000..e16277ba
--- /dev/null
+++ b/cdist/conf/type/__uci_section/manifest
@@ -0,0 +1,122 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+grep_line() { echo "$2" | grep -qxF "$1"; }
+unquote_lines() {
+	sed -e '/^".*"$/{s/^"//;s/"$//}' \
+	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
+}
+
+append_values() {
+	while read -r _value
+	do
+		set -- "$@" --value "${_value}"
+	done
+	unset _value
+	"$@" </dev/null
+}
+
+section=$(cat "${__object:?}/explorer/match")
+
+state_should=$(cat "${__object:?}/parameter/state")
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+case $state_should
+in
+	(present)
+		test -f "${__object:?}/parameter/type" || {
+			echo 'Parameter --type is required.' >&2
+			exit 1
+		}
+		type_is=$(cat "${__object:?}/explorer/type")
+		type_should=$(cat "${__object:?}/parameter/type")
+
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
+		if test -n "${type_is}"
+		then
+			if test "${type_is}" != "${type_should##*.}"
+			then
+				# Check if section type matches (section exists and --type provided)
+				printf 'Section type "%s" does not match --type "%s".\n' \
+					"${type_is}" "${type_should}" >&2
+				exit 1
+			fi
+			sect_type=${type_is}
+		else
+			sect_type=${type_should##*.}
+		fi
+
+		if test -z "${section}"
+		then
+			# No section exists and --match was used.
+			# So we generate a new section identifier from $__object_id.
+			case ${__object_id:?}
+			in
+				(*.*) section=${__object_id:?} ;;
+				(*) section="${type_should%%.*}.${__object_id:?}" ;;
+			esac
+		fi
+
+		# Make sure the section itself is present
+		__uci "${section}" --state present --transaction "${transaction_name}" \
+			--value "${sect_type}"
+		export require=__uci/"${section}"
+
+		# Delete options not in "should"
+		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		| while read -r optname
+		  do
+			  if ! grep_line "${optname}" "${optnames_should}"
+			  then
+				  __uci "${section}.${optname}" --state absent \
+					  --transaction "${transaction_name}" </dev/null
+			  fi
+		  done
+
+		# Set "should" options
+		echo "${optnames_should}" \
+		| while read -r optname
+		  do
+			  test -n "${optname}" || continue  # ignore empty lines
+
+			  grep "^${optname}=" <"${__object:?}/parameter/option" \
+			  | sed -e 's/^.*=//' \
+			  | unquote_lines \
+			  | append_values \
+				    __uci "${section}.${optname}" --state present \
+					    --transaction "${transaction_name}"
+		  done
+		;;
+	(absent)
+		# if explorer found no section there is nothing to delete
+		test -n "${section}" || exit 0
+
+		__uci "${section}" --state absent --transaction "${transaction_name}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__uci_section/parameter/default/state b/cdist/conf/type/__uci_section/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__uci_section/parameter/default/transaction b/cdist/conf/type/__uci_section/parameter/default/transaction
new file mode 100644
index 00000000..4ad96d51
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/default/transaction
@@ -0,0 +1 @@
+default
diff --git a/cdist/conf/type/__uci_section/parameter/optional b/cdist/conf/type/__uci_section/parameter/optional
new file mode 100644
index 00000000..8b956bae
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/optional
@@ -0,0 +1,4 @@
+match
+state
+transaction
+type
diff --git a/cdist/conf/type/__uci_section/parameter/optional_multiple b/cdist/conf/type/__uci_section/parameter/optional_multiple
new file mode 100644
index 00000000..01925a15
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/optional_multiple
@@ -0,0 +1 @@
+option

From 179815b5e93bd51b6757d92d14470808c5bee746 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 21 Jun 2020 18:22:54 +0200
Subject: [PATCH 433/743] [type/__uci_section] Ignore SC2015 error (notabug)

---
 cdist/conf/type/__uci_section/explorer/match | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index 9cf1ea3e..a3832bff 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -39,8 +39,8 @@ then
 	exit 0
 fi
 
-test -s "${__object:?}/parameter/match" \
-&& test -s "${__object:?}/parameter/type" \
+# shellcheck disable=SC2015
+test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type" \
 || {
 	echo 'Parameters --match and --type must be used together.' >&2
 	exit 1

From d453d964e1e0827dad217e1dd424dbaf82e434fc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 20 Jul 2020 11:21:06 +0200
Subject: [PATCH 434/743] [type/__uci_section] Fix in section matching

---
 cdist/conf/type/__uci_section/explorer/match | 23 +++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index a3832bff..a5619d4e 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -17,7 +17,8 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-# This explorer the "prefix" of the section matching --match.
+# This explorer determines the "prefix" of the --type section matching --match
+# if set, or __object_id otherwise.
 
 squote_values() {
 	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
@@ -28,7 +29,7 @@ RS=$(printf '\036')
 
 if ! test -e "${__object:?}/parameter/match"
 then
-	if echo "${__object_id:?}" | grep -qvE '^[^.]+\.[^.]+$'
+	if ! echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$'
 	then
 		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
 		exit 1
@@ -47,16 +48,28 @@ test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type
 }
 
 # Find by match
-match=$(cat "${__object:?}/parameter/match")
-sect_type_filter=$(cat "${__object:?}/parameter/type")
+# NOTE: Apart from section types all values are printed in single quotes by UCI.
+match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
 
+if ! grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' "${__object:?}/parameter/type"
+then
+	echo 'Section types are a package name and section type separated by a "." (period).' >&2
+	exit 1
+fi
+
+sect_type_filter=$(cat "${__object:?}/parameter/type")
 package_filter=${sect_type_filter%%.*}
-section_filter=${sect_type_filter##*.}
+section_filter=${sect_type_filter#*.}
+# FIXME: Does not work with named sections
 regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
 
 matched_sections=$(
 	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
 	| grep -e "${regex}" \
+	| while read -r _line
+	  do
+		  if test "${_line#*=}" = "${match#*=}"; then echo "${_line}"; fi
+	  done \
 	| sed -e 's/\.[^.]*=.*$//')
 
 if test "$(echo "${matched_sections}" | wc -l)" -gt 1

From f782a5a370a956995f444a677ecffe7f1564764d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 23 Oct 2020 11:45:37 +0200
Subject: [PATCH 435/743] [type/__uci] Refactor to do proper quoting of UCI
 commands

---
 cdist/conf/type/__uci/gencode-local | 40 ++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index ba27dc5e..44bf9c4f 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -18,15 +18,37 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
+in_list() {
+	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
+}
+
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
 
 uci_cmd() {
-	printf 'printf "%s\n"' "$1"
-	shift
-	printf " '%s'" "$@"
-	printf " >>'%s'\n" "${tmpfile}"
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	# Will output a shell command to be executed locally to add the given
+	# command to $tmpfile.
+	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
+	printf ' >>%s\n' "$(quote "${tmpfile}")"
 }
 
+
 config=${__object_id:?}
 
 state_is=$(cat "${__object:?}/explorer/state")
@@ -57,19 +79,19 @@ in
 
 			if test "${state_is}" != 'absent'
 			then
-				printf "uci delete '%s'\n" "${config}"
+				uci_cmd delete "${config}"
 			fi
 
 			while read -r value
 			do
-				uci_cmd "add_list '%s'='%s'" "${config}" "${value}"
+				uci_cmd add_list "${config}"="${value}"
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
 			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
 
 			value=$(cat "${__object:?}/parameter/value")
-			uci_cmd "set '%s'='%s'" "${config}" "${value}"
+			uci_cmd set "${config}"="${value}"
 		fi
 		;;
 	(absent)
@@ -79,7 +101,7 @@ in
 		fi
 
 		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
-		uci_cmd "delete '%s'" "${config}"
+		uci_cmd delete "${config}"
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2

From 3a6b0851459f08ad56f1c6a138b168fedfba993e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 23 Oct 2020 22:03:25 +0200
Subject: [PATCH 436/743] [type/__uci] Check __object_id for syntax errors

---
 cdist/conf/type/__uci/gencode-local | 45 +++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index 44bf9c4f..af91a698 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -48,8 +48,53 @@ uci_cmd() {
 	printf ' >>%s\n' "$(quote "${tmpfile}")"
 }
 
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+uci_validate_tuple() (
+	tok=${1:?}
+	case $tok
+	in
+		(*.*.*)
+			# check option
+			option=${tok##*.}
+			uci_validate_name "${option}" || {
+				printf 'Invalid option: %s\n' "${option}" >&2
+				return 1
+			}
+			tok=${tok%.*}
+			;;
+		(*.*)
+			# no option (section definition)
+			;;
+		(*)
+			printf 'Invalid tuple: %s\n' "$1" >&2
+			return 1
+			;;
+	esac
+
+	case ${tok#*.}
+	in
+		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
+		(*)  section=${tok#*.} ;;
+	esac
+	uci_validate_name "${section}" || {
+		printf 'Invalid section: %s\n' "${1#*.}" >&2
+		return 1
+	}
+
+	config=${tok%%.*}
+	uci_validate_name "${config}" || {
+		printf 'Invalid config: %s\n' "${config}" >&2
+		return 1
+	}
+)
+
 
 config=${__object_id:?}
+uci_validate_tuple "${config}"
 
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")

From c37253b852477a0a22568643bad461d20b58706f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 22 Oct 2020 20:32:48 +0200
Subject: [PATCH 437/743] [type/__uci_section] Check __object_id for syntax
 errors

---
 cdist/conf/type/__uci_section/manifest | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index e16277ba..1a59b970 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -33,6 +33,32 @@ append_values() {
 	"$@" </dev/null
 }
 
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+## Check section name and error if invalid!
+case ${__object_id:?}
+in
+	(*.*)
+		uci_validate_name "${__object_id%%.*}" || {
+			printf 'Invalid package name: %s\n' "${__object_id%%.*}" >&2
+			exit 1
+		}
+		uci_validate_name "${__object_id#*.}" || {
+			printf 'Invalid section name: %s\n' "${__object_id#*.}" >&2
+			exit 1
+		}
+		;;
+	(*)
+		uci_validate_name "${__object_id:?}" || {
+			printf 'Invalid section name: %s\n' "${__object_id:?}" >&2
+			exit 1
+		}
+		;;
+esac
+
 section=$(cat "${__object:?}/explorer/match")
 
 state_should=$(cat "${__object:?}/parameter/state")

From fe26c119b52c3822f009fe9aeb1b52b60139e33a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 10:01:49 +0200
Subject: [PATCH 438/743] [type/__uci*] Update man pages

---
 cdist/conf/type/__uci/man.rst         | 15 ++++----
 cdist/conf/type/__uci_commit/man.rst  | 12 ++++---
 cdist/conf/type/__uci_section/man.rst | 51 ++++++++++++++++++++-------
 3 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 2f64355b..cf184f9d 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -3,23 +3,22 @@ cdist-type__uci(7)
 
 NAME
 ----
-cdist-type__uci - Manage configuration values in OpenWrt's
-Unified Configuration Interface (UCI)
+cdist-type__uci - Manage configuration values in UCI
 
 
 DESCRIPTION
 -----------
-This cdist type can be used to alter configuration options in OpenWrt's UCI
-system.
+This cdist type can be used to alter configuration options in OpenWrt's
+Unified Configuration Interface (UCI) system.
 
-Options can be applied in batches if the `--transaction` parameter is used.
+Options can be applied in batches if the ``--transaction`` parameter is used.
 
 
 REQUIRED PARAMETERS
 -------------------
 value
     The value to be set. Can be used multiple times.
-    This parameter is allowed to be omitted if `--state` is `absent`.
+    This parameter is ignored if ``--state`` is ``absent``.
 
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
@@ -28,10 +27,10 @@ value
 OPTIONAL PARAMETERS
 -------------------
 state
-    `present` or `absent`, defaults to `present`.
+    ``present`` or ``absent``, defaults to ``present``.
 transaction
     The name of the transaction this option belongs to.
-    If none is given: "default" is used.
+    If none is given: ``default`` is used.
 
 
 BOOLEAN PARAMETERS
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
index c55d06e5..ea6611eb 100644
--- a/cdist/conf/type/__uci_commit/man.rst
+++ b/cdist/conf/type/__uci_commit/man.rst
@@ -3,14 +3,16 @@ cdist-type__uci_commit(7)
 
 NAME
 ----
-cdist-type__uci_commit - Commit a UCI transaction.
+cdist-type__uci_commit - Commit UCI transactions
 
 
 DESCRIPTION
 -----------
-This type executes the "uci commit" command on the target.
-It is usually not required to use this type. Use the `--transaction` parameter
-of `cdist-type__uci`\ (7) instead.
+This type executes the ``uci commit`` command on the target with the commands
+queued in a transaction.
+It is usually not required to use this type. Use the ``--transaction`` parameter
+of :strong:`cdist-type__uci`\ (7) and :strong:`cdist-type__uci_section`\ (7)
+instead.
 
 
 REQUIRED PARAMETERS
@@ -42,7 +44,7 @@ EXAMPLES
 
 SEE ALSO
 --------
-:strong:`cdist-type__uci`\ (7)
+:strong:`cdist-type__uci`\ (7), :strong:`cdist-type__uci_section`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 3468bfc3..86729316 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -3,21 +3,23 @@ cdist-type__uci_section(7)
 
 NAME
 ----
-cdist-type__uci_section - Manage configuration sections in OpenWrt's
-Unified Configuration Interface (UCI)
+cdist-type__uci_section - Manage configuration sections in UCI
 
 
 DESCRIPTION
 -----------
 This cdist type can be used to replace whole configuration sections in OpenWrt's
-UCI system.
-It can be thought of as syntactic sugar for `cdist-type__uci`\ (7), as this type
-will generate the required `__uci` objects to make the section contain exactly
-the options specified via ``--option``.
+Unified Configuration Interface (UCI) system.
+It can be thought of as syntactic sugar for :strong:`cdist-type__uci`\ (7),
+as this type will generate the required `__uci` objects to make the section
+contain exactly the options specified using ``--option``.
 
 Since many default UCI sections are unnamed, this type allows to find the
 matching section by one of its options using the ``--match`` parameter.
 
+**NOTE:** Options already present on the target and not listed in ``--option``
+or ``--list`` will be deleted.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -28,18 +30,18 @@ OPTIONAL PARAMETERS
 -------------------
 match
     Allows to find a section to "replace" through one of its parameters.
-    The value to this parameter is a ``<option>=<string>`` string.
+    The value to this parameter is a ``<option>=<value>`` string.
 option
     An option that should be present in the section.
     This parameter can be used multiple times to specify multiple options.
-    The value to this parameter is a ``<option>=<string>`` string.
+    The value to this parameter is a ``<option>=<value>`` string.
 
     Lists can be expressed by repeatedly using the same key.
 state
-    `present` or `absent`, defaults to `present`.
+    ``present`` or ``absent``, defaults to ``present``.
 transaction
     The name of the transaction this option belongs to.
-    The value will be forwarded to `cdist-type__uci`\ (7).
+    The value will be forwarded to :strong:`cdist-type__uci`\ (7).
 type
     The type of the section in the format: ``<config>.<section-type>``
 
@@ -54,13 +56,36 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # TODO
-    __uci_section ...
+    # Configure the dropbear daemon
+    __uci_section dropbear --type dropbear.dropbear --transaction sshd \
+        --match Port=22 --option Port=22 \
+        --option PasswordAuth=off \
+        --option RootPasswordAuth=off
+
+    # Block SSH access from the guest network
+    __uci_section firewall.block_ssh_from_guest --type firewall.rule \
+        --transaction fwrules \
+        --option name='Block-SSH-Access-from-Guest' \
+        --option src='guest' \
+        --option proto='tcp' \
+        --option dest_port='22' \
+        --option target='REJECT'
+
+    # Configure a Wi-Fi access point
+    __uci_section wireless.default_radio0 --type wireless.wifi-iface \
+        --transaction wifi \
+        --option device='radio0' \
+        --option mode='ap' \
+        --option network='wlan' \
+        --option ssid='mywifi' \
+        --option encryption="psk2' \
+        --option key='hunter2'
 
 
 SEE ALSO
 --------
-:strong:`cdist-type__uci`\ (7)
+- https://openwrt.org/docs/guide-user/base-system/uci
+- :strong:`cdist-type__uci`\ (7)
 
 
 AUTHORS

From 0840afce03c50824278d64cd204b86e79077b48e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 12:57:40 +0200
Subject: [PATCH 439/743] [type/__uci] Add --type parameter

---
 cdist/conf/type/__uci/gencode-local      |  54 ++++++++----
 cdist/conf/type/__uci/gencode-remote     | 101 +++++++++++++++++++++++
 cdist/conf/type/__uci/man.rst            |   9 +-
 cdist/conf/type/__uci/parameter/optional |   1 +
 4 files changed, 146 insertions(+), 19 deletions(-)
 create mode 100644 cdist/conf/type/__uci/gencode-remote

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index af91a698..34482a01 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -117,27 +117,45 @@ in
 			exit 0
 		fi
 
-		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
-		then
-			# "should" is a list
-			printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+		# Determine type
+		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
+		case ${type}
+		in
+			(option|list) ;;
+			('')
+				# Guess type by the number of values
+				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
+					 && type=list \
+					 || type=option
+				;;
+			(*)
+				printf 'Invalid --type: %s\n' "${type}" >&2
+				exit 1
+				;;
+		esac
 
-			if test "${state_is}" != 'absent'
-			then
-				uci_cmd delete "${config}"
-			fi
+		case ${type}
+		in
+			(list)
+				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
 
-			while read -r value
-			do
-				uci_cmd add_list "${config}"="${value}"
-			done <"${__object:?}/parameter/value"
-		else
-			# "should" is a scalar
-			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+				if test "${state_is}" != 'absent'
+				then
+					uci_cmd delete "${config}"
+				fi
 
-			value=$(cat "${__object:?}/parameter/value")
-			uci_cmd set "${config}"="${value}"
-		fi
+				while read -r value
+				do
+					uci_cmd add_list "${config}"="${value}"
+				done <"${__object:?}/parameter/value"
+				;;
+			(option)
+				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
+				value=$(cat "${__object:?}/parameter/value")
+				uci_cmd set "${config}"="${value}"
+				;;
+		esac
 		;;
 	(absent)
 		if in_list "${state_is}" 'absent'
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
new file mode 100644
index 00000000..348d15a1
--- /dev/null
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -0,0 +1,101 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+config=${__object_id:?}
+uci_validate_tuple "${config}"
+
+
+case ${state_should}
+in
+	(present)
+		if in_list "${state_is}" 'present' 'rearranged'
+		then
+			# NOTE: order is ignored so rearranged is also fine.
+			exit 0
+		fi
+
+		# Determine type
+		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
+		case ${type}
+		in
+			(option|list) ;;
+			('')
+				# Guess type by the number of values
+				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
+					 && type=list \
+					 || type=option
+				;;
+			(*)
+				printf 'Invalid --type: %s\n' "${type}" >&2
+				exit 1
+				;;
+		esac
+
+		case ${type}
+		in
+			(list)
+				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+
+				if test "${state_is}" != 'absent'
+				then
+					uci_cmd delete "${config}"
+				fi
+
+				while read -r value
+				do
+					uci_cmd add_list "${config}"="${value}"
+				done <"${__object:?}/parameter/value"
+				;;
+			(option)
+				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
+				value=$(cat "${__object:?}/parameter/value")
+				uci_cmd set "${config}"="${value}"
+				;;
+		esac
+		;;
+	(absent)
+		if in_list "${state_is}" 'absent'
+		then
+			exit 0
+		fi
+
+		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
+		uci_cmd delete "${config}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index cf184f9d..9d33d618 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -31,6 +31,10 @@ state
 transaction
     The name of the transaction this option belongs to.
     If none is given: ``default`` is used.
+type
+    If the type should generate an option or a list.
+    One of: ``option`` or ``list``.
+    Defaults to auto-detect based on the number of ``--value`` parameters.
 
 
 BOOLEAN PARAMETERS
@@ -46,10 +50,13 @@ EXAMPLES
     # Set the system hostname
     __uci system.@system[0].hostname --value 'OpenWrt'
 
+    # Set DHCP option 252: tell DHCP clients to not ask for proxy information.
+    __uci dhcp.lan.dhcp_option --type list --value '252,"\n"'
+
     # Enable NTP and NTPd (in one transaction)
     __uci system.ntp.enabled --value 1 --transaction ntp
     __uci system.ntp.enable_server --value 1 --transaction ntp
-    __uci system.ntp.server --transaction ntp \
+    __uci system.ntp.server --type list --transaction ntp \
         --value '0.openwrt.pool.ntp.org' \
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
index ddbbba16..2703b3df 100644
--- a/cdist/conf/type/__uci/parameter/optional
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -1,2 +1,3 @@
 state
 transaction
+type

From 49e867fab4c1bcb6549079e735abec2149e42cf6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 15:59:25 +0200
Subject: [PATCH 440/743] [type/__uci_section] Add more parameter checks

---
 cdist/conf/type/__uci_section/manifest | 42 +++++++++++++++++++-------
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 1a59b970..b5fa4a3b 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -19,11 +19,15 @@
 #
 
 grep_line() { echo "$2" | grep -qxF "$1"; }
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
+}
+validate_options() { grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"; }
 unquote_lines() {
 	sed -e '/^".*"$/{s/^"//;s/"$//}' \
 	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
 }
-
 append_values() {
 	while read -r _value
 	do
@@ -32,12 +36,23 @@ append_values() {
 	unset _value
 	"$@" </dev/null
 }
-
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+print_errors() {
+	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
+		BEGIN {
+			if (getline) {
+				print prefix
+				print
+				rc = 1
+			}
+		}
+		{ print }
+		END {
+			if (rc && suffix) print suffix
+			exit rc
+		}' >&2
 }
 
+
 ## Check section name and error if invalid!
 case ${__object_id:?}
 in
@@ -74,12 +89,6 @@ in
 		type_is=$(cat "${__object:?}/explorer/type")
 		type_should=$(cat "${__object:?}/parameter/type")
 
-		if test -f "${__object:?}/parameter/option"
-		then
-			optnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
-		fi
-
 		if test -n "${type_is}"
 		then
 			if test "${type_is}" != "${type_should##*.}"
@@ -105,6 +114,17 @@ in
 			esac
 		fi
 
+		# Check options for syntax errors
+		validate_options "${__object:?}/parameter/object" \
+		| print_errors 'Found erroneous options in arguments:'
+
+		# Collect option names
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
 		# Make sure the section itself is present
 		__uci "${section}" --state present --transaction "${transaction_name}" \
 			--value "${sect_type}"

From b99ca3cbdf7c761db05598e31e9788e5e4e81cc6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 16:01:02 +0200
Subject: [PATCH 441/743] [type/__uci_section] Split up --option and --list

---
 cdist/conf/type/__uci_section/man.rst         | 19 +++++++-
 cdist/conf/type/__uci_section/manifest        | 48 ++++++++++++-------
 .../__uci_section/parameter/optional_multiple |  1 +
 3 files changed, 50 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 86729316..b8829433 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -28,15 +28,21 @@ None.
 
 OPTIONAL PARAMETERS
 -------------------
+list
+    An option that is part of a list and should be present in the section (as
+    part of a list).  Lists with multiple options can be expressed by using the
+    same ``<option>`` repeatedly.
+
+    The value to this parameter is a ``<option>=<value>`` string.
 match
     Allows to find a section to "replace" through one of its parameters.
+
     The value to this parameter is a ``<option>=<value>`` string.
 option
     An option that should be present in the section.
     This parameter can be used multiple times to specify multiple options.
-    The value to this parameter is a ``<option>=<value>`` string.
 
-    Lists can be expressed by repeatedly using the same key.
+    The value to this parameter is a ``<option>=<value>`` string.
 state
     ``present`` or ``absent``, defaults to ``present``.
 transaction
@@ -62,6 +68,15 @@ EXAMPLES
         --option PasswordAuth=off \
         --option RootPasswordAuth=off
 
+    # Define a firewall zone comprised of lan and wlan networks
+    __uci_section firewall.internal --type firewall.zone --transaction fw \
+        --option name='internal' \
+        --list network='lan' \
+        --list network='wlan' \
+        --option input='ACCEPT' \
+        --option output='ACCEPT' \
+        --option forward='ACCEPT'
+
     # Block SSH access from the guest network
     __uci_section firewall.block_ssh_from_guest --type firewall.rule \
         --transaction fwrules \
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index b5fa4a3b..a7865315 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,7 +18,14 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-grep_line() { echo "$2" | grep -qxF "$1"; }
+grep_line() { { shift; printf '%s\n' "$@"; } | grep -qxF "$1"; }
+prefix_lines() {
+	while test $# -gt 0
+	do
+		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
+		shift; shift
+	done
+}
 uci_validate_name() {
 	# like util.c uci_validate_name()
 	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
@@ -115,14 +122,25 @@ in
 		fi
 
 		# Check options for syntax errors
-		validate_options "${__object:?}/parameter/object" \
+		validate_options "${__object:?}/parameter/list" "${__object:?}/parameter/object" \
 		| print_errors 'Found erroneous options in arguments:'
 
 		# Collect option names
+		if test -f "${__object:?}/parameter/list"
+		then
+			listnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
+		fi
+
 		if test -f "${__object:?}/parameter/option"
 		then
-			optnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+			optnames_should=$(sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort)
+
+			echo "${optnames_should}" \
+			| uniq -d \
+			| print_errors \
+				  'Found duplicate --options:' \
+				  "$(printf '\nUse --list for lists, instead.')"
 		fi
 
 		# Make sure the section itself is present
@@ -130,29 +148,27 @@ in
 			--value "${sect_type}"
 		export require=__uci/"${section}"
 
-		# Delete options not in "should"
+		# Delete options/lists not in "should"
 		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
 		| while read -r optname
 		  do
-			  if ! grep_line "${optname}" "${optnames_should}"
-			  then
-				  __uci "${section}.${optname}" --state absent \
-					  --transaction "${transaction_name}" </dev/null
-			  fi
+			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
+				  __uci "${section}.${optname}" --state absent --transaction "${transaction_name}"
+			  } </dev/null
 		  done
 
 		# Set "should" options
-		echo "${optnames_should}" \
-		| while read -r optname
+		prefix_lines option "${optnames_should}" list "${listnames_should}" \
+		| while read -r _type _optname
 		  do
-			  test -n "${optname}" || continue  # ignore empty lines
+			  test -n "${_type}${_optname}" || continue  # ignore empty lines
 
-			  grep "^${optname}=" <"${__object:?}/parameter/option" \
+			  grep "^${_optname}=" <"${__object:?}/parameter/${_type}" \
 			  | sed -e 's/^.*=//' \
 			  | unquote_lines \
 			  | append_values \
-				    __uci "${section}.${optname}" --state present \
-					    --transaction "${transaction_name}"
+				    __uci "${section}.${_optname}" --state present \
+					    --type "${_type}" --transaction "${transaction_name}"
 		  done
 		;;
 	(absent)
diff --git a/cdist/conf/type/__uci_section/parameter/optional_multiple b/cdist/conf/type/__uci_section/parameter/optional_multiple
index 01925a15..98af35e6 100644
--- a/cdist/conf/type/__uci_section/parameter/optional_multiple
+++ b/cdist/conf/type/__uci_section/parameter/optional_multiple
@@ -1 +1,2 @@
+list
 option

From 8728817af6ae1fa801a39456df3f84d420f87744 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 21:11:50 +0200
Subject: [PATCH 442/743] [type/__uci] Unquote UCI reported values

Without unquoting values printed in single quotes by UCI would always lead to
the state explorer reporting "different".
---
 cdist/conf/type/__uci/explorer/state | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
index 6fb4b173..d7363dbf 100644
--- a/cdist/conf/type/__uci/explorer/state
+++ b/cdist/conf/type/__uci/explorer/state
@@ -51,21 +51,34 @@ fi
 # strip off trailing newline
 printf '%s' "${values_is}" \
 | awk '
+function unquote(s) {
+	# simplified dequoting of single quoted strings
+	if (s ~ /^'\''.*'\''$/) {
+		s = substr(s, 2, length(s) - 2)
+		sub(/'"'\\\\''"'/, "'\''", s)
+	}
+	return s
+}
+
 BEGIN {
 	state = "present"  # assume all is fine
 }
 NR == FNR {
 	# memoize "should" state
 	should[FNR] = $0
+	should_count++
 
 	# go to next line (important!)
 	next
 }
 
 # compare "is" state
+
+{ $0 = unquote($0) }
+
 $0 == should[FNR] { next }
 
-FNR > length(should) {
+FNR > should_count {
 	# there are more "is" records than "should" -> definitely different
 	state = "different"
 	exit
@@ -87,7 +100,7 @@ FNR > length(should) {
 }
 
 END {
-	if (FNR < length(should)) {
+	if (FNR < should_count) {
 		# "is" was shorter than "should" -> different
 		state = "different"
 	}

From 4aebb1f12756593408d6f012adb923e4c9b81e1e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 21:18:22 +0200
Subject: [PATCH 443/743] [type/__uci*] Update man.rst regarding quoting
 requirements

---
 cdist/conf/type/__uci/man.rst         | 2 ++
 cdist/conf/type/__uci_section/man.rst | 8 ++++++++
 2 files changed, 10 insertions(+)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 9d33d618..c7f5f4f0 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -23,6 +23,8 @@ value
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
 
+    Values do not need special quoting for UCI. The only requirement is that the
+    value is passed to the type as a single shell argument.
 
 OPTIONAL PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index b8829433..753d00b3 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -34,6 +34,10 @@ list
     same ``<option>`` repeatedly.
 
     The value to this parameter is a ``<option>=<value>`` string.
+
+    ``<value>`` does not need special quoting for UCI.
+    The only requirement is that the value is passed to the type as a single
+    shell argument.
 match
     Allows to find a section to "replace" through one of its parameters.
 
@@ -43,6 +47,10 @@ option
     This parameter can be used multiple times to specify multiple options.
 
     The value to this parameter is a ``<option>=<value>`` string.
+
+    ``<value>`` does not need special quoting for UCI.
+    The only requirement is that the value is passed to the type as a single
+    shell argument.
 state
     ``present`` or ``absent``, defaults to ``present``.
 transaction

From 63d41a1053bd8b03ddda130d7129d233f97ac671 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 21:10:27 +0100
Subject: [PATCH 444/743] [type/__uci_section] Improve --match support with
 existing named sections

Use section if named section exists without --match option (e.g. empty section).
---
 cdist/conf/type/__uci_section/explorer/match | 80 +++++++++++++-------
 1 file changed, 51 insertions(+), 29 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index a5619d4e..0768e404 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -20,62 +20,84 @@
 # This explorer determines the "prefix" of the --type section matching --match
 # if set, or __object_id otherwise.
 
+RS=$(printf '\036')
+NL=$(printf '\n '); NL=${NL% }
+
 squote_values() {
 	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
 	    -e "/='.*'$/"'!{s/=/='\''/;s/$/'\''/}'
 }
+count_lines() (
+	IFS=${NL?}
+	# shellcheck disable=SC2048,SC2086
+	set -f -- $*; echo $#
+)
 
-RS=$(printf '\036')
-
-if ! test -e "${__object:?}/parameter/match"
-then
-	if ! echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$'
-	then
-		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
-		exit 1
-	fi
+echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' || {
+	echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
+	exit 1
+}
 
+test -s "${__object:?}/parameter/match" || {
 	# If no --match is given, we take the __object_id as the section identifier.
 	echo "${__object_id:?}"
 	exit 0
-fi
-
-# shellcheck disable=SC2015
-test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type" \
-|| {
+}
+test -s "${__object:?}/parameter/type" || {
 	echo 'Parameters --match and --type must be used together.' >&2
 	exit 1
 }
 
-# Find by match
-# NOTE: Apart from section types all values are printed in single quotes by UCI.
-match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
-
-if ! grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' "${__object:?}/parameter/type"
-then
+sect_type_param=$(cat "${__object:?}/parameter/type")
+expr "${sect_type_param}" : '[^.]\{1,\}\.[^.]\{1,\}$' >/dev/null 2>&1 || {
 	echo 'Section types are a package name and section type separated by a "." (period).' >&2
 	exit 1
+}
+package_filter=${sect_type_param%%.*}
+section_filter=${sect_type_param#*.}
+
+# Find by --match
+# NOTE: Apart from section types all values are printed in single quotes by uci show.
+match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
+
+if uci -s -N get "${__object_id:?}" >/dev/null 2>&1
+then
+	# Named section exists: ensure if --match applies to it
+	# if the "matched" option does not exist (e.g. empty section) we use the
+	# section unconditionally.
+	if match_value_is=$(uci -s -N get "${__object_id:?}.${match%%=*}" 2>/dev/null)
+	then
+		match_value_should=$(expr "${match}" : ".*='\\(.*\\)'$")
+
+		test "${match_value_is}" = "${match_value_should}" || {
+			printf 'Named section "%s" does not match --match "%s"\n' \
+				   "${__object_id:?}" "${match}" >&2
+			exit 1
+		}
+	fi
+
+	echo "${__object_id:?}"
+	exit 0
 fi
 
-sect_type_filter=$(cat "${__object:?}/parameter/type")
-package_filter=${sect_type_filter%%.*}
-section_filter=${sect_type_filter#*.}
-# FIXME: Does not work with named sections
-regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
+# No correctly named section exists already: find one to which --match applies
+regex="^${package_filter}\\.@${section_filter}\\[[0-9]\\{1,\\}\\]\\.${match%%=*}="
 
 matched_sections=$(
 	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
 	| grep -e "${regex}" \
 	| while read -r _line
 	  do
-		  if test "${_line#*=}" = "${match#*=}"; then echo "${_line}"; fi
+		  if test "${_line#*=}" = "${match#*=}"
+		  then
+			  echo "${_line}"
+		  fi
 	  done \
 	| sed -e 's/\.[^.]*=.*$//')
 
-if test "$(echo "${matched_sections}" | wc -l)" -gt 1
-then
+test "$(count_lines "${matched_sections}")" -le 1 || {
 	printf 'Found multiple matching sections:\n%s\n' "${matched_sections}" >&2
 	exit 1
-fi
+}
 
 echo "${matched_sections}"

From 7b301195040c4861dbd15c5f2514c4e9eeadbe61 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Oct 2020 18:52:43 +0100
Subject: [PATCH 445/743] [type/__uci] Externalise functions to separate file

---
 cdist/conf/type/__uci/files/functions.sh | 75 +++++++++++++++++++++
 cdist/conf/type/__uci/gencode-local      | 85 ++----------------------
 cdist/conf/type/__uci/manifest           |  4 +-
 3 files changed, 85 insertions(+), 79 deletions(-)
 create mode 100644 cdist/conf/type/__uci/files/functions.sh

diff --git a/cdist/conf/type/__uci/files/functions.sh b/cdist/conf/type/__uci/files/functions.sh
new file mode 100644
index 00000000..4ddb82e0
--- /dev/null
+++ b/cdist/conf/type/__uci/files/functions.sh
@@ -0,0 +1,75 @@
+# -*- mode: sh; indent-tabs-mode: t -*-
+
+in_list() {
+	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
+}
+
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
+
+uci_cmd() {
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	# Will output a shell command to be executed locally to add the given
+	# command to $tmpfile.
+	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
+	printf ' >>%s\n' "$(quote "${tmpfile}")"
+}
+
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+uci_validate_tuple() (
+	tok=${1:?}
+	case $tok
+	in
+		(*.*.*)
+			# check option
+			option=${tok##*.}
+			uci_validate_name "${option}" || {
+				printf 'Invalid option: %s\n' "${option}" >&2
+				return 1
+			}
+			tok=${tok%.*}
+			;;
+		(*.*)
+			# no option (section definition)
+			;;
+		(*)
+			printf 'Invalid tuple: %s\n' "$1" >&2
+			return 1
+			;;
+	esac
+
+	case ${tok#*.}
+	in
+		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
+		(*)  section=${tok#*.} ;;
+	esac
+	uci_validate_name "${section}" || {
+		printf 'Invalid section: %s\n' "${1#*.}" >&2
+		return 1
+	}
+
+	config=${tok%%.*}
+	uci_validate_name "${config}" || {
+		printf 'Invalid config: %s\n' "${config}" >&2
+		return 1
+	}
+)
diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index 34482a01..b7664e85 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -18,84 +18,6 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() {
-	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
-}
-
-quote() {
-	for _arg
-	do
-		shift
-		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
-		then
-			# needs quoting
-			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
-		else
-			set -- "$@" "${_arg}"
-		fi
-	done
-	unset _arg
-
-	# NOTE: Use printf because POSIX echo interprets escape sequences
-	printf '%s' "$*"
-}
-
-uci_cmd() {
-	# Usage: uci_cmd [UCI ARGUMENTS]...
-	# Will output a shell command to be executed locally to add the given
-	# command to $tmpfile.
-	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
-	printf ' >>%s\n' "$(quote "${tmpfile}")"
-}
-
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
-}
-
-uci_validate_tuple() (
-	tok=${1:?}
-	case $tok
-	in
-		(*.*.*)
-			# check option
-			option=${tok##*.}
-			uci_validate_name "${option}" || {
-				printf 'Invalid option: %s\n' "${option}" >&2
-				return 1
-			}
-			tok=${tok%.*}
-			;;
-		(*.*)
-			# no option (section definition)
-			;;
-		(*)
-			printf 'Invalid tuple: %s\n' "$1" >&2
-			return 1
-			;;
-	esac
-
-	case ${tok#*.}
-	in
-		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
-		(*)  section=${tok#*.} ;;
-	esac
-	uci_validate_name "${section}" || {
-		printf 'Invalid section: %s\n' "${1#*.}" >&2
-		return 1
-	}
-
-	config=${tok%%.*}
-	uci_validate_name "${config}" || {
-		printf 'Invalid config: %s\n' "${config}" >&2
-		return 1
-	}
-)
-
-
-config=${__object_id:?}
-uci_validate_tuple "${config}"
-
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
@@ -107,6 +29,13 @@ mkdir -p "${tmpdir}"
 
 tmpfile="${tmpdir}/${transaction_name}.txt"
 
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+
+config=${__object_id:?}
+uci_validate_tuple "${config}"
+
 
 case ${state_should}
 in
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index e56462e5..d0c8239e 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,7 +18,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; } }
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
 
 os=$(cat "${__global:?}/explorer/os")
 

From a6c37095f1eea72acbebebfa88bdf84f65f61f51 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Oct 2020 18:50:34 +0100
Subject: [PATCH 446/743] [type/__uci_section] Externalise functions to
 separate file

---
 .../type/__uci_section/files/functions.sh     | 52 +++++++++++++++++++
 cdist/conf/type/__uci_section/manifest        | 42 +--------------
 2 files changed, 54 insertions(+), 40 deletions(-)
 create mode 100644 cdist/conf/type/__uci_section/files/functions.sh

diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
new file mode 100644
index 00000000..9f510958
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -0,0 +1,52 @@
+# -*- mode: sh; indent-tabs-mode: t -*-
+
+append_values() {
+	while read -r _value
+	do
+		set -- "$@" --value "${_value}"
+	done
+	unset _value
+	"$@" </dev/null
+}
+
+grep_line() {
+	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
+}
+
+prefix_lines() {
+	while test $# -gt 0
+	do
+		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
+		shift; shift
+	done
+}
+
+print_errors() {
+	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
+		BEGIN {
+			if (getline) {
+				print prefix
+				print
+				rc = 1
+			}
+		}
+		{ print }
+		END {
+			if (rc && suffix) print suffix
+			exit rc
+		}' >&2
+}
+
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
+}
+
+unquote_lines() {
+	sed -e '/^".*"$/{s/^"//;s/"$//}' \
+	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
+}
+
+validate_options() {
+	grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"
+}
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index a7865315..208a2c4e 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,46 +18,8 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-grep_line() { { shift; printf '%s\n' "$@"; } | grep -qxF "$1"; }
-prefix_lines() {
-	while test $# -gt 0
-	do
-		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
-		shift; shift
-	done
-}
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
-}
-validate_options() { grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"; }
-unquote_lines() {
-	sed -e '/^".*"$/{s/^"//;s/"$//}' \
-	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
-}
-append_values() {
-	while read -r _value
-	do
-		set -- "$@" --value "${_value}"
-	done
-	unset _value
-	"$@" </dev/null
-}
-print_errors() {
-	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
-		BEGIN {
-			if (getline) {
-				print prefix
-				print
-				rc = 1
-			}
-		}
-		{ print }
-		END {
-			if (rc && suffix) print suffix
-			exit rc
-		}' >&2
-}
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
 
 
 ## Check section name and error if invalid!

From c1ae3ccb2f725d0d94eaec9eeae7c8f5f98a5e45 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:13:06 +0100
Subject: [PATCH 447/743] [type/__uci*] Remove public-facing transaction
 "interface"

---
 cdist/conf/type/__uci/man.rst          | 13 ++++---------
 cdist/conf/type/__uci_section/man.rst  |  9 ++-------
 cdist/conf/type/__uci_section/manifest | 10 ++++------
 3 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index c7f5f4f0..8c3f79f1 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -11,8 +11,6 @@ DESCRIPTION
 This cdist type can be used to alter configuration options in OpenWrt's
 Unified Configuration Interface (UCI) system.
 
-Options can be applied in batches if the ``--transaction`` parameter is used.
-
 
 REQUIRED PARAMETERS
 -------------------
@@ -30,9 +28,6 @@ OPTIONAL PARAMETERS
 -------------------
 state
     ``present`` or ``absent``, defaults to ``present``.
-transaction
-    The name of the transaction this option belongs to.
-    If none is given: ``default`` is used.
 type
     If the type should generate an option or a list.
     One of: ``option`` or ``list``.
@@ -55,10 +50,10 @@ EXAMPLES
     # Set DHCP option 252: tell DHCP clients to not ask for proxy information.
     __uci dhcp.lan.dhcp_option --type list --value '252,"\n"'
 
-    # Enable NTP and NTPd (in one transaction)
-    __uci system.ntp.enabled --value 1 --transaction ntp
-    __uci system.ntp.enable_server --value 1 --transaction ntp
-    __uci system.ntp.server --type list --transaction ntp \
+    # Enable NTP and NTPd (each is applied individually)
+    __uci system.ntp.enabled --value 1
+    __uci system.ntp.enable_server --value 1
+    __uci system.ntp.server --type list \
         --value '0.openwrt.pool.ntp.org' \
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 753d00b3..a0ab78e8 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -53,9 +53,6 @@ option
     shell argument.
 state
     ``present`` or ``absent``, defaults to ``present``.
-transaction
-    The name of the transaction this option belongs to.
-    The value will be forwarded to :strong:`cdist-type__uci`\ (7).
 type
     The type of the section in the format: ``<config>.<section-type>``
 
@@ -71,13 +68,13 @@ EXAMPLES
 .. code-block:: sh
 
     # Configure the dropbear daemon
-    __uci_section dropbear --type dropbear.dropbear --transaction sshd \
+    __uci_section dropbear --type dropbear.dropbear \
         --match Port=22 --option Port=22 \
         --option PasswordAuth=off \
         --option RootPasswordAuth=off
 
     # Define a firewall zone comprised of lan and wlan networks
-    __uci_section firewall.internal --type firewall.zone --transaction fw \
+    __uci_section firewall.internal --type firewall.zone \
         --option name='internal' \
         --list network='lan' \
         --list network='wlan' \
@@ -87,7 +84,6 @@ EXAMPLES
 
     # Block SSH access from the guest network
     __uci_section firewall.block_ssh_from_guest --type firewall.rule \
-        --transaction fwrules \
         --option name='Block-SSH-Access-from-Guest' \
         --option src='guest' \
         --option proto='tcp' \
@@ -96,7 +92,6 @@ EXAMPLES
 
     # Configure a Wi-Fi access point
     __uci_section wireless.default_radio0 --type wireless.wifi-iface \
-        --transaction wifi \
         --option device='radio0' \
         --option mode='ap' \
         --option network='wlan' \
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 208a2c4e..41e30a07 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -46,7 +46,6 @@ esac
 section=$(cat "${__object:?}/explorer/match")
 
 state_should=$(cat "${__object:?}/parameter/state")
-transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case $state_should
 in
@@ -106,8 +105,7 @@ in
 		fi
 
 		# Make sure the section itself is present
-		__uci "${section}" --state present --transaction "${transaction_name}" \
-			--value "${sect_type}"
+		__uci "${section}" --state present --value "${sect_type}"
 		export require=__uci/"${section}"
 
 		# Delete options/lists not in "should"
@@ -115,7 +113,7 @@ in
 		| while read -r optname
 		  do
 			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
-				  __uci "${section}.${optname}" --state absent --transaction "${transaction_name}"
+				  __uci "${section}.${optname}" --state absent
 			  } </dev/null
 		  done
 
@@ -130,14 +128,14 @@ in
 			  | unquote_lines \
 			  | append_values \
 				    __uci "${section}.${_optname}" --state present \
-					    --type "${_type}" --transaction "${transaction_name}"
+					    --type "${_type}"
 		  done
 		;;
 	(absent)
 		# if explorer found no section there is nothing to delete
 		test -n "${section}" || exit 0
 
-		__uci "${section}" --state absent --transaction "${transaction_name}"
+		__uci "${section}" --state absent
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2

From e264fb004f797efc540f74191a0e9630c66dd819 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:27:07 +0100
Subject: [PATCH 448/743] [type/__uci] Convert to immediate remote execution

---
 cdist/conf/type/__uci/files/functions.sh |   6 +-
 cdist/conf/type/__uci/gencode-local      | 102 -----------------------
 cdist/conf/type/__uci/gencode-remote     |   8 --
 3 files changed, 2 insertions(+), 114 deletions(-)
 delete mode 100755 cdist/conf/type/__uci/gencode-local
 mode change 100644 => 100755 cdist/conf/type/__uci/gencode-remote

diff --git a/cdist/conf/type/__uci/files/functions.sh b/cdist/conf/type/__uci/files/functions.sh
index 4ddb82e0..277f648c 100644
--- a/cdist/conf/type/__uci/files/functions.sh
+++ b/cdist/conf/type/__uci/files/functions.sh
@@ -24,10 +24,8 @@ quote() {
 
 uci_cmd() {
 	# Usage: uci_cmd [UCI ARGUMENTS]...
-	# Will output a shell command to be executed locally to add the given
-	# command to $tmpfile.
-	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
-	printf ' >>%s\n' "$(quote "${tmpfile}")"
+	mkdir -p "${__object:?}/files"
+	printf '%s\n' "$(quote "$@")" >>"${__object:?}/files/uci_batch.txt"
 }
 
 uci_validate_name() {
diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
deleted file mode 100755
index b7664e85..00000000
--- a/cdist/conf/type/__uci/gencode-local
+++ /dev/null
@@ -1,102 +0,0 @@
-#!/bin/sh -e
-#
-# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-state_is=$(cat "${__object:?}/explorer/state")
-state_should=$(cat "${__object:?}/parameter/state")
-
-transaction_name=$(cat "${__object:?}/parameter/transaction")
-
-tmpdir="${__global:?}/tmp/__uci"
-# HACK
-mkdir -p "${tmpdir}"
-
-tmpfile="${tmpdir}/${transaction_name}.txt"
-
-# shellcheck source=files/functions.sh
-. "${__type:?}/files/functions.sh"
-
-
-config=${__object_id:?}
-uci_validate_tuple "${config}"
-
-
-case ${state_should}
-in
-	(present)
-		if in_list "${state_is}" 'present' 'rearranged'
-		then
-			# NOTE: order is ignored so rearranged is also fine.
-			exit 0
-		fi
-
-		# Determine type
-		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
-		case ${type}
-		in
-			(option|list) ;;
-			('')
-				# Guess type by the number of values
-				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
-					 && type=list \
-					 || type=option
-				;;
-			(*)
-				printf 'Invalid --type: %s\n' "${type}" >&2
-				exit 1
-				;;
-		esac
-
-		case ${type}
-		in
-			(list)
-				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
-
-				if test "${state_is}" != 'absent'
-				then
-					uci_cmd delete "${config}"
-				fi
-
-				while read -r value
-				do
-					uci_cmd add_list "${config}"="${value}"
-				done <"${__object:?}/parameter/value"
-				;;
-			(option)
-				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
-
-				value=$(cat "${__object:?}/parameter/value")
-				uci_cmd set "${config}"="${value}"
-				;;
-		esac
-		;;
-	(absent)
-		if in_list "${state_is}" 'absent'
-		then
-			exit 0
-		fi
-
-		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
-		uci_cmd delete "${config}"
-		;;
-	(*)
-		printf 'Invalid --state: %s\n' "${state_should}" >&2
-		exit 1
-		;;
-esac
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
old mode 100644
new mode 100755
index 348d15a1..a15f50a6
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -91,11 +91,3 @@ in
 		exit 1
 		;;
 esac
-
-if test -s "${__object:?}/files/uci_batch.txt"
-then
-	cat "${__type:?}/files/uci_apply.sh"
-	printf "uci_apply <<'EOF'\n"
-	cat "${__object:?}/files/uci_batch.txt"
-	printf '\nEOF\n'
-fi

From dfe9e08c280a2a3bc0ded15069b7d60df5a757f8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:06:05 +0100
Subject: [PATCH 449/743] [type/__uci_commit] Delete type

---
 cdist/conf/type/__uci/man.rst               |  2 -
 cdist/conf/type/__uci/manifest              | 18 -----
 cdist/conf/type/__uci_commit/gencode-remote | 81 ---------------------
 cdist/conf/type/__uci_commit/man.rst        | 60 ---------------
 cdist/conf/type/__uci_commit/nonparallel    |  0
 5 files changed, 161 deletions(-)
 delete mode 100755 cdist/conf/type/__uci_commit/gencode-remote
 delete mode 100644 cdist/conf/type/__uci_commit/man.rst
 delete mode 100644 cdist/conf/type/__uci_commit/nonparallel

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 8c3f79f1..81a53473 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -58,13 +58,11 @@ EXAMPLES
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
         --value '3.openwrt.pool.ntp.org'
-    export require=__uci_commit/ntp
 
 
 SEE ALSO
 --------
 - https://openwrt.org/docs/guide-user/base-system/uci
-- :strong:`cdist-type__uci_commit`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index d0c8239e..26920011 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,15 +18,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
-. "${__type:?}/files/functions.sh"
-
-
 os=$(cat "${__global:?}/explorer/os")
 
-state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
-transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case ${os}
 in
@@ -40,8 +34,6 @@ in
 		;;
 esac
 
-changes_required=false
-
 case ${state_should}
 in
 	(present)
@@ -49,21 +41,11 @@ in
 			echo 'The parameter --value is required.' >&2
 			exit 1
 		}
-
-		# NOTE: order is ignored so rearranged is also fine.
-		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
 		;;
 	(absent)
-		in_list "${state_is}" 'absent' || changes_required=true
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
 		exit 1
 		;;
 esac
-
-if ${changes_required}
-then
-	# Make sure the changes are being committed
-	require=${__object_name:?} __uci_commit "${transaction_name}"
-fi
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
deleted file mode 100755
index e5c93966..00000000
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/bin/sh -e
-#
-# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-transaction_name=${__object_id:?}
-batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
-
-test -s "${batchfile}" || exit 0
-
-printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
-
-# NOTE: Uncommited changes are checked in code-remote instead of in an explorer
-#       because in cdist there is no interlocking between explorers and code
-#       execution.
-#       Checking for uncommited changes in an explorer leaves a time slot in
-#       which changes made are not detected by the code.
-#       Furthermore an explorer running concurrently with another transactions
-#       code-remote could lead to a false positive.
-
-cat <<CODE
-changes=\$(uci changes)
-
-if test -n "\${changes}"
-then
-	echo 'Uncommited UCI changes were found on the target:'
-	printf '%s\n\n' "\${changes}"
-	echo 'This can be caused by manual changes or due to a previous failed run.'
-	echo 'Please investigate the situation, revert or commit the changes, and try again.'
-	exit 1
-fi >&2
-
-check_errors() {
-	# reads stdin and forwards non-empty lines to stderr.
-	# returns 0 if stdin is empty, else 1.
-	! grep -e . >&2
-}
-
-commit() {
-	uci commit
-}
-
-rollback() {
-	echo >&2
-	echo 'An error occurred when trying to commit transaction ${transaction_name}!' >&2
-
-	uci changes \\
-	| sed -e 's/^-//' -e 's/\..*\$//' \\
-	| sort -u \\
-	| while read -r _package
-	  do
-		  uci revert "\${_package}"
-		  echo "\${_package}"  # for logging
-	  done \\
-	| awk '
-	  BEGIN { printf "Reverted changes in: " }
-	  { printf "%s%s", (FNR > 1 ? ", " : ""), \$0 }
-	  END { printf "\\n" }' >&2
-
-	return 1
-}
-
-uci batch <<'EOF' 2>&1 | check_errors && commit || rollback
-$(cat "${batchfile}")
-EOF
-CODE
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
deleted file mode 100644
index ea6611eb..00000000
--- a/cdist/conf/type/__uci_commit/man.rst
+++ /dev/null
@@ -1,60 +0,0 @@
-cdist-type__uci_commit(7)
-=========================
-
-NAME
-----
-cdist-type__uci_commit - Commit UCI transactions
-
-
-DESCRIPTION
------------
-This type executes the ``uci commit`` command on the target with the commands
-queued in a transaction.
-It is usually not required to use this type. Use the ``--transaction`` parameter
-of :strong:`cdist-type__uci`\ (7) and :strong:`cdist-type__uci_section`\ (7)
-instead.
-
-
-REQUIRED PARAMETERS
--------------------
-None.
-
-
-OPTIONAL PARAMETERS
--------------------
-None.
-
-
-BOOLEAN PARAMETERS
-------------------
-None.
-
-
-EXAMPLES
---------
-
-.. code-block:: sh
-
-    # Commit the default transaction
-    __uci_commit default
-
-    # Commit another transaction
-    __uci_commit my_transaction
-
-
-SEE ALSO
---------
-:strong:`cdist-type__uci`\ (7), :strong:`cdist-type__uci_section`\ (7)
-
-
-AUTHORS
--------
-Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
-
-
-COPYING
--------
-Copyright \(C) 2020 Dennis Camera. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_commit/nonparallel b/cdist/conf/type/__uci_commit/nonparallel
deleted file mode 100644
index e69de29b..00000000

From ec984f81b5a1db6b7f3e54255b570e07cf5282f9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:36:10 +0100
Subject: [PATCH 450/743] [type/__uci] Delete --transaction parameter

---
 cdist/conf/type/__uci/parameter/default/transaction | 1 -
 cdist/conf/type/__uci/parameter/optional            | 1 -
 2 files changed, 2 deletions(-)
 delete mode 100644 cdist/conf/type/__uci/parameter/default/transaction

diff --git a/cdist/conf/type/__uci/parameter/default/transaction b/cdist/conf/type/__uci/parameter/default/transaction
deleted file mode 100644
index 4ad96d51..00000000
--- a/cdist/conf/type/__uci/parameter/default/transaction
+++ /dev/null
@@ -1 +0,0 @@
-default
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
index 2703b3df..d9080e3a 100644
--- a/cdist/conf/type/__uci/parameter/optional
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -1,3 +1,2 @@
 state
-transaction
 type

From 3e5f18d409e448f69e4f259fc451d4a0e5a7a548 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 18:42:50 +0100
Subject: [PATCH 451/743] [type/__uci] Apply all commands in a single batch

---
 cdist/conf/type/__uci/files/uci_apply.sh | 43 ++++++++++++++++++++++++
 cdist/conf/type/__uci/gencode-remote     |  8 +++++
 cdist/conf/type/__uci/nonparallel        |  0
 3 files changed, 51 insertions(+)
 create mode 100644 cdist/conf/type/__uci/files/uci_apply.sh
 create mode 100644 cdist/conf/type/__uci/nonparallel

diff --git a/cdist/conf/type/__uci/files/uci_apply.sh b/cdist/conf/type/__uci/files/uci_apply.sh
new file mode 100644
index 00000000..63f94290
--- /dev/null
+++ b/cdist/conf/type/__uci/files/uci_apply.sh
@@ -0,0 +1,43 @@
+changes=$(uci changes)
+
+if test -n "${changes}"
+then
+	echo 'Uncommited UCI changes were found on the target:'
+	printf '%s\n\n' "${changes}"
+	echo 'This can be caused by manual changes or due to a previous failed run.'
+	echo 'Please investigate the situation, revert or commit the changes, and try again.'
+	exit 1
+fi >&2
+
+check_errors() {
+	# reads stdin and forwards non-empty lines to stderr.
+	# returns 0 if stdin is empty, else 1.
+	! grep -e . >&2
+}
+
+commit() {
+	uci commit
+}
+
+rollback() {
+	printf '\nAn error occurred when trying to commit UCI transaction!\n' >&2
+
+	uci changes \
+	| sed -e 's/^-//' -e 's/\..*\$//' \
+	| sort -u \
+	| while read -r _package
+	  do
+		  uci revert "${_package}"
+		  echo "${_package}"  # for logging
+	  done \
+	| awk '
+	  BEGIN { printf "Reverted changes in: " }
+	  { printf "%s%s", (FNR > 1 ? ", " : ""), $0 }
+	  END { printf "\n" }' >&2
+
+	return 1
+}
+
+uci_apply() {
+	uci batch 2>&1 | check_errors && commit || rollback
+}
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
index a15f50a6..348d15a1 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -91,3 +91,11 @@ in
 		exit 1
 		;;
 esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci/nonparallel b/cdist/conf/type/__uci/nonparallel
new file mode 100644
index 00000000..e69de29b

From 9d40500570875f29aa48784c6637683aa31afe4e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 20:40:29 +0100
Subject: [PATCH 452/743] [type/__uci_section] Apply all commands in a single
 batch

---
 .../type/__uci_section/files/functions.sh     |  31 +++-
 .../type/__uci_section/files/uci_apply.sh     |   1 +
 cdist/conf/type/__uci_section/gencode-remote  | 146 ++++++++++++++++++
 cdist/conf/type/__uci_section/manifest        |  76 ++-------
 cdist/conf/type/__uci_section/nonparallel     |   0
 5 files changed, 180 insertions(+), 74 deletions(-)
 create mode 120000 cdist/conf/type/__uci_section/files/uci_apply.sh
 create mode 100755 cdist/conf/type/__uci_section/gencode-remote
 create mode 100644 cdist/conf/type/__uci_section/nonparallel

diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
index 9f510958..17c64bbc 100644
--- a/cdist/conf/type/__uci_section/files/functions.sh
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -1,13 +1,6 @@
 # -*- mode: sh; indent-tabs-mode: t -*-
 
-append_values() {
-	while read -r _value
-	do
-		set -- "$@" --value "${_value}"
-	done
-	unset _value
-	"$@" </dev/null
-}
+NL=$(printf '\n '); NL=${NL% }
 
 grep_line() {
 	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
@@ -37,6 +30,28 @@ print_errors() {
 		}' >&2
 }
 
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+	printf '%s' "$*"
+}
+
+uci_cmd() {
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	mkdir -p "${__object:?}/files"
+	printf '%s\n' "$(quote "$@")" >>"${__object:?}/files/uci_batch.txt"
+}
+
 uci_validate_name() {
 	# like util.c uci_validate_name()
 	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
diff --git a/cdist/conf/type/__uci_section/files/uci_apply.sh b/cdist/conf/type/__uci_section/files/uci_apply.sh
new file mode 120000
index 00000000..4209151f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/uci_apply.sh
@@ -0,0 +1 @@
+../../__uci/files/uci_apply.sh
\ No newline at end of file
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
new file mode 100755
index 00000000..05bca0a7
--- /dev/null
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -0,0 +1,146 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+
+section=$(cat "${__object:?}/explorer/match")
+
+state_is=$(test -s "${__object:?}/explorer/type" && echo present || echo absent)
+state_should=$(cat "${__object:?}/parameter/state")
+
+case $state_should
+in
+	(present)
+		test -f "${__object:?}/parameter/type" || {
+			echo 'Parameter --type is required.' >&2
+			exit 1
+		}
+
+		type_is=$(cat "${__object:?}/explorer/type")
+		type_should=$(cat "${__object:?}/parameter/type")
+
+		if test -n "${type_is}"
+		then
+			sect_type=${type_is}
+		else
+			sect_type=${type_should##*.}
+		fi
+
+		if test -z "${section}"
+		then
+			# No section exists and --match was used.
+			# So we generate a new section identifier from $__object_id.
+			case ${__object_id:?}
+			in
+				(*.*) section=${__object_id:?} ;;
+				(*) section="${type_should%%.*}.${__object_id:?}" ;;
+			esac
+		fi
+
+		# Collect option names
+		if test -f "${__object:?}/parameter/list"
+		then
+			listnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
+		fi
+
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
+		# Make sure the section itself is present
+		if test "${state_is}" = absent
+		then
+			printf 'set %s\n' "${section}" >>"${__messages_out:?}"
+			# shellcheck disable=SC2140
+			uci_cmd set "${section}"="${sect_type}"
+		fi
+
+		# Delete options/lists not in "should"
+		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		| while read -r _optname
+		  do
+			  grep_line "${_optname}" "${listnames_should}" "${optnames_should}" || {
+				  printf 'delete %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+				  uci_cmd delete "${section}.${_optname}"
+			  } </dev/null
+		  done
+
+		# Set "should" options
+		prefix_lines option "${optnames_should}" list "${listnames_should}" \
+		| while read -r _type _optname
+		  do
+			  test -n "${_type}${_optname}" || continue  # ignore empty lines
+
+			  case $_type
+			  in
+				  (list)
+					  printf 'set_list %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  if grep -q -e "^${_optname}=" "${__object:?}/explorer/options"
+					  then
+						  uci_cmd delete "${section}.${_optname}"
+					  fi
+
+					  grep "^${_optname}=" "${__object:?}/parameter/list" \
+					  | sed -e 's/^.*=//' \
+					  | unquote_lines \
+					  | while read -r _value
+					    do
+						    # shellcheck disable=SC2140
+						    uci_cmd add_list "${section}.${_optname}"="${_value}"
+					    done
+					  ;;
+				  (option)
+					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  # shellcheck disable=SC2140
+					  uci_cmd set "${section}.${_optname}"="$(
+						  grep "^${_optname}=" "${__object:?}/parameter/option" \
+						  | sed -e 's/^.*=//' \
+						  | unquote_lines \
+						  | head -n 1)"
+					  ;;
+			  esac
+		  done
+		;;
+	(absent)
+		if test "${state_is}" = absent
+		then
+			# if explorer found no section there is nothing to delete
+			exit 0
+		fi
+
+		printf 'delete %s\n' "${section}" >>"${__messages_out:?}"
+		uci_cmd delete "${section}"
+		;;
+esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 41e30a07..7e23e443 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -43,8 +43,6 @@ in
 		;;
 esac
 
-section=$(cat "${__object:?}/explorer/match")
-
 state_should=$(cat "${__object:?}/parameter/state")
 
 case $state_should
@@ -57,85 +55,31 @@ in
 		type_is=$(cat "${__object:?}/explorer/type")
 		type_should=$(cat "${__object:?}/parameter/type")
 
-		if test -n "${type_is}"
+		if test -n "${type_is}" && test "${type_is}" != "${type_should##*.}"
 		then
-			if test "${type_is}" != "${type_should##*.}"
-			then
-				# Check if section type matches (section exists and --type provided)
-				printf 'Section type "%s" does not match --type "%s".\n' \
-					"${type_is}" "${type_should}" >&2
-				exit 1
-			fi
-			sect_type=${type_is}
-		else
-			sect_type=${type_should##*.}
-		fi
-
-		if test -z "${section}"
-		then
-			# No section exists and --match was used.
-			# So we generate a new section identifier from $__object_id.
-			case ${__object_id:?}
-			in
-				(*.*) section=${__object_id:?} ;;
-				(*) section="${type_should%%.*}.${__object_id:?}" ;;
-			esac
+			# Check if section type matches (section exists and --type provided)
+			printf 'Section type "%s" does not match --type "%s".\n' \
+				"${type_is}" "${type_should}" >&2
+			exit 1
 		fi
 
 		# Check options for syntax errors
 		validate_options "${__object:?}/parameter/list" "${__object:?}/parameter/object" \
 		| print_errors 'Found erroneous options in arguments:'
 
-		# Collect option names
-		if test -f "${__object:?}/parameter/list"
+		# Check for duplicate option names
+		if test -s "${__object:?}/parameter/option"
 		then
-			listnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
-		fi
-
-		if test -f "${__object:?}/parameter/option"
-		then
-			optnames_should=$(sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort)
-
-			echo "${optnames_should}" \
+			sed -e 's/=.*$//' "${__object:?}/parameter/option" \
+			| sort \
 			| uniq -d \
 			| print_errors \
 				  'Found duplicate --options:' \
 				  "$(printf '\nUse --list for lists, instead.')"
 		fi
-
-		# Make sure the section itself is present
-		__uci "${section}" --state present --value "${sect_type}"
-		export require=__uci/"${section}"
-
-		# Delete options/lists not in "should"
-		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
-		| while read -r optname
-		  do
-			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
-				  __uci "${section}.${optname}" --state absent
-			  } </dev/null
-		  done
-
-		# Set "should" options
-		prefix_lines option "${optnames_should}" list "${listnames_should}" \
-		| while read -r _type _optname
-		  do
-			  test -n "${_type}${_optname}" || continue  # ignore empty lines
-
-			  grep "^${_optname}=" <"${__object:?}/parameter/${_type}" \
-			  | sed -e 's/^.*=//' \
-			  | unquote_lines \
-			  | append_values \
-				    __uci "${section}.${_optname}" --state present \
-					    --type "${_type}"
-		  done
 		;;
 	(absent)
-		# if explorer found no section there is nothing to delete
-		test -n "${section}" || exit 0
-
-		__uci "${section}" --state absent
+		:
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
diff --git a/cdist/conf/type/__uci_section/nonparallel b/cdist/conf/type/__uci_section/nonparallel
new file mode 100644
index 00000000..e69de29b

From ade69729ddd64938ef69c08a522a85ec825628e9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 31 Oct 2020 18:18:15 +0100
Subject: [PATCH 453/743] [type/__uci_section] Only generate UCI commands if
 state differs

---
 .../conf/type/__uci_section/explorer/options  | 22 ++++-
 .../type/__uci_section/files/functions.sh     |  8 --
 .../type/__uci_section/files/option_state.awk | 91 +++++++++++++++++++
 cdist/conf/type/__uci_section/gencode-remote  | 68 ++++++++++----
 4 files changed, 160 insertions(+), 29 deletions(-)
 create mode 100644 cdist/conf/type/__uci_section/files/option_state.awk

diff --git a/cdist/conf/type/__uci_section/explorer/options b/cdist/conf/type/__uci_section/explorer/options
index 67b4f83f..e1e60668 100644
--- a/cdist/conf/type/__uci_section/explorer/options
+++ b/cdist/conf/type/__uci_section/explorer/options
@@ -25,4 +25,24 @@ section=$("${__type_explorer:?}/match")
 test -n "${section}" || exit 0
 
 uci -s -N -d "${RS}" show "${section}" 2>/dev/null \
-| grep -v -e "^${section}=" || true
+| awk -v VSEP="${RS}" '
+  {
+	  # Strip off the config and section parts
+	  is_opt = sub(/^([^.]*\.){2}/, "")
+
+	  if (!is_opt) {
+		  # this line represents the section -> skip
+		  next
+	  }
+
+	  if (index($0, VSEP)) {
+		  # Put values each on a line, like --option and --list parameters
+		  opt = substr($0, 1, index($0, "=") - 1)
+		  split(substr($0, length(opt) + 2), values, VSEP)
+		  for (i in values) {
+			  printf "%s=%s\n", opt, values[i]
+		  }
+	  } else {
+		  print
+	  }
+  }'
diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
index 17c64bbc..60cb9148 100644
--- a/cdist/conf/type/__uci_section/files/functions.sh
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -6,14 +6,6 @@ grep_line() {
 	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
 }
 
-prefix_lines() {
-	while test $# -gt 0
-	do
-		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
-		shift; shift
-	done
-}
-
 print_errors() {
 	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
 		BEGIN {
diff --git a/cdist/conf/type/__uci_section/files/option_state.awk b/cdist/conf/type/__uci_section/files/option_state.awk
new file mode 100644
index 00000000..97cd94fb
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/option_state.awk
@@ -0,0 +1,91 @@
+# -*- mode: awk; indent-tabs-mode:t -*-
+# Usage: awk -f option_state.awk option_type option_name
+# e.g. awk -f option_state.awk option title
+#      awk -f option_state.awk list entry
+
+function unquote(s) {
+	# simplified dequoting of single quoted strings
+	if (s ~ /^'.*'$/) {
+		s = substr(s, 2, length(s) - 2)
+		sub(/'\\''/, "'", s)
+	}
+	return s
+}
+
+function valueof(line) {
+	if (line !~ /^[[:alpha:]_]+=/) return 0
+	return unquote(substr(line, index(line, "=") + 1))
+}
+
+BEGIN {
+	__object = ENVIRON["__object"]
+	if (!__object) exit 1
+
+	opttype = ARGV[1]
+	optname = ARGV[2]
+
+	if (opttype !~ /^(option|list)/ || !optname) {
+		print "invalid"
+		exit (e=1)
+	}
+
+	ARGV[1] = __object "/parameter/" opttype
+	ARGV[2] = __object "/explorer/options"
+
+	state = "present"
+}
+
+NR == FNR {
+	# memoize "should" state
+	if (index($0, optname "=") == 1) {
+		should[++should_count] = valueof($0)
+	}
+
+	# go to next line (important!)
+	next
+}
+
+{
+	# compare "is" state
+	if (index($0, optname "=") != 1)
+		next
+	++is_count
+
+	v = valueof($0)
+
+	if (v == should[is_count]) {
+		# looks good, but can't say definitely just from this line
+	} else if (is_count > should_count) {
+		# there are more "is" records than "should" -> definitely different
+		state = "different"
+		exit
+	} else {
+		# see if we can find the "is" value somewhere in "should"
+		for (i in should) {
+			if (v == should[i]) {
+				# value found -> could be rearranged
+				# FIXME: Duplicate values are not properly handled here. Do they matter?
+				state = "rearranged"
+				next
+			}
+		}
+
+		# "is" value could not be found in "should" -> definitely different
+		state = "different"
+		exit
+	}
+}
+
+END {
+	if (e) exit
+
+	if (!is_count) {
+		# no "is" values -> absent
+		state = "absent"
+	} else if (is_count < should_count) {
+		# "is" was shorter than "should" -> different
+		state = "different"
+	}
+
+	print state
+}
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
index 05bca0a7..285659b0 100755
--- a/cdist/conf/type/__uci_section/gencode-remote
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -70,7 +70,8 @@ in
 		fi
 
 		# Make sure the section itself is present
-		if test "${state_is}" = absent
+		if test "${state_is}" = absent \
+			|| test "${type_is}" != "${type_should#*.}"
 		then
 			printf 'set %s\n' "${section}" >>"${__messages_out:?}"
 			# shellcheck disable=SC2140
@@ -78,7 +79,7 @@ in
 		fi
 
 		# Delete options/lists not in "should"
-		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		sed -e 's/=.*$//' "${__object:?}/explorer/options" \
 		| while read -r _optname
 		  do
 			  grep_line "${_optname}" "${listnames_should}" "${optnames_should}" || {
@@ -87,18 +88,55 @@ in
 			  } </dev/null
 		  done
 
-		# Set "should" options
-		prefix_lines option "${optnames_should}" list "${listnames_should}" \
-		| while read -r _type _optname
-		  do
-			  test -n "${_type}${_optname}" || continue  # ignore empty lines
+		opt_proc_error() {
+			printf 'An error occurred during processing of option %s\n' "${1:?}" >&2
+			exit 1
+		}
 
-			  case $_type
+		# Set "should" options
+		echo "${optnames_should}" \
+		| grep -e . \
+		| while read -r _optname
+		  do
+			  _opt_state=$(awk -f "${__type:?}/files/option_state.awk" option "${_optname}") \
+				  || opt_proc_error "${_optname}"
+			  case ${_opt_state}
 			  in
-				  (list)
+				  (invalid)
+					  opt_proc_error "${_optname}"
+					  ;;
+				  (present)
+					  ;;
+				  (*)
+					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  # shellcheck disable=SC2140
+					  uci_cmd set "${section}.${_optname}"="$(
+						  grep -e "^${_optname}=" "${__object:?}/parameter/option" \
+						  | sed -e 's/^.*=//' \
+						  | unquote_lines \
+						  | head -n 1)"
+					  ;;
+			  esac
+		  done
+
+		echo "${listnames_should}" \
+		| grep -e . \
+		| while read -r _optname
+		  do
+			  _list_state=$(awk -f "${__type:?}/files/option_state.awk" list "${_optname}") \
+				  || opt_proc_error "${_optname}"
+			  case ${_list_state}
+			  in
+				  (invalid)
+					  opt_proc_error "${_optname}"
+					  ;;
+				  (present)
+					  ;;
+				  (*)
 					  printf 'set_list %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
 
-					  if grep -q -e "^${_optname}=" "${__object:?}/explorer/options"
+					  if test "${_list_state}" != absent
 					  then
 						  uci_cmd delete "${section}.${_optname}"
 					  fi
@@ -112,16 +150,6 @@ in
 						    uci_cmd add_list "${section}.${_optname}"="${_value}"
 					    done
 					  ;;
-				  (option)
-					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
-
-					  # shellcheck disable=SC2140
-					  uci_cmd set "${section}.${_optname}"="$(
-						  grep "^${_optname}=" "${__object:?}/parameter/option" \
-						  | sed -e 's/^.*=//' \
-						  | unquote_lines \
-						  | head -n 1)"
-					  ;;
 			  esac
 		  done
 		;;

From 2be8c634584561158960fc5cf2f363f3f5c4cc4c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 3 Nov 2020 06:43:57 +0100
Subject: [PATCH 454/743] pycodestyle fixes

---
 cdist/argparse.py         |  2 +-
 cdist/scan/commandline.py |  2 +-
 cdist/scan/scan.py        | 12 ++++++++----
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index ff195e8c..88759d7b 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -473,7 +473,7 @@ def get_parsers():
 
     # Scan = config + further
     parser['scan'] = parser['sub'].add_parser('scan', add_help=False,
-                                                 parents=[parser['config']])
+                                              parents=[parser['config']])
 
     parser['scan'] = parser['sub'].add_parser(
             'scan', parents=[parser['loglevel'],
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 0fb718e5..eca4cf13 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -37,7 +37,7 @@ def commandline(args):
 
     if not args.mode:
         # By default scan and trigger, but do not call any action
-        args.mode = ['scan', 'trigger' ]
+        args.mode = ['scan', 'trigger', ]
 
     if 'trigger' in args.mode:
         t = scan.Trigger(interfaces=args.interfaces)
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index fcbf1899..0ce4dff3 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -30,9 +30,12 @@
 # Scanner logic
 #  - save results to configdir:
 #     basedir = ~/.cdist/scan/<ipv6-address>
-#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time or similar
-#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
-#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time
+#           or similar
+#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record
+#           unix time or similar
+#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record
+#           unix time or similar
 #
 #
 #
@@ -58,6 +61,7 @@ import datetime
 
 log = logging.getLogger("scan")
 
+
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
@@ -91,6 +95,7 @@ class Trigger(object):
         log.debug(f"Sending request on {interface}")
         send(packet, verbose=self.verbose)
 
+
 class Scanner(object):
     """
     Scan for replies of hosts, maintain the up-to-date database
@@ -134,7 +139,6 @@ class Scanner(object):
               prn=self.handle_pkg)
 
 
-
 if __name__ == '__main__':
     t = Trigger(interfaces=["wlan0"])
     t.start()

From df881c0f9859759c10339e9d4a5ac24ff77fbe1a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 4 Nov 2020 08:34:17 +0100
Subject: [PATCH 455/743] [type/__file] Fix --state pre-exists also for
 non-dry-runs

---
 cdist/conf/type/__file/gencode-remote | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index 2675b03a..f7a528fd 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -95,6 +95,10 @@ case "$state_should" in
         fi
     ;;
 
+    pre-exists)
+        :
+        ;;
+
     *)
         echo "Unknown state: $state_should" >&2
         exit 1

From d28a70a73c2affcf3268623a51df2b7a6cdd534a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 6 Nov 2020 08:32:40 +0100
Subject: [PATCH 456/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 5977d80e..335a0f9f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,8 @@ next:
 	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 	* Type __file: Fix state pre-exists (Dennis Camera)
+	* Type __apt_norecommends: Use 00InstallRecommends file as debian-installer does (Dennis Camera)
+	* Core: Introduce scanner (noninvasive, beta) (Nico Schottelius)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 0f1df5ef68a60c5b8077f437bf0d07f1f79788e7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:07:58 +0100
Subject: [PATCH 457/743] Fix shellcheck source directives

---
 cdist/conf/type/__uci/gencode-remote         | 2 +-
 cdist/conf/type/__uci_section/gencode-remote | 2 +-
 cdist/conf/type/__uci_section/manifest       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
index 348d15a1..70a3d3e0 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 state_is=$(cat "${__object:?}/explorer/state")
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
index 285659b0..50fdfa4e 100755
--- a/cdist/conf/type/__uci_section/gencode-remote
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci_section/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 7e23e443..0325b92c 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci_section/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 

From c7c3075f629191ed7e05a76395b77dce01e2928f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:10:14 +0100
Subject: [PATCH 458/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 335a0f9f..40b816a6 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -13,6 +13,7 @@ next:
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 	* Type __file: Fix state pre-exists (Dennis Camera)
 	* Type __apt_norecommends: Use 00InstallRecommends file as debian-installer does (Dennis Camera)
+	* New types: __uci, __uci_section (Dennis Camera)
 	* Core: Introduce scanner (noninvasive, beta) (Nico Schottelius)
 
 6.8.0: 2020-09-11

From 348c6eedc903ecd28b02c5dda3821e71ef255a4f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:12:20 +0100
Subject: [PATCH 459/743] Release 6.9.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 40b816a6..c47123d9 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.0: 2020-11-07
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)

From 10abe514b8fb7e6d3ce3587a55773a21e78ce9e6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 11 Jun 2020 21:27:53 +0200
Subject: [PATCH 460/743] [type/__hostname] Add support for OpenWrt

---
 cdist/conf/type/__hostname/gencode-remote | 3 +++
 cdist/conf/type/__hostname/manifest       | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index ae224611..02afcbfb 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -69,6 +69,9 @@ in
     centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void)
         echo "hostname '$name_should'"
     ;;
+    openwrt)
+        echo "echo '$name_should' >/proc/sys/kernel/hostname"
+    ;;
     macosx)
         echo "scutil --set HostName '$name_should'"
     ;;
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index e1e356a0..bf8a331c 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -149,6 +149,10 @@ in
     openbsd)
         echo "$name_should" | __file /etc/myname --source -
     ;;
+    openwrt)
+        __uci system.@system[0].hostname --value "$name_should"
+            # --transaction hostname
+    ;;
     slackware)
         # We write the FQDN into /etc/HOSTNAME.  But /etc/rc.d/rc.M will only
         # read the first component from this file and set it as the running

From b0f3bb3350be9575c1632c6e3f669e7ede431020 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 7 Nov 2020 16:49:09 +0100
Subject: [PATCH 461/743] New type __dpkg_architecture

This type handles foreign architectures added to dpkg.
---
 .../__dpkg_architecture/explorer/architecture |   8 ++
 .../explorer/foreign-architectures            |   8 ++
 .../type/__dpkg_architecture/gencode-remote   |  64 +++++++++++
 cdist/conf/type/__dpkg_architecture/man.rst   | 103 ++++++++++++++++++
 .../parameter/default/state                   |   1 +
 .../__dpkg_architecture/parameter/optional    |   1 +
 6 files changed, 185 insertions(+)
 create mode 100755 cdist/conf/type/__dpkg_architecture/explorer/architecture
 create mode 100755 cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
 create mode 100755 cdist/conf/type/__dpkg_architecture/gencode-remote
 create mode 100644 cdist/conf/type/__dpkg_architecture/man.rst
 create mode 100644 cdist/conf/type/__dpkg_architecture/parameter/default/state
 create mode 100644 cdist/conf/type/__dpkg_architecture/parameter/optional

diff --git a/cdist/conf/type/__dpkg_architecture/explorer/architecture b/cdist/conf/type/__dpkg_architecture/explorer/architecture
new file mode 100755
index 00000000..4478bc6e
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/explorer/architecture
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+# __dpkg_architecture/explorer/architecture
+
+# Get the main architecture of this machine
+
+
+# print or die in the gencode-remote
+dpkg --print-architecture || true
diff --git a/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
new file mode 100755
index 00000000..ff01672a
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+# __dpkg_architecture/explorer/foreign-architectures
+
+# Print all additional architectures
+
+
+# print or die in the gencode-remote
+dpkg --print-foreign-architectures || true
diff --git a/cdist/conf/type/__dpkg_architecture/gencode-remote b/cdist/conf/type/__dpkg_architecture/gencode-remote
new file mode 100755
index 00000000..adf9d655
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/gencode-remote
@@ -0,0 +1,64 @@
+#!/bin/sh -e
+# __dpkg_architecture/gencode-remote
+
+
+# Get parameter and explorer
+state_should="$(cat "$__object/parameter/state")"
+arch_wanted="$__object_id"
+main_arch="$(cat "$__object/explorer/architecture")"
+
+# Exit here if dpkg do not work (empty explorer)
+if [ -z "$main_arch" ]; then
+    echo "dpkg is not available or unable to detect a architecture!" >&2
+    exit 1
+fi
+
+
+# Check if requested architecture is the main one
+if [ "$arch_wanted" = "$main_arch" ]; then
+    # higher than present; we can not remove it
+    state_is="present"
+    caution="yes"
+
+# Check if the architecture not already used
+elif grep -qFx "$arch_wanted" "$__object/explorer/foreign-architectures"; then
+    state_is="present"
+
+# arch does not exist
+else
+    state_is="absent"
+fi
+
+
+# Check what to do
+if [ "$state_is" != "$state_should" ]; then
+    case "$state_should" in
+        present)
+            # print add code
+            printf "dpkg --add-architecture '%s'\n" "$arch_wanted"
+            # updating the index to make the new architecture available
+            echo "apt update"
+
+            echo added >> "$__messages_out"
+            ;;
+
+        absent)
+            if [ "$caution" ]; then
+                printf "can not remove the main arch '%s' of the system!\n" "$main_arch" >&2
+                exit 1
+            fi
+
+            # removing all existing packages for the architecture
+            printf "apt purge '.*:%s'\n" "$arch_wanted"
+            # print remove code
+            printf "dpkg --remove-architecture '%s'\n" "$arch_wanted"
+
+            echo removed >> "$__messages_out"
+            ;;
+
+        *)
+            printf "state '%s' is unknown!\n" "$state_should" >&2
+            exit 1
+            ;;
+    esac
+fi
diff --git a/cdist/conf/type/__dpkg_architecture/man.rst b/cdist/conf/type/__dpkg_architecture/man.rst
new file mode 100644
index 00000000..fa196229
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/man.rst
@@ -0,0 +1,103 @@
+cdist-type__dpkg_architecture(7)
+================================
+
+NAME
+----
+cdist-type__dpkg_architecture - Handles foreign architectures on debian-like
+systems managed by `dpkg`
+
+
+DESCRIPTION
+-----------
+This type handles foreign architectures on systems managed by
+:strong:`dpkg`\ (1). The object id is the name of the architecture accepted by
+`dpkg`, which should be added or removed.
+
+If the architecture is not setup on the system, it adds a new architecture as a
+new foreign architecture in `dpkg`. Then, it updates the apt package index to
+make packages from the new architecture available.
+
+If the architecture should be removed, it will remove it if it is not the base
+architecture on where the system was installed on. Before it, it will purge
+every package based on the "to be removed" architecture via `apt` to be able to
+remove the selected architecture.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    ``present`` or ``absent``. Defaults to ``present``.
+
+
+MESSAGES
+--------
+added
+   Added the specified architecture
+
+removed
+   Removed the specified architecture
+
+
+ABORTS
+------
+Aborts in the following cases:
+
+If :strong:`dpkg`\ (1) is not available. It will abort with a proper error
+message.
+
+If the architecture is the same as the base architecture the system is build
+upon it (returned by ``dpkg --print-architecture``) and it should be removed.
+
+It will fail if it can not execute :strong:`apt`\ (8). It is assumed that it is
+already installed.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+  # add i386 (32 bit) architecture
+  __dpkg_architecture i386
+
+  # remove it again :)
+  __dpkg_architecture i386 --state absent
+
+
+SEE ALSO
+--------
+`Multiarch on Debian systems <https://wiki.debian.org/Multiarch>`_
+
+`How to setup multiarch on Debian <https://wiki.debian.org/Multiarch/HOWTO>`_
+
+:strong:`dpkg`\ (1)
+:strong:`cdist-type__package_dpkg`\ (7)
+:strong:`cdist-type__package_apt`\ (7)
+
+Useful commands:
+
+.. code-block:: sh
+
+   # base architecture installed on this system
+   dpkg --print-architecture
+
+   # extra architectures added
+   dpkg --print-foreign-architectures
+
+
+AUTHORS
+-------
+Matthias Stecher <matthiasstecher at gmx.de>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Matthias Stecher. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+ublished by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__dpkg_architecture/parameter/default/state b/cdist/conf/type/__dpkg_architecture/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__dpkg_architecture/parameter/optional b/cdist/conf/type/__dpkg_architecture/parameter/optional
new file mode 100644
index 00000000..ff72b5c7
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/parameter/optional
@@ -0,0 +1 @@
+state

From 7777580d8ff94f2b0725dbbcef84d8957998acb6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 7 Nov 2020 20:56:17 +0100
Subject: [PATCH 462/743] __dpkg_architecture: add copyright headers

---
 .../__dpkg_architecture/explorer/architecture  | 18 ++++++++++++++++++
 .../explorer/foreign-architectures             | 18 ++++++++++++++++++
 .../type/__dpkg_architecture/gencode-remote    | 18 ++++++++++++++++++
 3 files changed, 54 insertions(+)

diff --git a/cdist/conf/type/__dpkg_architecture/explorer/architecture b/cdist/conf/type/__dpkg_architecture/explorer/architecture
index 4478bc6e..03e7e386 100755
--- a/cdist/conf/type/__dpkg_architecture/explorer/architecture
+++ b/cdist/conf/type/__dpkg_architecture/explorer/architecture
@@ -1,5 +1,23 @@
 #!/bin/sh -e
 # __dpkg_architecture/explorer/architecture
+#
+# 2020 Matthias Stecher <matthiasstecher at gmx.de>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 # Get the main architecture of this machine
 
diff --git a/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
index ff01672a..a150d307 100755
--- a/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
+++ b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
@@ -1,5 +1,23 @@
 #!/bin/sh -e
 # __dpkg_architecture/explorer/foreign-architectures
+#
+# 2020 Matthias Stecher <matthiasstecher at gmx.de>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 # Print all additional architectures
 
diff --git a/cdist/conf/type/__dpkg_architecture/gencode-remote b/cdist/conf/type/__dpkg_architecture/gencode-remote
index adf9d655..47fb24e7 100755
--- a/cdist/conf/type/__dpkg_architecture/gencode-remote
+++ b/cdist/conf/type/__dpkg_architecture/gencode-remote
@@ -1,5 +1,23 @@
 #!/bin/sh -e
 # __dpkg_architecture/gencode-remote
+#
+# 2020 Matthias Stecher <matthiasstecher at gmx.de>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 
 # Get parameter and explorer

From 91bcc2a2939508b8722c2a148c89c92aaa4f830f Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 7 Nov 2020 21:03:38 +0100
Subject: [PATCH 463/743] __dpkg_architecture: make type nonparallel

I think it's not good that dpkg or apt is running in parallel.
---
 cdist/conf/type/__dpkg_architecture/nonparallel | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 cdist/conf/type/__dpkg_architecture/nonparallel

diff --git a/cdist/conf/type/__dpkg_architecture/nonparallel b/cdist/conf/type/__dpkg_architecture/nonparallel
new file mode 100644
index 00000000..e69de29b

From 9fc6ee0948347a0de20e633ce4a0ee358dc92c00 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 8 Nov 2020 13:14:13 +0100
Subject: [PATCH 464/743] __package_apt: add --install-recommends parameter

For a good reason, __package_apt doesn't install recommended packages as
default. But the option --install-recommends comes handy if you want to
install a package where you want to install all recommended packages
(and not to install all of them separately).

Also, the manpage now explains that the type won't install recommended
packages by default.
---
 cdist/conf/type/__package_apt/gencode-remote    | 17 ++++++++++++-----
 cdist/conf/type/__package_apt/man.rst           | 15 +++++++++++++--
 cdist/conf/type/__package_apt/parameter/boolean |  1 +
 3 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote
index f3d91566..fbfca330 100755
--- a/cdist/conf/type/__package_apt/gencode-remote
+++ b/cdist/conf/type/__package_apt/gencode-remote
@@ -42,6 +42,13 @@ else
    target_release=""
 fi
 
+if [ -f "$__object/parameter/install-recommends" ]; then
+    # required if __apt_norecommends is used
+    recommendsparam="-o APT::Install-Recommends=1"
+else
+    recommendsparam="-o APT::Install-Recommends=0"
+fi
+
 if [ -f "$__object/parameter/purge-if-absent" ]; then
 	purgeparam="--purge"
 else
@@ -62,16 +69,16 @@ case "$state_is" in
     ;;
 esac
 
-# Hint if we need to avoid questions at some point:
-# DEBIAN_PRIORITY=critical can reduce the number of questions
-aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o APT::Install-Recommends=0 -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
-
 if [ "$state_is" = "$state_should" ]; then
     if [ -z "$version" ] || [ "$version" = "$version_is" ]; then
         exit 0;
     fi
 fi
 
+# Hint if we need to avoid questions at some point:
+# DEBIAN_PRIORITY=critical can reduce the number of questions
+aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
+
 case "$state_should" in
     present)
         # following is bit ugly, but important hack.
@@ -85,7 +92,7 @@ EOF
         if [ -n "$version" ]; then
             name="${name}=${version}"
         fi
-        echo "$aptget install $target_release '$name'"
+        echo "$aptget $recommendsparam install $target_release '$name'"
         echo "installed" >> "$__messages_out"
     ;;
     absent)
diff --git a/cdist/conf/type/__package_apt/man.rst b/cdist/conf/type/__package_apt/man.rst
index a1691eac..4e6101a5 100644
--- a/cdist/conf/type/__package_apt/man.rst
+++ b/cdist/conf/type/__package_apt/man.rst
@@ -9,7 +9,9 @@ cdist-type__package_apt - Manage packages with apt-get
 DESCRIPTION
 -----------
 apt-get is usually used on Debian and variants (like Ubuntu) to
-manage packages.
+manage packages. The package will be installed without recommended
+or suggested packages. If such packages are required, install them
+separatly or use the parameter ``--install-recommends``.
 
 This type will also update package index, if it is older
 than one day, to avoid missing package error messages.
@@ -23,7 +25,7 @@ None
 OPTIONAL PARAMETERS
 -------------------
 name
-   If supplied, use the name and not the object id as the package name.
+    If supplied, use the name and not the object id as the package name.
 
 state
     Either "present" or "absent", defaults to "present"
@@ -39,6 +41,15 @@ version
 
 BOOLEAN PARAMETERS
 ------------------
+install-recommends
+    If the package will be installed, it also installs recommended packages
+    with it. It will not install recommended packages if the original package
+    is already installed.
+
+    In most cases, it is recommended to install recommended packages separatly
+    to control which additional packages will be installed to avoid useless
+    installed packages.
+
 purge-if-absent
     If this parameter is given when state is `absent`, the package is
     purged from the system (using `--purge`).
diff --git a/cdist/conf/type/__package_apt/parameter/boolean b/cdist/conf/type/__package_apt/parameter/boolean
index f9a0f6b0..a2e433f3 100644
--- a/cdist/conf/type/__package_apt/parameter/boolean
+++ b/cdist/conf/type/__package_apt/parameter/boolean
@@ -1 +1,2 @@
+install-recommends
 purge-if-absent

From fded60bd0f4ca7b6b9047c9f3b1470fdbd20c263 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 8 Nov 2020 13:27:01 +0100
Subject: [PATCH 465/743] ++changelog

---
 docs/changelog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index c47123d9..f271142a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,12 @@
 Changelog
 ---------
 
+next:
+	* Type __file: Fix state pre-exists (Dennis Camera)
+	* Type __hostname: Add support for OpenWrt (Dennis Camera)
+	* New type: __dpkg_architecture (Matthias Stecher)
+	* Type __package_apt: Add --install-recommends parameter (Matthias Stecher)
+
 6.9.0: 2020-11-07
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)

From d2506ac04eac987caf0fe0c33f48866ee6e43c0d Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 8 Nov 2020 13:31:57 +0100
Subject: [PATCH 466/743] Release 6.9.1

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index f271142a..b6b8c22b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)
 	* Type __hostname: Add support for OpenWrt (Dennis Camera)
 	* New type: __dpkg_architecture (Matthias Stecher)

From a95eab77a5606f2552ac28c0b817945f72538c97 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 8 Nov 2020 15:18:04 +0100
Subject: [PATCH 467/743] __locale: add state explorer

.. so it doesn't execute code all the time.
---
 cdist/conf/type/__locale/explorer/state | 36 +++++++++++++++++++++++++
 cdist/conf/type/__locale/gencode-remote | 15 ++++++++---
 2 files changed, 47 insertions(+), 4 deletions(-)
 create mode 100755 cdist/conf/type/__locale/explorer/state

diff --git a/cdist/conf/type/__locale/explorer/state b/cdist/conf/type/__locale/explorer/state
new file mode 100755
index 00000000..4494fcbc
--- /dev/null
+++ b/cdist/conf/type/__locale/explorer/state
@@ -0,0 +1,36 @@
+#!/bin/sh -e
+# __locale/explorer/state
+#
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Check if the locale is already installed on the system.
+# Outputs 'present' or 'absent' depending if the locale exists.
+#
+
+
+# Get user-defined locale
+# locale name is echoed differently than the user propably set it (for UTF-8)
+locale="$(echo "$__object_id" | sed 's/UTF-8/utf8/')"
+
+# Check if the given locale exists on the system
+if localedef --list-archive | grep -qFx "$locale"; then
+    echo present
+else
+    echo absent
+fi
diff --git a/cdist/conf/type/__locale/gencode-remote b/cdist/conf/type/__locale/gencode-remote
index 1feb9884..4639cef8 100755
--- a/cdist/conf/type/__locale/gencode-remote
+++ b/cdist/conf/type/__locale/gencode-remote
@@ -23,6 +23,15 @@
 
 locale="$__object_id"
 
+state_is=$(cat "$__object/explorer/state")
+state_should=$(cat "$__object/parameter/state")
+
+# short circuit if there is nothing to do
+if [ "$state_is" = "$state_should" ]; then
+    exit 0
+fi
+
+
 # Hardcoded, create a pull request with
 # branching on $os in case it is at another location
 alias=/usr/share/locale/locale.alias
@@ -35,8 +44,6 @@ charmap=$(echo "$locale" | cut -d . -f 2)
 # W-T-F!
 locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
 
-state=$(cat "$__object/parameter/state")
-
 os=$(cat "$__global/explorer/os")
 
 # Nothing to be done on alpine
@@ -46,7 +53,7 @@ case "$os" in
         ;;
 esac
 
-case "$state" in
+case "$state_should" in
     present)
         echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
     ;;
@@ -54,7 +61,7 @@ case "$state" in
         echo localedef --delete-from-archive "$locale_remove"
     ;;
     *)
-        echo "Unsupported state: $state" >&2
+        echo "Unsupported state: $state_should" >&2
         exit 1
     ;;
 esac

From 792b4b10762c094d861282e78953dfd29b203a41 Mon Sep 17 00:00:00 2001
From: ssrq <ungleich@ssrq-sds-fds.ch>
Date: Mon, 9 Nov 2020 12:08:54 +0100
Subject: [PATCH 468/743] Add missing 'config' command

---
 docs/src/cdist-best-practice.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/src/cdist-best-practice.rst b/docs/src/cdist-best-practice.rst
index 39ec453e..f7677276 100644
--- a/docs/src/cdist-best-practice.rst
+++ b/docs/src/cdist-best-practice.rst
@@ -200,15 +200,15 @@ of cdist:
 .. code-block:: sh
 
     # Singleton type without parameter
-    echo __ungleich_munin_server | cdist --initial-manifest - munin.panter.ch
+    echo __ungleich_munin_server | cdist config --initial-manifest - munin.panter.ch
 
     # Singleton type with parameter
     echo __ungleich_munin_node --allow 1.2.3.4 | \
-        cdist --initial-manifest - rails-19.panter.ch
+        cdist config --initial-manifest - rails-19.panter.ch
 
     # Normal type
     echo __file /tmp/stdintest --mode 0644 | \
-        cdist --initial-manifest - cdist-dev-01.ungleich.ch
+        cdist config --initial-manifest - cdist-dev-01.ungleich.ch
 
 
 Other content in cdist repository

From ba906510529549446952bb3abfd8b49b05da7ca0 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 11 Nov 2020 07:49:32 +0100
Subject: [PATCH 469/743] ++changelog

---
 docs/changelog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index b6b8c22b..9fa0926c 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,11 @@
 Changelog
 ---------
 
+next:
+	* Documentation: Fix examples in best practice (Dennis Camera)
+	* Type __locale: Add state explorer (Matthias Stecher)
+	* Core: Reorganize scripts, version generation (Ander Punnar)
+
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)
 	* Type __hostname: Add support for OpenWrt (Dennis Camera)

From 3e48ef9e11795bf1c8a59ae202ed28f58cff1753 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:09:26 +0100
Subject: [PATCH 470/743] [type/__hostname] Lint

- Error if expected environment variables are unset
- Always wrap variable expansions in {}
---
 cdist/conf/type/__hostname/gencode-remote |  82 ++++++------
 cdist/conf/type/__hostname/manifest       | 147 +++++++++++-----------
 2 files changed, 114 insertions(+), 115 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 02afcbfb..9447daa9 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -20,26 +20,27 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$(cat "$__global/explorer/os")
-name_running=$(cat "$__global/explorer/hostname")
-has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
+os=$(cat "${__global:?}/explorer/os")
+name_running=$(cat "${__global:?}/explorer/hostname")
+has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl")
 
 
-if test -s "$__object/parameter/name"
+if test -s "${__object:?}/parameter/name"
 then
-    name_should=$(cat "$__object/parameter/name")
+    name_should=$(cat "${__object:?}/parameter/name")
 else
-    case $os
+    case ${os}
     in
         # RedHat-derivatives and BSDs
-        centos|fedora|redhat|scientific|freebsd|macosx|netbsd|openbsd)
+        (centos|fedora|redhat|scientific|freebsd|macosx|netbsd|openbsd)
             # Hostname is FQDN
-            name_should="${__target_host}"
-        ;;
-        *)
+            name_should=${__target_host:?}
+            ;;
+        (*)
             # Hostname is only first component of FQDN
-            name_should="${__target_host%%.*}"
-        ;;
+            name_should=${__target_host:?}
+            name_should=${name_should%%.*}
+            ;;
     esac
 fi
 
@@ -47,46 +48,46 @@ fi
 ################################################################################
 # Check if the (running) hostname is already correct
 #
-test "$name_running" != "$name_should" || exit 0
+test "${name_running}" != "${name_should}" || exit 0
 
 
 ################################################################################
 # Setup hostname
 #
-echo 'changed' >>"$__messages_out"
+echo 'changed' >>"${__messages_out:?}"
 
 # Use the good old way to set the hostname.
-case $os
+case ${os}
 in
-    alpine|debian|devuan|ubuntu)
+    (alpine|debian|devuan|ubuntu)
         echo 'hostname -F /etc/hostname'
-    ;;
-    archlinux)
+        ;;
+    (archlinux)
         echo 'command -v hostnamectl >/dev/null 2>&1' \
-            "&& hostnamectl set-hostname '$name_should'" \
-            "|| hostname '$name_should'"
-    ;;
-    centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void)
-        echo "hostname '$name_should'"
-    ;;
-    openwrt)
-        echo "echo '$name_should' >/proc/sys/kernel/hostname"
-    ;;
-    macosx)
-        echo "scutil --set HostName '$name_should'"
-    ;;
-    solaris)
-        echo "uname -S '$name_should'"
-    ;;
-    slackware|suse|opensuse-leap)
+            "&& hostnamectl set-hostname '${name_should}'" \
+            "|| hostname '${name_should}'"
+        ;;
+    (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void)
+        echo "hostname '${name_should}'"
+        ;;
+    (openwrt)
+        echo "echo '${name_should}' >/proc/sys/kernel/hostname"
+        ;;
+    (macosx)
+        echo "scutil --set HostName '${name_should}'"
+        ;;
+    (solaris)
+        echo "uname -S '${name_should}'"
+        ;;
+    (slackware|suse|opensuse-leap)
         # We do not read from /etc/HOSTNAME, because the running
         # hostname is the first component only while the file contains
         # the FQDN.
-        echo "hostname '$name_should'"
-    ;;
-    *)
+        echo "hostname '${name_should}'"
+        ;;
+    (*)
         # Fall back to set the hostname using hostnamectl, if available.
-        if test -n "$has_hostnamectl"
+        if test -n "${has_hostnamectl}"
         then
             # Don't use hostnamectl as the primary means to set the hostname for
             # systemd systems, because it cannot be trusted to work reliably and
@@ -97,7 +98,8 @@ in
             echo "test \"\$(hostname)\" = \"\$(cat /etc/hostname)\"" \
                  " || hostname -F /etc/hostname"
         else
-            printf "echo 'Unsupported OS: %s' >&2\nexit 1\n" "$os"
+            printf "echo 'Unsupported OS: %s' >&2\n" "${os}"
+            printf 'exit 1\n'
         fi
-    ;;
+        ;;
 esac
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index bf8a331c..125bb2fb 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -20,69 +20,65 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-not_supported() {
-    echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
-    echo "Please contribute an implementation for it if you can." >&2
-    exit 1
-}
-
 set_hostname_systemd() {
     echo "$1" | __file /etc/hostname --source -
 }
 
-os=$(cat "$__global/explorer/os")
-os_version=$(cat "$__global/explorer/os_version")
-os_major=$(echo "$os_version" | grep -o '^[0-9][0-9]*' || true)
+os=$(cat "${__global:?}/explorer/os")
+os_version=$(cat "${__global:?}/explorer/os_version")
+os_major=$(echo "${os_version}" | grep -o '^[0-9][0-9]*' || true)
 
-max_len=$(cat "$__object/explorer/max_len")
-has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
+max_len=$(cat "${__object:?}/explorer/max_len")
+has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl")
 
-if test -s "$__object/parameter/name"
+if test -s "${__object:?}/parameter/name"
 then
-    name_should=$(cat "$__object/parameter/name")
+    name_should=$(cat "${__object:?}/parameter/name")
 else
-    case $os
+    case ${os}
     in
         # RedHat-derivatives and BSDs
-        centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware)
+        (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware)
             # Hostname is FQDN
-            name_should="${__target_host}"
-        ;;
-        suse|opensuse-leap)
+            name_should=${__target_host:?}
+            ;;
+        (suse|opensuse-leap)
+            name_should=${__target_host:?}
+
             # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
             # systemd does not. The running hostname is the first
             # component in both cases.
             # In versions before 15.x, the FQDN is stored in /etc/hostname.
-            if test -n "$has_hostnamectl" && test "$os_major" -ge 15 \
-                    && test "$os_major" -ne 42
+            if test -n "${has_hostnamectl}" \
+                    && test "${os_major}" -ge 15 \
+                    && test "${os_major}" -ne 42
             then
-                name_should="${__target_host%%.*}"
-            else
-                name_should="${__target_host}"
+                name_should=${name_should%%.*}
             fi
-        ;;
+            ;;
         *)
             # Hostname is only first component of FQDN on all other systems.
-            name_should="${__target_host%%.*}"
-        ;;
+            name_should=${__target_host:?}
+            name_should=${name_should%%.*}
+            ;;
     esac
 fi
 
-if test -n "$max_len" && test "$(printf '%s' "$name_should" | wc -c)" -gt "$max_len"
+if test -n "${max_len}" && test "$(printf '%s' "${name_should}" | wc -c)" -gt "${max_len}"
 then
     printf "Host name too long. Up to %u characters allowed.\n" "${max_len}" >&2
     exit 1
 fi
 
-case $os
+case ${os}
 in
-    alpine|debian|devuan|ubuntu|void)
-        echo "$name_should" | __file /etc/hostname --source -
-    ;;
-    archlinux)
-        if test -n "$has_hostnamectl"
+    (alpine|debian|devuan|ubuntu|void)
+        echo "${name_should}" | __file /etc/hostname --source -
+        ;;
+    (archlinux)
+        if test -n "${has_hostnamectl}"
         then
-            set_hostname_systemd "$name_should"
+            set_hostname_systemd "${name_should}"
         else
             echo 'Ancient ArchLinux variants without hostnamectl are not supported.' >&2
             exit 1
@@ -97,8 +93,8 @@ in
             #     --value "\"$name_should\""
         fi
         ;;
-    centos|fedora|redhat|scientific)
-        if test -z "$has_hostnamectl"
+    (centos|fedora|redhat|scientific)
+        if test -z "${has_hostnamectl}"
         then
             # Only write to /etc/sysconfig/network on non-systemd versions.
             # On systemd-based versions this entry is ignored.
@@ -106,63 +102,62 @@ in
                 --file /etc/sysconfig/network \
                 --delimiter '=' --exact_delimiter \
                 --key HOSTNAME \
-                --value "\"$name_should\""
+                --value "\"${name_should}\""
         else
-            set_hostname_systemd "$name_should"
+            set_hostname_systemd "${name_should}"
         fi
-    ;;
-    gentoo)
+        ;;
+    (gentoo)
         # Only write to /etc/conf.d/hostname on OpenRC-based installations.
         # On systemd use hostnamectl(1) in gencode-remote.
-        if test -z "$has_hostnamectl"
+        if test -z "${has_hostnamectl}"
         then
             __key_value '/etc/conf.d/hostname:hostname' \
                 --file /etc/conf.d/hostname \
                 --delimiter '=' --exact_delimiter \
                 --key 'hostname' \
-                --value "\"$name_should\""
+                --value "\"${name_should}\""
         else
             set_hostname_systemd "$name_should"
         fi
-    ;;
-    freebsd)
+        ;;
+    (freebsd)
         __key_value '/etc/rc.conf:hostname' \
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
-            --value "\"$name_should\""
-    ;;
-    macosx)
+            --value "\"${name_should}\""
+        ;;
+    (macosx)
         # handled in gencode-remote
-        :
-    ;;
-    netbsd)
+        ;;
+    (netbsd)
         __key_value '/etc/rc.conf:hostname' \
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
-            --value "\"$name_should\""
+            --value "\"${name_should}\""
 
         # To avoid confusion, ensure that the hostname is only stored once.
         __file /etc/myname --state absent
-    ;;
-    openbsd)
-        echo "$name_should" | __file /etc/myname --source -
-    ;;
-    openwrt)
-        __uci system.@system[0].hostname --value "$name_should"
+        ;;
+    (openbsd)
+        echo "${name_should}" | __file /etc/myname --source -
+        ;;
+    (openwrt)
+        __uci system.@system[0].hostname --value "${name_should}"
             # --transaction hostname
-    ;;
-    slackware)
+        ;;
+    (slackware)
         # We write the FQDN into /etc/HOSTNAME.  But /etc/rc.d/rc.M will only
         # read the first component from this file and set it as the running
         # hostname on boot.
-        echo "$name_should" | __file /etc/HOSTNAME --source -
-    ;;
-    solaris)
-        echo "$name_should" | __file /etc/nodename --source -
-    ;;
-    suse|opensuse-leap)
+        echo "${name_should}" | __file /etc/HOSTNAME --source -
+        ;;
+    (solaris)
+        echo "${name_should}" | __file /etc/nodename --source -
+        ;;
+    (suse|opensuse-leap)
         # Modern SuSE provides /etc/HOSTNAME as a symlink for
         # backwards-compatibility. Unfortunately it cannot be used
         # here as __file does not follow the symlink.
@@ -171,23 +166,25 @@ in
         # not work correctly on openSUSE 12.x which provides
         # hostnamectl but not /etc/hostname.
 
-        if test -n "$has_hostnamectl" -a "$os_major" -gt 12
+        if test -n "${has_hostnamectl}" -a "${os_major}" -gt 12
         then
-            hostname_file='/etc/hostname'
+            hostname_file=/etc/hostname
         else
-            hostname_file='/etc/HOSTNAME'
+            hostname_file=/etc/HOSTNAME
         fi
 
-        echo "$name_should" | __file "$hostname_file" --source -
-    ;;
-    *)
+        echo "${name_should}" | __file "${hostname_file}" --source -
+        ;;
+    (*)
         # On other operating systems we fall back to systemd's
         # hostnamectl if available…
-        if test -n "$has_hostnamectl"
+        if test -n "${has_hostnamectl}"
         then
-            set_hostname_systemd "$name_should"
+            set_hostname_systemd "${name_should}"
         else
-            not_supported
+            echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
+            echo "Please contribute an implementation for it if you can." >&2
+            exit 1
         fi
-    ;;
+        ;;
 esac

From 702f3eba4f8f88b634a010b377d077d501a6f0e9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:15:02 +0100
Subject: [PATCH 471/743] [type/__hostname] Remove opensuse-leap OS string
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

everything should be suse now…
---
 cdist/conf/type/__hostname/gencode-remote | 2 +-
 cdist/conf/type/__hostname/manifest       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 9447daa9..c1a97ac8 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -79,7 +79,7 @@ in
     (solaris)
         echo "uname -S '${name_should}'"
         ;;
-    (slackware|suse|opensuse-leap)
+    (slackware|suse)
         # We do not read from /etc/HOSTNAME, because the running
         # hostname is the first component only while the file contains
         # the FQDN.
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 125bb2fb..d044072b 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -42,7 +42,7 @@ else
             # Hostname is FQDN
             name_should=${__target_host:?}
             ;;
-        (suse|opensuse-leap)
+        (suse)
             name_should=${__target_host:?}
 
             # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
@@ -157,7 +157,7 @@ in
     (solaris)
         echo "${name_should}" | __file /etc/nodename --source -
         ;;
-    (suse|opensuse-leap)
+    (suse)
         # Modern SuSE provides /etc/HOSTNAME as a symlink for
         # backwards-compatibility. Unfortunately it cannot be used
         # here as __file does not follow the symlink.

From 87a0d91587568594a616c7f3588bc74106312761 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:20:18 +0100
Subject: [PATCH 472/743] [type/__hostname] Fix OS version detection for SuSE
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

everything should be suse now…
---
 cdist/conf/type/__hostname/manifest | 39 ++++++++++++++++-------------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index d044072b..b80aa2ef 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -25,8 +25,6 @@ set_hostname_systemd() {
 }
 
 os=$(cat "${__global:?}/explorer/os")
-os_version=$(cat "${__global:?}/explorer/os_version")
-os_major=$(echo "${os_version}" | grep -o '^[0-9][0-9]*' || true)
 
 max_len=$(cat "${__object:?}/explorer/max_len")
 has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl")
@@ -38,24 +36,10 @@ else
     case ${os}
     in
         # RedHat-derivatives and BSDs
-        (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware)
+        (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware|suse)
             # Hostname is FQDN
             name_should=${__target_host:?}
             ;;
-        (suse)
-            name_should=${__target_host:?}
-
-            # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
-            # systemd does not. The running hostname is the first
-            # component in both cases.
-            # In versions before 15.x, the FQDN is stored in /etc/hostname.
-            if test -n "${has_hostnamectl}" \
-                    && test "${os_major}" -ge 15 \
-                    && test "${os_major}" -ne 42
-            then
-                name_should=${name_should%%.*}
-            fi
-            ;;
         *)
             # Hostname is only first component of FQDN on all other systems.
             name_should=${__target_host:?}
@@ -158,6 +142,27 @@ in
         echo "${name_should}" | __file /etc/nodename --source -
         ;;
     (suse)
+        if test -s "${__global:?}/explorer/os_release"
+        then
+            # shellcheck source=/dev/null
+            os_version=$(. "${__global:?}/explorer/os_release" && echo "${VERSION}")
+        else
+            os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global:?}/explorer/os_version")
+        fi
+        os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
+
+        # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
+        # systemd does not. The running hostname is the first
+        # component in both cases.
+        # In versions before 15.x, the FQDN is stored in /etc/hostname.
+        if test -n "${has_hostnamectl}" \
+                && test "${os_major}" -ge 15 \
+                && test "${os_major}" -ne 42
+        then
+            # strip away everything but the first part from $name_should
+            name_should=${name_should%%.*}
+        fi
+
         # Modern SuSE provides /etc/HOSTNAME as a symlink for
         # backwards-compatibility. Unfortunately it cannot be used
         # here as __file does not follow the symlink.

From 21dd500c05c91c4927b2a4166690da65b1c98f97 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:44:44 +0100
Subject: [PATCH 473/743] Make pycodestyle pipeline happy

---
 cdist/__init__.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index 1c60ae0f..44366cd0 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -33,14 +33,15 @@ try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
-    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+    cdist_dir = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), os.pardir))
     if os.path.isdir(os.path.join(cdist_dir, '.git')):
         try:
             VERSION = subprocess.check_output(
                 ['git', 'describe', '--always'],
                 cwd=cdist_dir,
                 universal_newlines=True)
-        except:
+        except Exception:
             pass
 
 BANNER = """

From e2d4f8037ab7d30187e4d153aa25ef36e8afc91c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:45:05 +0100
Subject: [PATCH 474/743] [bin/cdist-build-helper] Fix paths to ex scripts/
 scripts

---
 bin/cdist-build-helper | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper
index bdef0dbb..fb8a67a4 100755
--- a/bin/cdist-build-helper
+++ b/bin/cdist-build-helper
@@ -45,7 +45,7 @@ usage() {
         shellcheck-manifests
         shellcheck-local-gencodes
         shellcheck-remote-gencodes
-        shellcheck-scripts
+        shellcheck-bin
         shellcheck-gencodes
         shellcheck-types
         shellcheck
@@ -100,7 +100,7 @@ case "$option" in
                     if (\$0 ~ /^$end/) {
                         exit
                     } else {
-                        print \$0 
+                        print \$0
                     }
                 }
             }" "$basedir/docs/changelog"
@@ -135,7 +135,7 @@ case "$option" in
 
         version=$1; shift
 
-        ( 
+        (
         cat << eof
 Subject: cdist $version has been released
 
@@ -336,7 +336,7 @@ eof
         make docs-clean
         make docs
 
-        ############################################################# 
+        #############################################################
         # Everything green, let's do the release
 
         # Tag the current commit
@@ -405,7 +405,7 @@ eof
     ;;
 
     pycodestyle|pep8)
-        pycodestyle "${basedir}" "${basedir}/scripts/cdist"
+        pycodestyle "${basedir}" "${basedir}/bin/cdist"
     ;;
 
     check-pycodestyle)
@@ -460,9 +460,10 @@ eof
         test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
-    shellcheck-scripts)
+    # NOTE: shellcheck-scripts is kept for compatibility
+    shellcheck-bin|shellcheck-scripts)
         # shellcheck disable=SC2086
-        ${SHELLCHECKCMD} scripts/cdist-dump scripts/cdist-new-type > "${SHELLCHECKTMP}"
+        ${SHELLCHECKCMD} bin/cdist-dump bin/cdist-new-type > "${SHELLCHECKTMP}"
         test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
@@ -480,7 +481,7 @@ eof
     shellcheck)
         "$0" shellcheck-global-explorers || exit 1
         "$0" shellcheck-types || exit 1
-        "$0" shellcheck-scripts || exit 1
+        "$0" shellcheck-bin || exit 1
     ;;
 
     shellcheck-type-files)

From f82e0167aaf295b8451a07fa931506a1290efcdc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:49:04 +0100
Subject: [PATCH 475/743] [.gitlab-ci.yml] Make version before other targets

---
 .gitlab-ci.yml | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e48355ea..a4bc67aa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,20 +1,23 @@
+---
+image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
+
 stages:
     - test
 
-image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
+before_script:
+    - ./bin/cdist-build-helper version
 
-unit_tests:
+shellcheck:
     stage: test
     script:
-        - ./bin/cdist-build-helper version
-        - ./bin/cdist-build-helper test
+        - ./bin/cdist-build-helper shellcheck
 
 pycodestyle:
     stage: test
     script:
         - ./bin/cdist-build-helper pycodestyle
 
-shellcheck:
+unit_tests:
     stage: test
     script:
-        - ./bin/cdist-build-helper shellcheck
+        - ./bin/cdist-build-helper test

From 0ee3fda94deff1461ac488d218a2af98057675c5 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 15:04:51 +0100
Subject: [PATCH 476/743] Fix paths to cdist executable

---
 cdist/integration.py          | 5 ++---
 cdist/test/__init__.py        | 2 +-
 cdist/test/config/__init__.py | 4 ++--
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/cdist/integration.py b/cdist/integration.py
index 03e4167d..17b65f09 100644
--- a/cdist/integration.py
+++ b/cdist/integration.py
@@ -54,13 +54,12 @@ _mydir = os.path.dirname(__file__)
 def find_cdist_exec():
     """Search cdist executable starting from local lib directory.
 
-    Detect if ../scripts/cdist (from local lib direcotry) exists and
+    Detect if ../bin/cdist (from local lib directory) exists and
     if it is executable. If not then try to find cdist exec path in
     os.get_exec_path() entries. If no cdist path is found rasie
     cdist.Error.
     """
-    cdist_path = os.path.abspath(os.path.join(_mydir, '..', 'scripts',
-                                              'cdist'))
+    cdist_path = os.path.abspath(os.path.join(_mydir, '..', 'bin', 'cdist'))
     if os.access(cdist_path, os.X_OK):
         return cdist_path
     cdist_path = find_cdist_exec_in_path()
diff --git a/cdist/test/__init__.py b/cdist/test/__init__.py
index faa3686a..16a311e4 100644
--- a/cdist/test/__init__.py
+++ b/cdist/test/__init__.py
@@ -26,7 +26,7 @@ import tempfile
 cdist_base_path = os.path.abspath(
     os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../"))
 
-cdist_exec_path = os.path.join(cdist_base_path, "scripts/cdist")
+cdist_exec_path = os.path.join(cdist_base_path, "bin/cdist")
 
 global_fixtures_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                    "fixtures"))
diff --git a/cdist/test/config/__init__.py b/cdist/test/config/__init__.py
index 0ed614b1..c405207c 100644
--- a/cdist/test/config/__init__.py
+++ b/cdist/test/config/__init__.py
@@ -202,7 +202,7 @@ class ConfigRunTestCase(test.CdistTestCase):
             host_dir_name=self.hostdir,
             # exec_path can not derivated from sys.argv in case of unittest
             exec_path=os.path.abspath(os.path.join(
-                my_dir, '../../../scripts/cdist')),
+                my_dir, '../../../bin/cdist')),
             initial_manifest=os.path.join(fixtures,
                                           'manifest/dryrun_manifest'),
             add_conf_dirs=[fixtures])
@@ -219,7 +219,7 @@ class ConfigRunTestCase(test.CdistTestCase):
             base_root_path=self.host_base_path,
             host_dir_name=self.hostdir,
             exec_path=os.path.abspath(os.path.join(
-                my_dir, '../../../scripts/cdist')),
+                my_dir, '../../../bin/cdist')),
             initial_manifest=os.path.join(
                 fixtures, 'manifest/init-deps-resolver'),
             add_conf_dirs=[fixtures])

From c39eb1dbce281ed6f64f804a16fb28e5c8dceccd Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 15:16:33 +0100
Subject: [PATCH 477/743] [cdist.emulator] Fix setting of log level (tests OK)

---
 cdist/emulator.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cdist/emulator.py b/cdist/emulator.py
index 60b94880..a2bdc3d4 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -120,17 +120,18 @@ class Emulator:
                 level = logging.WARNING
         else:
             level = logging.WARNING
+        self.log = logging.getLogger(self.target_host[0])
         try:
             logging.root.setLevel(level)
+            self.log.setLevel(level)
         except (ValueError, TypeError):
             # if invalid __cdist_log_level value
             logging.root.setLevel(logging.WARNING)
+            self.log.setLevel(logging.WARNING)
 
         colored_log = self.env.get('__cdist_colored_log', 'false')
         cdist.log.CdistFormatter.USE_COLORS = colored_log == 'true'
 
-        self.log = logging.getLogger(self.target_host[0])
-
     def commandline(self):
         """Parse command line"""
 

From 2f70a8b540f8d4c283029207e98f7883557338f0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 15:25:46 +0100
Subject: [PATCH 478/743] [bin/cdist-build-helper] Keep going in shellcheck
 targets

---
 bin/cdist-build-helper | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper
index fb8a67a4..0380b3f8 100755
--- a/bin/cdist-build-helper
+++ b/bin/cdist-build-helper
@@ -468,20 +468,26 @@ eof
     ;;
 
     shellcheck-gencodes)
-        "$0" shellcheck-local-gencodes || exit 1
-        "$0" shellcheck-remote-gencodes || exit 1
+        errors=false
+        "$0" shellcheck-local-gencodes || errors=true
+        "$0" shellcheck-remote-gencodes || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck-types)
-        "$0" shellcheck-type-explorers || exit 1
-        "$0" shellcheck-manifests || exit 1
-        "$0" shellcheck-gencodes || exit 1
+        errors=false
+        "$0" shellcheck-type-explorers || errors=true
+        "$0" shellcheck-manifests || errors=true
+        "$0" shellcheck-gencodes || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck)
-        "$0" shellcheck-global-explorers || exit 1
-        "$0" shellcheck-types || exit 1
-        "$0" shellcheck-bin || exit 1
+        errors=false
+        "$0" shellcheck-global-explorers || errors=true
+        "$0" shellcheck-types || errors=true
+        "$0" shellcheck-bin || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck-type-files)
@@ -491,8 +497,10 @@ eof
     ;;
 
     shellcheck-with-files)
-        "$0" shellcheck || exit 1
-        "$0" shellcheck-type-files || exit 1
+        errors=false
+        "$0" shellcheck || errors=true
+        "$0" shellcheck-type-files || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck-build-helper)

From ebf471e8d0216cc81928fcdff1fe159584187808 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 13 Nov 2020 02:32:45 +0100
Subject: [PATCH 479/743] [type/__hwclock] Add new type

---
 .../conf/type/__hwclock/explorer/adjtime_mode |  28 +++
 .../__hwclock/explorer/timedatectl_localrtc   |  27 +++
 cdist/conf/type/__hwclock/gencode-remote      |  62 +++++
 cdist/conf/type/__hwclock/man.rst             |  63 +++++
 cdist/conf/type/__hwclock/manifest            | 222 ++++++++++++++++++
 cdist/conf/type/__hwclock/parameter/required  |   1 +
 cdist/conf/type/__hwclock/singleton           |   0
 7 files changed, 403 insertions(+)
 create mode 100755 cdist/conf/type/__hwclock/explorer/adjtime_mode
 create mode 100755 cdist/conf/type/__hwclock/explorer/timedatectl_localrtc
 create mode 100755 cdist/conf/type/__hwclock/gencode-remote
 create mode 100644 cdist/conf/type/__hwclock/man.rst
 create mode 100755 cdist/conf/type/__hwclock/manifest
 create mode 100644 cdist/conf/type/__hwclock/parameter/required
 create mode 100644 cdist/conf/type/__hwclock/singleton

diff --git a/cdist/conf/type/__hwclock/explorer/adjtime_mode b/cdist/conf/type/__hwclock/explorer/adjtime_mode
new file mode 100755
index 00000000..2b27bedc
--- /dev/null
+++ b/cdist/conf/type/__hwclock/explorer/adjtime_mode
@@ -0,0 +1,28 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Prints the clock mode read from the /etc/adjtime file, if present.
+#
+
+# not all operating systems use an adjfile
+test -f /etc/adjtime || exit 0
+
+# 3rd line is clock mode
+# adjtime(5) https://man7.org/linux/man-pages/man5/adjtime.5.html
+sed -n 3p /etc/adjtime
diff --git a/cdist/conf/type/__hwclock/explorer/timedatectl_localrtc b/cdist/conf/type/__hwclock/explorer/timedatectl_localrtc
new file mode 100755
index 00000000..8239122e
--- /dev/null
+++ b/cdist/conf/type/__hwclock/explorer/timedatectl_localrtc
@@ -0,0 +1,27 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Prints the LocalRTC property using timedatectl on systemd-based systems.
+#
+
+command -v timedatectl >/dev/null 2>&1 || exit 0
+
+# NOTE: Older versions of timedatectl do not support `timedatectl show'
+timedatectl --no-pager status \
+| awk -F': ' '$1 ~ "RTC in local TZ$" { sub(/[ \t]*$/, "", $2); print $2 }'
diff --git a/cdist/conf/type/__hwclock/gencode-remote b/cdist/conf/type/__hwclock/gencode-remote
new file mode 100755
index 00000000..5995fb23
--- /dev/null
+++ b/cdist/conf/type/__hwclock/gencode-remote
@@ -0,0 +1,62 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+mode=$(cat "${__object:?}/parameter/mode")
+
+timedatectl_localrtc=$(cat "${__object:?}/explorer/timedatectl_localrtc")
+adjtime_mode=$(cat "${__object:?}/explorer/adjtime_mode")
+
+
+case ${mode}
+in
+	(localtime)
+		adjtime_str=LOCAL
+		local_rtc_str=yes
+		;;
+	(UTC|utc)
+		adjtime_str=UTC
+		local_rtc_str=no
+		;;
+	(*)
+		printf 'Invalid value for --mode: %s\n' "${mode}" >&2
+		printf 'Acceptable values are: localtime, utc.\n' >&2
+		exit 1
+esac
+
+
+if test -n "${timedatectl_localrtc}"
+then
+	# systemd
+	timedatectl_should=${local_rtc_str}
+	if test "${timedatectl_localrtc}" != "${timedatectl_should}"
+	then
+		printf 'timedatectl set-local-rtc %s\n' "${timedatectl_should}"
+	fi
+elif test -n "${adjtime_mode}"
+then
+	# others (update /etc/adjtime if present)
+	if test "${adjtime_mode}" != "${adjtime_str}"
+	then
+		# Update /etc/adjtime (3rd line is clock mode)
+		# adjtime(5) https://man7.org/linux/man-pages/man5/adjtime.5.html
+		# FIXME: Should maybe add third line if adjfile only contains two lines
+		printf "sed -i '3c\\\\\\n%s\\n' /etc/adjtime\\n" "${adjtime_str}"
+	fi
+fi
diff --git a/cdist/conf/type/__hwclock/man.rst b/cdist/conf/type/__hwclock/man.rst
new file mode 100644
index 00000000..65eb648f
--- /dev/null
+++ b/cdist/conf/type/__hwclock/man.rst
@@ -0,0 +1,63 @@
+cdist-type__hwclock(7)
+======================
+
+NAME
+----
+cdist-type__hwclock - Manage the hardware real time clock.
+
+
+DESCRIPTION
+-----------
+This type can be used to control how the hardware clock is used by the operating
+system.
+
+
+REQUIRED PARAMETERS
+-------------------
+mode
+    What mode the hardware clock is in.
+
+    Acceptable values:
+
+    localtime
+        The hardware clock is set to local time (common for systems also running
+        Windows.)
+    UTC
+        The hardware clock is set to UTC (common on UNIX systems.)
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Make the operating system treat the time read from the hwclock as UTC.
+    __hwclock --mode UTC
+
+
+SEE ALSO
+--------
+:strong:`hwclock`\ (8)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__hwclock/manifest b/cdist/conf/type/__hwclock/manifest
new file mode 100755
index 00000000..7d9ab88f
--- /dev/null
+++ b/cdist/conf/type/__hwclock/manifest
@@ -0,0 +1,222 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# TODO: Consider supporting BADYEAR
+
+os=$(cat "${__global:?}/explorer/os")
+mode=$(cat "${__object:?}/parameter/mode")
+
+has_systemd_timedatectl=$(test -s "${__object:?}/explorer/timedatectl_localrtc" && echo true || echo false)
+
+
+case ${mode}
+in
+	(localtime)
+		local_clock=true
+		;;
+	(UTC|utc)
+		local_clock=false
+		;;
+	(*)
+		printf 'Invalid value for --mode: %s\n' "${mode}" >&2
+		printf 'Acceptable values are: UTC, localtime.\n' >&2
+		exit 1
+esac
+
+
+case ${os}
+in
+	(alpine|gentoo)
+		if ! $has_systemd_timedatectl
+		then
+			# NOTE: Gentoo also supports systemd, in which case /etc/conf.d is
+			#       not used. So we check for systemd presence here and only
+			#       update /etc/conf.d if systemd is not installed.
+			# https://wiki.gentoo.org/wiki/System_time#Hardware_clock
+
+			export CDIST_ORDER_DEPENDENCY=true
+			__file /etc/conf.d/hwclock --state present \
+				--owner root --group root --mode 0644
+			__key_value /etc/conf.d/hwclock:clock \
+				--file /etc/conf.d/hwclock \
+				--key clock \
+				--delimiter '=' --exact_delimiter \
+				--value "\"$($local_clock && echo local || echo UTC)\""
+			unset CDIST_ORDER_DEPENDENCY
+		fi
+		;;
+	(centos|fedora|redhat|scientific)
+		os_version=$(cat "${__global:?}/explorer/os_version")
+		os_major=$(expr "${os_version}" : '.* release \([0-9]*\)')
+		case ${os}
+		in
+			(centos|scientific)
+				update_sysconfig=$(test "${os_major}" -lt 6 && echo true || echo false)
+				;;
+			(fedora)
+				update_sysconfig=$(test "${os_major}" -lt 10 && echo true || echo false)
+				;;
+			(redhat|*)
+				case ${os_version}
+				in
+					('Red Hat Enterprise Linux'*)
+						update_sysconfig=$(test "${os_major}" -lt 6 && echo true || echo false)
+						;;
+					('Red Hat Linux'*)
+						update_sysconfig=true
+						;;
+					(*)
+						printf 'Could not determine Red Hat distribution.\n' >&2
+						printf "Please contribute an implementation for it if you can.\n" >&2
+						exit 1
+						;;
+				esac
+				;;
+		esac
+
+		if ${update_sysconfig:?}
+		then
+			export CDIST_ORDER_DEPENDENCY=true
+			__file /etc/sysconfig/clock --state present \
+				--owner root --group root --mode 0644
+			__key_value /etc/sysconfig/clock:UTC \
+				--file /etc/sysconfig/clock \
+				--key UTC \
+				--delimiter '=' --exact_delimiter \
+				--value "$($local_clock && echo false || echo true)"
+			unset CDIST_ORDER_DEPENDENCY
+		fi
+		;;
+	(debian|devuan|ubuntu)
+		os_major=$(sed 's/[^0-9].*$//' "${__global:?}/explorer/os_version")
+
+		case ${os}
+		in
+			(debian)
+				if test "${os_major}" -ge 7
+				then
+					update_rcS=false
+				elif test "${os_major}" -ge 3
+				then
+					update_rcS=true
+				else
+					# Debian 2.2 should be supportable using rcS.
+					# Debian 2.1 uses the ancient GMT key.
+					# Debian 1.3 does not have rcS.
+					printf "Your operating system (Debian %s) is currently not supported by this type (%s)\n" \
+						"$(cat "${__global:?}/explorer/os_version")" "${__type##*/}" >&2
+					printf "Please contribute an implementation for it if you can.\n" >&2
+					exit 1
+				fi
+				;;
+			(devuan)
+				update_rcS=false
+				;;
+			(ubuntu)
+				update_rcS=$(test "${os_major}" -lt 16 && echo true || echo false)
+				;;
+		esac
+
+		if ${update_rcS}
+		then
+			export CDIST_ORDER_DEPENDENCY=true
+			__file /etc/default/rcS --state present \
+				--owner root --group root --mode 0644
+			__key_value /etc/default/rcS:UTC \
+				--file /etc/default/rcS \
+				--key UTC \
+				--delimiter '=' --exact_delimiter \
+				--value "$($local_clock && echo no || echo yes)"
+			unset CDIST_ORDER_DEPENDENCY
+		fi		  
+		;;
+	(freebsd)
+		# cf. adjkerntz(8)
+		__file /etc/wall_cmos_clock \
+			--state "$($local_clock && echo present || echo absent)" \
+			--owner root --group wheel --mode 0444
+		;;
+	(netbsd)
+		# https://wiki.netbsd.org/guide/boot/#index9h2
+		__key_value /etc/rc.conf:rtclocaltime \
+			--file /etc/rc.conf \
+			--key rtclocaltime \
+			--delimiter '=' --exact_delimiter \
+			--value "$($local_clock && echo YES || echo NO)"
+		;;
+	(slackware)
+		__file /etc/hardwareclock --owner root --group root --mode 0644 \
+			--source - <<-EOF
+		# /etc/hardwareclock
+		#
+		# Tells how the hardware clock time is stored.
+		# This file is managed by cdist.
+
+		$($local_clock && echo localtime || echo UTC)
+		EOF
+		;;
+	(suse)
+		if test -s "${__global:?}/explorer/os_release"
+		then
+			# shellcheck source=/dev/null
+			os_version=$(. "${__global:?}/explorer/os_release" && echo "${VERSION}")
+		else
+			os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global:?}/explorer/os_version")
+		fi
+		os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
+
+		# TODO: Consider using `yast2 timezone set hwclock' instead
+		if expr "${os_major}" \< 12
+		then
+			# Starting with SuSE 12 (first systemd-based version)
+			# /etc/sysconfig/clock does not contain the HWCLOCK line
+			# anymore.
+			# With SuSE 13, it has been reduced to TIMEZONE configuration.
+			__key_value /etc/sysconfig/clock:HWCLOCK \
+				--file /etc/sysconfig/clock \
+				--delimiter '=' --exact_delimiter \
+				--key HWCLOCK \
+				--value "$($local_clock && echo '"--localtime"' || echo '"-u"')"
+		fi
+		;;
+	(void)
+		export CDIST_ORDER_DEPENDENCY=true
+		__file /etc/rc.conf \
+			--owner root --group root --mode 0644 \
+			--state present
+		__key_value /etc/rc.conf:HARDWARECLOCK \
+			--file /etc/rc.conf \
+			--delimiter '=' --exact_delimiter \
+			--key HARDWARECLOCK \
+			--value "\"$($local_clock && echo localtime || echo UTC)\""
+		unset CDIST_ORDER_DEPENDENCY
+		;;
+	(*)
+		if ! $has_systemd_timedatectl
+		then
+			printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2
+			printf "Please contribute an implementation for it if you can.\n" >&2
+			exit 1
+		fi
+		;;
+esac
+
+# NOTE: timedatectl set-local-rtc for systemd is in gencode-remote
+# NOTE: /etc/adjtime is also updated in gencode-remote
diff --git a/cdist/conf/type/__hwclock/parameter/required b/cdist/conf/type/__hwclock/parameter/required
new file mode 100644
index 00000000..17ab372f
--- /dev/null
+++ b/cdist/conf/type/__hwclock/parameter/required
@@ -0,0 +1 @@
+mode
diff --git a/cdist/conf/type/__hwclock/singleton b/cdist/conf/type/__hwclock/singleton
new file mode 100644
index 00000000..e69de29b

From a07a45887136105cc8be316307805ceaecdd4cf3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 13 Nov 2020 06:43:01 +0100
Subject: [PATCH 480/743] ++changelog

---
 docs/changelog | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 9fa0926c..ff411a46 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,7 +4,8 @@ Changelog
 next:
 	* Documentation: Fix examples in best practice (Dennis Camera)
 	* Type __locale: Add state explorer (Matthias Stecher)
-	* Core: Reorganize scripts, version generation (Ander Punnar)
+	* Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera)
+	* New type: __hwclock (Dennis Camera)
 
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)

From 50927527863fd382f5200287a808d8b36028ff87 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 14 Nov 2020 09:59:30 +0100
Subject: [PATCH 481/743] Update build helper script in .gitattributes

---
 .gitattributes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitattributes b/.gitattributes
index 45c10d7b..01d20f30 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -4,5 +4,5 @@
 docs/speeches export-ignore
 docs/video export-ignore
 docs/src/man7 export-ignore
-bin/build-helper export-ignore
+bin/cdist-build-helper export-ignore
 README-maintainers export-ignore

From 76aa00b12e9b675c797e997a06ec7597c53a85f3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 14 Nov 2020 10:13:58 +0100
Subject: [PATCH 482/743] Fix importing cdist module

Resolve #845.
---
 bin/cdist | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index 1f92f157..ddaffa7f 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -25,25 +25,23 @@ import logging
 import os
 import sys
 
-# try to import cdist and if that fails,
-# then add this file's parent dir to
-# module search path and try again.
-try:
-    import cdist
-except ModuleNotFoundError:
-    cdist_dir = os.path.realpath(
-        os.path.join(
-            os.path.dirname(os.path.realpath(__file__)),
-            os.pardir))
+# See if this file's parent is cdist module
+# and if so add it to module search path.
+cdist_dir = os.path.realpath(
+    os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        os.pardir))
+cdist_init_dir = os.path.join(cdist_dir, 'cdist', '__init__.py')
+if os.path.exists(cdist_init_dir):
     sys.path.insert(0, cdist_dir)
-    import cdist
 
-import cdist.argparse
-import cdist.banner
-import cdist.config
-import cdist.install
-import cdist.shell
-import cdist.inventory
+import cdist            # noqa 402
+import cdist.argparse   # noqa 402
+import cdist.banner     # noqa 402
+import cdist.config     # noqa 402
+import cdist.install    # noqa 402
+import cdist.shell      # noqa 402
+import cdist.inventory  # noqa 402
 
 
 def commandline():

From f75d4772090d3a9cfd798dd193cd324083a9b3a7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 13:45:12 +0100
Subject: [PATCH 483/743] Deprecate __locale and replace with __localedef

---
 cdist/conf/type/__locale/deprecated           |  1 +
 cdist/conf/type/__localedef/gencode-remote    | 60 +++++++++++++++++++
 cdist/conf/type/__localedef/man.rst           | 55 +++++++++++++++++
 cdist/conf/type/__localedef/manifest          | 41 +++++++++++++
 .../type/__localedef/parameter/default/state  |  1 +
 .../conf/type/__localedef/parameter/optional  |  1 +
 6 files changed, 159 insertions(+)
 create mode 100644 cdist/conf/type/__locale/deprecated
 create mode 100755 cdist/conf/type/__localedef/gencode-remote
 create mode 100644 cdist/conf/type/__localedef/man.rst
 create mode 100755 cdist/conf/type/__localedef/manifest
 create mode 100644 cdist/conf/type/__localedef/parameter/default/state
 create mode 100644 cdist/conf/type/__localedef/parameter/optional

diff --git a/cdist/conf/type/__locale/deprecated b/cdist/conf/type/__locale/deprecated
new file mode 100644
index 00000000..5a06b28e
--- /dev/null
+++ b/cdist/conf/type/__locale/deprecated
@@ -0,0 +1 @@
+This type is deprecated. Please use __localedef instead.
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
new file mode 100755
index 00000000..1feb9884
--- /dev/null
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -0,0 +1,60 @@
+#!/bin/sh -e
+#
+# 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Let localedef do the magic
+#
+
+locale="$__object_id"
+
+# Hardcoded, create a pull request with
+# branching on $os in case it is at another location
+alias=/usr/share/locale/locale.alias
+
+input=$(echo "$locale" | cut -d . -f 1)
+charmap=$(echo "$locale" | cut -d . -f 2)
+
+# Adding locale? The name is de_CH.UTF-8
+# Removing locale? The name is de_CH.utf8.
+# W-T-F!
+locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
+
+state=$(cat "$__object/parameter/state")
+
+os=$(cat "$__global/explorer/os")
+
+# Nothing to be done on alpine
+case "$os" in
+    alpine)
+        exit 0
+        ;;
+esac
+
+case "$state" in
+    present)
+        echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
+    ;;
+    absent)
+        echo localedef --delete-from-archive "$locale_remove"
+    ;;
+    *)
+        echo "Unsupported state: $state" >&2
+        exit 1
+    ;;
+esac
diff --git a/cdist/conf/type/__localedef/man.rst b/cdist/conf/type/__localedef/man.rst
new file mode 100644
index 00000000..0ccf484a
--- /dev/null
+++ b/cdist/conf/type/__localedef/man.rst
@@ -0,0 +1,55 @@
+cdist-type__localedef(7)
+========================
+
+NAME
+----
+cdist-type__localedef - Define and remove system locales
+
+
+DESCRIPTION
+-----------
+This cdist type allows you to define locales on the system using
+:strong:`localedef`\ (1) or remove them.
+On systems that don't support definition of new locales, the type will raise an
+error.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+   ``present`` or ``absent``. Defaults to ``present``.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Add locale de_CH.UTF-8
+    __localedef de_CH.UTF-8
+
+    # Same as above, but more explicit
+    __localedef de_CH.UTF-8 --state present
+
+    # Remove colourful British English
+    __localedef en_GB.UTF-8 --state absent
+
+
+SEE ALSO
+--------
+:strong:`locale`\ (1),
+:strong:`localedef`\ (1),
+:strong:`cdist-type__locale_system`\ (7)
+
+
+AUTHORS
+-------
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+
+
+COPYING
+-------
+Copyright \(C) 2013-2019 Nico Schottelius, 2020 Dennis Camera. Free use of this
+software is granted under the terms of the GNU General Public License version 3
+or later (GPLv3+).
diff --git a/cdist/conf/type/__localedef/manifest b/cdist/conf/type/__localedef/manifest
new file mode 100755
index 00000000..9f1e17ac
--- /dev/null
+++ b/cdist/conf/type/__localedef/manifest
@@ -0,0 +1,41 @@
+#!/bin/sh -e
+#
+# 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2015 David Hürlimann (david at ungleich.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Install required packages
+#
+
+os=$(cat "$__global/explorer/os")
+
+
+case "$os" in
+    debian|devuan)
+        # Debian needs a seperate package
+        __package locales --state present
+    ;;
+    archlinux|suse|ubuntu|scientific|centos|alpine)
+        :
+    ;;
+    *)
+        echo "Sorry, do not know how to handle os: $os" >&2
+        echo "Please edit the type ${__type##*/} to fix this." >&2
+        exit 1
+    ;;
+esac
diff --git a/cdist/conf/type/__localedef/parameter/default/state b/cdist/conf/type/__localedef/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__localedef/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__localedef/parameter/optional b/cdist/conf/type/__localedef/parameter/optional
new file mode 100644
index 00000000..ff72b5c7
--- /dev/null
+++ b/cdist/conf/type/__localedef/parameter/optional
@@ -0,0 +1 @@
+state

From 54e689f7c2b3bc25aedb2aaf9f5aece289ec5b66 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 13:49:04 +0100
Subject: [PATCH 484/743] [type/__localedef] Add state explorer

---
 cdist/conf/type/__localedef/explorer/state | 71 ++++++++++++++++++++++
 cdist/conf/type/__localedef/gencode-remote | 13 +++-
 2 files changed, 82 insertions(+), 2 deletions(-)
 create mode 100755 cdist/conf/type/__localedef/explorer/state

diff --git a/cdist/conf/type/__localedef/explorer/state b/cdist/conf/type/__localedef/explorer/state
new file mode 100755
index 00000000..d8db29c5
--- /dev/null
+++ b/cdist/conf/type/__localedef/explorer/state
@@ -0,0 +1,71 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer determines if the locale is defined on the target system.
+# Will print nothing on error.
+#
+# Possible output:
+#  present:
+#    the main locale (and possibly aliases) is present
+#  absent:
+#    neither the main locale nor any aliases are present
+#
+
+command -v locale >/dev/null 2>&1 || exit 0
+
+locales=$(locale -a)
+
+parse_locale() {
+	# This function will split locales into their parts. Locale strings are
+	# usually of the form: [language[_territory][.codeset][@modifier]]
+	# For simplicity, language and territory are not separated by this function.
+	# Old Linux systems were also using "english" or "german" as locale strings.
+	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
+	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
+	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
+	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
+}
+
+format_locale() {
+	# Usage: format_locale language codeset modifier
+	printf '%s' "$1"
+	test -z "$2" || printf '.%s' "$2"
+	test -z "$3" || printf '@%s' "$3"
+	printf '\n'
+}
+
+gnu_normalize_codeset() {
+	# reimplementation of glibc/locale/programs/localedef.c normalize_codeset()
+	echo "$*" | tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]'
+}
+
+locale_available() (
+	echo "${locales}" | grep -qxF "$1" || {
+		# glibc uses "normalized" locale names in archives.
+		# If a locale is stored in an archive, the normalized name will be
+		# printed by locale, so that needs to be checked, too.
+		localename=$(
+			parse_locale "$1" _lang _codeset _modifier \
+			&& format_locale "${_lang:?}" "$(gnu_normalize_codeset "${_codeset?}")" \
+			   "${_modifier?}")
+		echo "${locales}" | grep -qxF "${localename}"
+	}
+)
+
+locale_available "${__object_id:?}" && echo present || echo absent
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 1feb9884..9ea3f6f1 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -35,7 +35,8 @@ charmap=$(echo "$locale" | cut -d . -f 2)
 # W-T-F!
 locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
 
-state=$(cat "$__object/parameter/state")
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 
 os=$(cat "$__global/explorer/os")
 
@@ -46,7 +47,15 @@ case "$os" in
         ;;
 esac
 
-case "$state" in
+# NOTE: If state explorer fails (e.g. locale(1) missing), the following check
+#       will always fail and let definition/removal run.
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
+
+case ${state_should}
+in
     present)
         echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
     ;;

From cc29e54b851212568c3f6eddf4bf0a64c1eabb0d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 23 Jul 2020 23:20:39 +0200
Subject: [PATCH 485/743] [type/__localedef] Differentiate between OSes and
 better handling of normalized locale names

---
 cdist/conf/type/__localedef/gencode-remote | 117 +++++++++++++++------
 1 file changed, 82 insertions(+), 35 deletions(-)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 9ea3f6f1..b9cc68e8 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,35 +18,16 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+# Manage system locales using localedef(1).
 #
-# Let localedef do the magic
-#
-
-locale="$__object_id"
-
-# Hardcoded, create a pull request with
-# branching on $os in case it is at another location
-alias=/usr/share/locale/locale.alias
-
-input=$(echo "$locale" | cut -d . -f 1)
-charmap=$(echo "$locale" | cut -d . -f 2)
-
-# Adding locale? The name is de_CH.UTF-8
-# Removing locale? The name is de_CH.utf8.
-# W-T-F!
-locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
 
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
-os=$(cat "$__global/explorer/os")
-
-# Nothing to be done on alpine
-case "$os" in
-    alpine)
-        exit 0
-        ;;
-esac
+test "${state_should}" = 'present' -o "${state_should}" = 'absent' || {
+	printf 'Invalid state: %s\n' "${state_should}" >&2
+	exit 1
+}
 
 # NOTE: If state explorer fails (e.g. locale(1) missing), the following check
 #       will always fail and let definition/removal run.
@@ -54,16 +36,81 @@ then
 	exit 0
 fi
 
-case ${state_should}
+locale=${__object_id:?}
+os=$(cat "${__global:?}/explorer/os")
+
+if expr "${locale}" : '.*/' >/dev/null
+then
+	printf 'Paths as locales are not supported.\n' >&2
+	printf '__object_id is: %s\n' "${locale}" >&2
+	exit 1
+fi
+
+parse_locale() {
+	# This function will split locales into their parts. Locale strings are
+	# usually of the form: [language[_territory][.codeset][@modifier]]
+	# For simplicity, language and territory are not separated by this function.
+	# Old Linux systems were also using "english" or "german" as locale strings.
+	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
+	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
+	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
+	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
+}
+
+format_locale() {
+	# Usage: format_locale language codeset modifier
+	printf '%s' "$1"
+	test -z "$2" || printf '.%s' "$2"
+	test -z "$3" || printf '@%s' "$3"
+	printf '\n'
+}
+
+gnu_normalize_codeset() {
+	echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]'
+}
+
+
+: "${lang=}" "${codeset=}" "${modifier=}"  # declare variables for shellcheck
+parse_locale "${locale}" lang codeset modifier
+
+
+case ${os}
 in
-    present)
-        echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
-    ;;
-    absent)
-        echo localedef --delete-from-archive "$locale_remove"
-    ;;
-    *)
-        echo "Unsupported state: $state" >&2
-        exit 1
-    ;;
+	(alpine|openwrt)
+		printf '%s does not support locales.\n' "${os}" >&2
+		exit 1
+		;;
+	(archlinux|debian|devuan|ubuntu|suse|centos|fedora|redhat|scientific)
+		# FIXME: The code below only works for glibc-based installations.
+
+		# NOTE: Hardcoded, create a pull request in case it is at another
+		#       location for some opther distro.
+		# NOTE: locale.alias can be symlinked (e.g. Debian)
+		aliasfile='/usr/share/locale/locale.alias'
+
+		case ${state_should}
+		in
+			(present)
+				input=$(format_locale "${lang}" '' "${modifier}")
+				cat <<-EOF
+				set --
+				if test -e '${aliasfile}'
+				then
+					set -- -A '${aliasfile}'
+				fi
+
+				localedef -i '${input}' -f '${codeset}' "\$@" '${locale}'
+				EOF
+				;;
+			(absent)
+				localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}")
+				printf "localedef --delete-from-archive '%s'\n" "${localename}"
+				;;
+		esac
+		;;
+	(*)
+		echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
+		echo "Please contribute an implementation for it if you can." >&2
+		exit 1
+		;;
 esac

From f44888f192d78d8ce1e3583f70f94b0c2039c1e2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 14:02:08 +0100
Subject: [PATCH 486/743] [type/__localedef] Only install dependencies in
 manifest. OS checking moved to gencode-remote

---
 cdist/conf/type/__localedef/manifest | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__localedef/manifest b/cdist/conf/type/__localedef/manifest
index 9f1e17ac..3ab3ad8c 100755
--- a/cdist/conf/type/__localedef/manifest
+++ b/cdist/conf/type/__localedef/manifest
@@ -2,6 +2,7 @@
 #
 # 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
 # 2015 David Hürlimann (david at ungleich.ch)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,24 +19,12 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# Install required packages
+# Install required packages.
 #
 
-os=$(cat "$__global/explorer/os")
-
-
-case "$os" in
-    debian|devuan)
-        # Debian needs a seperate package
-        __package locales --state present
-    ;;
-    archlinux|suse|ubuntu|scientific|centos|alpine)
-        :
-    ;;
-    *)
-        echo "Sorry, do not know how to handle os: $os" >&2
-        echo "Please edit the type ${__type##*/} to fix this." >&2
-        exit 1
-    ;;
+case $(cat "${__global:?}/explorer/os")
+in
+	(debian|devuan)
+		__package_apt locales --state present
+		;;
 esac

From dcef2c19f5b87e3a9594f01fefab66cc7488dce7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 14:07:42 +0100
Subject: [PATCH 487/743] [type/__localedef] Add support for FreeBSD

---
 cdist/conf/type/__localedef/gencode-remote | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index b9cc68e8..80e7eb1c 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -108,6 +108,25 @@ in
 				;;
 		esac
 		;;
+	(freebsd)
+		case ${state_should}
+		in
+			(present)
+				if expr "$(grep -oe '^[0-9]*' "${__global:?}/explorer/os_version")" '>=' 11 >/dev/null
+				then
+					# localedef(1) is available with FreeBSD >= 11
+					printf "localedef -i '%s' -f '%s' '%s'\n" "${input}" "${codeset}" "${locale}"
+				else
+					printf 'localedef(1) was added to FreeBSD starting with version 11.\n' >&2
+					printf 'Please upgrade your FreeBSD installation to use %s.\n' "${__type##*/}" >&2
+					exit 1
+				fi
+				;;
+			(absent)
+				printf "rm -R '/usr/share/locale/%s'\n" "${locale}"
+				;;
+		esac
+		;;
 	(*)
 		echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
 		echo "Please contribute an implementation for it if you can." >&2

From c1c60e3374e0cb93ea182c4cbcb813150756c3e4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 15:24:46 +0100
Subject: [PATCH 488/743] [type/__localedef] Blacklist OpenBSD and NetBSD

---
 cdist/conf/type/__localedef/gencode-remote | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 80e7eb1c..af1a77f7 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -127,6 +127,12 @@ in
 				;;
 		esac
 		;;
+	(netbsd|openbsd)
+		# NetBSD/OpenBSD are missing localedef(1).
+		# We also do not delete defined locales because they can't be recreated.
+		echo "${os} is lacking localedef(1). Locale management unavailable." >&2
+		exit 1
+		;;
 	(*)
 		echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
 		echo "Please contribute an implementation for it if you can." >&2

From 575bb62dc507fb8c294319f8db20ba45595e6869 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 13 Nov 2020 18:42:04 +0100
Subject: [PATCH 489/743] [type/__localedef] Externalise functions to separate
 files

---
 .../conf/type/__localedef/files/lib/glibc.sh  |  5 ++++
 .../conf/type/__localedef/files/lib/locale.sh | 20 +++++++++++++
 cdist/conf/type/__localedef/gencode-remote    | 29 ++++---------------
 3 files changed, 30 insertions(+), 24 deletions(-)
 create mode 100644 cdist/conf/type/__localedef/files/lib/glibc.sh
 create mode 100644 cdist/conf/type/__localedef/files/lib/locale.sh

diff --git a/cdist/conf/type/__localedef/files/lib/glibc.sh b/cdist/conf/type/__localedef/files/lib/glibc.sh
new file mode 100644
index 00000000..6ace80d4
--- /dev/null
+++ b/cdist/conf/type/__localedef/files/lib/glibc.sh
@@ -0,0 +1,5 @@
+# -*- mode: sh; indent-tabs-mode: t -*-
+
+gnu_normalize_codeset() {
+	echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]'
+}
diff --git a/cdist/conf/type/__localedef/files/lib/locale.sh b/cdist/conf/type/__localedef/files/lib/locale.sh
new file mode 100644
index 00000000..b5e61374
--- /dev/null
+++ b/cdist/conf/type/__localedef/files/lib/locale.sh
@@ -0,0 +1,20 @@
+# -*- mode: sh; indent-tabs-mode:t -*-
+
+parse_locale() {
+	# This function will split locales into their parts. Locale strings are
+	# usually of the form: [language[_territory][.codeset][@modifier]]
+	# For simplicity, language and territory are not separated by this function.
+	# Old Linux systems were also using "english" or "german" as locale strings.
+	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
+	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
+	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
+	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
+}
+
+format_locale() {
+	# Usage: format_locale language codeset modifier
+	printf '%s' "$1"
+	test -z "$2" || printf '.%s' "$2"
+	test -z "$3" || printf '@%s' "$3"
+	printf '\n'
+}
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index af1a77f7..17941c63 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -21,6 +21,11 @@
 # Manage system locales using localedef(1).
 #
 
+# shellcheck source=cdist/conf/type/__localedef/files/lib/locale.sh
+. "${__type:?}/files/lib/locale.sh"
+# shellcheck source=cdist/conf/type/__localedef/files/lib/glibc.sh
+. "${__type:?}/files/lib/glibc.sh"
+
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
@@ -46,30 +51,6 @@ then
 	exit 1
 fi
 
-parse_locale() {
-	# This function will split locales into their parts. Locale strings are
-	# usually of the form: [language[_territory][.codeset][@modifier]]
-	# For simplicity, language and territory are not separated by this function.
-	# Old Linux systems were also using "english" or "german" as locale strings.
-	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
-	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
-	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
-	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
-}
-
-format_locale() {
-	# Usage: format_locale language codeset modifier
-	printf '%s' "$1"
-	test -z "$2" || printf '.%s' "$2"
-	test -z "$3" || printf '@%s' "$3"
-	printf '\n'
-}
-
-gnu_normalize_codeset() {
-	echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]'
-}
-
-
 : "${lang=}" "${codeset=}" "${modifier=}"  # declare variables for shellcheck
 parse_locale "${locale}" lang codeset modifier
 

From eeb98719197a09691af60fa26c0b64b3aaf8960e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 14 Nov 2020 09:54:01 +0100
Subject: [PATCH 490/743] [type/__localedef] glibc: Also delete aliases when
 removing a locale

---
 cdist/conf/type/__localedef/gencode-remote | 15 +++++++++++++--
 cdist/conf/type/__localedef/man.rst        |  5 +++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 17941c63..d7fd6942 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -84,8 +84,19 @@ in
 				EOF
 				;;
 			(absent)
-				localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}")
-				printf "localedef --delete-from-archive '%s'\n" "${localename}"
+				main_localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}")
+
+				cat <<-EOF
+				while read -r _alias _localename
+				do
+				    if test "\${_localename}" = '$(format_locale "${lang}" "${codeset}")'
+				    then
+				        localedef --delete-from-archive "\${_alias}"
+				    fi
+				done <'${aliasfile}'
+
+				localedef --delete-from-archive '${main_localename}'
+				EOF
 				;;
 		esac
 		;;
diff --git a/cdist/conf/type/__localedef/man.rst b/cdist/conf/type/__localedef/man.rst
index 0ccf484a..454ce9d1 100644
--- a/cdist/conf/type/__localedef/man.rst
+++ b/cdist/conf/type/__localedef/man.rst
@@ -13,6 +13,11 @@ This cdist type allows you to define locales on the system using
 On systems that don't support definition of new locales, the type will raise an
 error.
 
+**NB:** This type respects the glibc ``locale.alias`` file,
+i.e. it defines alias locales or deletes aliases of a locale when it is removed.
+It is not possible, however, to use alias names to define locales or only remove
+certain aliases of a locale.
+
 
 OPTIONAL PARAMETERS
 -------------------

From 87faffd8750cac94190707c2b37233229c3fbde2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 14 Nov 2020 10:41:50 +0100
Subject: [PATCH 491/743] [type/__localdef] Also check for aliases in state
 explorer

---
 cdist/conf/type/__localedef/explorer/state | 31 +++++++++++++++++++++-
 cdist/conf/type/__localedef/gencode-remote |  7 +++--
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__localedef/explorer/state b/cdist/conf/type/__localedef/explorer/state
index d8db29c5..3ba57661 100755
--- a/cdist/conf/type/__localedef/explorer/state
+++ b/cdist/conf/type/__localedef/explorer/state
@@ -25,8 +25,14 @@
 #    the main locale (and possibly aliases) is present
 #  absent:
 #    neither the main locale nor any aliases are present
+#  alias-present:
+#    the main locale is absent, but at least one of its aliases is present
 #
 
+# Hardcoded, create a pull request in case it is at another location for
+# some other distro. (cf. gencode-remote)
+aliasfile='/usr/share/locale/locale.alias'
+
 command -v locale >/dev/null 2>&1 || exit 0
 
 locales=$(locale -a)
@@ -68,4 +74,27 @@ locale_available() (
 	}
 )
 
-locale_available "${__object_id:?}" && echo present || echo absent
+if locale_available "${__object_id:?}"
+then
+	echo present
+else
+	# NOTE: locale.alias can be symlinked.
+	if test -e "${aliasfile}"
+	then
+		# Check if one of the aliases of the locale is defined
+		baselocale=$(
+			parse_locale "${__object_id:?}" _lang _codeset _modifiers \
+			&& format_locale "${_lang}" "${_codeset}")
+		while read -r _alias _localename
+		do
+			if test "${_localename}" = "${baselocale}" \
+				&& echo "${locales}" | grep -qxF "${_alias}"
+			then
+				echo alias-present
+				exit 0
+			fi
+		done <"${aliasfile}"
+	fi
+
+	echo absent
+fi
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index d7fd6942..4538151f 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -94,9 +94,12 @@ in
 				        localedef --delete-from-archive "\${_alias}"
 				    fi
 				done <'${aliasfile}'
-
-				localedef --delete-from-archive '${main_localename}'
 				EOF
+
+				if test "${state_is}" = present
+				then
+					printf "localedef --delete-from-archive '%s'\n" "${main_localename}"
+				fi
 				;;
 		esac
 		;;

From 9d4f69250ea0e6bb733cde1e42183480f48b5d8f Mon Sep 17 00:00:00 2001
From: ssrq <ungleich@ssrq-sds-fds.ch>
Date: Thu, 19 Nov 2020 19:33:47 +0100
Subject: [PATCH 492/743] __sshd config: New type

---
 cdist/conf/type/__sshd_config/explorer/state  | 121 ++++++++
 .../files/update_sshd_config.awk              | 293 ++++++++++++++++++
 cdist/conf/type/__sshd_config/gencode-remote  |  97 ++++++
 cdist/conf/type/__sshd_config/man.rst         |  94 ++++++
 cdist/conf/type/__sshd_config/manifest        |  48 +++
 .../type/__sshd_config/parameter/default/file |   1 +
 .../__sshd_config/parameter/default/state     |   1 +
 .../type/__sshd_config/parameter/optional     |   4 +
 .../__sshd_config/parameter/optional_multiple |   1 +
 9 files changed, 660 insertions(+)
 create mode 100644 cdist/conf/type/__sshd_config/explorer/state
 create mode 100644 cdist/conf/type/__sshd_config/files/update_sshd_config.awk
 create mode 100755 cdist/conf/type/__sshd_config/gencode-remote
 create mode 100644 cdist/conf/type/__sshd_config/man.rst
 create mode 100755 cdist/conf/type/__sshd_config/manifest
 create mode 100644 cdist/conf/type/__sshd_config/parameter/default/file
 create mode 100644 cdist/conf/type/__sshd_config/parameter/default/state
 create mode 100644 cdist/conf/type/__sshd_config/parameter/optional
 create mode 100644 cdist/conf/type/__sshd_config/parameter/optional_multiple

diff --git a/cdist/conf/type/__sshd_config/explorer/state b/cdist/conf/type/__sshd_config/explorer/state
new file mode 100644
index 00000000..75c68b8a
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/explorer/state
@@ -0,0 +1,121 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Determines the current state of the config option.
+# Possible output:
+#  - present: "should" option present in config file
+#  - default: the "should" option is the default -> don’t know if present
+#  - absent: no such option present in config file
+#
+
+joinlines() { sed -n -e H -e "\${x;s/^\\n//;s/\\n/${1:?}/g;p;}"; }
+trlower() { tr '[:upper:]' '[:lower:]'; }
+tolower() { printf '%s' "$*" | trlower; }
+
+default_value() {
+	sshd -T -f /dev/null -C "$(make_conn_spec)" \
+	| sed -n -e 's/^'"$(tolower "${1:?}")"'[[:blank:]]\{1,\}//p'
+}
+
+make_conn_spec() {
+	if test -s "${__object:?}/parameter/match"
+	then
+		_match_file="${__object:?}/parameter/match"
+	else
+		_match_file='/dev/null'
+	fi
+
+	for _kw in \
+		addr=Address \
+		user=User \
+		host=Host \
+		laddr=LocalAddress \
+		lport=LocalPort \
+		rdomain=RDomain
+	do
+		_specname=${_kw%%=*}
+		_confname=$(tolower "${_kw#*=}")
+		while read -r _k _v
+		do
+			if test "$(tolower "${_k}")" = "${_confname}"
+			then
+				printf '%s=%s\n' "${_specname}" "${_v}"
+				continue 2
+			fi
+		done <"${_match_file}"
+
+		# NOTE: Print test spec even for empty keys to suppress errors like:
+		#  'Match User' in configuration but 'user' not in connection test specification.
+		# except lport:
+		#  Invalid port '' in test mode specification lport=
+		test "${_specname}" = 'lport' || printf '%s=\n' "${_specname}"
+	done \
+	| joinlines ','
+	unset _match_file
+}
+
+sshd_config_file=$(cat "${__object:?}/parameter/file")
+state_should=$(cat "${__object:?}/parameter/state")
+
+if test -s "${__object:?}/parameter/option"
+then
+	option_name=$(cat "${__object:?}/parameter/option")
+else
+	option_name=${__object_id:?}
+fi
+
+value_should=$(cat "${__object:?}/parameter/value" 2>/dev/null) \
+|| test "${state_should}" = absent || exit 0  # param optional if --state absent
+
+command -v sshd >/dev/null 2>&1 || {
+	echo 'Cannot find sshd.' >&2
+	exit 1
+}
+
+test -e "${sshd_config_file}" || {
+	echo 'absent'
+	exit 0
+}
+
+value_is=$(
+	sshd -T -f "${sshd_config_file}" -C "$(make_conn_spec)" \
+	| sed -n -e 's/^'"$(tolower "${option_name}")"'[[:blank:]]\{1,\}//p')
+
+if printf '%s\n' "${value_is}" | {
+		if test -n "${value_should}"
+		then
+			grep -q -x -F "${value_should}"
+		else
+			# if no value provided, assume "any" value
+			grep -q -e .
+		fi
+	}
+then
+	if default_value "${option_name}" | grep -q -x -F "${value_is}"
+	then
+		# Might produce false positives for default values.
+		# TODO: Manual checking should be done, but for simplicity, this case is
+		#       currently ignored here.
+		echo default
+	else
+		echo present
+	fi
+else
+	echo absent
+fi
diff --git a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
new file mode 100644
index 00000000..d0bc2b4b
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
@@ -0,0 +1,293 @@
+# -*- mode: awk; indent-tabs-mode: t -*-
+
+function usage() {
+	print_err("Usage: awk -f update_sshd_config.awk -- -o set|unset [-m 'User git'] -l 'X11Forwarding no' /etc/ssh/sshd_config")
+}
+
+function print_err(s) { print s | "cat >&2" }
+
+function alength(a,    i) {
+	for (i = 0; (i + 1) in a; ++i);
+	return i
+}
+
+function join(sep, a, i,    s) {
+	for (i = i ? i : 1; i in a; i++)
+		s = s sep a[i]
+	return substr(s, 2)
+}
+
+function getopt(opts, argv, target, files,    i, c, lv, idx, nf) {
+	# trivial getopt(3) implementation; only basic functionality
+	if (argv[1] == "--") i++
+	for (i += 1; i in argv; i++) {
+		if (lv) { target[c] = argv[i]; lv = 0; continue }
+		if (argv[i] ~ /^-/) {
+			c = substr(argv[i], 2, 1)
+			idx = index(opts, c)
+			if (!idx) {
+				print_err(sprintf("invalid option -%c\n", c))
+				continue
+			}
+			if (substr(opts, idx + 1, 1) == ":") {
+				# option takes argument
+				if (length(argv[i]) > 2)
+					target[c] = substr(argv[i], 3)
+				else
+					lv = 1
+			} else {
+				target[c] = 1
+			}
+		} else
+			files[++nf] = argv[i]
+	}
+}
+
+# tokenise configuration line
+# this function mimics the counterpart in OpenSSH (misc.c)
+# but it returns two (next token SUBSEP rest) because I didn’t want to have to
+# simulate any pointer magic.
+function strdelim_internal(s, split_equals,    old) {
+	if (!s)
+		return ""
+
+	old = s
+
+	if (!match(s, WHITESPACE "|" QUOTE "" (split_equals ? "|" EQUALS : "")))
+		return s
+
+	s = substr(s, RSTART)
+	old = substr(old, 1, RSTART - 1)
+
+	if (s ~ "^" QUOTE) {
+		old = substr(old, 2)
+
+		# Find matching quote
+		if (match(s, QUOTE)) {
+			old = substr(old, 1, RSTART)
+			# s = substr()
+			if (match(s, "^" WHITESPACE "*"))
+				s = substr(s, RLENGTH)
+			return old
+		} else {
+			# no matching quote
+			return ""
+		}
+	}
+
+	if (match(s, "^" WHITESPACE "+")) {
+		sub("^" WHITESPACE "+", "", s)
+		if (split_equals)
+			sub(EQUALS WHITESPACE "*", "", s)
+	} else if (s ~ "^" EQUALS) {
+		s = substr(s, 2)
+	}
+
+	return old SUBSEP s
+}
+function strdelim(s) { return strdelim_internal(s, 1) }
+function strdelimw(s) { return strdelim_internal(s, 0) }
+
+function singleton_option(opt) {
+	return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|authenticationmethods|authorizedkeysfile|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/
+}
+
+function print_update() {
+	if (mode) {
+		if (match_only) printf "\t"
+		printf "%s\n", line_should
+		updated = 1
+	}
+}
+
+BEGIN {
+	FS = "\n"  # disable field splitting
+
+	WHITESPACE = "[ \t]"  # servconf.c, misc.c:strdelim_internal (without line breaks, cf. bugs)
+	QUOTE = "[\"]"  # misc.c:strdelim_internal
+	EQUALS = "[=]"
+
+	split("", opts)
+	split("", files)
+	getopt("ho:l:m:", ARGV, opts, files)
+
+	if (opts["h"]) { usage(); exit (e="0") }
+
+	line_should = opts["l"]
+	match_only = opts["m"]
+	num_files = alength(files)
+
+	if (num_files != 1 || !opts["o"] || !line_should) {
+		usage()
+		exit (e=126)
+	}
+
+	if (opts["o"] == "set") {
+		mode = 1
+	} else if (opts["o"] == "unset") {
+		mode = 0
+	} else {
+		print_err(sprintf("invalid mode %s\n", mode))
+		exit (e=1)
+	}
+
+	if (mode) {
+		# loop over sshd_config twice!
+		ARGV[2] = ARGV[1] = files[1]
+		ARGC = 3
+	} else {
+		# only loop once
+		ARGV[1] = files[1]
+		ARGC = 2
+	}
+
+	split(strdelim(line_should), should, SUBSEP)
+	option_should = tolower(should[1])
+	value_should = should[2]
+}
+
+{
+	line = $0
+
+	# Strip trailing whitespace. Allow \f (form feed) at EOL only
+	sub("(" WHITESPACE "|\f)*$", "", line)
+
+	# Strip leading whitespace
+	sub("^" WHITESPACE "*", "", line)
+
+	if (match(line, "^#" WHITESPACE "*")) {
+		prefix = substr(line, RSTART, RLENGTH)
+		line = substr(line, RSTART + RLENGTH)
+	} else {
+		prefix = ""
+	}
+
+	line_type = "invalid"
+	option_is = value_is = ""
+
+	if (line) {
+		split(strdelim(line), toks, SUBSEP)
+
+		if (tolower(toks[1]) == "match") {
+			MATCH = (prefix ~ /^#/ ? "#" : "") join(" ", toks, 2)
+			line_type = "match"
+		} else if (toks[1] ~ /^[A-Za-z][A-Za-z0-9]+$/) {
+			# This could be an option line
+			line_type = "option"
+			option_is = tolower(toks[1])
+			value_is = toks[2]
+		}
+	} else {
+		line_type = "empty"
+	}
+}
+
+# mode: unset
+
+!mode {
+	# delete matching config
+	if (prefix !~ /^#/)
+		if (MATCH == match_only && option_is == option_should)
+			if (!value_should || value_should == value_is)
+				next
+
+	print
+	next
+}
+
+
+# mode: set
+
+mode && NR == FNR {
+	if (line_type == "option") {
+		if (MATCH !~ /^#/) {
+			if (prefix ~ /^#/) {
+				# comment line
+				last_occ[MATCH, "#" option_is] = FNR
+			} else {
+				# option line
+				last_occ[MATCH, option_is] = FNR
+			}
+			last_occ[MATCH] = FNR
+		}
+	} else if (line_type == "invalid" && !prefix) {
+		# INVALID LINE
+		print_err(sprintf("%s: syntax error on line %u\n", ARGV[0], FNR))
+	}
+
+	next
+}
+
+# before second pass prepare hashes containing location information to be used
+# in the second pass.
+mode && NR > FNR && FNR == 1 {
+	# First we drop the locations of commented-out options if a non-commented
+	# option is available. If a non-commented option is available, we will
+	# append new config options there to have them all at one place.
+	for (k in last_occ) {
+		if (k ~ /^#/) {
+			# delete entries of commented out match blocks
+			delete last_occ[k]
+			continue
+		}
+
+		split(k, parts, SUBSEP)
+
+		if (parts[2] ~ /^#/ && ((parts[1], substr(parts[2], 2)) in last_occ))
+			delete last_occ[k]
+	}
+
+	# Reverse the option => line mapping. The line_map allows for easier lookups
+	# in the second pass.
+	# We only keep options, not top-level keywords, because we can only have
+	# one entry per line and there are conflicts with last lines of "sections".
+	for (k in last_occ) {
+		if (!index(k, SUBSEP)) continue
+		line_map[last_occ[k]] = k
+	}
+}
+
+# Second pass
+mode && line_map[FNR] == match_only SUBSEP option_should && !updated {
+	split(line_map[FNR], parts, SUBSEP)
+
+	# If option allows multiple values, print current value
+	if (!singleton_option(parts[2])) {
+		if (value_should != value_is)
+			print
+	}
+
+	print_update()
+
+	next
+}
+
+mode { print }
+
+# Is a comment option
+mode && line_map[FNR] == match_only SUBSEP "#" option_should && !updated {
+	print_update()
+}
+
+# Last line of the should match section
+mode && last_occ[match_only] == FNR && !updated {
+	# NOTE: Inserting empty lines is only cosmetic.  It is only done if
+	#       different options are next to each other and not in a match block
+	#       (match blocks are usually not in the default config and thus don’t
+	#       contain commented blocks.)
+	if (line && option_is != option_should && !MATCH)
+		print ""
+	print_update()
+}
+
+END {
+	if (e) exit e
+
+	if (mode && !updated) {
+		if (match_only && MATCH != match_only) {
+			printf "\nMatch %s\n", match_only
+		}
+
+		print_update()
+	}
+}
diff --git a/cdist/conf/type/__sshd_config/gencode-remote b/cdist/conf/type/__sshd_config/gencode-remote
new file mode 100755
index 00000000..0b44dfa7
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/gencode-remote
@@ -0,0 +1,97 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+joinlines() { sed -n -e H -e "\${x;s/^\\n//;s/\\n/${1:?}/g;p;}"; }
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+if test "${state_is}" = "${state_should}" -o "${state_is}" = 'default'
+then
+	# nothing to do (if the value is the default, ignore its state)
+	exit 0
+fi
+
+case ${state_should}
+in
+	(present)
+		mode='set'
+		;;
+	(absent)
+		mode='unset'
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
+
+sshd_config_file=$(cat "${__object:?}/parameter/file")
+
+quote() { printf "'%s'" "$(printf '%s' "$*" | sed -e "s/'/'\\\\''/g")"; }
+drop_awk_comments() { quote "$(sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@")"; }
+
+# Ensure the sshd_config file is there
+cat <<EOF
+test -e $(quote "${sshd_config_file}") || {
+	 : >$(quote "${sshd_config_file}")
+	 chown 0:0 $(quote "${sshd_config_file}")
+	 chmod 0644 $(quote "${sshd_config_file}")
+}
+
+EOF
+
+match_only=
+if test -s "${__object:?}/parameter/match"
+then
+	match_only=$(joinlines ' ' <"${__object:?}/parameter/match")
+fi
+
+if test -s "${__object:?}/parameter/option"
+then
+	option_line=$(cat "${__object:?}/parameter/option")
+else
+	option_line=${__object_id:?}
+fi
+
+if test -s "${__object:?}/parameter/value"
+then
+	option_line="${option_line} $(cat "${__object:?}/parameter/value")"
+fi
+
+# Send message on config update
+printf '%s%s %s\n' "${mode}" "${match_only:+ [${match_only}]}" \
+	"${option_line}" >>"${__messages_out:?}"
+
+# Update sshd_config (remote code)
+cat <<EOF
+awk $(drop_awk_comments "${__type:?}/files/update_sshd_config.awk") \\
+	-o ${mode} \\
+	-m $(quote "${match_only}") \\
+	-l $(quote "${option_line}") \\
+	$(quote "${sshd_config_file}") >$(quote "${sshd_config_file}.tmp") \\
+|| exit
+
+cmp -s $(quote "${sshd_config_file}") $(quote "${sshd_config_file}.tmp") || {
+	sshd -t -f $(quote "${sshd_config_file}.tmp") \\
+	&& cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}")
+}
+rm -f $(quote "${sshd_config_file}.tmp")
+EOF
diff --git a/cdist/conf/type/__sshd_config/man.rst b/cdist/conf/type/__sshd_config/man.rst
new file mode 100644
index 00000000..8b0069ac
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/man.rst
@@ -0,0 +1,94 @@
+cdist-type__sshd_config(7)
+==========================
+
+NAME
+----
+cdist-type__sshd_config - Manage options in sshd_config
+
+
+DESCRIPTION
+-----------
+This space intentionally left blank.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+file
+    The path to the sshd_config file to edit.
+    Defaults to ``/etc/ssh/sshd_config``.
+match
+    Restrict this option to apply only for certain connections.
+    Allowed values are what would be allowed to be written after a ``Match``
+    keyword in ``sshd_config``, e.g. ``--match 'User anoncvs'``.
+
+    Can be used multiple times. All of the values are ANDed together.
+option
+    The name of the option to manipulate. Defaults to ``__object_id``.
+state
+    Can be:
+
+    - ``present``: ensure a matching config line is present (or the default
+      value).
+    - ``absent``: ensure no matching config line is present.
+value
+    The option's value to be assigned to the option (if ``--state present``) or
+    removed (if ``--state absent``).
+
+    This option is required if ``--state present``. If not specified and
+    ``--state absent``, all values for the given option are removed.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Disallow root logins with password
+    __sshd_config PermitRootLogin --value without-password
+
+    # Disallow password-based authentication
+    __sshd_config PasswordAuthentication --value no
+
+    # Accept the EDITOR environment variable
+    __sshd_config AcceptEnv:EDITOR --option AcceptEnv --value EDITOR
+
+    # Force command for connections as git user
+    __sshd_config git@ForceCommand --match 'User git' --option ForceCommand \
+        --value 'cd ~git && exec git-shell ${SSH_ORIGINAL_COMMAND:+-c "${SSH_ORIGINAL_COMMAND}"}'
+
+
+SEE ALSO
+--------
+:strong:`sshd_config`\ (5)
+
+
+BUGS
+----
+- This type assumes a nicely formatted config file,
+  i.e. no config options spanning multiple lines.
+- ``Include`` directives are ignored.
+- Config options are not added/removed to/from the config file if their value is
+  the default value.
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest
new file mode 100755
index 00000000..566bde90
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/manifest
@@ -0,0 +1,48 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$(cat "${__global:?}/explorer/os")
+
+state_should=$(cat "${__object:?}/parameter/state")
+
+case ${os}
+in
+	(alpine|centos|fedora|redhat|scientific|debian|devuan|ubuntu)
+		if test "${state_should}" != 'absent'
+		then
+			__package openssh-server --state present
+		fi
+		;;
+	(archlinux|gentoo|slackware|suse)
+		if test "${state_should}" != 'absent'
+		then
+			__package openssh --state present
+		fi
+		;;
+	(freebsd|netbsd|openbsd)
+		# whitelist
+		;;
+	(*)
+		printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \
+			"${os}" "${__type##*/}" >&2
+		printf 'Please contribute an implementation for it if you can.\n' >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__sshd_config/parameter/default/file b/cdist/conf/type/__sshd_config/parameter/default/file
new file mode 100644
index 00000000..d8ea5dfc
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/default/file
@@ -0,0 +1 @@
+/etc/ssh/sshd_config
diff --git a/cdist/conf/type/__sshd_config/parameter/default/state b/cdist/conf/type/__sshd_config/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__sshd_config/parameter/optional b/cdist/conf/type/__sshd_config/parameter/optional
new file mode 100644
index 00000000..922ab093
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/optional
@@ -0,0 +1,4 @@
+file
+option
+state
+value
diff --git a/cdist/conf/type/__sshd_config/parameter/optional_multiple b/cdist/conf/type/__sshd_config/parameter/optional_multiple
new file mode 100644
index 00000000..02b1d1a9
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/optional_multiple
@@ -0,0 +1 @@
+match

From 82eadb69948e631041263938994be5aeae161ab0 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 19 Nov 2020 19:34:43 +0100
Subject: [PATCH 493/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index ff411a46..331acbeb 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,8 @@ next:
 	* Type __locale: Add state explorer (Matthias Stecher)
 	* Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera)
 	* New type: __hwclock (Dennis Camera)
+	* Type __hostname: Fix guessing SuSE OS version (Dennis Camera)
+	* New type: __sshd_config (Dennis Camera)
 
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)

From 803a9d62a7a6b7fd11844f6ea273f4b692c2334e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 20 Nov 2020 19:46:03 +0100
Subject: [PATCH 494/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 331acbeb..cd5dbb4f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,8 @@ next:
 	* New type: __hwclock (Dennis Camera)
 	* Type __hostname: Fix guessing SuSE OS version (Dennis Camera)
 	* New type: __sshd_config (Dennis Camera)
+	* New type: __localedef (Dennis Camera)
+	* Type __locale: Deprecate in favor of __localedef (Dennis Camera)
 
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)

From 23e0da521c785715357b27d66007bd03fe173bf3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 20 Nov 2020 19:46:55 +0100
Subject: [PATCH 495/743] Release 6.9.2

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index cd5dbb4f..9a91bf57 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.2: 2020-11-20
 	* Documentation: Fix examples in best practice (Dennis Camera)
 	* Type __locale: Add state explorer (Matthias Stecher)
 	* Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera)

From 84172550df9e1800bc795dc51da01c9d3a0be9e0 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 30 Nov 2020 20:29:51 +0100
Subject: [PATCH 496/743] __iptables*: add IPv6 support

Because it currently only support IPv4. To implement this, it falls back
to IPv4 for backward compatibilty, but now supports rules for IPv6 and
both protocols at the same time.
---
 .../type/__iptables_apply/files/init-script   | 84 +++++++++++++++----
 cdist/conf/type/__iptables_rule/man.rst       | 38 ++++++++-
 cdist/conf/type/__iptables_rule/manifest      | 40 ++++++++-
 .../type/__iptables_rule/parameter/boolean    |  3 +
 4 files changed, 140 insertions(+), 25 deletions(-)
 create mode 100644 cdist/conf/type/__iptables_rule/parameter/boolean

diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script
index d9c79ef7..196f019b 100644
--- a/cdist/conf/type/__iptables_apply/files/init-script
+++ b/cdist/conf/type/__iptables_apply/files/init-script
@@ -1,6 +1,27 @@
 #!/bin/sh
-# Nico Schottelius
-# Zürisee, Mon Sep  2 18:38:27 CEST 2013
+#
+# 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is distributed with cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Originally written by:
+#   Nico Schottelius
+#   Zürisee, Mon Sep  2 18:38:27 CEST 2013
 #
 ### BEGIN INIT INFO
 # Provides:          iptables
@@ -15,33 +36,60 @@
 ### END INIT INFO
 
 
+# Read files and execute the content with the given commands
+#
+# Arguments:
+#  1:    Directory
+#  2..n: Commands which should be used to execute the file content
+gothrough() {
+    cd "$1" || return
+    shift
+
+    # iterate through all rules and continue if it's not a file
+    for rule in *; do
+        [ -f "$rule" ] || continue
+        echo "Appling iptables rule $rule ..."
+
+        # execute it with all commands specificed
+        ruleparam="$(cat "$rule")"
+        for cmd in "$@"; do
+            # Command and Rule should be split.
+            # shellcheck disable=SC2046
+            command $cmd $ruleparam
+        done
+    done
+}
+
+# Shortcut for iptables command to do IPv4 and v6
+iptables() {
+    command iptables "$@"
+    command ip6tables "$@"
+}
+
 basedir=/etc/iptables.d
-status="${basedir}/.pre-start"
+status4="${basedir}/.pre-start"
+status6="${basedir}/.pre-start6"
 
 case $1 in
     start)
         # Save status
-        iptables-save > "$status"
+        iptables-save > "$status4"
+        ip6tables-save > "$status6"
 
         # Apply our ruleset
-        cd "$basedir" || exit
-        count="$(find . ! -name . -prune | wc -l)"
-
-        # Only do something if there are rules
-        if [ "$count" -ge 1 ]; then
-            for rule in *; do
-                echo "Applying iptables rule $rule ..."
-                # Rule should be split.
-                # shellcheck disable=SC2046
-                iptables $(cat "$rule")
-            done
-        fi
+        gothrough "$basedir"     iptables
+        #gothrough "$basedir/v4"  iptables  # conflicts with $basedir
+        gothrough "$basedir/v6"  ip6tables
+        gothrough "$basedir/all" iptables  ip6tables
     ;;
 
     stop)
         # Restore from status before, if there is something to restore
-        if [ -f "$status" ]; then
-            iptables-restore < "$status"
+        if [ -f "$status4" ]; then
+            iptables-restore < "$status4"
+        fi
+        if [ -f "$status6" ]; then
+            ip6tables-restore < "$status6"
         fi
     ;;
     restart)
diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst
index 92d8859f..75d0740b 100644
--- a/cdist/conf/type/__iptables_rule/man.rst
+++ b/cdist/conf/type/__iptables_rule/man.rst
@@ -25,6 +25,24 @@ state
    'present' or 'absent', defaults to 'present'
 
 
+BOOLEAN PARAMETERS
+------------------
+All rules without any of this parameter will be threaten like ``--v4`` because
+of backward compatibility.
+
+v4
+    Explicitly set it as rule for IPv4. If IPv6 is set, too, it will be
+    threaten like ``--all``. Will be the default if nothing else is set.
+
+v6
+    Explicitly set it as rule for IPv6. If IPv4 is set, too, it will be
+    threaten like ``--all``.
+
+all
+    Set the rule for both IPv4 and IPv6. It will be saved separately from the
+    other rules.
+
+
 EXAMPLES
 --------
 
@@ -48,6 +66,16 @@ EXAMPLES
         --state absent
 
 
+    # IPv4-only rule for ICMPv4
+    __iptables_rule icmp-v4 --v4 --rule "-A INPUT -p icmp -j ACCEPT"
+    # IPv6-only rule for ICMPv6
+    __iptables_rule icmp-v6 --v6 --rule "-A INPUT -p icmpv6 -j ACCEPT"
+
+    # doing something for the dual stack
+    __iptables_rule fwd-eth0-eth1 --v4 --v6 --rule "-A INPUT -i eth0 -o eth1 -j ACCEPT"
+    __iptables_rule fwd-eth1-eth0 --all --rule "-A -o eth1 -i eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT"
+
+
 SEE ALSO
 --------
 :strong:`cdist-type__iptables_apply`\ (7), :strong:`iptables`\ (8)
@@ -56,11 +84,13 @@ SEE ALSO
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2013 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2013 Nico Schottelius.
+Copyright \(C) 2020 Matthias Stecher.
+You can redistribute it and/or modify it under the terms of the GNU
+General Public License as published by the Free Software Foundation,
+either version 3 of the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest
index ed78787f..27d00024 100755
--- a/cdist/conf/type/__iptables_rule/manifest
+++ b/cdist/conf/type/__iptables_rule/manifest
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
 #
 # This file is part of cdist.
 #
@@ -24,12 +25,36 @@ base_dir=/etc/iptables.d
 name="$__object_id"
 state="$(cat "$__object/parameter/state")"
 
+if [ -f "$__object/parameter/v4" ]; then
+    only_v4="yes"
+    # $specific_dir is $base_dir
+fi
+if [ -f "$__object/parameter/v6" ]; then
+    only_v6="yes"
+    specific_dir="$base_dir/v6"
+fi
+# If rules should be set for both protocols
+if ([ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]) \
+    || [ -f "$__object/parameter/all" ]; then
+
+    # all to a specific directory
+    specific_dir="$base_dir/all"
+fi
+
+# set rule directory based on if it's the base or subdirectory
+rule_dir="${specific_dir:-$base_dir}"
+
 ################################################################################
 # Basic setup
 #
 
 __directory "$base_dir" --state present
 
+# sub-directory if required
+if [ "$specific_dir" ]; then
+    require="__directory/$base_dir" __directory "$specific_dir" --state present
+fi
+
 # Have apply do the real job
 require="$__object_name" __iptables_apply
 
@@ -37,6 +62,15 @@ require="$__object_name" __iptables_apply
 # The rule
 #
 
-require="__directory/$base_dir" __file "$base_dir/${name}" \
-    --source "$__object/parameter/rule" \
-    --state "$state"
+for dir in "$base_dir" "$base_dir/v6" "$base_dir/all"; do
+    # defaults to absent except the directory that should contain the file
+    if [ "$rule_dir" = "$dir" ]; then
+        curr_state="$state"
+    else
+        curr_state="absent"
+    fi
+
+    require="__directory/$rule_dir" __file "$dir/$name" \
+        --source "$__object/parameter/rule" \
+        --state "$curr_state"
+done
diff --git a/cdist/conf/type/__iptables_rule/parameter/boolean b/cdist/conf/type/__iptables_rule/parameter/boolean
new file mode 100644
index 00000000..76882272
--- /dev/null
+++ b/cdist/conf/type/__iptables_rule/parameter/boolean
@@ -0,0 +1,3 @@
+all
+v4
+v6

From f568462e4981c8b6437c29f96963c3d8e7bed742 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 2 Dec 2020 17:48:41 +0100
Subject: [PATCH 497/743] __iptables_rule: fix shellcheck SC2235

---
 cdist/conf/type/__iptables_rule/manifest | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest
index 27d00024..d4394c25 100755
--- a/cdist/conf/type/__iptables_rule/manifest
+++ b/cdist/conf/type/__iptables_rule/manifest
@@ -34,8 +34,8 @@ if [ -f "$__object/parameter/v6" ]; then
     specific_dir="$base_dir/v6"
 fi
 # If rules should be set for both protocols
-if ([ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]) \
-    || [ -f "$__object/parameter/all" ]; then
+if { [ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]; } ||
+    [ -f "$__object/parameter/all" ]; then
 
     # all to a specific directory
     specific_dir="$base_dir/all"

From bee255c1ae0039e3d532317007f06fcd2e64e3bc Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 2 Dec 2020 18:04:50 +0100
Subject: [PATCH 498/743] __iptables_apply: man updates

---
 .../type/__iptables_apply/files/init-script   |  1 +
 cdist/conf/type/__iptables_apply/man.rst      | 21 +++++++++++++------
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script
index 196f019b..7faa2f92 100644
--- a/cdist/conf/type/__iptables_apply/files/init-script
+++ b/cdist/conf/type/__iptables_apply/files/init-script
@@ -61,6 +61,7 @@ gothrough() {
 }
 
 # Shortcut for iptables command to do IPv4 and v6
+# only applies to the "reset" target
 iptables() {
     command iptables "$@"
     command ip6tables "$@"
diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst
index 76e1f6bf..db0e7869 100644
--- a/cdist/conf/type/__iptables_apply/man.rst
+++ b/cdist/conf/type/__iptables_apply/man.rst
@@ -10,7 +10,14 @@ DESCRIPTION
 -----------
 This cdist type deploys an init script that triggers
 the configured rules and also re-applies them on
-configuration.
+configuration. Rules are written from __iptables_rule
+into the folder ``/etc/iptables.d/``.
+
+It reads all rules from the base folder as rules for IPv4.
+Rules in the subfolder ``v6/`` are IPv6 rules. Rules in
+the subfolder ``all/`` are applied to both rule tables. All
+files contain the arguments for a single ``iptables`` and/or
+``ip6tables`` command.
 
 
 REQUIRED PARAMETERS
@@ -24,7 +31,7 @@ None
 EXAMPLES
 --------
 
-None (__iptables_apply is used by __iptables_rule)
+None (__iptables_apply is used by __iptables_rule automaticly)
 
 
 SEE ALSO
@@ -35,11 +42,13 @@ SEE ALSO
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2013 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2013 Nico Schottelius.
+Copyright \(C) 2020 Matthias Stecher.
+You can redistribute it and/or modify it under the terms of the GNU
+General Public License as published by the Free Software Foundation,
+either version 3 of the License, or (at your option) any later version.

From a1db5c3d0e7b5899f1a877c5bab28ac4d4796f8c Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 2 Dec 2020 18:22:31 +0100
Subject: [PATCH 499/743] __iptables*: Update manpages for execution order

To make some thinks clear if someone needs it ..
---
 cdist/conf/type/__iptables_apply/man.rst | 10 ++++++++++
 cdist/conf/type/__iptables_rule/man.rst  |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst
index db0e7869..4109e789 100644
--- a/cdist/conf/type/__iptables_apply/man.rst
+++ b/cdist/conf/type/__iptables_apply/man.rst
@@ -19,6 +19,16 @@ the subfolder ``all/`` are applied to both rule tables. All
 files contain the arguments for a single ``iptables`` and/or
 ``ip6tables`` command.
 
+Rules are applied in the following order:
+1. All IPv4 rules
+2. All IPv6 rules
+2. All rules that should be applied to both tables
+
+The order of the rules that will be applied are definite
+from the result the shell glob returns, which should be
+alphabetical. If rules must be applied in a special order,
+prefix them with a number like ``02-some-rule``.
+
 
 REQUIRED PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst
index 75d0740b..86d38a34 100644
--- a/cdist/conf/type/__iptables_rule/man.rst
+++ b/cdist/conf/type/__iptables_rule/man.rst
@@ -11,6 +11,10 @@ DESCRIPTION
 This cdist type allows you to manage iptable rules
 in a distribution independent manner.
 
+See :strong:`cdist-type__iptables_apply`\ (7) for the
+execution order of these rules. It will be executed
+automaticly to apply all rules non-volaite.
+
 
 REQUIRED PARAMETERS
 -------------------

From 1055e92545e74cf7845ede8094e783b4103efbcc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 2 Dec 2020 19:54:41 +0100
Subject: [PATCH 500/743] [setup.py] Add cdist.scan to packages

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 858c2c17..bfc8b495 100644
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@ os.chdir(cur)
 
 setup(
     name="cdist",
-    packages=["cdist", "cdist.core", "cdist.exec", "cdist.util", ],
+    packages=["cdist", "cdist.core", "cdist.exec", "cdist.scan", "cdist.util"],
     package_data={'cdist': package_data},
     scripts=["bin/cdist", "bin/cdist-dump", "bin/cdist-new-type"],
     version=cdist.version.VERSION,

From c7fa2efe6b780d63a779eed310bfa4ebf926d27e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 4 Dec 2020 15:30:08 +0100
Subject: [PATCH 501/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 9a91bf57..ad0aa75d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)
+
 6.9.2: 2020-11-20
 	* Documentation: Fix examples in best practice (Dennis Camera)
 	* Type __locale: Add state explorer (Matthias Stecher)

From d44b5cfdc990ce020ffe3af5085bbb93b8ce64a8 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 4 Dec 2020 15:31:35 +0100
Subject: [PATCH 502/743] Release 6.9.3

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index ad0aa75d..6c272fd7 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)
 
 6.9.2: 2020-11-20

From ba7d16a155cef46230d1dc650119f3d542a7f7f4 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 4 Dec 2020 17:57:55 +0100
Subject: [PATCH 503/743] __iptables_*: correct manpage spelling

---
 cdist/conf/type/__iptables_apply/man.rst | 2 +-
 cdist/conf/type/__iptables_rule/man.rst  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst
index 4109e789..3bef92cc 100644
--- a/cdist/conf/type/__iptables_apply/man.rst
+++ b/cdist/conf/type/__iptables_apply/man.rst
@@ -41,7 +41,7 @@ None
 EXAMPLES
 --------
 
-None (__iptables_apply is used by __iptables_rule automaticly)
+None (__iptables_apply is used by __iptables_rule automatically)
 
 
 SEE ALSO
diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst
index 86d38a34..afb71e01 100644
--- a/cdist/conf/type/__iptables_rule/man.rst
+++ b/cdist/conf/type/__iptables_rule/man.rst
@@ -31,7 +31,7 @@ state
 
 BOOLEAN PARAMETERS
 ------------------
-All rules without any of this parameter will be threaten like ``--v4`` because
+All rules without any of these parameters will be treated like ``--v4`` because
 of backward compatibility.
 
 v4

From 2d19856840400af57bd6667cc868ab314a3e07c2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 4 Dec 2020 18:26:03 +0100
Subject: [PATCH 504/743] [type/__package_pkgng_freebsd] Set ASSUME_ALWAYS_YES
 instead of -y

---
 cdist/conf/type/__package_pkgng_freebsd/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
index b5944177..05ba4cb2 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
+++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
@@ -75,7 +75,7 @@ execcmd(){
    esac
 
    if [ -z "${pkg_bootstrapped}" ]; then
-       echo "pkg bootstrap -y >/dev/null 2>&1"
+       echo "ASSUME_ALWAYS_YES=yes pkg bootstrap >/dev/null 2>&1"
    fi
 
    echo "$_cmd >/dev/null 2>&1"   # Silence the output of the command

From 087be130fa67d3fe195387ae0ab079f39c5066e1 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 4 Dec 2020 19:23:49 +0100
Subject: [PATCH 505/743] __iptables_apply: shorten copyright header

Do we need all the copyright header or is this sufficient? The licence
is given for cdist, but not on the target host. But it should be clear
anyway.
---
 .../type/__iptables_apply/files/init-script   | 34 ++++++-------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script
index 7faa2f92..e42017ae 100644
--- a/cdist/conf/type/__iptables_apply/files/init-script
+++ b/cdist/conf/type/__iptables_apply/files/init-script
@@ -1,28 +1,4 @@
 #!/bin/sh
-#
-# 2013 Nico Schottelius (nico-cdist at schottelius.org)
-# 2020 Matthias Stecher (matthiasstecher at gmx.de)
-#
-# This file is distributed with cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Originally written by:
-#   Nico Schottelius
-#   Zürisee, Mon Sep  2 18:38:27 CEST 2013
-#
 ### BEGIN INIT INFO
 # Provides:          iptables
 # Required-Start:    $local_fs $remote_fs
@@ -35,6 +11,16 @@
 #                    and saves/restores previous status
 ### END INIT INFO
 
+# Originally written by:
+#   Nico Schottelius
+#   Zürisee, Mon Sep  2 18:38:27 CEST 2013
+#
+# 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is distributed with cdist and licenced under the
+# GNU GPLv3+ WITHOUT ANY WARRANTY.
+
 
 # Read files and execute the content with the given commands
 #

From 3930f69456fd5a0d108a107e4ee61d87c9a73a56 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 6 Dec 2020 16:45:58 +0100
Subject: [PATCH 506/743] __block: fix escaping in here-doc

This changes the here-document to do not interpret any shell-things. It
also single-quotes some more strings that are printed to code-remote.

Fixes #838
---
 cdist/conf/type/__block/gencode-remote | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__block/gencode-remote b/cdist/conf/type/__block/gencode-remote
index 1f5cc033..acdb3286 100755
--- a/cdist/conf/type/__block/gencode-remote
+++ b/cdist/conf/type/__block/gencode-remote
@@ -46,10 +46,10 @@ fi
 
 remove_block() {
    cat << DONE
-tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
+tmpfile=\$(mktemp '${file}.cdist.XXXXXXXXXX')
 # preserve ownership and permissions of existing file
-if [ -f "$file" ]; then
-   cp -p "$file" "\$tmpfile"
+if [ -f '$file' ]; then
+   cp -p '$file' "\$tmpfile"
 fi
 awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
 {
@@ -63,8 +63,8 @@ awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
    } else {
       print
    }
-}' "$file" > "\$tmpfile"
-mv -f "\$tmpfile" "$file"
+}' '$file' > "\$tmpfile"
+mv -f "\$tmpfile" '$file'
 DONE
 }
 
@@ -77,7 +77,7 @@ case "$state_should" in
          echo add >> "$__messages_out"
       fi
       cat << DONE
-cat >> "$file" << ${__type##*/}_DONE
+cat >> '$file' << '${__type##*/}_DONE'
 $(cat "$block")
 ${__type##*/}_DONE
 DONE

From bed08c2c5c71ec4252743e0b64b75e7d8fedbd92 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sun, 6 Dec 2020 20:24:00 +0100
Subject: [PATCH 507/743] Deal with deprecation of imp module.

importlib has been a thing since Python 3.1, and imp has been deprecated since
3.4.

Insert random complaint here about not being able to use f-strings because they
were introduced in Python 3.6 and apparently we support Python 3.5 >,<.

Output diff before to after for ./bin/cdist-build-helper test (on heavy load):
```
1,2d0
< /usr/home/evilham/s/cdist/cdist/cdist/test/__main__.py:23: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
<   import imp
72c70
< ERROR: cdisttesthost: __file/tmp/foobar requires object __file without object id. Defined at /tmp/tmp.cdist.test.g87lx7c8/tmp.cdist.test.6ramsakx
---
> ERROR: cdisttesthost: __file/tmp/foobar requires object __file without object id. Defined at /tmp/tmp.cdist.test.aqdf6vjz/tmp.cdist.test.jgv3udel
76c74
< test_nonexistent_type_requirement (cdist.test.emulator.EmulatorTestCase) ... ERROR: cdisttesthost: __file/tmp/foobar requires object __does-not-exist/some-id, but type __does-not-exist does not exist. Defined at /tmp/tmp.cdist.test.mma5j8ln/tmp.cdist.test.3zg4by4d
---
> test_nonexistent_type_requirement (cdist.test.emulator.EmulatorTestCase) ... ERROR: cdisttesthost: __file/tmp/foobar requires object __does-not-exist/some-id, but type __does-not-exist does not exist. Defined at /tmp/tmp.cdist.test.t8d6ockr/tmp.cdist.test.uimxurg9
86c84
< test_initial_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running initial manifest /tmp/tmp.cdist.test.uvid60ij/759547ff4356de6e3d9e08522b0d0807/data/conf/manifest/dump_environment
---
> test_initial_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running initial manifest /tmp/tmp.cdist.test._cttcnrj/759547ff4356de6e3d9e08522b0d0807/data/conf/manifest/dump_environment
89c87
< test_type_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running type manifest /tmp/tmp.cdist.test.k1i2onpb/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__dump_environment/manifest for object __dump_environment/whatever
---
> test_type_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running type manifest /tmp/tmp.cdist.test.ukr7lrzd/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__dump_environment/manifest for object __dump_environment/whatever
272c270
< Ran 225 tests in 44.457s
---
> Ran 225 tests in 43.750s
```
---
 cdist/test/__main__.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cdist/test/__main__.py b/cdist/test/__main__.py
index c8c7df3b..8049c752 100644
--- a/cdist/test/__main__.py
+++ b/cdist/test/__main__.py
@@ -20,7 +20,7 @@
 #
 #
 
-import imp
+import importlib
 import os
 import sys
 import unittest
@@ -37,8 +37,9 @@ for possible_test in os.listdir(base_dir):
 
 suites = []
 for test_module in test_modules:
-    module_parameters = imp.find_module(test_module, [base_dir])
-    module = imp.load_module("cdist.test." + test_module, *module_parameters)
+    module_spec = importlib.util.find_spec("cdist.test.{}".format(test_module))
+    module = importlib.util.module_from_spec(module_spec)
+    module_spec.loader.exec_module(module)
 
     suite = unittest.defaultTestLoader.loadTestsFromModule(module)
     # print("Got suite: " + suite.__str__())

From 29662961731ef07fbc20213d0d87da1469f0d27c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 7 Dec 2020 19:47:52 +0100
Subject: [PATCH 508/743] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 6c272fd7..8b35901b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
+	* Core: Deal with deprecated imp in unit tests (Evil Ham)
+
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)
 

From c5ca4cd2e13516dfb55371c1600e32297c3343e9 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 7 Dec 2020 19:59:05 +0100
Subject: [PATCH 509/743] __block: securly quote via the quote function

Because the function already exists, it will be used for the file to be
changed, too. Therefor, no quotes are required for that value.

The prefix and suffix match was also improved: There is no regex check
any more (the regex did checked the whole line); instead it will do it
simple.
---
 cdist/conf/type/__block/gencode-remote | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__block/gencode-remote b/cdist/conf/type/__block/gencode-remote
index acdb3286..7a1f4064 100755
--- a/cdist/conf/type/__block/gencode-remote
+++ b/cdist/conf/type/__block/gencode-remote
@@ -46,28 +46,29 @@ fi
 
 remove_block() {
    cat << DONE
-tmpfile=\$(mktemp '${file}.cdist.XXXXXXXXXX')
+tmpfile=\$(mktemp ${quoted_file}.cdist.XXXXXXXXXX)
 # preserve ownership and permissions of existing file
-if [ -f '$file' ]; then
-   cp -p '$file' "\$tmpfile"
+if [ -f $quoted_file ]; then
+   cp -p $quoted_file "\$tmpfile"
 fi
-awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
+awk -v prefix=$(quote "$prefix") -v suffix=$(quote "$suffix") '
 {
-   if (match(\$0,prefix)) {
+   if (\$0 == prefix) {
       triggered=1
    }
    if (triggered) {
-      if (match(\$0,suffix)) {
+      if (\$0 == suffix) {
          triggered=0
       }
    } else {
       print
    }
-}' '$file' > "\$tmpfile"
-mv -f "\$tmpfile" '$file'
+}' $quoted_file > "\$tmpfile"
+mv -f "\$tmpfile" $quoted_file
 DONE
 }
 
+quoted_file="$(quote "$file")"
 case "$state_should" in
    present)
       if [ "$state_is" = "changed" ]; then
@@ -77,7 +78,7 @@ case "$state_should" in
          echo add >> "$__messages_out"
       fi
       cat << DONE
-cat >> '$file' << '${__type##*/}_DONE'
+cat >> $quoted_file << '${__type##*/}_DONE'
 $(cat "$block")
 ${__type##*/}_DONE
 DONE

From 14c81d6c7e20a2ed04c7318f6e8ecb51098390e8 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 8 Dec 2020 07:16:26 +0100
Subject: [PATCH 510/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8b35901b..4be08ae9 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
+	* Type __iptables: Add IPv6 support (Matthias Stecher)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From a58f5ffa7f59a7e28ba0a414097f154e4a497e88 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 8 Dec 2020 19:36:44 +0100
Subject: [PATCH 511/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 4be08ae9..3d6084f1 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
 	* Type __iptables: Add IPv6 support (Matthias Stecher)
+	* Type __block: Fix escaping in here-doc (Matthias Stecher)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From a5169ad858a4d6e9c378184db5736f55a86306e0 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 10 Dec 2020 21:24:26 +0100
Subject: [PATCH 512/743] new type __debian_backports

This new type will setup the backports distribution for the current
Debian release.
---
 cdist/conf/type/__debian_backports/man.rst    | 90 +++++++++++++++++++
 cdist/conf/type/__debian_backports/manifest   | 59 ++++++++++++
 .../parameter/default/mirror                  |  1 +
 .../parameter/default/state                   |  1 +
 .../__debian_backports/parameter/optional     |  2 +
 cdist/conf/type/__debian_backports/singleton  |  0
 6 files changed, 153 insertions(+)
 create mode 100644 cdist/conf/type/__debian_backports/man.rst
 create mode 100755 cdist/conf/type/__debian_backports/manifest
 create mode 100644 cdist/conf/type/__debian_backports/parameter/default/mirror
 create mode 100644 cdist/conf/type/__debian_backports/parameter/default/state
 create mode 100644 cdist/conf/type/__debian_backports/parameter/optional
 create mode 100644 cdist/conf/type/__debian_backports/singleton

diff --git a/cdist/conf/type/__debian_backports/man.rst b/cdist/conf/type/__debian_backports/man.rst
new file mode 100644
index 00000000..ba353f4e
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/man.rst
@@ -0,0 +1,90 @@
+cdist-type__debian_backports(7)
+===============================
+
+NAME
+----
+cdist-type__debian_backports - Install backports for Debain systems
+
+
+DESCRIPTION
+-----------
+This singleton type installs backports for the current Debian version.
+It aborts if backports are not supported for the specified os or no
+version codename could be fetched (like Debian unstable).
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    Represents the state of the backports repository. ``present`` or
+    ``absent``, defaults to ``present``.
+
+    Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
+
+mirror
+    The mirror to fetch the backports from. Will defaults to the Debian default
+    `<http://deb.debian.org/debian/>`_.
+
+    Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+MESSAGES
+--------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+   # setup the backports
+   __debian_backports
+   __debian_backports --state absent
+   __debian_backports --state present --mirror "http://ftp.de.debian.org/debian/"
+
+   # update
+   require="__debian_backports" __apt_update_index
+
+   # install a backports package
+   # currently for the buster release backports
+   require="__apt_update_index" __package_apt wireguard \
+        --target-release buster-backports
+
+
+ABORTS
+------
+Aborts if the detected os is not Debian.
+
+Aborts if no distribuition codename could be detected. This is common for the
+unstable distribution, but there is no backports repository for it already.
+
+
+SEE ALSO
+--------
+`Official Debian Backports site <https://backports.debian.org/>`_
+
+:strong:`cdist-type__apt_source`\ (7)
+
+
+AUTHORS
+-------
+Matthias Stecher <matthiasstecher at gmx.de>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Matthias Stecher. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__debian_backports/manifest
new file mode 100755
index 00000000..29bf9a43
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/manifest
@@ -0,0 +1,59 @@
+#!/bin/sh -e
+# __debian_backports/manifest
+#
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Enables/disables backports repository. Utilies __apt_source for it.
+#
+
+
+# detect backport distribution
+os="$(cat "$__global/explorer/os")"
+case "$os" in
+    debian)
+        # distribution codename from /etc/os-release
+        # lsb_release may not be given in all debian installations
+        dist="$(
+            . "$__global/explorer/os-release"
+            printf "%s" "$VERSION_CODENAME"
+        )"
+        ;;
+    *)
+        printf "Backports for %s are not supported!\n" "$os" >&2
+        exit 1
+        ;;
+esac
+
+# error if no codename given (e.g. on Debian unstable)
+if [ -z "$dist" ]; then
+    printf "No backports for unkown version of distribution %s!\n" "$os" >&2
+    exit 1
+fi
+
+
+# parameters
+state="$(cat "$__object/parameter/state")"
+mirror="$(cat "$__object/parameter/mirror")"
+
+# install the given backports repository
+__apt_source "${dist}-backports" \
+    --state "$state" \
+    --distribution "${dist}-backports" \
+    --component main \
+    --uri "$mirror"
diff --git a/cdist/conf/type/__debian_backports/parameter/default/mirror b/cdist/conf/type/__debian_backports/parameter/default/mirror
new file mode 100644
index 00000000..0965ef04
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/parameter/default/mirror
@@ -0,0 +1 @@
+http://deb.debian.org/debian/
diff --git a/cdist/conf/type/__debian_backports/parameter/default/state b/cdist/conf/type/__debian_backports/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__debian_backports/parameter/optional b/cdist/conf/type/__debian_backports/parameter/optional
new file mode 100644
index 00000000..4b05c235
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/parameter/optional
@@ -0,0 +1,2 @@
+state
+mirror
diff --git a/cdist/conf/type/__debian_backports/singleton b/cdist/conf/type/__debian_backports/singleton
new file mode 100644
index 00000000..e69de29b

From 0d96b31b5696f95b559450e17c71914d0746c0ce Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 11 Dec 2020 18:13:44 +0100
Subject: [PATCH 513/743] __debian_backports: pass shellcheck for sourced file

Because the sourced explorer can't be detected by shellcheck, it will be
completely disabled. Changing the path to /etc/os-release isn't
deterministic either.

The shellcheck wiki page suggests to use `source=/dev/null` instead of
`disable=SC1090`, but it was choosen to completely avoid that check ..
---
 cdist/conf/type/__debian_backports/manifest | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__debian_backports/manifest
index 29bf9a43..661e5281 100755
--- a/cdist/conf/type/__debian_backports/manifest
+++ b/cdist/conf/type/__debian_backports/manifest
@@ -30,6 +30,7 @@ case "$os" in
         # distribution codename from /etc/os-release
         # lsb_release may not be given in all debian installations
         dist="$(
+            # shellcheck disable=SC1090
             . "$__global/explorer/os-release"
             printf "%s" "$VERSION_CODENAME"
         )"

From bc2948a8a552e3bda3ab9ae8e8e199e831d81af2 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 11 Dec 2020 19:37:53 +0100
Subject: [PATCH 514/743] ++scan stuff

---
 cdist/scan/scan.py           | 15 +++++++++++++++
 docs/dev/logs/2020-10-29.org | 10 +++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index fcbf1899..00858581 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -56,6 +56,8 @@ from scapy.all import *
 # Datetime overwrites scapy.all.datetime - needs to be imported AFTER
 import datetime
 
+import cdist.config
+
 log = logging.getLogger("scan")
 
 class Trigger(object):
@@ -120,6 +122,19 @@ class Scanner(object):
             with open(fname, "w") as fd:
                 fd.write(f"{now}\n")
 
+    def config(self):
+        """
+        Configure a host
+
+        - Assume we are only called if necessary
+        - However we need to ensure to not run in parallel
+        - Maybe keep dict storing per host processes
+        - Save the result
+        - Save the output -> probably aligned to config mode
+
+        """
+
+
     def start(self):
         self.process = Process(target=self.scan)
         self.process.start()
diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org
index 4461be8c..03d6b3f4 100644
--- a/docs/dev/logs/2020-10-29.org
+++ b/docs/dev/logs/2020-10-29.org
@@ -54,4 +54,12 @@ VERBOSE: scan: Host fe80::f29f:c2ff:fe7c:275e is alive
 VERBOSE: scan: Host fe80::ba69:f4ff:fec5:8db7 is alive
 VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f863 is alive
 VERBOSE: scan: Host fe80::21b:fcff:feee:f4bc is alive
-...
+** Better usage -> saving the env
+    sudo -E cdist scan -b -I wlan0 -vv
+** TODO Implement actual configuration step
+   - Also serves as a nice PoC
+   - Might need to escape literal IPv6 addresses for scp
+** TODO Define how to map link local address to something useful
+   - via reverse DNS?
+   - via link local in manifest?
+** TODO define ignorehosts?

From c4d19a23193ec13eda96a3d7536ba338d0801825 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 09:36:17 +0100
Subject: [PATCH 515/743] __debian_backports -> __apt_backports; add wider os
 support

As discussed in the chat, this type now supports a broader list of OSes
which it supports backports for. Because of this, it was renamed to
something more generic. "apt" should fit in.
---
 .../man.rst                                   | 27 ++++++++----
 .../manifest                                  | 41 ++++++++++++++-----
 .../parameter/default/state                   |  0
 .../parameter/optional                        |  0
 .../singleton                                 |  0
 .../parameter/default/mirror                  |  1 -
 6 files changed, 49 insertions(+), 20 deletions(-)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/man.rst (66%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/manifest (59%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/parameter/default/state (100%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/parameter/optional (100%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/singleton (100%)
 delete mode 100644 cdist/conf/type/__debian_backports/parameter/default/mirror

diff --git a/cdist/conf/type/__debian_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
similarity index 66%
rename from cdist/conf/type/__debian_backports/man.rst
rename to cdist/conf/type/__apt_backports/man.rst
index ba353f4e..7d269fbb 100644
--- a/cdist/conf/type/__debian_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -3,13 +3,13 @@ cdist-type__debian_backports(7)
 
 NAME
 ----
-cdist-type__debian_backports - Install backports for Debain systems
+cdist-type__apt_backports - Install backports
 
 
 DESCRIPTION
 -----------
-This singleton type installs backports for the current Debian version.
-It aborts if backports are not supported for the specified os or no
+This singleton type installs backports for the current OS release.
+It aborts if backports are not supported for the specified OS or no
 version codename could be fetched (like Debian unstable).
 
 
@@ -27,8 +27,8 @@ state
     Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
 
 mirror
-    The mirror to fetch the backports from. Will defaults to the Debian default
-    `<http://deb.debian.org/debian/>`_.
+    The mirror to fetch the backports from. Will defaults to the generic
+    mirror of the current OS.
 
     Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
 
@@ -49,12 +49,12 @@ EXAMPLES
 .. code-block:: sh
 
    # setup the backports
-   __debian_backports
-   __debian_backports --state absent
-   __debian_backports --state present --mirror "http://ftp.de.debian.org/debian/"
+   __apt_backports
+   __apt_backports --state absent
+   __apt_backports --state present --mirror "http://ftp.de.debian.org/debian/"
 
    # update
-   require="__debian_backports" __apt_update_index
+   require="__apt_backports" __apt_update_index
 
    # install a backports package
    # currently for the buster release backports
@@ -70,6 +70,15 @@ Aborts if no distribuition codename could be detected. This is common for the
 unstable distribution, but there is no backports repository for it already.
 
 
+CAVEATS
+-------
+For Ubuntu, it setup all componenents for the backports repository: ``main``,
+``restricted``, ``universe`` and ``multiverse``. The user may not want to
+install proprietary packages, which will only be installed if the user
+explicitly uses the backports target-release. The user may change this behavior
+to install backports packages without the need of explicitly select it.
+
+
 SEE ALSO
 --------
 `Official Debian Backports site <https://backports.debian.org/>`_
diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__apt_backports/manifest
similarity index 59%
rename from cdist/conf/type/__debian_backports/manifest
rename to cdist/conf/type/__apt_backports/manifest
index 661e5281..e5358dea 100755
--- a/cdist/conf/type/__debian_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -1,5 +1,5 @@
 #!/bin/sh -e
-# __debian_backports/manifest
+# __apt_backports/manifest
 #
 # 2020 Matthias Stecher (matthiasstecher at gmx.de)
 #
@@ -23,18 +23,34 @@
 #
 
 
+# Get the distribution codename by /etc/os-release.
+#  is already executed in a subshell by string substitution
+#  lsb_release may not be given in all installations
+codename_os_release() {
+    # shellcheck disable=SC1090
+    . "$__global/explorer/os-release"
+    printf "%s" "$VERSION_CODENAME"
+}
+
 # detect backport distribution
 os="$(cat "$__global/explorer/os")"
 case "$os" in
     debian)
-        # distribution codename from /etc/os-release
-        # lsb_release may not be given in all debian installations
-        dist="$(
-            # shellcheck disable=SC1090
-            . "$__global/explorer/os-release"
-            printf "%s" "$VERSION_CODENAME"
-        )"
+        dist="$( codename_os_release )"
+        components="main"
+        mirror="http://deb.debian.org/debian/"
         ;;
+    devuan)
+        dist="$( codename_os_release )"
+        components="main"
+        mirror="http://deb.devuan.org/merged"
+        ;;
+    ubuntu)
+        dist="$( codename_os_release )"
+        components="main restricted universe multiverse"
+        mirror="http://archive.ubuntu.com/ubuntu"
+        ;;
+
     *)
         printf "Backports for %s are not supported!\n" "$os" >&2
         exit 1
@@ -50,11 +66,16 @@ fi
 
 # parameters
 state="$(cat "$__object/parameter/state")"
-mirror="$(cat "$__object/parameter/mirror")"
+
+# mirror already set for the os, only override user-values
+if [ -f "$__object/parameter/mirror" ]; then
+    mirror="$(cat "$__object/parameter/mirror")"
+fi
+
 
 # install the given backports repository
 __apt_source "${dist}-backports" \
     --state "$state" \
     --distribution "${dist}-backports" \
-    --component main \
+    --component "$components" \
     --uri "$mirror"
diff --git a/cdist/conf/type/__debian_backports/parameter/default/state b/cdist/conf/type/__apt_backports/parameter/default/state
similarity index 100%
rename from cdist/conf/type/__debian_backports/parameter/default/state
rename to cdist/conf/type/__apt_backports/parameter/default/state
diff --git a/cdist/conf/type/__debian_backports/parameter/optional b/cdist/conf/type/__apt_backports/parameter/optional
similarity index 100%
rename from cdist/conf/type/__debian_backports/parameter/optional
rename to cdist/conf/type/__apt_backports/parameter/optional
diff --git a/cdist/conf/type/__debian_backports/singleton b/cdist/conf/type/__apt_backports/singleton
similarity index 100%
rename from cdist/conf/type/__debian_backports/singleton
rename to cdist/conf/type/__apt_backports/singleton
diff --git a/cdist/conf/type/__debian_backports/parameter/default/mirror b/cdist/conf/type/__debian_backports/parameter/default/mirror
deleted file mode 100644
index 0965ef04..00000000
--- a/cdist/conf/type/__debian_backports/parameter/default/mirror
+++ /dev/null
@@ -1 +0,0 @@
-http://deb.debian.org/debian/

From 49aec0b5e40ba05e7703565731aae6b0a9e5936a Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 09:40:47 +0100
Subject: [PATCH 516/743] __apt_backports: list supported OSes

The manpage now lists all OSes where this type supports backports.
---
 cdist/conf/type/__apt_backports/man.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
index 7d269fbb..fd311361 100644
--- a/cdist/conf/type/__apt_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -12,6 +12,12 @@ This singleton type installs backports for the current OS release.
 It aborts if backports are not supported for the specified OS or no
 version codename could be fetched (like Debian unstable).
 
+It supports backports from following OSes:
+
+- Debian
+- Devuan
+- Ubuntu
+
 
 REQUIRED PARAMETERS
 -------------------

From fafa3d9ea55f38ae0a350a463d64b7ef1d70c7bb Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 10:00:23 +0100
Subject: [PATCH 517/743] __apt_backports: update index if required

This type now automatically calls the type __apt_update_index to update
the package index if something changed.
---
 cdist/conf/type/__apt_backports/man.rst  | 13 +++++++------
 cdist/conf/type/__apt_backports/manifest |  5 ++++-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
index fd311361..c578ce6b 100644
--- a/cdist/conf/type/__apt_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -9,8 +9,11 @@ cdist-type__apt_backports - Install backports
 DESCRIPTION
 -----------
 This singleton type installs backports for the current OS release.
-It aborts if backports are not supported for the specified OS or no
-version codename could be fetched (like Debian unstable).
+It aborts if backports are not supported for the specified OS or
+no version codename could be fetched (like Debian unstable).
+
+The package index will be automatically updated by the type
+:strong:`cdist-type__apt_update_index`\ (7) if required.
 
 It supports backports from following OSes:
 
@@ -59,12 +62,9 @@ EXAMPLES
    __apt_backports --state absent
    __apt_backports --state present --mirror "http://ftp.de.debian.org/debian/"
 
-   # update
-   require="__apt_backports" __apt_update_index
-
    # install a backports package
    # currently for the buster release backports
-   require="__apt_update_index" __package_apt wireguard \
+   require="__apt_backports" __package_apt wireguard \
         --target-release buster-backports
 
 
@@ -90,6 +90,7 @@ SEE ALSO
 `Official Debian Backports site <https://backports.debian.org/>`_
 
 :strong:`cdist-type__apt_source`\ (7)
+:strong:`cdist-type__apt_update_index`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index e5358dea..c490a103 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -19,7 +19,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Enables/disables backports repository. Utilies __apt_source for it.
+# Enables/disables backports repository. Utilises __apt_source for it.
 #
 
 
@@ -79,3 +79,6 @@ __apt_source "${dist}-backports" \
     --distribution "${dist}-backports" \
     --component "$components" \
     --uri "$mirror"
+
+# update the index if the source changed
+require="__apt_source/${dist}-backports" __apt_update_index

From 645734c62959c694597826598be3b34c0edc79d2 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 12 Dec 2020 12:15:17 +0100
Subject: [PATCH 518/743] [explorer/os_version] Improve FreeBSD support.

It looks like uname -r is not the most reliable way to get the target patch
level for the target system.

For more information see:
https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
---
 cdist/conf/explorer/os_version | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index a7b1d3bc..3b02dedd 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -70,6 +70,11 @@ case "$("$__explorer/os")" in
    macosx)
       sw_vers -productVersion
    ;;
+   freebsd)
+      # Apparently uname -r is not a reliable way to get the patch level.
+      # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
+      freebsd-version
+   ;;
    *bsd|solaris)
       uname -r
    ;;

From fca35fc858d09fc649dc5e7f0964cef4af9e09f0 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 17:29:58 +0100
Subject: [PATCH 519/743] __apt_backports: fix explorer call

s/-/_/ because the explorers are following an other convention :-)
---
 cdist/conf/type/__apt_backports/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index c490a103..c2943725 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -28,7 +28,7 @@
 #  lsb_release may not be given in all installations
 codename_os_release() {
     # shellcheck disable=SC1090
-    . "$__global/explorer/os-release"
+    . "$__global/explorer/os_release"
     printf "%s" "$VERSION_CODENAME"
 }
 

From 27aca06fb893c84601e14bc30890ea6be9300dcd Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 17:34:51 +0100
Subject: [PATCH 520/743] __apt_backports: undo __apt_update_index call

Becuase it is already done by __apt_source.
---
 cdist/conf/type/__apt_backports/man.rst  | 4 +---
 cdist/conf/type/__apt_backports/manifest | 3 ---
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
index c578ce6b..7036fb84 100644
--- a/cdist/conf/type/__apt_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -12,8 +12,7 @@ This singleton type installs backports for the current OS release.
 It aborts if backports are not supported for the specified OS or
 no version codename could be fetched (like Debian unstable).
 
-The package index will be automatically updated by the type
-:strong:`cdist-type__apt_update_index`\ (7) if required.
+The package index will be automatically updated if required.
 
 It supports backports from following OSes:
 
@@ -90,7 +89,6 @@ SEE ALSO
 `Official Debian Backports site <https://backports.debian.org/>`_
 
 :strong:`cdist-type__apt_source`\ (7)
-:strong:`cdist-type__apt_update_index`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index c2943725..bc47d8de 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -79,6 +79,3 @@ __apt_source "${dist}-backports" \
     --distribution "${dist}-backports" \
     --component "$components" \
     --uri "$mirror"
-
-# update the index if the source changed
-require="__apt_source/${dist}-backports" __apt_update_index

From 71f22831175ec41e9348a759fcdaf0eceb1a2a52 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 13 Dec 2020 16:03:39 +0100
Subject: [PATCH 521/743] ++changelog

---
 docs/changelog | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 3d6084f1..290f94d0 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -2,10 +2,12 @@ Changelog
 ---------
 
 next:
-	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
+	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
 	* Type __iptables: Add IPv6 support (Matthias Stecher)
 	* Type __block: Fix escaping in here-doc (Matthias Stecher)
+	* Explorer os_version: Improve FreeBSD support (Evil Ham)
+	* New type: __apt_backports (Matthias Stecher)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From 932e2496ed87b830a468a263cf1b1f5e6a88e2ef Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 15 Dec 2020 18:40:39 +0100
Subject: [PATCH 522/743] [type/__postgres_role] Lint

---
 .../conf/type/__postgres_role/explorer/state  | 32 ++++----
 .../conf/type/__postgres_role/gencode-remote  | 78 ++++++++++---------
 2 files changed, 58 insertions(+), 52 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index c8e1fa9d..110d29d5 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -11,32 +11,32 @@
 #
 # cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$("${__explorer}/os")"
+case $("${__explorer:?}/os")
 in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
+	(netbsd)
+		postgres_user='pgsql'
+		;;
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(*)
+		postgres_user='postgres'
+		;;
 esac
 
 
-name="$__object_id"
+rolename=${__object_id:?}
 
-if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='$name'\"")"
+if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='${rolename}'\"")"
 then
-    echo 'present'
+	echo 'present'
 else
-    echo 'absent'
+	echo 'absent'
 fi
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 282294c9..4fb761ed 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -11,55 +11,61 @@
 #
 # cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
+case $(cat "${__global:?}/explorer/os")
 in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
+	(netbsd)
+		postgres_user='pgsql'
+		;;
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(*)
+		postgres_user='postgres'
+		;;
 esac
 
 
-name="$__object_id"
-state_is="$(cat "$__object/explorer/state")"
-state_should="$(cat "$__object/parameter/state")"
+rolename=${__object_id:?}
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 
-[ "$state_is" = "$state_should" ] && exit 0
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
 
-case "$state_should" in
-    present)
-        if [ -f "$__object/parameter/password" ]; then
-            password="$(cat "$__object/parameter/password")"
-        fi
-        booleans=""
-        for boolean in login createdb createrole superuser; do
-            if [ ! -f "$__object/parameter/$boolean" ]; then
-                boolean="no${boolean}"
-            fi
-            upper=$(echo $boolean | tr '[:lower:]' '[:upper:]')
-            booleans="$booleans $upper"
-        done
+case ${state_should}
+in
+	(present)
+		if test -f "${__object:?}/parameter/password"
+		then
+			password=$(cat "${__object:?}/parameter/password")
+		fi
+		booleans=
+		for boolean in login createdb createrole superuser
+		do
+			if test ! -f "${__object:?}/parameter/${boolean}"
+			then
+				boolean="no${boolean}"
+			fi
+			booleans="${booleans} $(echo ${boolean} | tr '[:lower:]' '[:upper:]')"
+		done
 
-        [ -n "$password" ] && password="PASSWORD '$password'"
-        cat << EOF
-su - '$postgres_user' -c "psql postgres -wc \"CREATE ROLE \\\\\"$name\\\\\" WITH $password $booleans;\""
+		[ -n "${password}" ] && password="PASSWORD '${password}'"
+		cat << EOF
+su - '${postgres_user}' -c "psql postgres -wc 'CREATE ROLE \\"${rolename}\\" WITH ${password} ${booleans};'"
 EOF
-    ;;
-    absent)
-        cat << EOF
-su - '$postgres_user' -c "dropuser \"$name\""
+		;;
+	(absent)
+		cat << EOF
+su - '${postgres_user}' -c "dropuser '${rolename}'"
 EOF
-    ;;
+		;;
 esac

From c36df82882fedb6ce3630e4718c46992a7c153a0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 15 Dec 2020 21:05:55 +0100
Subject: [PATCH 523/743] [type/__postgres_role] ALTER ROLE when parameters
 change

---
 .../conf/type/__postgres_role/explorer/state  | 42 +++++++++++-
 .../conf/type/__postgres_role/gencode-remote  | 65 +++++++++++++++----
 2 files changed, 91 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 110d29d5..033afcb2 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -31,11 +32,48 @@ in
 		;;
 esac
 
-
 rolename=${__object_id:?}
 
-if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='${rolename}'\"")"
+role_properties=$(
+	cmd=$(printf "psql -F '\034' -R '\036' -wAc \"SELECT * FROM pg_roles WHERE rolname='%s'\"" "${rolename}")
+	su -l "${postgres_user}" -c "${cmd}" \
+	| awk '
+	  BEGIN { RS = "\036"; FS = "\034" }
+	  /^\([0-9]+ rows?\)/ { exit }
+	  NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next }
+	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }
+	  '
+)
+
+if test -n "${role_properties}"
 then
+	# Check if the user's properties match the parameters
+	for prop in login createdb createrole superuser
+	do
+		bool_should=$(test -f "${__object:?}/parameter/${prop}" && echo 't' || echo 'f')
+		bool_is=$(
+			printf '%s\n' "${role_properties}" |
+			awk -F '=' -v key="${prop}" '
+			BEGIN {
+				if (key == "login")
+					key = "canlogin"
+				else if (key == "superuser")
+					key = "super"
+				key = "rol" key
+			}
+			$1 == key {
+				sub(/^[^=]*=/, "")
+				print
+			}
+			'
+		)
+
+		test "${bool_is}" = "${bool_should}" || {
+			echo 'different'
+			exit 0
+		}
+	done
+
 	echo 'present'
 else
 	echo 'absent'
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 4fb761ed..7837976c 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,6 +19,15 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+quote() {
+	if test $# -gt 0
+	then
+		printf '%s' "$*"
+	else
+		cat -
+	fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
+}
+
 case $(cat "${__global:?}/explorer/os")
 in
 	(netbsd)
@@ -44,28 +54,55 @@ fi
 case ${state_should}
 in
 	(present)
-		if test -f "${__object:?}/parameter/password"
+		if test -s "${__object:?}/parameter/password"
 		then
-			password=$(cat "${__object:?}/parameter/password")
+			quoted_password=$(
+				delim='$$'
+				while grep -q -F "${delim}" "${__object:?}/parameter/password"
+				do
+					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
+				done
+
+				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
+				# shellcheck disable=SC2016
+				printf '%s%s%s' "${delim}" "${raw_passwd%?.}" "${delim}"
+			)
 		fi
+
 		booleans=
 		for boolean in login createdb createrole superuser
 		do
-			if test ! -f "${__object:?}/parameter/${boolean}"
-			then
-				boolean="no${boolean}"
-			fi
-			booleans="${booleans} $(echo ${boolean} | tr '[:lower:]' '[:upper:]')"
+			booleans="${booleans}${booleans:+ }$(
+				if test -f "${__object:?}/parameter/${boolean}"
+				then
+					echo "${boolean}"
+				else
+					echo "no${boolean}"
+				fi \
+				| tr '[:lower:]' '[:upper:]')"
 		done
 
-		[ -n "${password}" ] && password="PASSWORD '${password}'"
-		cat << EOF
-su - '${postgres_user}' -c "psql postgres -wc 'CREATE ROLE \\"${rolename}\\" WITH ${password} ${booleans};'"
-EOF
+		case ${state_is}
+		in
+			(absent)
+				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				;;
+			(different)
+				query=$(printf 'ALTER ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				;;
+			(*)
+				exit 1	# TODO: error msg
+				;;
+		esac
+
+		psql_cmd=$(printf 'psql postgres -wc %s' "$(quote "${query}")" | quote)
+		printf "su -l '%s' -c %s\\n" "${postgres_user}" "${psql_cmd}"
 		;;
 	(absent)
-		cat << EOF
-su - '${postgres_user}' -c "dropuser '${rolename}'"
-EOF
+		printf "su -l '%s' -c 'dropuser '\\\\'%s\\\\'\\n" \
+			"${postgres_user}" \
+			"$(quote "${rolename}")"
 		;;
 esac

From 7b7ca4d385ccd025cbefced58419468651652da8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 16 Dec 2020 19:06:50 +0100
Subject: [PATCH 524/743] [type/__postgres_role] Handle password changes

---
 .../conf/type/__postgres_role/explorer/state  | 76 +++++++++++++++++--
 .../conf/type/__postgres_role/gencode-remote  | 23 ++++--
 2 files changed, 88 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 033afcb2..4aadbdff 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -34,9 +34,23 @@ esac
 
 rolename=${__object_id:?}
 
+
+psql_query() {
+	su -l "${postgres_user}" -c "$(
+		printf "psql -F '\034' -R '\036' -wAc '%s'" \
+		"$(printf %s "$*" | sed "s/'/'\\\\''/g")"
+	)"
+}
+
+password_check_login() (
+	PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .)
+	PGPASSWORD=${PGPASSWORD%?.}
+	export PGPASSWORD
+	psql -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
+)
+
 role_properties=$(
-	cmd=$(printf "psql -F '\034' -R '\036' -wAc \"SELECT * FROM pg_roles WHERE rolname='%s'\"" "${rolename}")
-	su -l "${postgres_user}" -c "${cmd}" \
+	psql_query "SELECT * FROM pg_roles WHERE rolname = '${rolename}'" \
 	| awk '
 	  BEGIN { RS = "\036"; FS = "\034" }
 	  /^\([0-9]+ rows?\)/ { exit }
@@ -69,12 +83,62 @@ then
 		)
 
 		test "${bool_is}" = "${bool_should}" || {
-			echo 'different'
-			exit 0
+			state='different properties'
 		}
 	done
 
-	echo 'present'
+	# Check password
+	passwd_stored=$(
+		psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \
+		| awk 'BEGIN { RS = "\036" } NR == 2'
+		printf .
+	)
+	passwd_stored=${passwd_stored%?.}
+
+	passwd_should=$(cat "${__object}/parameter/password"; printf .)
+	passwd_should=${passwd_should%?.}
+
+	if expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
+	then
+		# SCRAM-SHA-256 "encrypted" password
+		# NOTE: There is currently no easy way to check SCRAM passwords
+		password_check_login || state="${state:-different} password"
+	elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null
+	then
+		# MD5 "encrypted" password
+		if command -v md5sum >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| md5sum - | sed -e 's/[^0-9a-f]*$//')
+		elif command -v gmd5sum >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| gmd5sum - | sed -e 's/[^0-9a-f]*$//')
+		elif command -v openssl >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| openssl dgst -md5 | sed 's/^.* //')
+		fi
+
+		if test -n "${should_md5}"
+		then
+			test "${passwd_stored}" = "md5${should_md5}" \
+			|| state="${state:-different} password"
+		else
+			password_check_login || state="${state:-different} password"
+		fi
+	else
+		# unencrypted password (unsupported since PostgreSQL 10)
+		test "${passwd_stored}" = "${passwd_should}" \
+		|| state="${state:-different} password"
+	fi
+
+	test -n "${state}" || state='present'
 else
-	echo 'absent'
+	state='absent'
 fi
+
+echo "${state}"
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 7837976c..fd590a4b 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -58,7 +58,9 @@ in
 		then
 			quoted_password=$(
 				delim='$$'
-				while grep -q -F "${delim}" "${__object:?}/parameter/password"
+				# NOTE: Strip away trailing $ because with it the check breaks
+				#       if the password ends with $ + random value.
+				while grep -q -F "${delim%$}" "${__object:?}/parameter/password"
 				do
 					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
 				done
@@ -88,12 +90,23 @@ in
 				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
 					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
 				;;
-			(different)
-				query=$(printf 'ALTER ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+			(different*)
+				query="ALTER ROLE \"${rolename}\" WITH"
+
+				if expr "${state_is}" : 'different.*properties' >/dev/null
+				then
+					query="${query} ${booleans}"
+				fi
+				if expr "${state_is}" : 'different.*password' >/dev/null
+				then
+					query="${query} PASSWORD ${quoted_password:-NULL}"
+				fi
+
+				query="${query};"
 				;;
 			(*)
-				exit 1	# TODO: error msg
+				printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2
+				exit 1
 				;;
 		esac
 

From 4859c27900d125841c703e0270dd8c0eae05d504 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 16:57:03 +0100
Subject: [PATCH 525/743] [type/__postgres_role] Refactor gencode-remote

---
 .../conf/type/__postgres_role/gencode-remote  | 65 ++++++++++---------
 1 file changed, 34 insertions(+), 31 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index fd590a4b..540eb606 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -51,9 +51,6 @@ then
 	exit 0
 fi
 
-case ${state_should}
-in
-	(present)
 		if test -s "${__object:?}/parameter/password"
 		then
 			quoted_password=$(
@@ -64,6 +61,11 @@ in
 				do
 					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
 				done
+psql_query() {
+	printf 'su -l %s -c %s\n' \
+		"$(quote "${postgres_user}")" \
+		"$(quote "psql postgres -wc $(quote "$1")")"
+}
 
 				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
 				# shellcheck disable=SC2016
@@ -71,51 +73,52 @@ in
 			)
 		fi
 
-		booleans=
-		for boolean in login createdb createrole superuser
-		do
-			booleans="${booleans}${booleans:+ }$(
-				if test -f "${__object:?}/parameter/${boolean}"
-				then
-					echo "${boolean}"
-				else
-					echo "no${boolean}"
-				fi \
-				| tr '[:lower:]' '[:upper:]')"
-		done
+role_properties_should() {
+	_props=
+	for _prop in login createdb createrole superuser
+	do
+		_props="${_props}${_props:+ }$(
+			if test -f "${__object:?}/parameter/${_prop}"
+			then
+				echo "${_prop}"
+			else
+				echo "no${_prop}"
+			fi \
+			| tr '[:lower:]' '[:upper:]')"
+	done
+	printf '%s\n' "${_props}"
+	unset _prop _props
+}
 
+case ${state_should}
+in
+	(present)
 		case ${state_is}
 		in
 			(absent)
-				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				psql_query "$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "$(role_properties_should)" "${quoted_password:-NULL}")"
 				;;
 			(different*)
-				query="ALTER ROLE \"${rolename}\" WITH"
-
 				if expr "${state_is}" : 'different.*properties' >/dev/null
 				then
-					query="${query} ${booleans}"
-				fi
-				if expr "${state_is}" : 'different.*password' >/dev/null
-				then
-					query="${query} PASSWORD ${quoted_password:-NULL}"
+					psql_query "ALTER ROLE \"${rolename}\" WITH $(role_properties_should);"
 				fi
 
-				query="${query};"
+				if expr "${state_is}" : 'different.*password' >/dev/null
+				then
+					psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD ${quoted_password:-NULL};"
+				fi
 				;;
 			(*)
 				printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2
 				exit 1
 				;;
 		esac
-
-		psql_cmd=$(printf 'psql postgres -wc %s' "$(quote "${query}")" | quote)
-		printf "su -l '%s' -c %s\\n" "${postgres_user}" "${psql_cmd}"
 		;;
 	(absent)
-		printf "su -l '%s' -c 'dropuser '\\\\'%s\\\\'\\n" \
-			"${postgres_user}" \
-			"$(quote "${rolename}")"
+		printf 'su -l %s -c %s\n' \
+			"$(quote "${postgres_user}")" \
+			"$(quote "dropuser $(quote "${rolename}")")"
 		;;
 esac

From 1180f13ed6c12e0277bebdf32bdd9840320d0fb9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 16:58:32 +0100
Subject: [PATCH 526/743] [type/__postgres_role] Fix setting password

We need to make sure that the password does not end up in ~/.psql_history.
---
 .../conf/type/__postgres_role/gencode-remote  | 38 ++++++++++---------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 540eb606..15c3e692 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -51,27 +51,29 @@ then
 	exit 0
 fi
 
-		if test -s "${__object:?}/parameter/password"
-		then
-			quoted_password=$(
-				delim='$$'
-				# NOTE: Strip away trailing $ because with it the check breaks
-				#       if the password ends with $ + random value.
-				while grep -q -F "${delim%$}" "${__object:?}/parameter/password"
-				do
-					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
-				done
 psql_query() {
 	printf 'su -l %s -c %s\n' \
 		"$(quote "${postgres_user}")" \
 		"$(quote "psql postgres -wc $(quote "$1")")"
 }
 
-				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
-				# shellcheck disable=SC2016
-				printf '%s%s%s' "${delim}" "${raw_passwd%?.}" "${delim}"
-			)
-		fi
+psql_set_password() {
+	# NOTE: Always make sure that the password does not end up in psql_history!
+	if test -s "${__object:?}/parameter/password"
+	then
+		cat <<-EOF
+		exec 3< "\${__object:?}/parameter/password"
+		su -l '${postgres_user}' -c 'psql -q postgres -w' <<'SQL'
+		\set HISTFILE /dev/null
+		\set pw \`cat <&3\`
+		ALTER ROLE "${rolename}" WITH PASSWORD :'pw';
+		SQL
+		exec 3<&-
+		EOF
+	else
+		psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD NULL;"
+	fi
+}
 
 role_properties_should() {
 	_props=
@@ -96,8 +98,8 @@ in
 		case ${state_is}
 		in
 			(absent)
-				psql_query "$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "$(role_properties_should)" "${quoted_password:-NULL}")"
+				psql_query "CREATE ROLE \"${rolename}\" WITH $(role_properties_should);"
+				psql_set_password
 				;;
 			(different*)
 				if expr "${state_is}" : 'different.*properties' >/dev/null
@@ -107,7 +109,7 @@ in
 
 				if expr "${state_is}" : 'different.*password' >/dev/null
 				then
-					psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD ${quoted_password:-NULL};"
+					psql_set_password
 				fi
 				;;
 			(*)

From 99d82fd0d5ab6d6ccd95fe8a7d3b24f5d82a4cce Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 17:05:58 +0100
Subject: [PATCH 527/743] [type/__postgres_role] Always set psql -q

---
 cdist/conf/type/__postgres_role/explorer/state | 4 ++--
 cdist/conf/type/__postgres_role/gencode-remote | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 4aadbdff..77779662 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -37,7 +37,7 @@ rolename=${__object_id:?}
 
 psql_query() {
 	su -l "${postgres_user}" -c "$(
-		printf "psql -F '\034' -R '\036' -wAc '%s'" \
+		printf "psql -q -F '\034' -R '\036' -wAc '%s'" \
 		"$(printf %s "$*" | sed "s/'/'\\\\''/g")"
 	)"
 }
@@ -46,7 +46,7 @@ password_check_login() (
 	PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .)
 	PGPASSWORD=${PGPASSWORD%?.}
 	export PGPASSWORD
-	psql -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
+	psql -q -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
 )
 
 role_properties=$(
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 15c3e692..877c8135 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -54,7 +54,7 @@ fi
 psql_query() {
 	printf 'su -l %s -c %s\n' \
 		"$(quote "${postgres_user}")" \
-		"$(quote "psql postgres -wc $(quote "$1")")"
+		"$(quote "psql postgres -q -w -c $(quote "$1")")"
 }
 
 psql_set_password() {
@@ -63,7 +63,7 @@ psql_set_password() {
 	then
 		cat <<-EOF
 		exec 3< "\${__object:?}/parameter/password"
-		su -l '${postgres_user}' -c 'psql -q postgres -w' <<'SQL'
+		su -l '${postgres_user}' -c 'psql -q -w postgres' <<'SQL'
 		\set HISTFILE /dev/null
 		\set pw \`cat <&3\`
 		ALTER ROLE "${rolename}" WITH PASSWORD :'pw';

From 8dc2c4207cdd826b9e64cc91cfc7c918cddd5f35 Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Fri, 18 Dec 2020 11:16:28 +0100
Subject: [PATCH 528/743] Added optional dirmode parameter to set the mode of
 (optional) the directory.

---
 cdist/conf/type/__dot_file/man.rst                   | 3 +++
 cdist/conf/type/__dot_file/manifest                  | 2 ++
 cdist/conf/type/__dot_file/parameter/default/dirmode | 1 +
 cdist/conf/type/__dot_file/parameter/optional        | 1 +
 4 files changed, 7 insertions(+)
 create mode 100644 cdist/conf/type/__dot_file/parameter/default/dirmode

diff --git a/cdist/conf/type/__dot_file/man.rst b/cdist/conf/type/__dot_file/man.rst
index ae65eb95..ba7621a1 100644
--- a/cdist/conf/type/__dot_file/man.rst
+++ b/cdist/conf/type/__dot_file/man.rst
@@ -25,6 +25,9 @@ user
 OPTIONAL PARAMETERS
 -------------------
 
+dirmode
+    forwarded to :strong:`__directory` type as mode
+
 mode
     forwarded to :strong:`__file` type
 
diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest
index 5e4957e5..02dadf05 100755
--- a/cdist/conf/type/__dot_file/manifest
+++ b/cdist/conf/type/__dot_file/manifest
@@ -19,6 +19,7 @@ set -eu
 user="$(cat "${__object}/parameter/user")"
 home="$(cat "${__object}/explorer/home")"
 primary_group="$(cat "${__object}/explorer/primary_group")"
+dirmode="$(cat "${__object}/parameter/dirmode")"
 
 # Create parent directory. Type __directory has flag 'parents', but it
 # will leave us with root-owned directory in user home, which is not
@@ -36,6 +37,7 @@ export CDIST_ORDER_DEPENDENCY
 for dir ; do
 	__directory "${home}/${dir}"   \
 	    --group "${primary_group}" \
+            --mode "${dirmode}" \
 	    --owner "${user}"
 done
 
diff --git a/cdist/conf/type/__dot_file/parameter/default/dirmode b/cdist/conf/type/__dot_file/parameter/default/dirmode
new file mode 100644
index 00000000..e9745d1f
--- /dev/null
+++ b/cdist/conf/type/__dot_file/parameter/default/dirmode
@@ -0,0 +1 @@
+0700
diff --git a/cdist/conf/type/__dot_file/parameter/optional b/cdist/conf/type/__dot_file/parameter/optional
index ccab9fa6..9f7f83fb 100644
--- a/cdist/conf/type/__dot_file/parameter/optional
+++ b/cdist/conf/type/__dot_file/parameter/optional
@@ -1,3 +1,4 @@
 state
 mode
 source
+dirmode

From 4bae2863dbc1f6b60b8e797c2f121e1616a94c94 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 18 Dec 2020 12:54:33 +0100
Subject: [PATCH 529/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 290f94d0..3a623236 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __block: Fix escaping in here-doc (Matthias Stecher)
 	* Explorer os_version: Improve FreeBSD support (Evil Ham)
 	* New type: __apt_backports (Matthias Stecher)
+	* Type __dot_file: Add dirmode parameter (Mark Verboom)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From 7cf85c465980c74049f8e0258758c0fd8ea178b2 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Dec 2020 19:21:51 +0100
Subject: [PATCH 530/743] Release 6.9.4

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 3a623236..35953a88 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
 	* Type __iptables: Add IPv6 support (Matthias Stecher)

From 766198912d558e4630b258bba62e78587b3f0efe Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 5 Jan 2021 15:50:21 +0100
Subject: [PATCH 531/743] [type/__sshd_config] Produce error if invalid config
 file is generated

Previously, cdist would silently swallow the error (no invalid config file was
generated).

Reason: `set -e` does not exit if a command in a sub-command group fails,
it merely returns with a non-zero exit status.

e.g. the following snippet does not abort the script if sshd -t returns with a
non-zero exit status:

    set -e
    cmp -s old new || {
        # check config file and update it
        sshd -t -f new \
        && cat new >old
    }

or compressed:

    set -e
    false || { false && true; }
    echo $?
    # prints 1
---
 cdist/conf/type/__sshd_config/gencode-remote | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sshd_config/gencode-remote b/cdist/conf/type/__sshd_config/gencode-remote
index 0b44dfa7..275db4aa 100755
--- a/cdist/conf/type/__sshd_config/gencode-remote
+++ b/cdist/conf/type/__sshd_config/gencode-remote
@@ -91,7 +91,8 @@ awk $(drop_awk_comments "${__type:?}/files/update_sshd_config.awk") \\
 
 cmp -s $(quote "${sshd_config_file}") $(quote "${sshd_config_file}.tmp") || {
 	sshd -t -f $(quote "${sshd_config_file}.tmp") \\
-	&& cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}")
+	&& cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}") \\
+	|| exit  # stop if sshd_config file check fails
 }
 rm -f $(quote "${sshd_config_file}.tmp")
 EOF

From 8753b7eedf022a052ddc0b27fbc58ebf8fd638e1 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 5 Jan 2021 15:57:39 +0100
Subject: [PATCH 532/743] [type/__sshd_config] Make AuthenticationMethods and
 AuthorizedKeysFile singleton options

They were incorrectly treated as non-singleton options before.

cf. https://github.com/openssh/openssh-portable/blob/V_8_4/servconf.c#L2273
and https://github.com/openssh/openssh-portable/blob/V_8_4/servconf.c#L1899 resp.
---
 cdist/conf/type/__sshd_config/files/update_sshd_config.awk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
index d0bc2b4b..f7f30e87 100644
--- a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
+++ b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
@@ -89,7 +89,7 @@ function strdelim(s) { return strdelim_internal(s, 1) }
 function strdelimw(s) { return strdelim_internal(s, 0) }
 
 function singleton_option(opt) {
-	return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|authenticationmethods|authorizedkeysfile|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/
+	return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/
 }
 
 function print_update() {

From bd8ab8f26fdc242f2eb77e58049d640f96d51a69 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 5 Jan 2021 17:00:55 +0100
Subject: [PATCH 533/743] [type/__sshd_config] Document "bug" in state explorer

---
 cdist/conf/type/__sshd_config/man.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cdist/conf/type/__sshd_config/man.rst b/cdist/conf/type/__sshd_config/man.rst
index 8b0069ac..c8e6b8ad 100644
--- a/cdist/conf/type/__sshd_config/man.rst
+++ b/cdist/conf/type/__sshd_config/man.rst
@@ -79,6 +79,10 @@ BUGS
 - ``Include`` directives are ignored.
 - Config options are not added/removed to/from the config file if their value is
   the default value.
+- | The explorer will incorrectly report ``absent`` if OpenSSH internally
+    transforms one value to another (e.g. ``permitrootlogin prohibit-password``
+    is transformed to ``permitrootlogin without-password``).
+  | Workaround: Use the value that OpenSSH uses internally.
 
 
 AUTHORS

From c819548343ad7bb24d023f94a32575e1b2520328 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 11 Jan 2021 09:50:12 +0100
Subject: [PATCH 534/743] Fix debug parameter

-d was removed from cdist in favor of mulitple -v and -l parameters, but
-d was not removed from preos.

Resolve #849.
---
 cdist/preos/debootstrap/files/code | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/preos/debootstrap/files/code b/cdist/preos/debootstrap/files/code
index 9e37003b..d836848c 100755
--- a/cdist/preos/debootstrap/files/code
+++ b/cdist/preos/debootstrap/files/code
@@ -22,7 +22,7 @@ set -e
 if [ "${debug}" ]
 then
     set -x
-    cdist_params="${cdist_params} -d"
+    cdist_params="${cdist_params} -l 3"
 fi
 
 bootstrap_dir="${target_dir}"

From 2954347771e00610a084c7b033016765f7a3d6d7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 14 Jan 2021 13:46:40 +0100
Subject: [PATCH 535/743] [type/__postgres_role] Add note regarding empty
 passwords

---
 cdist/conf/type/__postgres_role/gencode-remote | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 877c8135..d7631fbd 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -59,6 +59,8 @@ psql_query() {
 
 psql_set_password() {
 	# NOTE: Always make sure that the password does not end up in psql_history!
+	# NOTE: Never set an empty string as the password, because they can be
+	#       interpreted differently by different tooling.
 	if test -s "${__object:?}/parameter/password"
 	then
 		cat <<-EOF

From 6e9b13d94962e1bfb365efa849ebefc32854364b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 18 Jan 2021 06:22:32 +0100
Subject: [PATCH 536/743] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 35953a88..b2b35616 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
+	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
+
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)

From 92a50da4873ff4bfa85dde3b85e272704cda02c7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 18 Jan 2021 06:28:09 +0100
Subject: [PATCH 537/743] Fix pycodestyle issues

---
 cdist/scan/scan.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 5ca8fae2..b1d0e9e1 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -139,7 +139,6 @@ class Scanner(object):
 
         """
 
-
     def start(self):
         self.process = Process(target=self.scan)
         self.process.start()

From 35cde3e666c229ceb94a1c5074083f7ba905768a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 13:09:29 +0100
Subject: [PATCH 538/743] [type/__postgres_role] Fix state explorer when stored
 password is empty

---
 cdist/conf/type/__postgres_role/explorer/state | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 77779662..34069de9 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -95,13 +95,20 @@ then
 	)
 	passwd_stored=${passwd_stored%?.}
 
-	passwd_should=$(cat "${__object}/parameter/password"; printf .)
+	if test -f "${__object:?}/parameter/password"
+	then
+		passwd_should=$(cat "${__object:?}/parameter/password"; printf .)
+	fi
 	passwd_should=${passwd_should%?.}
 
-	if expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
+	if test -z "${passwd_stored}"
+	then
+		test -z "${passwd_should}" || state="${state:-different} password"
+	elif expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
 	then
 		# SCRAM-SHA-256 "encrypted" password
-		# NOTE: There is currently no easy way to check SCRAM passwords
+		# NOTE: There is currently no easy way to check SCRAM passwords without
+		#       logging in
 		password_check_login || state="${state:-different} password"
 	elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null
 	then

From c51d68a7375915154a90c22e716e832f635ca878 Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Thu, 3 Dec 2020 18:48:04 +0100
Subject: [PATCH 539/743] [type/__postgres_conf] New type based on ALTER SYSTEM
 command

---
 .../conf/type/__postgres_conf/gencode-remote  | 92 +++++++++++++++++++
 cdist/conf/type/__postgres_conf/man.rst       | 55 +++++++++++
 .../__postgres_conf/parameter/default/state   |  1 +
 .../type/__postgres_conf/parameter/optional   |  2 +
 4 files changed, 150 insertions(+)
 create mode 100755 cdist/conf/type/__postgres_conf/gencode-remote
 create mode 100644 cdist/conf/type/__postgres_conf/man.rst
 create mode 100644 cdist/conf/type/__postgres_conf/parameter/default/state
 create mode 100644 cdist/conf/type/__postgres_conf/parameter/optional

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
new file mode 100755
index 00000000..a09e1873
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -0,0 +1,92 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+# 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$(cat "${__global}/explorer/os")
+state_should=$(cat "${__object}/parameter/state")
+
+if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ]
+then
+	echo "Invalid state '${state_should}'." \
+	     'Only "present" and "absent" are acceptable' >&2
+	exit 1
+fi
+
+if [ "${state_should}" = 'present' ] && [ ! -f "${__object}/parameter/value" ]
+then
+	echo 'Missing required parameter "value"' >&2
+	exit 1
+fi
+
+# Parameters
+conf_name="${__object_id}"
+if [ -f "${__object}/parameter/value" ]
+then
+	conf_value=$(cat "${__object}/parameter/value")
+fi
+
+if [ "${state_should}" = 'present' ]
+then
+	set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'"
+	check_command="SHOW ${conf_name}"
+else
+	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
+fi
+
+case $os
+in
+	openbsd|devuan)
+		case $os
+		in
+			openbsd)
+				postgres_user='_postgresql'
+				restart_command='/etc/rc.d/postgresql restart'
+				;;
+			devuan)
+				postgres_user='postgres'
+				restart_command='/etc/init.d/postgresql restart'
+				;;
+		esac
+		# needs two separate psql commands because ALTER SYSTEM
+		# cannot run inside a transaction block
+		cat <<-EOF
+		su - ${postgres_user} -c "psql postgres -twAc \
+			\"${set_command}\""
+		su - ${postgres_user} -c "psql postgres -twAc \
+			\"SELECT pg_reload_conf()\""
+		EOF
+		# check success (makes only sense if setting to a non-default value)
+		# and restart server if needed
+		if [ "${state_should}" = 'present' ]
+		then
+			cat <<-EOF
+			if [ \$(su - ${postgres_user} -c "psql postgres -twAc \"SHOW ${conf_name}\"") != '${conf_value}' ]
+			then
+				${restart_command}
+			fi
+			EOF
+		fi
+		;;
+	*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst
new file mode 100644
index 00000000..76016c6c
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/man.rst
@@ -0,0 +1,55 @@
+cdist-type__postgres_conf(7)
+============================
+
+Configure a PostgreSQL server.
+
+NOTE: This type might need to be run multiple times to apply all bits of the
+configuration due to ordering requirements.
+
+SSRQ <cdist--@--ssrq-sds-fds.ch>
+
+
+DESCRIPTION
+-----------
+Configure a PostgreSQL server using ALTER SYSTEM.
+
+
+REQUIRED PARAMETERS
+-------------------
+value
+    The value to setup (can be omitted when state is set to "absent").
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    "present" or "absent". Defaults to "present".
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+	# set timezone
+    __postgres_conf timezone --value Europe/Zurich
+
+    # reset maximum number of concurrent connections to default (normally 100)
+    __postgres_conf max_connections --state absent
+
+
+SEE ALSO
+--------
+- `cdist-type(7) <cdist-type.html>`_
+
+
+COPYING
+-------
+Copyright \(C) 2020 SSRQ (www.ssrq-sds-fds.ch).
+Free use of this software is granted under the terms
+of the GNU General Public License version 3 (GPLv3).
diff --git a/cdist/conf/type/__postgres_conf/parameter/default/state b/cdist/conf/type/__postgres_conf/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__postgres_conf/parameter/optional b/cdist/conf/type/__postgres_conf/parameter/optional
new file mode 100644
index 00000000..d0460d86
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/parameter/optional
@@ -0,0 +1,2 @@
+state
+value

From 534d5f6bb5dd3d8e9cb2256e75ed182d6530a892 Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Fri, 4 Dec 2020 14:31:04 +0100
Subject: [PATCH 540/743] [type/__postgres_conf] Fix errors found by ShellCheck

---
 cdist/conf/type/__postgres_conf/gencode-remote | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index a09e1873..e25514d0 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -20,8 +20,8 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$(cat "${__global}/explorer/os")
-state_should=$(cat "${__object}/parameter/state")
+os=$(cat "${__global:?}/explorer/os")
+state_should=$(cat "${__object:?}/parameter/state")
 
 if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ]
 then
@@ -37,7 +37,7 @@ then
 fi
 
 # Parameters
-conf_name="${__object_id}"
+conf_name="${__object_id:?}"
 if [ -f "${__object}/parameter/value" ]
 then
 	conf_value=$(cat "${__object}/parameter/value")
@@ -46,7 +46,6 @@ fi
 if [ "${state_should}" = 'present' ]
 then
 	set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'"
-	check_command="SHOW ${conf_name}"
 else
 	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
 fi

From 50bcd951055d7599ec340b4dea80a3a5d11dd9b0 Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Thu, 10 Dec 2020 11:45:32 +0100
Subject: [PATCH 541/743] [type/__postgres_conf] Remove faulty quotes

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index e25514d0..8ccb3b42 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -45,7 +45,7 @@ fi
 
 if [ "${state_should}" = 'present' ]
 then
-	set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'"
+	set_command="ALTER SYSTEM SET ${conf_name} = ${conf_value}"
 else
 	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
 fi

From b4060720dc3c7cc58aa992285877e4c009caa6be Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Fri, 11 Dec 2020 11:12:16 +0100
Subject: [PATCH 542/743] [type/__postgres_conf] Fix psql options for ALTER
 command

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 8ccb3b42..7d86028e 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -67,7 +67,7 @@ in
 		# needs two separate psql commands because ALTER SYSTEM
 		# cannot run inside a transaction block
 		cat <<-EOF
-		su - ${postgres_user} -c "psql postgres -twAc \
+		su - ${postgres_user} -c "psql postgres -qwc \
 			\"${set_command}\""
 		su - ${postgres_user} -c "psql postgres -twAc \
 			\"SELECT pg_reload_conf()\""

From 1b49fec972ce5f486f2794571c8e892d1fa6cb51 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 19:15:49 +0100
Subject: [PATCH 543/743] [type/__postgres_conf] Refactor

---
 .../type/__postgres_conf/explorer/context     |  42 ++++++
 .../conf/type/__postgres_conf/explorer/state  |  60 ++++++++
 .../conf/type/__postgres_conf/gencode-remote  | 135 ++++++++++--------
 3 files changed, 180 insertions(+), 57 deletions(-)
 create mode 100644 cdist/conf/type/__postgres_conf/explorer/context
 create mode 100644 cdist/conf/type/__postgres_conf/explorer/state

diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context
new file mode 100644
index 00000000..3a2fa504
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/explorer/context
@@ -0,0 +1,42 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Returns the "context" of the configuration setting.
+# cf. also https://www.postgresql.org/docs/10/view-pg-settings.html
+
+os=$("${__explorer:?}/os")
+
+case ${os}
+in
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(devuan)
+		postgres_user='postgres'
+		;;
+	(*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
+
+conf_name=${__object_id:?}
+
+su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE name = '${conf_name}'\""
diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
new file mode 100644
index 00000000..da904b56
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -0,0 +1,60 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$("${__explorer:?}/os")
+
+case ${os}
+in
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(devuan)
+		postgres_user='postgres'
+		;;
+	(*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
+
+conf_name=${__object_id:?}
+
+if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
+	| cmp -s "${__object:?}/parameter/value" -
+then
+	echo present
+else
+	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT source FROM pg_settings WHERE name = '${conf_name}'\"")
+	in
+		('')
+			# invalid configuration parameter
+			# (error message was already printed by SHOW command above.
+			#  Yes, it's a hack)
+			exit 1
+			;;
+		(default)
+			echo absent
+			;;
+		(*)
+			echo different
+			;;
+	esac
+fi
diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 7d86028e..998b8582 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -1,7 +1,7 @@
 #!/bin/sh -e
 # -*- mode: sh; indent-tabs-mode: t -*-
 #
-# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+# 2019-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 # 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
@@ -21,71 +21,92 @@
 #
 
 os=$(cat "${__global:?}/explorer/os")
+state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
-if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ]
+conf_name=${__object_id:?}
+
+if test "${state_is}" = "${state_should}"
 then
-	echo "Invalid state '${state_should}'." \
-	     'Only "present" and "absent" are acceptable' >&2
-	exit 1
+	exit 0
 fi
 
-if [ "${state_should}" = 'present' ] && [ ! -f "${__object}/parameter/value" ]
-then
-	echo 'Missing required parameter "value"' >&2
-	exit 1
-fi
-
-# Parameters
-conf_name="${__object_id:?}"
-if [ -f "${__object}/parameter/value" ]
-then
-	conf_value=$(cat "${__object}/parameter/value")
-fi
-
-if [ "${state_should}" = 'present' ]
-then
-	set_command="ALTER SYSTEM SET ${conf_name} = ${conf_value}"
-else
-	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
-fi
-
-case $os
-in
-	openbsd|devuan)
-		case $os
-		in
-			openbsd)
-				postgres_user='_postgresql'
-				restart_command='/etc/rc.d/postgresql restart'
-				;;
-			devuan)
-				postgres_user='postgres'
-				restart_command='/etc/init.d/postgresql restart'
-				;;
-		esac
-		# needs two separate psql commands because ALTER SYSTEM
-		# cannot run inside a transaction block
-		cat <<-EOF
-		su - ${postgres_user} -c "psql postgres -qwc \
-			\"${set_command}\""
-		su - ${postgres_user} -c "psql postgres -twAc \
-			\"SELECT pg_reload_conf()\""
-		EOF
-		# check success (makes only sense if setting to a non-default value)
-		# and restart server if needed
-		if [ "${state_should}" = 'present' ]
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
 		then
-			cat <<-EOF
-			if [ \$(su - ${postgres_user} -c "psql postgres -twAc \"SHOW ${conf_name}\"") != '${conf_value}' ]
-			then
-				${restart_command}
-			fi
-			EOF
+			# needs quoting
+			set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'"
+		else
+			set -- "$@" "${_arg}"
 		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
+
+case ${os}
+in
+	(openbsd)
+		postgres_user='_postgresql'
+		restart_command='/etc/rc.d/postgresql restart'
 		;;
-	*)
+	(devuan)
+		postgres_user='postgres'
+		restart_command='/etc/init.d/postgresql restart'
+		;;
+	(*)
 		echo "Unsupported OS: ${os}" >&2
 		exit 1
 		;;
 esac
+
+
+psql_cmd() {
+	printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")"
+}
+
+case ${state_should}
+in
+	(present)
+		test -s "${__object:?}/parameter/value" || {
+			echo 'Missing required parameter --value' >&2
+			exit 1
+		}
+
+		cat <<-EOF
+		exec 3< "\${__object:?}/parameter/value"
+		$(psql_cmd postgres -tAw) <<'SQL'
+		\\set conf_value \`cat <&3\`
+		ALTER SYSTEM SET ${conf_name} = :'conf_value';
+		SELECT pg_reload_conf();
+		SQL
+		exec 3<&-
+		EOF
+		;;
+	(absent)
+		psql_cmd postgres -qwc "ALTER SYSTEM SET ${conf_name} TO DEFAULT"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		printf 'Only "present" and "absent" are acceptable.\n' >&2
+		exit 1
+		;;
+esac
+
+# check success (makes only sense if setting to a non-default value)
+# and restart server if needed
+if test "${state_should}" = 'present'
+then
+	cat <<-EOF
+
+	$(psql_cmd postgres -twAc "SHOW ${conf_name}") \\
+	| cmp -s "\${__object:?}/parameter/value" - || {
+		${restart_command}
+	}
+	EOF
+fi

From 803367b316f5c72c51153172a41b7a745f2d2d83 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 19:29:42 +0100
Subject: [PATCH 544/743] [type/__postgres_conf] Fix default detection when
 default is also set in config file

e.g. port is usually also set to the default value in postgresql.conf
---
 cdist/conf/type/__postgres_conf/explorer/state | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index da904b56..400b27e8 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -42,7 +42,7 @@ if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
 then
 	echo present
 else
-	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT source FROM pg_settings WHERE name = '${conf_name}'\"")
+	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE name = '${conf_name}'\"")
 	in
 		('')
 			# invalid configuration parameter

From 891c98567e1d94577dc4f20d3ec9f73f4927fd24 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 19:39:56 +0100
Subject: [PATCH 545/743] [type/__postgres_conf] Compare configuration
 parameter names case insensitively

---
 cdist/conf/type/__postgres_conf/explorer/context | 3 ++-
 cdist/conf/type/__postgres_conf/explorer/state   | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context
index 3a2fa504..def5f676 100644
--- a/cdist/conf/type/__postgres_conf/explorer/context
+++ b/cdist/conf/type/__postgres_conf/explorer/context
@@ -39,4 +39,5 @@ esac
 
 conf_name=${__object_id:?}
 
-su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE name = '${conf_name}'\""
+# NOTE: SHOW/SET are case-insentitive, so this command should also be.
+su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE lower(name) = lower('${conf_name}')\""
diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 400b27e8..a4930296 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -42,7 +42,8 @@ if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
 then
 	echo present
 else
-	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE name = '${conf_name}'\"")
+	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
+	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"")
 	in
 		('')
 			# invalid configuration parameter

From 5051d4f40b7097f58431ce334f9ddf5b171dbc7f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 16:36:44 +0100
Subject: [PATCH 546/743] [type/__postgres_conf] Catch invalid values

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 998b8582..cc9b4e99 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -80,7 +80,7 @@ in
 
 		cat <<-EOF
 		exec 3< "\${__object:?}/parameter/value"
-		$(psql_cmd postgres -tAw) <<'SQL'
+		$(psql_cmd postgres -tAw -v ON_ERROR_STOP=on) <<'SQL'
 		\\set conf_value \`cat <&3\`
 		ALTER SYSTEM SET ${conf_name} = :'conf_value';
 		SELECT pg_reload_conf();

From 0f2ff477381d67de27149a92744f873322e44fd7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 16:37:43 +0100
Subject: [PATCH 547/743] [type/__postgres_conf] Restart PostgreSQL server
 based on pending_restart column of pg_settings

---
 .../type/__postgres_conf/explorer/context     | 43 -------------------
 .../conf/type/__postgres_conf/gencode-remote  | 16 +++----
 2 files changed, 6 insertions(+), 53 deletions(-)
 delete mode 100644 cdist/conf/type/__postgres_conf/explorer/context

diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context
deleted file mode 100644
index def5f676..00000000
--- a/cdist/conf/type/__postgres_conf/explorer/context
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/sh -e
-# -*- mode: sh; indent-tabs-mode: t -*-
-#
-# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-# Returns the "context" of the configuration setting.
-# cf. also https://www.postgresql.org/docs/10/view-pg-settings.html
-
-os=$("${__explorer:?}/os")
-
-case ${os}
-in
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(devuan)
-		postgres_user='postgres'
-		;;
-	(*)
-		echo "Unsupported OS: ${os}" >&2
-		exit 1
-		;;
-esac
-
-conf_name=${__object_id:?}
-
-# NOTE: SHOW/SET are case-insentitive, so this command should also be.
-su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE lower(name) = lower('${conf_name}')\""
diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index cc9b4e99..fa930cc4 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -98,15 +98,11 @@ in
 		;;
 esac
 
-# check success (makes only sense if setting to a non-default value)
-# and restart server if needed
-if test "${state_should}" = 'present'
-then
-	cat <<-EOF
+# Restart PostgreSQL server if required to apply new configuration value
+cat <<EOF
 
-	$(psql_cmd postgres -twAc "SHOW ${conf_name}") \\
-	| cmp -s "\${__object:?}/parameter/value" - || {
-		${restart_command}
-	}
-	EOF
+if test 't' = "\$($(psql_cmd postgres -twAc "SELECT pending_restart FROM pg_settings WHERE lower(name) = lower('${conf_name}')"))"
+then
+	${restart_command}
 fi
+EOF

From 3f605c31ac7538d188a11b5d9d84cf2ed485bf23 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 17:18:27 +0100
Subject: [PATCH 548/743] [type/__postgres_conf] Add support for more init
 systems to restart service

---
 .../conf/type/__postgres_conf/gencode-remote  | 35 +++++++++++++++++--
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index fa930cc4..15d5bb3e 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -53,11 +53,9 @@ case ${os}
 in
 	(openbsd)
 		postgres_user='_postgresql'
-		restart_command='/etc/rc.d/postgresql restart'
 		;;
 	(devuan)
 		postgres_user='postgres'
-		restart_command='/etc/init.d/postgresql restart'
 		;;
 	(*)
 		echo "Unsupported OS: ${os}" >&2
@@ -103,6 +101,37 @@ cat <<EOF
 
 if test 't' = "\$($(psql_cmd postgres -twAc "SELECT pending_restart FROM pg_settings WHERE lower(name) = lower('${conf_name}')"))"
 then
-	${restart_command}
+	$(
+		init=$(cat "${__global:?}/explorer/init")
+		case ${init}
+		in
+			(systemd)
+				echo 'systemctl restart postgresql.service'
+				;;
+			(*openrc*)
+				echo 'rc-service postgresql restart'
+				;;
+			(sysvinit)
+				echo '/etc/init.d/postgresql restart'
+				;;
+			(init)
+				case $(cat "${__global:?}/explorer/kernel_name")
+				in
+					(FreeBSD)
+						echo 'service postgresql restart'
+						;;
+					(OpenBSD|NetBSD)
+						echo '/etc/rc.d/postgresql restart'
+						;;
+					(*)
+						echo "Unsupported operating system. Don't know how to restart services." >&2
+						exit 1
+				esac
+				;;
+			(*)
+				printf "Don't know how to restart services with your init (%s)\n" "${init}" >&2
+				exit 1
+		esac
+	)
 fi
 EOF

From 4967c7ebbb7e98af6aa2c7f8bc91511bcbbf9820 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 17:20:27 +0100
Subject: [PATCH 549/743] [type/__postgres_conf] Silence psql output

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 15d5bb3e..9b8227ee 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -78,7 +78,7 @@ in
 
 		cat <<-EOF
 		exec 3< "\${__object:?}/parameter/value"
-		$(psql_cmd postgres -tAw -v ON_ERROR_STOP=on) <<'SQL'
+		$(psql_cmd postgres -tAwq -o /dev/null -v ON_ERROR_STOP=on) <<'SQL'
 		\\set conf_value \`cat <&3\`
 		ALTER SYSTEM SET ${conf_name} = :'conf_value';
 		SELECT pg_reload_conf();

From f9ebb4333c85e41439635cb00a40c3e07d8a3683 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 17:21:50 +0100
Subject: [PATCH 550/743] [type/__postgres_conf] Add NetBSD PostgreSQL UNIX
 user

---
 cdist/conf/type/__postgres_conf/gencode-remote | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 9b8227ee..8f8a2bfe 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -51,15 +51,14 @@ quote() {
 
 case ${os}
 in
+	(netbsd)
+		postgres_user='pgsql'
+		;;
 	(openbsd)
 		postgres_user='_postgresql'
 		;;
-	(devuan)
-		postgres_user='postgres'
-		;;
 	(*)
-		echo "Unsupported OS: ${os}" >&2
-		exit 1
+		postgres_user='postgres'
 		;;
 esac
 

From 6b18cace759d1345cec6c2168c3bd8f238f23bca Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 21 Jan 2021 19:29:07 +0100
Subject: [PATCH 551/743] [type/__postgres_conf] Catch connection errors early

---
 cdist/conf/type/__postgres_conf/explorer/state | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index a4930296..589925de 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -37,6 +37,11 @@ esac
 
 conf_name=${__object_id:?}
 
+su - "${postgres_user}" -c 'psql postgres -c "SELECT 1"' >/dev/null || {
+	echo 'Connection to PostgreSQL server failed' >&2
+	exit 1
+}
+
 if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
 	| cmp -s "${__object:?}/parameter/value" -
 then

From 8eccacec59b0badd7a6e2a40a27f02f0ff355ccf Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 4 Feb 2021 19:09:26 +0100
Subject: [PATCH 552/743] __package_pip: add optional dependencies

This is a poor implementation of optional dependencies for pip packages.
It ensures to install them if the package will be installed, but does
not take into account if they must be added/removed after the package is
already installed. Also, it will not be autoremoved, as all dependencies
will not be removed.
---
 cdist/conf/type/__package_pip/gencode-remote        | 13 +++++++++++++
 cdist/conf/type/__package_pip/man.rst               | 13 +++++++++++++
 .../type/__package_pip/parameter/optional_multiple  |  1 +
 3 files changed, 27 insertions(+)
 create mode 100644 cdist/conf/type/__package_pip/parameter/optional_multiple

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index a1375c2d..58c89040 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -56,6 +56,19 @@ fi
 
 case "$state_should" in
     present)
+        if [ -f "$__object/parameter/extra" ]
+        then
+            while read extra
+            do
+                if [ "$extras" ]; then
+                    extras="$extras,$extra"
+                else
+                    extras="$extra"
+                fi
+            done < "$__object/parameter/extra"
+            name="${name}[${extras}]"
+        fi
+
         if [ "$runas" ]
         then
             echo "su -c '$pip install -q $name' $runas"
diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index 234ceee2..64ad0358 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -22,6 +22,11 @@ OPTIONAL PARAMETERS
 name
     If supplied, use the name and not the object id as the package name.
 
+extra
+    Extra optional dependencies which should be installed along the selected
+    package. Can be specified multiple times. Will only be applied if the
+    package actually will be installed, but will not explicitly checked.
+
 pip
     Instead of using pip from PATH, use the specific pip path.
 
@@ -46,6 +51,14 @@ EXAMPLES
     # Use pip in a virtualenv located at /foo/shinken_virtualenv as user foo
     __package_pip pyro --state present --pip /foo/shinken_virtualenv/bin/pip --runas foo
 
+    # Install package with optional dependencies
+    __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails
+    # or do a little cheating
+    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails
+
+    # or take all extras
+    __package_pip mautrix-telegram --extra all
+
 
 SEE ALSO
 --------
diff --git a/cdist/conf/type/__package_pip/parameter/optional_multiple b/cdist/conf/type/__package_pip/parameter/optional_multiple
new file mode 100644
index 00000000..0f228715
--- /dev/null
+++ b/cdist/conf/type/__package_pip/parameter/optional_multiple
@@ -0,0 +1 @@
+extra

From 73a03d75d7d23fea292b5fd60aeb059d8c72c4ba Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 4 Feb 2021 19:18:02 +0100
Subject: [PATCH 553/743] __package_pip: fix shellcheck

---
 cdist/conf/type/__package_pip/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 58c89040..925eaf24 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -58,7 +58,7 @@ case "$state_should" in
     present)
         if [ -f "$__object/parameter/extra" ]
         then
-            while read extra
+            while read -r extra
             do
                 if [ "$extras" ]; then
                     extras="$extras,$extra"

From cda17be38a1030742ed224b2536fca517bf8c09b Mon Sep 17 00:00:00 2001
From: ssrq <ungleich@ssrq-sds-fds.ch>
Date: Mon, 8 Feb 2021 08:27:03 +0100
Subject: [PATCH 554/743] [explorer/memory] Clean up, return kiB for all
 systems, add SunOS

BSDs were MiB before.
---
 cdist/conf/explorer/memory | 85 ++++++++++++++++++++++++++++++--------
 1 file changed, 68 insertions(+), 17 deletions(-)

diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory
index 5ea15ada..63aba9c6 100755
--- a/cdist/conf/explorer/memory
+++ b/cdist/conf/explorer/memory
@@ -1,8 +1,9 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2014 Daniel Heule  (hda at sfs.biz)
 # 2014 Thomas Oettli (otho at sfs.biz)
 # Copyright 2017, Philippe Gregoire <pg@pgregoire.xyz>
+# 2020 Dennis Camera <dennis.camera at ssrq-sds-fds.ch>
 #
 # This file is part of cdist.
 #
@@ -19,24 +20,74 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
+# Returns the amount of memory physically installed in the system, or if that
+# cannot be determined the amount available to the operating system kernel,
+# in kibibytes (kiB).
 
-# FIXME: other system types (not linux ...)
+str2bytes() {
+	awk -F' ' '
+	$2 ==   "B" || !$2 { print $1 }
+	$2 ==  "kB" { print $1 * 1000 }
+	$2 ==  "MB" { print $1 * 1000 * 1000 }
+	$2 ==  "GB" { print $1 * 1000 * 1000 * 1000 }
+	$2 ==  "TB" { print $1 * 1000 * 1000 * 1000 * 1000 }
+	$2 == "kiB" { print $1 * 1024 }
+	$2 == "MiB" { print $1 * 1024 * 1024 }
+	$2 == "GiB" { print $1 * 1024 * 1024 * 1024 }
+	$2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }'
+}
 
-os=$("$__explorer/os")
-case "$os" in
-    "macosx")
-        echo "$(sysctl -n hw.memsize)/1024" | bc
-    ;;
+bytes2kib() {
+	set -- "$(cat)"
+	test "$1" -gt 0 && echo $(($1 / 1024))
+}
 
-    *"bsd")
-        PATH=$(getconf PATH)
-        echo "$(sysctl -n hw.physmem) / 1048576" | bc
-    ;;
 
-    *)
-    if [ -r /proc/meminfo ]; then
-        grep "MemTotal:" /proc/meminfo | awk '{print $2}'
-    fi
-    ;;
+case $(uname -s)
+in
+	(Darwin)
+		sysctl -n hw.memsize | bytes2kib
+		;;
+	(FreeBSD)
+		sysctl -n hw.realmem | bytes2kib
+		;;
+	(NetBSD|OpenBSD)
+		# NOTE: This reports "usable" memory, not physically installed memory.
+		command -p sysctl -n hw.physmem | bytes2kib
+		;;
+	(SunOS)
+		# Make sure that awk from xpg4 is used for the scripts to work
+		export PATH="/usr/xpg4/bin:${PATH}"
+		prtconf \
+		| awk -F ': ' '
+		  $1 == "Memory size" { sub(/Megabytes/, "MiB", $2); print $2 }
+		  /^$/ { exit }' \
+		| str2bytes \
+		| bytes2kib
+		;;
+	(Linux)
+		if test -d /sys/devices/system/memory
+		then
+			# Use memory blocks if the architecture (e.g. x86, PPC64, s390)
+			# supports them (they denote physical memory)
+			num_mem_blocks=$(cat /sys/devices/system/memory/memory[0-9]*/state | grep -cxF online)
+			mem_block_size=$(cat /sys/devices/system/memory/block_size_bytes)
+
+			echo $((num_mem_blocks * 0x$mem_block_size)) | bytes2kib && exit
+		fi
+		if test -r /proc/meminfo
+		then
+			# Fall back to meminfo file on other architectures (e.g. ARM, MIPS,
+			# PowerPC)
+			# NOTE: This is "usable" memory, not physically installed memory.
+			awk -F ': +' '$1 == "MemTotal" { sub(/B$/, "iB", $2); print $2 }' /proc/meminfo \
+			| str2bytes \
+			| bytes2kib
+		fi
+		;;
+	(*)
+		printf "Your kernel (%s) is currently not supported by the memory explorer\n" "$(uname -s)" >&2
+		printf "Please contribute an implementation for it if you can.\n" >&2
+		exit 1
+		;;
 esac

From 65a6a2ed52c56f553d65ad0aa0aaba3ee4579e3c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 8 Feb 2021 08:28:31 +0100
Subject: [PATCH 555/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index b2b35616..52686617 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
+	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)

From bc145bbc27a45e382adb744e6774fb803f4fda0a Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 9 Feb 2021 19:58:47 +0100
Subject: [PATCH 556/743] [__letsencrypt_cert] Fix various issues with hooks.

Closes #853, see issue for full description / discussion.

Short summary:
- There was about 6.53% chances of `--renewal-hook` not being applied
- Using --automatic-renewal in one cert and not in another was an error.
- It was not possible to use different hooks for different certificates.
- FreeBSD support was utterly broken.
---
 cdist/conf/type/__letsencrypt_cert/man.rst    |  84 +++++++--
 cdist/conf/type/__letsencrypt_cert/manifest   | 171 ++++++++++++++----
 .../parameter/deprecated/automatic-renewal    |   2 +
 .../parameter/deprecated/renew-hook           |   2 +
 .../parameter/optional_multiple               |   3 +
 5 files changed, 220 insertions(+), 42 deletions(-)
 mode change 100755 => 100644 cdist/conf/type/__letsencrypt_cert/manifest
 create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal
 create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook

diff --git a/cdist/conf/type/__letsencrypt_cert/man.rst b/cdist/conf/type/__letsencrypt_cert/man.rst
index 85eb88ea..43be8424 100644
--- a/cdist/conf/type/__letsencrypt_cert/man.rst
+++ b/cdist/conf/type/__letsencrypt_cert/man.rst
@@ -1,16 +1,33 @@
 cdist-type__letsencrypt_cert(7)
 ===============================
 
+
 NAME
 ----
 
 cdist-type__letsencrypt_cert - Get an SSL certificate from Let's Encrypt
 
+
 DESCRIPTION
 -----------
 
 Automatically obtain a Let's Encrypt SSL certificate using Certbot.
 
+This type attempts to setup automatic renewals always. In many Linux
+distributions, that is the case out of the box, see:
+https://certbot.eff.org/docs/using.html#automated-renewals
+
+For Alpine Linux and Arch Linux, we setup a system-wide cronjob that
+attempts to renew certificates daily.
+
+If you are using FreeBSD, we configure periodic(8) as recommended by
+the port mantainer, so there will be a weekly attempt at renewal.
+
+If your OS is not mentioned here or on Certbot's docs as having
+support for automated renewals, please make sure you check your OS
+and possibly patch this type so the system-wide cronjob is installed.
+
+
 REQUIRED PARAMETERS
 -------------------
 
@@ -21,6 +38,7 @@ object id
 admin-email
     Where to send Let's Encrypt emails like "certificate needs renewal".
 
+
 OPTIONAL PARAMETERS
 -------------------
 
@@ -36,25 +54,68 @@ webroot
     The path to your webroot, as set up in your webserver config. If this
     parameter is not present, Certbot will be run in standalone mode.
 
+
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
 
-renew-hook
-    Renew hook command directly passed to Certbot in cron job.
-
 domain
     Domains to be included in the certificate. When specified then object id
     is not used as a domain.
 
+deploy-hook
+    Command to be executed only when the certificate associated with this
+    ``$__object_id`` is issued or renewed.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    For this command, the
+    shell variable ``$RENEWED_LINEAGE`` will point to the
+    config live subdirectory (for example,
+    ``/etc/letsencrypt/live/${__object_id}``) containing the
+    new certificates and keys; the shell variable
+    ``$RENEWED_DOMAINS`` will contain a space-delimited list
+    of renewed certificate domains (for example,
+    ``example.com www.example.com``)
+
+pre-hook
+    Command to be run in a shell before obtaining any
+    certificates.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    Note these run regardless of which certificate is attempted, you may want to
+    manage these system-wide hooks with ``__file`` in
+    ``/etc/letsencrypt/renewal-hooks/pre/``.
+
+    Intended primarily for renewal, where it
+    can be used to temporarily shut down a webserver that
+    might conflict with the standalone plugin. This will
+    only be called if a certificate is actually to be
+    obtained/renewed.
+
+post-hook
+    Command to be run in a shell after attempting to
+    obtain/renew certificates.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    Note these run regardless of which certificate was attempted, you may want to
+    manage these system-wide hooks with ``__file`` in
+    ``/etc/letsencrypt/renewal-hooks/post/``.
+
+    Can be used to deploy
+    renewed certificates, or to restart any servers that
+    were stopped by --pre-hook. This is only run if an
+    attempt was made to obtain/renew a certificate.
+
+
 BOOLEAN PARAMETERS
 ------------------
 
-automatic-renewal
-    Install a cron job, which attempts to renew certificates daily.
-
 staging
     Obtain a test certificate from a staging server.
 
+
 MESSAGES
 --------
 
@@ -67,6 +128,7 @@ create
 remove
     Certificate was removed.
 
+
 EXAMPLES
 --------
 
@@ -75,8 +137,7 @@ EXAMPLES
     # use object id as domain
     __letsencrypt_cert example.com \
         --admin-email root@example.com \
-        --automatic-renewal \
-        --renew-hook "service nginx reload" \
+        --deploy-hook "service nginx reload" \
         --webroot /data/letsencrypt/root
 
 .. code-block:: sh
@@ -85,11 +146,10 @@ EXAMPLES
     # and example.com needs to be included again with domain parameter
     __letsencrypt_cert example.com \
         --admin-email root@example.com \
-        --automatic-renewal \
         --domain example.com \
         --domain foo.example.com \
         --domain bar.example.com \
-        --renew-hook "service nginx reload" \
+        --deploy-hook "service nginx reload" \
         --webroot /data/letsencrypt/root
 
 AUTHORS
@@ -99,11 +159,13 @@ AUTHORS
 | Kamila Součková <kamila--@--ksp.sk>
 | Darko Poljak <darko.poljak--@--gmail.com>
 | Ľubomír Kučera <lubomir.kucera.jr at gmail.com>
+| Evilham <contact@evilham.com>
+
 
 COPYING
 -------
 
-Copyright \(C) 2017-2018 Nico Schottelius, Kamila Součková, Darko Poljak and
+Copyright \(C) 2017-2021 Nico Schottelius, Kamila Součková, Darko Poljak and
 Ľubomír Kučera. You can redistribute it and/or modify it under the terms of
 the GNU General Public License as published by the Free Software Foundation,
 either version 3 of the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
old mode 100755
new mode 100644
index b4464366..9c8cc043
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -1,18 +1,20 @@
 #!/bin/sh
 
 certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")"
+state=$(cat "${__object}/parameter/state")
+os="$(cat "${__global:?}/explorer/os")"
 
 if [ -z "${certbot_fullpath}" ]; then
-	os="$(cat "${__global:?}/explorer/os")"
 	os_version="$(cat "${__global}/explorer/os_version")"
-
+	# Use this, very common value, as a default. It is OS-dependent
+	certbot_fullpath="/usr/bin/certbot"
 	case "$os" in
-        archlinux)
-            __package certbot
-            ;;
-        alpine)
-            __package certbot
-            ;;
+		archlinux)
+			__package certbot
+		;;
+		alpine)
+			__package certbot
+		;;
 		debian)
 			case "$os_version" in
 				8*)
@@ -48,9 +50,7 @@ if [ -z "${certbot_fullpath}" ]; then
 					exit 1
 				;;
 			esac
-
-			certbot_fullpath=/usr/bin/certbot
-			;;
+		;;
 		devuan)
 			case "$os_version" in
 				jessie)
@@ -83,17 +83,14 @@ if [ -z "${certbot_fullpath}" ]; then
 					exit 1
 				;;
 			esac
-
-			certbot_fullpath=/usr/bin/certbot
 		;;
 		freebsd)
-			__package py27-certbot
-
-			certbot_fullpath=/usr/local/bin/certbot
+			__package py37-certbot
+			certbot_fullpath="/usr/local/bin/certbot"
 		;;
 		ubuntu)
-            __package certbot
-            ;;
+			__package certbot
+		;;
 		*)
 			echo "Unsupported os: $os" >&2
 			exit 1
@@ -101,18 +98,130 @@ if [ -z "${certbot_fullpath}" ]; then
 	esac
 fi
 
-if [ -f "${__object}/parameter/automatic-renewal" ]; then
-	renew_hook_param="${__object}/parameter/renew-hook"
-	renew_hook=""
-	if [ -f "${renew_hook_param}" ]; then
-		while read -r hook; do
-			renew_hook="${renew_hook} --renew-hook \"${hook}\""
-		done < "${renew_hook_param}"
-	fi
+# Other OS-dependent values that we want to set every time
+LE_DIR="/etc/letsencrypt"
+certbot_cronjob_state="absent"
+case "$os" in
+	archlinux|alpine)
+		certbot_cronjob_state="present"
+	;;
+	freebsd)
+		LE_DIR="/usr/local/etc/letsencrypt"
+		# FreeBSD uses periodic(8) instead of crontabs for this
+		__line "periodic.conf_weekly_certbot" \
+			--file "/etc/periodic.conf" \
+			--regex "^(#[[:space:]]*)?weekly_certbot_enable=.*" \
+			--state "replace" \
+			--line 'weekly_certbot_enable="YES"'
+	;;
+	*)
+	;;
+esac
 
-	__cron letsencrypt-certbot  \
-		--user root \
-		--command "${certbot_fullpath} renew -q ${renew_hook}" \
-		--hour 0 \
-		--minute 47
+# This is only necessary in certain OS
+__cron letsencrypt-certbot \
+	--user root \
+	--command "${certbot_fullpath} renew -q" \
+	--hour 0 \
+	--minute 47 \
+	--state "${certbot_cronjob_state}"
+
+# Ensure hook directories
+HOOKS_DIR="${LE_DIR}/renewal-hooks"
+__directory "${LE_DIR}" --mode 0755
+require="__directory/${LE_DIR}" __directory "${HOOKS_DIR}" --mode 0755
+
+if [ -f "${__object}/parameter/domain" ]; then
+	domains="$(sort "${__object}/parameter/domain")"
+else
+	domains="${__object_id}"
 fi
+
+# Install hooks as needed
+for hook in deploy pre post; do
+	# Using something unique and specific to this object
+	hook_file="${HOOKS_DIR}/${hook}/${__object_id}.cdist.sh"
+	# Reasonable defaults
+	hook_source="${__object}/parameter/${hook}-hook"
+	hook_state="absent"
+	hook_contents_head="#!/bin/sh -e"
+	hook_contents_logic=""
+	hook_contents_tail=""
+
+	# Backwards compatibility
+	# Remove this when renew-hook is removed
+	# Falling back to renew-hook if deploy-hook is not passed
+	if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then
+		hook_source="${__object}/parameter/renew-hook"
+	fi
+	if [ "${state}" = "present" ] && \
+		[ -f "${hook_source}" ]; then
+		# This hook is to be installed, let's generate it with some
+		# safety boilerplate
+		# Since certbot runs all hooks for all renewal processes
+		# (at each state for deploy, pre, post), it is up to us to
+		# differentiate whether or not the hook must run
+		hook_state="present"
+		hook_contents_head="$(cat <<EOF
+#!/bin/sh -e
+#
+# Managed remotely with https://cdi.st
+#
+# Domains for which this hook is supposed to apply
+lineage="${LE_DIR}/live/${__object_id}"
+domains="\$(cat <<eof
+${domains}
+eof
+)"
+EOF
+)"
+		case "${hook}" in
+			pre|post)
+				# Certbot is kind of terrible, we have
+				# no way of knowing what domain/lineage the
+				# hook is running for
+				hook_contents_logic="$(cat <<EOF
+# pre/post-hooks apply always due to a certbot limitation
+APPLY_HOOK="YES"
+EOF
+)"
+			;;
+			deploy)
+				hook_contents_logic="$(cat <<EOF
+# certbot defines these:
+# RENEWED_DOMAINS: DOMAIN1,DOMAIN2
+# RENEWED_LINEAGE: /etc/letsencrypt/live/__object_id
+# It feels more stable to use RENEWED_LINEAGE
+if [ "\${lineage}" = "\${RENEWED_LINEAGE}" ]; then
+	APPLY_HOOK="YES"
+fi
+EOF
+)"
+			;;
+			*)
+				echo "Unknown hook '${hook}'" >> /dev/stderr
+				exit 1
+			;;
+		esac
+
+		hook_contents_tail="$(cat <<EOF
+if [ -n "\${APPLY_HOOK}" ]; then
+$(sed -e 's/^/	//' "${hook_source}")
+fi
+EOF
+)"
+	fi
+	# Ensure hook directory exists
+	require="__directory/${HOOKS_DIR}" __directory "${HOOKS_DIR}/${hook}" \
+		--mode 0755
+	require="__directory/${HOOKS_DIR}/${hook}" __file "${hook_file}" \
+		--mode 0555 \
+		--source '-' \
+		--state "${hook_state}" <<EOF
+${hook_contents_head}
+
+${hook_contents_logic}
+
+${hook_contents_tail}
+EOF
+done
diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal
new file mode 100644
index 00000000..dbfc5fa7
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal
@@ -0,0 +1,2 @@
+Deprecated in favour of consistent behaviour. It has no effect, see:
+https://code.ungleich.ch/ungleich-public/cdist/-/issues/853
diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook
new file mode 100644
index 00000000..be4a3460
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook
@@ -0,0 +1,2 @@
+This parameter has been deprecated in favour of --deploy-hook.
+See: https://code.ungleich.ch/ungleich-public/cdist/-/issues/853
diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple b/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple
index 0e866d45..bebb19bd 100644
--- a/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple
+++ b/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple
@@ -1,2 +1,5 @@
+deploy-hook
 domain
+post-hook
+pre-hook
 renew-hook

From e49da474c4e92775477c132833fb8c517db6921a Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 9 Feb 2021 20:29:17 +0100
Subject: [PATCH 557/743] [__letsencrypt_cert] Remove problematic trailing
 slash in sed.

Happy fingers are happy and like adding slashes places.
---
 cdist/conf/type/__letsencrypt_cert/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 9c8cc043..04edea1e 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -206,7 +206,7 @@ EOF
 
 		hook_contents_tail="$(cat <<EOF
 if [ -n "\${APPLY_HOOK}" ]; then
-$(sed -e 's/^/	//' "${hook_source}")
+$(sed -e 's/^/	/' "${hook_source}")
 fi
 EOF
 )"

From b832af5e3bce7d631dc1aafd6c4d9c2786391684 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 9 Feb 2021 20:53:58 +0100
Subject: [PATCH 558/743] [__letsencrypt_cert] Don't mess with user script
 indentation

This could break in odd ways if they passed sth like:
cat <<eof
bla bla
eof
---
 cdist/conf/type/__letsencrypt_cert/manifest | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 04edea1e..84221a80 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -206,7 +206,8 @@ EOF
 
 		hook_contents_tail="$(cat <<EOF
 if [ -n "\${APPLY_HOOK}" ]; then
-$(sed -e 's/^/	/' "${hook_source}")
+# Messing with indentation can eff up the users' scripts, let's not
+$(cat "${hook_source}")
 fi
 EOF
 )"

From aa80c09c803e132c3a05a642e9ada82ccb0e7dd8 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Wed, 10 Feb 2021 10:10:21 +0100
Subject: [PATCH 559/743] [__letsencrypt_cert] Move hook contents generation
 out of manifest

While there address some minor issues in the comments in the hook contents.
---
 .../type/__letsencrypt_cert/files/gen_hook.sh | 84 +++++++++++++++++++
 cdist/conf/type/__letsencrypt_cert/manifest   | 78 +----------------
 2 files changed, 88 insertions(+), 74 deletions(-)
 create mode 100644 cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh

diff --git a/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh b/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh
new file mode 100644
index 00000000..81ea4856
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh
@@ -0,0 +1,84 @@
+#!/bin/sh -e
+
+# It is expected that this defines hook_contents
+
+# Reasonable defaults
+hook_source="${__object}/parameter/${hook}-hook"
+hook_state="absent"
+hook_contents_head="#!/bin/sh -e"
+hook_contents_logic=""
+hook_contents_tail=""
+
+# Backwards compatibility
+# Remove this when renew-hook is removed
+# Falling back to renew-hook if deploy-hook is not passed
+if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then
+	hook_source="${__object}/parameter/renew-hook"
+fi
+if [ "${state}" = "present" ] && \
+	[ -f "${hook_source}" ]; then
+	# This hook is to be installed, let's generate it with some
+	# safety boilerplate
+	# Since certbot runs all hooks for all renewal processes
+	# (at each state for deploy, pre, post), it is up to us to
+	# differentiate whether or not the hook must run
+	hook_state="present"
+	hook_contents_head="$(cat <<EOF
+#!/bin/sh -e
+#
+# Managed remotely with https://cdi.st
+#
+# Domains for which this hook is supposed to apply
+lineage="${LE_DIR}/live/${__object_id}"
+domains="\$(cat <<eof
+${domains}
+eof
+)"
+EOF
+)"
+	case "${hook}" in
+		pre|post)
+			# Certbot is kind of terrible, we have
+			# no way of knowing what domain/lineage the
+			# hook is running for
+			hook_contents_logic="$(cat <<EOF
+# pre/post-hooks apply always due to a certbot limitation
+APPLY_HOOK="YES"
+EOF
+)"
+		;;
+		deploy)
+			hook_contents_logic="$(cat <<EOF
+# certbot defines these environment variables:
+# RENEWED_DOMAINS="DOMAIN1 DOMAIN2"
+# RENEWED_LINEAGE="/etc/letsencrypt/live/__object_id"
+# It feels more stable to use RENEWED_LINEAGE
+if [ "\${lineage}" = "\${RENEWED_LINEAGE}" ]; then
+	APPLY_HOOK="YES"
+fi
+EOF
+)"
+		;;
+		*)
+			echo "Unknown hook '${hook}'" >> /dev/stderr
+			exit 1
+		;;
+	esac
+
+	hook_contents_tail="$(cat <<EOF
+if [ -n "\${APPLY_HOOK}" ]; then
+	# Messing with indentation can eff up the users' scripts, let's not
+$(cat "${hook_source}")
+fi
+EOF
+)"
+fi
+
+hook_contents="$(cat <<EOF
+${hook_contents_head}
+
+${hook_contents_logic}
+
+${hook_contents_tail}
+EOF
+)"
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 84221a80..1df3574a 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -141,77 +141,11 @@ fi
 for hook in deploy pre post; do
 	# Using something unique and specific to this object
 	hook_file="${HOOKS_DIR}/${hook}/${__object_id}.cdist.sh"
-	# Reasonable defaults
-	hook_source="${__object}/parameter/${hook}-hook"
-	hook_state="absent"
-	hook_contents_head="#!/bin/sh -e"
-	hook_contents_logic=""
-	hook_contents_tail=""
 
-	# Backwards compatibility
-	# Remove this when renew-hook is removed
-	# Falling back to renew-hook if deploy-hook is not passed
-	if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then
-		hook_source="${__object}/parameter/renew-hook"
-	fi
-	if [ "${state}" = "present" ] && \
-		[ -f "${hook_source}" ]; then
-		# This hook is to be installed, let's generate it with some
-		# safety boilerplate
-		# Since certbot runs all hooks for all renewal processes
-		# (at each state for deploy, pre, post), it is up to us to
-		# differentiate whether or not the hook must run
-		hook_state="present"
-		hook_contents_head="$(cat <<EOF
-#!/bin/sh -e
-#
-# Managed remotely with https://cdi.st
-#
-# Domains for which this hook is supposed to apply
-lineage="${LE_DIR}/live/${__object_id}"
-domains="\$(cat <<eof
-${domains}
-eof
-)"
-EOF
-)"
-		case "${hook}" in
-			pre|post)
-				# Certbot is kind of terrible, we have
-				# no way of knowing what domain/lineage the
-				# hook is running for
-				hook_contents_logic="$(cat <<EOF
-# pre/post-hooks apply always due to a certbot limitation
-APPLY_HOOK="YES"
-EOF
-)"
-			;;
-			deploy)
-				hook_contents_logic="$(cat <<EOF
-# certbot defines these:
-# RENEWED_DOMAINS: DOMAIN1,DOMAIN2
-# RENEWED_LINEAGE: /etc/letsencrypt/live/__object_id
-# It feels more stable to use RENEWED_LINEAGE
-if [ "\${lineage}" = "\${RENEWED_LINEAGE}" ]; then
-	APPLY_HOOK="YES"
-fi
-EOF
-)"
-			;;
-			*)
-				echo "Unknown hook '${hook}'" >> /dev/stderr
-				exit 1
-			;;
-		esac
+	# This defines hook_contents
+	# shellcheck source=cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh
+	. "${__type}/files/gen_hook.sh"
 
-		hook_contents_tail="$(cat <<EOF
-if [ -n "\${APPLY_HOOK}" ]; then
-# Messing with indentation can eff up the users' scripts, let's not
-$(cat "${hook_source}")
-fi
-EOF
-)"
-	fi
 	# Ensure hook directory exists
 	require="__directory/${HOOKS_DIR}" __directory "${HOOKS_DIR}/${hook}" \
 		--mode 0755
@@ -219,10 +153,6 @@ EOF
 		--mode 0555 \
 		--source '-' \
 		--state "${hook_state}" <<EOF
-${hook_contents_head}
-
-${hook_contents_logic}
-
-${hook_contents_tail}
+${hook_contents}
 EOF
 done

From 4717e5ceff078111748824e4fb365df7979e5fe7 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 10:31:07 +0100
Subject: [PATCH 560/743] __package_pip: add extras explorer

The two new explorers detect all installed extras for this package.
---
 .../type/__package_pip/explorer/distinfo-dir  | 45 +++++++++++++++++++
 cdist/conf/type/__package_pip/explorer/extras | 26 +++++++++++
 cdist/conf/type/__package_pip/explorer/state  |  0
 3 files changed, 71 insertions(+)
 create mode 100755 cdist/conf/type/__package_pip/explorer/distinfo-dir
 create mode 100755 cdist/conf/type/__package_pip/explorer/extras
 mode change 100644 => 100755 cdist/conf/type/__package_pip/explorer/state

diff --git a/cdist/conf/type/__package_pip/explorer/distinfo-dir b/cdist/conf/type/__package_pip/explorer/distinfo-dir
new file mode 100755
index 00000000..18e169ae
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/distinfo-dir
@@ -0,0 +1,45 @@
+#!/bin/sh
+#
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+nameparam="$__object/parameter/name"
+if [ -f "$nameparam" ]; then
+    name=$(cat "$nameparam")
+else
+    name="$__object_id"
+fi
+
+pipparam="$__object/parameter/pip"
+if [ -f "$pipparam" ]; then
+    pip=$(cat "$pipparam")
+else
+    pip="$( "$__type_explorer/pip" )"
+fi
+
+
+if command -v "$pip" >/dev/null 2>&1; then
+    # assemble the path where pip stores all pip package info
+    "$pip" show "$name" \
+        | awk -F': ' '
+            $1 == "Name" {name=$2; gsub(/-/,"_",name); next}
+            $1 == "Version" {version=$2; next}
+            $1 == "Location" {location=$2; next}
+            END {if (version != "") printf "%s/%s-%s.dist-info", location, name, version}'
+fi
diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
new file mode 100755
index 00000000..92fc4732
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -0,0 +1,26 @@
+#!/bin/sh
+#
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+distinfo_dir="$("$__type_explorer/distinfo-dir")"
+if [ "$distinfo_dir" ]; then
+    # output all extras that are installed
+    awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA"
+fi
diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state
old mode 100644
new mode 100755

From 8dc6ab97384f2240e9218fbd4463ba12b004f44c Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 13:49:53 +0100
Subject: [PATCH 561/743] __package_pip: install not found extras

Compares the explorer against the parameters and install those extras
that are not already installed.
---
 cdist/conf/type/__package_pip/gencode-remote | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 925eaf24..ad327e37 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -2,6 +2,7 @@
 #
 # 2012 Nico Schottelius (nico-cdist at schottelius.org)
 # 2016 Darko Poljak (darko.poljak at gmail.com)
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
 #
 # This file is part of cdist.
 #
@@ -58,14 +59,17 @@ case "$state_should" in
     present)
         if [ -f "$__object/parameter/extra" ]
         then
-            while read -r extra
-            do
-                if [ "$extras" ]; then
-                    extras="$extras,$extra"
-                else
-                    extras="$extra"
-                fi
-            done < "$__object/parameter/extra"
+            mkdir "$__object/files"
+            # sort and generalize all extras
+            sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params"
+            sort "$__object/explorer/extras" > "$__object/files/extras-explorer"
+
+            # assemble extras
+            # all extras are passed to pip in a comma-separated list
+            extras="$(comm -23 \
+                "$__object/files/extras-params" "$__object/files/extras-explorer" \
+                | sed ':a; $!N; s/\n/,/; ta'  # loops through all input lines and add commas between them
+            )"
             name="${name}[${extras}]"
         fi
 

From 2db0ef7c98226661e5e4b55c89180b31ba65b8bc Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 22:53:26 +0100
Subject: [PATCH 562/743] __package_pip: updating real detection of extras

As the previous detection took the wrong values, this explorer now
checks if packages for an extra are installed or not. If not, the extra
is not installed.

Based on the information of the explorer, it will install the package
again with the absent extras.
---
 cdist/conf/type/__package_pip/explorer/extras | 37 +++++++++++++++++--
 cdist/conf/type/__package_pip/gencode-remote  | 21 ++++-------
 2 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 92fc4732..2f5fcca6 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -17,10 +17,41 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+#
+# Checks if the given extras are really installed or not. It will be
+# done by querring all dependencies for that extra and return it as
+# "to be installed" if no dependency was found.
+#
 
 
 distinfo_dir="$("$__type_explorer/distinfo-dir")"
-if [ "$distinfo_dir" ]; then
-    # output all extras that are installed
-    awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA"
+
+# check if we have something to check
+if [ "$distinfo_dir" ] && [ -s "$__object/parameter/extra" ]
+then
+    # save cause freezing is slow
+    mkdir "$__object/files"
+    pip_freeze="$__object/files/pip-freeze.tmp"
+    pip3 freeze > "$pip_freeze"
+
+    for extra in $(cat "$__object/parameter/extra" | tr ',' '\n')
+    do
+        # create a grep BRE pattern to search all packages
+        grep_pattern="$(
+            awk -F'(: | ; )' -v check="$extra" '
+                $1 == "Requires-Dist" {
+                    split($2, r, " ");
+                    sub("extra == ", "", $3); gsub("'"'"'", "", $3);
+                    if($3 == check) print r[1]
+                }' "$distinfo_dir/METADATA" \
+                | sed ':a; $!N; s/\n/\\|/; ta'
+        )"
+
+        # echo the extra if no packages where found for it
+        # if there is no pattern, we don't need to search ;-)
+        if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze"
+        then
+            echo "$extra"
+        fi
+    done
 fi
diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index ad327e37..9abe28bf 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -26,7 +26,10 @@
 state_is=$(cat "$__object/explorer/state")
 state_should="$(cat "$__object/parameter/state")"
 
-[ "$state_is" = "$state_should" ] && exit 0
+# short circuit if state is the same and no extras to install
+[ "$state_is" = "$state_should" ] && ! [ -s "$__object/explorer/extras" ] \
+    && exit 0
+
 
 nameparam="$__object/parameter/name"
 if [ -f "$nameparam" ]; then
@@ -57,19 +60,11 @@ fi
 
 case "$state_should" in
     present)
-        if [ -f "$__object/parameter/extra" ]
+        if [ -s "$__object/explorer/extras" ]
         then
-            mkdir "$__object/files"
-            # sort and generalize all extras
-            sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params"
-            sort "$__object/explorer/extras" > "$__object/files/extras-explorer"
-
-            # assemble extras
-            # all extras are passed to pip in a comma-separated list
-            extras="$(comm -23 \
-                "$__object/files/extras-params" "$__object/files/extras-explorer" \
-                | sed ':a; $!N; s/\n/,/; ta'  # loops through all input lines and add commas between them
-            )"
+            # all extras are passed to pip in a comma-separated list in the name
+            # sed loops through all input lines and add commas between them
+            extras="$(sed ':a; $!N; s/\n/,/; ta' "$__object/explorer/extras")"
             name="${name}[${extras}]"
         fi
 

From 7398382890a4368e588e700183e6c777e90fb069 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 23:12:10 +0100
Subject: [PATCH 563/743] __package_pip: fix shellcheck

Useless `cat $file`, use `< $file` instead.
---
 cdist/conf/type/__package_pip/explorer/extras | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 2f5fcca6..d57c69db 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -34,7 +34,7 @@ then
     pip_freeze="$__object/files/pip-freeze.tmp"
     pip3 freeze > "$pip_freeze"
 
-    for extra in $(cat "$__object/parameter/extra" | tr ',' '\n')
+    for extra in $(tr ',' '\n' < "$__object/parameter/extra")
     do
         # create a grep BRE pattern to search all packages
         grep_pattern="$(

From a9d7dfb2ed40d0a61978cc706ff469c354a6c8f5 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 12 Feb 2021 09:17:02 +0100
Subject: [PATCH 564/743] __package_pip: split extra 'all' to a list of all
 extras

This will fix if a package will be upgraded from some extras to all
extras. Previously, it will not work because some dependencies of 'all'
are already installed, so the feature 'all' is already installed.

Now, it will use a list of all extras to iterate over them separatly. This
will result it will never install all extras via `[all]`, but rather
`[foo,bar]`.
---
 cdist/conf/type/__package_pip/explorer/extras | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index d57c69db..be945e36 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -34,7 +34,14 @@ then
     pip_freeze="$__object/files/pip-freeze.tmp"
     pip3 freeze > "$pip_freeze"
 
-    for extra in $(tr ',' '\n' < "$__object/parameter/extra")
+    # If all is set, it searches all available extras to separatly check them.
+    # It would work with just 'all' (cause dependencies are given), but will
+    # not update if one extra is already present. Side effect is that it will
+    # not use [all] but instead name all extras seperatly.
+    for extra in $(if grep -qFx all "$__object/parameter/extra";
+        then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA";
+        else tr ',' '\n' < "$__object/parameter/extra";
+        fi)
     do
         # create a grep BRE pattern to search all packages
         grep_pattern="$(

From 951712740f95f8f1277f1295bcd97c0b9bafdc94 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 12 Feb 2021 13:42:51 +0100
Subject: [PATCH 565/743] __package_pip: update man.rst

Adjusted comments for `explorer/extras` and updated the man page for the
new behaviour of updating the extras.
---
 cdist/conf/type/__package_pip/explorer/extras |  7 +++---
 cdist/conf/type/__package_pip/man.rst         | 25 ++++++++++++-------
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index be945e36..8104ed47 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -35,15 +35,16 @@ then
     pip3 freeze > "$pip_freeze"
 
     # If all is set, it searches all available extras to separatly check them.
-    # It would work with just 'all' (cause dependencies are given), but will
-    # not update if one extra is already present. Side effect is that it will
-    # not use [all] but instead name all extras seperatly.
+    # It would work with just 'all' (cause dependencies are specified for
+    # 'all'), but will not update if one extra is already present. Side effect
+    # is that it will not use [all] but instead name all extras seperatly.
     for extra in $(if grep -qFx all "$__object/parameter/extra";
         then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA";
         else tr ',' '\n' < "$__object/parameter/extra";
         fi)
     do
         # create a grep BRE pattern to search all packages
+        # maybe a file full of patterns for -F could be written
         grep_pattern="$(
             awk -F'(: | ; )' -v check="$extra" '
                 $1 == "Requires-Dist" {
diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index 64ad0358..c013695c 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -24,8 +24,14 @@ name
 
 extra
     Extra optional dependencies which should be installed along the selected
-    package. Can be specified multiple times. Will only be applied if the
-    package actually will be installed, but will not explicitly checked.
+    package. Can be specified multiple times. Multiple extra optional
+    dependencies can also be specified in a comma-separated list to provide
+    a more pip-natvie style.
+
+    Extra optional dependencies will be installed even when the base package
+    is already installed. Notice that the type will not remove installed extras
+    that are not explicitly named for the type because pip does not offer a
+    management for orphaned packages and they may be used by other packages.
 
 pip
     Instead of using pip from PATH, use the specific pip path.
@@ -53,8 +59,8 @@ EXAMPLES
 
     # Install package with optional dependencies
     __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails
-    # or do a little cheating
-    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails
+    # the extras can also be specified comma-separated
+    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails --extra postgres
 
     # or take all extras
     __package_pip mautrix-telegram --extra all
@@ -67,12 +73,13 @@ SEE ALSO
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2012 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2012 Nico Schottelius, 2021 Matthias Stecher. You can
+redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either
+version 3 of the License, or (at your option) any later version.

From 2ce1fce7677b64a8d17a86f29f676b1ba8475db2 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 15 Feb 2021 16:17:46 +0100
Subject: [PATCH 566/743] __package_pip: match package names case insensitive

Pip matches them insensitive, so we need to do the same to avoid
problems by saying extras are not installed but already is there in
place.
---
 cdist/conf/type/__package_pip/explorer/extras | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 8104ed47..bbdc17ab 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -57,7 +57,8 @@ then
 
         # echo the extra if no packages where found for it
         # if there is no pattern, we don't need to search ;-)
-        if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze"
+        # pip matches packages case-insensetive, we need to do that, too
+        if [ "$grep_pattern" ] && ! grep -qi "$grep_pattern" "$pip_freeze"
         then
             echo "$extra"
         fi

From 0835f414a5b2ce41309ee9b265cb80abb1362563 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 16 Feb 2021 16:03:23 +0100
Subject: [PATCH 567/743] [type/__postgres_conf] Extract PostgreSQL service
 user detection to separate explorer

---
 .../__postgres_conf/explorer/postgres_user    | 64 +++++++++++++++++++
 .../conf/type/__postgres_conf/explorer/state  | 16 +----
 .../conf/type/__postgres_conf/gencode-remote  | 17 +----
 3 files changed, 67 insertions(+), 30 deletions(-)
 create mode 100644 cdist/conf/type/__postgres_conf/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_conf/explorer/postgres_user b/cdist/conf/type/__postgres_conf/explorer/postgres_user
new file mode 100644
index 00000000..c6582dc4
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/explorer/postgres_user
@@ -0,0 +1,64 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$("${__explorer:?}/os")
+
+case ${os}
+in
+	(alpine)
+		echo 'postgres'
+		;;
+	(centos|rhel|scientific)
+		echo 'postgres'
+		;;
+	(debian|devuan|ubuntu)
+		echo 'postgres'
+		;;
+	(freebsd)
+		test -x /usr/local/etc/rc.d/postgresql || {
+			printf 'could not find postgresql rc script./n' >&2
+			exit 1
+		}
+		pg_status=$(/usr/local/etc/rc.d/postgresql onestatus) || {
+			printf 'postgresql daemon is not running.\n' >&2
+			exit 1
+		}
+		pg_pid=$(printf '%s\n' "${pg_status}" \
+			| sed -n 's/^pg_ctl:.*(PID: *\([0-9]*\))$/\1/p')
+
+		# PostgreSQL < 9.6: pgsql
+		# PostgreSQL >= 9.6: postgres
+		ps -o user -p "${pg_pid}" | sed -n '2p'
+		;;
+	(netbsd)
+		echo 'pgsql'
+		;;
+	(openbsd)
+		echo '_postgresql'
+		;;
+	(suse)
+		echo 'postgres'
+		;;
+	(*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 589925de..de6e8aa0 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -19,21 +19,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$("${__explorer:?}/os")
-
-case ${os}
-in
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(devuan)
-		postgres_user='postgres'
-		;;
-	(*)
-		echo "Unsupported OS: ${os}" >&2
-		exit 1
-		;;
-esac
+postgres_user=$("${__type_explorer:?}/postgres_user")
 
 conf_name=${__object_id:?}
 
diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 8f8a2bfe..d0d247b4 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -20,9 +20,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$(cat "${__global:?}/explorer/os")
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
 conf_name=${__object_id:?}
 
@@ -49,19 +49,6 @@ quote() {
 	printf '%s' "$*"
 }
 
-case ${os}
-in
-	(netbsd)
-		postgres_user='pgsql'
-		;;
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(*)
-		postgres_user='postgres'
-		;;
-esac
-
 
 psql_cmd() {
 	printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")"
@@ -117,7 +104,7 @@ then
 				case $(cat "${__global:?}/explorer/kernel_name")
 				in
 					(FreeBSD)
-						echo 'service postgresql restart'
+						echo '/usr/local/etc/rc.d/postgresql restart'
 						;;
 					(OpenBSD|NetBSD)
 						echo '/etc/rc.d/postgresql restart'

From d1f45d3524f8a025488d0d2f020433b6efb7edb6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 19 Feb 2021 09:03:56 +0100
Subject: [PATCH 568/743] __package_pip: corrected typo in man

.. by fully replacing it with a smaller sentence.
---
 cdist/conf/type/__package_pip/man.rst | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index c013695c..5a2bc673 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -24,9 +24,8 @@ name
 
 extra
     Extra optional dependencies which should be installed along the selected
-    package. Can be specified multiple times. Multiple extra optional
-    dependencies can also be specified in a comma-separated list to provide
-    a more pip-natvie style.
+    package. Can be specified multiple times. Multiple extras can be passed
+    in one `--extra` as a comma-separated list.
 
     Extra optional dependencies will be installed even when the base package
     is already installed. Notice that the type will not remove installed extras

From 5e0572189fb930764318227fa9fd9f4966b0da99 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 22 Feb 2021 09:11:22 +0100
Subject: [PATCH 569/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 52686617..59c58e65 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,8 @@ next:
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
+	* Type __postgres_role: Implement modification of roles (Dennis Camera)
+	* Type __letsencrypt_cert: Fix issues with hooks (Evil Ham)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)

From 22f637c15b6616899c848e2a08c4e1c1b13f0f0b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 23 Feb 2021 06:29:24 +0100
Subject: [PATCH 570/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 59c58e65..88dda0aa 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
 	* Type __postgres_role: Implement modification of roles (Dennis Camera)
 	* Type __letsencrypt_cert: Fix issues with hooks (Evil Ham)
+	* Type __package_pip: Add optional extra dependencies param (Matthias Stecher)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)

From 0734288483700e8e10cebd87c797b625aa83d55e Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Sun, 21 Feb 2021 19:59:57 +0000
Subject: [PATCH 571/743] First draft of __apt_pin

---
 cdist/conf/type/__apt_pin/man.rst             | 53 ++++++++++++++++
 cdist/conf/type/__apt_pin/manifest            | 63 +++++++++++++++++++
 .../type/__apt_pin/parameter/default/package  |  1 +
 .../type/__apt_pin/parameter/default/state    |  1 +
 cdist/conf/type/__apt_pin/parameter/optional  |  2 +
 cdist/conf/type/__apt_pin/parameter/required  |  2 +
 6 files changed, 122 insertions(+)
 create mode 100644 cdist/conf/type/__apt_pin/man.rst
 create mode 100755 cdist/conf/type/__apt_pin/manifest
 create mode 100644 cdist/conf/type/__apt_pin/parameter/default/package
 create mode 100644 cdist/conf/type/__apt_pin/parameter/default/state
 create mode 100644 cdist/conf/type/__apt_pin/parameter/optional
 create mode 100644 cdist/conf/type/__apt_pin/parameter/required

diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst
new file mode 100644
index 00000000..7fcae6f8
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/man.rst
@@ -0,0 +1,53 @@
+cdist-type__apt_pin(7)
+======================
+
+NAME
+----
+cdist-type__apt_pin - TODO
+
+
+DESCRIPTION
+-----------
+This space intentionally left blank.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # TODO
+    __apt_pin
+
+
+SEE ALSO
+--------
+:strong:`TODO`\ (7)
+
+
+AUTHORS
+-------
+Daniel Fancsali <fancsali@gmail.com>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Daniel Fancsali. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
new file mode 100755
index 00000000..8dd9770d
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -0,0 +1,63 @@
+#!/bin/sh -e
+#
+# 2021 Daniel Fancsali (fancsali@gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+os=$(cat "$__global/explorer/os")
+state="$(cat "$__object/parameter/state")"
+package="$(cat "$__object/parameter/package")"
+distribution="$(cat "$__object/parameter/distribution")"
+priority="$(cat "$__object/parameter/priority")"
+
+
+case "$os" in
+   debian|ubuntu|devuan)
+   ;;
+   *)
+      printf "This type is specific to Debian and it's derivatives" >&2
+      printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2
+      exit 1
+   ;;
+esac
+
+if [ "$package" = "*" ]; then
+    name="default"
+
+else
+    name="$__object_id"
+fi
+
+case $distribution in
+    stabletesting|unsatbel|experimental)
+        pin="release a=$distribution"
+        ;;
+    *)
+        pin="release n=$distribution"
+        ;;
+esac
+
+
+__file /etc/apt/preferences.d/$name \
+    --owner root --group root --mode 0644 \
+    --state "$state" \
+    --source - << EOF
+Package: $package
+Pin: $pin
+Pin-Priority: $priority
+EOF
diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package
new file mode 100644
index 00000000..72e8ffc0
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/default/package
@@ -0,0 +1 @@
+*
diff --git a/cdist/conf/type/__apt_pin/parameter/default/state b/cdist/conf/type/__apt_pin/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional
new file mode 100644
index 00000000..52f01fd2
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/optional
@@ -0,0 +1,2 @@
+state
+package
diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required
new file mode 100644
index 00000000..4b4e9741
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/required
@@ -0,0 +1,2 @@
+distribution
+priority

From 1a74470c4d9b30e41a72fbfc084dc54ea44e643b Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Tue, 23 Feb 2021 09:37:36 +0000
Subject: [PATCH 572/743] __apt_pin: Always use $__object_id as preferences.d
 filename

---
 cdist/conf/type/__apt_pin/manifest | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index 8dd9770d..162b523f 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -36,12 +36,7 @@ case "$os" in
    ;;
 esac
 
-if [ "$package" = "*" ]; then
-    name="default"
-
-else
-    name="$__object_id"
-fi
+name="$__object_id"
 
 case $distribution in
     stabletesting|unsatbel|experimental)

From dc66efa690e15ff32d6836e23baa1cbec5eee1ed Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Tue, 23 Feb 2021 11:59:09 +0000
Subject: [PATCH 573/743] Fix shellcheck issues

---
 cdist/conf/type/__apt_pin/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index 162b523f..b1372ad0 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -48,7 +48,7 @@ case $distribution in
 esac
 
 
-__file /etc/apt/preferences.d/$name \
+__file "/etc/apt/preferences.d/$name" \
     --owner root --group root --mode 0644 \
     --state "$state" \
     --source - << EOF

From 60fd7ba1f38382761c366cbc0425ddaec62f1164 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 28 Feb 2021 13:37:23 +0100
Subject: [PATCH 574/743] Release 6.9.5

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 88dda0aa..95e36b88 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)

From 8ef19d47f6b5fb6cdcba7a9a3b8890388dbd7023 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 1 Mar 2021 17:59:25 +0100
Subject: [PATCH 575/743] [type/__pyvenv] Fix example (--user -> --owner)

---
 cdist/conf/type/__pyvenv/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__pyvenv/man.rst b/cdist/conf/type/__pyvenv/man.rst
index 8085ff12..e2e4a1e6 100644
--- a/cdist/conf/type/__pyvenv/man.rst
+++ b/cdist/conf/type/__pyvenv/man.rst
@@ -61,7 +61,7 @@ EXAMPLES
     __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4
 
     # Create python virtualenv for user foo.
-    __pyvenv /home/foo/fooenv --group foo --user foo
+    __pyvenv /home/foo/fooenv --group foo --owner foo
 
     # Create python virtualenv with specific parameters.
     __pyvenv /home/services/djangoenv --venvparams "--copies --system-site-packages"

From e7d33891df945831d54d164eb13f2a9ae4f9dd8e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 2 Mar 2021 09:29:33 +0100
Subject: [PATCH 576/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 95e36b88..072fcb6b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __pyvenv: Fix user example in man page (Dennis Camera)
+
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)

From ea0126dd8117c051612e6b6abfe895f5f722c584 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Fri, 5 Mar 2021 16:11:49 +0100
Subject: [PATCH 577/743] Make local state dir available to custom remote
 scripts

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/config.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cdist/config.py b/cdist/config.py
index e84f6f84..19d5bd70 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -420,6 +420,9 @@ class Config:
                 exec_path=sys.argv[0],
                 save_output_streams=args.save_output_streams)
 
+            # Make __global state dir available to custom remote scripts.
+            os.environ['__global'] = local.base_path
+
             remote = cdist.exec.remote.Remote(
                 target_host=target_host,
                 remote_exec=remote_exec,

From ecba284fc8ce68486c5f1f87a863409fcad0f106 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Fri, 5 Mar 2021 16:13:02 +0100
Subject: [PATCH 578/743] changelog++

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 072fcb6b..ad93a595 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
+	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From fb19f342669d18e6decb21ee4cfb2b22e46748d9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 9 Mar 2021 21:15:26 +0100
Subject: [PATCH 579/743] [type/__ssh_authorized_key] Only grep if file exists

---
 cdist/conf/type/__ssh_authorized_key/explorer/entry | 1 +
 cdist/conf/type/__ssh_authorized_key/gencode-remote | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__ssh_authorized_key/explorer/entry b/cdist/conf/type/__ssh_authorized_key/explorer/entry
index ccab0afc..aca0f2b9 100755
--- a/cdist/conf/type/__ssh_authorized_key/explorer/entry
+++ b/cdist/conf/type/__ssh_authorized_key/explorer/entry
@@ -25,6 +25,7 @@ type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ]
 if [ -n "${type_and_key}" ]
 then
     file="$(cat "$__object/parameter/file")"
+    test -e "$file" || exit 0
 
     # get any entries that match the type and key
 
diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote
index f37aa565..61c77fb9 100755
--- a/cdist/conf/type/__ssh_authorized_key/gencode-remote
+++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote
@@ -37,9 +37,9 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
 # preserve ownership and permissions of existing file
 if [ -f "$file" ]; then
    cp -p "$file" "\$tmpfile"
+   grep -v -F -x '$line' '$file' >\$tmpfile
 fi
-grep -v -F -x '$line' '$file' > \$tmpfile || true
-mv -f "\$tmpfile" "$file"
+cat "\$tmpfile" >"$file"
 DONE
 }
 

From 31cc592aa10902c3e83357ee4a0b9300b5e34efa Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 10 Mar 2021 19:25:04 +0100
Subject: [PATCH 580/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index ad93a595..c63b901a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
+	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From e47c4dd8a4cc441b94fe73d362d79525d01a4bb1 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 9 Mar 2021 19:59:05 +0100
Subject: [PATCH 581/743] [type/__sshd_config] Whitelist OpenBMC in manifest

---
 cdist/conf/type/__sshd_config/manifest | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest
index 566bde90..e37afebb 100755
--- a/cdist/conf/type/__sshd_config/manifest
+++ b/cdist/conf/type/__sshd_config/manifest
@@ -39,7 +39,14 @@ in
 	(freebsd|netbsd|openbsd)
 		# whitelist
 		;;
+	(openbmc-phosphor)
+		# whitelist
+		# OpenBMC can be configured with dropbear and OpenSSH.
+		# If dropbear is used, the state explorer will already fail because it
+		# cannot find the sshd binary.
+		;;
 	(*)
+		: "${__type:?}"  # make shellcheck happy
 		printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \
 			"${os}" "${__type##*/}" >&2
 		printf 'Please contribute an implementation for it if you can.\n' >&2

From 10ca1c12fd25a9f2b92cebbc5d375a1723ed5e2f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 12 Mar 2021 08:21:03 +0100
Subject: [PATCH 582/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index c63b901a..42a74d04 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
+	* Type __sshd_config: Whitelist OpenBMC (Dennis Camera)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From 7a0b697f4c394f1ae8a8941a59937007cfe474aa Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 21 Mar 2021 18:10:06 +0100
Subject: [PATCH 583/743] Implement maintaining object relationship graph

For each object maintain parent-child relationship graph, i.e. list of
parent objects ('parents' property) and list of children objects ('children'
property).

Objects without parent(s) are objects specified in init manifest.
Objects without children are object of types that do not reuse other types.
---
 cdist/core/cdist_object.py |  7 +++++++
 cdist/emulator.py          | 19 +++++++++++++++++++
 docs/src/cdist-cache.rst   | 17 +++++++++++++++++
 3 files changed, 43 insertions(+)

diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index 51d61e04..eea2c255 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -247,6 +247,13 @@ class CdistObject:
             lambda obj: os.path.join(obj.absolute_path, 'typeorder'))
     typeorder_dep = fsproperty.FileListProperty(
             lambda obj: os.path.join(obj.absolute_path, 'typeorder_dep'))
+    # objects without parents are objects specified in init manifest
+    parents = fsproperty.FileListProperty(
+            lambda obj: os.path.join(obj.absolute_path, 'parents'))
+    # objects without children are object of types that do not reuse other
+    # types
+    children = fsproperty.FileListProperty(
+            lambda obj: os.path.join(obj.absolute_path, 'children'))
 
     def cleanup(self):
         try:
diff --git a/cdist/emulator.py b/cdist/emulator.py
index a2bdc3d4..f1db862e 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -106,6 +106,7 @@ class Emulator:
             self.save_stdin()
             self.record_requirements()
             self.record_auto_requirements()
+            self.record_parent_child_relationships()
             self.log.trace("Finished %s %s" % (
                 self.cdist_object.path, self.parameters))
 
@@ -420,3 +421,21 @@ class Emulator:
                 self.log.debug("Recording autorequirement %s for %s",
                                current_object.name, parent.name)
                 parent.autorequire.append(current_object.name)
+
+    def record_parent_child_relationships(self):
+        # __object_name is the name of the object whose type manifest is
+        # currently executed
+        __object_name = self.env.get('__object_name', None)
+        if __object_name:
+            # The object whose type manifest is currently run
+            parent = self.cdist_object.object_from_name(__object_name)
+            # The object currently being defined
+            current_object = self.cdist_object
+            if current_object.name not in parent.children:
+                self.log.debug("Recording child %s for %s",
+                               current_object.name, parent.name)
+                parent.children.append(current_object.name)
+            if parent.name not in current_object.parents:
+                self.log.debug("Recording parent %s for %s",
+                               parent.name, current_object.name)
+                current_object.parents.append(parent.name)
diff --git a/docs/src/cdist-cache.rst b/docs/src/cdist-cache.rst
index d2d2d56c..d4159e77 100644
--- a/docs/src/cdist-cache.rst
+++ b/docs/src/cdist-cache.rst
@@ -61,6 +61,14 @@ Object cache overview
 ~~~~~~~~~~~~~~~~~~~~~
 Each object under :strong:`object` directory has its own structure.
 
+autorequire
+    file containing a list of object auto requirements
+
+children
+    file containing a list of object children, i.e. objects of types that this
+    type reuses (along with 'parents' it is used for maintaining parent-child
+    relationship graph)
+
 code-local
     code generated from gencode-local, present only if something is
     generated
@@ -80,6 +88,15 @@ parameter
     directory containing type parameter named files containing parameter
     values   
 
+parents
+    file containing a list of object parents, i.e. objects of types that reuse
+    this type (along with 'children' it is used for maintaining parent-child
+    relationship graph); objects without parents are objects specified in init
+    manifest
+
+require
+    file containing a list of object requirements
+
 source
     this type's source (init manifest)
 

From 167c2ad7ea2b895cdc7d89c72968d88c7d1ab34f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 30 Mar 2021 13:24:56 +0200
Subject: [PATCH 584/743] [type/__git] Fix if --owner / --group is numeric

Before, if --owner and/or --group was numeric, gencode-remote would generate
`chown` code every time.
---
 cdist/conf/type/__git/explorer/group | 20 +++++++++++++++++---
 cdist/conf/type/__git/explorer/owner | 20 +++++++++++++++++---
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group
index 3ddf9656..1365c60d 100644
--- a/cdist/conf/type/__git/explorer/group
+++ b/cdist/conf/type/__git/explorer/group
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id/.git"
+destination="/${__object_id:?}/.git"
 
-stat --print "%G" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((group_gid + 1)) -ge 0
+then
+	group_should=$(cat "${__object:?}/parameter/group")
+
+	if expr "${group_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${group_gid}"
+	else
+		printf '%s\n' "$(id -u -n "${group_gid}")"
+	fi
+fi
diff --git a/cdist/conf/type/__git/explorer/owner b/cdist/conf/type/__git/explorer/owner
index 4c3cd431..4a4d0d13 100644
--- a/cdist/conf/type/__git/explorer/owner
+++ b/cdist/conf/type/__git/explorer/owner
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id/.git"
+destination="/${__object_id:?}/.git"
 
-stat --print "%U" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((owner_uid + 1)) -ge 0
+then
+	owner_should=$(cat "${__object:?}/parameter/owner")
+
+	if expr "${owner_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${owner_uid}"
+	else
+		printf '%s\n' "$(id -u -n "${owner_uid}")"
+	fi
+fi

From 985252585ce83605787d2612fdc2a84b9788d943 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 30 Mar 2021 13:26:21 +0200
Subject: [PATCH 585/743] [type/__pyvenv] Fix if --owner / --group is numeric

Before, if --owner and/or --group was numeric, gencode-remote would generate
`chown` code every time.
---
 cdist/conf/type/__pyvenv/explorer/group | 20 +++++++++++++++++---
 cdist/conf/type/__pyvenv/explorer/owner | 20 +++++++++++++++++---
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group
index a655bda7..f31a1cb7 100755
--- a/cdist/conf/type/__pyvenv/explorer/group
+++ b/cdist/conf/type/__pyvenv/explorer/group
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id"
+destination="/${__object_id:?}"
 
-stat --print "%G" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((group_gid + 1)) -ge 0
+then
+	group_should=$(cat "${__object:?}/parameter/group")
+
+	if expr "${group_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${group_gid}"
+	else
+		printf '%s\n' "$(id -u -n "${group_gid}")"
+	fi
+fi
diff --git a/cdist/conf/type/__pyvenv/explorer/owner b/cdist/conf/type/__pyvenv/explorer/owner
index 8b3c7f8e..ebec751f 100755
--- a/cdist/conf/type/__pyvenv/explorer/owner
+++ b/cdist/conf/type/__pyvenv/explorer/owner
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id"
+destination="/${__object_id:?}"
 
-stat --print "%U" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((owner_uid + 1)) -ge 0
+then
+	owner_should=$(cat "${__object:?}/parameter/owner")
+
+	if expr "${owner_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${owner_uid}"
+	else
+		printf '%s\n' "$(id -u -n "${owner_uid}")"
+	fi
+fi

From 1e765fcab7bae02d6b380ca02e7cb04ec69ebdc5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 31 Mar 2021 07:54:00 +0200
Subject: [PATCH 586/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 42a74d04..03ae7566 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
 	* Type __sshd_config: Whitelist OpenBMC (Dennis Camera)
+	* Core: Maintain object relationship graph in cdist cache (Darko Poljak)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From f984a918b959a46e49fe8456a327eca8d44313de Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 30 Mar 2021 07:56:38 +0200
Subject: [PATCH 587/743] Fix log message string formatting

Use logging message format with args, instead of direct `%` or `str.format`.

Resolve #855.
---
 cdist/argparse.py                      |  8 +--
 cdist/config.py                        | 87 ++++++++++++--------------
 cdist/core/cdist_type.py               |  6 +-
 cdist/core/explorer.py                 | 26 ++++----
 cdist/core/manifest.py                 |  2 +-
 cdist/emulator.py                      | 37 +++++------
 cdist/exec/local.py                    | 16 +++--
 cdist/exec/remote.py                   | 14 ++---
 cdist/install.py                       |  2 +-
 cdist/inventory.py                     | 36 +++++------
 cdist/preos.py                         |  6 +-
 cdist/preos/debootstrap/debootstrap.py | 41 ++++++------
 cdist/scan/scan.py                     |  4 +-
 cdist/util/ipaddr.py                   |  6 +-
 14 files changed, 140 insertions(+), 151 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 88759d7b..cadac39a 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -533,10 +533,10 @@ def parse_and_configure(argv, singleton=True):
 
     log = logging.getLogger("cdist")
 
-    log.verbose("version %s" % cdist.VERSION)
-    log.trace('command line args: {}'.format(cfg.command_line_args))
-    log.trace('configuration: {}'.format(cfg.get_config()))
-    log.trace('configured args: {}'.format(args))
+    log.verbose("version %s", cdist.VERSION)
+    log.trace('command line args: %s', cfg.command_line_args)
+    log.trace('configuration: %s', cfg.get_config())
+    log.trace('configured args: %s', args)
 
     check_beta(vars(args))
 
diff --git a/cdist/config.py b/cdist/config.py
index 19d5bd70..d6fec55f 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -273,15 +273,15 @@ class Config:
                     host_tags = None
             host_base_path, hostdir = cls.create_host_base_dirs(
                 host, base_root_path)
-            log.debug("Base root path for target host \"{}\" is \"{}\"".format(
-                host, host_base_path))
+            log.debug("Base root path for target host \"%s\" is \"%s\"",
+                      host, host_base_path)
 
             hostcnt += 1
             if args.parallel:
                 pargs = (host, host_tags, host_base_path, hostdir, args, True,
                          configuration)
-                log.trace(("Args for multiprocessing operation "
-                           "for host {}: {}".format(host, pargs)))
+                log.trace("Args for multiprocessing operation for host %s: %s",
+                          host, pargs)
                 process_args.append(pargs)
             else:
                 try:
@@ -298,10 +298,10 @@ class Config:
             except cdist.Error:
                 failed_hosts.append(host)
         elif args.parallel:
-            log.trace("Multiprocessing start method is {}".format(
-                multiprocessing.get_start_method()))
-            log.trace(("Starting multiprocessing Pool for {} "
-                       "parallel host operation".format(args.parallel)))
+            log.trace("Multiprocessing start method is %s",
+                      multiprocessing.get_start_method())
+            log.trace("Starting multiprocessing Pool for %d parallel host"
+                      " operation", args.parallel)
 
             results = mp_pool_run(cls.onehost,
                                   process_args,
@@ -396,16 +396,13 @@ class Config:
 
             remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds(
                 args)
-            log.debug("remote_exec for host \"{}\": {}".format(
-                host, remote_exec))
-            log.debug("remote_copy for host \"{}\": {}".format(
-                host, remote_copy))
+            log.debug("remote_exec for host \"%s\": %s", host, remote_exec)
+            log.debug("remote_copy for host \"%s\": %s", host, remote_copy)
 
             family = cls._address_family(args)
-            log.debug("address family: {}".format(family))
+            log.debug("address family: %s", family)
             target_host = cls.resolve_target_addresses(host, family)
-            log.debug("target_host for host \"{}\": {}".format(
-                host, target_host))
+            log.debug("target_host for host \"%s\": %s", host, target_host)
 
             local = cdist.exec.local.Local(
                 target_host=target_host,
@@ -474,8 +471,8 @@ class Config:
         """Do what is most often done: deploy & cleanup"""
         start_time = time.time()
 
-        self.log.info("Starting {} run".format(
-            'dry' if self.dry_run else 'configuration'))
+        self.log.info("Starting %s run",
+                      'dry' if self.dry_run else 'configuration')
 
         self._init_files_dirs()
 
@@ -493,9 +490,9 @@ class Config:
         self._remove_files_dirs()
 
         self.local.save_cache(start_time)
-        self.log.info("Finished {} run in {:.2f} seconds".format(
+        self.log.info("Finished %s run in %.2f seconds",
                       'dry' if self.dry_run else 'successful',
-                      time.time() - start_time))
+                      time.time() - start_time)
 
     def cleanup(self):
         self.log.debug("Running cleanup commands")
@@ -519,8 +516,8 @@ class Config:
                 self.local.object_path, self.local.type_path,
                 self.local.object_marker_name):
             if cdist_object.cdist_type.is_install:
-                self.log.debug(("Running in config mode, ignoring install "
-                                "object: {0}").format(cdist_object))
+                self.log.debug("Running in config mode, ignoring install "
+                               "object: %s", cdist_object)
             else:
                 yield cdist_object
 
@@ -565,8 +562,7 @@ class Config:
         return objects_changed
 
     def _iterate_once_parallel(self):
-        self.log.debug("Iteration in parallel mode in {} jobs".format(
-            self.jobs))
+        self.log.debug("Iteration in parallel mode in %d jobs", self.jobs)
         objects_changed = False
 
         cargo = []
@@ -588,8 +584,8 @@ class Config:
             self.object_prepare(cargo[0])
             objects_changed = True
         elif cargo:
-            self.log.trace("Multiprocessing start method is {}".format(
-                multiprocessing.get_start_method()))
+            self.log.trace("Multiprocessing start method is %s",
+                           multiprocessing.get_start_method())
 
             self.log.trace("Multiprocessing cargo: %s", cargo)
 
@@ -603,9 +599,8 @@ class Config:
                                 "sequentially"))
                 self.explorer.transfer_type_explorers(cargo_types.pop())
             else:
-                self.log.trace(("Starting multiprocessing Pool for {} "
-                                "parallel types explorers transferring".format(
-                                    nt)))
+                self.log.trace("Starting multiprocessing Pool for %d "
+                               "parallel types explorers transferring", nt)
                 args = [
                     (ct, ) for ct in cargo_types
                 ]
@@ -614,8 +609,8 @@ class Config:
                 self.log.trace(("Multiprocessing for parallel transferring "
                                 "types' explorers finished"))
 
-            self.log.trace(("Starting multiprocessing Pool for {} parallel "
-                            "objects preparation".format(n)))
+            self.log.trace("Starting multiprocessing Pool for %d parallel "
+                           "objects preparation", n)
             args = [
                 (c, False, ) for c in cargo
             ]
@@ -667,10 +662,10 @@ class Config:
                 self.object_run(chunk[0])
                 objects_changed = True
             elif chunk:
-                self.log.trace("Multiprocessing start method is {}".format(
-                    multiprocessing.get_start_method()))
-                self.log.trace(("Starting multiprocessing Pool for {} "
-                               "parallel object run".format(n)))
+                self.log.trace("Multiprocessing start method is %s",
+                               multiprocessing.get_start_method())
+                self.log.trace("Starting multiprocessing Pool for %d "
+                               "parallel object run", n)
                 args = [
                     (c, ) for c in chunk
                 ]
@@ -794,9 +789,9 @@ class Config:
     def object_prepare(self, cdist_object, transfer_type_explorers=True):
         """Prepare object: Run type explorer + manifest"""
         self._handle_deprecation(cdist_object)
-        self.log.verbose("Preparing object {}".format(cdist_object.name))
-        self.log.verbose(
-            "Running manifest and explorers for " + cdist_object.name)
+        self.log.verbose("Preparing object %s", cdist_object.name)
+        self.log.verbose("Running manifest and explorers for %s",
+                         cdist_object.name)
         self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
         try:
             self.manifest.run_type_manifest(cdist_object)
@@ -810,13 +805,13 @@ class Config:
     def object_run(self, cdist_object):
         """Run gencode and code for an object"""
         try:
-            self.log.verbose("Running object " + cdist_object.name)
+            self.log.verbose("Running object %s", cdist_object.name)
             if cdist_object.state == core.CdistObject.STATE_DONE:
                 raise cdist.Error(("Attempting to run an already finished "
-                                   "object: %s"), cdist_object)
+                                   "object: {}").format(cdist_object))
 
             # Generate
-            self.log.debug("Generating code for %s" % (cdist_object.name))
+            self.log.debug("Generating code for %s", cdist_object.name)
             cdist_object.code_local = self.code.run_gencode_local(cdist_object)
             cdist_object.code_remote = self.code.run_gencode_remote(
                 cdist_object)
@@ -825,20 +820,20 @@ class Config:
 
             # Execute
             if cdist_object.code_local or cdist_object.code_remote:
-                self.log.info("Processing %s" % (cdist_object.name))
+                self.log.info("Processing %s", cdist_object.name)
             if not self.dry_run:
                 if cdist_object.code_local:
-                    self.log.trace("Executing local code for %s"
-                                   % (cdist_object.name))
+                    self.log.trace("Executing local code for %s",
+                                   cdist_object.name)
                     self.code.run_code_local(cdist_object)
                 if cdist_object.code_remote:
-                    self.log.trace("Executing remote code for %s"
-                                   % (cdist_object.name))
+                    self.log.trace("Executing remote code for %s",
+                                   cdist_object.name)
                     self.code.transfer_code_remote(cdist_object)
                     self.code.run_code_remote(cdist_object)
 
             # Mark this object as done
-            self.log.trace("Finishing run of " + cdist_object.name)
+            self.log.trace("Finishing run of %s", cdist_object.name)
             cdist_object.state = core.CdistObject.STATE_DONE
         except cdist.Error as e:
             raise cdist.CdistObjectError(cdist_object, e)
diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py
index c0329c8a..274de989 100644
--- a/cdist/core/cdist_type.py
+++ b/cdist/core/cdist_type.py
@@ -82,9 +82,9 @@ class CdistType:
                 yield cls(base_path, name)
             except InvalidTypeError as e:
                 # ignore invalid type, log warning and continue
-                msg = "Ignoring invalid type '%s' at '%s' defined at '%s'" % (
-                    e.type_path, e.type_absolute_path, e.source_path)
-                cls.log.warning(msg)
+                cls.log.warning("Ignoring invalid type '%s' at '%s' defined"
+                                " at '%s'", e.type_path, e.type_absolute_path,
+                                e.source_path)
                 # remove invalid from runtime conf dir
                 os.remove(e.type_absolute_path)
 
diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py
index a3baa959..caa12a7d 100644
--- a/cdist/core/explorer.py
+++ b/cdist/core/explorer.py
@@ -131,18 +131,17 @@ class Explorer:
             self._run_global_explorer(explorer, out_path)
 
     def _run_global_explorers_parallel(self, out_path):
-        self.log.debug("Running global explorers in {} parallel jobs".format(
-            self.jobs))
-        self.log.trace("Multiprocessing start method is {}".format(
-            multiprocessing.get_start_method()))
-        self.log.trace(("Starting multiprocessing Pool for global "
-                       "explorers run"))
+        self.log.debug("Running global explorers in %s parallel jobs",
+                       self.jobs)
+        self.log.trace("Multiprocessing start method is %s",
+                       multiprocessing.get_start_method())
+        self.log.trace("Starting multiprocessing Pool for global explorers"
+                       " run")
         args = [
             (e, out_path, ) for e in self.list_global_explorer_names()
         ]
         mp_pool_run(self._run_global_explorer, args, jobs=self.jobs)
-        self.log.trace(("Multiprocessing run for global explorers "
-                        "finished"))
+        self.log.trace("Multiprocessing run for global explorers finished")
 
     # logger is not pickable, so remove it when we pickle
     def __getstate__(self):
@@ -184,15 +183,14 @@ class Explorer:
         in the object.
 
         """
-        self.log.verbose("Running type explorers for {}".format(
-            cdist_object.cdist_type))
+        self.log.verbose("Running type explorers for %s",
+                         cdist_object.cdist_type)
         if transfer_type_explorers:
             self.log.trace("Transferring type explorers for type: %s",
                            cdist_object.cdist_type)
             self.transfer_type_explorers(cdist_object.cdist_type)
         else:
-            self.log.trace(("No need for transferring type explorers for "
-                            "type: %s"),
+            self.log.trace("No need for transferring type explorers for %s",
                            cdist_object.cdist_type)
         self.log.trace("Transferring object parameters for object: %s",
                        cdist_object.name)
@@ -236,8 +234,8 @@ class Explorer:
            remote side."""
         if cdist_type.explorers:
             if cdist_type.name in self._type_explorers_transferred:
-                self.log.trace(("Skipping retransfer of type explorers "
-                                "for: %s"), cdist_type)
+                self.log.trace("Skipping retransfer of type explorers for: %s",
+                               cdist_type)
             else:
                 source = os.path.join(self.local.type_path,
                                       cdist_type.explorer_path)
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 390340d4..3148d66c 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -161,7 +161,7 @@ class Manifest:
             raise NoInitialManifestError(initial_manifest, user_supplied)
 
         message_prefix = "initialmanifest"
-        self.log.verbose("Running initial manifest " + initial_manifest)
+        self.log.verbose("Running initial manifest %s", initial_manifest)
         which = "init"
         if self.local.save_output_streams:
             stderr_path = os.path.join(self.local.stderr_base_path, which)
diff --git a/cdist/emulator.py b/cdist/emulator.py
index f1db862e..f09c282d 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -107,8 +107,8 @@ class Emulator:
             self.record_requirements()
             self.record_auto_requirements()
             self.record_parent_child_relationships()
-            self.log.trace("Finished %s %s" % (
-                self.cdist_object.path, self.parameters))
+            self.log.trace("Finished %s %s", self.cdist_object.path,
+                           self.parameters)
 
     def __init_log(self):
         """Setup logging facility"""
@@ -170,7 +170,7 @@ class Emulator:
 
         # And finally parse/verify parameter
         self.args = parser.parse_args(self.argv[1:])
-        self.log.trace('Args: %s' % self.args)
+        self.log.trace('Args: %s', self.args)
 
     def init_object(self):
         # Initialize object - and ensure it is not in args
@@ -241,8 +241,8 @@ class Emulator:
                 raise cdist.Error(errmsg)
         else:
             if self.cdist_object.exists:
-                self.log.debug(('Object %s override forced with '
-                                'CDIST_OVERRIDE'), self.cdist_object.name)
+                self.log.debug('Object %s override forced with CDIST_OVERRIDE',
+                               self.cdist_object.name)
                 self.cdist_object.create(True)
             else:
                 self.cdist_object.create()
@@ -260,8 +260,8 @@ class Emulator:
             parent = self.cdist_object.object_from_name(__object_name)
             parent.typeorder.append(self.cdist_object.name)
             if self._order_dep_on():
-                self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
-                                ' for %s'), depname, parent.name)
+                self.log.trace('[ORDER_DEP] Adding %s to typeorder dep for %s',
+                               depname, parent.name)
                 parent.typeorder_dep.append(depname)
         elif self._order_dep_on():
             self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
@@ -301,16 +301,14 @@ class Emulator:
         try:
             cdist_object = self.cdist_object.object_from_name(requirement)
         except core.cdist_type.InvalidTypeError as e:
-            self.log.error(("%s requires object %s, but type %s does not"
-                            " exist. Defined at %s" % (
-                                self.cdist_object.name,
-                                requirement, e.name, self.object_source)))
+            self.log.error("%s requires object %s, but type %s does not"
+                           " exist. Defined at %s", self.cdist_object.name,
+                           requirement, e.name, self.object_source)
             raise
         except core.cdist_object.MissingObjectIdError:
-            self.log.error(("%s requires object %s without object id."
-                            " Defined at %s" % (self.cdist_object.name,
-                                                requirement,
-                                                self.object_source)))
+            self.log.error("%s requires object %s without object id."
+                           " Defined at %s", self.cdist_object.name,
+                           requirement, self.object_source)
             raise
 
         self.log.debug("Recording requirement %s for %s",
@@ -380,10 +378,9 @@ class Emulator:
                         self.env['require'] += " " + lastcreatedtype
                 else:
                     self.env['require'] = lastcreatedtype
-                self.log.debug(("Injecting require for "
-                                "CDIST_ORDER_DEPENDENCY: %s for %s"),
-                               lastcreatedtype,
-                               self.cdist_object.name)
+                self.log.debug("Injecting require for"
+                               " CDIST_ORDER_DEPENDENCY: %s for %s",
+                               lastcreatedtype, self.cdist_object.name)
             except IndexError:
                 # if no second last line, we are on the first type,
                 # so do not set a requirement
@@ -391,7 +388,7 @@ class Emulator:
 
         if "require" in self.env:
             requirements = self.env['require']
-            self.log.debug("reqs = " + requirements)
+            self.log.debug("reqs = %s", requirements)
             for requirement in self._parse_require(requirements):
                 # Ignore empty fields - probably the only field anyway
                 if len(requirement) == 0:
diff --git a/cdist/exec/local.py b/cdist/exec/local.py
index e0aab190..6713cd13 100644
--- a/cdist/exec/local.py
+++ b/cdist/exec/local.py
@@ -154,8 +154,8 @@ class Local:
         with open(self.object_marker_file, 'w') as fd:
             fd.write("%s\n" % self.object_marker_name)
 
-        self.log.trace("Object marker %s saved in %s" % (
-            self.object_marker_name, self.object_marker_file))
+        self.log.trace("Object marker %s saved in %s",
+                       self.object_marker_name, self.object_marker_file)
 
     def _init_cache_dir(self, cache_dir):
         home_dir = cdist.home_dir()
@@ -289,14 +289,12 @@ class Local:
         return cache_subpath
 
     def save_cache(self, start_time=time.time()):
-        self.log.trace("cache subpath pattern: {}".format(
-            self.cache_path_pattern))
+        self.log.trace("cache subpath pattern: %s", self.cache_path_pattern)
         cache_subpath = self._cache_subpath(start_time,
                                             self.cache_path_pattern)
-        self.log.debug("cache subpath: {}".format(cache_subpath))
+        self.log.debug("cache subpath: %s", cache_subpath)
         destination = os.path.join(self.cache_path, cache_subpath)
-        self.log.trace(("Saving cache: " + self.base_path + " to " +
-                        destination))
+        self.log.trace("Saving cache %s to %s", self.base_path, destination)
 
         if not os.path.exists(destination):
             shutil.move(self.base_path, destination)
@@ -332,7 +330,7 @@ class Local:
 
         # Iterate over all directories and link the to the output dir
         for conf_dir in self.conf_dirs:
-            self.log.debug("Checking conf_dir %s ..." % (conf_dir))
+            self.log.debug("Checking conf_dir %s ...", conf_dir)
             for sub_dir in CONF_SUBDIRS_LINKED:
                 current_dir = os.path.join(conf_dir, sub_dir)
 
@@ -350,7 +348,7 @@ class Local:
                     if os.path.exists(dst):
                         os.unlink(dst)
 
-                    self.log.trace("Linking %s to %s ..." % (src, dst))
+                    self.log.trace("Linking %s to %s ...", src, dst)
                     try:
                         os.symlink(src, dst)
                     except OSError as e:
diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py
index e5af2f34..ea85da4c 100644
--- a/cdist/exec/remote.py
+++ b/cdist/exec/remote.py
@@ -176,19 +176,19 @@ class Remote:
                 # create archive
                 tarpath, fcnt = autil.tar(source, self.archiving_mode)
                 if tarpath is None:
-                    self.log.trace(("Files count {} is lower than {} limit, "
-                                    "skipping archiving").format(
-                                        fcnt, autil.FILES_LIMIT))
+                    self.log.trace("Files count %d is lower than %d limit, "
+                                   "skipping archiving",
+                                   fcnt, autil.FILES_LIMIT)
                 else:
-                    self.log.trace(("Archiving mode, tarpath: %s, file count: "
-                                    "%s"), tarpath, fcnt)
+                    self.log.trace("Archiving mode, tarpath: %s, file count: "
+                                   "%s", tarpath, fcnt)
                     # get archive name
                     tarname = os.path.basename(tarpath)
                     self.log.trace("Archiving mode tarname: %s", tarname)
                     # archive path at the remote
                     desttarpath = os.path.join(destination, tarname)
-                    self.log.trace(
-                        "Archiving mode desttarpath: %s", desttarpath)
+                    self.log.trace("Archiving mode desttarpath: %s",
+                                   desttarpath)
                     # transfer archive to the remote side
                     self.log.trace("Archiving mode: transferring")
                     self._transfer_file(tarpath, desttarpath)
diff --git a/cdist/install.py b/cdist/install.py
index 7c894fe5..d077baef 100644
--- a/cdist/install.py
+++ b/cdist/install.py
@@ -47,4 +47,4 @@ class Install(cdist.config.Config):
                 yield cdist_object
             else:
                 self.log.debug("Running in install mode, ignoring non install"
-                               "object: {0}".format(cdist_object))
+                               "object: %s", cdist_object)
diff --git a/cdist/inventory.py b/cdist/inventory.py
index 0387f326..106052a2 100644
--- a/cdist/inventory.py
+++ b/cdist/inventory.py
@@ -92,7 +92,7 @@ class Inventory:
         self.init_db()
 
     def init_db(self):
-        self.log.trace("Init db: {}".format(self.db_basedir))
+        self.log.trace("Init db: %s", self.db_basedir)
         if not os.path.exists(self.db_basedir):
             os.makedirs(self.db_basedir, exist_ok=True)
         elif not os.path.isdir(self.db_basedir):
@@ -182,9 +182,9 @@ class Inventory:
         configuration = cfg.get_config(section='GLOBAL')
         determine_default_inventory_dir(args, configuration)
 
-        log.debug("Using inventory: {}".format(args.inventory_dir))
-        log.trace("Inventory args: {}".format(vars(args)))
-        log.trace("Inventory command: {}".format(args.subcommand))
+        log.debug("Using inventory: %s", args.inventory_dir)
+        log.trace("Inventory args: %s", vars(args))
+        log.trace("Inventory command: %s", args.subcommand)
 
         if args.subcommand == "list":
             c = InventoryList(hosts=args.host, istag=args.tag,
@@ -237,16 +237,16 @@ class InventoryList(Inventory):
     def _do_list(self, it_tags, it_hosts, check_func):
         if (it_tags is not None):
             param_tags = set(it_tags)
-            self.log.trace("param_tags: {}".format(param_tags))
+            self.log.trace("param_tags: %s", param_tags)
         else:
             param_tags = set()
         for host in it_hosts:
-            self.log.trace("host: {}".format(host))
+            self.log.trace("host: %s", host)
             tags = self._get_host_tags(host)
             if tags is None:
-                self.log.debug("Host \'{}\' not found, skipped".format(host))
+                self.log.debug("Host \'%s\' not found, skipped", host)
                 continue
-            self.log.trace("tags: {}".format(tags))
+            self.log.trace("tags: %s", tags)
             if check_func(tags, param_tags):
                 yield host, tags
 
@@ -308,11 +308,11 @@ class InventoryHost(Inventory):
 
     def _action(self, host):
         if self.action == "add":
-            self.log.debug("Adding host \'{}\'".format(host))
+            self.log.debug("Adding host \'%s\'", host)
         elif self.action == "del":
-            self.log.debug("Deleting host \'{}\'".format(host))
+            self.log.debug("Deleting host \'%s\'", host)
         hostpath = self._host_path(host)
-        self.log.trace("hostpath: {}".format(hostpath))
+        self.log.trace("hostpath: %s", hostpath)
         if self.action == "add" and not os.path.exists(hostpath):
             self._new_hostpath(hostpath)
         else:
@@ -372,23 +372,23 @@ class InventoryTag(Inventory):
             print("Host \'{}\' does not exist, skipping".format(host),
                   file=sys.stderr)
             return
-        self.log.trace("existing host_tags: {}".format(host_tags))
+        self.log.trace("existing host_tags: %s", host_tags)
         if self.action == "del" and self.all:
             host_tags = set()
         else:
             for tag in self.input_tags:
                 if self.action == "add":
-                    self.log.debug("Adding tag \'{}\' for host \'{}\'".format(
-                        tag, host))
+                    self.log.debug("Adding tag \'%s\' for host \'%s\'",
+                                   tag, host)
                     host_tags.add(tag)
                 elif self.action == "del":
-                    self.log.debug("Deleting tag \'{}\' for host "
-                                   "\'{}\'".format(tag, host))
+                    self.log.debug("Deleting tag \'%s\' for host \'%s\'",
+                                   tag, host)
                     if tag in host_tags:
                         host_tags.remove(tag)
-        self.log.trace("new host tags: {}".format(host_tags))
+        self.log.trace("new host tags: %s", host_tags)
         if not self._write_host_tags(host, host_tags):
-            self.log.trace("{} does not exist, skipped".format(host))
+            self.log.trace("%s does not exist, skipped", host)
 
     def run(self):
         if self.allhosts:
diff --git a/cdist/preos.py b/cdist/preos.py
index f8a5dd67..45711a41 100644
--- a/cdist/preos.py
+++ b/cdist/preos.py
@@ -49,7 +49,7 @@ def scan_preos_dir_plugins(dir):
                 c = cm[1]
                 yield from preos_plugin(c)
         except ImportError as e:
-            log.warning("Cannot import '{}': {}".format(module_name, e))
+            log.warning("Cannot import '%s': %s", module_name, e)
 
 
 def find_preos_plugins():
@@ -102,7 +102,7 @@ class PreOS:
         parser.add_argument('remainder_args', nargs=argparse.REMAINDER)
         args = parser.parse_args(argv[1:])
         cdist.argparse.handle_loglevel(args)
-        log.debug("preos args : {}".format(args))
+        log.debug("preos args : %s", args)
 
         conf_dirs = util.resolve_conf_dirs_from_config_and_args(args)
 
@@ -122,7 +122,7 @@ class PreOS:
                 func_args = [preos, args.remainder_args, ]
             else:
                 func_args = [args.remainder_args, ]
-            log.info("Running preos : {}".format(preos_name))
+            log.info("Running preos : %s", preos_name)
             func(*func_args)
         else:
             raise cdist.Error(
diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py
index a20cdb9c..f1f750ee 100644
--- a/cdist/preos/debootstrap/debootstrap.py
+++ b/cdist/preos/debootstrap/debootstrap.py
@@ -166,7 +166,7 @@ class Debian:
             args.pxe_boot_dir = os.path.realpath(args.pxe_boot_dir)
 
         cdist.argparse.handle_loglevel(args)
-        log.debug("preos: {}, args: {}".format(cls._preos_name, args))
+        log.debug("preos: %s, args: %s", cls._preos_name, args)
         try:
             env = vars(args)
             new_env = {}
@@ -190,27 +190,30 @@ class Debian:
             env = new_env
             env.update(os.environ)
             cls.update_env(env)
-            log.debug("preos: {} env: {}".format(cls._preos_name, env))
+            log.debug("preos: %s env: %s", cls._preos_name, env)
+
+            if log.getEffectiveLevel() <= logging.INFO:
+                info_msg = ["Running preos: {}, suite: {}, arch: {}".format(
+                    cls._preos_name, args.suite, args.arch), ]
+                if args.mirror:
+                    info_msg.append("mirror: {}".format(args.mirror))
+                if args.script:
+                    info_msg.append("script: {}".format(args.script))
+                if args.bootstrap:
+                    info_msg.append("bootstrapping")
+                if args.configure:
+                    info_msg.append("configuring")
+                if args.pxe_boot_dir:
+                    info_msg.append("creating PXE")
+                if args.drive:
+                    info_msg.append("creating bootable drive")
+                log.info(info_msg)
+
             cmd = os.path.join(cls._files_dir, "code")
-            info_msg = ["Running preos: {}, suite: {}, arch: {}".format(
-                cls._preos_name, args.suite, args.arch), ]
-            if args.mirror:
-                info_msg.append("mirror: {}".format(args.mirror))
-            if args.script:
-                info_msg.append("script: {}".format(args.script))
-            if args.bootstrap:
-                info_msg.append("bootstrapping")
-            if args.configure:
-                info_msg.append("configuring")
-            if args.pxe_boot_dir:
-                info_msg.append("creating PXE")
-            if args.drive:
-                info_msg.append("creating bootable drive")
-            log.info(info_msg)
-            log.debug("cmd={}".format(cmd))
+            log.debug("cmd=%s", cmd)
             subprocess.check_call(cmd, env=env, shell=True)
         except subprocess.CalledProcessError as e:
-            log.error("preos {} failed: {}".format(cls._preos_name, e))
+            log.error("preos %s failed: %s", cls._preos_name, e)
 
 
 class Ubuntu(Debian):
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index b1d0e9e1..9a0bcc52 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -94,7 +94,7 @@ class Trigger(object):
 
     def trigger(self, interface):
         packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
-        log.debug(f"Sending request on {interface}")
+        log.debug("Sending request on %s", interface)
         send(packet, verbose=self.verbose)
 
 
@@ -114,7 +114,7 @@ class Scanner(object):
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
             host = pkg['IPv6'].src
-            log.verbose(f"Host {host} is alive")
+            log.verbose("Host %s is alive", host)
 
             dir = os.path.join(self.outdir, host)
             fname = os.path.join(dir, "last_seen")
diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py
index 95ca74ee..d9e5f498 100644
--- a/cdist/util/ipaddr.py
+++ b/cdist/util/ipaddr.py
@@ -42,8 +42,7 @@ def resolve_target_host_name(host, family=0):
         # gethostbyaddr returns triple
         # (hostname, aliaslist, ipaddrlist)
         host_name = socket.gethostbyaddr(ip_addr)[0]
-        log.debug("derived host_name for host \"{}\": {}".format(
-            host, host_name))
+        log.debug("derived host_name for host \"%s\": %s", host, host_name)
     except (socket.gaierror, socket.herror) as e:
         # in case of error provide empty value
         host_name = ''
@@ -54,8 +53,7 @@ def resolve_target_fqdn(host):
     log = logging.getLogger(host)
     try:
         host_fqdn = socket.getfqdn(host)
-        log.debug("derived host_fqdn for host \"{}\": {}".format(
-            host, host_fqdn))
+        log.debug("derived host_fqdn for host \"%s\": %s", host, host_fqdn)
     except socket.herror as e:
         # in case of error provide empty value
         host_fqdn = ''

From 4c2d273f07e6cedb3dcb539d132cb0c3f8176d42 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 30 Mar 2021 07:56:38 +0200
Subject: [PATCH 588/743] Unify string formatting

Use one way of string formatting: replace old `%` style with new `str.format`.

Resolve #855.
---
 cdist/config.py                     |  4 ++--
 cdist/core/cdist_object.py          | 14 +++++++-------
 cdist/core/cdist_type.py            |  4 ++--
 cdist/core/code.py                  |  8 ++++----
 cdist/core/explorer.py              |  6 +++---
 cdist/core/manifest.py              | 11 +++++------
 cdist/emulator.py                   | 12 ++++++------
 cdist/exec/local.py                 | 11 ++++++-----
 cdist/exec/remote.py                | 11 +++++------
 cdist/message.py                    |  2 +-
 cdist/scan/scan.py                  |  2 +-
 cdist/shell.py                      |  2 +-
 cdist/test/cdist_object/__init__.py | 27 +++++++++++++++------------
 cdist/test/emulator/__init__.py     |  4 ++--
 cdist/test/exec/remote.py           |  8 ++++----
 cdist/test/message/__init__.py      |  2 +-
 cdist/util/fsproperty.py            |  4 ++--
 17 files changed, 67 insertions(+), 65 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index d6fec55f..adc460de 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -190,7 +190,7 @@ class Config:
                     fd.write(sys.stdin.read())
             except (IOError, OSError) as e:
                 raise cdist.Error(("Creating tempfile for stdin data "
-                                   "failed: %s" % e))
+                                   "failed: {}").format(e))
 
             args.manifest = initial_manifest_temp_path
             atexit.register(lambda: os.remove(initial_manifest_temp_path))
@@ -764,7 +764,7 @@ class Config:
 
             raise cdist.UnresolvableRequirementsError(
                     ("The requirements of the following objects could not be "
-                     "resolved:\n%s") % ("\n".join(info_string)))
+                     "resolved:\n{}").format("\n".join(info_string)))
 
     def _handle_deprecation(self, cdist_object):
         cdist_type = cdist_object.cdist_type
diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index eea2c255..ccf7392d 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -34,17 +34,17 @@ class IllegalObjectIdError(cdist.Error):
         self.message = message or 'Illegal object id'
 
     def __str__(self):
-        return '%s: %s' % (self.message, self.object_id)
+        return '{}: {}'.format(self.message, self.object_id)
 
 
 class MissingObjectIdError(cdist.Error):
     def __init__(self, type_name):
         self.type_name = type_name
-        self.message = ("Type %s requires object id (is not a "
-                        "singleton type)") % self.type_name
+        self.message = ("Type {} requires object id (is not a "
+                        "singleton type)").format(self.type_name)
 
     def __str__(self):
-        return '%s' % (self.message)
+        return '{}'.format(self.message)
 
 
 class CdistObject:
@@ -142,7 +142,7 @@ class CdistObject:
             if self.object_marker in self.object_id.split(os.sep):
                 raise IllegalObjectIdError(
                         self.object_id, ('object_id may not contain '
-                                         '\'%s\'') % self.object_marker)
+                                         '\'{}\'').format(self.object_marker))
             if '//' in self.object_id:
                 raise IllegalObjectIdError(
                         self.object_id, 'object_id may not contain //')
@@ -189,7 +189,7 @@ class CdistObject:
                               object_id=object_id)
 
     def __repr__(self):
-        return '<CdistObject %s>' % self.name
+        return '<CdistObject {}>'.format(self.name)
 
     def __eq__(self, other):
         """define equality as 'name is the same'"""
@@ -277,7 +277,7 @@ class CdistObject:
                 os.makedirs(path, exist_ok=allow_overwrite)
         except EnvironmentError as error:
             raise cdist.Error(('Error creating directories for cdist object: '
-                               '%s: %s') % (self, error))
+                               '{}: {}').format(self, error))
 
     def requirements_unfinished(self, requirements):
         """Return state whether requirements are satisfied"""
diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py
index 274de989..b68448bc 100644
--- a/cdist/core/cdist_type.py
+++ b/cdist/core/cdist_type.py
@@ -34,7 +34,7 @@ class InvalidTypeError(cdist.Error):
         self.source_path = os.path.realpath(self.type_absolute_path)
 
     def __str__(self):
-        return "Invalid type '%s' at '%s' defined at '%s'" % (
+        return "Invalid type '{}' at '{}' defined at '{}'".format(
                 self.type_path, self.type_absolute_path, self.source_path)
 
 
@@ -109,7 +109,7 @@ class CdistType:
         return cls._instances[name]
 
     def __repr__(self):
-        return '<CdistType %s>' % self.name
+        return '<CdistType {}>'.format(self.name)
 
     def __eq__(self, other):
         return isinstance(other, self.__class__) and self.name == other.name
diff --git a/cdist/core/code.py b/cdist/core/code.py
index 1e9b4f80..12888ba4 100644
--- a/cdist/core/code.py
+++ b/cdist/core/code.py
@@ -122,8 +122,8 @@ class Code:
 
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.cdist_type
-        script = os.path.join(self.local.type_path,
-                              getattr(cdist_type, 'gencode_%s_path' % which))
+        gencode_attr = getattr(cdist_type, 'gencode_{}_path'.format(which))
+        script = os.path.join(self.local.type_path, gencode_attr)
         if os.path.isfile(script):
             env = os.environ.copy()
             env.update(self.env)
@@ -167,8 +167,8 @@ class Code:
 
     def _run_code(self, cdist_object, which, env=None):
         which_exec = getattr(self, which)
-        script = os.path.join(which_exec.object_path,
-                              getattr(cdist_object, 'code_%s_path' % which))
+        code_attr = getattr(cdist_object, 'code_{}_path'.format(which))
+        script = os.path.join(which_exec.object_path, code_attr)
         if which_exec.save_output_streams:
             stderr_path = os.path.join(cdist_object.stderr_path,
                                        'code-' + which)
diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py
index caa12a7d..48414ade 100644
--- a/cdist/core/explorer.py
+++ b/cdist/core/explorer.py
@@ -160,8 +160,8 @@ class Explorer:
         self.remote.transfer(self.local.global_explorer_path,
                              self.remote.global_explorer_path,
                              self.jobs)
-        self.remote.run(["chmod", "0700",
-                         "%s/*" % (self.remote.global_explorer_path)])
+        self.remote.run(["chmod", "0700", "{}/*".format(
+            self.remote.global_explorer_path)])
 
     def run_global_explorer(self, explorer):
         """Run the given global explorer and return it's output."""
@@ -242,7 +242,7 @@ class Explorer:
                 destination = os.path.join(self.remote.type_path,
                                            cdist_type.explorer_path)
                 self.remote.transfer(source, destination)
-                self.remote.run(["chmod", "0700", "%s/*" % (destination)])
+                self.remote.run(["chmod", "0700", "{}/*".format(destination)])
                 self._type_explorers_transferred.append(cdist_type.name)
 
     def transfer_object_parameters(self, cdist_object):
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 3148d66c..09e74dac 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -80,13 +80,12 @@ class NoInitialManifestError(cdist.Error):
 
         if user_supplied:
             if os.path.islink(manifest_path):
-                self.message = "%s: %s -> %s" % (
-                        msg_header, manifest_path,
-                        os.path.realpath(manifest_path))
+                self.message = "{}: {} -> {}".format(
+                    msg_header, manifest_path, os.path.realpath(manifest_path))
             else:
-                self.message = "%s: %s" % (msg_header, manifest_path)
+                self.message = "{}: {}".format(msg_header, manifest_path)
         else:
-            self.message = "%s" % (msg_header)
+            self.message = "{}".format(msg_header)
 
     def __str__(self):
         return repr(self.message)
@@ -107,7 +106,7 @@ class Manifest:
         self._open_logger()
 
         self.env = {
-            'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
+            'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']),
             # for use in type emulator
             '__cdist_type_base_path': self.local.type_path,
             '__global': self.local.base_path,
diff --git a/cdist/emulator.py b/cdist/emulator.py
index f09c282d..c55b47d2 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -36,8 +36,8 @@ from cdist.core.manifest import Manifest
 class MissingRequiredEnvironmentVariableError(cdist.Error):
     def __init__(self, name):
         self.name = name
-        self.message = ("Emulator requires the environment variable %s to be "
-                        "setup" % self.name)
+        self.message = ("Emulator requires the environment variable {} to be "
+                        "setup").format(self.name)
 
     def __str__(self):
         return self.message
@@ -231,13 +231,13 @@ class Emulator:
         if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env:
             obj_params = self._object_params_in_context()
             if obj_params != self.parameters:
-                errmsg = ("Object %s already exists with conflicting "
-                          "parameters:\n%s: %s\n%s: %s" % (
+                errmsg = ("Object {} already exists with conflicting "
+                          "parameters:\n{}: {}\n{}: {}").format(
                               self.cdist_object.name,
                               " ".join(self.cdist_object.source),
                               obj_params,
                               self.object_source,
-                              self.parameters))
+                              self.parameters)
                 raise cdist.Error(errmsg)
         else:
             if self.cdist_object.exists:
@@ -292,7 +292,7 @@ class Emulator:
                         fd.write(chunk)
                         chunk = self._read_stdin()
             except EnvironmentError as e:
-                raise cdist.Error('Failed to read from stdin: %s' % e)
+                raise cdist.Error('Failed to read from stdin: {}'.format(e))
 
     def record_requirement(self, requirement):
         """record requirement and return recorded requirement"""
diff --git a/cdist/exec/local.py b/cdist/exec/local.py
index 6713cd13..370336ad 100644
--- a/cdist/exec/local.py
+++ b/cdist/exec/local.py
@@ -152,7 +152,7 @@ class Local:
 
     def _setup_object_marker_file(self):
         with open(self.object_marker_file, 'w') as fd:
-            fd.write("%s\n" % self.object_marker_name)
+            fd.write("{}\n".format(self.object_marker_name))
 
         self.log.trace("Object marker %s saved in %s",
                        self.object_marker_name, self.object_marker_file)
@@ -184,7 +184,7 @@ class Local:
 
         """
         assert isinstance(command, (list, tuple)), (
-                "list or tuple argument expected, got: %s" % command)
+                "list or tuple argument expected, got: {}".format(command))
 
         quiet = self.quiet_mode or quiet_mode
         do_save_output = save_output and not quiet and self.save_output_streams
@@ -352,8 +352,9 @@ class Local:
                     try:
                         os.symlink(src, dst)
                     except OSError as e:
-                        raise cdist.Error("Linking %s %s to %s failed: %s" % (
-                            sub_dir, src, dst, e.__str__()))
+                        raise cdist.Error(
+                            "Linking {} {} to {} failed: {}".format(
+                                sub_dir, src, dst, e.__str__()))
 
     def _link_types_for_emulator(self):
         """Link emulator to types"""
@@ -366,5 +367,5 @@ class Local:
                 os.symlink(src, dst)
             except OSError as e:
                 raise cdist.Error(
-                        "Linking emulator from %s to %s failed: %s" % (
+                        "Linking emulator from {} to {} failed: {}".format(
                             src, dst, e.__str__()))
diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py
index ea85da4c..af9e70cb 100644
--- a/cdist/exec/remote.py
+++ b/cdist/exec/remote.py
@@ -24,12 +24,10 @@ import os
 import glob
 import subprocess
 import logging
-import multiprocessing
 
 import cdist
 import cdist.exec.util as util
 import cdist.util.ipaddr as ipaddr
-from cdist.mputil import mp_pool_run
 
 
 def _wrap_addr(addr):
@@ -262,9 +260,10 @@ class Remote:
         # remotely in e.g. csh and setting up CDIST_REMOTE_SHELL to e.g.
         # /bin/csh will execute this script in the right way.
         if env:
-            remote_env = [" export %s=%s;" % item for item in env.items()]
-            string_cmd = ("/bin/sh -c '" + " ".join(remote_env) +
-                          " ".join(command) + "'")
+            remote_env = [" export {env[0]}={env[1]};".format(env=item)
+                          for item in env.items()]
+            string_cmd = ("/bin/sh -c '{}{}'").format(" ".join(remote_env),
+                                                      " ".join(command))
             cmd.append(string_cmd)
         else:
             cmd.extend(command)
@@ -278,7 +277,7 @@ class Remote:
 
         """
         assert isinstance(command, (list, tuple)), (
-                "list or tuple argument expected, got: %s" % command)
+                "list or tuple argument expected, got: {}".format(command))
 
         close_stdout = False
         close_stderr = False
diff --git a/cdist/message.py b/cdist/message.py
index ffa8c2bb..0c4e21a6 100644
--- a/cdist/message.py
+++ b/cdist/message.py
@@ -70,7 +70,7 @@ class Message:
 
         with open(self.global_messages, 'a') as fd:
             for line in content:
-                fd.write("%s:%s" % (self.prefix, line))
+                fd.write("{}:{}".format(self.prefix, line))
 
     def merge_messages(self):
         self._merge_messages()
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 9a0bcc52..faee8a56 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -93,7 +93,7 @@ class Trigger(object):
             time.sleep(self.sleeptime)
 
     def trigger(self, interface):
-        packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
+        packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest()
         log.debug("Sending request on %s", interface)
         send(packet, verbose=self.verbose)
 
diff --git a/cdist/shell.py b/cdist/shell.py
index 04a68937..05803556 100644
--- a/cdist/shell.py
+++ b/cdist/shell.py
@@ -65,7 +65,7 @@ class Shell:
     def _init_environment(self):
         self.env = os.environ.copy()
         additional_env = {
-            'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
+            'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']),
             # for use in type emulator
             '__cdist_type_base_path': self.local.type_path,
             '__cdist_manifest': "cdist shell",
diff --git a/cdist/test/cdist_object/__init__.py b/cdist/test/cdist_object/__init__.py
index a9c20cd3..11619e96 100644
--- a/cdist/test/cdist_object/__init__.py
+++ b/cdist/test/cdist_object/__init__.py
@@ -86,8 +86,7 @@ class ObjectClassTestCase(test.CdistTestCase):
 
     def test_create_singleton(self):
         """Check whether creating an object without id (singleton) works"""
-        singleton = self.expected_objects[0].object_from_name(
-                "__test_singleton")
+        self.expected_objects[0].object_from_name("__test_singleton")
         # came here - everything fine
 
     def test_create_singleton_not_singleton_type(self):
@@ -126,16 +125,16 @@ class ObjectIdTestCase(test.CdistTestCase):
 
     def test_object_id_contains_object_marker(self):
         cdist_type = core.CdistType(type_base_path, '__third')
-        illegal_object_id = (
-            'object_id/may/not/contain/%s/anywhere' % OBJECT_MARKER_NAME)
+        illegal_object_id = 'object_id/may/not/contain/{}/anywhere'.format(
+            OBJECT_MARKER_NAME)
         with self.assertRaises(core.IllegalObjectIdError):
             core.CdistObject(cdist_type, self.object_base_path,
                              OBJECT_MARKER_NAME, illegal_object_id)
 
     def test_object_id_contains_object_marker_string(self):
         cdist_type = core.CdistType(type_base_path, '__third')
-        illegal_object_id = (
-            'object_id/may/contain_%s_in_filename' % OBJECT_MARKER_NAME)
+        illegal_object_id = 'object_id/may/contain_{}_in_filename'.format(
+            OBJECT_MARKER_NAME)
         core.CdistObject(cdist_type, self.object_base_path,
                          OBJECT_MARKER_NAME, illegal_object_id)
         # if we get here, the test passed
@@ -195,28 +194,32 @@ class ObjectTestCase(test.CdistTestCase):
 
     def test_path(self):
         self.assertEqual(self.cdist_object.path,
-                         "__third/moon/%s" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}".format(OBJECT_MARKER_NAME))
 
     def test_absolute_path(self):
         self.assertEqual(self.cdist_object.absolute_path,
                          os.path.join(self.object_base_path,
-                                      "__third/moon/%s" % OBJECT_MARKER_NAME))
+                                      "__third/moon/{}".format(
+                                          OBJECT_MARKER_NAME)))
 
     def test_code_local_path(self):
         self.assertEqual(self.cdist_object.code_local_path,
-                         "__third/moon/%s/code-local" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/code-local".format(
+                             OBJECT_MARKER_NAME))
 
     def test_code_remote_path(self):
         self.assertEqual(self.cdist_object.code_remote_path,
-                         "__third/moon/%s/code-remote" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/code-remote".format(
+                             OBJECT_MARKER_NAME))
 
     def test_parameter_path(self):
         self.assertEqual(self.cdist_object.parameter_path,
-                         "__third/moon/%s/parameter" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/parameter".format(
+                             OBJECT_MARKER_NAME))
 
     def test_explorer_path(self):
         self.assertEqual(self.cdist_object.explorer_path,
-                         "__third/moon/%s/explorer" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/explorer".format(OBJECT_MARKER_NAME))
 
     def test_parameters(self):
         expected_parameters = {'planet': 'Saturn', 'name': 'Prometheus'}
diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py
index befd7b57..4b2bc3ba 100644
--- a/cdist/test/emulator/__init__.py
+++ b/cdist/test/emulator/__init__.py
@@ -84,8 +84,8 @@ class EmulatorTestCase(test.CdistTestCase):
 
     def test_illegal_object_id_requirement(self):
         argv = ['__file', '/tmp/foobar']
-        self.env['require'] = (
-                "__file/bad/id/with/%s/inside") % self.local.object_marker_name
+        self.env['require'] = "__file/bad/id/with/{}/inside".format(
+            self.local.object_marker_name)
         emu = emulator.Emulator(argv, env=self.env)
         self.assertRaises(core.IllegalObjectIdError, emu.run)
 
diff --git a/cdist/test/exec/remote.py b/cdist/test/exec/remote.py
index a7fe384d..b23f8447 100644
--- a/cdist/test/exec/remote.py
+++ b/cdist/test/exec/remote.py
@@ -47,8 +47,8 @@ class RemoteTestCase(test.CdistTestCase):
             args = (self.target_host,)
         kwargs.setdefault('base_path', self.base_path)
         user = getpass.getuser()
-        kwargs.setdefault('remote_exec', 'ssh -o User=%s -q' % user)
-        kwargs.setdefault('remote_copy', 'scp -o User=%s -q' % user)
+        kwargs.setdefault('remote_exec', 'ssh -o User={} -q'.format(user))
+        kwargs.setdefault('remote_copy', 'scp -o User={} -q'.format(user))
         if 'stdout_base_path' not in kwargs:
             stdout_path = os.path.join(self.temp_dir, 'stdout')
             os.makedirs(stdout_path, exist_ok=True)
@@ -170,7 +170,7 @@ class RemoteTestCase(test.CdistTestCase):
         r = self.create_remote(remote_exec=remote_exec,
                                remote_copy=remote_copy)
         self.assertEqual(r.run('true', return_output=True),
-                         "%s\n" % self.target_host[0])
+                         "{}\n".format(self.target_host[0]))
 
     def test_run_script_target_host_in_env(self):
         handle, remote_exec_path = self.mkstemp(dir=self.temp_dir)
@@ -185,7 +185,7 @@ class RemoteTestCase(test.CdistTestCase):
         with os.fdopen(handle, "w") as fd:
             fd.writelines(["#!/bin/sh\n", "true"])
         self.assertEqual(r.run_script(script, return_output=True),
-                         "%s\n" % self.target_host[0])
+                         "{}\n".format(self.target_host[0]))
 
     def test_run_script_with_env_target_host_in_env(self):
         handle, script = self.mkstemp(dir=self.temp_dir)
diff --git a/cdist/test/message/__init__.py b/cdist/test/message/__init__.py
index 61cd5d97..55040fc9 100644
--- a/cdist/test/message/__init__.py
+++ b/cdist/test/message/__init__.py
@@ -67,7 +67,7 @@ class MessageTestCase(test.CdistTestCase):
     def test_message_merge_prefix(self):
         """Ensure messages are merged and are prefixed"""
 
-        expectedcontent = "%s:%s" % (self.prefix, self.content)
+        expectedcontent = "{}:{}".format(self.prefix, self.content)
 
         out = self.message.env['__messages_out']
 
diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py
index 1d76fd76..09e9cc19 100644
--- a/cdist/util/fsproperty.py
+++ b/cdist/util/fsproperty.py
@@ -30,7 +30,7 @@ class AbsolutePathRequiredError(cdist.Error):
         self.path = path
 
     def __str__(self):
-        return 'Absolute path required, got: %s' % self.path
+        return 'Absolute path required, got: {}'.format(self.path)
 
 
 class FileList(collections.MutableSequence):
@@ -218,7 +218,7 @@ class FileBasedProperty:
 
     def _get_attribute(self, instance, owner):
         name = self._get_property_name(owner)
-        attribute_name = '__%s' % name
+        attribute_name = '__{}'.format(name)
         if not hasattr(instance, attribute_name):
             path = self._get_path(instance)
             attribute_instance = self.attribute_class(path)

From ab811ad282882558a4639986e75c800e1b63c2e0 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 1 Apr 2021 15:37:11 +0200
Subject: [PATCH 589/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 03ae7566..db60c0c4 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,8 @@ next:
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
 	* Type __sshd_config: Whitelist OpenBMC (Dennis Camera)
 	* Core: Maintain object relationship graph in cdist cache (Darko Poljak)
+	* Type __git: Fix numeric owner and group handling (Dennis Camera)
+	* Type __pyvenv: Fix numeric owner and group handling (Dennis Camera)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From 199effb7ef23980bc69f0233ff6d7ee465623f40 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 6 Apr 2021 19:35:14 +0200
Subject: [PATCH 590/743] Improve unfinished object requirements bool check

When we need only boolean value for unfinished object requirements then
we don't need to determine the whole list of unfinished objects.
---
 cdist/config.py            | 12 +++++++-----
 cdist/core/cdist_object.py | 13 ++++++++++++-
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index adc460de..f7da9b1d 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -537,7 +537,7 @@ class Config:
         objects_changed = False
 
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(
+            if cdist_object.has_requirements_unfinished(
                     cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
@@ -548,7 +548,7 @@ class Config:
                 self.object_prepare(cdist_object)
                 objects_changed = True
 
-            if cdist_object.requirements_unfinished(
+            if cdist_object.has_requirements_unfinished(
                     cdist_object.autorequire):
                 """The previous step created objects we depend on -
                     wait for them
@@ -567,7 +567,8 @@ class Config:
 
         cargo = []
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(cdist_object.requirements):
+            if cdist_object.has_requirements_unfinished(
+                    cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
 
@@ -621,12 +622,13 @@ class Config:
 
         del cargo[:]
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(cdist_object.requirements):
+            if cdist_object.has_requirements_unfinished(
+                    cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
 
             if cdist_object.state == core.CdistObject.STATE_PREPARED:
-                if cdist_object.requirements_unfinished(
+                if cdist_object.has_requirements_unfinished(
                         cdist_object.autorequire):
                     """The previous step created objects we depend on -
                     wait for them
diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index ccf7392d..bb3a65bd 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -280,7 +280,7 @@ class CdistObject:
                                '{}: {}').format(self, error))
 
     def requirements_unfinished(self, requirements):
-        """Return state whether requirements are satisfied"""
+        """Return unsatisfied requirements"""
 
         object_list = []
 
@@ -291,3 +291,14 @@ class CdistObject:
                 object_list.append(cdist_object)
 
         return object_list
+
+    def has_requirements_unfinished(self, requirements):
+        """Return whether requirements are satisfied"""
+
+        for requirement in requirements:
+            cdist_object = self.object_from_name(requirement)
+
+            if cdist_object.state != self.STATE_DONE:
+                return True
+
+        return False

From 750c71fb5a9fcf4a30af0700b7a09e3501988a27 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 7 Apr 2021 09:07:29 +0200
Subject: [PATCH 591/743] Minor refactoring and remove code duplication

---
 cdist/config.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index f7da9b1d..638fdf0e 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -699,20 +699,22 @@ class Config:
             check for cycles.
         '''
         graph = {}
-        for cdist_object in self.object_list():
+
+        def _add_requirements(cdist_object, requirements):
             obj_name = cdist_object.name
             if obj_name not in graph:
                 graph[obj_name] = []
+
+            for requirement in cdist_object.requirements_unfinished(
+                    requirements):
+                graph[obj_name].append(requirement.name)
+
+        for cdist_object in self.object_list():
             if cdist_object.state == cdist_object.STATE_DONE:
                 continue
 
-            for requirement in cdist_object.requirements_unfinished(
-                    cdist_object.requirements):
-                graph[obj_name].append(requirement.name)
-
-            for requirement in cdist_object.requirements_unfinished(
-                    cdist_object.autorequire):
-                graph[obj_name].append(requirement.name)
+            _add_requirements(cdist_object, cdist_object.requirements)
+            _add_requirements(cdist_object, cdist_object.autorequire)
         return graph_check_cycle(graph)
 
     def iterate_until_finished(self):

From d2eec6066876eeb4a295b81a670c70b0ff8ad2a7 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 11 Apr 2021 23:05:48 +0300
Subject: [PATCH 592/743] [__download] make --sum optional

---
 cdist/conf/type/__download/explorer/state     |  6 ++++++
 cdist/conf/type/__download/man.rst            | 15 ++++++---------
 cdist/conf/type/__download/parameter/optional |  1 +
 cdist/conf/type/__download/parameter/required |  1 -
 4 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 00362545..68b517c5 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -8,6 +8,12 @@ then
     exit 0
 fi
 
+if [ ! -f "$__object/parameter/sum" ]
+then
+    echo 'present'
+    exit 0
+fi
+
 sum_should="$( cat "$__object/parameter/sum" )"
 
 if [ -f "$__object/parameter/cmd-sum" ]
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index 54503470..a1278cfb 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -8,9 +8,6 @@ cdist-type__download - Download a file
 
 DESCRIPTION
 -----------
-Destination (``$__object_id``) in target host must be persistent storage
-in order to calculate checksum and decide if file must be (re-)downloaded.
-
 By default type will try to use ``wget``, ``curl`` or ``fetch``.
 If download happens in target (see ``--download``) then type will
 fallback to (and install) ``wget``.
@@ -25,14 +22,14 @@ REQUIRED PARAMETERS
 url
    File's URL.
 
-sum
-   Checksum of file going to be downloaded.
-   By default output of ``cksum`` without filename is expected.
-   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
-
 
 OPTIONAL PARAMETERS
 -------------------
+sum
+   Checksum is used to decide if existing destination file must be redownloaded.
+   By default output of ``cksum`` without filename is expected.
+   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
+
 download
    If ``local`` (default), then download file to local storage and copy
    it to target host. If ``remote``, then download happens in target.
@@ -81,7 +78,7 @@ Ander Punnar <ander-at-kvlt-dot-ee>
 
 COPYING
 -------
-Copyright \(C) 2020 Ander Punnar. You can redistribute it
+Copyright \(C) 2021 Ander Punnar. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index 838e2fbf..d69e083e 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,3 +1,4 @@
+sum
 cmd-get
 cmd-sum
 download
diff --git a/cdist/conf/type/__download/parameter/required b/cdist/conf/type/__download/parameter/required
index 6ea4c38f..96cdd3b9 100644
--- a/cdist/conf/type/__download/parameter/required
+++ b/cdist/conf/type/__download/parameter/required
@@ -1,2 +1 @@
 url
-sum

From 9ec01d9f971339ee67c866c4a248047966f73ce1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 13 Apr 2021 12:22:45 +0200
Subject: [PATCH 593/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index db60c0c4..c8cb44da 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,7 @@ next:
 	* Core: Maintain object relationship graph in cdist cache (Darko Poljak)
 	* Type __git: Fix numeric owner and group handling (Dennis Camera)
 	* Type __pyvenv: Fix numeric owner and group handling (Dennis Camera)
+	* Type __download: Make sum parameter optional (Ander Punnar)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From 92b8942a8c37bb985ef42991f3b7aaef0cd8073f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 09:00:32 +0200
Subject: [PATCH 594/743] [type/__postgres_conf] Add psql_exec function to
 state explorer

---
 cdist/conf/type/__postgres_conf/explorer/state | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index de6e8aa0..1a58751a 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -20,21 +20,24 @@
 #
 
 postgres_user=$("${__type_explorer:?}/postgres_user")
-
 conf_name=${__object_id:?}
 
-su - "${postgres_user}" -c 'psql postgres -c "SELECT 1"' >/dev/null || {
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_exec() {
+	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
+}
+
+psql_exec 'SELECT 1' >/dev/null || {
 	echo 'Connection to PostgreSQL server failed' >&2
 	exit 1
 }
 
-if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
-	| cmp -s "${__object:?}/parameter/value" -
+if psql_exec "SHOW ${conf_name}" | cmp -s "${__object:?}/parameter/value" -
 then
 	echo present
 else
 	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
-	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"")
+	case $(psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')")
 	in
 		('')
 			# invalid configuration parameter

From 2ccc03fef1e28c759a7918fa00ffcb45c76dc57a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 09:02:40 +0200
Subject: [PATCH 595/743] [type/__postgres_conf] Add psql_conf_cmp function to
 state explorer

---
 cdist/conf/type/__postgres_conf/explorer/state | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 1a58751a..be881be6 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -27,12 +27,16 @@ psql_exec() {
 	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
 }
 
+psql_conf_cmp() {
+	test "$(psql_exec "SHOW $1")" = "$2"
+}
+
 psql_exec 'SELECT 1' >/dev/null || {
 	echo 'Connection to PostgreSQL server failed' >&2
 	exit 1
 }
 
-if psql_exec "SHOW ${conf_name}" | cmp -s "${__object:?}/parameter/value" -
+if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
 then
 	echo present
 else

From e0416403c4862cb8bc0f41a7c6f8a33d1a8656a5 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 09:04:07 +0200
Subject: [PATCH 596/743] [type/__postgres_conf] Add psql_conf_source function
 to state explorer

---
 cdist/conf/type/__postgres_conf/explorer/state | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index be881be6..9f54724a 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -27,6 +27,10 @@ psql_exec() {
 	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
 }
 
+psql_conf_source() {
+	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
+	psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')"
+}
 psql_conf_cmp() {
 	test "$(psql_exec "SHOW $1")" = "$2"
 }
@@ -40,8 +44,7 @@ if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
 then
 	echo present
 else
-	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
-	case $(psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')")
+	case $(psql_conf_source "${conf_name}")
 	in
 		('')
 			# invalid configuration parameter

From 12c2995494ce582cb3adf1d942b36f6df035370a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 14:46:36 +0200
Subject: [PATCH 597/743] [type/__postgres_conf] Implement complex state
 compare logic

---
 .../conf/type/__postgres_conf/explorer/state  | 166 +++++++++++++++++-
 1 file changed, 163 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 9f54724a..e76540c5 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -22,6 +22,52 @@
 postgres_user=$("${__type_explorer:?}/postgres_user")
 conf_name=${__object_id:?}
 
+tolower() { printf '%s' "$*" | tr '[:upper:]' '[:lower:]'; }
+
+tobytes() {
+	# NOTE: This function treats everything as base 2.
+	#       It is not compatible with SI units.
+	awk 'BEGIN { FS = "\n" }
+	/TB$/ { $0 = ($0 * 1024) "GB" }
+	/GB$/ { $0 = ($0 * 1024) "MB" }
+	/MB$/ { $0 = ($0 * 1024) "kB" }
+	/kB$/ { $0 = ($0 * 1024) "B" }
+	 /B?$/ { sub(/ *B?$/, "") }
+	($0*1) == $0  # is number
+	' <<-EOF
+	$1
+	EOF
+}
+
+tomillisecs() {
+	awk 'BEGIN { FS = "\n" }
+	    /d$/ { $0 = ($0 * 24) "h" }
+	    /h$/ { $0 = ($0 * 60) "min" }
+	  /min$/ { $0 = ($0 * 60) "s" }
+	/[^m]s$/ { $0 = ($0 * 1000) "ms" }
+	   /ms$/ { $0 *= 1 }
+	($0*1) == $0  # is number
+	' <<-EOF
+	$1
+	EOF
+}
+
+tobool() {
+	# prints either 'on' or 'off'
+	case $(tolower "$1")
+	in
+		(t|true|y|yes|on|1)
+			echo 'on' ;;
+		(f|false|n|no|off|0)
+			echo 'off' ;;
+		(*)
+			printf 'Inavlid bool value: %s\n' "$2" >&2
+			return 1
+			;;
+	esac
+	return 0
+}
+
 quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
 psql_exec() {
 	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
@@ -31,9 +77,123 @@ psql_conf_source() {
 	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
 	psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')"
 }
-psql_conf_cmp() {
-	test "$(psql_exec "SHOW $1")" = "$2"
-}
+psql_conf_cmp() (
+	IFS='|' read -r lower_name vartype setting unit <<-EOF
+	$(psql_exec "SELECT lower(name), vartype, setting, unit FROM pg_settings WHERE lower(name) = lower('$1')")
+	EOF
+
+	should_value=$2
+	is_value=${setting}
+
+	# The following case contains special cases for special settings.
+	case ${lower_name}
+	in
+		(archive_command)
+			if test "${setting}" = '(disabled)'
+			then
+				# DAFUQ PostgreSQL?!
+				# PostgreSQL returns (disabled) if the feature is inactive.
+				# We cannot compare the values unless it is enabled, first.
+				return 0
+			fi
+			;;
+		(archive_mode|backslash_quote|constraint_exclusion|force_parallel_mode|huge_pages|synchronous_commit)
+			# Although only 'on', 'off' are documented, PostgreSQL accepts all
+			# the "likely" variants of "on" and "off".
+			case $(tolower "${should_value}")
+			in
+				(on|off|true|false|yes|no|1|0)
+					should_value=$(tobool "${should_value}")
+					;;
+			esac
+			;;
+	esac
+
+	case ${vartype}
+	in
+		(bool)
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported for vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			should_value=$(tobool "${should_value}")
+
+			test "${is_value}" = "${should_value}"
+			;;
+		(enum)
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported with vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			# NOTE: All enums that are currently defined are lower case, but
+			#       PostgreSQL also accepts upper case spelling.
+			should_value=$(tolower "$2")
+
+			test "${is_value}" = "${should_value}"
+			;;
+		(integer)
+			# split multiples from unit, first (e.g. 8kB -> 8, kB)
+			case ${unit}
+			in
+				([0-9]*)
+					multiple=${unit%%[!0-9]*}
+					unit=${unit##*[0-9 ]}
+					;;
+				(*) multiple=1 ;;
+			esac
+
+			is_value=$((setting * multiple))${unit}
+
+			if expr "${should_value}" : '-\{0,1\}[0-9]*$' >/dev/null
+			then
+				# default unit
+				should_value=$((should_value * multiple))${unit}
+			fi
+
+			# then, do conversion
+			# NOTE: these conversions work for integers only!
+			case ${unit}
+			in
+				(B|[kMGT]B)
+					# bytes
+					is_bytes=$(tobytes "${is_value}")
+					should_bytes=$(tobytes "${should_value}")
+
+					test $((is_bytes)) -eq $((should_bytes))
+					;;
+				(ms|s|min|h|d)
+					# seconds
+					is_ms=$(tomillisecs "${is_value}")
+					should_ms=$(tomillisecs "${should_value}")
+
+					test $((is_ms)) -eq $((should_ms))
+					;;
+				('')
+					# no unit
+					is_int=${is_value}
+					should_int=${should_value}
+
+					test $((is_int)) -eq $((should_int))
+					;;
+			esac
+			;;
+		(real|string)
+			# NOTE: reals could possibly have units, but currently there none.
+
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported with vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			test "${is_value}" = "${should_value}"
+			;;
+	esac
+)
 
 psql_exec 'SELECT 1' >/dev/null || {
 	echo 'Connection to PostgreSQL server failed' >&2

From bef1433ba3adb022f8b50eb9df0fbbbf90757d76 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 14:46:50 +0200
Subject: [PATCH 598/743] [type/__postgres_conf] Accept empty values

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index d0d247b4..27651600 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -57,7 +57,7 @@ psql_cmd() {
 case ${state_should}
 in
 	(present)
-		test -s "${__object:?}/parameter/value" || {
+		test -n "${__object:?}/parameter/value" || {
 			echo 'Missing required parameter --value' >&2
 			exit 1
 		}

From 686e4f0f2d498ee39a69e43f7535171f16f5fcbc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 15:01:38 +0200
Subject: [PATCH 599/743] [type/__postgres_conf] Reverse state logic (decide
 based on source first)

---
 .../conf/type/__postgres_conf/explorer/state  | 39 ++++++++++---------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index e76540c5..4b7b0a43 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -200,23 +200,24 @@ psql_exec 'SELECT 1' >/dev/null || {
 	exit 1
 }
 
-if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
-then
-	echo present
-else
-	case $(psql_conf_source "${conf_name}")
-	in
-		('')
-			# invalid configuration parameter
-			# (error message was already printed by SHOW command above.
-			#  Yes, it's a hack)
-			exit 1
-			;;
-		(default)
-			echo absent
-			;;
-		(*)
+case $(psql_conf_source "${conf_name}")
+in
+	('')
+		printf 'Invalid configuration parameter: %s\n' "${conf_name}" >&2
+		exit 1
+		;;
+	(default)
+		echo absent
+		;;
+	(*)
+		if ! test -f "${__object:?}/parameter/value"
+		then
+			echo present
+		elif psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
+		then
+			echo present
+		else
 			echo different
-			;;
-	esac
-fi
+		fi
+		;;
+esac

From 19bf37be1a8b0b099a7199d1565b74598010f614 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 15:55:52 +0200
Subject: [PATCH 600/743] [type/__postgres_conf] Update man.rst

---
 cdist/conf/type/__postgres_conf/man.rst | 39 ++++++++++++++-----------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst
index 76016c6c..e035f080 100644
--- a/cdist/conf/type/__postgres_conf/man.rst
+++ b/cdist/conf/type/__postgres_conf/man.rst
@@ -1,29 +1,27 @@
 cdist-type__postgres_conf(7)
 ============================
 
-Configure a PostgreSQL server.
-
-NOTE: This type might need to be run multiple times to apply all bits of the
-configuration due to ordering requirements.
-
-SSRQ <cdist--@--ssrq-sds-fds.ch>
+NAME
+----
+cdist-type__postgres_conf - Alter PostgreSQL configuration
 
 
 DESCRIPTION
 -----------
-Configure a PostgreSQL server using ALTER SYSTEM.
+Configure a running PostgreSQL server using ``ALTER SYSTEM``.
 
 
 REQUIRED PARAMETERS
 -------------------
 value
-    The value to setup (can be omitted when state is set to "absent").
+   The value to set (can be omitted if ``--state`` is set to ``absent``).
 
 
 OPTIONAL PARAMETERS
 -------------------
 state
-    "present" or "absent". Defaults to "present".
+   ``present`` or ``absent``.
+   Defaults to ``present``.
 
 
 BOOLEAN PARAMETERS
@@ -36,20 +34,27 @@ EXAMPLES
 
 .. code-block:: sh
 
-	# set timezone
-    __postgres_conf timezone --value Europe/Zurich
+   # set timezone
+   __postgres_conf timezone --value Europe/Zurich
 
-    # reset maximum number of concurrent connections to default (normally 100)
-    __postgres_conf max_connections --state absent
+   # reset maximum number of concurrent connections to default (normally 100)
+   __postgres_conf max_connections --state absent
 
 
 SEE ALSO
 --------
-- `cdist-type(7) <cdist-type.html>`_
+None.
+
+
+AUTHORS
+-------
+Beni Ruef (bernhard.ruef--@--ssrq-sds-fds.ch)
+Dennis Camera (dennis.camera--@--ssrq-sds-fds.ch)
 
 
 COPYING
 -------
-Copyright \(C) 2020 SSRQ (www.ssrq-sds-fds.ch).
-Free use of this software is granted under the terms
-of the GNU General Public License version 3 (GPLv3).
+Copyright \(C) 2019-2021 SSRQ (www.ssrq-sds-fds.ch).
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.

From 1c047353a95e7ac079ccf89af8fc232451b8f891 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 17 Apr 2021 09:57:10 +0200
Subject: [PATCH 601/743] [bin/cdist] Fix Python version check

---
 bin/cdist         | 8 +++++---
 cdist/__init__.py | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index ddaffa7f..adb06a8d 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -72,9 +72,11 @@ def commandline():
 
 
 if __name__ == "__main__":
-    if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION:
-        print('Python >= {} is required on the source host.'.format(
-                cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr)
+    if sys.version_info[:3] < cdist.MIN_SUPPORTED_PYTHON_VERSION:
+        print(
+            'Python >= {} is required on the source host.'.format(
+                ".".join(map(str, cdist.MIN_SUPPORTED_PYTHON_VERSION))),
+            file=sys.stderr)
         sys.exit(1)
 
     exit_code = 0
diff --git a/cdist/__init__.py b/cdist/__init__.py
index 44366cd0..31d49889 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -64,7 +64,7 @@ REMOTE_EXEC = "ssh -o User=root"
 REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
 
 
-MIN_SUPPORTED_PYTHON_VERSION = '3.5'
+MIN_SUPPORTED_PYTHON_VERSION = (3, 5)
 
 
 class Error(Exception):

From 1bb696a4100f161afada9b1e8d8ad2b8ab190f54 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 20 Apr 2021 07:33:07 +0200
Subject: [PATCH 602/743] Release 6.9.6

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index c8cb44da..70d0b960 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)

From acf9bf91f17fb4ad0e5813c6e4b0b40fc7e027b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Thu, 22 Apr 2021 08:55:14 +0200
Subject: [PATCH 603/743] [scanner] error to stderr and exit when scapy is not
 available

---
 cdist/scan/commandline.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index eca4cf13..0d7fb0ca 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -20,6 +20,7 @@
 #
 
 import logging
+import sys
 
 log = logging.getLogger("scan")
 
@@ -31,7 +32,9 @@ def commandline(args):
     try:
         import cdist.scan.scan as scan
     except ModuleNotFoundError:
-        print('cdist scan requires scapy to be installed')
+        print('cdist scan requires scapy to be installed! Exiting.',
+                file=sys.stderr)
+        sys.exit(1)
 
     processes = []
 

From a4464209b6741f2c1764aa432c56cfa8128852a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Thu, 22 Apr 2021 09:29:53 +0200
Subject: [PATCH 604/743] [scanner] add minimal error handling, consolidate CLI
 args processing

---
 cdist/argparse.py         | 10 ++++--
 cdist/scan/commandline.py | 19 ++++++-----
 cdist/scan/scan.py        | 71 +++++++++++----------------------------
 3 files changed, 37 insertions(+), 63 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index cadac39a..153e7864 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -492,12 +492,16 @@ def get_parsers():
         help='Try to configure detected hosts')
     parser['scan'].add_argument(
         '-I', '--interfaces',
-        action='append',  default=[],
+        action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
     parser['scan'].add_argument(
         '-d', '--delay',
-        action='store',  default=3600,
-        help='How long to wait before reconfiguring after last try')
+        action='store',  default=3600, type=int,
+        help='How long (seconds) to wait before reconfiguring after last try')
+    parser['scan'].add_argument(
+        '-t', '--trigger-delay',
+        action='store',  default=5, type=int,
+        help='How long (seconds) to wait between ICMPv6 echo requests')
     parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
 
     for p in parser:
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 0d7fb0ca..dead5292 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -24,26 +24,29 @@ import sys
 
 log = logging.getLogger("scan")
 
-
-# define this outside of the class to not handle scapy import errors by default
+# CLI processing is defined outside of the main scan class to handle
+# non-available optional scapy dependency (instead of crashing mid-flight).
 def commandline(args):
     log.debug(args)
 
+    # Check if we have the optional scapy dependency available.
     try:
         import cdist.scan.scan as scan
     except ModuleNotFoundError:
-        print('cdist scan requires scapy to be installed! Exiting.',
-                file=sys.stderr)
+        log.error('cdist scan requires scapy to be installed. Exiting.')
         sys.exit(1)
 
-    processes = []
-
+    # Default operation mode.
     if not args.mode:
-        # By default scan and trigger, but do not call any action
+        # By default scan and trigger, but do not call any action.
         args.mode = ['scan', 'trigger', ]
 
+    # We run each component in a separate process since they
+    # must not block on each other.
+    processes = []
+
     if 'trigger' in args.mode:
-        t = scan.Trigger(interfaces=args.interfaces)
+        t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay)
         t.start()
         processes.append(t)
         log.debug("Trigger started")
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index faee8a56..633a5c06 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -61,20 +61,22 @@ import datetime
 
 import cdist.config
 
+logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("scan")
 
-
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
     """
 
-    def __init__(self, interfaces=None, verbose=False):
+    def __init__(self, interfaces, sleeptime, verbose=False):
         self.interfaces = interfaces
+
+        # Used by scapy / send in trigger/2.
         self.verbose = verbose
 
-        # Wait 5 seconds before triggering again - FIXME: add parameter
-        self.sleeptime = 5
+        # Delay in seconds between sent ICMPv6EchoRequests.
+        self.sleeptime = sleeptime
 
     def start(self):
         self.processes = []
@@ -93,9 +95,12 @@ class Trigger(object):
             time.sleep(self.sleeptime)
 
     def trigger(self, interface):
-        packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest()
-        log.debug("Sending request on %s", interface)
-        send(packet, verbose=self.verbose)
+        try:
+            log.debug("Sending ICMPv6EchoRequest on %s", interface)
+            packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest()
+            send(packet, verbose=self.verbose)
+        except Exception as e:
+            log.error( "Could not send ICMPv6EchoRequest: %s", e)
 
 
 class Scanner(object):
@@ -103,7 +108,7 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces=None, args=None, outdir=None):
+    def __init__(self, interfaces, args=None, outdir=None):
         self.interfaces = interfaces
 
         if outdir:
@@ -148,47 +153,9 @@ class Scanner(object):
 
     def scan(self):
         log.debug("Scanning - zzzzz")
-        sniff(iface=self.interfaces,
-              filter="icmp6",
-              prn=self.handle_pkg)
-
-
-if __name__ == '__main__':
-    t = Trigger(interfaces=["wlan0"])
-    t.start()
-
-    # Scanner can listen on many interfaces at the same time
-    s = Scanner(interfaces=["wlan0"])
-    s.scan()
-
-    # Join back the trigger processes
-    t.join()
-
-    # Test in my lan shows:
-    # [18:48] bridge:cdist% ls -1d fe80::*
-    # fe80::142d:f0a5:725b:1103
-    # fe80::20d:b9ff:fe49:ac11
-    # fe80::20d:b9ff:fe4c:547d
-    # fe80::219:d2ff:feb2:2e12
-    # fe80::21b:fcff:feee:f446
-    # fe80::21b:fcff:feee:f45c
-    # fe80::21b:fcff:feee:f4b1
-    # fe80::21b:fcff:feee:f4ba
-    # fe80::21b:fcff:feee:f4bc
-    # fe80::21b:fcff:feee:f4c1
-    # fe80::21d:72ff:fe86:46b
-    # fe80::42b0:34ff:fe6f:f6f0
-    # fe80::42b0:34ff:fe6f:f863
-    # fe80::42b0:34ff:fe6f:f9b2
-    # fe80::4a5d:60ff:fea1:e55f
-    # fe80::77a3:5e3f:82cc:f2e5
-    # fe80::9e93:4eff:fe6c:c1f4
-    # fe80::ba69:f4ff:fec5:6041
-    # fe80::ba69:f4ff:fec5:8db7
-    # fe80::bad8:12ff:fe65:313d
-    # fe80::bad8:12ff:fe65:d9b1
-    # fe80::ce2d:e0ff:fed4:2611
-    # fe80::ce32:e5ff:fe79:7ea7
-    # fe80::d66d:6dff:fe33:e00
-    # fe80::e2ff:f7ff:fe00:20e6
-    # fe80::f29f:c2ff:fe7c:275e
+        try:
+            sniff(iface=self.interfaces,
+                  filter="icmp6",
+                  prn=self.handle_pkg)
+        except Exception as e:
+            log.error( "Could not start listener: %s", e)

From bb24d632d62da8390dd1a724fc325a57526da40d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Thu, 22 Apr 2021 10:20:49 +0200
Subject: [PATCH 605/743] [scanner] implement the --list flag

---
 cdist/argparse.py         |  4 +++
 cdist/scan/commandline.py | 63 ++++++++++++++++++++++++++++-----------
 cdist/scan/scan.py        | 17 +++++++++++
 3 files changed, 67 insertions(+), 17 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 153e7864..f390a974 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -486,6 +486,10 @@ def get_parsers():
         '-m', '--mode', help='Which modes should run',
         action='append', default=[],
         choices=['scan', 'trigger'])
+    parser['scan'].add_argument(
+        '--list',
+        action='store_true',
+        help='List the known hosts and exit')
     parser['scan'].add_argument(
         '--config',
         action='store_true',
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index dead5292..331694e4 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -21,26 +21,11 @@
 
 import logging
 import sys
+from datetime import datetime
 
 log = logging.getLogger("scan")
 
-# CLI processing is defined outside of the main scan class to handle
-# non-available optional scapy dependency (instead of crashing mid-flight).
-def commandline(args):
-    log.debug(args)
-
-    # Check if we have the optional scapy dependency available.
-    try:
-        import cdist.scan.scan as scan
-    except ModuleNotFoundError:
-        log.error('cdist scan requires scapy to be installed. Exiting.')
-        sys.exit(1)
-
-    # Default operation mode.
-    if not args.mode:
-        # By default scan and trigger, but do not call any action.
-        args.mode = ['scan', 'trigger', ]
-
+def run(scan, args):
     # We run each component in a separate process since they
     # must not block on each other.
     processes = []
@@ -59,3 +44,47 @@ def commandline(args):
 
     for process in processes:
         process.join()
+
+def list(scan, args):
+    s = scan.Scanner(interfaces=args.interfaces, args=args)
+    hosts = s.list()
+
+    # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
+    # 6 colons.
+    ipv6_max_size = 8 * 4 + 10
+    # We format dates as follow: YYYY-MM-DD HH:MM:SS
+    date_max_size =              8 + 2    + 6 + 2
+
+    print("{} | {}".format(
+        'link-local address'.ljust(ipv6_max_size),
+        'last seen'.ljust(date_max_size)))
+    print('=' * (ipv6_max_size + 3 + date_max_size))
+    for addr in hosts:
+        last_seen = datetime.strftime(
+                datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'),
+                '%Y-%m-%d %H:%M:%S')
+        print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size)))
+
+# CLI processing is defined outside of the main scan class to handle
+# non-available optional scapy dependency (instead of crashing mid-flight).
+def commandline(args):
+    log.debug(args)
+
+    # Check if we have the optional scapy dependency available.
+    try:
+        import cdist.scan.scan as scan
+    except ModuleNotFoundError:
+        log.error('cdist scan requires scapy to be installed. Exiting.')
+        sys.exit(1)
+
+    # Set default operation mode.
+    if not args.mode:
+        # By default scan and trigger, but do not call any action.
+        args.mode = ['scan', 'trigger', ]
+
+    # Print known hosts and exit is --list is specified - do not start
+    # the scanner.
+    if args.list:
+        list(scan, args)
+    else:
+        run(scan, args)
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 633a5c06..f3976370 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -132,6 +132,23 @@ class Scanner(object):
             with open(fname, "w") as fd:
                 fd.write(f"{now}\n")
 
+    def list(self):
+        hosts = dict()
+        for linklocal_addr in os.listdir(self.outdir):
+            workdir = os.path.join(self.outdir, linklocal_addr)
+            # We ignore any (unexpected) file in this directory.
+            if os.path.isdir(workdir):
+                last_seen='-'
+                last_seen_file = os.path.join(workdir, 'last_seen')
+                if os.path.isfile(last_seen_file):
+                    with open(last_seen_file, "r") as fd:
+                        last_seen = fd.readline()
+
+                hosts[linklocal_addr] = {'last_seen': last_seen}
+
+        return hosts
+
+
     def config(self):
         """
         Configure a host

From 13e2ad175f01b6cfbdb21ee50e85b6f3ba6fe750 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Sun, 25 Apr 2021 12:45:34 +0200
Subject: [PATCH 606/743] [scanner] add host class, name mapper and pre-config
 logic

---
 cdist/argparse.py         |   6 +-
 cdist/scan/commandline.py |  37 ++++++++----
 cdist/scan/scan.py        | 122 +++++++++++++++++++++++++++++---------
 3 files changed, 124 insertions(+), 41 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index f390a974..bedb23ac 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -485,7 +485,7 @@ def get_parsers():
     parser['scan'].add_argument(
         '-m', '--mode', help='Which modes should run',
         action='append', default=[],
-        choices=['scan', 'trigger'])
+        choices=['scan', 'trigger', 'config'])
     parser['scan'].add_argument(
         '--list',
         action='store_true',
@@ -498,6 +498,10 @@ def get_parsers():
         '-I', '--interfaces',
         action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
+    parser['scan'].add_argument(
+        '--name-mapper',
+        action='store',  default=None,
+        help='Map addresses to names, required for config mode')
     parser['scan'].add_argument(
         '-d', '--delay',
         action='store',  default=3600, type=int,
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 331694e4..1a0ab0a7 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -37,7 +37,10 @@ def run(scan, args):
         log.debug("Trigger started")
 
     if 'scan' in args.mode:
-        s = scan.Scanner(interfaces=args.interfaces, args=args)
+        s = scan.Scanner(
+                autoconfigure='config' in args.mode,
+                interfaces=args.interfaces,
+                name_mapper=args.name_mapper)
         s.start()
         processes.append(s)
         log.debug("Scanner started")
@@ -46,24 +49,27 @@ def run(scan, args):
         process.join()
 
 def list(scan, args):
-    s = scan.Scanner(interfaces=args.interfaces, args=args)
+    s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper)
     hosts = s.list()
 
     # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
     # 6 colons.
     ipv6_max_size = 8 * 4 + 10
-    # We format dates as follow: YYYY-MM-DD HH:MM:SS
-    date_max_size =              8 + 2    + 6 + 2
+    date_max_size = len(datetime.now().strftime(scan.datetime_format))
+    name_max_size = 25
 
-    print("{} | {}".format(
-        'link-local address'.ljust(ipv6_max_size),
-        'last seen'.ljust(date_max_size)))
-    print('=' * (ipv6_max_size + 3 + date_max_size))
-    for addr in hosts:
-        last_seen = datetime.strftime(
-                datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'),
-                '%Y-%m-%d %H:%M:%S')
-        print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size)))
+    print("{} | {} | {} | {}".format(
+        'name'.ljust(name_max_size),
+        'address'.ljust(ipv6_max_size),
+        'last seen'.ljust(date_max_size),
+        'last configured'.ljust(date_max_size)))
+    print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
+    for host in hosts:
+        print("{} | {} | {} | {}".format(
+            host.name(default='-').ljust(name_max_size),
+            host.address().ljust(ipv6_max_size),
+            host.last_seen().ljust(date_max_size),
+            host.last_configured().ljust(date_max_size)))
 
 # CLI processing is defined outside of the main scan class to handle
 # non-available optional scapy dependency (instead of crashing mid-flight).
@@ -82,6 +88,11 @@ def commandline(args):
         # By default scan and trigger, but do not call any action.
         args.mode = ['scan', 'trigger', ]
 
+    if 'config' in args.mode and args.name_mapper == None:
+        print('--name-mapper must be specified for scanner config mode.',
+                file=sys.stderr)
+        sys.exit(1)
+
     # Print known hosts and exit is --list is specified - do not start
     # the scanner.
     if args.list:
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index f3976370..459138e2 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -63,6 +63,69 @@ import cdist.config
 
 logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("scan")
+datetime_format = '%Y-%m-%d %H:%M:%S'
+
+class Host(object):
+    def __init__(self, addr, outdir, name_mapper=None):
+        self.addr = addr
+        self.workdir = os.path.join(outdir, addr)
+        self.name_mapper = name_mapper
+
+        os.makedirs(self.workdir, exist_ok=True)
+
+    def __get(self, key, default=None):
+        fname = os.path.join(self.workdir, key)
+        value=default
+        if os.path.isfile(fname):
+            with open(fname, "r") as fd:
+                value = fd.readline()
+        return value
+
+    def __set(self, key, value):
+        fname = os.path.join(self.workdir, key)
+        with open(fname, "w") as fd:
+            fd.write(f"{value}")
+
+    def name(self, default=None):
+        if self.name_mapper == None:
+            return default
+
+        fpath = os.path.join(os.getcwd(), self.name_mapper)
+        if os.path.isfile(fpath) and os.access(fpath, os.X_OK):
+             out = subprocess.run([fpath, self.addr], capture_output=True)
+             if out.returncode != 0:
+                 return default
+             else:
+                value = out.stdout.decode()
+                return (None if len(value) == 0 else value)
+        else:
+            return default
+
+    def address(self):
+        return self.addr
+
+    def last_seen(self, default=None):
+        raw = self.__get('last_seen')
+        if raw:
+            return datetime.datetime.strptime(raw, datetime_format)
+        else:
+            return default
+
+    def last_configured(self, default=None):
+        raw = self.__get('last_configured')
+        if raw:
+            return datetime.datetime.strptime(raw, datetime_format)
+        else:
+            return default
+
+    def seen(self):
+        now = datetime.datetime.now().strftime(datetime_format)
+        self.__set('last_seen', now)
+
+    def configure(self):
+        # TODO: configure.
+        now = datetime.datetime.now().strftime(datetime_format)
+        self.__set('last_configured', now)
 
 class Trigger(object):
     """
@@ -108,48 +171,43 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces, args=None, outdir=None):
+    def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None):
         self.interfaces = interfaces
+        self.autoconfigure=autoconfigure
+        self.name_mapper = name_mapper
+        self.config_delay = datetime.timedelta(seconds=3600)
 
         if outdir:
             self.outdir = outdir
         else:
             self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan')
+        os.makedirs(self.outdir, exist_ok=True)
+
+        self.running_configs = {}
 
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
-            host = pkg['IPv6'].src
-            log.verbose("Host %s is alive", host)
+            host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper)
+            if host.name():
+                log.verbose("Host %s (%s) is alive", host.name(), host.address())
+            else:
+                log.verbose("Host %s is alive", host.address())
+            host.seen()
 
-            dir = os.path.join(self.outdir, host)
-            fname = os.path.join(dir, "last_seen")
-
-            now = datetime.datetime.now()
-
-            os.makedirs(dir, exist_ok=True)
-
-            # FIXME: maybe adjust the format so we can easily parse again
-            with open(fname, "w") as fd:
-                fd.write(f"{now}\n")
+            # TODO check last config.
+            if self.autoconfigure and \
+                    host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now():
+                self.config(host)
 
     def list(self):
-        hosts = dict()
-        for linklocal_addr in os.listdir(self.outdir):
-            workdir = os.path.join(self.outdir, linklocal_addr)
-            # We ignore any (unexpected) file in this directory.
-            if os.path.isdir(workdir):
-                last_seen='-'
-                last_seen_file = os.path.join(workdir, 'last_seen')
-                if os.path.isfile(last_seen_file):
-                    with open(last_seen_file, "r") as fd:
-                        last_seen = fd.readline()
-
-                hosts[linklocal_addr] = {'last_seen': last_seen}
+        hosts = []
+        for addr in os.listdir(self.outdir):
+            hosts.append(Host(addr, self.outdir, self.name_mapper))
 
         return hosts
 
 
-    def config(self):
+    def config(self, host):
         """
         Configure a host
 
@@ -158,9 +216,19 @@ class Scanner(object):
         - Maybe keep dict storing per host processes
         - Save the result
         - Save the output -> probably aligned to config mode
-
         """
 
+        if host.name() == None:
+            log.debug("config - could not resolve name for %s, aborting.", host.address())
+            return
+
+        if self.running_configs.get(host.name()) != None:
+            log.debug("config - is already running for %s, aborting.", host.name())
+
+        log.info("config - running against host %s.", host.name())
+        p = host.configure()
+        self.running_configs[host.name()] = p
+
     def start(self):
         self.process = Process(target=self.scan)
         self.process.start()

From 512e9b23c0035b09ac1e7e76f706fee2e2ec481f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 25 Apr 2021 15:53:40 +0200
Subject: [PATCH 607/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 70d0b960..c41743cd 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
+
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong

From 6ac8cbf98f5d5018373db6eb9c038ef1c599ff2f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:33:56 +0200
Subject: [PATCH 608/743] [type/__postgres_database] Include postgres_user
 explorer from __postgres_conf

---
 .../explorer/postgres_user                    |  1 +
 .../type/__postgres_database/explorer/state   | 23 +++++--------------
 .../type/__postgres_database/gencode-remote   | 13 +----------
 3 files changed, 8 insertions(+), 29 deletions(-)
 create mode 120000 cdist/conf/type/__postgres_database/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_database/explorer/postgres_user b/cdist/conf/type/__postgres_database/explorer/postgres_user
new file mode 120000
index 00000000..714e7237
--- /dev/null
+++ b/cdist/conf/type/__postgres_database/explorer/postgres_user
@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user
\ No newline at end of file
diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state
index d68d4120..bb044e0e 100755
--- a/cdist/conf/type/__postgres_database/explorer/state
+++ b/cdist/conf/type/__postgres_database/explorer/state
@@ -1,6 +1,7 @@
 #!/bin/sh
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,25 +19,13 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$("${__explorer}/os")"
-in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
-esac
+postgres_user=$("${__type_explorer:?}/postgres_user")
 
+name=${__object_id:?}
 
-name="$__object_id"
-
-if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='$name'\"")"
+if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='${name}'\"")"
 then
-    echo 'present'
+	echo 'present'
 else
-    echo 'absent'
+	echo 'absent'
 fi
diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote
index 0f11cff4..d70fc436 100755
--- a/cdist/conf/type/__postgres_database/gencode-remote
+++ b/cdist/conf/type/__postgres_database/gencode-remote
@@ -18,23 +18,12 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
-in
-   netbsd)
-      postgres_user='pgsql'
-      ;;
-   openbsd)
-      postgres_user='_postgresql'
-      ;;
-   *)
-      postgres_user='postgres'
-      ;;
-esac
 
 
 name="$__object_id"
 state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
 if [ "$state_should" != "$state_is" ]; then
    case "$state_should" in

From 58b279a8d0121dcbef05a4999d1d88208656aeed Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:34:17 +0200
Subject: [PATCH 609/743] [type/__postgres_database] Improve quoting

---
 .../type/__postgres_database/gencode-remote   | 95 +++++++++++--------
 1 file changed, 55 insertions(+), 40 deletions(-)

diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote
index d70fc436..7d7d6fa2 100755
--- a/cdist/conf/type/__postgres_database/gencode-remote
+++ b/cdist/conf/type/__postgres_database/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,49 +19,63 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
 
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
 
-name="$__object_id"
-state_should="$(cat "$__object/parameter/state")"
-state_is="$(cat "$__object/explorer/state")"
 postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
-if [ "$state_should" != "$state_is" ]; then
-   case "$state_should" in
-      present)
-         owner=""
-         if [ -f "$__object/parameter/owner" ]; then
-            owner="-O \"$(cat "$__object/parameter/owner")\""
-         fi
+dbname=${__object_id:?}
+state_should=$(cat "${__object:?}/parameter/state")
+state_is=$(cat "${__object:?}/explorer/state")
 
-         template=""
-         if [ -f "$__object/parameter/template" ]; then
-            template="--template \"$(cat "$__object/parameter/template")\""
-         fi
-
-         encoding=""
-         if [ -f "$__object/parameter/encoding" ]; then
-            encoding="--encoding \"$(cat "$__object/parameter/encoding")\""
-         fi
-
-         lc_collate=""
-         if [ -f "$__object/parameter/lc-collate" ]; then
-            lc_collate="--lc-collate \"$(cat "$__object/parameter/lc-collate")\""
-         fi
-
-         lc_ctype=""
-         if [ -f "$__object/parameter/lc-ctype" ]; then
-            lc_ctype="--lc-ctype \"$(cat "$__object/parameter/lc-ctype")\""
-         fi
-
-         cat << EOF
-su - '$postgres_user' -c "createdb $owner \"$name\" $template $encoding $lc_collate $lc_ctype"
-EOF
-      ;;
-      absent)
-         cat << EOF
-su - '$postgres_user' -c "dropdb \"$name\""
-EOF
-      ;;
-   esac
+if test "${state_should}" = "$state_is"
+then
+	exit 0
 fi
+
+case ${state_should}
+in
+	(present)
+		set --
+
+		while read -r param_name opt
+		do
+			if test -f "${__object:?}/parameter/${param_name}"
+			then
+				set -- "$@" "${opt}" "$(cat "${__object:?}/parameter/${param_name}")"
+			fi
+		done <<-'EOF'
+		owner -O
+		template --template
+		encoding --encoding
+		lc_collate --lc-collate
+		lc_ctype --lc-ctype
+		EOF
+
+		set -- "$@" "${dbname}"
+
+		cat <<-EOF
+		su - $(quote "${postgres_user}") -c $(quote "$(quote createdb "$@")")
+		EOF
+		;;
+	(absent)
+		cat <<-EOF
+		su - $(quote "${postgres_user}") -c $(quote "$(quote dropdb "${dbname}")")
+		EOF
+		;;
+esac

From beb8da6d5fd141ac63eacfb526cfcc43cbf22d8f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:48:51 +0200
Subject: [PATCH 610/743] [type/__postgres_role] Include postgres_user explorer
 from __postgres_conf

---
 .../type/__postgres_role/explorer/postgres_user   |  1 +
 cdist/conf/type/__postgres_role/explorer/state    | 14 +-------------
 cdist/conf/type/__postgres_role/gencode-remote    | 15 +--------------
 3 files changed, 3 insertions(+), 27 deletions(-)
 create mode 120000 cdist/conf/type/__postgres_role/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_role/explorer/postgres_user b/cdist/conf/type/__postgres_role/explorer/postgres_user
new file mode 120000
index 00000000..714e7237
--- /dev/null
+++ b/cdist/conf/type/__postgres_role/explorer/postgres_user
@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user
\ No newline at end of file
diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 34069de9..0b264e6f 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -19,19 +19,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case $("${__explorer:?}/os")
-in
-	(netbsd)
-		postgres_user='pgsql'
-		;;
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(*)
-		postgres_user='postgres'
-		;;
-esac
-
+postgres_user=$("${__type_explorer:?}/postgres_user")
 rolename=${__object_id:?}
 
 
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index d7631fbd..7324b80c 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -28,20 +28,7 @@ quote() {
 	fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
 }
 
-case $(cat "${__global:?}/explorer/os")
-in
-	(netbsd)
-		postgres_user='pgsql'
-		;;
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(*)
-		postgres_user='postgres'
-		;;
-esac
-
-
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 rolename=${__object_id:?}
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")

From 3cf93249c38a6f27c097877add619b5d6499a049 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:55:11 +0200
Subject: [PATCH 611/743] [type/__postgres_extension] Include postgres_user
 explorer from __postgres_conf

---
 .../__postgres_extension/explorer/postgres_user     |  1 +
 cdist/conf/type/__postgres_extension/gencode-remote | 13 +------------
 2 files changed, 2 insertions(+), 12 deletions(-)
 create mode 120000 cdist/conf/type/__postgres_extension/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_extension/explorer/postgres_user b/cdist/conf/type/__postgres_extension/explorer/postgres_user
new file mode 120000
index 00000000..714e7237
--- /dev/null
+++ b/cdist/conf/type/__postgres_extension/explorer/postgres_user
@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user
\ No newline at end of file
diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote
index af9c97f1..a1c84828 100755
--- a/cdist/conf/type/__postgres_extension/gencode-remote
+++ b/cdist/conf/type/__postgres_extension/gencode-remote
@@ -22,18 +22,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
-in
-   netbsd)
-      postgres_user='pgsql'
-      ;;
-   openbsd)
-      postgres_user='_postgresql'
-      ;;
-   *)
-      postgres_user='postgres'
-      ;;
-esac
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
 
 dbname=$(    echo "$__object_id" | cut -d":" -f1 )

From 8296051653a8187374ad51a3a487315767b73a19 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 16 Apr 2021 19:22:58 +0200
Subject: [PATCH 612/743] [type/__postgres_extension] Add state explorer

---
 .../type/__postgres_extension/explorer/state  | 41 ++++++++++++++
 .../type/__postgres_extension/gencode-remote  | 44 ++++++++++-----
 cdist/conf/type/__postgres_extension/man.rst  | 55 +++++++++++--------
 3 files changed, 104 insertions(+), 36 deletions(-)
 create mode 100644 cdist/conf/type/__postgres_extension/explorer/state

diff --git a/cdist/conf/type/__postgres_extension/explorer/state b/cdist/conf/type/__postgres_extension/explorer/state
new file mode 100644
index 00000000..9d156be7
--- /dev/null
+++ b/cdist/conf/type/__postgres_extension/explorer/state
@@ -0,0 +1,41 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Prints "present" if the extension is currently installed.
+# "absent" otherwise.
+
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+
+postgres_user=$("${__type_explorer:?}/postgres_user")
+
+IFS=: read -r dbname extname <<EOF
+${__object_id:?}
+EOF
+
+psql_exec() {
+	su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")"
+}
+
+if psql_exec "${dbname}" 'SELECT extname FROM pg_extension' | grep -qFx "${extname}"
+then
+	echo present
+else
+	echo absent
+fi
diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote
index a1c84828..4ce72622 100755
--- a/cdist/conf/type/__postgres_extension/gencode-remote
+++ b/cdist/conf/type/__postgres_extension/gencode-remote
@@ -2,9 +2,10 @@
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2013 Tomas Pospisek (tpo_deb at sourcepole.ch)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This type was created by Tomas Pospisek based on the
-#__postgres_role type by Steven Armstrong
+# __postgres_role type by Steven Armstrong.
 #
 # This file is part of cdist.
 #
@@ -24,19 +25,36 @@
 
 postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_cmd() {
+	printf 'su - %s -c %s\n' \
+		"$(quote "${postgres_user}")" \
+		"$(quote psql "$(quote "$1")" -c "$(quote "$2")")"
+}
 
-dbname=$(    echo "$__object_id" | cut -d":" -f1 )
-extension=$( echo "$__object_id" | cut -d":" -f2 )
 
-state_should=$( cat "$__object/parameter/state" )
+IFS=: read -r dbname extname <<EOF
+${__object_id:?}
+EOF
 
-case "$state_should" in
-    present)
-         cmd="CREATE EXTENSION IF NOT EXISTS $extension"
-         echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'"
-    ;;
-    absent)
-         cmd="DROP EXTENSION IF EXISTS $extension"
-         echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'"
-    ;;
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
+
+case ${state_should}
+in
+	(present)
+		psql_cmd "${dbname}" "CREATE EXTENSION ${extname}"
+		;;
+	(absent)
+		psql_cmd "${dbname}" "DROP EXTENSION ${extname}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
 esac
diff --git a/cdist/conf/type/__postgres_extension/man.rst b/cdist/conf/type/__postgres_extension/man.rst
index 79645b2b..442239f6 100644
--- a/cdist/conf/type/__postgres_extension/man.rst
+++ b/cdist/conf/type/__postgres_extension/man.rst
@@ -3,32 +3,36 @@ cdist-type__postgres_extension(7)
 
 NAME
 ----
-cdist-type__postgres_extension - manage postgres extensions
+cdist-type__postgres_extension - Manage PostgreSQL extensions
 
 
 DESCRIPTION
 -----------
-This cdist type allows you to create or drop postgres extensions.
+This cdist type allows you to manage PostgreSQL extensions.
 
-The object you need to pass to __postgres_extension consists of
-the database name and the extension name joined by a colon in the
-following form:
-
-.. code-block:: sh
-
-    dbname:extension
-
-f.ex.
+The ``__object_id`` to pass to ``__postgres_extension`` is of the form
+``dbname:extension``, e.g.:
 
 .. code-block:: sh
 
     rails_test:unaccent
 
 
+**CAUTION!** Be careful when installing extensions from (untrusted) third-party
+sources:
+
+   | Installing an extension as superuser requires trusting that the extension's
+     author wrote the extension installation script in a secure fashion. It is
+     not terribly difficult for a malicious user to create trojan-horse objects
+     that will compromise later execution of a carelessly-written extension
+     script, allowing that user to acquire superuser privileges.
+   | – `<https://www.postgresql.org/docs/13/sql-createextension.html#id-1.9.3.64.7>`_
+
+
 OPTIONAL PARAMETERS
 -------------------
 state
-    either "present" or "absent", defaults to "present"
+    either ``present`` or ``absent``, defaults to ``present``.
 
 
 EXAMPLES
@@ -36,24 +40,29 @@ EXAMPLES
 
 .. code-block:: sh
 
-    __postgres_extension           rails_test:unaccent
-    __postgres_extension --present rails_test:unaccent
-    __postgres_extension --absent  rails_test:unaccent
+   # Install extension unaccent into database rails_test
+   __postgres_extension rails_test:unaccent
+
+   # Drop extension unaccent from database fails_test
+   __postgres_extension rails_test:unaccent --state absent
 
 
 SEE ALSO
 --------
-:strong:`cdist-type__postgre_database`\ (7)
+- :strong:`cdist-type__postgres_database`\ (7)
+- PostgreSQL "CREATE EXTENSION" documentation at:
+  `<http://www.postgresql.org/docs/current/static/sql-createextension.html>`_.
 
-Postgres "Create Extension" documentation at: <http://www.postgresql.org/docs/current/static/sql-createextension.html>.
 
-AUTHOR
+AUTHORS
 -------
-Tomas Pospisek <tpo_deb--@--sourcepole.ch>
+| Tomas Pospisek <tpo_deb--@--sourcepole.ch>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+
 
 COPYING
 -------
-Copyright \(C) 2014 Tomas Pospisek. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2014 Tomas Pospisek, 2021 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.

From 0d33407b182964672b1cfc3ffc49c7fa7cf462a0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 16 Apr 2021 19:35:26 +0200
Subject: [PATCH 613/743] [type/__postgres_database] Proper quoting in state
 explorer

---
 cdist/conf/type/__postgres_database/explorer/state | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state
index bb044e0e..6a25df86 100755
--- a/cdist/conf/type/__postgres_database/explorer/state
+++ b/cdist/conf/type/__postgres_database/explorer/state
@@ -21,9 +21,14 @@
 
 postgres_user=$("${__type_explorer:?}/postgres_user")
 
-name=${__object_id:?}
+dbname=${__object_id:?}
 
-if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='${name}'\"")"
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_exec() {
+	su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")"
+}
+
+if psql_exec postgres "SELECT datname FROM pg_database" | grep -qFx "${dbname}"
 then
 	echo 'present'
 else

From 0f05f38384c94b79232ca0917d1323eeefd6e1ed Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 19 Apr 2021 19:09:46 +0200
Subject: [PATCH 614/743] [type/__postgres_role] Treat --password '' like no
 --password

---
 cdist/conf/type/__postgres_role/explorer/state | 11 ++++-------
 cdist/conf/type/__postgres_role/gencode-remote |  2 +-
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 0b264e6f..822816c1 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -43,8 +43,7 @@ role_properties=$(
 	  BEGIN { RS = "\036"; FS = "\034" }
 	  /^\([0-9]+ rows?\)/ { exit }
 	  NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next }
-	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }
-	  '
+	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }'
 )
 
 if test -n "${role_properties}"
@@ -78,12 +77,10 @@ then
 	# Check password
 	passwd_stored=$(
 		psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \
-		| awk 'BEGIN { RS = "\036" } NR == 2'
-		printf .
-	)
-	passwd_stored=${passwd_stored%?.}
+		| awk 'BEGIN { RS = "\036" } NR == 2 { printf "%s.", $0 }')
+	passwd_stored=${passwd_stored%.}
 
-	if test -f "${__object:?}/parameter/password"
+	if test -s "${__object:?}/parameter/password"
 	then
 		passwd_should=$(cat "${__object:?}/parameter/password"; printf .)
 	fi
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 7324b80c..4cb78330 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -46,7 +46,7 @@ psql_query() {
 
 psql_set_password() {
 	# NOTE: Always make sure that the password does not end up in psql_history!
-	# NOTE: Never set an empty string as the password, because they can be
+	# NOTE: Never set an empty string as the password, because it can be
 	#       interpreted differently by different tooling.
 	if test -s "${__object:?}/parameter/password"
 	then

From 92fff7cb77271009f25d999b6451dc1e43e9bb6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 26 Apr 2021 12:09:20 +0200
Subject: [PATCH 615/743] [scanner] fix crash on --list with name mapper
 provided

---
 cdist/scan/commandline.py | 10 ++++++++--
 cdist/scan/scan.py        |  2 +-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 1a0ab0a7..3eb7eec4 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -65,11 +65,17 @@ def list(scan, args):
         'last configured'.ljust(date_max_size)))
     print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
     for host in hosts:
+        last_seen = host.last_seen()
+        last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-'
+
+        last_configured = host.last_configured()
+        last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-'
+
         print("{} | {} | {} | {}".format(
             host.name(default='-').ljust(name_max_size),
             host.address().ljust(ipv6_max_size),
-            host.last_seen().ljust(date_max_size),
-            host.last_configured().ljust(date_max_size)))
+            last_seen.ljust(date_max_size),
+            last_configured.ljust(date_max_size)))
 
 # CLI processing is defined outside of the main scan class to handle
 # non-available optional scapy dependency (instead of crashing mid-flight).
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 459138e2..69a4121d 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -97,7 +97,7 @@ class Host(object):
                  return default
              else:
                 value = out.stdout.decode()
-                return (None if len(value) == 0 else value)
+                return (default if len(value) == 0 else value)
         else:
             return default
 

From 3a9dd5b1669fa29e1713cdadec1596d859d377f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 26 Apr 2021 12:09:55 +0200
Subject: [PATCH 616/743] [scanner] add minimal (non-configurable) config mode

---
 cdist/scan/scan.py | 35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 69a4121d..152abb4e 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -122,8 +122,20 @@ class Host(object):
         now = datetime.datetime.now().strftime(datetime_format)
         self.__set('last_seen', now)
 
+    # XXX: There's no easy way to use the config module without feeding it with
+    # CLI args. Might as well call everything from scratch!
     def configure(self):
-        # TODO: configure.
+        target = self.name() or self.address()
+        cmd = ['cdist', 'config', '-v', target ]
+
+        fname = os.path.join(self.workdir, 'last_configuration_log')
+        with open(fname, "w") as fd:
+            log.debug("Executing: %s", cmd)
+            completed_process = subprocess.run(cmd, stdout=fd, stderr=fd)
+            if completed_process.returncode != 0:
+                log.error("%s return with non-zero code %i - see %s for details.",
+                        cmd, completed_process.returncode, fname)
+
         now = datetime.datetime.now().strftime(datetime_format)
         self.__set('last_configured', now)
 
@@ -194,7 +206,7 @@ class Scanner(object):
                 log.verbose("Host %s is alive", host.address())
             host.seen()
 
-            # TODO check last config.
+            # Configure if needed.
             if self.autoconfigure and \
                     host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now():
                 self.config(host)
@@ -206,27 +218,18 @@ class Scanner(object):
 
         return hosts
 
-
     def config(self, host):
-        """
-        Configure a host
-
-        - Assume we are only called if necessary
-        - However we need to ensure to not run in parallel
-        - Maybe keep dict storing per host processes
-        - Save the result
-        - Save the output -> probably aligned to config mode
-        """
-
         if host.name() == None:
             log.debug("config - could not resolve name for %s, aborting.", host.address())
             return
 
-        if self.running_configs.get(host.name()) != None:
+        previous_config_process = self.running_configs.get(host.name())
+        if previous_config_process != None and previous_config_process.is_alive():
             log.debug("config - is already running for %s, aborting.", host.name())
 
-        log.info("config - running against host %s.", host.name())
-        p = host.configure()
+        log.info("config - running against host %s (%s).", host.name(), host.address())
+        p = Process(target=host.configure())
+        p.start()
         self.running_configs[host.name()] = p
 
     def start(self):

From 2232435c2206b2b3a9a5f2b976df16540b3f1eeb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 26 Apr 2021 14:39:26 +0200
Subject: [PATCH 617/743] [scanner] initial documentation

Note: still needs to patch main cdist(1) manpage
---
 cdist/scan/scan.py      | 32 -------------
 docs/src/cdist-scan.rst | 99 +++++++++++++++++++++++++++++++++++++++++
 docs/src/index.rst      |  1 +
 3 files changed, 100 insertions(+), 32 deletions(-)
 create mode 100644 docs/src/cdist-scan.rst

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 152abb4e..2912dab3 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -19,38 +19,6 @@
 #
 #
 
-#
-# Interface to be implemented:
-# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated
-#   scan: scan / listen for icmp6 replies
-#   trigger: send trigger to multicast
-#   config: configure newly detected hosts
-#   install: install newly detected hosts
-#
-# Scanner logic
-#  - save results to configdir:
-#     basedir = ~/.cdist/scan/<ipv6-address>
-#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time
-#           or similar
-#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record
-#           unix time or similar
-#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record
-#           unix time or similar
-#
-#
-#
-#
-# cdist scan --list
-#       Show all known hosts including last seen flag
-#
-# Logic for reconfiguration:
-#
-#  - record when configured last time
-#  - introduce a parameter --reconfigure-after that takes time argument
-#  - reconfigure if a) host alive and b) reconfigure-after time passed
-#
-
-
 from multiprocessing import Process
 import os
 import logging
diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst
new file mode 100644
index 00000000..02193456
--- /dev/null
+++ b/docs/src/cdist-scan.rst
@@ -0,0 +1,99 @@
+Scan
+=====
+
+Description
+-----------
+Runs cdist as a daemon that discover/watch on hosts and reconfigure them
+periodically. It is especially useful in netboot-based environment where hosts
+boot unconfigured, and to ensure your infrastructure stays in sync with your
+configuration.
+
+This feature is still consider to be in **beta** stage.
+
+Usage (Examples)
+----------------
+
+Discover hosts on local network and configure those whose name is resolved by
+the name mapper script.
+
+.. code-block:: sh
+
+    $ cdist scan --beta --interface eth0 \
+      --mode scan --name-mapper path/to/script \
+      --mode trigger --mode config
+
+List known hosts and exit.
+
+.. code-block:: sh
+
+    $ cdist scan --beta --list --name-mapper path/to/script
+
+Please refer to `cdist(1)` for a detailed list of parameters.
+
+Modes
+-----
+
+The scanner has 3 modes that can be independently toggled. If the `--mode`
+parameter is not specified, only `tigger` and `scan` are enabled (= hosts are
+not configured).
+
+trigger
+  Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to
+  trigger detection by the `scan` module.
+
+scan
+  Watch for incoming ICMPv6 replies and optionally configure detected hosts.
+
+config
+  Enable configuration of hosts detected by `scan`.
+
+Name Mapper Script
+------------------
+
+The name mapper script takes an IPv6 address as first argument and writes the
+resolved name to stdout - if any. The script must be executable.
+
+Simplest script:
+
+.. code-block:: sh
+  #!/bin/sh
+
+  case "$1" in
+  	"fe80::20d:b9ff:fe57:3524")
+  		printf "my-host-01"
+  		;;
+  	"fe80::7603:bdff:fe05:89bb")
+  		printf "my-host-02"
+  		;;
+  esac
+
+Resolving name from `PTR` DNS record:
+
+.. code-block:: sh
+  #!/bin/sh
+
+  for cmd in dig sed; do
+  	if ! command -v $cmd > /dev/null; then
+  		exit 1
+  	fi
+  done
+
+  dig +short -x "$1" | sed -e 's/.$//'
+
+
+Trigger Source Script
+---------------------
+
+This script returns a list of addresses (separated by a newline) to be used by
+`trigger` mode. It is not used to map names. The script must be executable.
+
+Simplest script:
+
+.. code-block:: sh
+  #!/bin/sh
+
+  cat << EOF
+  server1.domain.tld
+  server2.domain.tld
+  server3.domain.tld
+  EOF
diff --git a/docs/src/index.rst b/docs/src/index.rst
index 31c044dc..369d5309 100644
--- a/docs/src/index.rst
+++ b/docs/src/index.rst
@@ -34,6 +34,7 @@ It natively supports IPv6 since the first release.
    cdist-parallelization
    cdist-inventory
    cdist-preos
+   cdist-scan
    cdist-integration
    cdist-reference
    cdist-best-practice

From a4122882f2a14034d1a219bd0314892176c5cb64 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Apr 2021 16:39:51 +0200
Subject: [PATCH 618/743] [type/__debconf_set_selections] Add state explorer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…and to make it work, replace --file with --line.

--file is deprecated because it does not work with the state explorer as the
contents of the file are not available on the target.
---
 .../__debconf_set_selections/explorer/state   | 124 ++++++++++++++++++
 .../__debconf_set_selections/gencode-remote   |  37 ++++--
 .../type/__debconf_set_selections/man.rst     |  49 ++++---
 .../type/__debconf_set_selections/manifest    |  21 +++
 .../parameter/{required => deprecated}        |   0
 .../parameter/optional_multiple               |   1 +
 6 files changed, 206 insertions(+), 26 deletions(-)
 create mode 100644 cdist/conf/type/__debconf_set_selections/explorer/state
 create mode 100755 cdist/conf/type/__debconf_set_selections/manifest
 rename cdist/conf/type/__debconf_set_selections/parameter/{required => deprecated} (100%)
 create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/optional_multiple

diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state
new file mode 100644
index 00000000..68d28941
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/explorer/state
@@ -0,0 +1,124 @@
+#!/bin/sh -e
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Determine current debconf selections' state.
+# Prints one of:
+#   present: all selections are already set as they should.
+#   different: one or more of the selections have a different value.
+#   absent: one or more of the selections are not (currently) defined.
+#
+
+test -x /usr/bin/perl || {
+	# cannot find perl (no perl ~ no debconf)
+	echo 'absent'
+	exit 0
+}
+
+linesfile="${__object:?}/parameter/line"
+test -s "${linesfile}" || {
+	if test -s "${__object:?}/parameter/file"
+	then
+		echo absent
+	else
+		echo present
+	fi
+	exit 0
+}
+
+/usr/bin/perl -- - "${linesfile}" <<'EOF'
+use strict;
+use warnings "all";
+
+use Debconf::Db;
+use Debconf::Question;
+
+# Extract @known... arrays from debconf-set-selections
+# These values are required to distinguish flags and values in the given lines.
+# DC: I couldn't think of a more ugly solution to the problem…
+my @knownflags;
+my @knowntypes;
+my $debconf_set_selections = '/usr/bin/debconf-set-selections';
+if (-e $debconf_set_selections) {
+	my $sed_known = 's/^my \(@known\(flags\|types\) = qw([a-z ]*);\).*$/\1/p';
+	eval `sed -n '$sed_known' '$debconf_set_selections'`;
+}
+
+sub mungeline ($) {
+	my $line = shift;
+	chomp $line;
+	$line =~ s/\r$//;
+	return $line;
+}
+
+sub fatal { printf STDERR @_; exit 1; }
+
+my $state = 'present';
+
+sub state {
+	my $new = shift;
+	if ($state eq 'present'
+	    or ($state eq 'different' and $new eq 'absent')) {
+		$state = $new;
+	}
+}
+
+Debconf::Db->load(readonly => 'true');
+
+while (<>) {
+	# Read and process lines (taken from debconf-set-selections)
+	$_ = mungeline($_);
+	while (/\\$/ && ! eof) {
+		s/\\$//;
+		$_ .= mungeline(<>);
+	}
+	next if /^\s*$/ || /^\s*\#/;
+
+	my ($owner, $label, $type, $content) = /^\s*(\S+)\s+(\S+)\s+(\S+)(?:\s(.*))?/
+		or fatal "invalid line: %s\n", $_;
+	$content = '' unless defined $content;
+
+
+	# Compare is and should state
+	my $q = Debconf::Question->get($label);
+
+	unless (defined $q) {
+		# probably a preseed
+		state 'absent';
+		next;
+	}
+
+	if (grep { $_ eq $q->type } @knownflags) {
+		# This line wants to set a flag, presumably.
+		if ($q->flag($q->type) ne $content) {
+			state 'different';
+		}
+	} else {
+		# Otherwise, it's probably a value…
+		if ($q->value ne $content) {
+			state 'different';
+		}
+
+		unless (grep { $_ eq $owner } (split /, /, $q->owners)) {
+			state 'different';
+		}
+	}
+}
+
+printf "%s\n", $state;
+EOF
diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote
index e99aef40..50d898c6 100755
--- a/cdist/conf/type/__debconf_set_selections/gencode-remote
+++ b/cdist/conf/type/__debconf_set_selections/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,16 +18,32 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# Setup selections
-#
 
-filename="$(cat "$__object/parameter/file")"
-
-if [ "$filename" = "-" ]; then
-    filename="$__object/stdin"
+if test -f "${__object:?}/parameter/line"
+then
+	filename="${__object:?}/parameter/line"
+elif test -s "${__object:?}/parameter/file"
+then
+	filename=$(cat "${__object:?}/parameter/file")
+	if test "${filename}" = '-'
+	then
+		filename="${__object:?}/stdin"
+	fi
+else
+	printf 'Neither --line nor --file set.\n' >&2
+	exit 1
 fi
 
-echo "debconf-set-selections << __file-eof"
-cat "$filename"
-echo "__file-eof"
+# setting no lines makes no sense
+test -s "${filename}" || exit 0
+
+state_is=$(cat "${__object:?}/explorer/state")
+
+if test "${state_is}" != 'present'
+then
+	cat <<-CODE
+	debconf-set-selections <<'EOF'
+	$(cat "${filename}")
+	EOF
+	CODE
+fi
diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst
index 58c25b81..690e3e49 100644
--- a/cdist/conf/type/__debconf_set_selections/man.rst
+++ b/cdist/conf/type/__debconf_set_selections/man.rst
@@ -8,15 +8,33 @@ cdist-type__debconf_set_selections - Setup debconf selections
 
 DESCRIPTION
 -----------
-On Debian and alike systems debconf-set-selections(1) can be used
+On Debian and alike systems :strong:`debconf-set-selections`\ (1) can be used
 to setup configuration parameters.
 
 
 REQUIRED PARAMETERS
 -------------------
+cf. ``--line``.
+
+
+OPTIONAL PARAMETERS
+-------------------
 file
-   Use the given filename as input for debconf-set-selections(1)
-   If filename is "-", read from stdin.
+   Use the given filename as input for :strong:`debconf-set-selections`\ (1)
+   If filename is ``-``, read from stdin.
+
+   **This parameter is deprecated, because it doesn't work with state detection.**
+line
+   A line in :strong:`debconf-set-selections`\ (1) compatible format.
+   This parameter can be used multiple times to set multiple options.
+
+   (This parameter is actually required, but marked optional because the
+   deprecated ``--file`` is still accepted.)
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
 
 
 EXAMPLES
@@ -24,30 +42,29 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # Setup configuration for nslcd
-    __debconf_set_selections nslcd --file /path/to/file
+    # Setup gitolite's gituser
+    __debconf_set_selections nslcd --line 'gitolite gitolite/gituser string git'
 
-    # Setup configuration for nslcd from another type
-    __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd"
-
-    __debconf_set_selections nslcd --file - << eof
-    gitolite gitolite/gituser string git
-    eof
+    # Setup configuration for nslcd from a file.
+    # NB: Multiple lines can be passed to --line, although this can be considered a hack.
+    __debconf_set_selections nslcd --line "$(cat "${__files:?}/preseed/nslcd.debconf")"
 
 
 SEE ALSO
 --------
-:strong:`debconf-set-selections`\ (1), :strong:`cdist-type__update_alternatives`\ (7)
+- :strong:`cdist-type__update_alternatives`\ (7)
+- :strong:`debconf-set-selections`\ (1)
 
 
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2014 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2011-2014 Nico Schottelius, 2021 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__debconf_set_selections/manifest b/cdist/conf/type/__debconf_set_selections/manifest
new file mode 100755
index 00000000..0f4fb2e2
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/manifest
@@ -0,0 +1,21 @@
+#!/bin/sh -e
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+__package_apt debconf
diff --git a/cdist/conf/type/__debconf_set_selections/parameter/required b/cdist/conf/type/__debconf_set_selections/parameter/deprecated
similarity index 100%
rename from cdist/conf/type/__debconf_set_selections/parameter/required
rename to cdist/conf/type/__debconf_set_selections/parameter/deprecated
diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple
new file mode 100644
index 00000000..a999a0c2
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple
@@ -0,0 +1 @@
+line

From 9cf19388abe95a5f8df9f5c94d2e54fa1060b2ef Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Apr 2021 16:47:44 +0200
Subject: [PATCH 619/743] [type/__debconf_set_selections] Send message about
 each debconf setting that is changed

---
 cdist/conf/type/__debconf_set_selections/gencode-remote | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote
index 50d898c6..9ba28f09 100755
--- a/cdist/conf/type/__debconf_set_selections/gencode-remote
+++ b/cdist/conf/type/__debconf_set_selections/gencode-remote
@@ -46,4 +46,9 @@ then
 	$(cat "${filename}")
 	EOF
 	CODE
+
+	awk '
+		{
+			printf "set %s %s %s %s\n", $1, $2, $3, $4
+		}' "${filename}" >>"${__messages_out:?}"
 fi

From 3a25b804664963e092d1f75c1fdded25d7016ef3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 26 Apr 2021 21:27:15 +0200
Subject: [PATCH 620/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index c41743cd..497ae91a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
+	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From a42ebc7a78e09e138954b1376376f7cb643420e0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 27 Apr 2021 19:41:10 +0200
Subject: [PATCH 621/743] [type/__debconf_set_selections] Synchronise objects

Works around locking error:

	debconf: DbDriver "config": /var/cache/debconf/config.dat is locked by another process: Resource temporarily unavailable
---
 .../__debconf_set_selections/explorer/state   | 20 ++++++++++++++++++-
 .../type/__debconf_set_selections/nonparallel |  0
 2 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 cdist/conf/type/__debconf_set_selections/nonparallel

diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state
index 68d28941..f8a3f6c8 100644
--- a/cdist/conf/type/__debconf_set_selections/explorer/state
+++ b/cdist/conf/type/__debconf_set_selections/explorer/state
@@ -41,10 +41,15 @@ test -s "${linesfile}" || {
 	exit 0
 }
 
+# assert __type_explorer is set (because it is used by the Perl script)
+: "${__type_explorer:?}"
+
 /usr/bin/perl -- - "${linesfile}" <<'EOF'
 use strict;
 use warnings "all";
 
+use Fcntl qw(:DEFAULT :flock);
+
 use Debconf::Db;
 use Debconf::Question;
 
@@ -78,7 +83,20 @@ sub state {
 	}
 }
 
-Debconf::Db->load(readonly => 'true');
+
+# Load Debconf DB but manually lock on the state explorer script,
+# because Debconf aborts immediately if executed concurrently.
+# This is not really an ideal solution because the Debconf DB could be locked by
+# another process (e.g. apt-get), but no way to achieve this could be found.
+# If you know how to, please provide a patch.
+my $lockfile = "%ENV{'__type_explorer'}/state";
+if (open my $lock_fh, '+<', $lockfile) {
+   flock $lock_fh, LOCK_EX or die "Cannot lock $lockfile";
+}
+{
+	Debconf::Db->load(readonly => 'true');
+}
+
 
 while (<>) {
 	# Read and process lines (taken from debconf-set-selections)
diff --git a/cdist/conf/type/__debconf_set_selections/nonparallel b/cdist/conf/type/__debconf_set_selections/nonparallel
new file mode 100644
index 00000000..e69de29b

From c00c8c20127af9881b4e295b7732524e7e1c5031 Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Mon, 10 May 2021 12:08:22 +0200
Subject: [PATCH 622/743] [__apt_key*] Deprecate __apt_key_uri and improve
 __apt_key

Previously this type was falling back to using the deprecated apt-key(8) by
checking for existence of files/directories on the controller host in
gencode-remote.

Adding `--use-deprecated-apt-key` as an explicit boolean serves two purposes:
1. It prevents fallbacks that might end up doing the wrong thing
   (as was the case)
2. It allows for a simple way to remove keys from the keyring that were
   previously added with apt-key(8) to /etc/apt/trusted.gpg

This parameter is added marked as deprecated as is only intended use is to
migrate to directory-based keyrings as recommended by Debian for a few releases.
It will be removed when Debian 11 stops being supported.

During the review process of this merge request, it was noted that the state of
PGP Key Servers is somewhat suboptimal, that the examples encouraged bad
practise (it is trivial to produce collisions for short key IDs), and that
this use does not require the Web of Trust, but instead only the public key
that is signing the repository.

That is why this also adds `--source` as an argument allowing for in-type or
in-manifest provision of such public keys by the type/manifest maintainer and
the use of Key Servers is still supported, but discouraged.
---
 cdist/conf/type/__apt_key/explorer/state      |  27 +++--
 cdist/conf/type/__apt_key/gencode-remote      |  53 ++++-----
 cdist/conf/type/__apt_key/man.rst             |  84 ++++++++++----
 cdist/conf/type/__apt_key/manifest            | 104 +++++++++++++++++-
 cdist/conf/type/__apt_key/parameter/boolean   |   1 +
 .../deprecated/use-deprecated-apt-key         |   3 +
 cdist/conf/type/__apt_key/parameter/optional  |   5 +-
 cdist/conf/type/__apt_key_uri/deprecated      |   1 +
 8 files changed, 209 insertions(+), 69 deletions(-)
 create mode 100644 cdist/conf/type/__apt_key/parameter/boolean
 create mode 100644 cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key
 create mode 100644 cdist/conf/type/__apt_key_uri/deprecated

diff --git a/cdist/conf/type/__apt_key/explorer/state b/cdist/conf/type/__apt_key/explorer/state
index 38f1bd3c..8ab268c1 100755
--- a/cdist/conf/type/__apt_key/explorer/state
+++ b/cdist/conf/type/__apt_key/explorer/state
@@ -27,18 +27,25 @@ else
    keyid="$__object_id"
 fi
 
+# From apt-key(8):
+#   Use of apt-key is deprecated, except for the use of apt-key del in
+#   maintainer scripts to remove existing keys from the main keyring.
+#   If such usage of apt-key is desired the additional installation of
+#   the GNU Privacy Guard suite (packaged in gnupg) is required.
+if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then
+   if apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK"
+      then echo present
+      else echo absent
+   fi
+   exit
+fi
+
 keydir="$(cat "$__object/parameter/keydir")"
 keyfile="$keydir/$__object_id.gpg"
 
-if [ -d "$keydir" ]
+if [ -f "$keyfile" ]
 then
-   if [ -f "$keyfile" ]
-   then echo present
-   else echo absent
-   fi
-else
-   # fallback to deprecated apt-key
-   apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \
-      && echo present \
-      || echo absent
+   echo present
+   exit
 fi
+echo absent
diff --git a/cdist/conf/type/__apt_key/gencode-remote b/cdist/conf/type/__apt_key/gencode-remote
index 0c96ff67..17dc9bfc 100755
--- a/cdist/conf/type/__apt_key/gencode-remote
+++ b/cdist/conf/type/__apt_key/gencode-remote
@@ -25,11 +25,7 @@ else
 fi
 state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
-
-if [ "$state_should" = "$state_is" ]; then
-   # nothing to do
-   exit 0
-fi
+method="$(cat "$__object/key_method")"
 
 keydir="$(cat "$__object/parameter/keydir")"
 keyfile="$keydir/$__object_id.gpg"
@@ -37,30 +33,18 @@ keyfile="$keydir/$__object_id.gpg"
 case "$state_should" in
    present)
       keyserver="$(cat "$__object/parameter/keyserver")"
-
-      if [ -f "$__object/parameter/uri" ]; then
-         uri="$(cat "$__object/parameter/uri")"
-
-         if [ -d "$keydir" ]; then
-            cat << EOF
-
-curl -s -L \\
-    -o "$keyfile" \\
-    "$uri"
-
-key="\$( cat "$keyfile" )"
-
-if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK'
-then
-    echo "\$key" | gpg --dearmor > "$keyfile"
-fi
-
-EOF
-         else
-            # fallback to deprecated apt-key
-            echo "curl -s -L '$uri' | apt-key add -"
+      # Using __download or __file as key source
+      # Propagate messages if needed
+      if [ "${method}" = "uri" ] || [ "${method}" = "source" ]; then
+         if grep -Eq "^__(file|download)$keyfile" "$__messages_in"; then
+            echo "added '$keyid'" >> "$__messages_out"
          fi
-      elif [ -d "$keydir" ]; then
+         exit 0
+      elif [ "${state_is}" = "present" ]; then
+         exit 0
+      fi
+      # Using key servers to fetch the key
+      if [ ! -f "$__object/parameter/use-deprecated-apt-key" ]; then
          # we need to kill gpg after 30 seconds, because gpg
          # can get stuck if keyserver is not responding.
          # exporting env var and not exit 1,
@@ -100,13 +84,16 @@ EOF
       echo "added '$keyid'" >> "$__messages_out"
    ;;
    absent)
-      if [ -f "$keyfile" ]; then
-         echo "rm '$keyfile'"
-      else
+      # Removal for keys added from a keyserver without this flag
+      # is done in the manifest
+      if [ "$state_is" != "absent" ] && \
+            [ -f "$__object/parameter/use-deprecated-apt-key" ]; then
          # fallback to deprecated apt-key
          echo "apt-key del \"$keyid\""
+         echo "removed '$keyid'" >> "$__messages_out"
+      # Propagate messages if needed
+      elif grep -Eq "^__file$keyfile" "$__messages_in"; then
+         echo "removed '$keyid'" >> "$__messages_out"
       fi
-
-      echo "removed '$keyid'" >> "$__messages_out"
    ;;
 esac
diff --git a/cdist/conf/type/__apt_key/man.rst b/cdist/conf/type/__apt_key/man.rst
index 234bc715..e35eaa0f 100644
--- a/cdist/conf/type/__apt_key/man.rst
+++ b/cdist/conf/type/__apt_key/man.rst
@@ -10,6 +10,14 @@ DESCRIPTION
 -----------
 Manages the list of keys used by apt to authenticate packages.
 
+This is done by placing the requested key in a file named
+``$__object_id.gpg`` in the ``keydir`` directory.
+
+This is supported by modern releases of Debian-based distributions.
+
+In order of preference, exactly one of: ``source``, ``uri`` or ``keyid``
+must be specified.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -18,21 +26,49 @@ None.
 
 OPTIONAL PARAMETERS
 -------------------
+keydir
+   keyring directory, defaults to ``/etc/apt/trusted.pgp.d``, which is
+   enabled system-wide by default.
+
+source
+   path to a file containing the GPG key of the repository.
+   Using this is recommended as it ensures that the manifest/type manintainer
+   has validated the key.
+   If ``-``, the GPG key is read from the type's stdin.
+
 state
    'present' or 'absent'. Defaults to 'present'
 
+uri
+   the URI from which to download the key.
+   It is highly recommended that you only use protocols with TLS like HTTPS.
+   This uses ``__download`` but does not use checksums, if you want to ensure
+   that the key doesn't change, you are better off downloading it and using
+   ``--source``.
+
+
+DEPRECATED OPTIONAL PARAMETERS
+------------------------------
 keyid
-   the id of the key to add. Defaults to __object_id
+   the id of the key to download from the ``keyserver``.
+   This is to be used in absence of ``--source`` and ``--uri`` or together
+   with ``--use-deprecated-apt-key`` for key removal.
+   Defaults to ``$__object_id``.
 
 keyserver
-   the keyserver from which to fetch the key. If omitted the default set
-   in ./parameter/default/keyserver is used.
+   the keyserver from which to fetch the key.
+   Defaults to ``pool.sks-keyservers.net``.
 
-keydir
-   key save location, defaults to ``/etc/apt/trusted.pgp.d``
 
-uri
-   the URI from which to download the key
+DEPRECATED BOOLEAN PARAMETERS
+-----------------------------
+use-deprecated-apt-key
+   ``apt-key(8)`` will last be available in Debian 11 and Ubuntu 22.04.
+   You can use this parameter to force usage of ``apt-key(8)``.
+   Please only use this parameter to *remove* keys from the keyring,
+   in order to prepare for removal of ``apt-key``.
+   Adding keys should be done without this parameter.
+   This parameter will be removed when Debian 11 stops being supported.
 
 
 EXAMPLES
@@ -40,33 +76,39 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # Add Ubuntu Archive Automatic Signing Key
-    __apt_key 437D05B5
-    # Same thing
-    __apt_key 437D05B5 --state present
-    # Get rid of it
-    __apt_key 437D05B5 --state absent
+    # add a key that has been verified by a type maintainer
+    __apt_key jitsi_meet_2021 \
+       --source cdist-contrib/type/__jitsi_meet/files/apt_2021.gpg
 
-    # same thing with human readable name and explicit keyid
-    __apt_key UbuntuArchiveKey --keyid 437D05B5
+    # remove an old, deprecated or expired key
+    __apt_key jitsi_meet_2016 --state absent
 
-    # same thing with other keyserver
-    __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com
+    # Get rid of a key that might have been added to
+    # /etc/apt/trusted.gpg with apt-key
+    __apt_key 0x40976EAF437D05B5 --use-deprecated-apt-key --state absent
 
-    # download key from the internet
-    __apt_key rabbitmq \
-       --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc
+    # add a key that we define in-line
+    __apt_key jitsi_meet_2021 --source '-' <<EOF
+    -----BEGIN PGP PUBLIC KEY BLOCK-----
+    [...]
+    -----END PGP PUBLIC KEY BLOCK-----
+    EOF
+
+    # download or update key from the internet
+    __apt_key rabbitmq_2007 \
+       --uri https://www.rabbitmq.com/rabbitmq-signing-key-public.asc
 
 
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
 Ander Punnar <ander-at-kvlt-dot-ee>
+Evilham <contact~~@~~evilham.com>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can
+Copyright \(C) 2011-2021 Steven Armstrong, Ander Punnar and Evilham. You can
 redistribute it and/or modify it under the terms of the GNU General Public
 License as published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__apt_key/manifest b/cdist/conf/type/__apt_key/manifest
index 010357cd..889a764a 100755
--- a/cdist/conf/type/__apt_key/manifest
+++ b/cdist/conf/type/__apt_key/manifest
@@ -2,7 +2,105 @@
 
 __package gnupg
 
-if [ -f "$__object/parameter/uri" ]
-then __package curl
-else __package dirmngr
+state_should="$(cat "${__object}/parameter/state")"
+
+incompatible_args()
+{
+	cat >> /dev/stderr <<-EOF
+	This type does not support --${1} and --${method} simultaneously.
+	EOF
+	exit 1
+}
+
+if [ -f "${__object}/parameter/source" ]; then
+	method="source"
+	src="$(cat "${__object}/parameter/source")"
+	if [ "${src}" = "-" ]; then
+		src="${__object}/stdin"
+	fi
+fi
+if [ -f "${__object}/parameter/uri" ]; then
+	if [ -n "${method}" ]; then
+		incompatible_args uri
+	fi
+	method="uri"
+	src="$(cat "${__object}/parameter/uri")"
+fi
+if [ -f "${__object}/parameter/keyid" ]; then
+	if [ -n "${method}" ]; then
+		incompatible_args keyid
+	fi
+	method="keyid"
+fi
+# Keep old default
+if [ -z "${method}" ]; then
+	method="keyid"
+fi
+# Save this for later in gencode-remote
+echo "${method}" > "${__object}/key_method"
+
+# Required remotely (most likely already installed)
+__package dirmngr
+# We need this in case a key has to be dearmor'd
+__package gnupg
+export require="__package/gnupg"
+
+if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then
+	# This is required if apt-key(8) is to be used
+	if [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then
+		incompatible_args use-deprecated-apt-key
+	fi
+else
+	if [ "${state_should}" = "absent" ] && \
+			[ -f "${__object}/parameter/keyid" ]; then
+		cat >> /dev/stderr <<EOF
+You can't reliably remove by keyid without --use-deprecated-apt-key.
+This would very likely do something you do not intend.
+EOF
+		exit 1
+	fi
+fi
+
+keydir="$(cat "${__object}/parameter/keydir")"
+keyfile="${keydir}/${__object_id}.gpg"
+keyfilecdist="${keyfile}.cdist"
+if [ "${state_should}" != "absent" ]; then
+	# Ensure keydir exists
+	__directory "${keydir}" --state exists --mode 0755
+fi
+
+if [ "${state_should}" = "absent" ]; then
+	__file "${keyfile}" --state "absent"
+	__file "${keyfilecdist}" --state "absent"
+elif [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then
+	dearmor="$(cat <<-EOF
+	if [ '${state_should}' = 'present' ]; then
+		# Dearmor if necessary
+		if grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK' '${keyfilecdist}'; then
+			gpg --dearmor < '${keyfilecdist}' > '${keyfile}'
+		else
+			cp '${keyfilecdist}' '${keyfile}'
+		fi
+		# Ensure permissions
+		chown root '${keyfile}'
+		chmod 0444 '${keyfile}'
+	fi
+	EOF
+	)"
+
+	if [ "${method}" = "uri" ]; then
+		__download "${keyfilecdist}" \
+			--url "${src}" \
+			--onchange "${dearmor}"
+		require="__download${keyfilecdist}" \
+			__file "${keyfile}" \
+				--owner root \
+				--mode 0444 \
+				--state pre-exists
+	else
+		__file "${keyfilecdist}" --state "${state_should}" \
+			--mode 0444 \
+			--source "${src}" \
+			--onchange "${dearmor}"
+	fi
 fi
diff --git a/cdist/conf/type/__apt_key/parameter/boolean b/cdist/conf/type/__apt_key/parameter/boolean
new file mode 100644
index 00000000..7263e1b2
--- /dev/null
+++ b/cdist/conf/type/__apt_key/parameter/boolean
@@ -0,0 +1 @@
+use-deprecated-apt-key
diff --git a/cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key b/cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key
new file mode 100644
index 00000000..d18ebe57
--- /dev/null
+++ b/cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key
@@ -0,0 +1,3 @@
+apt-key(8) will last be available in Debian 11 and Ubuntu 22.04.
+Use this flag *only* to migrate to placing a keyring directly in the
+/etc/apt/trusted.gpg.d/ directory with a descriptive name.
diff --git a/cdist/conf/type/__apt_key/parameter/optional b/cdist/conf/type/__apt_key/parameter/optional
index de647375..975d1fd8 100644
--- a/cdist/conf/type/__apt_key/parameter/optional
+++ b/cdist/conf/type/__apt_key/parameter/optional
@@ -1,5 +1,6 @@
-state
+keydir
 keyid
 keyserver
-keydir
+source
+state
 uri
diff --git a/cdist/conf/type/__apt_key_uri/deprecated b/cdist/conf/type/__apt_key_uri/deprecated
new file mode 100644
index 00000000..05550587
--- /dev/null
+++ b/cdist/conf/type/__apt_key_uri/deprecated
@@ -0,0 +1 @@
+Please migrate to using __apt_key key_id --uri URI.

From a696f3cf0026ec96e4cf7235ee8634391ccfecdb Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Mon, 10 May 2021 12:10:00 +0200
Subject: [PATCH 623/743] [__letsencrypt_cert] Revamp explorers, add locking.

This would fix #839

Certbot uses locking [1] even for read-only operations and does not properly
use exit codes, which means that sometimes it would print:
"Another instance of Certbot is already running" and exit with success.

However, the previous explorers would take that as the certificate being absent
and would trigger code generation.

The issue was made worse by having many explorers running certbot, so for N
certificates, we'd run certbot N*4 times, potentially "in parallel".

[1]: https://certbot.eff.org/docs/using.html#id5

This patch joins all explorers in one to avoid starting multiple remote python
processes and uses a cdist-specific lock in /tmp/certbot.cdist.lock with a
60 seconds timeout.

It has been tested with certbot 0.31.0 and 0.17 that the:

    from certbot.main import main

trick works. It is somewhat well documented so it can be somewhat relied upon.
---
 .../__letsencrypt_cert/explorer/certbot-path  |  3 -
 .../explorer/certificate-data                 | 78 +++++++++++++++++++
 .../explorer/certificate-domains              |  8 --
 .../explorer/certificate-exists               | 13 ----
 .../explorer/certificate-is-test              | 14 ----
 .../type/__letsencrypt_cert/gencode-remote    | 11 ++-
 cdist/conf/type/__letsencrypt_cert/manifest   |  2 +-
 7 files changed, 87 insertions(+), 42 deletions(-)
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certbot-path
 create mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-data
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test

diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path
deleted file mode 100755
index 3c6076df..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh -e
-
-command -v certbot 2>/dev/null || true
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data
new file mode 100755
index 00000000..ff62e742
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data
@@ -0,0 +1,78 @@
+#!/bin/sh -e
+certbot_path="$(command -v certbot 2>/dev/null || true)"
+# Defaults
+certificate_exists="no"
+certificate_is_test="no"
+
+if [ -n "${certbot_path}" ]; then
+	# Find python executable that has access to certbot's module
+	python_path=$(sed -n '1s/^#! *//p' "${certbot_path}")
+
+	# Use a lock for cdist due to certbot not exiting with failure
+	# or having any flags for concurrent use.
+	_certbot() {
+		${python_path} - 2>/dev/null <<EOF
+from certbot.main import main
+import fcntl
+lock_file = "/tmp/certbot.cdist.lock"
+timeout=60
+with open(lock_file, 'w') as fd:
+    for i in range(timeout):
+        try:
+            # Get exclusive lock
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            break
+        except:
+            # Wait if that fails
+            import time
+            time.sleep(1)
+    else:
+        # Timed out, exit with failure
+        import sys
+        sys.exit(1)
+    # Do list certificates
+    main(["certificates", "--cert-name", "${__object_id:?}"])
+EOF
+	}
+
+
+	_certificate_exists() {
+		if grep -q "  Certificate Name: ${__object_id:?}$"; then
+			echo yes
+		else
+			echo no
+		fi
+	}
+
+	_certificate_is_test() {
+		if grep -q 'INVALID: TEST_CERT'; then
+			echo yes
+		else
+			echo no
+		fi
+	}
+
+	_certificate_domains() {
+		grep '    Domains: ' | cut -d ' ' -f 6- | tr ' ' '\n'
+	}
+
+	# Get data about all available certificates
+	certificates="$(_certbot)"
+
+	# Check whether or not the certificate exists
+	certificate_exists="$(echo "${certificates}" | _certificate_exists)"
+
+	# Check whether or not the certificate is for testing
+	certificate_is_test="$(echo "${certificates}" | _certificate_is_test)"
+
+	# Get domains for certificate
+	certificate_domains="$(echo "${certificates}" | _certificate_domains)"
+fi
+
+# Return received data
+cat <<EOF
+certbot_path:${certbot_path}
+certificate_exists:${certificate_exists}
+certificate_is_test:${certificate_is_test}
+${certificate_domains}
+EOF
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains
deleted file mode 100755
index db605b63..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	certbot certificates --cert-name "${__object_id:?}" | grep '    Domains: ' | \
-		cut -d ' ' -f 6- | tr ' ' '\n'
-fi
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists
deleted file mode 100755
index 4e6f44db..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	if certbot certificates | grep -q "  Certificate Name: ${__object_id:?}$"; then
-		echo yes
-	else
-		echo no
-	fi
-else
-	echo no
-fi
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test
deleted file mode 100755
index 9b445059..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	if certbot certificates --cert-name "${__object_id:?}" | \
-		grep -q 'INVALID: TEST_CERT'; then
-		echo yes
-	else
-		echo no
-	fi
-else
-	echo no
-fi
diff --git a/cdist/conf/type/__letsencrypt_cert/gencode-remote b/cdist/conf/type/__letsencrypt_cert/gencode-remote
index 375570a4..60a77ec3 100755
--- a/cdist/conf/type/__letsencrypt_cert/gencode-remote
+++ b/cdist/conf/type/__letsencrypt_cert/gencode-remote
@@ -1,6 +1,10 @@
 #!/bin/sh -e
 
-certificate_exists=$(cat "${__object:?}/explorer/certificate-exists")
+_explorer_var() {
+	grep "^$1:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-
+}
+
+certificate_exists="$(_explorer_var certificate_exists)"
 name="${__object_id:?}"
 state=$(cat "${__object}/parameter/state")
 
@@ -29,8 +33,9 @@ case "${state}" in
 		fi
 
 		if [ "${certificate_exists}" = "yes" ]; then
-			existing_domains="${__object}/explorer/certificate-domains"
-			certificate_is_test=$(cat "${__object}/explorer/certificate-is-test")
+			existing_domains=$(mktemp "${TMPDIR:-/tmp}/existing_domains.cdist.XXXXXXXXXX")
+			tail -n +4 "${__object:?}/explorer/certificate-data" | grep -v '^$' > "${existing_domains}"
+			certificate_is_test="$(_explorer_var certificate_is_test)"
 
 			sort -uo "${requested_domains}" "${requested_domains}"
 			sort -uo "${existing_domains}" "${existing_domains}"
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 1df3574a..6394f629 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")"
+certbot_fullpath="$(grep "^certbot_path:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-)"
 state=$(cat "${__object}/parameter/state")
 os="$(cat "${__global:?}/explorer/os")"
 

From f14623e45f56aec4bdce53bc2b367e11ff157865 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 10 May 2021 12:17:08 +0200
Subject: [PATCH 624/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 497ae91a..5c3b547a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,8 @@ Changelog
 next:
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
+	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)
+	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 6210cccb28ace8ec2807ffe6ca9f9e4d9007990b Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 10 May 2021 12:34:04 +0200
Subject: [PATCH 625/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 5c3b547a..a7310788 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)
 	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
+	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking.
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 503a06ed28f743aad47797a0989be735c67b07a6 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 28 Apr 2021 13:32:10 +0300
Subject: [PATCH 626/743] [__git] fix group explorer

group name from numberic id wasn't resolved correctly.

try to use getent and fallback to reading /etc/group directly.
---
 cdist/conf/type/__git/explorer/group | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group
index 1365c60d..ab4396b1 100644
--- a/cdist/conf/type/__git/explorer/group
+++ b/cdist/conf/type/__git/explorer/group
@@ -14,6 +14,11 @@ then
 	then
 		printf '%u\n' "${group_gid}"
 	else
-		printf '%s\n' "$(id -u -n "${group_gid}")"
+		if command -v getent > /dev/null
+		then
+			getent group "${group_gid}" | cut -d : -f 1
+		else
+			awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group
+		fi
 	fi
 fi

From 75c71f69c1fed4371c5891ddcca0eaf28ad928e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 10:17:48 +0200
Subject: [PATCH 627/743] [scanner] pycodestyle compliance

---
 cdist/scan/commandline.py | 20 +++++++++++----
 cdist/scan/scan.py        | 54 ++++++++++++++++++++++++---------------
 2 files changed, 48 insertions(+), 26 deletions(-)

diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 3eb7eec4..b42bc7b2 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -25,13 +25,15 @@ from datetime import datetime
 
 log = logging.getLogger("scan")
 
+
 def run(scan, args):
     # We run each component in a separate process since they
     # must not block on each other.
     processes = []
 
     if 'trigger' in args.mode:
-        t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay)
+        t = scan.Trigger(interfaces=args.interfaces,
+                         sleeptime=args.trigger_delay)
         t.start()
         processes.append(t)
         log.debug("Trigger started")
@@ -48,6 +50,7 @@ def run(scan, args):
     for process in processes:
         process.join()
 
+
 def list(scan, args):
     s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper)
     hosts = s.list()
@@ -66,10 +69,16 @@ def list(scan, args):
     print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
     for host in hosts:
         last_seen = host.last_seen()
-        last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-'
+        if last_seen:
+            last_seen = last_seen.strftime(scan.datetime_format)
+        else:
+            last_seen = '-'
 
         last_configured = host.last_configured()
-        last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-'
+        if last_configured:
+            last_configured = last_configured.strftime(scan.datetime_format)
+        else:
+            '-'
 
         print("{} | {} | {} | {}".format(
             host.name(default='-').ljust(name_max_size),
@@ -77,6 +86,7 @@ def list(scan, args):
             last_seen.ljust(date_max_size),
             last_configured.ljust(date_max_size)))
 
+
 # CLI processing is defined outside of the main scan class to handle
 # non-available optional scapy dependency (instead of crashing mid-flight).
 def commandline(args):
@@ -94,9 +104,9 @@ def commandline(args):
         # By default scan and trigger, but do not call any action.
         args.mode = ['scan', 'trigger', ]
 
-    if 'config' in args.mode and args.name_mapper == None:
+    if 'config' in args.mode and args.name_mapper is None:
         print('--name-mapper must be specified for scanner config mode.',
-                file=sys.stderr)
+              file=sys.stderr)
         sys.exit(1)
 
     # Print known hosts and exit is --list is specified - do not start
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 2912dab3..4a20f511 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -33,6 +33,7 @@ logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("scan")
 datetime_format = '%Y-%m-%d %H:%M:%S'
 
+
 class Host(object):
     def __init__(self, addr, outdir, name_mapper=None):
         self.addr = addr
@@ -43,7 +44,7 @@ class Host(object):
 
     def __get(self, key, default=None):
         fname = os.path.join(self.workdir, key)
-        value=default
+        value = default
         if os.path.isfile(fname):
             with open(fname, "r") as fd:
                 value = fd.readline()
@@ -55,15 +56,15 @@ class Host(object):
             fd.write(f"{value}")
 
     def name(self, default=None):
-        if self.name_mapper == None:
+        if self.name_mapper is None:
             return default
 
         fpath = os.path.join(os.getcwd(), self.name_mapper)
         if os.path.isfile(fpath) and os.access(fpath, os.X_OK):
-             out = subprocess.run([fpath, self.addr], capture_output=True)
-             if out.returncode != 0:
-                 return default
-             else:
+            out = subprocess.run([fpath, self.addr], capture_output=True)
+            if out.returncode != 0:
+                return default
+            else:
                 value = out.stdout.decode()
                 return (default if len(value) == 0 else value)
         else:
@@ -94,19 +95,20 @@ class Host(object):
     # CLI args. Might as well call everything from scratch!
     def configure(self):
         target = self.name() or self.address()
-        cmd = ['cdist', 'config', '-v', target ]
+        cmd = ['cdist', 'config', '-v', target]
 
         fname = os.path.join(self.workdir, 'last_configuration_log')
         with open(fname, "w") as fd:
             log.debug("Executing: %s", cmd)
             completed_process = subprocess.run(cmd, stdout=fd, stderr=fd)
             if completed_process.returncode != 0:
-                log.error("%s return with non-zero code %i - see %s for details.",
-                        cmd, completed_process.returncode, fname)
+                log.error("%s return with non-zero code %i - see %s for \
+                        details.", cmd, completed_process.returncode, fname)
 
         now = datetime.datetime.now().strftime(datetime_format)
         self.__set('last_configured', now)
 
+
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
@@ -140,10 +142,12 @@ class Trigger(object):
     def trigger(self, interface):
         try:
             log.debug("Sending ICMPv6EchoRequest on %s", interface)
-            packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest()
+            packet = IPv6(
+                    dst="ff02::1%{}".format(interface)
+                    ) / ICMPv6EchoRequest()
             send(packet, verbose=self.verbose)
         except Exception as e:
-            log.error( "Could not send ICMPv6EchoRequest: %s", e)
+            log.error("Could not send ICMPv6EchoRequest: %s", e)
 
 
 class Scanner(object):
@@ -151,9 +155,10 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None):
+    def __init__(self, interfaces, autoconfigure=False, outdir=None,
+                 name_mapper=None):
         self.interfaces = interfaces
-        self.autoconfigure=autoconfigure
+        self.autoconfigure = autoconfigure
         self.name_mapper = name_mapper
         self.config_delay = datetime.timedelta(seconds=3600)
 
@@ -169,14 +174,17 @@ class Scanner(object):
         if ICMPv6EchoReply in pkg:
             host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper)
             if host.name():
-                log.verbose("Host %s (%s) is alive", host.name(), host.address())
+                log.verbose("Host %s (%s) is alive", host.name(),
+                            host.address())
             else:
                 log.verbose("Host %s is alive", host.address())
+
             host.seen()
 
             # Configure if needed.
             if self.autoconfigure and \
-                    host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now():
+                    host.last_configured(default=datetime.datetime.min) + \
+                    self.config_delay < datetime.datetime.now():
                 self.config(host)
 
     def list(self):
@@ -187,15 +195,19 @@ class Scanner(object):
         return hosts
 
     def config(self, host):
-        if host.name() == None:
-            log.debug("config - could not resolve name for %s, aborting.", host.address())
+        if host.name() is None:
+            log.debug("config - could not resolve name for %s, aborting.",
+                      host.address())
             return
 
         previous_config_process = self.running_configs.get(host.name())
-        if previous_config_process != None and previous_config_process.is_alive():
-            log.debug("config - is already running for %s, aborting.", host.name())
+        if previous_config_process is not None and \
+                previous_config_process.is_alive():
+            log.debug("config - is already running for %s, aborting.",
+                      host.name())
 
-        log.info("config - running against host %s (%s).", host.name(), host.address())
+        log.info("config - running against host %s (%s).", host.name(),
+                 host.address())
         p = Process(target=host.configure())
         p.start()
         self.running_configs[host.name()] = p
@@ -214,4 +226,4 @@ class Scanner(object):
                   filter="icmp6",
                   prn=self.handle_pkg)
         except Exception as e:
-            log.error( "Could not start listener: %s", e)
+            log.error("Could not start listener: %s", e)

From ab10b453f275d4289f3d37988b3caec204227194 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 11:15:41 +0200
Subject: [PATCH 628/743] [scanner] populate cdist(1)

---
 docs/src/man1/cdist.rst | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 0ecb4a61..599ec3b7 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -88,6 +88,9 @@ SYNOPSIS
     cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t]
                [pattern]
 
+    cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list]
+               [-d CONFIG_DELAY] [-t TRIGGER_DELAY]
+
 
 DESCRIPTION
 -----------
@@ -641,6 +644,31 @@ Display information for cdist (global explorers, types).
 **-t, --types**
     Display info for types.
 
+SCAN
+----
+
+Runs cdist as a daemon that discover/watch on hosts and reconfigure them
+periodically.
+
+**-I INTERFACE, --interfaces INTERFACE**
+    Interface to listen on. Can be specified multiple times.
+
+**-m MODE, --mode MODE**
+    Scanner components to enable. Can be specified multiple time to enable more
+    than one component. Supported modes are: scan, trigger and config. Defaults
+    to tiggger and scan.
+
+**--name-mapper PATH_TO_SCRIPT**
+    Path to script used to resolve a remote host name from an IPv6 address.
+
+**--list**
+    List known hosts and exit.
+
+**-d CONFIG_DELAY, --config-delay CONFIG_DELAY**
+    How long (seconds) to wait before reconfiguring after last try (config mode only).
+
+**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY**
+    How long (seconds) to wait between ICMPv6 echo requests (trigger mode only).
 
 CONFIGURATION
 -------------

From b8733c65f52776facce7a8de0265538a856ead64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 11:26:35 +0200
Subject: [PATCH 629/743] [scanner] fix minor CLI handling and --list bugs /
 typo

---
 cdist/argparse.py         |  4 ++--
 cdist/scan/commandline.py | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index bedb23ac..f17315e7 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -495,7 +495,7 @@ def get_parsers():
         action='store_true',
         help='Try to configure detected hosts')
     parser['scan'].add_argument(
-        '-I', '--interfaces',
+        '-I', '--interface',
         action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
     parser['scan'].add_argument(
@@ -503,7 +503,7 @@ def get_parsers():
         action='store',  default=None,
         help='Map addresses to names, required for config mode')
     parser['scan'].add_argument(
-        '-d', '--delay',
+        '-d', '--config-delay',
         action='store',  default=3600, type=int,
         help='How long (seconds) to wait before reconfiguring after last try')
     parser['scan'].add_argument(
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index b42bc7b2..ddbe4933 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -32,7 +32,7 @@ def run(scan, args):
     processes = []
 
     if 'trigger' in args.mode:
-        t = scan.Trigger(interfaces=args.interfaces,
+        t = scan.Trigger(interfaces=args.interface,
                          sleeptime=args.trigger_delay)
         t.start()
         processes.append(t)
@@ -41,7 +41,7 @@ def run(scan, args):
     if 'scan' in args.mode:
         s = scan.Scanner(
                 autoconfigure='config' in args.mode,
-                interfaces=args.interfaces,
+                interfaces=args.interface,
                 name_mapper=args.name_mapper)
         s.start()
         processes.append(s)
@@ -52,7 +52,7 @@ def run(scan, args):
 
 
 def list(scan, args):
-    s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper)
+    s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper)
     hosts = s.list()
 
     # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
@@ -75,10 +75,10 @@ def list(scan, args):
             last_seen = '-'
 
         last_configured = host.last_configured()
-        if last_configured:
+        if last_configured is not None:
             last_configured = last_configured.strftime(scan.datetime_format)
         else:
-            '-'
+            last_configured = '-'
 
         print("{} | {} | {} | {}".format(
             host.name(default='-').ljust(name_max_size),

From e0c52d0e1dfbaa3814a2ff26482177c2909a15ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 11:27:11 +0200
Subject: [PATCH 630/743] [scanner] remove mention of non-implemented trigger
 soruce script

---
 docs/src/cdist-scan.rst | 21 ++-------------------
 1 file changed, 2 insertions(+), 19 deletions(-)

diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst
index 02193456..064e65ff 100644
--- a/docs/src/cdist-scan.rst
+++ b/docs/src/cdist-scan.rst
@@ -8,7 +8,8 @@ periodically. It is especially useful in netboot-based environment where hosts
 boot unconfigured, and to ensure your infrastructure stays in sync with your
 configuration.
 
-This feature is still consider to be in **beta** stage.
+This feature is still consider to be in **beta** stage, and only operate on
+IPv6 (including link-local).
 
 Usage (Examples)
 ----------------
@@ -79,21 +80,3 @@ Resolving name from `PTR` DNS record:
   done
 
   dig +short -x "$1" | sed -e 's/.$//'
-
-
-Trigger Source Script
----------------------
-
-This script returns a list of addresses (separated by a newline) to be used by
-`trigger` mode. It is not used to map names. The script must be executable.
-
-Simplest script:
-
-.. code-block:: sh
-  #!/bin/sh
-
-  cat << EOF
-  server1.domain.tld
-  server2.domain.tld
-  server3.domain.tld
-  EOF

From defa3c22eaef88652a2bec6135d038a84710c41e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 29 May 2021 11:21:34 +0200
Subject: [PATCH 631/743] ++changelog

---
 docs/changelog | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index a7310788..403e7653 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,7 +6,8 @@ next:
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)
 	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
-	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking.
+	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham)
+	* Type __git: Fix group explorer (Ander Punnar)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From d596986af894749877fb3dcadf90628c9cfa9d13 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 31 May 2021 09:06:52 +0200
Subject: [PATCH 632/743] [type/__pyvenv] Fix group explorer

---
 cdist/conf/type/__pyvenv/explorer/group | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group
index f31a1cb7..922ce3df 100755
--- a/cdist/conf/type/__pyvenv/explorer/group
+++ b/cdist/conf/type/__pyvenv/explorer/group
@@ -14,6 +14,11 @@ then
 	then
 		printf '%u\n' "${group_gid}"
 	else
-		printf '%s\n' "$(id -u -n "${group_gid}")"
+		if command -v getent >/dev/null 2>&1
+		then
+			getent group "${group_gid}" | cut -d : -f 1
+		else
+			awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group
+		fi
 	fi
 fi

From 6ede76b08b96b7504c65ca2dc6737e31be7e7f24 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 8 Jun 2021 16:20:55 +0200
Subject: [PATCH 633/743] [type/__debconf_set_selections] man.rst: Fix line
 break in AUTHORS

---
 cdist/conf/type/__debconf_set_selections/man.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst
index 690e3e49..fd0040ae 100644
--- a/cdist/conf/type/__debconf_set_selections/man.rst
+++ b/cdist/conf/type/__debconf_set_selections/man.rst
@@ -58,8 +58,8 @@ SEE ALSO
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
-Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING

From c308a2896971ac4808be0152630ef41b7f96a2de Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 10 Jun 2021 06:39:55 +0200
Subject: [PATCH 634/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 403e7653..97ea204b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
 	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham)
 	* Type __git: Fix group explorer (Ander Punnar)
+	* Type __pyvenv: Fix group explorer (Dennis Camera)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From a3102022e18a23ce9b54eeaf7415b55361f80bd1 Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Fri, 11 Jun 2021 15:05:17 +0100
Subject: [PATCH 635/743] More sensible defaults; reword debian-only error
 message

---
 cdist/conf/type/__apt_pin/manifest                  | 13 +++++++++----
 cdist/conf/type/__apt_pin/nonparallel               |  0
 cdist/conf/type/__apt_pin/parameter/default/package |  1 -
 3 files changed, 9 insertions(+), 5 deletions(-)
 create mode 100644 cdist/conf/type/__apt_pin/nonparallel
 delete mode 100644 cdist/conf/type/__apt_pin/parameter/default/package

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index b1372ad0..909bc80d 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -19,9 +19,17 @@
 #
 
 
+name="$__object_id"
+
 os=$(cat "$__global/explorer/os")
 state="$(cat "$__object/parameter/state")"
-package="$(cat "$__object/parameter/package")"
+
+if [ -f "$__object/parameter/package" ]; then
+    package="$(cat "$__object/parameter/package")"
+else
+    package=$name
+fi
+
 distribution="$(cat "$__object/parameter/distribution")"
 priority="$(cat "$__object/parameter/priority")"
 
@@ -31,13 +39,10 @@ case "$os" in
    ;;
    *)
       printf "This type is specific to Debian and it's derivatives" >&2
-      printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2
       exit 1
    ;;
 esac
 
-name="$__object_id"
-
 case $distribution in
     stabletesting|unsatbel|experimental)
         pin="release a=$distribution"
diff --git a/cdist/conf/type/__apt_pin/nonparallel b/cdist/conf/type/__apt_pin/nonparallel
new file mode 100644
index 00000000..e69de29b
diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package
deleted file mode 100644
index 72e8ffc0..00000000
--- a/cdist/conf/type/__apt_pin/parameter/default/package
+++ /dev/null
@@ -1 +0,0 @@
-*

From b726697e070e5266eae38c7951ced14a2305acb2 Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Fri, 11 Jun 2021 15:05:33 +0100
Subject: [PATCH 636/743] Add documentation

---
 cdist/conf/type/__apt_pin/man.rst | 37 ++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst
index 7fcae6f8..0c91cdec 100644
--- a/cdist/conf/type/__apt_pin/man.rst
+++ b/cdist/conf/type/__apt_pin/man.rst
@@ -13,11 +13,21 @@ This space intentionally left blank.
 
 REQUIRED PARAMETERS
 -------------------
-None.
+distribution
+   Specifies what distribution the package should be pinned to. Accepts both codenames (buster/bullseye/sid) and suite names (stable/testing/...).
 
 
 OPTIONAL PARAMETERS
 -------------------
+package
+   Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used.
+
+priority
+   The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority)
+
+state
+   Will be passed to underlying `__file` type; see there for valid values and defaults.
+
 None.
 
 
@@ -31,14 +41,31 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # TODO
-    __apt_pin
+   # Add the bullseye repo to buster, but do not install any pacakges by default
+   # only if explicitely asked for
+    __apt_pin bullseye-default \
+       --package "*" \
+       --distribution bullseye \
+       --priority -1
+
+    require="__apt_pin/bullseye-default" __apt_source bullseye \
+       --uri http://deb.debian.org/debian/ \
+       --distribution bullseye \
+       --component main
+       # TODO
+       __apt_pin
+
+    __apt_pin foo --package "foo foo-*" --distribution bullseye
+
+    __foo # Installs the `foo` package internally
+
+    __package foo-plugin-extras
 
 
 SEE ALSO
 --------
-:strong:`TODO`\ (7)
-
+:strong:`apt_preferences`\ (7)
+:strong:`cdist-type__file`\ (7)
 
 AUTHORS
 -------

From 7b3f268df25922c515174862da95569aea547367 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 22 Jun 2021 16:36:30 +0300
Subject: [PATCH 637/743] [__download] improvements

1. post download checksum verification
2. detect hashes without prefix
3. add optional --destination
4. updated man
---
 .../conf/type/__download/explorer/remote_cmd  |  19 ---
 .../type/__download/explorer/remote_cmd_get   |  16 +++
 .../type/__download/explorer/remote_cmd_sum   |  82 +++++++++++
 cdist/conf/type/__download/explorer/state     |  60 ++------
 cdist/conf/type/__download/gencode-local      | 131 +++++++++++++++---
 cdist/conf/type/__download/gencode-remote     |  46 +++++-
 cdist/conf/type/__download/man.rst            |  31 ++++-
 cdist/conf/type/__download/manifest           |   2 +-
 cdist/conf/type/__download/parameter/optional |   3 +-
 9 files changed, 292 insertions(+), 98 deletions(-)
 delete mode 100755 cdist/conf/type/__download/explorer/remote_cmd
 create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_get
 create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_sum

diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd
deleted file mode 100755
index e3e35b45..00000000
--- a/cdist/conf/type/__download/explorer/remote_cmd
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh -e
-
-if [ -f "$__object/parameter/cmd-get" ]
-then
-    cmd="$( cat "$__object/parameter/cmd-get" )"
-
-elif command -v curl > /dev/null
-then
-    cmd="curl -L -o - '%s'"
-
-elif command -v fetch > /dev/null
-then
-    cmd="fetch -o - '%s'"
-
-else
-    cmd="wget -O - '%s'"
-fi
-
-echo "$cmd"
diff --git a/cdist/conf/type/__download/explorer/remote_cmd_get b/cdist/conf/type/__download/explorer/remote_cmd_get
new file mode 100755
index 00000000..9f1cd59c
--- /dev/null
+++ b/cdist/conf/type/__download/explorer/remote_cmd_get
@@ -0,0 +1,16 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cat "$__object/parameter/cmd-get"
+elif
+    command -v curl > /dev/null
+then
+    echo "curl -sSL -o - '%s'"
+elif
+    command -v fetch > /dev/null
+then
+    echo "fetch -o - '%s'"
+else
+    echo "wget -O - '%s'"
+fi
diff --git a/cdist/conf/type/__download/explorer/remote_cmd_sum b/cdist/conf/type/__download/explorer/remote_cmd_sum
new file mode 100755
index 00000000..84df663c
--- /dev/null
+++ b/cdist/conf/type/__download/explorer/remote_cmd_sum
@@ -0,0 +1,82 @@
+#!/bin/sh -e
+
+if [ ! -f "$__object/parameter/sum" ]
+then
+    exit 0
+fi
+
+if [ -f "$__object/parameter/cmd-sum" ]
+then
+    cat "$__object/parameter/cmd-sum"
+    exit 0
+fi
+
+sum_should="$( cat "$__object/parameter/sum" )"
+
+if echo "$sum_should" | grep -Fq ':'
+then
+    sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
+else
+    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+    then
+        sum_hash='cksum'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
+    then
+        sum_hash='md5'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
+    then
+        sum_hash='sha1'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
+    then
+        sum_hash='sha256'
+    else
+        echo 'hash format detection failed' >&2
+        exit 1
+    fi
+fi
+
+os="$( "$__explorer/os" )"
+
+case "$sum_hash" in
+    cksum)
+        echo "cksum %s | awk '{print \$1\" \"\$2}'"
+    ;;
+    md5)
+        case "$os" in
+            freebsd)
+                echo "md5 -q %s"
+            ;;
+            *)
+                echo "md5sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    sha1)
+        case "$os" in
+            freebsd)
+                echo "sha1 -q %s"
+            ;;
+            *)
+                echo "sha1sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    sha256)
+        case "$os" in
+            freebsd)
+                echo "sha256 -q %s"
+            ;;
+            *)
+                echo "sha256sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    *)
+        # we arrive here only if --sum is given with unknown format prefix
+        echo "unknown hash format: $sum_hash" >&2
+        exit 1
+    ;;
+esac
diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 68b517c5..881a1c09 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -1,6 +1,11 @@
 #!/bin/sh -e
 
-dst="/$__object_id"
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
+else
+    dst="/$__object_id"
+fi
 
 if [ ! -f "$dst" ]
 then
@@ -16,57 +21,18 @@ fi
 
 sum_should="$( cat "$__object/parameter/sum" )"
 
-if [ -f "$__object/parameter/cmd-sum" ]
+if echo "$sum_should" | grep -Fq ':'
 then
-    # shellcheck disable=SC2059
-    sum_is="$( eval "$( printf \
-        "$( cat "$__object/parameter/cmd-sum" )" \
-        "$dst" )" )"
-else
-    os="$( "$__explorer/os" )"
-
-    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
-    then
-        sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
-
-    elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="md5:$( md5 -q "$dst" )"
-            ;;
-            *)
-                sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-
-    elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="sha1:$( sha1 -q "$dst" )"
-            ;;
-            *)
-                sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-
-    elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="sha256:$( sha256 -q "$dst" )"
-            ;;
-            *)
-                sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-    fi
+    sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
 fi
 
+sum_cmd="$( "$__type_explorer/remote_cmd_sum" )"
+
+sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )"
+
 if [ -z "$sum_is" ]
 then
-    echo 'no checksum from target' >&2
+    echo 'existing destination checksum failed' >&2
     exit 1
 fi
 
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 571d2c3c..d1b0d0d5 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -11,34 +11,133 @@ fi
 
 url="$( cat "$__object/parameter/url" )"
 
-tmp="$( mktemp )"
-
-dst="/$__object_id"
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
+else
+    dst="/$__object_id"
+fi
 
 if [ -f "$__object/parameter/cmd-get" ]
 then
     cmd="$( cat "$__object/parameter/cmd-get" )"
 
-elif command -v wget > /dev/null
-then
-    cmd="wget -O - '%s'"
-
 elif command -v curl > /dev/null
 then
-    cmd="curl -L -o - '%s'"
+    cmd="curl -sSL -o - '%s'"
 
 elif command -v fetch > /dev/null
 then
     cmd="fetch -o - '%s'"
 
+elif command -v wget > /dev/null
+then
+    cmd="wget -O - '%s'"
+
 else
-    echo 'no usable locally installed utility for downloading' >&2
+    echo 'local download failed, no usable utility' >&2
     exit 1
 fi
 
-printf "$cmd > %s\n" \
-    "$url" \
-    "$tmp"
+echo "download_tmp=\"\$( mktemp )\""
+
+# shellcheck disable=SC2059
+printf "$cmd > \"\$download_tmp\"\n" "$url"
+
+if [ -f "$__object/parameter/sum" ]
+then
+    sum_should="$( cat "$__object/parameter/sum" )"
+
+    if [ -f "$__object/parameter/cmd-sum" ]
+    then
+        local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
+    else
+        if echo "$sum_should" | grep -Fq ':'
+        then
+            sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
+
+            sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
+        else
+            if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+            then
+                sum_hash='cksum'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
+            then
+                sum_hash='md5'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
+            then
+                sum_hash='sha1'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
+            then
+                sum_hash='sha256'
+            else
+                echo 'hash format detection failed' >&2
+                exit 1
+            fi
+        fi
+
+        case "$sum_hash" in
+            cksum)
+                local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'"
+            ;;
+            md5)
+                if command -v md5 > /dev/null
+                then
+                    local_cmd_sum="md5 -q %s"
+                elif
+                    command -v md5sum > /dev/null
+                then
+                    local_cmd_sum="md5sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            sha1)
+                if command -v sha1 > /dev/null
+                then
+                    local_cmd_sum="sha1 -q %s"
+                elif
+                    command -v sha1sum > /dev/null
+                then
+                    local_cmd_sum="sha1sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            sha256)
+                if command -v sha256 > /dev/null
+                then
+                    local_cmd_sum="sha256 -q %s"
+                elif
+                    command -v sha256sum > /dev/null
+                then
+                    local_cmd_sum="sha256sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            *)
+                # we arrive here only if --sum is given with unknown format prefix
+                echo "unknown hash format: $sum_hash" >&2
+                exit 1
+            ;;
+        esac
+
+        if [ -z "$local_cmd_sum" ]
+        then
+            echo 'local checksum verification failed, no usable utility' >&2
+            exit 1
+        fi
+    fi
+
+    # shellcheck disable=SC2059
+    echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\""
+
+    echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
+
+    echo "echo 'local download checksum mismatch' >&2"
+
+    echo "rm -f \"\$download_tmp\""
+
+    echo 'exit 1; fi'
+fi
 
 if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$'
 then
@@ -47,12 +146,10 @@ else
     target_host="$__target_host"
 fi
 
-printf '%s %s %s:%s\n' \
+# shellcheck disable=SC2016
+printf '%s "$download_tmp" %s:%s\n' \
     "$__remote_copy" \
-    "$tmp" \
     "$target_host" \
     "$dst"
 
-echo "rm -f '$tmp'"
-
-echo 'downloaded' > "$__messages_out"
+echo "rm -f \"\$download_tmp\""
diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
index 029a0801..e49bcec3 100755
--- a/cdist/conf/type/__download/gencode-remote
+++ b/cdist/conf/type/__download/gencode-remote
@@ -6,17 +6,51 @@ state_is="$( cat "$__object/explorer/state" )"
 
 if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
 then
-    cmd="$( cat "$__object/explorer/remote_cmd" )"
+    cmd_get="$( cat "$__object/explorer/remote_cmd_get" )"
 
     url="$( cat "$__object/parameter/url" )"
 
-    dst="/$__object_id"
+    if [ -f "$__object/parameter/destination" ]
+    then
+        dst="$( cat "$__object/parameter/destination" )"
+    else
+        dst="/$__object_id"
+    fi
 
-    printf "$cmd > %s\n" \
-        "$url" \
-        "$dst"
+    echo "download_tmp=\"\$( mktemp )\""
 
-    echo 'downloaded' > "$__messages_out"
+    # shellcheck disable=SC2059
+    printf "$cmd_get > \"\$download_tmp\"\n" "$url"
+
+    if [ -f "$__object/parameter/sum" ]
+    then
+        sum_should="$( cat "$__object/parameter/sum" )"
+
+        if [ -f "$__object/parameter/cmd-sum" ]
+        then
+            remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
+        else
+            remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )"
+
+            if echo "$sum_should" | grep -Fq ':'
+            then
+                sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
+            fi
+        fi
+
+        # shellcheck disable=SC2059
+        echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\""
+
+        echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
+
+        echo "echo 'remote download checksum mismatch' >&2"
+
+        echo "rm -f \"\$download_tmp\""
+
+        echo 'exit 1; fi'
+    fi
+
+    echo "mv \"\$download_tmp\" '$dst'"
 fi
 
 if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ]
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index a1278cfb..c16510a9 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -8,7 +8,7 @@ cdist-type__download - Download a file
 
 DESCRIPTION
 -----------
-By default type will try to use ``wget``, ``curl`` or ``fetch``.
+By default type will try to use ``curl``, ``fetch`` or ``wget``.
 If download happens in target (see ``--download``) then type will
 fallback to (and install) ``wget``.
 
@@ -16,6 +16,8 @@ If download happens in local machine, then environment variables like
 ``{http,https,ftp}_proxy`` etc can be used on cdist execution
 (``http_proxy=foo cdist config ...``).
 
+To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -25,14 +27,29 @@ url
 
 OPTIONAL PARAMETERS
 -------------------
+destination
+   Downloaded file's destination in target. If unset, ``$__object_id`` is used.
+
 sum
-   Checksum is used to decide if existing destination file must be redownloaded.
-   By default output of ``cksum`` without filename is expected.
-   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
+   Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256.
+
+   Type tries to detect hash format with regexes, but prefixes
+   ``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported.
+
+   Checksum have two purposes - state check and post-download verification.
+   In state check, if destination checksum mismatches, then content of URL
+   will be downloaded to temporary file. If downloaded temporary file's
+   checksum matches, then it will be moved to destination (overwritten).
+
+   For local downloads it is expected that usable utilities for checksum
+   calculation exist in the system.
 
 download
-   If ``local`` (default), then download file to local storage and copy
-   it to target host. If ``remote``, then download happens in target.
+   If ``local`` (default), then file is downloaded to local storage and copied
+   to target host. If ``remote``, then download happens in target.
+
+   For local downloads it is expected that usable utilities for downloading
+   exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``.
 
 cmd-get
    Command used for downloading.
@@ -62,7 +79,7 @@ EXAMPLES
     require='__directory/opt/cpma' \
         __download /opt/cpma/cnq3.zip \
             --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
-            --sum md5:46da3021ca9eace277115ec9106c5b46
+            --sum 46da3021ca9eace277115ec9106c5b46
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest
index 7ec8d86d..3d4c498b 100755
--- a/cdist/conf/type/__download/manifest
+++ b/cdist/conf/type/__download/manifest
@@ -1,6 +1,6 @@
 #!/bin/sh -e
 
-if grep -Eq '^wget' "$__object/explorer/remote_cmd"
+if grep -Eq '^wget' "$__object/explorer/remote_cmd_get"
 then
     __package wget
 fi
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index d69e083e..e809ef78 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,5 +1,6 @@
-sum
 cmd-get
 cmd-sum
+destination
 download
 onchange
+sum

From 2db40d8d704b427768307fbea29384bd3dc8dbd7 Mon Sep 17 00:00:00 2001
From: fancsali <fancsali@gmail.com>
Date: Mon, 28 Jun 2021 12:54:20 +0200
Subject: [PATCH 638/743] Use $__remote_exec and thus the ssh multiplexing

---
 cdist/conf/type/__rsync/gencode-local | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index e36ded2f..36addc36 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -36,4 +36,5 @@ fi
 
 echo rsync -a \
     --no-owner --no-group \
+    -e "$__remote_exec" \
     -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"

From d937d53f3dfd10830a07aee0450596eea62f2a1a Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Mon, 28 Jun 2021 18:09:35 +0100
Subject: [PATCH 639/743] Add quotes to rsync command

---
 cdist/conf/type/__rsync/gencode-local | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index 36addc36..f1bddc16 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -36,5 +36,5 @@ fi
 
 echo rsync -a \
     --no-owner --no-group \
-    -e "$__remote_exec" \
+    -e \"$__remote_exec\" \
     -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"

From 60753ddfcc3ac49303d572da7f6a68f398c02227 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 1 Jul 2021 14:42:10 +0300
Subject: [PATCH 640/743] fix shellcheck

---
 cdist/conf/type/__download/explorer/state | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 881a1c09..8c5d5ce1 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -28,6 +28,7 @@ fi
 
 sum_cmd="$( "$__type_explorer/remote_cmd_sum" )"
 
+# shellcheck disable=SC2059
 sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )"
 
 if [ -z "$sum_is" ]

From a90e642c1354cf01be7c9a1ba8a468ad624c4202 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 1 Jul 2021 14:50:40 +0300
Subject: [PATCH 641/743] update README

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index de6901c7..a468dd86 100644
--- a/README.md
+++ b/README.md
@@ -24,8 +24,8 @@ For community-maintained types there is
 
 ## Participating
 
-IRC: ``#cdist`` @ freenode
+IRC: ``#cdist`` @ [libera](https://libera.chat)
 
 Matrix: ``#cdist:ungleich.ch``
 
-Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist
+Matrix and IRC are bridged.

From 243a4b904a2de638d00bac57f6e762a076f9ae54 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 2 Jul 2021 06:50:02 +0200
Subject: [PATCH 642/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 97ea204b..d48c657e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,8 @@ next:
 	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham)
 	* Type __git: Fix group explorer (Ander Punnar)
 	* Type __pyvenv: Fix group explorer (Dennis Camera)
+	* Type __download: Improve checksum verification, add optional --destination (Ander Punnar)
+	* Type __debconf_set_selections: Add state explorer (Dennis Camera)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 30ba796d060fd3d0c4affd4167f14784156b5354 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 1 Jul 2021 11:49:07 +0300
Subject: [PATCH 643/743] new type: __snakeoil_cert

---
 .../__snakeoil_cert/explorer/ssl-cert-group   |  8 ++
 .../conf/type/__snakeoil_cert/explorer/state  | 24 ++++++
 .../conf/type/__snakeoil_cert/gencode-remote  | 73 +++++++++++++++++++
 cdist/conf/type/__snakeoil_cert/man.rst       | 60 +++++++++++++++
 .../parameter/default/cert-path               |  1 +
 .../parameter/default/key-path                |  1 +
 .../parameter/default/key-type                |  1 +
 .../type/__snakeoil_cert/parameter/optional   |  4 +
 8 files changed, 172 insertions(+)
 create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group
 create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/state
 create mode 100755 cdist/conf/type/__snakeoil_cert/gencode-remote
 create mode 100644 cdist/conf/type/__snakeoil_cert/man.rst
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/cert-path
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-path
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-type
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/optional

diff --git a/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group
new file mode 100755
index 00000000..a6cb3dfd
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+if grep -Eq '^ssl-cert:' /etc/group
+then
+    echo 'present'
+else
+    echo 'absent'
+fi
diff --git a/cdist/conf/type/__snakeoil_cert/explorer/state b/cdist/conf/type/__snakeoil_cert/explorer/state
new file mode 100755
index 00000000..cc5aae0b
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/explorer/state
@@ -0,0 +1,24 @@
+#!/bin/sh -e
+
+key_path="$( cat "$__object/parameter/key-path" )"
+
+if echo "$key_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    key_path="$( printf "$key_path" "$__object_id" )"
+fi
+
+cert_path="$( cat "$__object/parameter/cert-path" )"
+
+if echo "$cert_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    cert_path="$( printf "$cert_path" "$__object_id" )"
+fi
+
+if [ ! -f "$key_path" ] || [ ! -f "$cert_path" ]
+then
+    echo 'absent'
+else
+    echo 'present'
+fi
diff --git a/cdist/conf/type/__snakeoil_cert/gencode-remote b/cdist/conf/type/__snakeoil_cert/gencode-remote
new file mode 100755
index 00000000..8ffbfad1
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/gencode-remote
@@ -0,0 +1,73 @@
+#!/bin/sh -e
+
+state="$( cat "$__object/explorer/state" )"
+
+if [ "$state" = 'present' ]
+then
+    exit 0
+fi
+
+if [ -f "$__object/parameter/common-name" ]
+then
+    common_name="$( cat "$__object/parameter/common-name" )"
+else
+    common_name="$__object_id"
+fi
+
+key_path="$( cat "$__object/parameter/key-path" )"
+
+if echo "$key_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    key_path="$( printf "$key_path" "$__object_id" )"
+fi
+
+cert_path="$( cat "$__object/parameter/cert-path" )"
+
+if echo "$cert_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    cert_path="$( printf "$cert_path" "$__object_id" )"
+fi
+
+key_type="$( cat "$__object/parameter/key-type" )"
+
+key_type_arg="$( echo "$key_type" | cut -d : -f 2 )"
+
+case "$key_type" in
+    rsa:*)
+        echo "openssl genrsa -out '$key_path' $key_type_arg"
+    ;;
+    ec:*)
+        echo "openssl ecparam -name $key_type_arg -genkey -noout -out '$key_path'"
+    ;;
+esac
+
+# shellcheck disable=SC2016
+echo 'csr_path="$( mktemp )"'
+
+echo "openssl req -new -subj '/CN=$common_name' -key '$key_path' -out \"\$csr_path\""
+
+echo "openssl x509 -req -sha256 -days 3650 -in \"\$csr_path\" -signkey '$key_path' -out '$cert_path'"
+
+# shellcheck disable=SC2016
+echo 'rm -f "$csr_path"'
+
+if [ "$( cat "$__object/explorer/ssl-cert-group" )" = 'present' ]
+then
+    key_group='ssl-cert'
+else
+    key_group='root'
+fi
+
+echo "chmod 640 '$key_path'"
+
+echo "chown root '$key_path'"
+
+echo "chgrp $key_group '$key_path'"
+
+echo "chmod 644 '$cert_path'"
+
+echo "chown root '$cert_path'"
+
+echo "chgrp root '$cert_path'"
diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst
new file mode 100644
index 00000000..0b547804
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/man.rst
@@ -0,0 +1,60 @@
+cdist-type__snakeoil_cert(7)
+============================
+
+NAME
+----
+cdist-type__snakeoil_cert - Generate self-signed certificate
+
+
+DESCRIPTION
+-----------
+The purpose of this type is to generate **self-signed** certificate and private key
+for **testing purposes**. Certificate will expire in 3650 days.
+
+Certificate's and key's access bits will be ``644`` and ``640`` respectively.
+If target system has ``ssl-cert`` group, then it will be used as key's group.
+Use ``require='__snakeoil_cert/...' __file ...`` to override.
+
+
+OPTIONAL PARAMETERS
+-------------------
+common-name
+   Defaults to ``$__object_id``.
+
+key-path
+   ``%s`` in path will be replaced with ``$__object_id``.
+   Defaults to ``/etc/ssl/private/%s.pem``.
+
+key-type
+   Possible values are ``rsa:$bits`` and ``ec:$name``.
+   For possible EC names see ``openssl ecparam -list_curves``.
+   Defaults to ``rsa:2048``.
+
+cert-path
+   ``%s`` in path will be replaced with ``$__object_id``.
+   Defaults to ``/etc/ssl/certs/%s.pem``.
+
+
+EXAMPLES
+--------
+.. code-block:: sh
+	__snakeoil_cert localhost-rsa \
+	    --common-name localhost \
+	    --key-type rsa:4096
+
+	__snakeoil_cert localhost-ec \
+	    --common-name localhost \
+	    --key-type ec:prime256v1
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path
new file mode 100644
index 00000000..4bbae089
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path
@@ -0,0 +1 @@
+/etc/ssl/certs/%s.pem
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-path b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path
new file mode 100644
index 00000000..86eb9359
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path
@@ -0,0 +1 @@
+/etc/ssl/private/%s.pem
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-type b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type
new file mode 100644
index 00000000..f13f8ada
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type
@@ -0,0 +1 @@
+rsa:2048
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/optional b/cdist/conf/type/__snakeoil_cert/parameter/optional
new file mode 100644
index 00000000..76d08c0a
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/optional
@@ -0,0 +1,4 @@
+common-name
+key-path
+key-type
+cert-path

From 853e5cf7b4f615e1470c7ec2cfa8fe68cfa85ce7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 5 Jul 2021 09:07:06 +0200
Subject: [PATCH 644/743] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index d48c657e..0c3c64e1 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,8 @@ next:
 	* Type __pyvenv: Fix group explorer (Dennis Camera)
 	* Type __download: Improve checksum verification, add optional --destination (Ander Punnar)
 	* Type __debconf_set_selections: Add state explorer (Dennis Camera)
+	* Core: Implement usable cdist scan (Timothée Floure)
+	* New type: __snakeoil_cert (Ander Punnar)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From be92731c5c8a8543448f0d87fafae67e22ac76a1 Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Mon, 5 Jul 2021 12:38:26 +0100
Subject: [PATCH 645/743] Shell check quoting

We're actually echo-ing the command, hence the escape in front of the
quotes - the issue Shellcheck alludes too would actually occur, had the
escaping bakcslashes been omitted.
---
 cdist/conf/type/__rsync/gencode-local | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index f1bddc16..be4feabb 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -34,7 +34,8 @@ if [ -f "$__object/parameter/rsync-opts" ]; then
     done < "$__object/parameter/rsync-opts"
 fi
 
+# shellcheck disable=SC2086
 echo rsync -a \
     --no-owner --no-group \
-    -e \"$__remote_exec\" \
+    -e \"${__remote_exec}\" \
     -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"

From 521241d74102e37fae5f55552e1ef565d26ea9d2 Mon Sep 17 00:00:00 2001
From: fancsali <fancsali@gmail.com>
Date: Mon, 5 Jul 2021 15:28:05 +0200
Subject: [PATCH 646/743] Refine docs even more

---
 cdist/conf/type/__apt_pin/man.rst | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst
index 0c91cdec..4229c0cd 100644
--- a/cdist/conf/type/__apt_pin/man.rst
+++ b/cdist/conf/type/__apt_pin/man.rst
@@ -3,12 +3,12 @@ cdist-type__apt_pin(7)
 
 NAME
 ----
-cdist-type__apt_pin - TODO
+cdist-type__apt_pin - Manage apt pinning rules
 
 
 DESCRIPTION
 -----------
-This space intentionally left blank.
+Adds/removes/edits rules to pin some packages to a specific distribution. Useful if using multiple debian repositories at the same time. (Useful, if one wants to use a few specific packages from backports or perhaps Debain testing... or even sid.)
 
 
 REQUIRED PARAMETERS
@@ -20,7 +20,7 @@ distribution
 OPTIONAL PARAMETERS
 -------------------
 package
-   Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used.
+   Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used.
 
 priority
    The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority)
@@ -28,7 +28,6 @@ priority
 state
    Will be passed to underlying `__file` type; see there for valid values and defaults.
 
-None.
 
 
 BOOLEAN PARAMETERS
@@ -41,8 +40,8 @@ EXAMPLES
 
 .. code-block:: sh
 
-   # Add the bullseye repo to buster, but do not install any pacakges by default
-   # only if explicitely asked for
+   # Add the bullseye repo to buster, but do not install any packages by default,
+   # only if explicitely asked for (-1 means "never" for apt)
     __apt_pin bullseye-default \
        --package "*" \
        --distribution bullseye \
@@ -52,19 +51,19 @@ EXAMPLES
        --uri http://deb.debian.org/debian/ \
        --distribution bullseye \
        --component main
-       # TODO
-       __apt_pin
 
     __apt_pin foo --package "foo foo-*" --distribution bullseye
 
-    __foo # Installs the `foo` package internally
+    __foo # Assuming, this installs the `foo` package internally
 
-    __package foo-plugin-extras
+    __package foo-plugin-extras # Assuming we also need some extra stuff
 
 
 SEE ALSO
 --------
-:strong:`apt_preferences`\ (7)
+:strong:`apt_preferences`\ (5)
+:strong:`cdist-type__apt_source`\ (7)
+:strong:`cdist-type__apt_backports`\ (7)
 :strong:`cdist-type__file`\ (7)
 
 AUTHORS

From 166b58aeea09da41086525849291c70a3c3a571c Mon Sep 17 00:00:00 2001
From: fancsali <fancsali@gmail.com>
Date: Mon, 5 Jul 2021 15:32:27 +0200
Subject: [PATCH 647/743] Fix typo in distro names...

---
 cdist/conf/type/__apt_pin/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index 909bc80d..e72a8fdd 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -44,7 +44,7 @@ case "$os" in
 esac
 
 case $distribution in
-    stabletesting|unsatbel|experimental)
+    stable|testing|unstable|experimental)
         pin="release a=$distribution"
         ;;
     *)

From 485283f2e531f5fa23a876fa52411e0030f32589 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 7 Jul 2021 20:47:22 +0300
Subject: [PATCH 648/743] new type: __sed

---
 cdist/conf/type/__sed/explorer/file           | 22 +++++++++
 cdist/conf/type/__sed/gencode-remote          | 46 +++++++++++++++++
 cdist/conf/type/__sed/man.rst                 | 49 +++++++++++++++++++
 cdist/conf/type/__sed/parameter/boolean       |  1 +
 cdist/conf/type/__sed/parameter/optional      |  1 +
 .../type/__sed/parameter/required_multiple    |  1 +
 6 files changed, 120 insertions(+)
 create mode 100755 cdist/conf/type/__sed/explorer/file
 create mode 100755 cdist/conf/type/__sed/gencode-remote
 create mode 100644 cdist/conf/type/__sed/man.rst
 create mode 100644 cdist/conf/type/__sed/parameter/boolean
 create mode 100644 cdist/conf/type/__sed/parameter/optional
 create mode 100644 cdist/conf/type/__sed/parameter/required_multiple

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
new file mode 100755
index 00000000..a40f71b3
--- /dev/null
+++ b/cdist/conf/type/__sed/explorer/file
@@ -0,0 +1,22 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/file" ]
+then
+    file="$( cat "$__object/parameter/file" )"
+else
+    file="/$__object_id"
+fi
+
+if [ -f "$file" ]
+then
+    if [ -s "$file" ]
+    then
+        cat "$file"
+    else
+        echo "$file is empty" >&2
+        exit 1
+    fi
+else
+    echo "$file do not exist" >&2
+    exit 1
+fi
diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
new file mode 100755
index 00000000..391e3bc1
--- /dev/null
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -0,0 +1,46 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/file" ]
+then
+    file="$( cat "$__object/parameter/file" )"
+else
+    file="/$__object_id"
+fi
+
+script="$( cat "$__object/parameter/script" )"
+
+if [ "$script" = '-' ]
+then
+    script="$( cat "$__object/stdin" )"
+elif
+    [ -f "$script" ]
+then
+    script="$( cat "$script" )"
+fi
+
+file_from_target="$__object/explorer/file"
+
+sed_cmd='sed'
+
+if [ -f "$__object/parameter/regexp-extended" ]
+then
+    sed_cmd="$sed_cmd --regexp-extended"
+fi
+
+if ! echo "$script" \
+    | "$sed_cmd" -f - "$file_from_target" \
+    | diff "$file_from_target" - \
+    > /dev/null
+then
+    echo 'tmp="$( mktemp )"'
+
+    echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
+
+    echo "$script"
+
+    echo 'EOF'
+
+    echo "cp \"\$tmp\" '$file'"
+    
+    echo 'rm -f "$tmp"'
+fi
diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst
new file mode 100644
index 00000000..fbac212d
--- /dev/null
+++ b/cdist/conf/type/__sed/man.rst
@@ -0,0 +1,49 @@
+cdist-type__sed(7)
+==================
+
+NAME
+----
+cdist-type__sed - Transform text files with ``sed``
+
+
+DESCRIPTION
+-----------
+TODO
+
+
+REQUIRED MULTIPLE PARAMETERS
+----------------------------
+script
+   TODO
+
+
+OPTIONAL PARAMETERS
+-------------------
+file
+   TODO
+
+
+BOOLEAN PARAMETERS
+------------------
+regexp-extended
+   TODO
+
+
+EXAMPLES
+--------
+.. code-block:: sh
+
+   true
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
diff --git a/cdist/conf/type/__sed/parameter/boolean b/cdist/conf/type/__sed/parameter/boolean
new file mode 100644
index 00000000..1ad75c5d
--- /dev/null
+++ b/cdist/conf/type/__sed/parameter/boolean
@@ -0,0 +1 @@
+regexp-extended
diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional
new file mode 100644
index 00000000..f73f3093
--- /dev/null
+++ b/cdist/conf/type/__sed/parameter/optional
@@ -0,0 +1 @@
+file
diff --git a/cdist/conf/type/__sed/parameter/required_multiple b/cdist/conf/type/__sed/parameter/required_multiple
new file mode 100644
index 00000000..84f7e31d
--- /dev/null
+++ b/cdist/conf/type/__sed/parameter/required_multiple
@@ -0,0 +1 @@
+script

From 7a5896acfad0f992d6e7fe9cbe0bd08035ce19db Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 7 Jul 2021 21:23:25 +0300
Subject: [PATCH 649/743] add --onchange, fix shellcheck

---
 cdist/conf/type/__sed/gencode-remote     | 9 ++++++++-
 cdist/conf/type/__sed/man.rst            | 3 +++
 cdist/conf/type/__sed/parameter/optional | 1 +
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 391e3bc1..6b1bd216 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -32,6 +32,7 @@ if ! echo "$script" \
     | diff "$file_from_target" - \
     > /dev/null
 then
+    # shellcheck disable=SC2016
     echo 'tmp="$( mktemp )"'
 
     echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
@@ -41,6 +42,12 @@ then
     echo 'EOF'
 
     echo "cp \"\$tmp\" '$file'"
-    
+
+    # shellcheck disable=SC2016
     echo 'rm -f "$tmp"'
+
+    if [ -f "$__object/parameter/onchange" ]
+    then
+        cat "$__object/parameter/onchange"
+    fi
 fi
diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst
index fbac212d..4a728184 100644
--- a/cdist/conf/type/__sed/man.rst
+++ b/cdist/conf/type/__sed/man.rst
@@ -22,6 +22,9 @@ OPTIONAL PARAMETERS
 file
    TODO
 
+onchange
+   TODO
+
 
 BOOLEAN PARAMETERS
 ------------------
diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional
index f73f3093..fa86f917 100644
--- a/cdist/conf/type/__sed/parameter/optional
+++ b/cdist/conf/type/__sed/parameter/optional
@@ -1 +1,2 @@
 file
+onchange

From cf0032d667eb4bd8b7fe1bebacec341bbf71ed42 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 7 Jul 2021 21:28:00 +0300
Subject: [PATCH 650/743] add messaging and exit earlier

---
 cdist/conf/type/__sed/gencode-remote | 42 +++++++++++++++-------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 6b1bd216..836506f0 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -27,27 +27,31 @@ then
     sed_cmd="$sed_cmd --regexp-extended"
 fi
 
-if ! echo "$script" \
+if echo "$script" \
     | "$sed_cmd" -f - "$file_from_target" \
     | diff "$file_from_target" - \
     > /dev/null
 then
-    # shellcheck disable=SC2016
-    echo 'tmp="$( mktemp )"'
-
-    echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
-
-    echo "$script"
-
-    echo 'EOF'
-
-    echo "cp \"\$tmp\" '$file'"
-
-    # shellcheck disable=SC2016
-    echo 'rm -f "$tmp"'
-
-    if [ -f "$__object/parameter/onchange" ]
-    then
-        cat "$__object/parameter/onchange"
-    fi
+    exit 0
+fi
+
+# shellcheck disable=SC2016
+echo 'tmp="$( mktemp )"'
+
+echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
+
+echo "$script"
+
+echo 'EOF'
+
+echo "cp \"\$tmp\" '$file'"
+
+# shellcheck disable=SC2016
+echo 'rm -f "$tmp"'
+
+echo 'change' >> "$__messages_out"
+
+if [ -f "$__object/parameter/onchange" ]
+then
+    cat "$__object/parameter/onchange"
 fi

From 3e76d1cd3fbe53fa77c460cb2ce8416698b101bd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 8 Jul 2021 08:09:05 +0200
Subject: [PATCH 651/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 0c3c64e1..04f826f0 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -13,6 +13,7 @@ next:
 	* Type __debconf_set_selections: Add state explorer (Dennis Camera)
 	* Core: Implement usable cdist scan (Timothée Floure)
 	* New type: __snakeoil_cert (Ander Punnar)
+	* Type __rsync: Honour $__remote_exec env var (Daniel Fancsali)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 77dab4c5c63070aef962875af7fe8b1565f5ba78 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 10 Jul 2021 20:37:02 +0200
Subject: [PATCH 652/743] Release 6.9.7

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 04f826f0..284293c1 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)

From 65c43d3c1db938ac0063f54b1cb6b5090bb0a665 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 10 Jul 2021 21:02:27 +0200
Subject: [PATCH 653/743] Fix docs code block errors

---
 cdist/conf/type/__snakeoil_cert/man.rst | 1 +
 docs/src/cdist-scan.rst                 | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst
index 0b547804..b0b0a2e9 100644
--- a/cdist/conf/type/__snakeoil_cert/man.rst
+++ b/cdist/conf/type/__snakeoil_cert/man.rst
@@ -38,6 +38,7 @@ cert-path
 EXAMPLES
 --------
 .. code-block:: sh
+
 	__snakeoil_cert localhost-rsa \
 	    --common-name localhost \
 	    --key-type rsa:4096
diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst
index 064e65ff..86b7fab6 100644
--- a/docs/src/cdist-scan.rst
+++ b/docs/src/cdist-scan.rst
@@ -57,6 +57,7 @@ resolved name to stdout - if any. The script must be executable.
 Simplest script:
 
 .. code-block:: sh
+
   #!/bin/sh
 
   case "$1" in
@@ -71,6 +72,7 @@ Simplest script:
 Resolving name from `PTR` DNS record:
 
 .. code-block:: sh
+
   #!/bin/sh
 
   for cmd in dig sed; do

From 0e611af2a6388572eef5112c2ffaed082803965c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 13 Jul 2021 00:13:22 +0300
Subject: [PATCH 654/743] [__rsync] rewrite

---
 cdist/conf/type/__rsync/gencode-local         | 123 +++++++++++++-----
 cdist/conf/type/__rsync/gencode-remote        |  37 ------
 cdist/conf/type/__rsync/man.rst               | 107 +++++----------
 cdist/conf/type/__rsync/manifest              |  18 ---
 .../type/__rsync/parameter/default/options    |   1 +
 cdist/conf/type/__rsync/parameter/optional    |   4 +-
 .../type/__rsync/parameter/optional_multiple  |   2 +-
 7 files changed, 130 insertions(+), 162 deletions(-)
 delete mode 100755 cdist/conf/type/__rsync/gencode-remote
 create mode 100644 cdist/conf/type/__rsync/parameter/default/options

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index be4feabb..612d237e 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -1,41 +1,100 @@
 #!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
-source=$(cat "$__object/parameter/source")
-remote_user=$(cat "$__object/parameter/remote-user")
+if ! command -v rsync > /dev/null
+then
+    echo 'rsync is missing in local machine' >&2
+    exit 1
+fi
 
-if [ -f "$__object/parameter/destination" ]; then
-    destination=$(cat "$__object/parameter/destination")
+src="$( cat "$__object/parameter/source" )"
+
+if [ ! -e "$src" ]
+then
+    echo "$src not found" >&2
+    exit 1
+fi
+
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
 else
-    destination="/$__object_id"
+    dst="/$__object_id"
 fi
 
-set --
-if [ -f "$__object/parameter/rsync-opts" ]; then
-    while read -r opts; do
-        set -- "$@" "--$opts"
-    done < "$__object/parameter/rsync-opts"
+# if source is directory, then make sure that
+# source and destination are ending with slash,
+# because this is what you almost always want when
+# rsyncing two directories.
+
+if [ -d "$src" ]
+then
+    if ! echo "$src" | grep -Eq '/$'
+    then
+        src="$src/"
+    fi
+
+    if ! echo "$dst" | grep -Eq '/$'
+    then
+        dst="$dst/"
+    fi
 fi
 
+remote_user="$( cat "$__object/parameter/remote-user" )"
+
+options="$( cat "$__object/parameter/options" )"
+
+if [ -f "$__object/parameter/option" ]
+then
+    while read -r l
+    do
+        # there's a limitation in argparse: value can't begin with '-'.
+        # to workaround this, let's prefix opts with '\' in manifest and remove here.
+        # read more about argparse issue: https://bugs.python.org/issue9334
+
+        options="$options $( echo "$l" | sed 's/\\//g' )"
+    done \
+        < "$__object/parameter/option"
+fi
+
+if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ]
+then
+    options="$options --chown="
+
+    if [ -f "$__object/parameter/owner" ]
+    then
+        owner="$( cat "$__object/parameter/owner" )"
+        options="$options$owner"
+    fi
+
+    if [ -f "$__object/parameter/group" ]
+    then
+        group="$( cat "$__object/parameter/group" )"
+        options="$options:$group"
+    fi
+fi
+
+if [ -f "$__object/parameter/mode" ]
+then
+    mode="$( cat "$__object/parameter/mode" )"
+    options="$options --chmod=$mode"
+fi
+
+# IMPORTANT
+#
+# 1. we first dry-run rsync with change summary to find out
+#    if there are any changes and code generation is needed.
+# 2. normally, to get current state or target host, we run
+#    such operations in type explorers, but that's not
+#    possible due to how rsync works.
+# 3. redirecting output of dry-run to stderr to ease debugging.
+# 4. to understand how that cryptic regex works, please
+#    open rsync manpage and read about --itemize-changes.
+
 # shellcheck disable=SC2086
-echo rsync -a \
-    --no-owner --no-group \
-    -e \"${__remote_exec}\" \
-    -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"
+if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \
+    | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2
+then
+    exit 0
+fi
+
+echo "rsync $options $src $remote_user@$__target_host:$dst"
diff --git a/cdist/conf/type/__rsync/gencode-remote b/cdist/conf/type/__rsync/gencode-remote
deleted file mode 100755
index 074246af..00000000
--- a/cdist/conf/type/__rsync/gencode-remote
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-if [ -f "$__object/parameter/destination" ]; then
-    destination=$(cat "$__object/parameter/destination")
-else
-    destination="/$__object_id"
-fi
-
-ownergroup=""
-if [ -f "$__object/parameter/owner" ]; then
-    ownergroup=$(cat "$__object/parameter/owner")
-fi
-if [ -f "$__object/parameter/group" ]; then
-    ownergroup="${ownergroup}:$(cat "$__object/parameter/group")"
-fi
-
-if [ "$ownergroup" ]; then
-    echo chown -R "$ownergroup" "$destination"
-fi
diff --git a/cdist/conf/type/__rsync/man.rst b/cdist/conf/type/__rsync/man.rst
index 94b06d63..88019c92 100644
--- a/cdist/conf/type/__rsync/man.rst
+++ b/cdist/conf/type/__rsync/man.rst
@@ -3,112 +3,73 @@ cdist-type__rsync(7)
 
 NAME
 ----
-cdist-type__rsync - Mirror directories using rsync
+cdist-type__rsync - Mirror directories using ``rsync``
 
 
 DESCRIPTION
 -----------
-WARNING: This type is of BETA quality:
-
-- it has not been tested widely
-- interfaces *may* change
-- if there is a better approach to solve the problem -> the type may even vanish
-
-If you are fine with these constraints, please read on.
-
-
-This cdist type allows you to mirror local directories to the
-target host using rsync. Rsync will be installed in the manifest of the type.
-If group or owner are giveng, a recursive chown will be executed on the 
-target host.
-
-A slash will be appended to the source directory so that only the contents
-of the directory are taken and not the directory name itself.
+The purpose of this type is to bring power of ``rsync`` into ``cdist``.
 
 
 REQUIRED PARAMETERS
 -------------------
 source
-    Where to take files from
+   Source directory in local machine.
+   If source is directory, slash (``/``) will be added to source and destination paths.
 
 
 OPTIONAL PARAMETERS
 -------------------
-group
-   Group to chgrp to.
+destination
+   Destination directory. Defaults to ``$__object_id``.
 
 owner
-   User to chown to.
+   Will be passed to ``rsync`` as ``--chown=OWNER``.
+   Read ``rsync(1)`` for more details.
 
-destination
-    Use this as the base destination instead of the object id
+group
+   Will be passed to ``rsync`` as ``--chown=:GROUP``.
+   Read ``rsync(1)`` for more details.
+
+mode
+   Will be passed to ``rsync`` as ``--chmod=MODE``.
+   Read ``rsync(1)`` for more details.
+
+options
+   Defaults to ``--recursive --links --perms --times``.
+   Due to `bug in Python's argparse<https://bugs.python.org/issue9334>`_, value must be prefixed with ``\``.
 
 remote-user
-    Use this user instead of the default "root" for rsync operations.
+   Defaults to ``root``.
 
 
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
-rsync-opts
-    Use this option to give rsync options with.
-    See rsync(1) for available options.
-    Only "--" options are supported.
-    Write the options without the beginning "--"
-    Can be specified multiple times.
-
-
-MESSAGES
---------
-NONE
+option
+   Pass additional options to ``rsync``.
+   See ``rsync(1)`` for all possible options.
+   Due to `bug in Python's argparse<https://bugs.python.org/issue9334>`_, value must be prefixed with ``\``.
 
 
 EXAMPLES
 --------
-
 .. code-block:: sh
 
-    # You can use any source directory
-    __rsync /tmp/testdir \
-        --source /etc
-
-    # Use source from type
-    __rsync /etc \
-        --source "$__type/files/package"
-
-    # Allow multiple __rsync objects to write to the same dir
-    __rsync mystuff \
-        --destination /usr/local/bin \
-        --source "$__type/files/package"
-
-    __rsync otherstuff \
-        --destination /usr/local/bin \
-        --source "$__type/files/package2"
-
-    # Use rsync option --exclude
-    __rsync /tmp/testdir \
-        --source /etc \
-        --rsync-opts exclude=sshd_conf
-
-    # Use rsync with multiple options --exclude --dry-run
-    __rsync /tmp/testing \
-        --source /home/tester \
-        --rsync-opts exclude=id_rsa \
-        --rsync-opts dry-run
-
-
-SEE ALSO
---------
-:strong:`rsync`\ (1)
+    __rsync /var/www/example.com \
+        --owner root \
+        --group www-data \
+        --mode 'D750,F640' \
+        --source "$__files/example.com/www"
 
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+Ander Punnar <ander-at-kvlt-dot-ee>
 
 
 COPYING
 -------
-Copyright \(C) 2015 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
diff --git a/cdist/conf/type/__rsync/manifest b/cdist/conf/type/__rsync/manifest
index 9bd44c6d..64fa804e 100755
--- a/cdist/conf/type/__rsync/manifest
+++ b/cdist/conf/type/__rsync/manifest
@@ -1,21 +1,3 @@
 #!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
 __package rsync
diff --git a/cdist/conf/type/__rsync/parameter/default/options b/cdist/conf/type/__rsync/parameter/default/options
new file mode 100644
index 00000000..d967b110
--- /dev/null
+++ b/cdist/conf/type/__rsync/parameter/default/options
@@ -0,0 +1 @@
+--recursive --links --perms --times
diff --git a/cdist/conf/type/__rsync/parameter/optional b/cdist/conf/type/__rsync/parameter/optional
index ac2b2390..833e9bbe 100644
--- a/cdist/conf/type/__rsync/parameter/optional
+++ b/cdist/conf/type/__rsync/parameter/optional
@@ -1,4 +1,6 @@
 destination
-owner
 group
+mode
+options
+owner
 remote-user
diff --git a/cdist/conf/type/__rsync/parameter/optional_multiple b/cdist/conf/type/__rsync/parameter/optional_multiple
index fdb7cd88..01925a15 100644
--- a/cdist/conf/type/__rsync/parameter/optional_multiple
+++ b/cdist/conf/type/__rsync/parameter/optional_multiple
@@ -1 +1 @@
-rsync-opts
+option

From 46b5c24cd240cd9006d93cfcc12a4d81b46a5238 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 18 Jul 2021 16:25:00 +0300
Subject: [PATCH 655/743] use $__remote_exec for RSYNC_RSH

---
 cdist/conf/type/__rsync/gencode-local | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index 612d237e..e9f3c131 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -90,6 +90,8 @@ fi
 # 4. to understand how that cryptic regex works, please
 #    open rsync manpage and read about --itemize-changes.
 
+export RSYNC_RSH="$__remote_exec"
+
 # shellcheck disable=SC2086
 if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \
     | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2
@@ -97,4 +99,6 @@ then
     exit 0
 fi
 
+echo "export RSYNC_RSH='$__remote_exec'"
+
 echo "rsync $options $src $remote_user@$__target_host:$dst"

From 5229337611b7e7afb2597729a786c637a8bec1f6 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 18 Jul 2021 17:41:29 +0200
Subject: [PATCH 656/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 284293c1..f0746218 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __rsync: Rewrite (Ander Punnar)
+
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)

From de116661613002e91b54194ac8f0d5c3dd3eebbd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 18 Jul 2021 17:45:19 +0200
Subject: [PATCH 657/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index f0746218..ea51ac1b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __rsync: Rewrite (Ander Punnar)
+	* New type: __apt_pin (Daniel Fancsali)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 24c9406ea0f2c6c8edc87e5bbc1be25b5e8e1572 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 19 Jul 2021 12:13:23 +0200
Subject: [PATCH 658/743] [explorer/os_version] Convert Devuan ceres to version
 number

Conversion of Devuan ceres to version numbers is done based on Devuan codenames.
The version number is the version number of the final release - 0.01.

Analogous to Debian.
---
 cdist/conf/explorer/os_version | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 3b02dedd..6c94915c 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -1,6 +1,7 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,12 +18,11 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
 # All os variables are lower case
 #
-#
 
-case "$("$__explorer/os")" in
+case $("${__explorer:?}/os")
+in
    amazon)
       cat /etc/system-release
    ;;
@@ -59,7 +59,23 @@ case "$("$__explorer/os")" in
       esac
    ;;
    devuan)
-      cat /etc/devuan_version
+      devuan_version=$(cat /etc/devuan_version)
+      case ${devuan_version}
+      in
+         (*/ceres)
+            # ceres versions don't have a number, so we decode by codename:
+            case ${devuan_version}
+            in
+               (chimaera/ceres) echo 3.99 ;;
+               (beowulf/ceres) echo 2.99 ;;
+               (ascii/ceres) echo 1.99 ;;
+               (*) exit 1
+            esac
+            ;;
+         (*)
+            echo "${devuan_version}"
+            ;;
+      esac
    ;;
    fedora)
       cat /etc/fedora-release

From fbc9594729ece74b2f8612fe8046ddea966e8c05 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 20 Jul 2021 06:38:46 +0200
Subject: [PATCH 659/743] ++changelog

---
 docs/changelog | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/docs/changelog b/docs/changelog
index ea51ac1b..55d7cb73 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)
+	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
@@ -146,7 +147,7 @@ next:
 	* Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham)
 	* Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham)
 	* Configuration: Add notes to cdist.cfg.skeleton (Evil Ham)
-	* Explorers cpu_cores, memory: Improve *BSD support (Evil Ham)
+	* Explorers cpu_cores, memory: Improve BSD support (Evil Ham)
 	* Core: Remove debug logging noise (Evil Ham)
 
 6.5.4: 2020-04-11
@@ -211,7 +212,7 @@ next:
 	* Documentation: PreOS english nitpicking (Evil Ham)
 	* Documentation: Add installing from source with signature verification (Darko Poljak)
 	* Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak)
-	* Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham)
+	* Type __start_on_boot: Docs: remove unsupported BSD claim (Evil Ham)
 	* New type: __openldap_server (Evil Ham)
 
 6.2.0: 2019-11-30
@@ -1070,9 +1071,9 @@ next:
 	* Removed type __removeline (replaced by __line) (Nico Schottelius)
 	* Type __directory: Parameter --parents and --recursive are now boolean (Nico Schottelius)
 	* Type __package_apt, __package_luarocks, __package_opkg,
-		__package_pacman, __package_pkg_freebsd, __package_pkg_openbsd,
-		__package_rubygem, __package_yum, __process:
-			Parameter state accepts only "present" and "absent" (Nico Schottelius)
+	  __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd,
+	  __package_rubygem, __package_yum, __process:
+	  Parameter state accepts only "present" and "absent" (Nico Schottelius)
 	* Dist: Initial support for pypi packaging (Nico Schottelius)
 
 2.0.15: 2012-11-02

From c7daaabc6c28cbfaebaf6c620cc3cac130684d2c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 20 Jul 2021 09:03:16 +0200
Subject: [PATCH 660/743] [docs] Bump copyright year to 2021

---
 docs/src/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/conf.py b/docs/src/conf.py
index 47765413..a3dfafca 100644
--- a/docs/src/conf.py
+++ b/docs/src/conf.py
@@ -56,7 +56,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'cdist'
-copyright = 'ungleich GmbH 2020'
+copyright = 'ungleich GmbH 2021'
 # author = 'Darko Poljak'
 
 # The version info for the project you're documenting, acts as replacement for

From fed01ded83b217a3ad147f2403ca863be373eeaa Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 22 Jul 2021 11:17:41 +0200
Subject: [PATCH 661/743] [cdist.log] Define custom log functions on
 logging.Logger

Define out custom logger functions on logging.Logger so that they are passed on
to all other loggers.

Also, the logger functions need to take a self argument so that they can log on
the corrent Logger.
---
 cdist/log.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/cdist/log.py b/cdist/log.py
index 113f3b4c..62e457fe 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -36,25 +36,27 @@ import threading
 logging.OFF = logging.CRITICAL + 10  # disable logging
 logging.addLevelName(logging.OFF, 'OFF')
 
+
 logging.VERBOSE = logging.INFO - 5
 logging.addLevelName(logging.VERBOSE, 'VERBOSE')
 
 
-def _verbose(msg, *args, **kwargs):
-    logging.log(logging.VERBOSE, msg, *args, **kwargs)
+def _verbose(self, msg, *args, **kwargs):
+    self.log(logging.VERBOSE, msg, args, **kwargs)
 
 
-logging.verbose = _verbose
+logging.Logger.verbose = _verbose
+
 
 logging.TRACE = logging.DEBUG - 5
 logging.addLevelName(logging.TRACE, 'TRACE')
 
 
-def _trace(msg, *args, **kwargs):
-    logging.log(logging.TRACE, msg, *args, **kwargs)
+def _trace(self, msg, *args, **kwargs):
+    self.log(logging.TRACE, msg, *args, **kwargs)
 
 
-logging.trace = _trace
+logging.Logger.trace = _trace
 
 
 class CdistFormatter(logging.Formatter):

From 71fee1fd6b4874abffee5173a96c31842a9583bd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 23 Jul 2021 08:06:45 +0200
Subject: [PATCH 662/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 55d7cb73..1273e432 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
+	* Core: Fix logging bug (Dennis Camera)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 67bcc6cae38e2d480c3336b6a4d59d01196190e3 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 24 Jul 2021 02:37:58 +0200
Subject: [PATCH 663/743] Improve Makefile compatibility and build docs

We now use `$(MAKE)` for subsequent calls to `make`.
This means that systems that do not default to GNU make can run `gmake man` and
produce the man pages.

While there also document a dependency on the rtd theme for sphinx.
---
 Makefile                   | 6 +++---
 docs/src/cdist-install.rst | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 3712511c..89286310 100644
--- a/Makefile
+++ b/Makefile
@@ -35,9 +35,9 @@ DOCS_SRC_DIR=./docs/src
 SPEECHDIR=./docs/speeches
 TYPEDIR=./cdist/conf/type
 
-SPHINXM=make -C $(DOCS_SRC_DIR) man
-SPHINXH=make -C $(DOCS_SRC_DIR) html
-SPHINXC=make -C $(DOCS_SRC_DIR) clean
+SPHINXM=$(MAKE) -C $(DOCS_SRC_DIR) man
+SPHINXH=$(MAKE) -C $(DOCS_SRC_DIR) html
+SPHINXC=$(MAKE) -C $(DOCS_SRC_DIR) clean
 
 ################################################################################
 # Manpages
diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst
index 18863145..390ab9ec 100644
--- a/docs/src/cdist-install.rst
+++ b/docs/src/cdist-install.rst
@@ -12,7 +12,7 @@ This is the machine from which you will configure target hosts.
  * /bin/sh: A POSIX like shell (for instance bash, dash, zsh)
  * Python >= 3.5
  * SSH client
- * sphinx (for building html docs and/or the man pages)
+ * sphinx with the rtd theme (for building html docs and/or the man pages)
 
 Target Hosts
 ~~~~~~~~~~~~

From cb8695cc88478640d7e76992438d90d3d6a68a90 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 24 Jul 2021 12:53:39 +0200
Subject: [PATCH 664/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 1273e432..181c4139 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* New type: __apt_pin (Daniel Fancsali)
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
 	* Core: Fix logging bug (Dennis Camera)
+	* Build: Improve Makefile compatibility (Evilham)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 4156fea9001aa267c8b173247cada2f919511c1b Mon Sep 17 00:00:00 2001
From: Joachim Desroches <joachim.desroches@epfl.ch>
Date: Wed, 28 Jul 2021 12:56:39 +0200
Subject: [PATCH 665/743] [filesystem] Add ubuntu as supported distribution.

---
 cdist/conf/type/__filesystem/explorer/lsblk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk
index 9be3c575..d376c09f 100644
--- a/cdist/conf/type/__filesystem/explorer/lsblk
+++ b/cdist/conf/type/__filesystem/explorer/lsblk
@@ -27,7 +27,7 @@ else
 fi
 
 case "$os" in
-    alpine|centos|fedora|redhat|suse|gentoo)
+    alpine|centos|fedora|gentoo|redhat|suse|ubuntu)
         if [ ! -x "$(command -v lsblk)" ]; then
             echo "lsblk is required for __filesystem type" >&2
             exit 1

From 542674dae81ab03a4de2c4ad0b2eaf264ee2c442 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 30 Jul 2021 10:30:33 +0200
Subject: [PATCH 666/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 181c4139..9fc10b20 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
 	* Core: Fix logging bug (Dennis Camera)
 	* Build: Improve Makefile compatibility (Evilham)
+	* Type __filesystem: Support ubuntu (Joachim Desroches)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 53334fb4eb311550af7f5b73f279a2e86fa1c504 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 4 Aug 2021 19:50:10 +0200
Subject: [PATCH 667/743] [explorer/os_version] Fix for FreeBSD < 10.0 (again)

---
 cdist/conf/explorer/os_version | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 6c94915c..cc976608 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -89,7 +89,14 @@ in
    freebsd)
       # Apparently uname -r is not a reliable way to get the patch level.
       # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
-      freebsd-version
+      if command -v freebsd-version >/dev/null 2>&1
+      then
+         # get userland version
+         freebsd-version -u
+      else
+         # fallback to kernel release for FreeBSD < 10.0
+         uname -r
+      fi
    ;;
    *bsd|solaris)
       uname -r

From e108cbc205cb5e7ac0d2e07b82cef4c83eaa285f Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Tue, 3 Aug 2021 13:20:43 +0200
Subject: [PATCH 668/743] [explorer/os_version] Ubuntu: fall back to
 os-release/lsb-release files

---
 cdist/conf/explorer/os_version | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 6c94915c..e70fe7f4 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -21,6 +21,17 @@
 # All os variables are lower case
 #
 
+rc_getvar() {
+   awk -F= -v varname="$2" '
+      function unquote(s) {
+         if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/)
+            return substr(s, 2, length(s) - 2)
+         else
+            return s
+      }
+      $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1"
+}
+
 case $("${__explorer:?}/os")
 in
    amazon)
@@ -114,7 +125,20 @@ in
       fi
    ;;
    ubuntu)
-      lsb_release -sr
+      if command -v lsb_release >/dev/null 2>&1
+      then
+         lsb_release -sr
+      elif test -r /usr/lib/os-release
+      then
+         # fallback to /usr/lib/os-release if lsb_release is not present (like
+         # on minimized Ubuntu installations)
+         rc_getvar /usr/lib/os-release VERSION_ID
+      elif test -r /etc/lsb-release
+      then
+         # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old
+         # versions without /usr/lib/os-release.
+         rc_getvar /etc/lsb-release DISTRIB_RELEASE
+      fi
    ;;
    alpine)
        cat /etc/alpine-release

From 83fe6e9f5b2537db73d5c6a142b6d24eef75ac58 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Tue, 3 Aug 2021 19:26:55 +0200
Subject: [PATCH 669/743] [explorer/memory] Fix conversion of large numbers (>=
 2GiB)

At least mawk uses scientific notation when using print for
numbers >=2^31 (INT_MAX of a signed 32-bit int).

`printf "%.f\n"` works around this.
---
 cdist/conf/explorer/memory | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory
index 63aba9c6..c6d113cf 100755
--- a/cdist/conf/explorer/memory
+++ b/cdist/conf/explorer/memory
@@ -27,19 +27,18 @@
 str2bytes() {
 	awk -F' ' '
 	$2 ==   "B" || !$2 { print $1 }
-	$2 ==  "kB" { print $1 * 1000 }
-	$2 ==  "MB" { print $1 * 1000 * 1000 }
-	$2 ==  "GB" { print $1 * 1000 * 1000 * 1000 }
-	$2 ==  "TB" { print $1 * 1000 * 1000 * 1000 * 1000 }
-	$2 == "kiB" { print $1 * 1024 }
-	$2 == "MiB" { print $1 * 1024 * 1024 }
-	$2 == "GiB" { print $1 * 1024 * 1024 * 1024 }
-	$2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }'
+	$2 ==  "kB" { printf "%.f\n", ($1 * 1000) }
+	$2 ==  "MB" { printf "%.f\n", ($1 * 1000 * 1000) }
+	$2 ==  "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) }
+	$2 ==  "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) }
+	$2 == "kiB" { printf "%.f\n", ($1 * 1024) }
+	$2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) }
+	$2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) }
+	$2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }'
 }
 
 bytes2kib() {
-	set -- "$(cat)"
-	test "$1" -gt 0 && echo $(($1 / 1024))
+	awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }'
 }
 
 

From a7d6481a7ddc7cb72b1a55bfca7fdfed20514a62 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Mon, 2 Aug 2021 21:23:50 +0200
Subject: [PATCH 670/743] [type/__update_alternatives] Secure cdist-defined
 environment variables with :?

---
 .../__update_alternatives/explorer/alternatives     |  2 +-
 cdist/conf/type/__update_alternatives/explorer/link |  6 +++---
 .../type/__update_alternatives/explorer/path_is     |  4 ++--
 .../explorer/path_should_state                      |  2 +-
 .../conf/type/__update_alternatives/gencode-remote  | 13 ++++++-------
 5 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives
index 34aaca56..ecc62f4b 100755
--- a/cdist/conf/type/__update_alternatives/explorer/alternatives
+++ b/cdist/conf/type/__update_alternatives/explorer/alternatives
@@ -1,4 +1,4 @@
 #!/bin/sh -e
 
-update-alternatives --display "$__object_id" 2>/dev/null \
+update-alternatives --display "${__object_id:?}" 2>/dev/null \
     | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link
index 6519e7c2..c6fd1c98 100755
--- a/cdist/conf/type/__update_alternatives/explorer/link
+++ b/cdist/conf/type/__update_alternatives/explorer/link
@@ -18,12 +18,12 @@ for altdir in \
     /var/lib/dpkg/alternatives \
     /var/lib/alternatives
 do
-    if [ ! -f "$altdir/$__object_id" ]
+    if [ ! -f "$altdir/${__object_id:?}" ]
     then
         continue
     fi
 
-    link="$( awk 'NR==2' "$altdir/$__object_id" )"
+    link="$( awk 'NR==2' "$altdir/${__object_id:?}" )"
 
     if [ -n "$link" ]
     then
@@ -33,7 +33,7 @@ done
 
 if [ -z "$link" ]
 then
-    echo "unable to get link for $__object_id" >&2
+    echo "unable to get link for ${__object_id:?}" >&2
     exit 1
 fi
 
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
index fc304d5d..a24bd40e 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_is
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -1,11 +1,11 @@
 #!/bin/sh -e
 
-path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \
+path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \
     | awk '/link currently points to/ {print $5}' )"
 
 if [ -z "$path_is" ]
 then
-    echo "unable to get current path for $__object_id" >&2
+    echo "unable to get current path for ${__object_id:?}" >&2
     exit 1
 fi
 
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state
index 59e015c5..b74a7ee8 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_should_state
+++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state
@@ -1,6 +1,6 @@
 #!/bin/sh -e
 
-if [ -f "$( cat "$__object/parameter/path" )" ]
+if [ -f "$( cat "${__object:?}/parameter/path" )" ]
 then
     echo 'present'
 else
diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index e393cdef..13666805 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -18,26 +18,25 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 
-path_is="$( cat "$__object/explorer/path_is" )"
+path_is="$( cat "${__object:?}/explorer/path_is" )"
 
-path_should="$( cat "$__object/parameter/path" )"
+path_should="$( cat "${__object:?}/parameter/path" )"
 
 if [ "$path_is" = "$path_should" ]
 then
     exit 0
 fi
 
-if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ]
+if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \
+    && [ -z "${__cdist_dry_run+dry run}" ]
 then
     echo "$path_should does not exist in target" >&2
     exit 1
 fi
 
-name="$__object_id"
+name=${__object_id:?}
 
-alternatives="$( cat "$__object/explorer/alternatives" )"
-
-if ! echo "$alternatives" | grep -Fxq "$path_should"
+if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives"
 then
     if [ ! -f "$__object/parameter/install" ]
     then

From 0b3b47396f2aafa377e3d5d9a13f51ace2303d41 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Mon, 2 Aug 2021 21:25:08 +0200
Subject: [PATCH 671/743] [type/__update_alternatives] dry-run fixes

---
 cdist/conf/type/__update_alternatives/explorer/link |  5 ++++-
 .../type/__update_alternatives/explorer/path_is     |  5 ++++-
 .../conf/type/__update_alternatives/gencode-remote  | 13 ++++++++-----
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link
index c6fd1c98..d1087c75 100755
--- a/cdist/conf/type/__update_alternatives/explorer/link
+++ b/cdist/conf/type/__update_alternatives/explorer/link
@@ -31,8 +31,11 @@ do
     fi
 done
 
-if [ -z "$link" ]
+if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then
+    # NOTE: ignore error for dry-runs because a package providing the link
+    #       might be managed by another cdist object (which wasn't executed,
+    #       because dry run…).
     echo "unable to get link for ${__object_id:?}" >&2
     exit 1
 fi
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
index a24bd40e..9208df7b 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_is
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -3,8 +3,11 @@
 path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \
     | awk '/link currently points to/ {print $5}' )"
 
-if [ -z "$path_is" ]
+if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then
+    # NOTE: ignore error for dry-runs because a package providing the
+    #       alternative might be managed by another cdist object (which
+    #       wasn't executed, because dry run…).
     echo "unable to get current path for ${__object_id:?}" >&2
     exit 1
 fi
diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index 13666805..e91ea78f 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -38,16 +38,19 @@ name=${__object_id:?}
 
 if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives"
 then
-    if [ ! -f "$__object/parameter/install" ]
+    if [ -f "${__object:?}/parameter/install" ]
     then
+        link="$( cat "${__object:?}/explorer/link" )"
+        echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
+    elif [ -z "${__cdist_dry_run+dry run}" ]
+    then
+        # NOTE: ignore error for dry-runs because a package providing the link
+        #       to be installed might be managed by another cdist object (which
+        #       wasn't executed, because dry run…).
         echo "$path_should is not in $name alternatives." >&2
         echo 'Please install missing packages or use --install to add path to alternatives.' >&2
         exit 1
     fi
-
-    link="$( cat "$__object/explorer/link" )"
-
-    echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
 fi
 
 echo "update-alternatives --set '$name' '$path_should'"

From bbcc81a9841f2619e1b9e13b25a941337489a681 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 4 Aug 2021 21:44:04 +0200
Subject: [PATCH 672/743] [type/__update_alternatives] Fix for non-English
 locales

Since update-alternatives(1) is localized, screen scraping its output breaks
if the locale is set to non-English.
---
 cdist/conf/type/__update_alternatives/explorer/alternatives | 4 ++--
 cdist/conf/type/__update_alternatives/explorer/path_is      | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives
index ecc62f4b..bb1619a9 100755
--- a/cdist/conf/type/__update_alternatives/explorer/alternatives
+++ b/cdist/conf/type/__update_alternatives/explorer/alternatives
@@ -1,4 +1,4 @@
 #!/bin/sh -e
 
-update-alternatives --display "${__object_id:?}" 2>/dev/null \
-    | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
+LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \
+| awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
index 9208df7b..5cf4fa4b 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_is
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -1,7 +1,8 @@
 #!/bin/sh -e
 
-path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \
-    | awk '/link currently points to/ {print $5}' )"
+path_is=$(
+    LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \
+    | awk '/link currently points to/ { print $5 }')
 
 if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then

From 2a0c073d4021206c7459015cefaba218004235ce Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 4 Aug 2021 21:54:17 +0200
Subject: [PATCH 673/743] [explorer/os_version] Fix for legacy Mac OS X
 versions

---
 cdist/conf/explorer/os_version | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 3b02dedd..96eca1ee 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -68,7 +68,8 @@ case "$("$__explorer/os")" in
       cat /etc/gentoo-release
    ;;
    macosx)
-      sw_vers -productVersion
+      # NOTE: Legacy versions (< 10.3) do not support options
+      sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }'
    ;;
    freebsd)
       # Apparently uname -r is not a reliable way to get the patch level.

From 3ae5a606ca7182ec7fe13134670400506e198ec1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 5 Aug 2021 10:27:51 +0200
Subject: [PATCH 674/743] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 9fc10b20..f9409d7e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,10 @@ next:
 	* Core: Fix logging bug (Dennis Camera)
 	* Build: Improve Makefile compatibility (Evilham)
 	* Type __filesystem: Support ubuntu (Joachim Desroches)
+	* Explorer os_version: Fall back to os-release/lsb-release file on Ubuntu (Dennis Camera)
+	* Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera)
+	* Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera)
+	* Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From edcac70b2a99ce532dd5bca9a0b8fc5bf6dc5148 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 21 Jul 2021 13:22:34 +0200
Subject: [PATCH 675/743] [explorer/machine_type] Reimplement

---
 cdist/conf/explorer/machine_type | 986 ++++++++++++++++++++++++++++---
 1 file changed, 904 insertions(+), 82 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 1c84f4d7..29f98849 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -1,8 +1,6 @@
-#!/bin/sh
+#!/bin/sh -e
 #
-# 2014 Daniel Heule  (hda at sfs.biz)
-# 2014 Thomas Oettli (otho at sfs.biz)
-# 2020 Evilham (contact at evilham.com)
+# 2021 Dennis Camera (cdist at dtnr.ch)
 #
 # This file is part of cdist.
 #
@@ -19,91 +17,915 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+# This explorer tries to determine what type of machine the target to be
+# configured is (container, virtual machine, bare-metal).
+#
+# It will print one line for each layer it can detect.
+# The format of all lines is: TYPE[ VERB VENDOR]
+#
+# VERB does not have a special meaning, it is just for better readability.
+#
+# e.g.
+# container on lxc
+# virtual by kvm-spapr
+#
+# The third word of each line can be composed of different parts concatenated with a `-'
+# (minus) character, with each component being a specification of the previous,
+# e.g.:
+#  - lxc-libvirt             (LXC container, managed by libvirt)
+#  - lpar-s390 / lpar-power  (LPAR running on IBM S/390 or POWER, respectively)
+#  - xen-hvm / xen-pv        (Xen HVM vs para-virtualization)
+#
+# If this explorer cannot determine any information it will print nothing.
+#
 
-os=$("$__explorer/os")
+# Add /sbin and /usr/sbin to the path so we can find system
+# binaries like dmidecode.
+PATH=$(getconf PATH 2>/dev/null) || PATH='/usr/bin:/bin'
+PATH="/sbin:/usr/sbin:${PATH}"
+export PATH
 
-vendor_string_to_machine_type() {
-    for vendor in vmware bochs kvm qemu virtualbox bhyve; do
-        if echo "${1}" | grep -q -i "${vendor}"; then
-            if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then
-                vendor="kvm"
-            fi
-            echo "virtual_by_${vendor}"
-            exit
-        fi
-    done
+arch=$(uname -m | sed -e 's/i.86/i386/' -e 's/arm.*/arm/')
+uname_s=$(uname -s)
+
+
+is_command() { command -v "$1" >/dev/null 2>&1; }
+
+is_oneof() (
+	x=$1; shift
+	for y
+	do
+		test "${x}" = "${y}" || continue
+		return 0
+	done
+	return 1
+)
+
+tolower() { LC_ALL=C tr '[:upper:]' '[:lower:]'; }
+
+# shellcheck disable=SC2086
+glob_exists() { set -- $1; test -e "$1"; }
+
+get_dmi_field() {
+	if is_oneof "${uname_s}" NetBSD
+	then
+		case $1
+		in
+			(system-manufacturer) _mib=machdep.dmi.system-vendor ;;
+			(system-product-name) _mib=machdep.dmi.system-product ;;
+			(system-version|system-uuid) _mib=machdep.dmi.$1 ;;
+			(bios-vendor|bios-version) _mib=machdep.dmi.$1 ;;
+			(biod-release-date) _mib=machdep.dmi.bios-date ;;
+			(*) _mib= ;;
+		esac
+
+		test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return
+	fi
+
+	if is_command dmidecode
+	then
+		dmidecode -s "$1"
+	elif test -d "${dmi_sysfs-}"
+	then
+		case $1
+		in
+			(system-manufacturer) _filename=sys_vendor ;;
+			(system-product-name) _filename=product_name ;;
+			(*) _filename=$(echo "$1" | tr - _) ;;
+		esac
+		if test -r "${dmi_sysfs-}/${_filename}"
+		then
+			cat "${dmi_sysfs}/${_filename}"
+		fi
+		unset _filename
+	elif test "${uname_s}" = OpenBSD
+	then
+		# NOTE: something similar to system-manufacutrer and system-product-name
+		#       is available on OpenBSD in sysctl
+		case $1
+		in
+			(system-manufacturer) _mib=hw.vendor ;;
+			(system-product-name) _mib=hw.product ;;
+			(*) _mib= ;;
+		esac
+
+		test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return
+	fi
+
+	return 1
 }
 
-case "$os" in
-    "freebsd")
-        # FreeBSD does not have /proc/cpuinfo even when procfs is used.
-        # Instead there is a sysctl kern.vm_guest.
-        # Which is 'none' if physical, else the virtualisation.
-        vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)"
-        if [ -n "${vm_guest}" ]; then
-            if [ "${vm_guest}" = "none" ]; then
-                echo "physical"
-                exit
-            fi
-            echo "virtual_by_${vm_guest}"
-            exit
-        fi
-    ;;
+has_cpuinfo() { test -e /proc/cpuinfo; }
 
-    "openbsd")
-        # OpenBSD can also use the sysctl's: hw.vendor or hw.product.
-        # Note we can be reasonably sure about a machine being virtualised
-        # as long as we can identify the virtualisation technology.
-        # But not so much about it being physical...
-        # Patches are welcome / reach out if you have better ideas.
-        for sysctl in hw.vendor hw.product; do
-            # This exits if we can make a reasonable judgement
-            vendor_string_to_machine_type "$(sysctl -n "${sysctl}")"
-        done
-    ;;
+get_sysctl() {
+	is_command sysctl && sysctl -n "$1" 2>/dev/null
+}
 
-    *)
-        # Defaulting to linux for compatibility with previous cdist behaviour
 
-        if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then
-            echo openvz
-            exit
-        fi
+# Check for container
 
-        if [ -e "/proc/1/environ" ] &&
-            tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then
-            echo lxc
-            exit
-        fi
+has_ct_pid_1() {
+	test -r /run/systemd/container -o -r /proc/1/environ
+}
 
-        if [ -r /proc/cpuinfo ]; then
-            # this should only exist on virtual guest machines,
-            # tested on vmware, xen, kvm, bhyve
-            if grep -q "hypervisor" /proc/cpuinfo; then
-                # this file is aviable in xen guest systems
-                if [ -r /sys/hypervisor/type ]; then
-                    if grep -q -i "xen" /sys/hypervisor/type; then
-                        echo virtual_by_xen
-                        exit
-                    fi
-                else
-                    for vendor_file in /sys/class/dmi/id/product_name \
-                                       /sys/class/dmi/id/sys_vendor \
-                                       /sys/class/dmi/id/chasis_vendor; do
-                        if [ -r ${vendor_file} ]; then
-                            # This exits if we can make a reasonable judgement
-                            vendor_string_to_machine_type "$(cat "${vendor_file}")"
-                        fi
-                    done
-                fi
-                echo "virtual_by_unknown"
-                exit
-            else
-                echo "physical"
-                exit
-            fi
-        fi
-    ;;
-esac
+translate_container_name() {
+	case $1
+	in
+		('lxc')
+			echo lxc ;;
+		('lxc-libvirt')
+			echo lxc-libvirt ;;
+		('podman')
+			echo podman ;;
+		('systemd-nspawn')
+			echo systemd_nspawn ;;
+		(*)
+			return 1 ;;
+	esac
+	return 0
+}
 
-echo "unknown"
+check_ct_pid_1() {
+	if test -r /run/systemd/container
+	then
+		translate_container_name "$(head -n1 /run/systemd/container)" \
+		&& return 0
+	fi
+
+	if test -r /proc/1/environ
+	then
+		translate_container_name "$(
+			LC_ALL=C tr '\000' '\n' </proc/1/environ \
+			| sed -n -e 's/^container=//p')" \
+		&& return 0
+	fi
+
+	return 1
+}
+
+has_ct_cgroup() {
+	test -r /proc/self/cgroup
+}
+
+check_ct_cgroup() {
+	if grep -q -e ':/docker/' /proc/self/cgroup
+	then
+		echo docker
+	elif grep -q -e ':/lxc/.*\.libvirt-lxc$' /proc/self/cgroup
+	then
+		echo libvirt-lxc
+	elif grep -q -F '/libpod-' /proc/self/cgroup
+	then
+		echo podman
+	else
+		return 1
+	fi
+}
+
+check_ct_files() {
+	if test -e /.dockerenv -o -e /run/.dockerenv -o -e /.dockerinit
+	then
+		echo docker
+	elif test -f /run/.containerenv
+	then
+		# https://github.com/containers/podman/issues/3586#issuecomment-661918679
+		echo podman
+	elif test -d /proc/vz -a ! -d /proc/bc -a -f /proc/user_beancounters \
+		|| test -d /.cpt_hardlink_dir_*/
+	then
+		# Virtuozzo / OpenVZ
+		#
+		# /proc/vz: always exists if OpenVZ kernel is running
+		# /proc/bc: exists only on node (not inside container)
+		# /proc/user_beancounters - exists inside container
+		#
+		# /.cpt_hardlink_dir_*/ may be (?) created by vzctl:
+		# https://github.com/kolyshkin/vzctl/commit/09e974f
+
+		echo openvz
+	elif grep -qie 'Microsoft\|WSL' /proc/sys/kernel/osrelease /proc/version 2>/dev/null
+	then
+		# https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364
+		echo wsl
+	elif test -d /var/.cagefs
+	then
+		# https://docs.cloudlinux.com/cloudlinux_os_components/#cagefs
+		# CageFS is not "really" a container, but it isn't a chroot either.
+		echo cagefs
+	elif test -e /proc/self/status && grep -q -e '^VxID: [0-9]\{1,\}' /proc/self/status
+	then
+		# Linux-VServer
+		if grep -q -x -F 'VxID: 0' /proc/self/status
+		then
+			# host
+			return 1
+		else
+			# guest
+			echo linux_vserver
+		fi
+	else
+		return 1
+	fi
+}
+
+check_ct_os_specific() (
+	if jailed=$(get_sysctl security.jail.jailed) && test "${jailed}" = 1
+	then
+		# FreeBSD jail
+		echo jail
+		return 0
+	fi
+
+	if is_command zonename && test "$(zonename)" != global
+	then
+		# Solaris zone
+		echo zone
+		return 0
+	fi
+
+	return 1
+)
+
+
+# Check for hypervisor
+
+guess_hypervisor_from_cpu_model() {
+	case $1
+	in
+		(*\ KVM\ *)
+			echo kvm ;;
+		(*\ QEMU\ *|QEMU\ *)
+			echo qemu ;;
+		(*)
+			return 1 ;;
+	esac
+}
+
+has_vm_cpuinfo() { has_cpuinfo; }
+
+check_vm_cpuinfo() {
+	if grep -q -F 'User Mode Linux' /proc/cpuinfo \
+		|| grep -q -F 'UML' /proc/cpuinfo
+	then
+		# User Mode Linux
+		echo uml
+	elif grep -q -e '^vendor_id.*: PowerVM Lx86' /proc/cpuinfo
+	then
+		# IBM PowerVM Lx86 (Linux/x86 emulator)
+		echo powervm_lx86
+	elif grep -q -e '^vendor_id.*: IBM/S390' /proc/cpuinfo
+	then
+		# IBM SystemZ (S/390)
+		if test -f /proc/sysinfo
+		then
+			if grep -q -e '^VM[0-9]* Control Program: KVM/Linux' /proc/sysinfo
+			then
+				echo kvm-s390
+				return 0
+			elif grep -q -e '^VM[0-9]* Control Program: z/VM' /proc/sysinfo
+			then
+				echo zvm
+				return 0
+			elif grep -q -e '^LPAR ' /proc/sysinfo
+			then
+				echo zvm-lpar
+				return 0
+			fi
+		fi
+		return 1
+	else
+		if grep -q -e '^model name.*:' /proc/cpuinfo
+		then
+			sed -n -e 's/^model name[^:]*: *//p' /proc/cpuinfo \
+			| while read -r _cpu_model
+			  do
+				  guess_hypervisor_from_cpu_model "${_cpu_model}"
+			  done \
+			| sort \
+			| uniq -c \
+			| awk '
+			  { if ($1 > most_c) { most_c = $1; most_s = $2 } }
+			  END {
+				if (most_s) print most_s
+				exit !most_s
+			  }' \
+			&& return 0
+		fi
+		return 1
+	fi
+}
+
+check_vm_arch_specific() {
+	case ${arch}
+	in
+		(ppc64|ppc64le)
+			# Check PPC64 LPAR, KVM
+
+			# example /proc/cpuinfo line indicating 'not baremetal'
+			# platform	: pSeries
+			#
+			# example /proc/ppc64/lparcfg systemtype line
+			# system_type=IBM pSeries (emulated by qemu)
+
+			if has_cpuinfo && grep -q -e 'platform.**pSeries' /proc/cpuinfo
+			then
+				if test -e /proc/ppc64/lparcfg
+				then
+					# Assume LPAR, now detect shared or dedicated
+					if grep -q -x -F 'shared_processor_mode=1' /proc/ppc64/lparcfg
+					then
+						echo powervm-shared
+						return 0
+					else
+						echo powervm-dedicated
+						return 0
+					fi
+				fi
+			fi
+			;;
+		(sparc*)
+			# Check for SPARC LDoms
+
+			if test -e /dev/mdesc
+			then
+				if test -d /sys/class/vlds/ctrl -a -d /sys/class/vlds/sp
+				then
+					# control LDom
+					return 1
+				else
+					# guest LDom
+					echo ldom-sparc
+				fi
+
+				# MDPROP=/usr/lib/ldoms/mdprop.py
+				# if test -x "${MDPROP}"
+				# then
+				# 	if test -n "$("${MDPROP}" -v iodevice device-type=pciex)"
+				# 	then
+				# 		echo ldoms-root
+				# 		echo ldoms-io
+				# 	elif test -n "$("${MDPROP}" -v iov-device vf-id=0)"
+				# 	then
+				# 		echo ldoms-io
+				# 	fi
+				# fi
+				return 0
+			fi
+			;;
+		(i?86|x86*|amd64|i86pc)
+			# Check CPUID
+			#
+			# Many fullvirt hypervisors give an indication through CPUID.  Use
+			# the virt-what helper program to get this information if available.
+
+			for CPUID_HELPER in \
+				$(command -v virt-what-cpuid-helper 2>/dev/null) \
+				/usr/lib/x86_64-*/virt-what-cpuid-helper \
+				/usr/lib/i?86-*/virt-what-cpuid-helper \
+				/usr/lib/virt-what/virt-what-cpuid-helper
+			do
+				if test -x "${CPUID_HELPER:?}"; then break; fi
+			done
+
+			if test -x "${CPUID_HELPER-}"
+			then
+				case $(command "${CPUID_HELPER}")
+				in
+					('bhyve bhyve ')
+						echo bhyve
+						;;
+					('LKVMLKVMLKVM')
+						echo lkvm
+						;;
+					('KVMKVMKVM')
+						echo kvm
+						;;
+					('TCGTCGTCGTCG')
+						echo qemu-tcg
+						;;
+					('Microsoft Hv')
+						# http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx
+						echo hyperv
+						;;
+					('OpenBSDVMM58')
+						# OpenBSD/VMM
+						echo openbsd_vmm
+						;;
+					('VMwareVMware')
+						# check added by Chetan Loke.
+						echo vmware
+						;;
+					('XenVMMXenVMM')
+						if has dmi
+						then
+							# https://access.redhat.com/solutions/222903
+							echo xen-hvm
+						else
+							echo xen-paravirt
+						fi
+						;;
+					(*)
+						return 1 ;;
+				esac
+				return 0
+			fi
+
+			unset CPUID_HELPER
+
+			# VMM CPUID flag denotes that this system is running under a VMM
+			if is_oneof "${uname_s}" Darwin
+			then
+				get_sysctl machdep.cpu.features | tr ' ' '\n' | grep -qixF VMM \
+				&& return 0
+			fi
+			if has_cpuinfo \
+				&& grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo
+			then
+				return 0
+			fi
+			;;
+		(ia64)
+			if test -d /sys/bus/xen -a ! -d /sys/bus/xen-backend
+			then
+				# PV-on-HVM drivers installed in a Xen guest
+				echo xen-hvm
+				return 0
+			fi
+			;;
+	esac
+	return 1
+}
+
+has_vm_dmi() {
+	# Check for various products in SMBIOS/DMI.
+	# Note that DMI doesn't exist on all architectures (only x86 and some ARM).
+	# On other architectures the $dmi variable will be empty.
+
+	if test -d /sys/class/dmi/id/
+	then
+		dmi_sysfs=/sys/class/dmi/id
+	elif test -d /sys/devices/virtual/dmi/id/
+	then
+		dmi_sysfs=/sys/devices/virtual/dmi/id
+	fi
+
+	# shellcheck disable=SC2015
+	{
+		is_command dmidecode \
+		&& (
+			# dmidecode needs to exit 0 and not print the No SMBIOS/DMI line
+			dmi_out=$(dmidecode 2>&1) \
+			&& ! printf '%s\n' "${dmi_out}" \
+				 | grep -qF 'No SMBIOS nor DMI entry point found, sorry.'
+			) \
+		|| test -d "${dmi_sysfs}"
+	}
+}
+
+check_vm_dmi() {
+	case $(get_dmi_field system-product-name)
+	in
+		(*.metal)
+			if test "$(get_dmi_field system-manufacturer)" = 'Amazon EC2'
+			then
+				# AWS EC2 bare metal -> no virtualisation
+				return 1
+			fi
+			;;
+		('BHYVE')
+			echo bhyve
+			return 0
+			;;
+		('Google Compute Engine')
+			echo gce
+			return 0
+			;;
+		('RHEV Hypervisor')
+			# Red Hat Enterprise Virtualization
+			echo rhev
+			return 0
+			;;
+		('KVM'|'Bochs'|'KVM Virtual Machine')
+			echo kvm
+			return 0
+			;;
+		('Parallels Virtual Platform')
+			echo parallels
+			return 0
+			;;
+		('VirtualBox')
+			echo virtualbox
+			return 0
+			;;
+		('VMware Virtual Platform')
+			echo vmware
+			return 0
+			;;
+	esac
+
+	case $(get_dmi_field system-manufacturer)
+	in
+		('Alibaba'*)
+			case $(get_dmi_field system-product-name)
+			in
+				('Alibaba Cloud ECS')
+					echo alibaba-ecs
+					;;
+				(*)
+					echo alibaba
+					;;
+			esac
+			return 0
+			;;
+		('Amazon EC2')
+			# AWS on bare-metal or KVM
+			echo aws-ec2
+			return 0
+			;;
+		('innotek GmbH'|'Oracle Corporation')
+			echo virtualbox
+			return 0
+			;;
+		('Joyent')
+			if test "$(get_dmi_field system-product-name)" = 'SmartDC HVM'
+			then
+				# SmartOS KVM
+				echo kvm-smartdc_hvm
+				return 0
+			fi
+			;;
+		('Microsoft Corporation'*)
+			if test "$(get_dmi_field system-product-name)" = 'Virtual Machine'
+			then
+				if test -e /proc/irq/7/hyperv \
+				|| expr "$(get_dmi_field bios-version)" : 'VRTUAL.*' >/dev/null
+				then
+					echo hyperv
+					return 0
+				fi
+
+				case $(get_dmi_field system-version)
+				in
+					(VPC[0-9]*|VS2005*|*[Vv]irtual*[Pp][Cc]*)
+						echo virtualpc
+						return 0
+						;;
+					(*)
+						echo hyperv
+						return 0
+						;;
+				esac
+			fi
+			;;
+		('Nutanix')
+			# Nutanix AHV. Similar to KVM.
+			if test "$(get_dmi_field system-product-name)" = 'AHV'
+			then
+				echo nutanix_ahv
+				return 0
+			fi
+			;;
+		('oVirt')
+			echo ovirt
+			return 0
+			;;
+		('Parallels Software International Inc.')
+			echo parallels
+			return 0
+			;;
+		('QEMU')
+			echo qemu
+			return 0
+			;;
+		('VMware, Inc.')
+			echo vmware
+			return 0
+			;;
+	esac
+
+	case $(get_dmi_field bios-vendor)
+	in
+		('Amazon EC2')
+			# AWS on bare-metal or KVM
+			echo aws-ec2
+			return 0
+			;;
+		('BHYVE')
+			echo bhyve
+			return 0
+			;;
+		('innotek GmbH')
+			echo virtualbox
+			return 0
+			;;
+		('Parallels Software International Inc.')
+			echo parallels
+			return 0
+			;;
+		('Xen')
+			if get_dmi_field bios-version | grep -q -e '\([0-9]\{1,\}\.\)\{2\}amazon'
+			then
+				# AWS on Xen
+				echo aws-xen
+				return 0
+			fi
+			;;
+	esac
+
+	return 1
+}
+
+check_vm_hyp_specific() {
+	if is_command vmware-checkvm && vmware-checkvm >/dev/null
+	then
+		# vmware-checkvm is provided by VMware's open-vm-tools
+		echo vmware
+		return 0
+	elif test -d /proc/xen
+	then
+		test -r /proc/xen/capabilities &&
+		if grep -q -F 'control_d' /proc/xen/capabilities 2>/dev/null
+		then
+			# Xen dom0
+			return 1
+		else
+			# Xen domU
+			echo xen
+			return 0
+		fi
+	fi
+	return 1
+}
+
+has_vm_dt() {
+	# OpenFirmware/Das U-Boot device-tree
+	test -d /proc/device-tree
+}
+
+check_vm_dt() {
+	case ${arch}
+	in
+		(arm|aarch64)
+			if test -r /proc/device-tree/hypervisor/compatible
+			then
+				if grep -q -F 'xen' /proc/device-tree/hypervisor/compatible
+				then
+					echo xen
+					return 0
+				elif grep -q -F 'vmware' /proc/device-tree/hypervisor/compatible
+				then
+					# e.g. VMware ESXi on ARM
+					echo vmware
+					return 0
+				fi
+			fi
+			if glob_exists /proc/device-tree/fw-cfg@*/compatible
+			then
+				# qemu,fw-cfg-mmio
+				sed -e 's/,.*$//' /proc/device-tree/fw-cfg@*/compatible | head -n1
+				return 0
+			fi
+			if grep -q -F 'dummy-virt' /proc/device-tree/compatible
+			then
+				echo lkvm
+				return 0
+			fi
+			;;
+		(ppc64*)
+			if test -d /proc/device-tree/hypervisor \
+				&& grep -qF 'linux,kvm' /proc/device-tree/hypervisor/compatible
+			then
+				# We are running as a spapr KVM guest on ppc64
+				echo kvm-spapr
+				return 0
+			fi
+			if test -r /proc/device-tree/ibm,partition-name \
+				&& test -r /proc/device-tree/hmc-managed\? \
+				&& test -r /proc/device-tree/chosen/qemu,graphic-width
+			then
+				echo powervm
+			fi
+			;;
+	esac
+	return 1
+}
+
+has_vm_sys_hypervisor() {
+	test -d /sys/hypervisor/
+}
+
+check_vm_sys_hypervisor() {
+	test -r /sys/hypervisor/type &&
+	case $(head -n1 /sys/hypervisor/type)
+	in
+		(xen)
+			# Ordinary kernel with pv_ops.	There does not seem to be
+			# enough information at present to tell whether this is dom0
+			# or domU.
+			echo xen
+			return 0
+			;;
+	esac
+	return 1
+}
+
+check_vm_os_specific() {
+	_hyp_generic=false
+
+	case ${uname_s}
+	in
+		(Darwin)
+			if hv_vmm_present=$(get_sysctl kern.hv_vmm_present) \
+				&& test "${hv_vmm_present}" -ne 0
+			then
+				_hyp_generic=true
+			fi
+			;;
+		(FreeBSD)
+			# FreeBSD does not have /proc/cpuinfo even when procfs is used.
+			# Instead there is a sysctl kern.vm_guest.
+			# Which is 'none' if physical, else the virtualisation.
+			vm_guest=$(get_sysctl kern.vm_guest | tolower) &&
+			case ${vm_guest}
+			in
+				(none) ;;
+				(generic) _hyp_generic=true ;;
+				(*)
+					# kernel could detect hypervisor
+					case ${vm_guest}
+					in
+						(hv) echo hyperv ;;
+						(vbox) echo virtualbox ;;
+						(*) echo "${vm_guest}" ;;
+					esac
+					return 0
+					;;
+			esac
+			;;
+		(NetBSD)
+			machdep_hv=$(get_sysctl machdep.hypervisor | tolower) &&
+			case ${machdep_hv}
+			in
+				(none) ;;
+				(generic) _hyp_generic=true ;;
+				(*)
+					# kernel could detect hypervisor
+					case ${machdep_hv}
+					in
+						(hyper-v) echo hyperv ;;
+						(xenhvm*) echo xen-hvm ;;
+						(xenpv*) echo xen-pv ;;
+						(xen*) echo xen ;;
+						(*) echo "${machdep_hv}" ;;
+					esac
+					return 0
+					;;
+			esac
+			;;
+		(OpenBSD)
+			if is_command hostctl && glob_exists /dev/pvbus[0-9]*
+			then
+				for _pvbus in /dev/pvbus[0-9]*
+				do
+					_h_out=$(hostctl -f "${_pvbus}" -t 2>/dev/null) || continue
+					case $(expr "${_h_out}" : '[^:]*: *\(.*\)$')
+					in
+						(KVM) echo kvm ;;
+						(Hyper-V) echo hyperv ;;
+						(VMware) echo vmware ;;
+						(Xen) echo xen ;;
+						(bhyve) echo bhyve ;;
+						(OpenBSD) echo openbsd_vmm ;;
+					esac
+					return 0
+				done
+			fi
+			;;
+		(SunOS)
+			diag_conf=$(prtdiag | sed -n -e 's/.*Configuration: *//p' -e '/^$/q')
+			# NOTE: Don't use -e or -F in Solaris grep
+			if printf '%s\n' "${diag_conf}" | grep -q -i QEMU
+			then
+				echo qemu
+				return 0
+			elif printf '%s\n' "${diag_conf}" | grep -q -i VMware
+			then
+				echo vmware
+				return 0
+			fi
+			;;
+		(Linux)
+			if is_command dmesg
+			then
+				while read -r line
+				do
+					case ${line}
+					in
+						('Booting paravirtualized kernel on ')
+							case $(expr "${line}" : '.* kernel on \(.*\)')
+							in
+								('Xen')
+									echo xen-pv; return 0 ;;
+								('bare hardware')
+									return 1 ;;
+							esac
+							;;
+						('Hypervisor detected')
+							case $(expr "${line}" : '.*: *\(.*\)')
+							in
+								('ACRN')
+									echo acrn ;;
+								('Jailhouse')
+									echo jailhouse ;;
+								('KVM')
+									echo kvm ;;
+								('Microsoft Hyper-V')
+									echo hyperv ;;
+								('VMware')
+									echo vmware ;;
+								('Xen HVM')
+									echo xen-hvm ;;
+								('Xen PV')
+									echo xen-pv ;;
+							esac
+							return 0
+							;;
+						(lpar:*' under hypervisor')
+							return 0 ;;
+					esac
+				done <<-EOF
+				$(dmesg 2>/dev/null | awk '
+				/Booting paravirtualized kernel on /
+				/Hypervisor detected: /
+				/lpar: .* under hypervisor/
+				')
+				EOF
+			fi
+	esac
+
+	# Try to guess hypervisor based on CPU model (sysctl hw.model if available)
+	if cpu_model=$(get_sysctl hw.model)
+	then
+		guess_hypervisor_from_cpu_model "${cpu_model}" && return 0
+	fi
+
+	if ${_hyp_generic}
+	then
+		# cannot say which hypervisor, but one was detected
+		return 0
+	else
+		return 1
+	fi
+}
+
+run_stage() {
+	if type "has_$1_$2" >/dev/null 2>&1
+	then
+		"has_$1_$2"
+	else
+		true
+	fi \
+	&& "check_$1_$2"
+}
+
+
+# Execute container stages
+
+for stage in \
+	pid_1 cgroup files os_specific
+do
+	ctengine=$(run_stage ct ${stage}) || continue
+	is_contained=true
+	if test -n "${ctengine}"
+	then
+		echo container on "${ctengine}"
+		break
+	fi
+done
+if ${is_contained:-false} && test -z "${ctengine}"
+then
+	# none of the stages could determine the specific container engine, but
+	# we are running in some container.
+	echo container
+fi
+
+
+# Execute virtual machine / hypervisor stages
+
+for stage in \
+	os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
+do
+	hypervisor=$(run_stage vm ${stage}) || continue
+	is_virtual=true
+	if test -n "${hypervisor}"
+	then
+		echo virtual by "${hypervisor}"
+		break
+	fi
+done
+if ${is_virtual:-false} && test -z "${hypervisor}"
+then
+	# none of the stages could determine the specific hypervisor, but
+	# we are virtual.
+	echo virtual
+fi

From abc6d009b21b0d1ce3fc5107201e30740f127200 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 19:29:41 +0200
Subject: [PATCH 676/743] [explorer/machine_type] Print top most machine layer
 as first line (fallback to physical)

---
 cdist/conf/explorer/machine_type | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 29f98849..90c441da 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -26,17 +26,19 @@
 # VERB does not have a special meaning, it is just for better readability.
 #
 # e.g.
+# container
 # container on lxc
 # virtual by kvm-spapr
 #
-# The third word of each line can be composed of different parts concatenated with a `-'
-# (minus) character, with each component being a specification of the previous,
-# e.g.:
+# The third word of each line (except the first) can be composed of different
+# parts concatenated with a `-' (minus) character, with each component being
+# a specification of the previous, e.g.:
 #  - lxc-libvirt             (LXC container, managed by libvirt)
 #  - lpar-s390 / lpar-power  (LPAR running on IBM S/390 or POWER, respectively)
 #  - xen-hvm / xen-pv        (Xen HVM vs para-virtualization)
 #
-# If this explorer cannot determine any information it will print nothing.
+# If this explorer cannot collect enough information about virtualization it
+# will fall back to 'physical'.
 #
 
 # Add /sbin and /usr/sbin to the path so we can find system
@@ -121,6 +123,10 @@ get_sysctl() {
 	is_command sysctl && sysctl -n "$1" 2>/dev/null
 }
 
+detected_layer() {
+	test -n "${_toplayer:-}" || echo "${_toplayer:=${1:?}}"
+}
+
 
 # Check for container
 
@@ -895,6 +901,7 @@ for stage in \
 	pid_1 cgroup files os_specific
 do
 	ctengine=$(run_stage ct ${stage}) || continue
+	detected_layer 'container'
 	is_contained=true
 	if test -n "${ctengine}"
 	then
@@ -916,6 +923,7 @@ for stage in \
 	os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
 do
 	hypervisor=$(run_stage vm ${stage}) || continue
+	detected_layer 'virtual machine'
 	is_virtual=true
 	if test -n "${hypervisor}"
 	then
@@ -929,3 +937,8 @@ then
 	# we are virtual.
 	echo virtual
 fi
+
+
+# Fallback
+
+detected_layer physical

From 2ffa895f578f12d615ed65b9a93f3e2ae07f1d07 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 19:57:24 +0200
Subject: [PATCH 677/743] [explorer/machine_type] Remove CPUID check

it's a lot of code and depends on a binary helper unlikely to be installed.
---
 cdist/conf/explorer/machine_type | 59 --------------------------------
 1 file changed, 59 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 90c441da..7ce035e3 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -379,65 +379,6 @@ check_vm_arch_specific() {
 			fi
 			;;
 		(i?86|x86*|amd64|i86pc)
-			# Check CPUID
-			#
-			# Many fullvirt hypervisors give an indication through CPUID.  Use
-			# the virt-what helper program to get this information if available.
-
-			for CPUID_HELPER in \
-				$(command -v virt-what-cpuid-helper 2>/dev/null) \
-				/usr/lib/x86_64-*/virt-what-cpuid-helper \
-				/usr/lib/i?86-*/virt-what-cpuid-helper \
-				/usr/lib/virt-what/virt-what-cpuid-helper
-			do
-				if test -x "${CPUID_HELPER:?}"; then break; fi
-			done
-
-			if test -x "${CPUID_HELPER-}"
-			then
-				case $(command "${CPUID_HELPER}")
-				in
-					('bhyve bhyve ')
-						echo bhyve
-						;;
-					('LKVMLKVMLKVM')
-						echo lkvm
-						;;
-					('KVMKVMKVM')
-						echo kvm
-						;;
-					('TCGTCGTCGTCG')
-						echo qemu-tcg
-						;;
-					('Microsoft Hv')
-						# http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx
-						echo hyperv
-						;;
-					('OpenBSDVMM58')
-						# OpenBSD/VMM
-						echo openbsd_vmm
-						;;
-					('VMwareVMware')
-						# check added by Chetan Loke.
-						echo vmware
-						;;
-					('XenVMMXenVMM')
-						if has dmi
-						then
-							# https://access.redhat.com/solutions/222903
-							echo xen-hvm
-						else
-							echo xen-paravirt
-						fi
-						;;
-					(*)
-						return 1 ;;
-				esac
-				return 0
-			fi
-
-			unset CPUID_HELPER
-
 			# VMM CPUID flag denotes that this system is running under a VMM
 			if is_oneof "${uname_s}" Darwin
 			then

From 23fbfaf0352eadaabd43504d3ee074bb9f696fcd Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 21:29:24 +0200
Subject: [PATCH 678/743] [explorer/machine_type] Use systemd-detect-virt (if
 available) to detect containers and VMs

---
 cdist/conf/explorer/machine_type | 49 ++++++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 7ce035e3..1a38fda0 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -130,6 +130,25 @@ detected_layer() {
 
 # Check for container
 
+has_ct_systemd() {
+	is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^  -c'
+}
+
+check_ct_systemd() (
+	_ctengine=$(systemd-detect-virt -c 2>/dev/null) &&
+	case ${_ctengine}
+	in
+		(''|'none')
+			return 1 ;;
+		('container-other')
+			return 0 ;;
+		('systemd-nspawn')
+			echo systemd_nspawn ;;
+		(*)
+			echo "${_ctengine}" ;;
+	esac
+)
+
 has_ct_pid_1() {
 	test -r /run/systemd/container -o -r /proc/1/environ
 }
@@ -267,6 +286,32 @@ guess_hypervisor_from_cpu_model() {
 	esac
 }
 
+has_vm_systemd() {
+	is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^  -v'
+}
+
+check_vm_systemd() (
+	_hypervisor=$(systemd-detect-virt -v 2>/dev/null) &&
+	case ${_hypervisor}
+	in
+		(''|'none')
+			return 1 ;;
+		('amazon')
+			echo aws ;;
+		('bochs')
+			echo kvm ;;
+		('microsoft')
+			# assumption
+			echo hyperv ;;
+		('oracle')
+			echo virtualbox ;;
+		('vm-other')
+			return 0 ;;
+		(*)
+			echo "${_hypervisor}" ;;
+	esac
+)
+
 has_vm_cpuinfo() { has_cpuinfo; }
 
 check_vm_cpuinfo() {
@@ -839,7 +884,7 @@ run_stage() {
 # Execute container stages
 
 for stage in \
-	pid_1 cgroup files os_specific
+	systemd pid_1 cgroup files os_specific
 do
 	ctengine=$(run_stage ct ${stage}) || continue
 	detected_layer 'container'
@@ -861,7 +906,7 @@ fi
 # Execute virtual machine / hypervisor stages
 
 for stage in \
-	os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
+	systemd os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
 do
 	hypervisor=$(run_stage vm ${stage}) || continue
 	detected_layer 'virtual machine'

From 4a05669765c2c00c19af0ef1b607b0f2efa10f42 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 22:01:28 +0200
Subject: [PATCH 679/743] [explorer/machine_type] Implement chroot detection

---
 cdist/conf/explorer/machine_type | 53 ++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 1a38fda0..10c914ba 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -53,6 +53,21 @@ uname_s=$(uname -s)
 
 is_command() { command -v "$1" >/dev/null 2>&1; }
 
+files_same() {
+	# shellcheck disable=SC2012
+	LC_ALL=C df -P "$1" "$2" 2>/dev/null | {
+		read -r _  # skip header line
+		read -r fs1 _ _ _ _ mp1
+		read -r fs2 _ _ _ _ mp2
+		test "${fs1}" = "${fs2}" -a "${mp1}" = "${mp2}" || return 1
+	} &&
+	ls -1Ldi "$1" "$2" 2>/dev/null | {
+		read -r ino1 _
+		read -r ino2 _
+		test "${ino1}" = "${ino2}" || return 1
+	}
+}
+
 is_oneof() (
 	x=$1; shift
 	for y
@@ -128,6 +143,32 @@ detected_layer() {
 }
 
 
+# Check for chroot
+
+has_chroot_systemd() {
+	is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^  -r'
+}
+
+check_chroot_systemd() {
+	systemd-detect-virt -r
+}
+
+has_chroot_debian_ischroot() {
+	is_command ischroot
+}
+
+check_chroot_debian_ischroot() {
+	ischroot --default-false
+}
+
+has_chroot_procfs() {
+	test -d /proc/
+}
+
+check_chroot_procfs() {
+	test -e /proc/1/root && ! files_same /proc/1/root /
+}
+
 # Check for container
 
 has_ct_systemd() {
@@ -881,6 +922,18 @@ run_stage() {
 }
 
 
+# Execute chroot stages
+
+for stage in \
+	procfs debian_ischroot systemd
+do
+	run_stage chroot ${stage} || continue
+	detected_layer 'chroot'
+	echo chroot
+	break
+done
+
+
 # Execute container stages
 
 for stage in \

From 5af1317c2969995871da1d17b951de60dbc3fd09 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sun, 1 Aug 2021 16:40:20 +0200
Subject: [PATCH 680/743] [explorer/machine_type] Try to detect chroot path

---
 cdist/conf/explorer/machine_type | 38 ++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 10c914ba..fa68ec4d 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -166,7 +166,28 @@ has_chroot_procfs() {
 }
 
 check_chroot_procfs() {
-	test -e /proc/1/root && ! files_same /proc/1/root /
+	if test -e /proc/1/root && ! files_same /proc/1/root /
+	then
+		# try to determine where the chroot has been mounted
+		(
+			rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}')
+
+			if test -e "${rootdev}"
+			then
+				# escape chroot to determine where the device containing the
+				# chroot's / is mounted
+				rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}')
+
+				# shellcheck disable=SC2012
+				root_ino=$(ls -1di / | awk '{print $1}')
+
+				# Get mount point
+				chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}"
+			fi
+		)
+		return 0
+	fi
+	return 1
 }
 
 # Check for container
@@ -927,11 +948,20 @@ run_stage() {
 for stage in \
 	procfs debian_ischroot systemd
 do
-	run_stage chroot ${stage} || continue
+	chrootpnt=$(run_stage chroot ${stage}) || continue
+	is_chrooted=true
 	detected_layer 'chroot'
-	echo chroot
-	break
+	if test -n "${chrootpnt}"
+	then
+		echo chroot at "${chrootpnt}"
+		break
+	fi
 done
+if ${is_chrooted:-false} && test -z "${chrootpnt}"
+then
+	# could determine chroot, but not its mount point
+	echo chroot
+fi
 
 
 # Execute container stages

From 05c2a62191f533ceff1e074df571278a103d75ab Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sun, 1 Aug 2021 23:09:02 +0200
Subject: [PATCH 681/743] [explorer/machine_type] Implement chroot detection
 using /proc/.../mountinfo

---
 cdist/conf/explorer/machine_type | 64 +++++++++++++++++++++-----------
 1 file changed, 43 insertions(+), 21 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index fa68ec4d..00646c75 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -165,30 +165,52 @@ has_chroot_procfs() {
 	test -d /proc/
 }
 
-check_chroot_procfs() {
+check_chroot_procfs() (
+	is_chroot=false  # default
 	if test -e /proc/1/root && ! files_same /proc/1/root /
 	then
-		# try to determine where the chroot has been mounted
-		(
-			rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}')
-
-			if test -e "${rootdev}"
-			then
-				# escape chroot to determine where the device containing the
-				# chroot's / is mounted
-				rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}')
-
-				# shellcheck disable=SC2012
-				root_ino=$(ls -1di / | awk '{print $1}')
-
-				# Get mount point
-				chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}"
-			fi
-		)
-		return 0
+		is_chroot=true
 	fi
-	return 1
-}
+	if test -e /proc/1/mountinfo -a -e /proc/self/mountinfo
+	then
+		has_mountinfo=true
+		cmp -s /proc/1/mountinfo /proc/self/mountinfo || is_chroot=true
+	fi
+
+	if ${is_chroot}
+	then
+		# try to determine where the chroot has been mounted
+		rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}')
+
+		if test -e "${rootdev}"
+		then
+			# escape chroot to determine where the device containing the
+			# chroot's / is mounted
+			rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}')
+
+			# shellcheck disable=SC2012
+			root_ino=$(ls -1di / | awk '{print $1}')
+
+			# escape chroot and find mount point by inode
+			chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}"
+		elif ${has_mountinfo}
+		then
+			while read -r mntid _ _ _ cmntpnt _
+			do
+				read -r _ _ _ _ hmntpnt _ <<-EOF
+				$(grep -e "^$((mntid)) " /proc/1/mountinfo)
+				EOF
+				printf '%s\n' "${hmntpnt%${cmntpnt}}"
+			done </proc/self/mountinfo \
+			| sort -u \
+			| head -n 1  # just take the first...
+		fi
+
+		return 0
+	else
+		return 1
+	fi
+)
 
 # Check for container
 

From 67f85546ec3bf865d368f268fdbbc121ccfd5742 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 23 Aug 2021 09:57:20 +0300
Subject: [PATCH 682/743] [explorer/os_version] add new debian code names:
 bookworm and trixie

---
 cdist/conf/explorer/os_version | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 7bc6dd6b..aea3c43f 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -54,6 +54,8 @@ in
               # sid versions don't have a number, so we decode by codename:
               case $(expr "$debian_version" : '\([a-z]\{1,\}\)/')
               in
+                  trixie) echo 12.99 ;;
+                  bookworm) echo 11.99 ;;
                   bullseye) echo 10.99 ;;
                   buster) echo 9.99 ;;
                   stretch) echo 8.99 ;;

From e1e134899811e51e4694e89e421790e41cb894bf Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 23 Aug 2021 10:44:48 +0300
Subject: [PATCH 683/743] [explorer/os_version] use 99.99 as fallback for
 unknown code names in */sid

---
 cdist/conf/explorer/os_version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index aea3c43f..bbc9e4f0 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -63,7 +63,7 @@ in
                   wheezy) echo 6.99 ;;
                   squeeze) echo 5.99 ;;
                   lenny) echo 4.99 ;;
-                  *) exit 1
+                  *) echo 99.99 ;;
               esac
               ;;
           *)

From 46ed48d546af6ff663b131c803a9027b19798ba1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 08:09:47 +0200
Subject: [PATCH 684/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index f9409d7e..8507c663 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,7 @@ next:
 	* Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera)
 	* Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera)
 	* Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera)
+	* Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 0546283d0ebc0058dbff68cec9d18b9281c26edd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 20:32:44 +0200
Subject: [PATCH 685/743] Update shellcheck disable

---
 cdist/conf/type/__apt_backports/manifest             | 1 +
 cdist/conf/type/__package_pkg_freebsd/gencode-remote | 1 +
 cdist/conf/type/__ssh_authorized_keys/explorer/keys  | 1 +
 3 files changed, 3 insertions(+)

diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index bc47d8de..6fcd9212 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -28,6 +28,7 @@
 #  lsb_release may not be given in all installations
 codename_os_release() {
     # shellcheck disable=SC1090
+    # shellcheck disable=SC1091
     . "$__global/explorer/os_release"
     printf "%s" "$VERSION_CODENAME"
 }
diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote
index 3f88f6bc..ca9aa45a 100755
--- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote
+++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote
@@ -37,6 +37,7 @@ assert ()                 #  If condition false,
 	then
 		echo "Assertion failed:  \"$1\""
 		# shellcheck disable=SC2039
+		# shellcheck disable=SC3044
 		echo "File \"$0\", line $lineno, called by $(caller 0)"
 		exit $E_ASSERT_FAILED
 	fi
diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
index cec25746..9694a64b 100755
--- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys
+++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 
 # shellcheck disable=SC1090
+# shellcheck disable=SC1091
 file="$( . "$__type_explorer/file" )"
 
 if [ -f "$file" ]

From 44741e714b16f7a00bf84bd54ae33eacc7593192 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 20:25:49 +0200
Subject: [PATCH 686/743] Release 6.9.8

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 8507c663..dcdc4b3d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)

From b8eb6e984c1638e8e167394c6b3aa482cb8aad49 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 20:47:50 +0200
Subject: [PATCH 687/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index dcdc4b3d..693d028f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Explorer machine_type: Rewrite (Dennis Camera)
+
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)

From d7fdc8006f747be86b64a2790266b50219bab507 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 21:54:45 +0300
Subject: [PATCH 688/743] allow empty file

---
 cdist/conf/type/__sed/explorer/file | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
index a40f71b3..5f0f4edd 100755
--- a/cdist/conf/type/__sed/explorer/file
+++ b/cdist/conf/type/__sed/explorer/file
@@ -9,13 +9,7 @@ fi
 
 if [ -f "$file" ]
 then
-    if [ -s "$file" ]
-    then
-        cat "$file"
-    else
-        echo "$file is empty" >&2
-        exit 1
-    fi
+    cat "$file"
 else
     echo "$file do not exist" >&2
     exit 1

From 0f6e48dbc657a9d9145b965eacf79f021a1419f5 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:24:26 +0300
Subject: [PATCH 689/743] use $__object/tempfile in target instead of mktemp,
 add comments

---
 cdist/conf/type/__sed/gencode-remote | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 836506f0..df929b42 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -35,8 +35,11 @@ then
     exit 0
 fi
 
+# we can't use -i, because it's not posix, so we fly with tempfile and cp
+# and we use cp because we want to preserve destination file's attributes
+
 # shellcheck disable=SC2016
-echo 'tmp="$( mktemp )"'
+echo 'tmp="$__object/tempfile"'
 
 echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
 

From 90488fcebcc00f3b4ea260867f8b31625df61e46 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:27:42 +0300
Subject: [PATCH 690/743] use -e

---
 cdist/conf/type/__sed/explorer/file | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
index 5f0f4edd..9262b613 100755
--- a/cdist/conf/type/__sed/explorer/file
+++ b/cdist/conf/type/__sed/explorer/file
@@ -7,10 +7,10 @@ else
     file="/$__object_id"
 fi
 
-if [ -f "$file" ]
+if [ ! -e "$file" ]
 then
-    cat "$file"
-else
     echo "$file do not exist" >&2
     exit 1
 fi
+
+cat "$file"

From b7f392fa371ff78591abb910f7e224e7587c81be Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:38:55 +0300
Subject: [PATCH 691/743] use -E for better compat (not really sure if it is
 posix at all)

---
 cdist/conf/type/__sed/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index df929b42..625448e6 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -24,7 +24,7 @@ sed_cmd='sed'
 
 if [ -f "$__object/parameter/regexp-extended" ]
 then
-    sed_cmd="$sed_cmd --regexp-extended"
+    sed_cmd="$sed_cmd -E"
 fi
 
 if echo "$script" \

From aabef7f44a98ad144a7e1fe68bfb562079916ef8 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:40:06 +0300
Subject: [PATCH 692/743] remove reading script from file

---
 cdist/conf/type/__sed/gencode-remote | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 625448e6..be7690e1 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -12,10 +12,6 @@ script="$( cat "$__object/parameter/script" )"
 if [ "$script" = '-' ]
 then
     script="$( cat "$__object/stdin" )"
-elif
-    [ -f "$script" ]
-then
-    script="$( cat "$script" )"
 fi
 
 file_from_target="$__object/explorer/file"

From 5bf0c71e7af49ff6e7c1cccec192f763a019503e Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:45:36 +0300
Subject: [PATCH 693/743] update man

---
 cdist/conf/type/__sed/man.rst | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst
index 4a728184..86789363 100644
--- a/cdist/conf/type/__sed/man.rst
+++ b/cdist/conf/type/__sed/man.rst
@@ -8,35 +8,40 @@ cdist-type__sed - Transform text files with ``sed``
 
 DESCRIPTION
 -----------
-TODO
+Transform text files with ``sed``.
 
 
 REQUIRED MULTIPLE PARAMETERS
 ----------------------------
 script
-   TODO
+   ``sed`` script.
+   If ``-`` then the script is read from ``stdin``.
 
 
 OPTIONAL PARAMETERS
 -------------------
 file
-   TODO
+   Path to the file. Defaults to ``$__object_id``.
 
 onchange
-   TODO
+   Execute this command if ``sed`` changes file.
 
 
 BOOLEAN PARAMETERS
 ------------------
 regexp-extended
-   TODO
+   Use extended regular expressions in the script.
+   Might not be supported with every ``sed`` version.
 
 
 EXAMPLES
 --------
+
 .. code-block:: sh
 
-   true
+   __sed /tmp/foobar --script 's/foo/bar/'
+
+   echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script -
 
 
 AUTHORS

From cd4acde67ec9c3967e6e08450734cf65040e787f Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 15 Sep 2021 09:22:27 +0300
Subject: [PATCH 694/743] grammar

---
 cdist/conf/type/__sed/explorer/file | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
index 9262b613..ec3d0fe8 100755
--- a/cdist/conf/type/__sed/explorer/file
+++ b/cdist/conf/type/__sed/explorer/file
@@ -9,7 +9,7 @@ fi
 
 if [ ! -e "$file" ]
 then
-    echo "$file do not exist" >&2
+    echo "$file does not exist" >&2
     exit 1
 fi
 

From 7b6789ddeb74450f70991efe957e3eb0f11043f6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 15:04:12 +0200
Subject: [PATCH 695/743] __package_update_index: fix complain about suite
 change

2 of 4th fix for ticket #861
---
 .../type/__package_update_index/gencode-remote     | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote
index 803468b5..a10c16d3 100755
--- a/cdist/conf/type/__package_update_index/gencode-remote
+++ b/cdist/conf/type/__package_update_index/gencode-remote
@@ -41,7 +41,19 @@ fi
 case "$type" in
     yum) ;;
     apt)
-        echo "apt-get --quiet update"
+        # There are special arguments to apt(8) to prevent aborts if apt woudn't been
+        # updated after the 19th April 2021 till the bullseye release. The additional
+        # arguments acknoledge the happend suite change (the apt(8) update does the
+        # same by itself).
+        #
+        # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+        # allows backward compatablility to pre-buster Debian versions.
+        #
+        # See more: ticket #861
+        # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+        apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
+        echo "apt-get --quiet $apt_opts update"
         echo "apt-cache updated (age was: $currage)" >> "$__messages_out"
         ;;
     pacman)

From 12787ffe2c48f5c6d534d10f8faadde30db477ac Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 15:12:20 +0200
Subject: [PATCH 696/743] __apt_source: fix complain about suite change

3 of 4th fix for ticket #861
---
 cdist/conf/type/__apt_source/gencode-remote | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_source/gencode-remote b/cdist/conf/type/__apt_source/gencode-remote
index 1e8592c6..973b0f6c 100755
--- a/cdist/conf/type/__apt_source/gencode-remote
+++ b/cdist/conf/type/__apt_source/gencode-remote
@@ -22,7 +22,21 @@
 name="$__object_id"
 destination="/etc/apt/sources.list.d/${name}.list"
 
+# There are special arguments to apt(8) to prevent aborts if apt woudn't been
+# updated after the 19th April 2021 till the bullseye release. The additional
+# arguments acknoledge the happend suite change (the apt(8) update does the
+# same by itself).
+#
+# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+# allows backward compatablility to pre-buster Debian versions.
+#
+# See more: ticket #861
+# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
+# run 'apt-get update' only if something changed with our sources.list file
+#   it will be run a second time on error as a redundancy messure to success
 if grep -q "^__file${destination}" "$__messages_in"; then
-   printf 'apt-get update || apt-get update\n'
+   printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts"
 fi
 

From d246e067109dc8b85ea89668f742affbb221d1a6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 14:49:23 +0200
Subject: [PATCH 697/743] __apt_update_index: fix complain about suite change

1 of 4th fix for ticket #861
---
 .../conf/type/__apt_update_index/gencode-remote  | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_update_index/gencode-remote b/cdist/conf/type/__apt_update_index/gencode-remote
index 70b59710..2d7f9030 100755
--- a/cdist/conf/type/__apt_update_index/gencode-remote
+++ b/cdist/conf/type/__apt_update_index/gencode-remote
@@ -18,9 +18,23 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+
+# There are special arguments to apt(8) to prevent aborts if apt woudn't been
+# updated after the 19th April 2021 till the bullseye release. The additional
+# arguments acknoledge the happend suite change (the apt(8) update does the
+# same by itself).
+#
+# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+# allows backward compatablility to pre-buster Debian versions.
+#
+# See more: ticket #861
+# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
 # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists
+#   it will be run a second time on error as a redundancy messure to success
 cat << DONE
 if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then
-   apt-get update || apt-get update
+   apt-get $apt_opts update || apt-get $apt_opts update
 fi
 DONE

From 3d7b31cbb43067271c2c510281f90f19dc5ec5fb Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 15:22:16 +0200
Subject: [PATCH 698/743] __package_apt: fix complain about suite change

the last fix for ticket #861 :-)
---
 cdist/conf/type/__package_apt/gencode-remote | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote
index fbfca330..79c0d9d3 100755
--- a/cdist/conf/type/__package_apt/gencode-remote
+++ b/cdist/conf/type/__package_apt/gencode-remote
@@ -81,12 +81,24 @@ aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::=
 
 case "$state_should" in
     present)
+        # There are special arguments to apt(8) to prevent aborts if apt woudn't been
+        # updated after the 19th April 2021 till the bullseye release. The additional
+        # arguments acknoledge the happend suite change (the apt(8) update does the
+        # same by itself).
+        #
+        # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+        # allows backward compatablility to pre-buster Debian versions.
+        #
+        # See more: ticket #861
+        # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+        apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
         # following is bit ugly, but important hack.
         # due to how cdist config run works, there isn't
         # currently better way to do it :(
         cat << EOF
 if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ]
-then echo apt-get update > /dev/null 2>&1 || true
+then echo apt-get $apt_opts update > /dev/null 2>&1 || true
 fi
 EOF
         if [ -n "$version" ]; then

From 72ff48154c958ebdedc8b6d919b6f372efa63be2 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 16 Sep 2021 21:36:39 +0300
Subject: [PATCH 699/743] add comments, add -u to diff

---
 cdist/conf/type/__sed/gencode-remote | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index be7690e1..f99c5a88 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -14,6 +14,8 @@ then
     script="$( cat "$__object/stdin" )"
 fi
 
+# since stdin is not available in explorer, we pull file from target with explorer
+
 file_from_target="$__object/explorer/file"
 
 sed_cmd='sed'
@@ -23,10 +25,10 @@ then
     sed_cmd="$sed_cmd -E"
 fi
 
-if echo "$script" \
-    | "$sed_cmd" -f - "$file_from_target" \
-    | diff "$file_from_target" - \
-    > /dev/null
+# do sed dry run, diff result and if no change, then there's nothing to do
+# also redirect diff's output to stderr for debugging purposes
+
+if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2
 then
     exit 0
 fi

From bf222d05435000063930ff199bd0dec94d7d8cd5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 21 Sep 2021 08:55:54 +0200
Subject: [PATCH 700/743] ++changelog

---
 docs/changelog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 693d028f..ab637f5f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,11 @@ Changelog
 
 next:
 	* Explorer machine_type: Rewrite (Dennis Camera)
+	* New type: __sed (Ander Punnar)
+	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From 15c642a9b7af42c368eb5a8e22cab5f07b39b95e Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 12:06:45 +0200
Subject: [PATCH 701/743] [__debconf_set_selections] Fix --file not being
 supported

Even if deprecated, the parameter *must* be supported, which isn't the case
right now.

This was due to a misunderstanding of how deprecating parameters work, see:
https://www.cdi.st/manual/latest/cdist-type.html#deprecated-parameters
---
 .../conf/type/__debconf_set_selections/parameter/deprecated/file | 1 +
 .../__debconf_set_selections/parameter/{deprecated => optional}  | 0
 2 files changed, 1 insertion(+)
 create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/deprecated/file
 rename cdist/conf/type/__debconf_set_selections/parameter/{deprecated => optional} (100%)

diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file
new file mode 100644
index 00000000..09db545a
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file
@@ -0,0 +1 @@
+'file' has been deprecated in favour of 'line' in order to provide idempotency.
diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated b/cdist/conf/type/__debconf_set_selections/parameter/optional
similarity index 100%
rename from cdist/conf/type/__debconf_set_selections/parameter/deprecated
rename to cdist/conf/type/__debconf_set_selections/parameter/optional

From 5b7cca99f73e4ec3dd5cdb90e4dbfd40ec8fe349 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 12:09:42 +0200
Subject: [PATCH 702/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index ab637f5f..142602c8 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
+	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From fc9bd40c9a34cf54e2d6cfd2fec089bd25e1b172 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 13:14:57 +0200
Subject: [PATCH 703/743] Improve bullseye support, perticularly
 __letsencrypt_cert

---
 cdist/conf/type/__grafana_dashboard/manifest | 2 +-
 cdist/conf/type/__letsencrypt_cert/manifest  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest
index d145c4c3..0d944482 100755
--- a/cdist/conf/type/__grafana_dashboard/manifest
+++ b/cdist/conf/type/__grafana_dashboard/manifest
@@ -15,7 +15,7 @@ case $os in
                 # Differntation not needed anymore
                 apt_source_distribution=stable
                 ;;
-            10*)
+            10*|11*)
                 # Differntation not needed anymore
                 apt_source_distribution=stable
                 ;;
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 6394f629..638a99e0 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -41,7 +41,7 @@ if [ -z "${certbot_fullpath}" ]; then
 					require="__apt_source/stretch-backports" __package_apt certbot \
 						--target-release stretch-backports
 				;;
-				10*)
+				10*|11*)
 					__package_apt certbot
 				;;
 

From 560374a6861281695b61c8c2298123be41d92326 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 13:16:11 +0200
Subject: [PATCH 704/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 142602c8..6f717cf4 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,7 @@ next:
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
+	* Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From c33d99ee120ad180daba144be80a25d77b473f56 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sun, 31 Oct 2021 17:38:10 +0100
Subject: [PATCH 705/743] [__haproxy_dualstack] New type with PROXY protocol
 support

This is backwards compatible with what is already used internally @ungleich, but
adds on top of that the ability to customise ports and, most importantly, it
adds PROXY protocol support.
---
 .../conf/type/__haproxy_dualstack/files/http  |   8 +
 .../conf/type/__haproxy_dualstack/files/https |  10 ++
 .../conf/type/__haproxy_dualstack/files/imaps |  12 ++
 .../conf/type/__haproxy_dualstack/files/smtps |  12 ++
 cdist/conf/type/__haproxy_dualstack/man.rst   | 121 ++++++++++++++
 cdist/conf/type/__haproxy_dualstack/manifest  | 155 ++++++++++++++++++
 .../parameter/default/protocol                |   1 +
 .../parameter/optional_multiple               |   3 +
 cdist/conf/type/__haproxy_dualstack/singleton |   0
 9 files changed, 322 insertions(+)
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/http
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/https
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/imaps
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/smtps
 create mode 100644 cdist/conf/type/__haproxy_dualstack/man.rst
 create mode 100644 cdist/conf/type/__haproxy_dualstack/manifest
 create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/default/protocol
 create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple
 create mode 100644 cdist/conf/type/__haproxy_dualstack/singleton

diff --git a/cdist/conf/type/__haproxy_dualstack/files/http b/cdist/conf/type/__haproxy_dualstack/files/http
new file mode 100644
index 00000000..0508a465
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/http
@@ -0,0 +1,8 @@
+frontend http
+	bind	BIND@:80
+	mode	http
+	option	httplog
+	default_backend	http
+
+backend http
+	mode	http
diff --git a/cdist/conf/type/__haproxy_dualstack/files/https b/cdist/conf/type/__haproxy_dualstack/files/https
new file mode 100644
index 00000000..73deac46
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/https
@@ -0,0 +1,10 @@
+frontend https
+	bind	BIND@:443
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	https
+
+backend https
+	mode	tcp
diff --git a/cdist/conf/type/__haproxy_dualstack/files/imaps b/cdist/conf/type/__haproxy_dualstack/files/imaps
new file mode 100644
index 00000000..b1ec3793
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/imaps
@@ -0,0 +1,12 @@
+frontend imaps
+	bind	BIND@:143
+	bind	BIND@:993
+
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	imaps
+
+backend imaps
+	mode	tcp
diff --git a/cdist/conf/type/__haproxy_dualstack/files/smtps b/cdist/conf/type/__haproxy_dualstack/files/smtps
new file mode 100644
index 00000000..dce6ed4a
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/smtps
@@ -0,0 +1,12 @@
+frontend smtps
+	bind	BIND@:25
+	bind	BIND@:465
+
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	smtps
+
+backend smtps
+	mode	tcp
diff --git a/cdist/conf/type/__haproxy_dualstack/man.rst b/cdist/conf/type/__haproxy_dualstack/man.rst
new file mode 100644
index 00000000..6c131cbe
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/man.rst
@@ -0,0 +1,121 @@
+cdist-type__haproxy_dualstack(7)
+================================
+
+
+NAME
+----
+cdist-type__haproxy_dualstack - Proxy services from a dual-stack server
+
+
+DESCRIPTION
+-----------
+This (singleton) type installs and configures haproxy to act as a dual-stack
+proxy for single-stack services.
+
+This can be useful to add IPv4 support to IPv6-only services while only using
+one IPv4 for many such services.
+
+By default this type uses the plain TCP proxy mode, which means that there is no
+need for TLS termination on this host when SNI is supported.
+This also means that proxied services will not receive the client's IP address,
+but will see the proxy's IP address instead (that of `$__target_host`).
+
+This can be solved by using the PROXY protocol, but do take into account that,
+e.g. nginx cannot serve both regular HTTP(S) and PROXY protocols on the same
+port, so you will need to use other ports for that.
+
+As a recommendation in this type: use TCP ports 8080 and 591 respectively to
+serve HTTP and HTTPS using the PROXY protocol.
+
+See the EXAMPLES for more details.
+
+
+OPTIONAL PARAMETERS
+-------------------
+v4proxy
+    Proxy incoming IPv4 connections to the equivalent IPv6 endpoint.
+    In its simplest use, it must be a NAME with an `AAAA` DNS entry, which is
+    the IP address actually providing the proxied services.
+    The full format of this argument is:
+    `[proxy:]NAME[[:PROTOCOL_1=PORT_1]...[:PROTOCOL_N=PORT_N]]`
+    Where starting with `proxy:` determines that the PROXY protocol must be
+    used and each `:PROTOCOL=PORT` (e.g. `:http=8080` or `:https=591`) is a PORT
+    override for the given PROTOCOL (see `--protocol`), if not present the
+    PROTOCOL's default port will be used.
+
+
+v6proxy
+    Proxy incoming IPv6 connections to the equivalent IPv4 endpoint.
+    In its simplest use, it must be a NAME with an `A` DNS entry, which is
+    the IP address actually providing the proxied services.
+    See `--v4proxy` for more options and details.
+
+protocol
+    Can be passed multiple times or as a space-separated list of protocols.
+    Currently supported protocols are: `http`, `https`, `imaps`, `smtps`.
+    This defaults to: `http https imaps smtps`.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Proxy the IPv6-only services so IPv4-only clients can access them
+    # This uses HAProxy's TCP mode for http, https, imaps and smtps
+    __haproxy_dualstack \
+        --v4proxy ipv6.chat \
+        --v4proxy matrix.ungleich.ch
+
+    # Proxy the IPv6-only HTTP(S) services so IPv4-only clients can access them
+    # Note this means that the backend IPv6-only server will only see
+    # the IPv6 address of the haproxy host managed by cdist, which can be
+    # troublesome if this information is relevant for analytics/security/...
+    # See the PROXY example below
+    __haproxy_dualstack \
+        --protocol http --protocol https \
+        --v4proxy ipv6.chat \
+        --v4proxy matrix.ungleich.ch
+
+    # Use the PROXY protocol to proxy the IPv6-only HTTP(S) services enabling
+    # IPv4-only clients to access them while maintaining the client's IP address
+    __haproxy_dualstack \
+        --protocol http --protocol https \
+        --v4proxy proxy:ipv6.chat:http=8080:https=591 \
+        --v4proxy proxy:matrix.ungleich.ch:http=8080:https=591
+    # Note however that the PROXY protocol is not compatible with regular
+    # HTTP(S) protocols, so your nginx will have to listen on different ports
+    # with the PROXY settings.
+    # Note that you will need to restrict access to the 8080 port to prevent
+    # Client IP spoofing.
+    # This can be something like:
+    # server {
+    #     # listen for regular HTTP connections
+    #     listen [::]:80 default_server;
+    #     listen 80 default_server;
+    #     # listen for PROXY HTTP connections
+    #     listen [::]:8080 proxy_protocol;
+    #     # Accept the Client's IP from the PROXY protocol
+    #     real_ip_header proxy_protocol;
+    # }
+
+
+SEE ALSO
+--------
+- https://www.haproxy.com/blog/enhanced-ssl-load-balancing-with-server-name-indication-sni-tls-extension/
+- https://www.haproxy.com/blog/haproxy/proxy-protocol/
+- https://docs.nginx.com/nginx/admin-guide/load-balancer/using-proxy-protocol/
+
+
+AUTHORS
+-------
+ungleich <foss--@--ungleich.ch>
+Evilham <cvs--@--evilham.com>
+
+
+COPYING
+-------
+Copyright \(C) 2021 ungleich glarus ag. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__haproxy_dualstack/manifest b/cdist/conf/type/__haproxy_dualstack/manifest
new file mode 100644
index 00000000..d110eea6
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/manifest
@@ -0,0 +1,155 @@
+#!/bin/sh -eu
+
+__package haproxy
+require="__package/haproxy" __start_on_boot haproxy
+
+tmpdir="$__object/files"
+mkdir "$tmpdir"
+configtmp="$__object/files/haproxy.cfg"
+
+os=$(cat "$__global/explorer/os")
+case $os in
+	freebsd)
+		CONFIG_FILE="/usr/local/etc/haproxy.conf"
+		cat <<EOF > "$configtmp"
+global
+	maxconn	4000
+	user	nobody
+	group	nogroup
+	daemon
+
+EOF
+
+		;;
+	*)
+		CONFIG_FILE="/etc/haproxy/haproxy.cfg"
+		cat <<EOF > "$configtmp"
+global
+	log	[::1] local2
+	chroot	/var/lib/haproxy
+	pidfile	/var/run/haproxy.pid
+	maxconn	4000
+	user	haproxy
+	group	haproxy
+	daemon
+
+	# turn on stats unix socket
+	stats socket	/var/lib/haproxy/stats
+
+EOF
+		;;
+esac
+
+cat <<EOF >> "$configtmp"
+defaults
+	retries	3
+	log	global
+	timeout http-request	10s
+	timeout queue		1m
+	timeout connect		10s
+	timeout client		1m
+	timeout server		1m
+	timeout http-keep-alive	10s
+	timeout check		10s
+EOF
+
+dig_cmd="$(command -v dig || true)"
+get_ip() {
+	# Usage: get_ip (ipv4|ipv6) NAME
+	# uses "dig" if available, else fallback to "host"
+	case $1 in
+		ipv4)
+			if [ -n "${dig_cmd}" ]; then
+				${dig_cmd} +short A "$2"
+			else
+				host -t A "$2" | cut -d ' ' -f 4 | grep -v 'found:'
+			fi
+			;;
+		ipv6)
+			if [ -n "${dig_cmd}" ]; then
+				${dig_cmd} +short AAAA "$2"
+			else
+				host -t AAAA "$2" | cut -d ' ' -f 5 | grep -v 'NXDOMAIN'
+			fi
+			;;
+	esac
+}
+
+PROTOCOLS="$(cat "$__object/parameter/protocol")"
+
+for proxy in v4proxy v6proxy; do
+	param=$__object/parameter/$proxy
+	# no backend? skip generating code
+	if [ ! -f "$param" ]; then
+		continue
+	fi
+
+	# turn backend name into bind parameter: v4backend -> ipv4@
+	bind=$(echo $proxy | sed -e 's/^/ip/' -e 's/proxy//')
+
+	case $bind in
+		ipv4)
+			backendproto=ipv6
+			;;
+		ipv6)
+			backendproto=ipv4
+			;;
+	esac
+
+	for proto in ${PROTOCOLS}; do
+		# Add protocol "header"
+		printf "\n# %s %s \n" "${bind}" "${proto}" >> "$configtmp"
+
+		sed -e "s/BIND/$bind/" \
+			-e "s/\(frontend[[:space:]].*\)/\1$bind/" \
+			-e "s/\(backend[[:space:]].*\)/\\1$bind/" \
+			"$__type/files/$proto" >> "$configtmp"
+
+		while read -r hostdefinition; do
+			if echo "$hostdefinition" | grep -qE '^proxy:'; then
+				# Proxy protocol was requested
+				host="$(echo "$hostdefinition" | sed -E 's/^proxy:([^:]+).*$/\1/')"
+				send_proxy=" send-proxy"
+			else
+				# Just use tcp proxy mode
+				host="$hostdefinition"
+				send_proxy=""
+			fi
+			if echo "$hostdefinition" | grep -qE ":${proto}="; then
+				# Use custom port definition if requested
+				port="$(echo "$hostdefinition" | sed -E "s/^(.*:)?${proto}=([0-9]+).*$/:\2/")"
+			else
+				# Else use the default
+				port=""
+			fi
+			servername=$host
+
+			res=$(get_ip "$bind" "$servername")
+
+			if [ -z "$res" ]; then
+				echo "$servername does not resolve - aborting config" >&2
+				exit 1
+			fi
+
+			# Treat protocols without TLS+SNI specially
+			if [ "$proto" = http ]; then
+				echo "	use-server $servername if { hdr(host) -i $host }" >> "$configtmp"
+			else
+				echo "	use-server $servername if { req_ssl_sni -i $host }" >> "$configtmp"
+			fi
+
+			# Create the "server" itself.
+			# Note that port and send_proxy will be empty unless
+			# they were requested by the type user
+			echo "	server $servername ${backendproto}@${host}${port}${send_proxy}" >> "$configtmp"
+
+		done < "$param"
+	done
+done
+
+# Create config file
+require="__package/haproxy" __file ${CONFIG_FILE} --source "$configtmp" --mode 0644
+
+require="__file${CONFIG_FILE}" __check_messages "haproxy_reload" \
+	--pattern "^__file${CONFIG_FILE}" \
+	--execute "service haproxy reload || service haproxy restart"
diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol
new file mode 100644
index 00000000..dc8bb7bf
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol
@@ -0,0 +1 @@
+http https imaps smtps
diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple
new file mode 100644
index 00000000..8c482bd4
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple
@@ -0,0 +1,3 @@
+protocol
+v4proxy
+v6proxy
diff --git a/cdist/conf/type/__haproxy_dualstack/singleton b/cdist/conf/type/__haproxy_dualstack/singleton
new file mode 100644
index 00000000..e69de29b

From e2500248f2ddc83129e77f2e6b8dffb64904dbae Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Wed, 3 Nov 2021 11:03:33 +0100
Subject: [PATCH 706/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 6f717cf4..99a8c08b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Explorer machine_type: Rewrite (Dennis Camera)
 	* New type: __sed (Ander Punnar)
+	* New type: __haproxy_dualstack (Evilham and ungleich)
 	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)

From 3a321469a8ba5aea55220bd70bd4900de732e917 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 16 Nov 2021 11:11:45 +0100
Subject: [PATCH 707/743] Python 3.10: collections.X -> collections.abc.X

---
 cdist/integration.py     | 2 +-
 cdist/util/fsproperty.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/integration.py b/cdist/integration.py
index 17b65f09..04470ea7 100644
--- a/cdist/integration.py
+++ b/cdist/integration.py
@@ -84,7 +84,7 @@ def _process_hosts_simple(action, host, manifest, verbose,
     """
     if isinstance(host, str):
         hosts = [host, ]
-    elif isinstance(host, collections.Iterable):
+    elif isinstance(host, collections.abc.Iterable):
         hosts = host
     else:
         raise cdist.Error('Invalid host argument: {}'.format(host))
diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py
index 09e9cc19..6bf935e8 100644
--- a/cdist/util/fsproperty.py
+++ b/cdist/util/fsproperty.py
@@ -33,7 +33,7 @@ class AbsolutePathRequiredError(cdist.Error):
         return 'Absolute path required, got: {}'.format(self.path)
 
 
-class FileList(collections.MutableSequence):
+class FileList(collections.abc.MutableSequence):
     """A list that stores it's state in a file.
 
     """
@@ -102,7 +102,7 @@ class FileList(collections.MutableSequence):
         self.__write(lines)
 
 
-class DirectoryDict(collections.MutableMapping):
+class DirectoryDict(collections.abc.MutableMapping):
     """A dict that stores it's items as files in a directory.
 
     """

From 6e3ad11ea0177865e7e0288b1765267df8d5d020 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Thu, 23 Dec 2021 20:07:28 +0100
Subject: [PATCH 708/743] [__package_upgrade_all] Add new --apt-with-new-pkgs
 argument

---
 cdist/conf/type/__package_upgrade_all/gencode-remote    | 6 +++++-
 cdist/conf/type/__package_upgrade_all/man.rst           | 8 ++++++++
 cdist/conf/type/__package_upgrade_all/parameter/boolean | 1 +
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote
index 38aa001e..d332e851 100755
--- a/cdist/conf/type/__package_upgrade_all/gencode-remote
+++ b/cdist/conf/type/__package_upgrade_all/gencode-remote
@@ -28,6 +28,10 @@ apt_clean="$__object/parameter/apt-clean"
 
 apt_dist_upgrade="$__object/parameter/apt-dist-upgrade"
 
+if [ -f "$__object/parameter/apt-with-new-pkgs" ]; then
+	apt_with_new_pkgs="--with-new-pkgs"
+fi
+
 if [ -f "$type" ]; then
     type="$(cat "$type")"
 else
@@ -54,7 +58,7 @@ case "$type" in
     apt)
         if [ -f "$apt_dist_upgrade" ]
         then echo "$aptget dist-upgrade"
-        else echo "$aptget upgrade"
+        else echo "$aptget $apt_with_new_pkgs upgrade"
         fi
 
         if [ -f "$apt_clean" ]
diff --git a/cdist/conf/type/__package_upgrade_all/man.rst b/cdist/conf/type/__package_upgrade_all/man.rst
index e9e2b8ce..0c116bac 100644
--- a/cdist/conf/type/__package_upgrade_all/man.rst
+++ b/cdist/conf/type/__package_upgrade_all/man.rst
@@ -33,6 +33,14 @@ BOOLEAN PARAMETERS
 apt-dist-upgrade
     Do dist-upgrade instead of upgrade.
 
+apt-with-new-pkg
+    Allow installing new packages when used in conjunction with
+    upgrade. This is useful if the update of an installed package
+    requires new dependencies to be installed. Instead of holding the
+    package back upgrade will upgrade the package and install the new
+    dependencies. Note that upgrade with this option will never remove
+    packages, only allow adding new ones.
+
 apt-clean
     Clean out the local repository of retrieved package files.
 
diff --git a/cdist/conf/type/__package_upgrade_all/parameter/boolean b/cdist/conf/type/__package_upgrade_all/parameter/boolean
index 7a56a34b..cd22eb90 100644
--- a/cdist/conf/type/__package_upgrade_all/parameter/boolean
+++ b/cdist/conf/type/__package_upgrade_all/parameter/boolean
@@ -1,2 +1,3 @@
 apt-clean
 apt-dist-upgrade
+apt-with-new-pkgs

From c2c5668b704e1648ff6c8fb88219badddd028346 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Thu, 23 Dec 2021 20:08:49 +0100
Subject: [PATCH 709/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 99a8c08b..26d89057 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* New type: __haproxy_dualstack (Evilham and ungleich)
 	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_upgrade_all: Add new --apt-with-new-pkgs argument (Evilham)
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)

From 08ff41efded6e3112fc462ba13d9166e620b4082 Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Tue, 8 Mar 2022 12:04:58 +0100
Subject: [PATCH 710/743] Added rm of tmpfile.

---
 cdist/conf/type/__ssh_authorized_key/gencode-remote | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote
index 61c77fb9..cbffde94 100755
--- a/cdist/conf/type/__ssh_authorized_key/gencode-remote
+++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote
@@ -40,6 +40,7 @@ if [ -f "$file" ]; then
    grep -v -F -x '$line' '$file' >\$tmpfile
 fi
 cat "\$tmpfile" >"$file"
+rm -f "\$tmpfile"
 DONE
 }
 

From e0150e779681e232f95bdbefd957c666f05daa89 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Wed, 9 Mar 2022 16:16:44 +0100
Subject: [PATCH 711/743] ++changes

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 26d89057..81be51f6 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,8 @@ next:
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
 	* Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham)
+	* Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom)
+	* Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From bd44c023d33eb51a09afad12b18c082f6a3ae36d Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Fri, 11 Jun 2021 11:22:31 +0100
Subject: [PATCH 712/743] Fix typos; add default priority; comments in
 generated files

---
 cdist/conf/type/__apt_pin/manifest                   | 5 +++++
 cdist/conf/type/__apt_pin/parameter/default/priority | 1 +
 cdist/conf/type/__apt_pin/parameter/optional         | 1 +
 cdist/conf/type/__apt_pin/parameter/required         | 1 -
 4 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 cdist/conf/type/__apt_pin/parameter/default/priority

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index e72a8fdd..983b2b42 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -57,6 +57,11 @@ __file "/etc/apt/preferences.d/$name" \
     --owner root --group root --mode 0644 \
     --state "$state" \
     --source - << EOF
+# Created by cdist ${__type##*/}
+# Do not change. Changes will be overwritten.
+#
+
+# $name
 Package: $package
 Pin: $pin
 Pin-Priority: $priority
diff --git a/cdist/conf/type/__apt_pin/parameter/default/priority b/cdist/conf/type/__apt_pin/parameter/default/priority
new file mode 100644
index 00000000..1b79f38e
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/default/priority
@@ -0,0 +1 @@
+500
diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional
index 52f01fd2..847e703d 100644
--- a/cdist/conf/type/__apt_pin/parameter/optional
+++ b/cdist/conf/type/__apt_pin/parameter/optional
@@ -1,2 +1,3 @@
 state
 package
+priority
diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required
index 4b4e9741..c8572d92 100644
--- a/cdist/conf/type/__apt_pin/parameter/required
+++ b/cdist/conf/type/__apt_pin/parameter/required
@@ -1,2 +1 @@
 distribution
-priority

From 22039284f57f575defa0bc65c46b8bbcbe016cd8 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@armstrong.cc>
Date: Sun, 10 Apr 2022 23:52:53 +0200
Subject: [PATCH 713/743] __file: make file uploading and attribute changes
 more atomic

Fixes https://code.ungleich.ch/ungleich-public/cdist/pulls/331

Signed-off-by: Steven Armstrong <steven@armstrong.cc>
---
 cdist/conf/type/__file/gencode-local  | 28 +++++++++++++++++++--------
 cdist/conf/type/__file/gencode-remote | 14 +++++++++++++-
 2 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local
index 231b6927..bea3d79c 100755
--- a/cdist/conf/type/__file/gencode-local
+++ b/cdist/conf/type/__file/gencode-local
@@ -1,7 +1,7 @@
 #!/bin/sh -e
 #
 # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
-# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -89,10 +89,26 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then
       touch "$__object/files/set-attributes"
 
       # upload file to temp location
-      tempfile_template="${destination}.cdist.XXXXXXXXXX"
+      upload_destination="$(mktemp -u "${destination}.cdist.XXXXXXXXXX")"
+      # Yes, we are aware that this is a race condition.
+      # However:
+      # a) cdist usually writes to directories that are not user writable
+      #    (probably > 99.9%)
+      # b) if they are user owned, the user / attacker always wins
+      #    (probably < 0.1%)
+      # c) the only case which we could improve are tmp directories and we
+      #    don't think managing tmp directories with cdist is a typical case
+      #    ("the rest %)"
       cat << DONE
-destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template")"
+$__remote_exec $__target_host test -e $upload_destination && {
+   echo "Refusing to upload file to existing destination: $upload_destination" >&2
+   exit 1
+}
 DONE
+      # Tell gencode-remote to where we uploaded the file so it can move
+      # it to its final destination.
+      echo "$upload_destination" > "$__object/files/upload-destination"
+
       if [ "$upload_file" ]; then
          echo upload >> "$__messages_out"
          # IPv6 fix
@@ -103,12 +119,8 @@ DONE
              my_target_host="${__target_host}"
          fi
          cat << DONE
-$__remote_copy "$source" "${my_target_host}:\$destination_upload"
+$__remote_copy "$source" "${my_target_host}:${upload_destination}"
 DONE
       fi
-# move uploaded file into place
-cat << DONE
-$__remote_exec $__target_host "rm -rf \"$destination\"; mv \"\$destination_upload\" \"$destination\""
-DONE
    fi
 fi
diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index f7a528fd..136520a7 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -1,7 +1,7 @@
 #!/bin/sh -e
 #
 # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
-# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -62,6 +62,13 @@ set_mode() {
 
 case "$state_should" in
     present|exists)
+        if [ -f "$__object/files/upload-destination" ]; then
+            final_destination="$destination"
+            # We change the 'global' $destination variable here so we can
+            # change attributes of the new/uploaded file before moving it
+            # to it's final destination.
+            destination="$(cat "$__object/files/upload-destination")"
+        fi
         # Note: Mode - needs to happen last as a chown/chgrp can alter mode by
         #  clearing S_ISUID and S_ISGID bits (see chown(2))
         for attribute in group owner mode; do
@@ -81,6 +88,11 @@ case "$state_should" in
                 fi
             fi
         done
+        if [ -f "$__object/files/upload-destination" ]; then
+            # move uploaded file into place
+            printf 'rm -rf "%s"\n' "$final_destination"
+            printf 'mv -T "%s" "%s"\n' "$destination" "$final_destination"
+        fi
         if [ -f "$__object/files/set-attributes" ]; then
             # set-attributes is created if file is created or uploaded in gencode-local
             fire_onchange=1

From af54fe6febc186b0db6da00f2d06d5e87f4012f9 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@armstrong.cc>
Date: Mon, 11 Apr 2022 00:04:41 +0200
Subject: [PATCH 714/743] changelog++

Signed-off-by: Steven Armstrong <steven@armstrong.cc>
---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 81be51f6..3363be27 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -14,6 +14,7 @@ next:
 	* Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham)
 	* Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom)
 	* Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali)
+	* Type __file: make file uploading and attribute changes more atomic	(Steven Armstrong)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From cb0fa0f2e4d99891653b8871309e22f62ff393dd Mon Sep 17 00:00:00 2001
From: Romain Dartigues <romain.dartigues@gmail.com>
Date: Fri, 24 Dec 2021 15:39:29 +0100
Subject: [PATCH 715/743] force add-apt-repository to act in non-interactive
 mode

---
 cdist/conf/type/__apt_ppa/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote
index 84ebebfe..094366b9 100755
--- a/cdist/conf/type/__apt_ppa/gencode-remote
+++ b/cdist/conf/type/__apt_ppa/gencode-remote
@@ -29,7 +29,7 @@ fi
 
 case "$state_should" in
    present)
-      echo "add-apt-repository '$name'"
+      echo "add-apt-repository -y '$name'"
    ;;
    absent)
       echo "remove-apt-repository '$name'"

From 54a5cb17b762a7fbb61225b36621c8a45e83536b Mon Sep 17 00:00:00 2001
From: Romain Dartigues <romain.dartigues@gmail.com>
Date: Fri, 24 Dec 2021 15:41:03 +0100
Subject: [PATCH 716/743] use add-apt-repository instead of add-apt-repository

Remove `remove-apt-repository` which is now no longer needed;
use `add-apt-repository` which allow removal through the `-r` flag.
---
 .../__apt_ppa/files/remove-apt-repository     | 55 -------------------
 cdist/conf/type/__apt_ppa/gencode-remote      |  2 +-
 cdist/conf/type/__apt_ppa/manifest            |  5 --
 3 files changed, 1 insertion(+), 61 deletions(-)
 delete mode 100755 cdist/conf/type/__apt_ppa/files/remove-apt-repository

diff --git a/cdist/conf/type/__apt_ppa/files/remove-apt-repository b/cdist/conf/type/__apt_ppa/files/remove-apt-repository
deleted file mode 100755
index 3eb7d491..00000000
--- a/cdist/conf/type/__apt_ppa/files/remove-apt-repository
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/usr/bin/env python
-#
-# Remove the given apt repository.
-#
-# Exit with:
-#   0: if it worked
-#   1: if not
-#   2: on other error
-
-import os
-import sys
-from aptsources import distro, sourceslist
-from softwareproperties import ppa
-from softwareproperties.SoftwareProperties import SoftwareProperties
-
-
-def remove_if_empty(file_name):
-    with open(file_name, 'r') as f:
-        if f.read().strip():
-            return
-        os.unlink(file_name)
-
-def remove_repository(repository):
-    #print 'repository:', repository
-    codename = distro.get_distro().codename
-    #print 'codename:', codename
-    (line, file) = ppa.expand_ppa_line(repository.strip(), codename)
-    #print 'line:', line
-    #print 'file:', file
-    deb_source_entry = sourceslist.SourceEntry(line, file)
-    src_source_entry = sourceslist.SourceEntry('deb-src{}'.format(line[3:]), file)
-
-    try:
-        sp = SoftwareProperties()
-        sp.remove_source(deb_source_entry)
-        try:
-            # If there's a deb-src entry, remove that too
-            sp.remove_source(src_source_entry)
-        except:
-            pass
-        remove_if_empty(file)
-        return True
-    except ValueError:
-        print >> sys.stderr, "Error: '%s' doesn't exists in a sourcelist file" % line
-        return False
-
-if __name__ == '__main__':
-    if (len(sys.argv) != 2):
-        print >> sys.stderr, 'Error: need a repository as argument'
-        sys.exit(2)
-    repository = sys.argv[1]
-    if remove_repository(repository):
-        sys.exit(0)
-    else:
-        sys.exit(1)
diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote
index 094366b9..e41341b8 100755
--- a/cdist/conf/type/__apt_ppa/gencode-remote
+++ b/cdist/conf/type/__apt_ppa/gencode-remote
@@ -32,6 +32,6 @@ case "$state_should" in
       echo "add-apt-repository -y '$name'"
    ;;
    absent)
-      echo "remove-apt-repository '$name'"
+      echo "add-apt-repository -r -y '$name'"
    ;;
 esac
diff --git a/cdist/conf/type/__apt_ppa/manifest b/cdist/conf/type/__apt_ppa/manifest
index c6f4e876..57e85442 100755
--- a/cdist/conf/type/__apt_ppa/manifest
+++ b/cdist/conf/type/__apt_ppa/manifest
@@ -20,9 +20,4 @@
 
 __package software-properties-common
 
-require="__package/software-properties-common" \
-   __file /usr/local/bin/remove-apt-repository \
-   --source "$__type/files/remove-apt-repository" \
-   --mode 0755
-
 require="$__object_name" __apt_update_index

From 6f8c774cb08e29c8f0282a8a40a43f1d09c7248e Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@armstrong.cc>
Date: Thu, 14 Apr 2022 00:16:10 +0200
Subject: [PATCH 717/743] workaround mktemp -u checking for write access

Signed-off-by: Steven Armstrong <steven@armstrong.cc>
---
 cdist/conf/type/__file/gencode-local | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local
index bea3d79c..42c9d1e7 100755
--- a/cdist/conf/type/__file/gencode-local
+++ b/cdist/conf/type/__file/gencode-local
@@ -89,7 +89,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then
       touch "$__object/files/set-attributes"
 
       # upload file to temp location
-      upload_destination="$(mktemp -u "${destination}.cdist.XXXXXXXXXX")"
+      upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')"
       # Yes, we are aware that this is a race condition.
       # However:
       # a) cdist usually writes to directories that are not user writable

From 2df2578e36c688fcbdb32a87609bc263d2e7db9c Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@armstrong.cc>
Date: Thu, 14 Apr 2022 00:27:28 +0200
Subject: [PATCH 718/743] __file: remove the questionable check for uploadfile
 existence

Signed-off-by: Steven Armstrong <steven@armstrong.cc>
---
 cdist/conf/type/__file/gencode-local | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local
index 42c9d1e7..aa7149c1 100755
--- a/cdist/conf/type/__file/gencode-local
+++ b/cdist/conf/type/__file/gencode-local
@@ -99,12 +99,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then
       # c) the only case which we could improve are tmp directories and we
       #    don't think managing tmp directories with cdist is a typical case
       #    ("the rest %)"
-      cat << DONE
-$__remote_exec $__target_host test -e $upload_destination && {
-   echo "Refusing to upload file to existing destination: $upload_destination" >&2
-   exit 1
-}
-DONE
+
       # Tell gencode-remote to where we uploaded the file so it can move
       # it to its final destination.
       echo "$upload_destination" > "$__object/files/upload-destination"

From 8b915b15b5a969b7de308e2b1b124d28548349dc Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@armstrong.cc>
Date: Thu, 14 Apr 2022 00:46:13 +0200
Subject: [PATCH 719/743] __file: make the create-empty-file case work again

Signed-off-by: Steven Armstrong <steven@armstrong.cc>
---
 cdist/conf/type/__file/gencode-local | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local
index aa7149c1..f9acdb17 100755
--- a/cdist/conf/type/__file/gencode-local
+++ b/cdist/conf/type/__file/gencode-local
@@ -72,6 +72,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then
          if [ "$type" != "file" ]; then
             # destination is not a regular file, upload source to replace it
             upload_file=1
+            echo upload >> "$__messages_out"
          else
             local_cksum="$(cksum < "$source")"
             remote_cksum="$(cat "$__object/explorer/cksum")"
@@ -88,6 +89,14 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then
       mkdir "$__object/files"
       touch "$__object/files/set-attributes"
 
+      if [ "$create_file" ]; then
+         # When creating an empty file we create it locally and then
+         # upload it so that permissions can be set before moving the file
+         # into place.
+         source="$__object/files/empty"
+         touch "$source"
+      fi
+
       # upload file to temp location
       upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')"
       # Yes, we are aware that this is a race condition.
@@ -104,18 +113,15 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then
       # it to its final destination.
       echo "$upload_destination" > "$__object/files/upload-destination"
 
-      if [ "$upload_file" ]; then
-         echo upload >> "$__messages_out"
-         # IPv6 fix
-         if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$'
-         then
-             my_target_host="[${__target_host}]"
-         else
-             my_target_host="${__target_host}"
-         fi
-         cat << DONE
+      # IPv6 fix
+      if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$'
+      then
+          my_target_host="[${__target_host}]"
+      else
+          my_target_host="${__target_host}"
+      fi
+      cat << DONE
 $__remote_copy "$source" "${my_target_host}:${upload_destination}"
 DONE
-      fi
    fi
 fi

From abbc7dfc376ffb6554bfaad2d5a6ec93955f235f Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@armstrong.cc>
Date: Sat, 16 Apr 2022 19:05:31 +0200
Subject: [PATCH 720/743] since we already remove the destination, we have no
 need to use -T on move, fixes #333

Signed-off-by: Steven Armstrong <steven@armstrong.cc>
---
 cdist/conf/type/__file/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index 136520a7..1a9ff69c 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -91,7 +91,7 @@ case "$state_should" in
         if [ -f "$__object/files/upload-destination" ]; then
             # move uploaded file into place
             printf 'rm -rf "%s"\n' "$final_destination"
-            printf 'mv -T "%s" "%s"\n' "$destination" "$final_destination"
+            printf 'mv "%s" "%s"\n' "$destination" "$final_destination"
         fi
         if [ -f "$__object/files/set-attributes" ]; then
             # set-attributes is created if file is created or uploaded in gencode-local

From 6c8c692a22c886bf82b18f51a133f88a8ab547b7 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@armstrong.cc>
Date: Mon, 2 May 2022 23:25:59 +0200
Subject: [PATCH 721/743] __file: kiss and fix regression on Mac OSX

Signed-off-by: Steven Armstrong <steven@armstrong.cc>
---
 cdist/conf/type/__file/gencode-local | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local
index f9acdb17..5a303308 100755
--- a/cdist/conf/type/__file/gencode-local
+++ b/cdist/conf/type/__file/gencode-local
@@ -98,7 +98,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then
       fi
 
       # upload file to temp location
-      upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')"
+      upload_destination="${destination}.cdist.${__cdist_object_marker}.$$"
       # Yes, we are aware that this is a race condition.
       # However:
       # a) cdist usually writes to directories that are not user writable

From 3d58c9b24fede2d1dafb15fc36424c52c466bb0d Mon Sep 17 00:00:00 2001
From: Stephan Leemburg <sleemburg@it-functions.nl>
Date: Fri, 20 May 2022 13:48:07 +0200
Subject: [PATCH 722/743] add optional file parameter to allow for use in a
 loop without object_id clashes

---
 cdist/conf/type/__dot_file/man.rst  | 15 +++++++++++++++
 cdist/conf/type/__dot_file/manifest | 10 ++++++++--
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__dot_file/man.rst b/cdist/conf/type/__dot_file/man.rst
index ba7621a1..c8f36712 100644
--- a/cdist/conf/type/__dot_file/man.rst
+++ b/cdist/conf/type/__dot_file/man.rst
@@ -37,6 +37,12 @@ state
 source
     forwarded to :strong:`__file` type
 
+file
+    forwarded to :strong:`__file` type
+    This can be used if multiple users need to have a dotfile updated,
+    which will result in duplicate object id errors. When using the
+    file parameter the object id can be some unique value.
+
 MESSAGES
 --------
 
@@ -61,6 +67,15 @@ EXAMPLES
     # Install default xmonad config for user 'eve'. Parent directory is created automatically.
     __dot_file .xmonad/xmonad.hs --user eve --state exists --source "$__files/xmonad.hs"
 
+    # install .vimrc for root and some users
+    for user in root userx usery userz; do
+        __dot_file "${user}_dot_vimrc" \
+            --user $user \
+            --file .vimrc \
+            --state exists \
+            --source "$__files/$user/.vimrc"
+    done
+
 SEE ALSO
 --------
 
diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest
index 02dadf05..a38ed943 100755
--- a/cdist/conf/type/__dot_file/manifest
+++ b/cdist/conf/type/__dot_file/manifest
@@ -20,13 +20,19 @@ user="$(cat "${__object}/parameter/user")"
 home="$(cat "${__object}/explorer/home")"
 primary_group="$(cat "${__object}/explorer/primary_group")"
 dirmode="$(cat "${__object}/parameter/dirmode")"
+if [ -f "${__object}/parameter/file" ]; then
+    file="$(cat "${__object}/parameter/file")"
+else
+    file="${__object_id}"
+fi
+
 
 # Create parent directory. Type __directory has flag 'parents', but it
 # will leave us with root-owned directory in user home, which is not
 # acceptable. So we create parent directories one-by-one. XXX: maybe
 # it should be fixed in '__directory'?
 set --
-subpath=${__object_id}
+subpath=${file}
 while subpath="$(dirname "${subpath}")" ; do
 	[ "${subpath}" = . ] && break
 	set -- "${subpath}" "$@"
@@ -64,4 +70,4 @@ if [ "${source}" = "-" ] ; then
 fi
 unset source
 
-__file "${home}/${__object_id}" --owner "$user" --group "$primary_group" "$@"
+__file "${home}/${file}" --owner "$user" --group "$primary_group" "$@"

From 9839c2d8ec63719c30493870851f1e1ab8079fb1 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 20 May 2022 14:55:12 +0200
Subject: [PATCH 723/743] ++changelog

Signed-off-by: Nico Schottelius <nico@nico-notebook.schottelius.org>
---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 3363be27..29507443 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -15,6 +15,7 @@ next:
 	* Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom)
 	* Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali)
 	* Type __file: make file uploading and attribute changes more atomic	(Steven Armstrong)
+	* Type __dot_file: Add support for using --file parameter (Stephan Leemburg)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From 77d9a757ec0e6986388aa368b77f065ae6be68c8 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 20 May 2022 14:58:45 +0200
Subject: [PATCH 724/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 29507443..8c9e1028 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -16,6 +16,7 @@ next:
 	* Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali)
 	* Type __file: make file uploading and attribute changes more atomic	(Steven Armstrong)
 	* Type __dot_file: Add support for using --file parameter (Stephan Leemburg)
+	* Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From 0ae37b3445c6662a968c49972e0102c75d6f14c2 Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Fri, 1 Jul 2022 14:27:33 +0100
Subject: [PATCH 725/743] Handle signed-by option in __apt_source

Allow users to specify a GPG key fingerprint or keyring file to be
included as the 'signed-by' option.
---
 .../type/__apt_source/files/source.list.template   |  3 ++-
 cdist/conf/type/__apt_source/man.rst               |  8 ++++++++
 cdist/conf/type/__apt_source/manifest              | 14 ++++++++++----
 cdist/conf/type/__apt_source/parameter/optional    |  3 ++-
 4 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__apt_source/files/source.list.template b/cdist/conf/type/__apt_source/files/source.list.template
index d4420e96..a28bb45f 100755
--- a/cdist/conf/type/__apt_source/files/source.list.template
+++ b/cdist/conf/type/__apt_source/files/source.list.template
@@ -2,13 +2,14 @@
 set -u
 
 entry="$uri $distribution $component"
+
 cat << DONE
 # Created by cdist ${__type##*/}
 # Do not change. Changes will be overwritten.
 #
 
 # $name
-deb ${forcedarch} $entry
+deb ${options} $entry
 DONE
 if [ -f "$__object/parameter/include-src" ]; then
    echo "deb-src $entry"
diff --git a/cdist/conf/type/__apt_source/man.rst b/cdist/conf/type/__apt_source/man.rst
index d1acb388..d317a135 100644
--- a/cdist/conf/type/__apt_source/man.rst
+++ b/cdist/conf/type/__apt_source/man.rst
@@ -23,6 +23,9 @@ OPTIONAL PARAMETERS
 arch
    set this if you need to force and specific arch (ubuntu specific)
 
+signed-by
+   provide a GPG key fingerprint or keyring path for signature checks
+
 state
    'present' or 'absent', defaults to 'present'
 
@@ -56,6 +59,11 @@ EXAMPLES
        --uri http://archive.canonical.com/ \
        --component partner --state present
 
+    __apt_source goaccess \
+       --uri http://deb.goaccess.io/ \
+       --component main \
+       --signed-by C03B48887D5E56B046715D3297BD1A0133449C3D
+
 
 AUTHORS
 -------
diff --git a/cdist/conf/type/__apt_source/manifest b/cdist/conf/type/__apt_source/manifest
index 35f15909..cdb526d3 100755
--- a/cdist/conf/type/__apt_source/manifest
+++ b/cdist/conf/type/__apt_source/manifest
@@ -31,9 +31,15 @@ fi
 component="$(cat "$__object/parameter/component")"
 
 if [ -f "$__object/parameter/arch" ]; then
-   forcedarch="[arch=$(cat "$__object/parameter/arch")]"
-else
-   forcedarch=""
+   options="arch=$(cat "$__object/parameter/arch")"
+fi
+
+if [ -f "$__object/parameter/signed-by" ]; then
+   options="$options signed-by=$(cat "$__object/parameter/signed-by")"
+fi
+
+if [ "$options" ]; then
+    options="[$options]"
 fi
 
 # export variables for use in template
@@ -41,7 +47,7 @@ export name
 export uri
 export distribution
 export component
-export forcedarch
+export options
 
 # generate file from template
 mkdir "$__object/files"
diff --git a/cdist/conf/type/__apt_source/parameter/optional b/cdist/conf/type/__apt_source/parameter/optional
index 87537335..0b5470a1 100644
--- a/cdist/conf/type/__apt_source/parameter/optional
+++ b/cdist/conf/type/__apt_source/parameter/optional
@@ -1,4 +1,5 @@
 state
 distribution
 component
-arch
\ No newline at end of file
+arch
+signed-by

From 339ca9347b8dc3d9167393bf754a92c4dedbd704 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Sat, 2 Jul 2022 19:21:27 +0200
Subject: [PATCH 726/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8c9e1028..3d9cb704 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -17,6 +17,7 @@ next:
 	* Type __file: make file uploading and attribute changes more atomic	(Steven Armstrong)
 	* Type __dot_file: Add support for using --file parameter (Stephan Leemburg)
 	* Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues)
+	* Type __apt_source: Add signed-by parameter (Daniel Fancsali)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From 483f0c161440a018efe48bc96910dcb5834c32a1 Mon Sep 17 00:00:00 2001
From: Stephan Leemburg <sleemburg@it-functions.nl>
Date: Wed, 13 Jul 2022 14:50:17 +0200
Subject: [PATCH 727/743] add Check Point Gaia

---
 cdist/conf/explorer/lsb_codename    | 3 +++
 cdist/conf/explorer/lsb_description | 3 +++
 cdist/conf/explorer/lsb_id          | 3 +++
 cdist/conf/explorer/lsb_release     | 3 +++
 cdist/conf/explorer/os              | 7 +++++++
 cdist/conf/explorer/os_release      | 4 ++++
 cdist/conf/explorer/os_version      | 3 +++
 7 files changed, 26 insertions(+)

diff --git a/cdist/conf/explorer/lsb_codename b/cdist/conf/explorer/lsb_codename
index 26bb8e3d..c9fb5cdf 100755
--- a/cdist/conf/explorer/lsb_codename
+++ b/cdist/conf/explorer/lsb_codename
@@ -21,6 +21,9 @@
 
 set +e
 case "$("$__explorer/os")" in
+   checkpoint)
+       awk '{printf("%s\n", $(NF-1))}' /etc/cp-release
+   ;;
    openwrt)
       # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_CODENAME")
diff --git a/cdist/conf/explorer/lsb_description b/cdist/conf/explorer/lsb_description
index b1009627..7279a9c2 100755
--- a/cdist/conf/explorer/lsb_description
+++ b/cdist/conf/explorer/lsb_description
@@ -21,6 +21,9 @@
 
 set +e
 case "$("$__explorer/os")" in
+   checkpoint)
+       cat /etc/cp-release
+   ;;
    openwrt)
       # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION")
diff --git a/cdist/conf/explorer/lsb_id b/cdist/conf/explorer/lsb_id
index 82ff9977..1f91cc40 100755
--- a/cdist/conf/explorer/lsb_id
+++ b/cdist/conf/explorer/lsb_id
@@ -21,6 +21,9 @@
 
 set +e
 case "$("$__explorer/os")" in
+   checkpoint)
+       echo "CheckPoint"
+   ;;
    openwrt)
       # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_ID")
diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release
index 5ebfff1a..d9100569 100755
--- a/cdist/conf/explorer/lsb_release
+++ b/cdist/conf/explorer/lsb_release
@@ -21,6 +21,9 @@
 
 set +e
 case "$("$__explorer/os")" in
+   checkpoint)
+       cat /etc/cp-release|sed -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/'
+   ;;
    openwrt)
       # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_RELEASE")
diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os
index 46d87f3e..b9232ee4 100755
--- a/cdist/conf/explorer/os
+++ b/cdist/conf/explorer/os
@@ -116,6 +116,13 @@ if [ -f /etc/slackware-version ]; then
    exit 0
 fi
 
+# Appliances
+
+if grep -q '^Check Point Gaia' /etc/cp-release 2>/dev/null; then
+    echo checkpoint
+    exit 0
+fi
+
 uname_s="$(uname -s)"
 
 # Assume there is no tr on the client -> do lower case ourselves
diff --git a/cdist/conf/explorer/os_release b/cdist/conf/explorer/os_release
index 6489446b..ec85046f 100644
--- a/cdist/conf/explorer/os_release
+++ b/cdist/conf/explorer/os_release
@@ -34,5 +34,9 @@ elif test -f /var/run/os-release
 then
 	# FreeBSD (created by os-release service)
 	cat /var/run/os-release
+elif test -f /etc/cp-release
+then
+        # Checkpoint firewall or management (actually linux based)
+        cat /etc/cp-release
 fi
 
diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index bbc9e4f0..430200ae 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -41,6 +41,9 @@ in
       # empty, but well...
       cat /etc/arch-release
    ;;
+   checkpoint)
+       awk '{version=$NF; printf("%s\n", substr(version, 2))}' /etc/cp-release
+    ;;
    debian)
       debian_version=$(cat /etc/debian_version)
       case $debian_version

From 624316564531cc7575d2ee0504e3a5f684c74e3d Mon Sep 17 00:00:00 2001
From: Stephan Leemburg <sleemburg@it-functions.nl>
Date: Thu, 28 Jul 2022 16:27:12 +0200
Subject: [PATCH 728/743] add create and ifexists to line type

---
 cdist/conf/type/__line/gencode-remote | 12 ++++++++++--
 cdist/conf/type/__line/man.rst        |  6 ++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote
index a89886da..95c2360d 100755
--- a/cdist/conf/type/__line/gencode-remote
+++ b/cdist/conf/type/__line/gencode-remote
@@ -34,8 +34,16 @@ state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
 
 if [ -z "$state_is" ]; then
-   printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2
-   exit 1
+   if [ -f "$__object/parameter/create" ]; then
+      echo "touch $file"
+   else
+      # only bark if the file should exists
+      if [ -f "$__object/parameter/ifexists" ]; then
+          exit 0
+      fi
+      printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2
+      exit 1
+   fi
 fi
 
 if [ "$state_should" = "$state_is" ] || \
diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst
index 70490f68..dd41fc40 100644
--- a/cdist/conf/type/__line/man.rst
+++ b/cdist/conf/type/__line/man.rst
@@ -21,6 +21,9 @@ OPTIONAL PARAMETERS
 after
     Insert the given line after this pattern.
 
+create
+    It the file does not exist then create an empty file
+
 before
     Insert the given line before this pattern.
 
@@ -28,6 +31,9 @@ file
     If supplied, use this as the destination file.
     Otherwise the object_id is used.
 
+ifexists
+    Only apply the line if the file exists.
+
 line
     Specifies the line which should be absent or present.
 

From 7d8fc8a5c38c47a36ea87cd98d798d22f91c3185 Mon Sep 17 00:00:00 2001
From: Stephan Leemburg <sleemburg@it-functions.nl>
Date: Thu, 28 Jul 2022 17:18:41 +0200
Subject: [PATCH 729/743] improve checkpoint sed, add __line changes

---
 cdist/conf/explorer/lsb_release                   | 2 +-
 cdist/conf/type/__line/parameter/boolean          | 2 ++
 cdist/conf/type/__line/parameter/default/create   | 1 +
 cdist/conf/type/__line/parameter/default/ifexists | 1 +
 4 files changed, 5 insertions(+), 1 deletion(-)
 create mode 100644 cdist/conf/type/__line/parameter/boolean
 create mode 100644 cdist/conf/type/__line/parameter/default/create
 create mode 100644 cdist/conf/type/__line/parameter/default/ifexists

diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release
index d9100569..0bb9f7fe 100755
--- a/cdist/conf/explorer/lsb_release
+++ b/cdist/conf/explorer/lsb_release
@@ -22,7 +22,7 @@
 set +e
 case "$("$__explorer/os")" in
    checkpoint)
-       cat /etc/cp-release|sed -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/'
+       sed /etc/cp-release -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/'
    ;;
    openwrt)
       # shellcheck disable=SC1091
diff --git a/cdist/conf/type/__line/parameter/boolean b/cdist/conf/type/__line/parameter/boolean
new file mode 100644
index 00000000..182a5da6
--- /dev/null
+++ b/cdist/conf/type/__line/parameter/boolean
@@ -0,0 +1,2 @@
+create
+ifexists
diff --git a/cdist/conf/type/__line/parameter/default/create b/cdist/conf/type/__line/parameter/default/create
new file mode 100644
index 00000000..c508d536
--- /dev/null
+++ b/cdist/conf/type/__line/parameter/default/create
@@ -0,0 +1 @@
+false
diff --git a/cdist/conf/type/__line/parameter/default/ifexists b/cdist/conf/type/__line/parameter/default/ifexists
new file mode 100644
index 00000000..c508d536
--- /dev/null
+++ b/cdist/conf/type/__line/parameter/default/ifexists
@@ -0,0 +1 @@
+false

From 17466452f0762fc3a259a66742a59943c5af2510 Mon Sep 17 00:00:00 2001
From: Stephan Leemburg <sleemburg@it-functions.nl>
Date: Thu, 28 Jul 2022 17:53:41 +0200
Subject: [PATCH 730/743] revert __line for clean PR history

---
 cdist/conf/type/__line/gencode-remote             | 12 ++----------
 cdist/conf/type/__line/man.rst                    |  6 ------
 cdist/conf/type/__line/parameter/boolean          |  2 --
 cdist/conf/type/__line/parameter/default/create   |  1 -
 cdist/conf/type/__line/parameter/default/ifexists |  1 -
 5 files changed, 2 insertions(+), 20 deletions(-)
 delete mode 100644 cdist/conf/type/__line/parameter/boolean
 delete mode 100644 cdist/conf/type/__line/parameter/default/create
 delete mode 100644 cdist/conf/type/__line/parameter/default/ifexists

diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote
index 95c2360d..a89886da 100755
--- a/cdist/conf/type/__line/gencode-remote
+++ b/cdist/conf/type/__line/gencode-remote
@@ -34,16 +34,8 @@ state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
 
 if [ -z "$state_is" ]; then
-   if [ -f "$__object/parameter/create" ]; then
-      echo "touch $file"
-   else
-      # only bark if the file should exists
-      if [ -f "$__object/parameter/ifexists" ]; then
-          exit 0
-      fi
-      printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2
-      exit 1
-   fi
+   printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2
+   exit 1
 fi
 
 if [ "$state_should" = "$state_is" ] || \
diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst
index dd41fc40..70490f68 100644
--- a/cdist/conf/type/__line/man.rst
+++ b/cdist/conf/type/__line/man.rst
@@ -21,9 +21,6 @@ OPTIONAL PARAMETERS
 after
     Insert the given line after this pattern.
 
-create
-    It the file does not exist then create an empty file
-
 before
     Insert the given line before this pattern.
 
@@ -31,9 +28,6 @@ file
     If supplied, use this as the destination file.
     Otherwise the object_id is used.
 
-ifexists
-    Only apply the line if the file exists.
-
 line
     Specifies the line which should be absent or present.
 
diff --git a/cdist/conf/type/__line/parameter/boolean b/cdist/conf/type/__line/parameter/boolean
deleted file mode 100644
index 182a5da6..00000000
--- a/cdist/conf/type/__line/parameter/boolean
+++ /dev/null
@@ -1,2 +0,0 @@
-create
-ifexists
diff --git a/cdist/conf/type/__line/parameter/default/create b/cdist/conf/type/__line/parameter/default/create
deleted file mode 100644
index c508d536..00000000
--- a/cdist/conf/type/__line/parameter/default/create
+++ /dev/null
@@ -1 +0,0 @@
-false
diff --git a/cdist/conf/type/__line/parameter/default/ifexists b/cdist/conf/type/__line/parameter/default/ifexists
deleted file mode 100644
index c508d536..00000000
--- a/cdist/conf/type/__line/parameter/default/ifexists
+++ /dev/null
@@ -1 +0,0 @@
-false

From d4bf41ce3b7d186e7e85c030c5960ddd38855a29 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 29 Jul 2022 10:57:01 +0200
Subject: [PATCH 731/743] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 3d9cb704..db2faef0 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -18,6 +18,7 @@ next:
 	* Type __dot_file: Add support for using --file parameter (Stephan Leemburg)
 	* Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues)
 	* Type __apt_source: Add signed-by parameter (Daniel Fancsali)
+	* Explorer: add support for checkpoint (Stephan Leemburg)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From be6e7fcc08d5c4e98f7ce30b157ee087010cd126 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Sun, 31 Jul 2022 21:59:35 +0200
Subject: [PATCH 732/743] Prepare release of cdist 7.0.0

---
 bin/cdist-build-helper | 2 +-
 docs/changelog         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper
index 0380b3f8..cadddae7 100755
--- a/bin/cdist-build-helper
+++ b/bin/cdist-build-helper
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011-2022 Nico Schottelius (nico-cdist at schottelius.org)
 # 2016-2019 Darko Poljak (darko.poljak at gmail.com)
 #
 # This file is part of cdist.
diff --git a/docs/changelog b/docs/changelog
index db2faef0..00defc2a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+7.0.0: 2022-07-31
 	* Explorer machine_type: Rewrite (Dennis Camera)
 	* New type: __sed (Ander Punnar)
 	* New type: __haproxy_dualstack (Evilham and ungleich)

From 90488d2e9e21b4e6bf3b7f0d8c44ebfe29c517b7 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Mon, 1 Aug 2022 00:03:51 +0200
Subject: [PATCH 733/743] [doc] add release process documentation

---
 docs/dev/release-process.org | 90 ++++++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)
 create mode 100644 docs/dev/release-process.org

diff --git a/docs/dev/release-process.org b/docs/dev/release-process.org
new file mode 100644
index 00000000..42b4f5c5
--- /dev/null
+++ b/docs/dev/release-process.org
@@ -0,0 +1,90 @@
+* Install requirements (Alpine)
+  - apk add py3-pycodestyle shellcheck py3-sphinx py3-sphinx_rtd_theme \
+    py3-build twine
+* Ensure your gpg setup works with the email used in the git commit!
+  - For me this is nico@nico-notebook.schottelius.org
+  - Signature / id is on nb2
+* Create ~/.pypirc
+[distutils]
+  index-servers =
+    pypi
+    cdist
+
+[pypi]
+  username = __token__
+  password = ...
+
+[cdist]
+  repository = https://upload.pypi.org/legacy/
+  username = __token__
+  password = ...
+
+* Add date in docs/changelog
+* Run ./bin/cdist-build-helper
+* TODO Move to "build"
+  - python3 -m build
+* DONE git tag: when?
+CLOSED: [2022-07-31 Sun 23:58]
+** Asked during release process: ok
+* DONE Pypi error with distutils: do not use distutils anymore
+CLOSED: [2022-07-31 Sun 23:58]
+python3 setup.py sdist upload
+...
+Creating tar archive
+removing 'cdist-7.0.0' (and everything under it)
+running upload
+Submitting dist/cdist-7.0.0.tar.gz to https://upload.pypi.org/legacy/
+Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest.
+error: Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest.
+(venv2) [22:50] nb2:cdist%
+
+* DONE Pypi error with twine: fixed in twine 4.0.1
+CLOSED: [2022-07-31 Sun 23:58]
+
+Seeing:
+
+(venv2) [22:47] nb2:cdist% twine upload dist/cdist-7.0.0*
+Uploading distributions to https://upload.pypi.org/legacy/
+Traceback (most recent call last):
+  File "/usr/bin/twine", line 8, in <module>
+    sys.exit(main())
+  File "/usr/lib/python3.10/site-packages/twine/__main__.py", line 28, in main
+    result = cli.dispatch(sys.argv[1:])
+  File "/usr/lib/python3.10/site-packages/twine/cli.py", line 68, in dispatch
+    return main(args.args)
+  File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 197, in main
+    return upload(upload_settings, parsed_args.dists)
+  File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 141, in upload
+    resp = repository.upload(package)
+  File "/usr/lib/python3.10/site-packages/twine/repository.py", line 189, in upload
+    resp = self._upload(package)
+  File "/usr/lib/python3.10/site-packages/twine/repository.py", line 144, in _upload
+    data = package.metadata_dictionary()
+  File "/usr/lib/python3.10/site-packages/twine/package.py", line 181, in metadata_dictionary
+    "dynamic": meta.dynamic,
+AttributeError: 'Wheel' object has no attribute 'dynamic'
+
+
+Fix:
+
+
+(venv2) [23:43] nb2:cdist% pipx run twine upload dist/*
+⚠️  twine is already on your PATH and installed at /home/nico/venv2/bin/twine. Downloading and running anyway.
+Uploading distributions to https://upload.pypi.org/legacy/
+Uploading cdist-7.0.0-py3-none-any.whl
+100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 868.6/868.6 kB • 00:04 • 221.3 kB/s
+Uploading cdist-7.0.0.tar.gz
+100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.5/1.5 MB • 00:08 • 169.3 kB/s
+
+View at:
+https://pypi.org/project/cdist/7.0.0/
+* TODO cdist web
+  - on staticweb-2022
+  - Should be moved to sftp/k8s
+
+
+  Manual steps:
+
+  ~/bin/permissions.public html/
+  rsync -a html/ staticweb.ungleich.ch:/home/services/www/nico/www.cdi.st/www/manual/7.0.0/
+  ssh staticweb.ungleich.ch "cd /home/services/www/nico/www.cdi.st/www/manual; ln -sf 7.0.0 latest"

From c85184dcb48118ea5edbda2799badb5779253f7c Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Sun, 18 Sep 2022 08:49:37 +0200
Subject: [PATCH 734/743] Make sure flag is followed by end of line or space.

---
 cdist/conf/explorer/machine_type | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 00646c75..0f212e73 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -515,7 +515,7 @@ check_vm_arch_specific() {
 				&& return 0
 			fi
 			if has_cpuinfo \
-				&& grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo
+				&& grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)\( \|$\) /proc/cpuinfo
 			then
 				return 0
 			fi

From 62db96bb376354c829cdde290c915cffc24612b0 Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Thu, 29 Sep 2022 16:19:07 +0200
Subject: [PATCH 735/743] Initialise options variable so expansion when running
 files/source.list.template there will not be an error when the variable is
 not set.

---
 cdist/conf/type/__apt_source/manifest | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__apt_source/manifest b/cdist/conf/type/__apt_source/manifest
index cdb526d3..fd1ec47f 100755
--- a/cdist/conf/type/__apt_source/manifest
+++ b/cdist/conf/type/__apt_source/manifest
@@ -21,6 +21,7 @@
 name="$__object_id"
 state="$(cat "$__object/parameter/state")"
 uri="$(cat "$__object/parameter/uri")"
+options=""
 
 if [ -f "$__object/parameter/distribution" ]; then
    distribution="$(cat "$__object/parameter/distribution")"

From ffeaafe9b68a944418ec8496c7db9f347dd4b1ee Mon Sep 17 00:00:00 2001
From: marcoduif <marco@vduijvenbode.nl>
Date: Fri, 7 Oct 2022 07:22:31 +0000
Subject: [PATCH 736/743] Make grep more specific

package name should be an exact match, not a substring
---
 cdist/conf/type/__apt_mark/explorer/state | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_mark/explorer/state b/cdist/conf/type/__apt_mark/explorer/state
index b7fe08fa..b464179a 100755
--- a/cdist/conf/type/__apt_mark/explorer/state
+++ b/cdist/conf/type/__apt_mark/explorer/state
@@ -24,4 +24,4 @@ else
     name="$__object_id"
 fi
 
-apt-mark showhold | grep -Fq "$name" && echo hold || echo unhold
+apt-mark showhold | grep -q "^${name}$" && echo hold || echo unhold

From f36069754c0101d1eccbef59fa4f060a19784d26 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Tue, 20 Dec 2022 18:03:15 +0100
Subject: [PATCH 737/743] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 00defc2a..af41368d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+7.0.1:
+	* Type __apt_mark: Narrow down grep for hold packages (marcoduif)
+
 7.0.0: 2022-07-31
 	* Explorer machine_type: Rewrite (Dennis Camera)
 	* New type: __sed (Ander Punnar)

From bdfd92dc3707004503af4a03aea0819ad587a947 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Wed, 21 Dec 2022 09:41:33 +0100
Subject: [PATCH 738/743] ++changes

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index af41368d..e8090131 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 7.0.1:
 	* Type __apt_mark: Narrow down grep for hold packages (marcoduif)
+	* Type __apt_source: Set required options variable (Mark Verboom)
 
 7.0.0: 2022-07-31
 	* Explorer machine_type: Rewrite (Dennis Camera)

From b974969f28f4d007f75904757e80e4f663e3d134 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Mon, 26 Dec 2022 20:59:16 +0100
Subject: [PATCH 739/743] Remove double definition of scan parser

Fixes #353
---
 cdist/argparse.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index f17315e7..8f7bbb85 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -472,9 +472,6 @@ def get_parsers():
     parser['info'].set_defaults(func=cdist.info.Info.commandline)
 
     # Scan = config + further
-    parser['scan'] = parser['sub'].add_parser('scan', add_help=False,
-                                              parents=[parser['config']])
-
     parser['scan'] = parser['sub'].add_parser(
             'scan', parents=[parser['loglevel'],
                              parser['beta'],

From ed3da3c829d47c7c578733482920f111e8bc8869 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Mon, 26 Dec 2022 21:02:41 +0100
Subject: [PATCH 740/743] ++changes

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index e8090131..6ade5d47 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 7.0.1:
 	* Type __apt_mark: Narrow down grep for hold packages (marcoduif)
 	* Type __apt_source: Set required options variable (Mark Verboom)
+	* Core: Remove double definition of scan parser (Nico Schottelius)
 
 7.0.0: 2022-07-31
 	* Explorer machine_type: Rewrite (Dennis Camera)

From 08a6b467fa7afcd2f2a0a2497441cee773166d6e Mon Sep 17 00:00:00 2001
From: Michelle <michelle.quintero@camilion.eu>
Date: Wed, 25 Jan 2023 16:06:35 -0500
Subject: [PATCH 741/743] Added support for Devuan Daedalus

Added one line that allows cdist to support Devuan Daedelus version
---
 cdist/conf/explorer/os_version | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 430200ae..fc59fd14 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -82,6 +82,7 @@ in
             # ceres versions don't have a number, so we decode by codename:
             case ${devuan_version}
             in
+               (daedalus/ceres) echo 4.99 ;;
                (chimaera/ceres) echo 3.99 ;;
                (beowulf/ceres) echo 2.99 ;;
                (ascii/ceres) echo 1.99 ;;

From 1450861e26c8c5dd249517fa14fa8ea0f4961d3d Mon Sep 17 00:00:00 2001
From: Michelle <michelle.quintero@camilion.eu>
Date: Thu, 2 Feb 2023 18:21:43 -0500
Subject: [PATCH 742/743] Updated the python version of cerbot freebsd

The package referenced for cerbot to be install in the FreeBSD platform
used python 3.7 package, updated to python 3.9
---
 cdist/conf/type/__letsencrypt_cert/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 638a99e0..39067f3b 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -85,7 +85,7 @@ if [ -z "${certbot_fullpath}" ]; then
 			esac
 		;;
 		freebsd)
-			__package py37-certbot
+			__package py39-certbot
 			certbot_fullpath="/usr/local/bin/certbot"
 		;;
 		ubuntu)

From 7dd2d1025ae3b005e5d44d11e1ca62536119f36c Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 3 Feb 2023 22:54:13 +0100
Subject: [PATCH 743/743] ++changelog

---
 docs/changelog | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 6ade5d47..4e9a8a6d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -2,9 +2,12 @@ Changelog
 ---------
 
 7.0.1:
+	* Core: Remove double definition of scan parser (Nico Schottelius)
 	* Type __apt_mark: Narrow down grep for hold packages (marcoduif)
 	* Type __apt_source: Set required options variable (Mark Verboom)
-	* Core: Remove double definition of scan parser (Nico Schottelius)
+	* Type __letsencrypt_cert: Update python version (Michelle)
+	* Explorer os_version: Add support for Daedalus (Michelle)
+	* Explorer machine_type: Correct incorrect VMM matching (Mark Verboom)
 
 7.0.0: 2022-07-31
 	* Explorer machine_type: Rewrite (Dennis Camera)