Merge branch 'master' into beta

cdist/conf/explorer/memory

@@ -1,8 +1,9 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2014 Daniel Heule  (hda at sfs.biz)
 # 2014 Thomas Oettli (otho at sfs.biz)
 # Copyright 2017, Philippe Gregoire <pg@pgregoire.xyz>
+# 2020 Dennis Camera <dennis.camera at ssrq-sds-fds.ch>
 #
 # This file is part of cdist.
 #
@@ -19,24 +20,74 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
+# Returns the amount of memory physically installed in the system, or if that
+# cannot be determined the amount available to the operating system kernel,
+# in kibibytes (kiB).
 
-# FIXME: other system types (not linux ...)
+str2bytes() {
+	awk -F' ' '
+	$2 ==   "B" || !$2 { print $1 }
+	$2 ==  "kB" { print $1 * 1000 }
+	$2 ==  "MB" { print $1 * 1000 * 1000 }
+	$2 ==  "GB" { print $1 * 1000 * 1000 * 1000 }
+	$2 ==  "TB" { print $1 * 1000 * 1000 * 1000 * 1000 }
+	$2 == "kiB" { print $1 * 1024 }
+	$2 == "MiB" { print $1 * 1024 * 1024 }
+	$2 == "GiB" { print $1 * 1024 * 1024 * 1024 }
+	$2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }'
+}
 
-os=$("$__explorer/os")
+bytes2kib() {
-case "$os" in
+	set -- "$(cat)"
-    "macosx")
+	test "$1" -gt 0 && echo $(($1 / 1024))
-        echo "$(sysctl -n hw.memsize)/1024" | bc
+}
-    ;;
 
-    *"bsd")
-        PATH=$(getconf PATH)
-        echo "$(sysctl -n hw.physmem) / 1048576" | bc
-    ;;
 
-    *)
+case $(uname -s)
-    if [ -r /proc/meminfo ]; then
+in
-        grep "MemTotal:" /proc/meminfo | awk '{print $2}'
+	(Darwin)
-    fi
+		sysctl -n hw.memsize | bytes2kib
-    ;;
+		;;
+	(FreeBSD)
+		sysctl -n hw.realmem | bytes2kib
+		;;
+	(NetBSD|OpenBSD)
+		# NOTE: This reports "usable" memory, not physically installed memory.
+		command -p sysctl -n hw.physmem | bytes2kib
+		;;
+	(SunOS)
+		# Make sure that awk from xpg4 is used for the scripts to work
+		export PATH="/usr/xpg4/bin:${PATH}"
+		prtconf \
+		| awk -F ': ' '
+		  $1 == "Memory size" { sub(/Megabytes/, "MiB", $2); print $2 }
+		  /^$/ { exit }' \
+		| str2bytes \
+		| bytes2kib
+		;;
+	(Linux)
+		if test -d /sys/devices/system/memory
+		then
+			# Use memory blocks if the architecture (e.g. x86, PPC64, s390)
+			# supports them (they denote physical memory)
+			num_mem_blocks=$(cat /sys/devices/system/memory/memory[0-9]*/state | grep -cxF online)
+			mem_block_size=$(cat /sys/devices/system/memory/block_size_bytes)
+
+			echo $((num_mem_blocks * 0x$mem_block_size)) | bytes2kib && exit
+		fi
+		if test -r /proc/meminfo
+		then
+			# Fall back to meminfo file on other architectures (e.g. ARM, MIPS,
+			# PowerPC)
+			# NOTE: This is "usable" memory, not physically installed memory.
+			awk -F ': +' '$1 == "MemTotal" { sub(/B$/, "iB", $2); print $2 }' /proc/meminfo \
+			| str2bytes \
+			| bytes2kib
+		fi
+		;;
+	(*)
+		printf "Your kernel (%s) is currently not supported by the memory explorer\n" "$(uname -s)" >&2
+		printf "Please contribute an implementation for it if you can.\n" >&2
+		exit 1
+		;;
 esac

cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh

@@ -0,0 +1,84 @@
+#!/bin/sh -e
+
+# It is expected that this defines hook_contents
+
+# Reasonable defaults
+hook_source="${__object}/parameter/${hook}-hook"
+hook_state="absent"
+hook_contents_head="#!/bin/sh -e"
+hook_contents_logic=""
+hook_contents_tail=""
+
+# Backwards compatibility
+# Remove this when renew-hook is removed
+# Falling back to renew-hook if deploy-hook is not passed
+if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then
+	hook_source="${__object}/parameter/renew-hook"
+fi
+if [ "${state}" = "present" ] && \
+	[ -f "${hook_source}" ]; then
+	# This hook is to be installed, let's generate it with some
+	# safety boilerplate
+	# Since certbot runs all hooks for all renewal processes
+	# (at each state for deploy, pre, post), it is up to us to
+	# differentiate whether or not the hook must run
+	hook_state="present"
+	hook_contents_head="$(cat <<EOF
+#!/bin/sh -e
+#
+# Managed remotely with https://cdi.st
+#
+# Domains for which this hook is supposed to apply
+lineage="${LE_DIR}/live/${__object_id}"
+domains="\$(cat <<eof
+${domains}
+eof
+)"
+EOF
+)"
+	case "${hook}" in
+		pre|post)
+			# Certbot is kind of terrible, we have
+			# no way of knowing what domain/lineage the
+			# hook is running for
+			hook_contents_logic="$(cat <<EOF
+# pre/post-hooks apply always due to a certbot limitation
+APPLY_HOOK="YES"
+EOF
+)"
+		;;
+		deploy)
+			hook_contents_logic="$(cat <<EOF
+# certbot defines these environment variables:
+# RENEWED_DOMAINS="DOMAIN1 DOMAIN2"
+# RENEWED_LINEAGE="/etc/letsencrypt/live/__object_id"
+# It feels more stable to use RENEWED_LINEAGE
+if [ "\${lineage}" = "\${RENEWED_LINEAGE}" ]; then
+	APPLY_HOOK="YES"
+fi
+EOF
+)"
+		;;
+		*)
+			echo "Unknown hook '${hook}'" >> /dev/stderr
+			exit 1
+		;;
+	esac
+
+	hook_contents_tail="$(cat <<EOF
+if [ -n "\${APPLY_HOOK}" ]; then
+	# Messing with indentation can eff up the users' scripts, let's not
+$(cat "${hook_source}")
+fi
+EOF
+)"
+fi
+
+hook_contents="$(cat <<EOF
+${hook_contents_head}
+
+${hook_contents_logic}
+
+${hook_contents_tail}
+EOF
+)"

cdist/conf/type/__letsencrypt_cert/man.rst

@@ -1,16 +1,33 @@
 cdist-type__letsencrypt_cert(7)
 ===============================
 
+
 NAME
 ----
 
 cdist-type__letsencrypt_cert - Get an SSL certificate from Let's Encrypt
 
+
 DESCRIPTION
 -----------
 
 Automatically obtain a Let's Encrypt SSL certificate using Certbot.
 
+This type attempts to setup automatic renewals always. In many Linux
+distributions, that is the case out of the box, see:
+https://certbot.eff.org/docs/using.html#automated-renewals
+
+For Alpine Linux and Arch Linux, we setup a system-wide cronjob that
+attempts to renew certificates daily.
+
+If you are using FreeBSD, we configure periodic(8) as recommended by
+the port mantainer, so there will be a weekly attempt at renewal.
+
+If your OS is not mentioned here or on Certbot's docs as having
+support for automated renewals, please make sure you check your OS
+and possibly patch this type so the system-wide cronjob is installed.
+
+
 REQUIRED PARAMETERS
 -------------------
 
@@ -21,6 +38,7 @@ object id
 admin-email
     Where to send Let's Encrypt emails like "certificate needs renewal".
 
+
 OPTIONAL PARAMETERS
 -------------------
 
@@ -36,25 +54,68 @@ webroot
     The path to your webroot, as set up in your webserver config. If this
     parameter is not present, Certbot will be run in standalone mode.
 
+
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
 
-renew-hook
-    Renew hook command directly passed to Certbot in cron job.
-
 domain
     Domains to be included in the certificate. When specified then object id
     is not used as a domain.
 
+deploy-hook
+    Command to be executed only when the certificate associated with this
+    ``$__object_id`` is issued or renewed.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    For this command, the
+    shell variable ``$RENEWED_LINEAGE`` will point to the
+    config live subdirectory (for example,
+    ``/etc/letsencrypt/live/${__object_id}``) containing the
+    new certificates and keys; the shell variable
+    ``$RENEWED_DOMAINS`` will contain a space-delimited list
+    of renewed certificate domains (for example,
+    ``example.com www.example.com``)
+
+pre-hook
+    Command to be run in a shell before obtaining any
+    certificates.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    Note these run regardless of which certificate is attempted, you may want to
+    manage these system-wide hooks with ``__file`` in
+    ``/etc/letsencrypt/renewal-hooks/pre/``.
+
+    Intended primarily for renewal, where it
+    can be used to temporarily shut down a webserver that
+    might conflict with the standalone plugin. This will
+    only be called if a certificate is actually to be
+    obtained/renewed.
+
+post-hook
+    Command to be run in a shell after attempting to
+    obtain/renew certificates.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    Note these run regardless of which certificate was attempted, you may want to
+    manage these system-wide hooks with ``__file`` in
+    ``/etc/letsencrypt/renewal-hooks/post/``.
+
+    Can be used to deploy
+    renewed certificates, or to restart any servers that
+    were stopped by --pre-hook. This is only run if an
+    attempt was made to obtain/renew a certificate.
+
+
 BOOLEAN PARAMETERS
 ------------------
 
-automatic-renewal
-    Install a cron job, which attempts to renew certificates daily.
-
 staging
     Obtain a test certificate from a staging server.
 
+
 MESSAGES
 --------
 
@@ -67,6 +128,7 @@ create
 remove
     Certificate was removed.
 
+
 EXAMPLES
 --------
 
@@ -75,8 +137,7 @@ EXAMPLES
     # use object id as domain
     __letsencrypt_cert example.com \
         --admin-email root@example.com \
-        --automatic-renewal \
+        --deploy-hook "service nginx reload" \
-        --renew-hook "service nginx reload" \
         --webroot /data/letsencrypt/root
 
 .. code-block:: sh
@@ -85,11 +146,10 @@ EXAMPLES
     # and example.com needs to be included again with domain parameter
     __letsencrypt_cert example.com \
         --admin-email root@example.com \
-        --automatic-renewal \
         --domain example.com \
         --domain foo.example.com \
         --domain bar.example.com \
-        --renew-hook "service nginx reload" \
+        --deploy-hook "service nginx reload" \
         --webroot /data/letsencrypt/root
 
 AUTHORS
@@ -99,11 +159,13 @@ AUTHORS
 | Kamila Součková <kamila--@--ksp.sk>
 | Darko Poljak <darko.poljak--@--gmail.com>
 | Ľubomír Kučera <lubomir.kucera.jr at gmail.com>
+| Evilham <contact@evilham.com>
+
 
 COPYING
 -------
 
-Copyright \(C) 2017-2018 Nico Schottelius, Kamila Součková, Darko Poljak and
+Copyright \(C) 2017-2021 Nico Schottelius, Kamila Součková, Darko Poljak and
 Ľubomír Kučera. You can redistribute it and/or modify it under the terms of
 the GNU General Public License as published by the Free Software Foundation,
 either version 3 of the License, or (at your option) any later version.

cdist/conf/type/__letsencrypt_cert/manifest

@@ -1,18 +1,20 @@
 #!/bin/sh
 
 certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")"
+state=$(cat "${__object}/parameter/state")
+os="$(cat "${__global:?}/explorer/os")"
 
 if [ -z "${certbot_fullpath}" ]; then
-	os="$(cat "${__global:?}/explorer/os")"
 	os_version="$(cat "${__global}/explorer/os_version")"
-
+	# Use this, very common value, as a default. It is OS-dependent
+	certbot_fullpath="/usr/bin/certbot"
 	case "$os" in
-        archlinux)
+		archlinux)
-            __package certbot
+			__package certbot
-            ;;
+		;;
-        alpine)
+		alpine)
-            __package certbot
+			__package certbot
-            ;;
+		;;
 		debian)
 			case "$os_version" in
 				8*)
@@ -48,9 +50,7 @@ if [ -z "${certbot_fullpath}" ]; then
 					exit 1
 				;;
 			esac
-
+		;;
-			certbot_fullpath=/usr/bin/certbot
-			;;
 		devuan)
 			case "$os_version" in
 				jessie)
@@ -83,17 +83,14 @@ if [ -z "${certbot_fullpath}" ]; then
 					exit 1
 				;;
 			esac
-
-			certbot_fullpath=/usr/bin/certbot
 		;;
 		freebsd)
-			__package py27-certbot
+			__package py37-certbot
-
+			certbot_fullpath="/usr/local/bin/certbot"
-			certbot_fullpath=/usr/local/bin/certbot
 		;;
 		ubuntu)
-            __package certbot
+			__package certbot
-            ;;
+		;;
 		*)
 			echo "Unsupported os: $os" >&2
 			exit 1
@@ -101,18 +98,61 @@ if [ -z "${certbot_fullpath}" ]; then
 	esac
 fi
 
-if [ -f "${__object}/parameter/automatic-renewal" ]; then
+# Other OS-dependent values that we want to set every time
-	renew_hook_param="${__object}/parameter/renew-hook"
+LE_DIR="/etc/letsencrypt"
-	renew_hook=""
+certbot_cronjob_state="absent"
-	if [ -f "${renew_hook_param}" ]; then
+case "$os" in
-		while read -r hook; do
+	archlinux|alpine)
-			renew_hook="${renew_hook} --renew-hook \"${hook}\""
+		certbot_cronjob_state="present"
-		done < "${renew_hook_param}"
+	;;
-	fi
+	freebsd)
+		LE_DIR="/usr/local/etc/letsencrypt"
+		# FreeBSD uses periodic(8) instead of crontabs for this
+		__line "periodic.conf_weekly_certbot" \
+			--file "/etc/periodic.conf" \
+			--regex "^(#[[:space:]]*)?weekly_certbot_enable=.*" \
+			--state "replace" \
+			--line 'weekly_certbot_enable="YES"'
+	;;
+	*)
+	;;
+esac
 
-	__cron letsencrypt-certbot  \
+# This is only necessary in certain OS
-		--user root \
+__cron letsencrypt-certbot \
-		--command "${certbot_fullpath} renew -q ${renew_hook}" \
+	--user root \
-		--hour 0 \
+	--command "${certbot_fullpath} renew -q" \
-		--minute 47
+	--hour 0 \
+	--minute 47 \
+	--state "${certbot_cronjob_state}"
+
+# Ensure hook directories
+HOOKS_DIR="${LE_DIR}/renewal-hooks"
+__directory "${LE_DIR}" --mode 0755
+require="__directory/${LE_DIR}" __directory "${HOOKS_DIR}" --mode 0755
+
+if [ -f "${__object}/parameter/domain" ]; then
+	domains="$(sort "${__object}/parameter/domain")"
+else
+	domains="${__object_id}"
 fi
+
+# Install hooks as needed
+for hook in deploy pre post; do
+	# Using something unique and specific to this object
+	hook_file="${HOOKS_DIR}/${hook}/${__object_id}.cdist.sh"
+
+	# This defines hook_contents
+	# shellcheck source=cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh
+	. "${__type}/files/gen_hook.sh"
+
+	# Ensure hook directory exists
+	require="__directory/${HOOKS_DIR}" __directory "${HOOKS_DIR}/${hook}" \
+		--mode 0755
+	require="__directory/${HOOKS_DIR}/${hook}" __file "${hook_file}" \
+		--mode 0555 \
+		--source '-' \
+		--state "${hook_state}" <<EOF
+${hook_contents}
+EOF
+done

cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal

@@ -0,0 +1,2 @@
+Deprecated in favour of consistent behaviour. It has no effect, see:
+https://code.ungleich.ch/ungleich-public/cdist/-/issues/853

cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook

@@ -0,0 +1,2 @@
+This parameter has been deprecated in favour of --deploy-hook.
+See: https://code.ungleich.ch/ungleich-public/cdist/-/issues/853

cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple

@@ -1,2 +1,5 @@
+deploy-hook
 domain
+post-hook
+pre-hook
 renew-hook

cdist/conf/type/__package_pip/explorer/distinfo-dir

@@ -0,0 +1,45 @@
+#!/bin/sh
+#
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+nameparam="$__object/parameter/name"
+if [ -f "$nameparam" ]; then
+    name=$(cat "$nameparam")
+else
+    name="$__object_id"
+fi
+
+pipparam="$__object/parameter/pip"
+if [ -f "$pipparam" ]; then
+    pip=$(cat "$pipparam")
+else
+    pip="$( "$__type_explorer/pip" )"
+fi
+
+
+if command -v "$pip" >/dev/null 2>&1; then
+    # assemble the path where pip stores all pip package info
+    "$pip" show "$name" \
+        | awk -F': ' '
+            $1 == "Name" {name=$2; gsub(/-/,"_",name); next}
+            $1 == "Version" {version=$2; next}
+            $1 == "Location" {location=$2; next}
+            END {if (version != "") printf "%s/%s-%s.dist-info", location, name, version}'
+fi

cdist/conf/type/__package_pip/explorer/extras

@@ -0,0 +1,66 @@
+#!/bin/sh
+#
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Checks if the given extras are really installed or not. It will be
+# done by querring all dependencies for that extra and return it as
+# "to be installed" if no dependency was found.
+#
+
+
+distinfo_dir="$("$__type_explorer/distinfo-dir")"
+
+# check if we have something to check
+if [ "$distinfo_dir" ] && [ -s "$__object/parameter/extra" ]
+then
+    # save cause freezing is slow
+    mkdir "$__object/files"
+    pip_freeze="$__object/files/pip-freeze.tmp"
+    pip3 freeze > "$pip_freeze"
+
+    # If all is set, it searches all available extras to separatly check them.
+    # It would work with just 'all' (cause dependencies are specified for
+    # 'all'), but will not update if one extra is already present. Side effect
+    # is that it will not use [all] but instead name all extras seperatly.
+    for extra in $(if grep -qFx all "$__object/parameter/extra";
+        then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA";
+        else tr ',' '\n' < "$__object/parameter/extra";
+        fi)
+    do
+        # create a grep BRE pattern to search all packages
+        # maybe a file full of patterns for -F could be written
+        grep_pattern="$(
+            awk -F'(: | ; )' -v check="$extra" '
+                $1 == "Requires-Dist" {
+                    split($2, r, " ");
+                    sub("extra == ", "", $3); gsub("'"'"'", "", $3);
+                    if($3 == check) print r[1]
+                }' "$distinfo_dir/METADATA" \
+                | sed ':a; $!N; s/\n/\\|/; ta'
+        )"
+
+        # echo the extra if no packages where found for it
+        # if there is no pattern, we don't need to search ;-)
+        # pip matches packages case-insensetive, we need to do that, too
+        if [ "$grep_pattern" ] && ! grep -qi "$grep_pattern" "$pip_freeze"
+        then
+            echo "$extra"
+        fi
+    done
+fi

cdist/conf/type/__package_pip/gencode-remote

@@ -2,6 +2,7 @@
 #
 # 2012 Nico Schottelius (nico-cdist at schottelius.org)
 # 2016 Darko Poljak (darko.poljak at gmail.com)
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
 #
 # This file is part of cdist.
 #
@@ -25,7 +26,10 @@
 state_is=$(cat "$__object/explorer/state")
 state_should="$(cat "$__object/parameter/state")"
 
-[ "$state_is" = "$state_should" ] && exit 0
+# short circuit if state is the same and no extras to install
+[ "$state_is" = "$state_should" ] && ! [ -s "$__object/explorer/extras" ] \
+    && exit 0
+
 
 nameparam="$__object/parameter/name"
 if [ -f "$nameparam" ]; then
@@ -56,6 +60,14 @@ fi
 
 case "$state_should" in
     present)
+        if [ -s "$__object/explorer/extras" ]
+        then
+            # all extras are passed to pip in a comma-separated list in the name
+            # sed loops through all input lines and add commas between them
+            extras="$(sed ':a; $!N; s/\n/,/; ta' "$__object/explorer/extras")"
+            name="${name}[${extras}]"
+        fi
+
         if [ "$runas" ]
         then
             echo "su -c '$pip install -q $name' $runas"

cdist/conf/type/__package_pip/man.rst

@@ -22,6 +22,16 @@ OPTIONAL PARAMETERS
 name
     If supplied, use the name and not the object id as the package name.
 
+extra
+    Extra optional dependencies which should be installed along the selected
+    package. Can be specified multiple times. Multiple extras can be passed
+    in one `--extra` as a comma-separated list.
+
+    Extra optional dependencies will be installed even when the base package
+    is already installed. Notice that the type will not remove installed extras
+    that are not explicitly named for the type because pip does not offer a
+    management for orphaned packages and they may be used by other packages.
+
 pip
     Instead of using pip from PATH, use the specific pip path.
 
@@ -46,6 +56,14 @@ EXAMPLES
     # Use pip in a virtualenv located at /foo/shinken_virtualenv as user foo
     __package_pip pyro --state present --pip /foo/shinken_virtualenv/bin/pip --runas foo
 
+    # Install package with optional dependencies
+    __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails
+    # the extras can also be specified comma-separated
+    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails --extra postgres
+
+    # or take all extras
+    __package_pip mautrix-telegram --extra all
+
 
 SEE ALSO
 --------
@@ -54,12 +72,13 @@ SEE ALSO
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2012 Nico Schottelius. You can redistribute it
+Copyright \(C) 2012 Nico Schottelius, 2021 Matthias Stecher. You can
-and/or modify it under the terms of the GNU General Public License as
+redistribute it and/or modify it under the terms of the GNU General
-published by the Free Software Foundation, either version 3 of the
+Public License as published by the Free Software Foundation, either
-License, or (at your option) any later version.
+version 3 of the License, or (at your option) any later version.

cdist/conf/type/__package_pip/parameter/optional_multiple

@@ -0,0 +1 @@
+extra

cdist/conf/type/__postgres_role/explorer/state

@@ -1,6 +1,7 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -11,32 +12,140 @@
 #
 # cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$("${__explorer}/os")"
+case $("${__explorer:?}/os")
 in
-    netbsd)
+	(netbsd)
-        postgres_user='pgsql'
+		postgres_user='pgsql'
-        ;;
+		;;
-    openbsd)
+	(openbsd)
-        postgres_user='_postgresql'
+		postgres_user='_postgresql'
-        ;;
+		;;
-    *)
+	(*)
-        postgres_user='postgres'
+		postgres_user='postgres'
-        ;;
+		;;
 esac
 
+rolename=${__object_id:?}
 
-name="$__object_id"
 
-if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='$name'\"")"
+psql_query() {
+	su -l "${postgres_user}" -c "$(
+		printf "psql -q -F '\034' -R '\036' -wAc '%s'" \
+		"$(printf %s "$*" | sed "s/'/'\\\\''/g")"
+	)"
+}
+
+password_check_login() (
+	PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .)
+	PGPASSWORD=${PGPASSWORD%?.}
+	export PGPASSWORD
+	psql -q -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
+)
+
+role_properties=$(
+	psql_query "SELECT * FROM pg_roles WHERE rolname = '${rolename}'" \
+	| awk '
+	  BEGIN { RS = "\036"; FS = "\034" }
+	  /^\([0-9]+ rows?\)/ { exit }
+	  NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next }
+	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }
+	  '
+)
+
+if test -n "${role_properties}"
 then
-    echo 'present'
+	# Check if the user's properties match the parameters
+	for prop in login createdb createrole superuser
+	do
+		bool_should=$(test -f "${__object:?}/parameter/${prop}" && echo 't' || echo 'f')
+		bool_is=$(
+			printf '%s\n' "${role_properties}" |
+			awk -F '=' -v key="${prop}" '
+			BEGIN {
+				if (key == "login")
+					key = "canlogin"
+				else if (key == "superuser")
+					key = "super"
+				key = "rol" key
+			}
+			$1 == key {
+				sub(/^[^=]*=/, "")
+				print
+			}
+			'
+		)
+
+		test "${bool_is}" = "${bool_should}" || {
+			state='different properties'
+		}
+	done
+
+	# Check password
+	passwd_stored=$(
+		psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \
+		| awk 'BEGIN { RS = "\036" } NR == 2'
+		printf .
+	)
+	passwd_stored=${passwd_stored%?.}
+
+	if test -f "${__object:?}/parameter/password"
+	then
+		passwd_should=$(cat "${__object:?}/parameter/password"; printf .)
+	fi
+	passwd_should=${passwd_should%?.}
+
+	if test -z "${passwd_stored}"
+	then
+		test -z "${passwd_should}" || state="${state:-different} password"
+	elif expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
+	then
+		# SCRAM-SHA-256 "encrypted" password
+		# NOTE: There is currently no easy way to check SCRAM passwords without
+		#       logging in
+		password_check_login || state="${state:-different} password"
+	elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null
+	then
+		# MD5 "encrypted" password
+		if command -v md5sum >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| md5sum - | sed -e 's/[^0-9a-f]*$//')
+		elif command -v gmd5sum >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| gmd5sum - | sed -e 's/[^0-9a-f]*$//')
+		elif command -v openssl >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| openssl dgst -md5 | sed 's/^.* //')
+		fi
+
+		if test -n "${should_md5}"
+		then
+			test "${passwd_stored}" = "md5${should_md5}" \
+			|| state="${state:-different} password"
+		else
+			password_check_login || state="${state:-different} password"
+		fi
+	else
+		# unencrypted password (unsupported since PostgreSQL 10)
+		test "${passwd_stored}" = "${passwd_should}" \
+		|| state="${state:-different} password"
+	fi
+
+	test -n "${state}" || state='present'
 else
-    echo 'absent'
+	state='absent'
 fi
+
+echo "${state}"

cdist/conf/type/__postgres_role/gencode-remote

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -11,55 +12,117 @@
 #
 # cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
+quote() {
+	if test $# -gt 0
+	then
+		printf '%s' "$*"
+	else
+		cat -
+	fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
+}
+
+case $(cat "${__global:?}/explorer/os")
 in
-    netbsd)
+	(netbsd)
-        postgres_user='pgsql'
+		postgres_user='pgsql'
-        ;;
+		;;
-    openbsd)
+	(openbsd)
-        postgres_user='_postgresql'
+		postgres_user='_postgresql'
-        ;;
+		;;
-    *)
+	(*)
-        postgres_user='postgres'
+		postgres_user='postgres'
-        ;;
+		;;
 esac
 
 
-name="$__object_id"
+rolename=${__object_id:?}
-state_is="$(cat "$__object/explorer/state")"
+state_is=$(cat "${__object:?}/explorer/state")
-state_should="$(cat "$__object/parameter/state")"
+state_should=$(cat "${__object:?}/parameter/state")
 
-[ "$state_is" = "$state_should" ] && exit 0
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
 
-case "$state_should" in
+psql_query() {
-    present)
+	printf 'su -l %s -c %s\n' \
-        if [ -f "$__object/parameter/password" ]; then
+		"$(quote "${postgres_user}")" \
-            password="$(cat "$__object/parameter/password")"
+		"$(quote "psql postgres -q -w -c $(quote "$1")")"
-        fi
+}
-        booleans=""
-        for boolean in login createdb createrole superuser; do
-            if [ ! -f "$__object/parameter/$boolean" ]; then
-                boolean="no${boolean}"
-            fi
-            upper=$(echo $boolean | tr '[:lower:]' '[:upper:]')
-            booleans="$booleans $upper"
-        done
 
-        [ -n "$password" ] && password="PASSWORD '$password'"
+psql_set_password() {
-        cat << EOF
+	# NOTE: Always make sure that the password does not end up in psql_history!
-su - '$postgres_user' -c "psql postgres -wc \"CREATE ROLE \\\\\"$name\\\\\" WITH $password $booleans;\""
+	# NOTE: Never set an empty string as the password, because they can be
-EOF
+	#       interpreted differently by different tooling.
-    ;;
+	if test -s "${__object:?}/parameter/password"
-    absent)
+	then
-        cat << EOF
+		cat <<-EOF
-su - '$postgres_user' -c "dropuser \"$name\""
+		exec 3< "\${__object:?}/parameter/password"
-EOF
+		su -l '${postgres_user}' -c 'psql -q -w postgres' <<'SQL'
-    ;;
+		\set HISTFILE /dev/null
+		\set pw \`cat <&3\`
+		ALTER ROLE "${rolename}" WITH PASSWORD :'pw';
+		SQL
+		exec 3<&-
+		EOF
+	else
+		psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD NULL;"
+	fi
+}
+
+role_properties_should() {
+	_props=
+	for _prop in login createdb createrole superuser
+	do
+		_props="${_props}${_props:+ }$(
+			if test -f "${__object:?}/parameter/${_prop}"
+			then
+				echo "${_prop}"
+			else
+				echo "no${_prop}"
+			fi \
+			| tr '[:lower:]' '[:upper:]')"
+	done
+	printf '%s\n' "${_props}"
+	unset _prop _props
+}
+
+case ${state_should}
+in
+	(present)
+		case ${state_is}
+		in
+			(absent)
+				psql_query "CREATE ROLE \"${rolename}\" WITH $(role_properties_should);"
+				psql_set_password
+				;;
+			(different*)
+				if expr "${state_is}" : 'different.*properties' >/dev/null
+				then
+					psql_query "ALTER ROLE \"${rolename}\" WITH $(role_properties_should);"
+				fi
+
+				if expr "${state_is}" : 'different.*password' >/dev/null
+				then
+					psql_set_password
+				fi
+				;;
+			(*)
+				printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2
+				exit 1
+				;;
+		esac
+		;;
+	(absent)
+		printf 'su -l %s -c %s\n' \
+			"$(quote "${postgres_user}")" \
+			"$(quote "dropuser $(quote "${rolename}")")"
+		;;
 esac

cdist/conf/type/__sshd_config/files/update_sshd_config.awk

@@ -89,7 +89,7 @@ function strdelim(s) { return strdelim_internal(s, 1) }
 function strdelimw(s) { return strdelim_internal(s, 0) }
 
 function singleton_option(opt) {
-	return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|authenticationmethods|authorizedkeysfile|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/
+	return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/
 }
 
 function print_update() {

cdist/conf/type/__sshd_config/gencode-remote

@@ -91,7 +91,8 @@ awk $(drop_awk_comments "${__type:?}/files/update_sshd_config.awk") \\
 
 cmp -s $(quote "${sshd_config_file}") $(quote "${sshd_config_file}.tmp") || {
 	sshd -t -f $(quote "${sshd_config_file}.tmp") \\
-	&& cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}")
+	&& cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}") \\
+	|| exit  # stop if sshd_config file check fails
 }
 rm -f $(quote "${sshd_config_file}.tmp")
 EOF

cdist/conf/type/__sshd_config/man.rst

@@ -79,6 +79,10 @@ BUGS
 - ``Include`` directives are ignored.
 - Config options are not added/removed to/from the config file if their value is
   the default value.
+- | The explorer will incorrectly report ``absent`` if OpenSSH internally
+    transforms one value to another (e.g. ``permitrootlogin prohibit-password``
+    is transformed to ``permitrootlogin without-password``).
+  | Workaround: Use the value that OpenSSH uses internally.
 
 
 AUTHORS

cdist/preos/debootstrap/files/code

@@ -22,7 +22,7 @@ set -e
 if [ "${debug}" ]
 then
     set -x
-    cdist_params="${cdist_params} -d"
+    cdist_params="${cdist_params} -l 3"
 fi
 
 bootstrap_dir="${target_dir}"

cdist/scan/scan.py

@@ -59,6 +59,8 @@ from scapy.all import *
 # Datetime overwrites scapy.all.datetime - needs to be imported AFTER
 import datetime
 
+import cdist.config
+
 log = logging.getLogger("scan")
 
 
@@ -125,6 +127,18 @@ class Scanner(object):
             with open(fname, "w") as fd:
                 fd.write(f"{now}\n")
 
+    def config(self):
+        """
+        Configure a host
+
+        - Assume we are only called if necessary
+        - However we need to ensure to not run in parallel
+        - Maybe keep dict storing per host processes
+        - Save the result
+        - Save the output -> probably aligned to config mode
+
+        """
+
     def start(self):
         self.process = Process(target=self.scan)
         self.process.start()

docs/changelog

@@ -5,6 +5,14 @@ next:
 	* Core: Add trigger functionality (Nico Schottelius, Darko Poljak)
 	* Core: Implement core support for python types (Darko Poljak)
 
+6.9.5: 2021-02-28
+	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
+	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
+	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
+	* Type __postgres_role: Implement modification of roles (Dennis Camera)
+	* Type __letsencrypt_cert: Fix issues with hooks (Evil Ham)
+	* Type __package_pip: Add optional extra dependencies param (Matthias Stecher)
+
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)

docs/dev/logs/2020-10-29.org

@@ -54,4 +54,12 @@ VERBOSE: scan: Host fe80::f29f:c2ff:fe7c:275e is alive
 VERBOSE: scan: Host fe80::ba69:f4ff:fec5:8db7 is alive
 VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f863 is alive
 VERBOSE: scan: Host fe80::21b:fcff:feee:f4bc is alive
-...
+** Better usage -> saving the env
+    sudo -E cdist scan -b -I wlan0 -vv
+** TODO Implement actual configuration step
+   - Also serves as a nice PoC
+   - Might need to escape literal IPv6 addresses for scp
+** TODO Define how to map link local address to something useful
+   - via reverse DNS?
+   - via link local in manifest?
+** TODO define ignorehosts?