From 932e2496ed87b830a468a263cf1b1f5e6a88e2ef Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 15 Dec 2020 18:40:39 +0100
Subject: [PATCH 01/29] [type/__postgres_role] Lint

---
 .../conf/type/__postgres_role/explorer/state  | 32 ++++----
 .../conf/type/__postgres_role/gencode-remote  | 78 ++++++++++---------
 2 files changed, 58 insertions(+), 52 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index c8e1fa9d..110d29d5 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -11,32 +11,32 @@
 #
 # cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$("${__explorer}/os")"
+case $("${__explorer:?}/os")
 in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
+	(netbsd)
+		postgres_user='pgsql'
+		;;
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(*)
+		postgres_user='postgres'
+		;;
 esac
 
 
-name="$__object_id"
+rolename=${__object_id:?}
 
-if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='$name'\"")"
+if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='${rolename}'\"")"
 then
-    echo 'present'
+	echo 'present'
 else
-    echo 'absent'
+	echo 'absent'
 fi
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 282294c9..4fb761ed 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -11,55 +11,61 @@
 #
 # cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
+case $(cat "${__global:?}/explorer/os")
 in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
+	(netbsd)
+		postgres_user='pgsql'
+		;;
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(*)
+		postgres_user='postgres'
+		;;
 esac
 
 
-name="$__object_id"
-state_is="$(cat "$__object/explorer/state")"
-state_should="$(cat "$__object/parameter/state")"
+rolename=${__object_id:?}
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 
-[ "$state_is" = "$state_should" ] && exit 0
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
 
-case "$state_should" in
-    present)
-        if [ -f "$__object/parameter/password" ]; then
-            password="$(cat "$__object/parameter/password")"
-        fi
-        booleans=""
-        for boolean in login createdb createrole superuser; do
-            if [ ! -f "$__object/parameter/$boolean" ]; then
-                boolean="no${boolean}"
-            fi
-            upper=$(echo $boolean | tr '[:lower:]' '[:upper:]')
-            booleans="$booleans $upper"
-        done
+case ${state_should}
+in
+	(present)
+		if test -f "${__object:?}/parameter/password"
+		then
+			password=$(cat "${__object:?}/parameter/password")
+		fi
+		booleans=
+		for boolean in login createdb createrole superuser
+		do
+			if test ! -f "${__object:?}/parameter/${boolean}"
+			then
+				boolean="no${boolean}"
+			fi
+			booleans="${booleans} $(echo ${boolean} | tr '[:lower:]' '[:upper:]')"
+		done
 
-        [ -n "$password" ] && password="PASSWORD '$password'"
-        cat << EOF
-su - '$postgres_user' -c "psql postgres -wc \"CREATE ROLE \\\\\"$name\\\\\" WITH $password $booleans;\""
+		[ -n "${password}" ] && password="PASSWORD '${password}'"
+		cat << EOF
+su - '${postgres_user}' -c "psql postgres -wc 'CREATE ROLE \\"${rolename}\\" WITH ${password} ${booleans};'"
 EOF
-    ;;
-    absent)
-        cat << EOF
-su - '$postgres_user' -c "dropuser \"$name\""
+		;;
+	(absent)
+		cat << EOF
+su - '${postgres_user}' -c "dropuser '${rolename}'"
 EOF
-    ;;
+		;;
 esac

From c36df82882fedb6ce3630e4718c46992a7c153a0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 15 Dec 2020 21:05:55 +0100
Subject: [PATCH 02/29] [type/__postgres_role] ALTER ROLE when parameters
 change

---
 .../conf/type/__postgres_role/explorer/state  | 42 +++++++++++-
 .../conf/type/__postgres_role/gencode-remote  | 65 +++++++++++++++----
 2 files changed, 91 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 110d29d5..033afcb2 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -31,11 +32,48 @@ in
 		;;
 esac
 
-
 rolename=${__object_id:?}
 
-if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='${rolename}'\"")"
+role_properties=$(
+	cmd=$(printf "psql -F '\034' -R '\036' -wAc \"SELECT * FROM pg_roles WHERE rolname='%s'\"" "${rolename}")
+	su -l "${postgres_user}" -c "${cmd}" \
+	| awk '
+	  BEGIN { RS = "\036"; FS = "\034" }
+	  /^\([0-9]+ rows?\)/ { exit }
+	  NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next }
+	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }
+	  '
+)
+
+if test -n "${role_properties}"
 then
+	# Check if the user's properties match the parameters
+	for prop in login createdb createrole superuser
+	do
+		bool_should=$(test -f "${__object:?}/parameter/${prop}" && echo 't' || echo 'f')
+		bool_is=$(
+			printf '%s\n' "${role_properties}" |
+			awk -F '=' -v key="${prop}" '
+			BEGIN {
+				if (key == "login")
+					key = "canlogin"
+				else if (key == "superuser")
+					key = "super"
+				key = "rol" key
+			}
+			$1 == key {
+				sub(/^[^=]*=/, "")
+				print
+			}
+			'
+		)
+
+		test "${bool_is}" = "${bool_should}" || {
+			echo 'different'
+			exit 0
+		}
+	done
+
 	echo 'present'
 else
 	echo 'absent'
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 4fb761ed..7837976c 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,6 +19,15 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+quote() {
+	if test $# -gt 0
+	then
+		printf '%s' "$*"
+	else
+		cat -
+	fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
+}
+
 case $(cat "${__global:?}/explorer/os")
 in
 	(netbsd)
@@ -44,28 +54,55 @@ fi
 case ${state_should}
 in
 	(present)
-		if test -f "${__object:?}/parameter/password"
+		if test -s "${__object:?}/parameter/password"
 		then
-			password=$(cat "${__object:?}/parameter/password")
+			quoted_password=$(
+				delim='$$'
+				while grep -q -F "${delim}" "${__object:?}/parameter/password"
+				do
+					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
+				done
+
+				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
+				# shellcheck disable=SC2016
+				printf '%s%s%s' "${delim}" "${raw_passwd%?.}" "${delim}"
+			)
 		fi
+
 		booleans=
 		for boolean in login createdb createrole superuser
 		do
-			if test ! -f "${__object:?}/parameter/${boolean}"
-			then
-				boolean="no${boolean}"
-			fi
-			booleans="${booleans} $(echo ${boolean} | tr '[:lower:]' '[:upper:]')"
+			booleans="${booleans}${booleans:+ }$(
+				if test -f "${__object:?}/parameter/${boolean}"
+				then
+					echo "${boolean}"
+				else
+					echo "no${boolean}"
+				fi \
+				| tr '[:lower:]' '[:upper:]')"
 		done
 
-		[ -n "${password}" ] && password="PASSWORD '${password}'"
-		cat << EOF
-su - '${postgres_user}' -c "psql postgres -wc 'CREATE ROLE \\"${rolename}\\" WITH ${password} ${booleans};'"
-EOF
+		case ${state_is}
+		in
+			(absent)
+				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				;;
+			(different)
+				query=$(printf 'ALTER ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				;;
+			(*)
+				exit 1	# TODO: error msg
+				;;
+		esac
+
+		psql_cmd=$(printf 'psql postgres -wc %s' "$(quote "${query}")" | quote)
+		printf "su -l '%s' -c %s\\n" "${postgres_user}" "${psql_cmd}"
 		;;
 	(absent)
-		cat << EOF
-su - '${postgres_user}' -c "dropuser '${rolename}'"
-EOF
+		printf "su -l '%s' -c 'dropuser '\\\\'%s\\\\'\\n" \
+			"${postgres_user}" \
+			"$(quote "${rolename}")"
 		;;
 esac

From 7b7ca4d385ccd025cbefced58419468651652da8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 16 Dec 2020 19:06:50 +0100
Subject: [PATCH 03/29] [type/__postgres_role] Handle password changes

---
 .../conf/type/__postgres_role/explorer/state  | 76 +++++++++++++++++--
 .../conf/type/__postgres_role/gencode-remote  | 23 ++++--
 2 files changed, 88 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 033afcb2..4aadbdff 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -34,9 +34,23 @@ esac
 
 rolename=${__object_id:?}
 
+
+psql_query() {
+	su -l "${postgres_user}" -c "$(
+		printf "psql -F '\034' -R '\036' -wAc '%s'" \
+		"$(printf %s "$*" | sed "s/'/'\\\\''/g")"
+	)"
+}
+
+password_check_login() (
+	PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .)
+	PGPASSWORD=${PGPASSWORD%?.}
+	export PGPASSWORD
+	psql -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
+)
+
 role_properties=$(
-	cmd=$(printf "psql -F '\034' -R '\036' -wAc \"SELECT * FROM pg_roles WHERE rolname='%s'\"" "${rolename}")
-	su -l "${postgres_user}" -c "${cmd}" \
+	psql_query "SELECT * FROM pg_roles WHERE rolname = '${rolename}'" \
 	| awk '
 	  BEGIN { RS = "\036"; FS = "\034" }
 	  /^\([0-9]+ rows?\)/ { exit }
@@ -69,12 +83,62 @@ then
 		)
 
 		test "${bool_is}" = "${bool_should}" || {
-			echo 'different'
-			exit 0
+			state='different properties'
 		}
 	done
 
-	echo 'present'
+	# Check password
+	passwd_stored=$(
+		psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \
+		| awk 'BEGIN { RS = "\036" } NR == 2'
+		printf .
+	)
+	passwd_stored=${passwd_stored%?.}
+
+	passwd_should=$(cat "${__object}/parameter/password"; printf .)
+	passwd_should=${passwd_should%?.}
+
+	if expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
+	then
+		# SCRAM-SHA-256 "encrypted" password
+		# NOTE: There is currently no easy way to check SCRAM passwords
+		password_check_login || state="${state:-different} password"
+	elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null
+	then
+		# MD5 "encrypted" password
+		if command -v md5sum >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| md5sum - | sed -e 's/[^0-9a-f]*$//')
+		elif command -v gmd5sum >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| gmd5sum - | sed -e 's/[^0-9a-f]*$//')
+		elif command -v openssl >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| openssl dgst -md5 | sed 's/^.* //')
+		fi
+
+		if test -n "${should_md5}"
+		then
+			test "${passwd_stored}" = "md5${should_md5}" \
+			|| state="${state:-different} password"
+		else
+			password_check_login || state="${state:-different} password"
+		fi
+	else
+		# unencrypted password (unsupported since PostgreSQL 10)
+		test "${passwd_stored}" = "${passwd_should}" \
+		|| state="${state:-different} password"
+	fi
+
+	test -n "${state}" || state='present'
 else
-	echo 'absent'
+	state='absent'
 fi
+
+echo "${state}"
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 7837976c..fd590a4b 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -58,7 +58,9 @@ in
 		then
 			quoted_password=$(
 				delim='$$'
-				while grep -q -F "${delim}" "${__object:?}/parameter/password"
+				# NOTE: Strip away trailing $ because with it the check breaks
+				#       if the password ends with $ + random value.
+				while grep -q -F "${delim%$}" "${__object:?}/parameter/password"
 				do
 					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
 				done
@@ -88,12 +90,23 @@ in
 				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
 					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
 				;;
-			(different)
-				query=$(printf 'ALTER ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+			(different*)
+				query="ALTER ROLE \"${rolename}\" WITH"
+
+				if expr "${state_is}" : 'different.*properties' >/dev/null
+				then
+					query="${query} ${booleans}"
+				fi
+				if expr "${state_is}" : 'different.*password' >/dev/null
+				then
+					query="${query} PASSWORD ${quoted_password:-NULL}"
+				fi
+
+				query="${query};"
 				;;
 			(*)
-				exit 1	# TODO: error msg
+				printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2
+				exit 1
 				;;
 		esac
 

From 4859c27900d125841c703e0270dd8c0eae05d504 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 16:57:03 +0100
Subject: [PATCH 04/29] [type/__postgres_role] Refactor gencode-remote

---
 .../conf/type/__postgres_role/gencode-remote  | 65 ++++++++++---------
 1 file changed, 34 insertions(+), 31 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index fd590a4b..540eb606 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -51,9 +51,6 @@ then
 	exit 0
 fi
 
-case ${state_should}
-in
-	(present)
 		if test -s "${__object:?}/parameter/password"
 		then
 			quoted_password=$(
@@ -64,6 +61,11 @@ in
 				do
 					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
 				done
+psql_query() {
+	printf 'su -l %s -c %s\n' \
+		"$(quote "${postgres_user}")" \
+		"$(quote "psql postgres -wc $(quote "$1")")"
+}
 
 				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
 				# shellcheck disable=SC2016
@@ -71,51 +73,52 @@ in
 			)
 		fi
 
-		booleans=
-		for boolean in login createdb createrole superuser
-		do
-			booleans="${booleans}${booleans:+ }$(
-				if test -f "${__object:?}/parameter/${boolean}"
-				then
-					echo "${boolean}"
-				else
-					echo "no${boolean}"
-				fi \
-				| tr '[:lower:]' '[:upper:]')"
-		done
+role_properties_should() {
+	_props=
+	for _prop in login createdb createrole superuser
+	do
+		_props="${_props}${_props:+ }$(
+			if test -f "${__object:?}/parameter/${_prop}"
+			then
+				echo "${_prop}"
+			else
+				echo "no${_prop}"
+			fi \
+			| tr '[:lower:]' '[:upper:]')"
+	done
+	printf '%s\n' "${_props}"
+	unset _prop _props
+}
 
+case ${state_should}
+in
+	(present)
 		case ${state_is}
 		in
 			(absent)
-				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				psql_query "$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "$(role_properties_should)" "${quoted_password:-NULL}")"
 				;;
 			(different*)
-				query="ALTER ROLE \"${rolename}\" WITH"
-
 				if expr "${state_is}" : 'different.*properties' >/dev/null
 				then
-					query="${query} ${booleans}"
-				fi
-				if expr "${state_is}" : 'different.*password' >/dev/null
-				then
-					query="${query} PASSWORD ${quoted_password:-NULL}"
+					psql_query "ALTER ROLE \"${rolename}\" WITH $(role_properties_should);"
 				fi
 
-				query="${query};"
+				if expr "${state_is}" : 'different.*password' >/dev/null
+				then
+					psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD ${quoted_password:-NULL};"
+				fi
 				;;
 			(*)
 				printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2
 				exit 1
 				;;
 		esac
-
-		psql_cmd=$(printf 'psql postgres -wc %s' "$(quote "${query}")" | quote)
-		printf "su -l '%s' -c %s\\n" "${postgres_user}" "${psql_cmd}"
 		;;
 	(absent)
-		printf "su -l '%s' -c 'dropuser '\\\\'%s\\\\'\\n" \
-			"${postgres_user}" \
-			"$(quote "${rolename}")"
+		printf 'su -l %s -c %s\n' \
+			"$(quote "${postgres_user}")" \
+			"$(quote "dropuser $(quote "${rolename}")")"
 		;;
 esac

From 1180f13ed6c12e0277bebdf32bdd9840320d0fb9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 16:58:32 +0100
Subject: [PATCH 05/29] [type/__postgres_role] Fix setting password

We need to make sure that the password does not end up in ~/.psql_history.
---
 .../conf/type/__postgres_role/gencode-remote  | 38 ++++++++++---------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 540eb606..15c3e692 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -51,27 +51,29 @@ then
 	exit 0
 fi
 
-		if test -s "${__object:?}/parameter/password"
-		then
-			quoted_password=$(
-				delim='$$'
-				# NOTE: Strip away trailing $ because with it the check breaks
-				#       if the password ends with $ + random value.
-				while grep -q -F "${delim%$}" "${__object:?}/parameter/password"
-				do
-					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
-				done
 psql_query() {
 	printf 'su -l %s -c %s\n' \
 		"$(quote "${postgres_user}")" \
 		"$(quote "psql postgres -wc $(quote "$1")")"
 }
 
-				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
-				# shellcheck disable=SC2016
-				printf '%s%s%s' "${delim}" "${raw_passwd%?.}" "${delim}"
-			)
-		fi
+psql_set_password() {
+	# NOTE: Always make sure that the password does not end up in psql_history!
+	if test -s "${__object:?}/parameter/password"
+	then
+		cat <<-EOF
+		exec 3< "\${__object:?}/parameter/password"
+		su -l '${postgres_user}' -c 'psql -q postgres -w' <<'SQL'
+		\set HISTFILE /dev/null
+		\set pw \`cat <&3\`
+		ALTER ROLE "${rolename}" WITH PASSWORD :'pw';
+		SQL
+		exec 3<&-
+		EOF
+	else
+		psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD NULL;"
+	fi
+}
 
 role_properties_should() {
 	_props=
@@ -96,8 +98,8 @@ in
 		case ${state_is}
 		in
 			(absent)
-				psql_query "$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "$(role_properties_should)" "${quoted_password:-NULL}")"
+				psql_query "CREATE ROLE \"${rolename}\" WITH $(role_properties_should);"
+				psql_set_password
 				;;
 			(different*)
 				if expr "${state_is}" : 'different.*properties' >/dev/null
@@ -107,7 +109,7 @@ in
 
 				if expr "${state_is}" : 'different.*password' >/dev/null
 				then
-					psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD ${quoted_password:-NULL};"
+					psql_set_password
 				fi
 				;;
 			(*)

From 99d82fd0d5ab6d6ccd95fe8a7d3b24f5d82a4cce Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 17:05:58 +0100
Subject: [PATCH 06/29] [type/__postgres_role] Always set psql -q

---
 cdist/conf/type/__postgres_role/explorer/state | 4 ++--
 cdist/conf/type/__postgres_role/gencode-remote | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 4aadbdff..77779662 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -37,7 +37,7 @@ rolename=${__object_id:?}
 
 psql_query() {
 	su -l "${postgres_user}" -c "$(
-		printf "psql -F '\034' -R '\036' -wAc '%s'" \
+		printf "psql -q -F '\034' -R '\036' -wAc '%s'" \
 		"$(printf %s "$*" | sed "s/'/'\\\\''/g")"
 	)"
 }
@@ -46,7 +46,7 @@ password_check_login() (
 	PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .)
 	PGPASSWORD=${PGPASSWORD%?.}
 	export PGPASSWORD
-	psql -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
+	psql -q -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
 )
 
 role_properties=$(
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 15c3e692..877c8135 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -54,7 +54,7 @@ fi
 psql_query() {
 	printf 'su -l %s -c %s\n' \
 		"$(quote "${postgres_user}")" \
-		"$(quote "psql postgres -wc $(quote "$1")")"
+		"$(quote "psql postgres -q -w -c $(quote "$1")")"
 }
 
 psql_set_password() {
@@ -63,7 +63,7 @@ psql_set_password() {
 	then
 		cat <<-EOF
 		exec 3< "\${__object:?}/parameter/password"
-		su -l '${postgres_user}' -c 'psql -q postgres -w' <<'SQL'
+		su -l '${postgres_user}' -c 'psql -q -w postgres' <<'SQL'
 		\set HISTFILE /dev/null
 		\set pw \`cat <&3\`
 		ALTER ROLE "${rolename}" WITH PASSWORD :'pw';

From 2954347771e00610a084c7b033016765f7a3d6d7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 14 Jan 2021 13:46:40 +0100
Subject: [PATCH 07/29] [type/__postgres_role] Add note regarding empty
 passwords

---
 cdist/conf/type/__postgres_role/gencode-remote | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 877c8135..d7631fbd 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -59,6 +59,8 @@ psql_query() {
 
 psql_set_password() {
 	# NOTE: Always make sure that the password does not end up in psql_history!
+	# NOTE: Never set an empty string as the password, because they can be
+	#       interpreted differently by different tooling.
 	if test -s "${__object:?}/parameter/password"
 	then
 		cat <<-EOF

From 35cde3e666c229ceb94a1c5074083f7ba905768a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 13:09:29 +0100
Subject: [PATCH 08/29] [type/__postgres_role] Fix state explorer when stored
 password is empty

---
 cdist/conf/type/__postgres_role/explorer/state | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 77779662..34069de9 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -95,13 +95,20 @@ then
 	)
 	passwd_stored=${passwd_stored%?.}
 
-	passwd_should=$(cat "${__object}/parameter/password"; printf .)
+	if test -f "${__object:?}/parameter/password"
+	then
+		passwd_should=$(cat "${__object:?}/parameter/password"; printf .)
+	fi
 	passwd_should=${passwd_should%?.}
 
-	if expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
+	if test -z "${passwd_stored}"
+	then
+		test -z "${passwd_should}" || state="${state:-different} password"
+	elif expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
 	then
 		# SCRAM-SHA-256 "encrypted" password
-		# NOTE: There is currently no easy way to check SCRAM passwords
+		# NOTE: There is currently no easy way to check SCRAM passwords without
+		#       logging in
 		password_check_login || state="${state:-different} password"
 	elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null
 	then

From 8eccacec59b0badd7a6e2a40a27f02f0ff355ccf Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 4 Feb 2021 19:09:26 +0100
Subject: [PATCH 09/29] __package_pip: add optional dependencies

This is a poor implementation of optional dependencies for pip packages.
It ensures to install them if the package will be installed, but does
not take into account if they must be added/removed after the package is
already installed. Also, it will not be autoremoved, as all dependencies
will not be removed.
---
 cdist/conf/type/__package_pip/gencode-remote        | 13 +++++++++++++
 cdist/conf/type/__package_pip/man.rst               | 13 +++++++++++++
 .../type/__package_pip/parameter/optional_multiple  |  1 +
 3 files changed, 27 insertions(+)
 create mode 100644 cdist/conf/type/__package_pip/parameter/optional_multiple

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index a1375c2d..58c89040 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -56,6 +56,19 @@ fi
 
 case "$state_should" in
     present)
+        if [ -f "$__object/parameter/extra" ]
+        then
+            while read extra
+            do
+                if [ "$extras" ]; then
+                    extras="$extras,$extra"
+                else
+                    extras="$extra"
+                fi
+            done < "$__object/parameter/extra"
+            name="${name}[${extras}]"
+        fi
+
         if [ "$runas" ]
         then
             echo "su -c '$pip install -q $name' $runas"
diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index 234ceee2..64ad0358 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -22,6 +22,11 @@ OPTIONAL PARAMETERS
 name
     If supplied, use the name and not the object id as the package name.
 
+extra
+    Extra optional dependencies which should be installed along the selected
+    package. Can be specified multiple times. Will only be applied if the
+    package actually will be installed, but will not explicitly checked.
+
 pip
     Instead of using pip from PATH, use the specific pip path.
 
@@ -46,6 +51,14 @@ EXAMPLES
     # Use pip in a virtualenv located at /foo/shinken_virtualenv as user foo
     __package_pip pyro --state present --pip /foo/shinken_virtualenv/bin/pip --runas foo
 
+    # Install package with optional dependencies
+    __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails
+    # or do a little cheating
+    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails
+
+    # or take all extras
+    __package_pip mautrix-telegram --extra all
+
 
 SEE ALSO
 --------
diff --git a/cdist/conf/type/__package_pip/parameter/optional_multiple b/cdist/conf/type/__package_pip/parameter/optional_multiple
new file mode 100644
index 00000000..0f228715
--- /dev/null
+++ b/cdist/conf/type/__package_pip/parameter/optional_multiple
@@ -0,0 +1 @@
+extra

From 73a03d75d7d23fea292b5fd60aeb059d8c72c4ba Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 4 Feb 2021 19:18:02 +0100
Subject: [PATCH 10/29] __package_pip: fix shellcheck

---
 cdist/conf/type/__package_pip/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 58c89040..925eaf24 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -58,7 +58,7 @@ case "$state_should" in
     present)
         if [ -f "$__object/parameter/extra" ]
         then
-            while read extra
+            while read -r extra
             do
                 if [ "$extras" ]; then
                     extras="$extras,$extra"

From 4717e5ceff078111748824e4fb365df7979e5fe7 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 10:31:07 +0100
Subject: [PATCH 11/29] __package_pip: add extras explorer

The two new explorers detect all installed extras for this package.
---
 .../type/__package_pip/explorer/distinfo-dir  | 45 +++++++++++++++++++
 cdist/conf/type/__package_pip/explorer/extras | 26 +++++++++++
 cdist/conf/type/__package_pip/explorer/state  |  0
 3 files changed, 71 insertions(+)
 create mode 100755 cdist/conf/type/__package_pip/explorer/distinfo-dir
 create mode 100755 cdist/conf/type/__package_pip/explorer/extras
 mode change 100644 => 100755 cdist/conf/type/__package_pip/explorer/state

diff --git a/cdist/conf/type/__package_pip/explorer/distinfo-dir b/cdist/conf/type/__package_pip/explorer/distinfo-dir
new file mode 100755
index 00000000..18e169ae
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/distinfo-dir
@@ -0,0 +1,45 @@
+#!/bin/sh
+#
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+nameparam="$__object/parameter/name"
+if [ -f "$nameparam" ]; then
+    name=$(cat "$nameparam")
+else
+    name="$__object_id"
+fi
+
+pipparam="$__object/parameter/pip"
+if [ -f "$pipparam" ]; then
+    pip=$(cat "$pipparam")
+else
+    pip="$( "$__type_explorer/pip" )"
+fi
+
+
+if command -v "$pip" >/dev/null 2>&1; then
+    # assemble the path where pip stores all pip package info
+    "$pip" show "$name" \
+        | awk -F': ' '
+            $1 == "Name" {name=$2; gsub(/-/,"_",name); next}
+            $1 == "Version" {version=$2; next}
+            $1 == "Location" {location=$2; next}
+            END {if (version != "") printf "%s/%s-%s.dist-info", location, name, version}'
+fi
diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
new file mode 100755
index 00000000..92fc4732
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -0,0 +1,26 @@
+#!/bin/sh
+#
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+distinfo_dir="$("$__type_explorer/distinfo-dir")"
+if [ "$distinfo_dir" ]; then
+    # output all extras that are installed
+    awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA"
+fi
diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state
old mode 100644
new mode 100755

From 8dc6ab97384f2240e9218fbd4463ba12b004f44c Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 13:49:53 +0100
Subject: [PATCH 12/29] __package_pip: install not found extras

Compares the explorer against the parameters and install those extras
that are not already installed.
---
 cdist/conf/type/__package_pip/gencode-remote | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 925eaf24..ad327e37 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -2,6 +2,7 @@
 #
 # 2012 Nico Schottelius (nico-cdist at schottelius.org)
 # 2016 Darko Poljak (darko.poljak at gmail.com)
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
 #
 # This file is part of cdist.
 #
@@ -58,14 +59,17 @@ case "$state_should" in
     present)
         if [ -f "$__object/parameter/extra" ]
         then
-            while read -r extra
-            do
-                if [ "$extras" ]; then
-                    extras="$extras,$extra"
-                else
-                    extras="$extra"
-                fi
-            done < "$__object/parameter/extra"
+            mkdir "$__object/files"
+            # sort and generalize all extras
+            sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params"
+            sort "$__object/explorer/extras" > "$__object/files/extras-explorer"
+
+            # assemble extras
+            # all extras are passed to pip in a comma-separated list
+            extras="$(comm -23 \
+                "$__object/files/extras-params" "$__object/files/extras-explorer" \
+                | sed ':a; $!N; s/\n/,/; ta'  # loops through all input lines and add commas between them
+            )"
             name="${name}[${extras}]"
         fi
 

From 2db0ef7c98226661e5e4b55c89180b31ba65b8bc Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 22:53:26 +0100
Subject: [PATCH 13/29] __package_pip: updating real detection of extras

As the previous detection took the wrong values, this explorer now
checks if packages for an extra are installed or not. If not, the extra
is not installed.

Based on the information of the explorer, it will install the package
again with the absent extras.
---
 cdist/conf/type/__package_pip/explorer/extras | 37 +++++++++++++++++--
 cdist/conf/type/__package_pip/gencode-remote  | 21 ++++-------
 2 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 92fc4732..2f5fcca6 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -17,10 +17,41 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+#
+# Checks if the given extras are really installed or not. It will be
+# done by querring all dependencies for that extra and return it as
+# "to be installed" if no dependency was found.
+#
 
 
 distinfo_dir="$("$__type_explorer/distinfo-dir")"
-if [ "$distinfo_dir" ]; then
-    # output all extras that are installed
-    awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA"
+
+# check if we have something to check
+if [ "$distinfo_dir" ] && [ -s "$__object/parameter/extra" ]
+then
+    # save cause freezing is slow
+    mkdir "$__object/files"
+    pip_freeze="$__object/files/pip-freeze.tmp"
+    pip3 freeze > "$pip_freeze"
+
+    for extra in $(cat "$__object/parameter/extra" | tr ',' '\n')
+    do
+        # create a grep BRE pattern to search all packages
+        grep_pattern="$(
+            awk -F'(: | ; )' -v check="$extra" '
+                $1 == "Requires-Dist" {
+                    split($2, r, " ");
+                    sub("extra == ", "", $3); gsub("'"'"'", "", $3);
+                    if($3 == check) print r[1]
+                }' "$distinfo_dir/METADATA" \
+                | sed ':a; $!N; s/\n/\\|/; ta'
+        )"
+
+        # echo the extra if no packages where found for it
+        # if there is no pattern, we don't need to search ;-)
+        if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze"
+        then
+            echo "$extra"
+        fi
+    done
 fi
diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index ad327e37..9abe28bf 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -26,7 +26,10 @@
 state_is=$(cat "$__object/explorer/state")
 state_should="$(cat "$__object/parameter/state")"
 
-[ "$state_is" = "$state_should" ] && exit 0
+# short circuit if state is the same and no extras to install
+[ "$state_is" = "$state_should" ] && ! [ -s "$__object/explorer/extras" ] \
+    && exit 0
+
 
 nameparam="$__object/parameter/name"
 if [ -f "$nameparam" ]; then
@@ -57,19 +60,11 @@ fi
 
 case "$state_should" in
     present)
-        if [ -f "$__object/parameter/extra" ]
+        if [ -s "$__object/explorer/extras" ]
         then
-            mkdir "$__object/files"
-            # sort and generalize all extras
-            sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params"
-            sort "$__object/explorer/extras" > "$__object/files/extras-explorer"
-
-            # assemble extras
-            # all extras are passed to pip in a comma-separated list
-            extras="$(comm -23 \
-                "$__object/files/extras-params" "$__object/files/extras-explorer" \
-                | sed ':a; $!N; s/\n/,/; ta'  # loops through all input lines and add commas between them
-            )"
+            # all extras are passed to pip in a comma-separated list in the name
+            # sed loops through all input lines and add commas between them
+            extras="$(sed ':a; $!N; s/\n/,/; ta' "$__object/explorer/extras")"
             name="${name}[${extras}]"
         fi
 

From 7398382890a4368e588e700183e6c777e90fb069 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 23:12:10 +0100
Subject: [PATCH 14/29] __package_pip: fix shellcheck

Useless `cat $file`, use `< $file` instead.
---
 cdist/conf/type/__package_pip/explorer/extras | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 2f5fcca6..d57c69db 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -34,7 +34,7 @@ then
     pip_freeze="$__object/files/pip-freeze.tmp"
     pip3 freeze > "$pip_freeze"
 
-    for extra in $(cat "$__object/parameter/extra" | tr ',' '\n')
+    for extra in $(tr ',' '\n' < "$__object/parameter/extra")
     do
         # create a grep BRE pattern to search all packages
         grep_pattern="$(

From a9d7dfb2ed40d0a61978cc706ff469c354a6c8f5 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 12 Feb 2021 09:17:02 +0100
Subject: [PATCH 15/29] __package_pip: split extra 'all' to a list of all
 extras

This will fix if a package will be upgraded from some extras to all
extras. Previously, it will not work because some dependencies of 'all'
are already installed, so the feature 'all' is already installed.

Now, it will use a list of all extras to iterate over them separatly. This
will result it will never install all extras via `[all]`, but rather
`[foo,bar]`.
---
 cdist/conf/type/__package_pip/explorer/extras | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index d57c69db..be945e36 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -34,7 +34,14 @@ then
     pip_freeze="$__object/files/pip-freeze.tmp"
     pip3 freeze > "$pip_freeze"
 
-    for extra in $(tr ',' '\n' < "$__object/parameter/extra")
+    # If all is set, it searches all available extras to separatly check them.
+    # It would work with just 'all' (cause dependencies are given), but will
+    # not update if one extra is already present. Side effect is that it will
+    # not use [all] but instead name all extras seperatly.
+    for extra in $(if grep -qFx all "$__object/parameter/extra";
+        then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA";
+        else tr ',' '\n' < "$__object/parameter/extra";
+        fi)
     do
         # create a grep BRE pattern to search all packages
         grep_pattern="$(

From 951712740f95f8f1277f1295bcd97c0b9bafdc94 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 12 Feb 2021 13:42:51 +0100
Subject: [PATCH 16/29] __package_pip: update man.rst

Adjusted comments for `explorer/extras` and updated the man page for the
new behaviour of updating the extras.
---
 cdist/conf/type/__package_pip/explorer/extras |  7 +++---
 cdist/conf/type/__package_pip/man.rst         | 25 ++++++++++++-------
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index be945e36..8104ed47 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -35,15 +35,16 @@ then
     pip3 freeze > "$pip_freeze"
 
     # If all is set, it searches all available extras to separatly check them.
-    # It would work with just 'all' (cause dependencies are given), but will
-    # not update if one extra is already present. Side effect is that it will
-    # not use [all] but instead name all extras seperatly.
+    # It would work with just 'all' (cause dependencies are specified for
+    # 'all'), but will not update if one extra is already present. Side effect
+    # is that it will not use [all] but instead name all extras seperatly.
     for extra in $(if grep -qFx all "$__object/parameter/extra";
         then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA";
         else tr ',' '\n' < "$__object/parameter/extra";
         fi)
     do
         # create a grep BRE pattern to search all packages
+        # maybe a file full of patterns for -F could be written
         grep_pattern="$(
             awk -F'(: | ; )' -v check="$extra" '
                 $1 == "Requires-Dist" {
diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index 64ad0358..c013695c 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -24,8 +24,14 @@ name
 
 extra
     Extra optional dependencies which should be installed along the selected
-    package. Can be specified multiple times. Will only be applied if the
-    package actually will be installed, but will not explicitly checked.
+    package. Can be specified multiple times. Multiple extra optional
+    dependencies can also be specified in a comma-separated list to provide
+    a more pip-natvie style.
+
+    Extra optional dependencies will be installed even when the base package
+    is already installed. Notice that the type will not remove installed extras
+    that are not explicitly named for the type because pip does not offer a
+    management for orphaned packages and they may be used by other packages.
 
 pip
     Instead of using pip from PATH, use the specific pip path.
@@ -53,8 +59,8 @@ EXAMPLES
 
     # Install package with optional dependencies
     __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails
-    # or do a little cheating
-    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails
+    # the extras can also be specified comma-separated
+    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails --extra postgres
 
     # or take all extras
     __package_pip mautrix-telegram --extra all
@@ -67,12 +73,13 @@ SEE ALSO
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2012 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2012 Nico Schottelius, 2021 Matthias Stecher. You can
+redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either
+version 3 of the License, or (at your option) any later version.

From 2ce1fce7677b64a8d17a86f29f676b1ba8475db2 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 15 Feb 2021 16:17:46 +0100
Subject: [PATCH 17/29] __package_pip: match package names case insensitive

Pip matches them insensitive, so we need to do the same to avoid
problems by saying extras are not installed but already is there in
place.
---
 cdist/conf/type/__package_pip/explorer/extras | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 8104ed47..bbdc17ab 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -57,7 +57,8 @@ then
 
         # echo the extra if no packages where found for it
         # if there is no pattern, we don't need to search ;-)
-        if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze"
+        # pip matches packages case-insensetive, we need to do that, too
+        if [ "$grep_pattern" ] && ! grep -qi "$grep_pattern" "$pip_freeze"
         then
             echo "$extra"
         fi

From d1f45d3524f8a025488d0d2f020433b6efb7edb6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 19 Feb 2021 09:03:56 +0100
Subject: [PATCH 18/29] __package_pip: corrected typo in man

.. by fully replacing it with a smaller sentence.
---
 cdist/conf/type/__package_pip/man.rst | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index c013695c..5a2bc673 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -24,9 +24,8 @@ name
 
 extra
     Extra optional dependencies which should be installed along the selected
-    package. Can be specified multiple times. Multiple extra optional
-    dependencies can also be specified in a comma-separated list to provide
-    a more pip-natvie style.
+    package. Can be specified multiple times. Multiple extras can be passed
+    in one `--extra` as a comma-separated list.
 
     Extra optional dependencies will be installed even when the base package
     is already installed. Notice that the type will not remove installed extras

From 5e0572189fb930764318227fa9fd9f4966b0da99 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 22 Feb 2021 09:11:22 +0100
Subject: [PATCH 19/29] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 52686617..59c58e65 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,8 @@ next:
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
+	* Type __postgres_role: Implement modification of roles (Dennis Camera)
+	* Type __letsencrypt_cert: Fix issues with hooks (Evil Ham)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)

From 22f637c15b6616899c848e2a08c4e1c1b13f0f0b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 23 Feb 2021 06:29:24 +0100
Subject: [PATCH 20/29] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 59c58e65..88dda0aa 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
 	* Type __postgres_role: Implement modification of roles (Dennis Camera)
 	* Type __letsencrypt_cert: Fix issues with hooks (Evil Ham)
+	* Type __package_pip: Add optional extra dependencies param (Matthias Stecher)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)

From 60fd7ba1f38382761c366cbc0425ddaec62f1164 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 28 Feb 2021 13:37:23 +0100
Subject: [PATCH 21/29] Release 6.9.5

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 88dda0aa..95e36b88 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)

From 8ef19d47f6b5fb6cdcba7a9a3b8890388dbd7023 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 1 Mar 2021 17:59:25 +0100
Subject: [PATCH 22/29] [type/__pyvenv] Fix example (--user -> --owner)

---
 cdist/conf/type/__pyvenv/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__pyvenv/man.rst b/cdist/conf/type/__pyvenv/man.rst
index 8085ff12..e2e4a1e6 100644
--- a/cdist/conf/type/__pyvenv/man.rst
+++ b/cdist/conf/type/__pyvenv/man.rst
@@ -61,7 +61,7 @@ EXAMPLES
     __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4
 
     # Create python virtualenv for user foo.
-    __pyvenv /home/foo/fooenv --group foo --user foo
+    __pyvenv /home/foo/fooenv --group foo --owner foo
 
     # Create python virtualenv with specific parameters.
     __pyvenv /home/services/djangoenv --venvparams "--copies --system-site-packages"

From e7d33891df945831d54d164eb13f2a9ae4f9dd8e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 2 Mar 2021 09:29:33 +0100
Subject: [PATCH 23/29] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 95e36b88..072fcb6b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __pyvenv: Fix user example in man page (Dennis Camera)
+
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)

From ea0126dd8117c051612e6b6abfe895f5f722c584 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Fri, 5 Mar 2021 16:11:49 +0100
Subject: [PATCH 24/29] Make local state dir available to custom remote scripts

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/config.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cdist/config.py b/cdist/config.py
index e84f6f84..19d5bd70 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -420,6 +420,9 @@ class Config:
                 exec_path=sys.argv[0],
                 save_output_streams=args.save_output_streams)
 
+            # Make __global state dir available to custom remote scripts.
+            os.environ['__global'] = local.base_path
+
             remote = cdist.exec.remote.Remote(
                 target_host=target_host,
                 remote_exec=remote_exec,

From ecba284fc8ce68486c5f1f87a863409fcad0f106 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Fri, 5 Mar 2021 16:13:02 +0100
Subject: [PATCH 25/29] changelog++

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 072fcb6b..ad93a595 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
+	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From fb19f342669d18e6decb21ee4cfb2b22e46748d9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 9 Mar 2021 21:15:26 +0100
Subject: [PATCH 26/29] [type/__ssh_authorized_key] Only grep if file exists

---
 cdist/conf/type/__ssh_authorized_key/explorer/entry | 1 +
 cdist/conf/type/__ssh_authorized_key/gencode-remote | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__ssh_authorized_key/explorer/entry b/cdist/conf/type/__ssh_authorized_key/explorer/entry
index ccab0afc..aca0f2b9 100755
--- a/cdist/conf/type/__ssh_authorized_key/explorer/entry
+++ b/cdist/conf/type/__ssh_authorized_key/explorer/entry
@@ -25,6 +25,7 @@ type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ]
 if [ -n "${type_and_key}" ]
 then
     file="$(cat "$__object/parameter/file")"
+    test -e "$file" || exit 0
 
     # get any entries that match the type and key
 
diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote
index f37aa565..61c77fb9 100755
--- a/cdist/conf/type/__ssh_authorized_key/gencode-remote
+++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote
@@ -37,9 +37,9 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
 # preserve ownership and permissions of existing file
 if [ -f "$file" ]; then
    cp -p "$file" "\$tmpfile"
+   grep -v -F -x '$line' '$file' >\$tmpfile
 fi
-grep -v -F -x '$line' '$file' > \$tmpfile || true
-mv -f "\$tmpfile" "$file"
+cat "\$tmpfile" >"$file"
 DONE
 }
 

From 31cc592aa10902c3e83357ee4a0b9300b5e34efa Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 10 Mar 2021 19:25:04 +0100
Subject: [PATCH 27/29] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index ad93a595..c63b901a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
+	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From e47c4dd8a4cc441b94fe73d362d79525d01a4bb1 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 9 Mar 2021 19:59:05 +0100
Subject: [PATCH 28/29] [type/__sshd_config] Whitelist OpenBMC in manifest

---
 cdist/conf/type/__sshd_config/manifest | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest
index 566bde90..e37afebb 100755
--- a/cdist/conf/type/__sshd_config/manifest
+++ b/cdist/conf/type/__sshd_config/manifest
@@ -39,7 +39,14 @@ in
 	(freebsd|netbsd|openbsd)
 		# whitelist
 		;;
+	(openbmc-phosphor)
+		# whitelist
+		# OpenBMC can be configured with dropbear and OpenSSH.
+		# If dropbear is used, the state explorer will already fail because it
+		# cannot find the sshd binary.
+		;;
 	(*)
+		: "${__type:?}"  # make shellcheck happy
 		printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \
 			"${os}" "${__type##*/}" >&2
 		printf 'Please contribute an implementation for it if you can.\n' >&2

From 10ca1c12fd25a9f2b92cebbc5d375a1723ed5e2f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 12 Mar 2021 08:21:03 +0100
Subject: [PATCH 29/29] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index c63b901a..42a74d04 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
+	* Type __sshd_config: Whitelist OpenBMC (Dennis Camera)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)