++changelog

__sshd_config: Whitelist OpenBMC

See merge request ungleich-public/cdist!980

cdist/conf/type/__package_pip/explorer/distinfo-dir

@@ -0,0 +1,45 @@
+#!/bin/sh
+#
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+nameparam="$__object/parameter/name"
+if [ -f "$nameparam" ]; then
+    name=$(cat "$nameparam")
+else
+    name="$__object_id"
+fi
+
+pipparam="$__object/parameter/pip"
+if [ -f "$pipparam" ]; then
+    pip=$(cat "$pipparam")
+else
+    pip="$( "$__type_explorer/pip" )"
+fi
+
+
+if command -v "$pip" >/dev/null 2>&1; then
+    # assemble the path where pip stores all pip package info
+    "$pip" show "$name" \
+        | awk -F': ' '
+            $1 == "Name" {name=$2; gsub(/-/,"_",name); next}
+            $1 == "Version" {version=$2; next}
+            $1 == "Location" {location=$2; next}
+            END {if (version != "") printf "%s/%s-%s.dist-info", location, name, version}'
+fi

cdist/conf/type/__package_pip/explorer/extras

@@ -0,0 +1,66 @@
+#!/bin/sh
+#
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Checks if the given extras are really installed or not. It will be
+# done by querring all dependencies for that extra and return it as
+# "to be installed" if no dependency was found.
+#
+
+
+distinfo_dir="$("$__type_explorer/distinfo-dir")"
+
+# check if we have something to check
+if [ "$distinfo_dir" ] && [ -s "$__object/parameter/extra" ]
+then
+    # save cause freezing is slow
+    mkdir "$__object/files"
+    pip_freeze="$__object/files/pip-freeze.tmp"
+    pip3 freeze > "$pip_freeze"
+
+    # If all is set, it searches all available extras to separatly check them.
+    # It would work with just 'all' (cause dependencies are specified for
+    # 'all'), but will not update if one extra is already present. Side effect
+    # is that it will not use [all] but instead name all extras seperatly.
+    for extra in $(if grep -qFx all "$__object/parameter/extra";
+        then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA";
+        else tr ',' '\n' < "$__object/parameter/extra";
+        fi)
+    do
+        # create a grep BRE pattern to search all packages
+        # maybe a file full of patterns for -F could be written
+        grep_pattern="$(
+            awk -F'(: | ; )' -v check="$extra" '
+                $1 == "Requires-Dist" {
+                    split($2, r, " ");
+                    sub("extra == ", "", $3); gsub("'"'"'", "", $3);
+                    if($3 == check) print r[1]
+                }' "$distinfo_dir/METADATA" \
+                | sed ':a; $!N; s/\n/\\|/; ta'
+        )"
+
+        # echo the extra if no packages where found for it
+        # if there is no pattern, we don't need to search ;-)
+        # pip matches packages case-insensetive, we need to do that, too
+        if [ "$grep_pattern" ] && ! grep -qi "$grep_pattern" "$pip_freeze"
+        then
+            echo "$extra"
+        fi
+    done
+fi

cdist/conf/type/__package_pip/gencode-remote

@@ -2,6 +2,7 @@
 #
 # 2012 Nico Schottelius (nico-cdist at schottelius.org)
 # 2016 Darko Poljak (darko.poljak at gmail.com)
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
 #
 # This file is part of cdist.
 #
@@ -25,7 +26,10 @@
 state_is=$(cat "$__object/explorer/state")
 state_should="$(cat "$__object/parameter/state")"
 
-[ "$state_is" = "$state_should" ] && exit 0
+# short circuit if state is the same and no extras to install
+[ "$state_is" = "$state_should" ] && ! [ -s "$__object/explorer/extras" ] \
+    && exit 0
+
 
 nameparam="$__object/parameter/name"
 if [ -f "$nameparam" ]; then
@@ -56,6 +60,14 @@ fi
 
 case "$state_should" in
     present)
+        if [ -s "$__object/explorer/extras" ]
+        then
+            # all extras are passed to pip in a comma-separated list in the name
+            # sed loops through all input lines and add commas between them
+            extras="$(sed ':a; $!N; s/\n/,/; ta' "$__object/explorer/extras")"
+            name="${name}[${extras}]"
+        fi
+
         if [ "$runas" ]
         then
             echo "su -c '$pip install -q $name' $runas"

cdist/conf/type/__package_pip/man.rst

@@ -22,6 +22,16 @@ OPTIONAL PARAMETERS
 name
     If supplied, use the name and not the object id as the package name.
 
+extra
+    Extra optional dependencies which should be installed along the selected
+    package. Can be specified multiple times. Multiple extras can be passed
+    in one `--extra` as a comma-separated list.
+
+    Extra optional dependencies will be installed even when the base package
+    is already installed. Notice that the type will not remove installed extras
+    that are not explicitly named for the type because pip does not offer a
+    management for orphaned packages and they may be used by other packages.
+
 pip
     Instead of using pip from PATH, use the specific pip path.
 
@@ -46,6 +56,14 @@ EXAMPLES
     # Use pip in a virtualenv located at /foo/shinken_virtualenv as user foo
     __package_pip pyro --state present --pip /foo/shinken_virtualenv/bin/pip --runas foo
 
+    # Install package with optional dependencies
+    __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails
+    # the extras can also be specified comma-separated
+    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails --extra postgres
+
+    # or take all extras
+    __package_pip mautrix-telegram --extra all
+
 
 SEE ALSO
 --------
@@ -54,12 +72,13 @@ SEE ALSO
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2012 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2012 Nico Schottelius, 2021 Matthias Stecher. You can
+redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either
+version 3 of the License, or (at your option) any later version.

cdist/conf/type/__package_pip/parameter/optional_multiple

@@ -0,0 +1 @@
+extra

cdist/conf/type/__postgres_role/explorer/state

@@ -1,6 +1,7 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -11,32 +12,140 @@
 #
 # cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$("${__explorer}/os")"
+case $("${__explorer:?}/os")
 in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
+	(netbsd)
+		postgres_user='pgsql'
+		;;
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(*)
+		postgres_user='postgres'
+		;;
 esac
 
+rolename=${__object_id:?}
 
-name="$__object_id"
 
-if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='$name'\"")"
+psql_query() {
+	su -l "${postgres_user}" -c "$(
+		printf "psql -q -F '\034' -R '\036' -wAc '%s'" \
+		"$(printf %s "$*" | sed "s/'/'\\\\''/g")"
+	)"
+}
+
+password_check_login() (
+	PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .)
+	PGPASSWORD=${PGPASSWORD%?.}
+	export PGPASSWORD
+	psql -q -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
+)
+
+role_properties=$(
+	psql_query "SELECT * FROM pg_roles WHERE rolname = '${rolename}'" \
+	| awk '
+	  BEGIN { RS = "\036"; FS = "\034" }
+	  /^\([0-9]+ rows?\)/ { exit }
+	  NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next }
+	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }
+	  '
+)
+
+if test -n "${role_properties}"
 then
-    echo 'present'
+	# Check if the user's properties match the parameters
+	for prop in login createdb createrole superuser
+	do
+		bool_should=$(test -f "${__object:?}/parameter/${prop}" && echo 't' || echo 'f')
+		bool_is=$(
+			printf '%s\n' "${role_properties}" |
+			awk -F '=' -v key="${prop}" '
+			BEGIN {
+				if (key == "login")
+					key = "canlogin"
+				else if (key == "superuser")
+					key = "super"
+				key = "rol" key
+			}
+			$1 == key {
+				sub(/^[^=]*=/, "")
+				print
+			}
+			'
+		)
+
+		test "${bool_is}" = "${bool_should}" || {
+			state='different properties'
+		}
+	done
+
+	# Check password
+	passwd_stored=$(
+		psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \
+		| awk 'BEGIN { RS = "\036" } NR == 2'
+		printf .
+	)
+	passwd_stored=${passwd_stored%?.}
+
+	if test -f "${__object:?}/parameter/password"
+	then
+		passwd_should=$(cat "${__object:?}/parameter/password"; printf .)
+	fi
+	passwd_should=${passwd_should%?.}
+
+	if test -z "${passwd_stored}"
+	then
+		test -z "${passwd_should}" || state="${state:-different} password"
+	elif expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
+	then
+		# SCRAM-SHA-256 "encrypted" password
+		# NOTE: There is currently no easy way to check SCRAM passwords without
+		#       logging in
+		password_check_login || state="${state:-different} password"
+	elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null
+	then
+		# MD5 "encrypted" password
+		if command -v md5sum >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| md5sum - | sed -e 's/[^0-9a-f]*$//')
+		elif command -v gmd5sum >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| gmd5sum - | sed -e 's/[^0-9a-f]*$//')
+		elif command -v openssl >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| openssl dgst -md5 | sed 's/^.* //')
+		fi
+
+		if test -n "${should_md5}"
+		then
+			test "${passwd_stored}" = "md5${should_md5}" \
+			|| state="${state:-different} password"
+		else
+			password_check_login || state="${state:-different} password"
+		fi
+	else
+		# unencrypted password (unsupported since PostgreSQL 10)
+		test "${passwd_stored}" = "${passwd_should}" \
+		|| state="${state:-different} password"
+	fi
+
+	test -n "${state}" || state='present'
 else
-    echo 'absent'
+	state='absent'
 fi
+
+echo "${state}"

cdist/conf/type/__postgres_role/gencode-remote

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -11,55 +12,117 @@
 #
 # cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
+quote() {
+	if test $# -gt 0
+	then
+		printf '%s' "$*"
+	else
+		cat -
+	fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
+}
+
+case $(cat "${__global:?}/explorer/os")
 in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
+	(netbsd)
+		postgres_user='pgsql'
+		;;
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(*)
+		postgres_user='postgres'
+		;;
 esac
 
 
-name="$__object_id"
-state_is="$(cat "$__object/explorer/state")"
-state_should="$(cat "$__object/parameter/state")"
+rolename=${__object_id:?}
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 
-[ "$state_is" = "$state_should" ] && exit 0
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
 
-case "$state_should" in
-    present)
-        if [ -f "$__object/parameter/password" ]; then
-            password="$(cat "$__object/parameter/password")"
-        fi
-        booleans=""
-        for boolean in login createdb createrole superuser; do
-            if [ ! -f "$__object/parameter/$boolean" ]; then
-                boolean="no${boolean}"
-            fi
-            upper=$(echo $boolean | tr '[:lower:]' '[:upper:]')
-            booleans="$booleans $upper"
-        done
+psql_query() {
+	printf 'su -l %s -c %s\n' \
+		"$(quote "${postgres_user}")" \
+		"$(quote "psql postgres -q -w -c $(quote "$1")")"
+}
 
-        [ -n "$password" ] && password="PASSWORD '$password'"
-        cat << EOF
-su - '$postgres_user' -c "psql postgres -wc \"CREATE ROLE \\\\\"$name\\\\\" WITH $password $booleans;\""
-EOF
-    ;;
-    absent)
-        cat << EOF
-su - '$postgres_user' -c "dropuser \"$name\""
-EOF
-    ;;
+psql_set_password() {
+	# NOTE: Always make sure that the password does not end up in psql_history!
+	# NOTE: Never set an empty string as the password, because they can be
+	#       interpreted differently by different tooling.
+	if test -s "${__object:?}/parameter/password"
+	then
+		cat <<-EOF
+		exec 3< "\${__object:?}/parameter/password"
+		su -l '${postgres_user}' -c 'psql -q -w postgres' <<'SQL'
+		\set HISTFILE /dev/null
+		\set pw \`cat <&3\`
+		ALTER ROLE "${rolename}" WITH PASSWORD :'pw';
+		SQL
+		exec 3<&-
+		EOF
+	else
+		psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD NULL;"
+	fi
+}
+
+role_properties_should() {
+	_props=
+	for _prop in login createdb createrole superuser
+	do
+		_props="${_props}${_props:+ }$(
+			if test -f "${__object:?}/parameter/${_prop}"
+			then
+				echo "${_prop}"
+			else
+				echo "no${_prop}"
+			fi \
+			| tr '[:lower:]' '[:upper:]')"
+	done
+	printf '%s\n' "${_props}"
+	unset _prop _props
+}
+
+case ${state_should}
+in
+	(present)
+		case ${state_is}
+		in
+			(absent)
+				psql_query "CREATE ROLE \"${rolename}\" WITH $(role_properties_should);"
+				psql_set_password
+				;;
+			(different*)
+				if expr "${state_is}" : 'different.*properties' >/dev/null
+				then
+					psql_query "ALTER ROLE \"${rolename}\" WITH $(role_properties_should);"
+				fi
+
+				if expr "${state_is}" : 'different.*password' >/dev/null
+				then
+					psql_set_password
+				fi
+				;;
+			(*)
+				printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2
+				exit 1
+				;;
+		esac
+		;;
+	(absent)
+		printf 'su -l %s -c %s\n' \
+			"$(quote "${postgres_user}")" \
+			"$(quote "dropuser $(quote "${rolename}")")"
+		;;
 esac

cdist/conf/type/__pyvenv/man.rst

@@ -61,7 +61,7 @@ EXAMPLES
     __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4
 
     # Create python virtualenv for user foo.
-    __pyvenv /home/foo/fooenv --group foo --user foo
+    __pyvenv /home/foo/fooenv --group foo --owner foo
 
     # Create python virtualenv with specific parameters.
     __pyvenv /home/services/djangoenv --venvparams "--copies --system-site-packages"

cdist/conf/type/__ssh_authorized_key/explorer/entry

@@ -25,6 +25,7 @@ type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ]
 if [ -n "${type_and_key}" ]
 then
     file="$(cat "$__object/parameter/file")"
+    test -e "$file" || exit 0
 
     # get any entries that match the type and key
 

cdist/conf/type/__ssh_authorized_key/gencode-remote

@@ -37,9 +37,9 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
 # preserve ownership and permissions of existing file
 if [ -f "$file" ]; then
    cp -p "$file" "\$tmpfile"
+   grep -v -F -x '$line' '$file' >\$tmpfile
 fi
-grep -v -F -x '$line' '$file' > \$tmpfile || true
-mv -f "\$tmpfile" "$file"
+cat "\$tmpfile" >"$file"
 DONE
 }
 

cdist/conf/type/__sshd_config/manifest

@@ -39,7 +39,14 @@ in
 	(freebsd|netbsd|openbsd)
 		# whitelist
 		;;
+	(openbmc-phosphor)
+		# whitelist
+		# OpenBMC can be configured with dropbear and OpenSSH.
+		# If dropbear is used, the state explorer will already fail because it
+		# cannot find the sshd binary.
+		;;
 	(*)
+		: "${__type:?}"  # make shellcheck happy
 		printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \
 			"${os}" "${__type##*/}" >&2
 		printf 'Please contribute an implementation for it if you can.\n' >&2

cdist/config.py

@@ -420,6 +420,9 @@ class Config:
                 exec_path=sys.argv[0],
                 save_output_streams=args.save_output_streams)
 
+            # Make __global state dir available to custom remote scripts.
+            os.environ['__global'] = local.base_path
+
             remote = cdist.exec.remote.Remote(
                 target_host=target_host,
                 remote_exec=remote_exec,

docs/changelog

@@ -2,9 +2,18 @@ Changelog
 ---------
 
 next:
+	* Type __pyvenv: Fix user example in man page (Dennis Camera)
+	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
+	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
+	* Type __sshd_config: Whitelist OpenBMC (Dennis Camera)
+
+6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
+	* Type __postgres_role: Implement modification of roles (Dennis Camera)
+	* Type __letsencrypt_cert: Fix issues with hooks (Evil Ham)
+	* Type __package_pip: Add optional extra dependencies param (Matthias Stecher)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)