++changelog

Reviewed-on: ungleich-public/cdist#323

cdist/conf/type/__apt_source/gencode-remote

@@ -22,7 +22,21 @@
 name="$__object_id"
 destination="/etc/apt/sources.list.d/${name}.list"
 
+# There are special arguments to apt(8) to prevent aborts if apt woudn't been
+# updated after the 19th April 2021 till the bullseye release. The additional
+# arguments acknoledge the happend suite change (the apt(8) update does the
+# same by itself).
+#
+# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+# allows backward compatablility to pre-buster Debian versions.
+#
+# See more: ticket #861
+# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
+# run 'apt-get update' only if something changed with our sources.list file
+#   it will be run a second time on error as a redundancy messure to success
 if grep -q "^__file${destination}" "$__messages_in"; then
-   printf 'apt-get update || apt-get update\n'
+   printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts"
 fi
 

cdist/conf/type/__apt_update_index/gencode-remote

@@ -18,9 +18,23 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+
+# There are special arguments to apt(8) to prevent aborts if apt woudn't been
+# updated after the 19th April 2021 till the bullseye release. The additional
+# arguments acknoledge the happend suite change (the apt(8) update does the
+# same by itself).
+#
+# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+# allows backward compatablility to pre-buster Debian versions.
+#
+# See more: ticket #861
+# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
 # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists
+#   it will be run a second time on error as a redundancy messure to success
 cat << DONE
 if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then
-   apt-get update || apt-get update
+   apt-get $apt_opts update || apt-get $apt_opts update
 fi
 DONE

cdist/conf/type/__debconf_set_selections/parameter/deprecated/file

@@ -0,0 +1 @@
+'file' has been deprecated in favour of 'line' in order to provide idempotency.

cdist/conf/type/__grafana_dashboard/manifest

@@ -15,7 +15,7 @@ case $os in
                 # Differntation not needed anymore
                 apt_source_distribution=stable
                 ;;
-            10*)
+            10*|11*)
                 # Differntation not needed anymore
                 apt_source_distribution=stable
                 ;;

cdist/conf/type/__haproxy_dualstack/files/http

@@ -0,0 +1,8 @@
+frontend http
+	bind	BIND@:80
+	mode	http
+	option	httplog
+	default_backend	http
+
+backend http
+	mode	http

cdist/conf/type/__haproxy_dualstack/files/https

@@ -0,0 +1,10 @@
+frontend https
+	bind	BIND@:443
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	https
+
+backend https
+	mode	tcp

cdist/conf/type/__haproxy_dualstack/files/imaps

@@ -0,0 +1,12 @@
+frontend imaps
+	bind	BIND@:143
+	bind	BIND@:993
+
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	imaps
+
+backend imaps
+	mode	tcp

cdist/conf/type/__haproxy_dualstack/files/smtps

@@ -0,0 +1,12 @@
+frontend smtps
+	bind	BIND@:25
+	bind	BIND@:465
+
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	smtps
+
+backend smtps
+	mode	tcp

cdist/conf/type/__haproxy_dualstack/man.rst

@@ -0,0 +1,121 @@
+cdist-type__haproxy_dualstack(7)
+================================
+
+
+NAME
+----
+cdist-type__haproxy_dualstack - Proxy services from a dual-stack server
+
+
+DESCRIPTION
+-----------
+This (singleton) type installs and configures haproxy to act as a dual-stack
+proxy for single-stack services.
+
+This can be useful to add IPv4 support to IPv6-only services while only using
+one IPv4 for many such services.
+
+By default this type uses the plain TCP proxy mode, which means that there is no
+need for TLS termination on this host when SNI is supported.
+This also means that proxied services will not receive the client's IP address,
+but will see the proxy's IP address instead (that of `$__target_host`).
+
+This can be solved by using the PROXY protocol, but do take into account that,
+e.g. nginx cannot serve both regular HTTP(S) and PROXY protocols on the same
+port, so you will need to use other ports for that.
+
+As a recommendation in this type: use TCP ports 8080 and 591 respectively to
+serve HTTP and HTTPS using the PROXY protocol.
+
+See the EXAMPLES for more details.
+
+
+OPTIONAL PARAMETERS
+-------------------
+v4proxy
+    Proxy incoming IPv4 connections to the equivalent IPv6 endpoint.
+    In its simplest use, it must be a NAME with an `AAAA` DNS entry, which is
+    the IP address actually providing the proxied services.
+    The full format of this argument is:
+    `[proxy:]NAME[[:PROTOCOL_1=PORT_1]...[:PROTOCOL_N=PORT_N]]`
+    Where starting with `proxy:` determines that the PROXY protocol must be
+    used and each `:PROTOCOL=PORT` (e.g. `:http=8080` or `:https=591`) is a PORT
+    override for the given PROTOCOL (see `--protocol`), if not present the
+    PROTOCOL's default port will be used.
+
+
+v6proxy
+    Proxy incoming IPv6 connections to the equivalent IPv4 endpoint.
+    In its simplest use, it must be a NAME with an `A` DNS entry, which is
+    the IP address actually providing the proxied services.
+    See `--v4proxy` for more options and details.
+
+protocol
+    Can be passed multiple times or as a space-separated list of protocols.
+    Currently supported protocols are: `http`, `https`, `imaps`, `smtps`.
+    This defaults to: `http https imaps smtps`.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Proxy the IPv6-only services so IPv4-only clients can access them
+    # This uses HAProxy's TCP mode for http, https, imaps and smtps
+    __haproxy_dualstack \
+        --v4proxy ipv6.chat \
+        --v4proxy matrix.ungleich.ch
+
+    # Proxy the IPv6-only HTTP(S) services so IPv4-only clients can access them
+    # Note this means that the backend IPv6-only server will only see
+    # the IPv6 address of the haproxy host managed by cdist, which can be
+    # troublesome if this information is relevant for analytics/security/...
+    # See the PROXY example below
+    __haproxy_dualstack \
+        --protocol http --protocol https \
+        --v4proxy ipv6.chat \
+        --v4proxy matrix.ungleich.ch
+
+    # Use the PROXY protocol to proxy the IPv6-only HTTP(S) services enabling
+    # IPv4-only clients to access them while maintaining the client's IP address
+    __haproxy_dualstack \
+        --protocol http --protocol https \
+        --v4proxy proxy:ipv6.chat:http=8080:https=591 \
+        --v4proxy proxy:matrix.ungleich.ch:http=8080:https=591
+    # Note however that the PROXY protocol is not compatible with regular
+    # HTTP(S) protocols, so your nginx will have to listen on different ports
+    # with the PROXY settings.
+    # Note that you will need to restrict access to the 8080 port to prevent
+    # Client IP spoofing.
+    # This can be something like:
+    # server {
+    #     # listen for regular HTTP connections
+    #     listen [::]:80 default_server;
+    #     listen 80 default_server;
+    #     # listen for PROXY HTTP connections
+    #     listen [::]:8080 proxy_protocol;
+    #     # Accept the Client's IP from the PROXY protocol
+    #     real_ip_header proxy_protocol;
+    # }
+
+
+SEE ALSO
+--------
+- https://www.haproxy.com/blog/enhanced-ssl-load-balancing-with-server-name-indication-sni-tls-extension/
+- https://www.haproxy.com/blog/haproxy/proxy-protocol/
+- https://docs.nginx.com/nginx/admin-guide/load-balancer/using-proxy-protocol/
+
+
+AUTHORS
+-------
+ungleich <foss--@--ungleich.ch>
+Evilham <cvs--@--evilham.com>
+
+
+COPYING
+-------
+Copyright \(C) 2021 ungleich glarus ag. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__haproxy_dualstack/manifest

@@ -0,0 +1,155 @@
+#!/bin/sh -eu
+
+__package haproxy
+require="__package/haproxy" __start_on_boot haproxy
+
+tmpdir="$__object/files"
+mkdir "$tmpdir"
+configtmp="$__object/files/haproxy.cfg"
+
+os=$(cat "$__global/explorer/os")
+case $os in
+	freebsd)
+		CONFIG_FILE="/usr/local/etc/haproxy.conf"
+		cat <<EOF > "$configtmp"
+global
+	maxconn	4000
+	user	nobody
+	group	nogroup
+	daemon
+
+EOF
+
+		;;
+	*)
+		CONFIG_FILE="/etc/haproxy/haproxy.cfg"
+		cat <<EOF > "$configtmp"
+global
+	log	[::1] local2
+	chroot	/var/lib/haproxy
+	pidfile	/var/run/haproxy.pid
+	maxconn	4000
+	user	haproxy
+	group	haproxy
+	daemon
+
+	# turn on stats unix socket
+	stats socket	/var/lib/haproxy/stats
+
+EOF
+		;;
+esac
+
+cat <<EOF >> "$configtmp"
+defaults
+	retries	3
+	log	global
+	timeout http-request	10s
+	timeout queue		1m
+	timeout connect		10s
+	timeout client		1m
+	timeout server		1m
+	timeout http-keep-alive	10s
+	timeout check		10s
+EOF
+
+dig_cmd="$(command -v dig || true)"
+get_ip() {
+	# Usage: get_ip (ipv4|ipv6) NAME
+	# uses "dig" if available, else fallback to "host"
+	case $1 in
+		ipv4)
+			if [ -n "${dig_cmd}" ]; then
+				${dig_cmd} +short A "$2"
+			else
+				host -t A "$2" | cut -d ' ' -f 4 | grep -v 'found:'
+			fi
+			;;
+		ipv6)
+			if [ -n "${dig_cmd}" ]; then
+				${dig_cmd} +short AAAA "$2"
+			else
+				host -t AAAA "$2" | cut -d ' ' -f 5 | grep -v 'NXDOMAIN'
+			fi
+			;;
+	esac
+}
+
+PROTOCOLS="$(cat "$__object/parameter/protocol")"
+
+for proxy in v4proxy v6proxy; do
+	param=$__object/parameter/$proxy
+	# no backend? skip generating code
+	if [ ! -f "$param" ]; then
+		continue
+	fi
+
+	# turn backend name into bind parameter: v4backend -> ipv4@
+	bind=$(echo $proxy | sed -e 's/^/ip/' -e 's/proxy//')
+
+	case $bind in
+		ipv4)
+			backendproto=ipv6
+			;;
+		ipv6)
+			backendproto=ipv4
+			;;
+	esac
+
+	for proto in ${PROTOCOLS}; do
+		# Add protocol "header"
+		printf "\n# %s %s \n" "${bind}" "${proto}" >> "$configtmp"
+
+		sed -e "s/BIND/$bind/" \
+			-e "s/\(frontend[[:space:]].*\)/\1$bind/" \
+			-e "s/\(backend[[:space:]].*\)/\\1$bind/" \
+			"$__type/files/$proto" >> "$configtmp"
+
+		while read -r hostdefinition; do
+			if echo "$hostdefinition" | grep -qE '^proxy:'; then
+				# Proxy protocol was requested
+				host="$(echo "$hostdefinition" | sed -E 's/^proxy:([^:]+).*$/\1/')"
+				send_proxy=" send-proxy"
+			else
+				# Just use tcp proxy mode
+				host="$hostdefinition"
+				send_proxy=""
+			fi
+			if echo "$hostdefinition" | grep -qE ":${proto}="; then
+				# Use custom port definition if requested
+				port="$(echo "$hostdefinition" | sed -E "s/^(.*:)?${proto}=([0-9]+).*$/:\2/")"
+			else
+				# Else use the default
+				port=""
+			fi
+			servername=$host
+
+			res=$(get_ip "$bind" "$servername")
+
+			if [ -z "$res" ]; then
+				echo "$servername does not resolve - aborting config" >&2
+				exit 1
+			fi
+
+			# Treat protocols without TLS+SNI specially
+			if [ "$proto" = http ]; then
+				echo "	use-server $servername if { hdr(host) -i $host }" >> "$configtmp"
+			else
+				echo "	use-server $servername if { req_ssl_sni -i $host }" >> "$configtmp"
+			fi
+
+			# Create the "server" itself.
+			# Note that port and send_proxy will be empty unless
+			# they were requested by the type user
+			echo "	server $servername ${backendproto}@${host}${port}${send_proxy}" >> "$configtmp"
+
+		done < "$param"
+	done
+done
+
+# Create config file
+require="__package/haproxy" __file ${CONFIG_FILE} --source "$configtmp" --mode 0644
+
+require="__file${CONFIG_FILE}" __check_messages "haproxy_reload" \
+	--pattern "^__file${CONFIG_FILE}" \
+	--execute "service haproxy reload || service haproxy restart"

cdist/conf/type/__haproxy_dualstack/parameter/default/protocol

@@ -0,0 +1 @@
+http https imaps smtps

cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple

@@ -0,0 +1,3 @@
+protocol
+v4proxy
+v6proxy

cdist/conf/type/__letsencrypt_cert/manifest

@@ -41,7 +41,7 @@ if [ -z "${certbot_fullpath}" ]; then
 					require="__apt_source/stretch-backports" __package_apt certbot \
 						--target-release stretch-backports
 				;;
-				10*)
+				10*|11*)
 					__package_apt certbot
 				;;
 

cdist/conf/type/__package_apt/gencode-remote

@@ -81,12 +81,24 @@ aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::=
 
 case "$state_should" in
     present)
+        # There are special arguments to apt(8) to prevent aborts if apt woudn't been
+        # updated after the 19th April 2021 till the bullseye release. The additional
+        # arguments acknoledge the happend suite change (the apt(8) update does the
+        # same by itself).
+        #
+        # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+        # allows backward compatablility to pre-buster Debian versions.
+        #
+        # See more: ticket #861
+        # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+        apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
         # following is bit ugly, but important hack.
         # due to how cdist config run works, there isn't
         # currently better way to do it :(
         cat << EOF
 if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ]
-then echo apt-get update > /dev/null 2>&1 || true
+then echo apt-get $apt_opts update > /dev/null 2>&1 || true
 fi
 EOF
         if [ -n "$version" ]; then

cdist/conf/type/__package_update_index/gencode-remote

@@ -41,7 +41,19 @@ fi
 case "$type" in
     yum) ;;
     apt)
-        echo "apt-get --quiet update"
+        # There are special arguments to apt(8) to prevent aborts if apt woudn't been
+        # updated after the 19th April 2021 till the bullseye release. The additional
+        # arguments acknoledge the happend suite change (the apt(8) update does the
+        # same by itself).
+        #
+        # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+        # allows backward compatablility to pre-buster Debian versions.
+        #
+        # See more: ticket #861
+        # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+        apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
+        echo "apt-get --quiet $apt_opts update"
         echo "apt-cache updated (age was: $currage)" >> "$__messages_out"
         ;;
     pacman)

cdist/conf/type/__package_upgrade_all/gencode-remote

@@ -28,6 +28,10 @@ apt_clean="$__object/parameter/apt-clean"
 
 apt_dist_upgrade="$__object/parameter/apt-dist-upgrade"
 
+if [ -f "$__object/parameter/apt-with-new-pkgs" ]; then
+	apt_with_new_pkgs="--with-new-pkgs"
+fi
+
 if [ -f "$type" ]; then
     type="$(cat "$type")"
 else
@@ -54,7 +58,7 @@ case "$type" in
     apt)
         if [ -f "$apt_dist_upgrade" ]
         then echo "$aptget dist-upgrade"
-        else echo "$aptget upgrade"
+        else echo "$aptget $apt_with_new_pkgs upgrade"
         fi
 
         if [ -f "$apt_clean" ]

cdist/conf/type/__package_upgrade_all/man.rst

@@ -33,6 +33,14 @@ BOOLEAN PARAMETERS
 apt-dist-upgrade
     Do dist-upgrade instead of upgrade.
 
+apt-with-new-pkg
+    Allow installing new packages when used in conjunction with
+    upgrade. This is useful if the update of an installed package
+    requires new dependencies to be installed. Instead of holding the
+    package back upgrade will upgrade the package and install the new
+    dependencies. Note that upgrade with this option will never remove
+    packages, only allow adding new ones.
+
 apt-clean
     Clean out the local repository of retrieved package files.
 

cdist/conf/type/__package_upgrade_all/parameter/boolean

@@ -1,2 +1,3 @@
 apt-clean
 apt-dist-upgrade
+apt-with-new-pkgs

cdist/conf/type/__sed/explorer/file

@@ -0,0 +1,16 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/file" ]
+then
+    file="$( cat "$__object/parameter/file" )"
+else
+    file="/$__object_id"
+fi
+
+if [ ! -e "$file" ]
+then
+    echo "$file does not exist" >&2
+    exit 1
+fi
+
+cat "$file"

cdist/conf/type/__sed/gencode-remote

@@ -0,0 +1,58 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/file" ]
+then
+    file="$( cat "$__object/parameter/file" )"
+else
+    file="/$__object_id"
+fi
+
+script="$( cat "$__object/parameter/script" )"
+
+if [ "$script" = '-' ]
+then
+    script="$( cat "$__object/stdin" )"
+fi
+
+# since stdin is not available in explorer, we pull file from target with explorer
+
+file_from_target="$__object/explorer/file"
+
+sed_cmd='sed'
+
+if [ -f "$__object/parameter/regexp-extended" ]
+then
+    sed_cmd="$sed_cmd -E"
+fi
+
+# do sed dry run, diff result and if no change, then there's nothing to do
+# also redirect diff's output to stderr for debugging purposes
+
+if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2
+then
+    exit 0
+fi
+
+# we can't use -i, because it's not posix, so we fly with tempfile and cp
+# and we use cp because we want to preserve destination file's attributes
+
+# shellcheck disable=SC2016
+echo 'tmp="$__object/tempfile"'
+
+echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
+
+echo "$script"
+
+echo 'EOF'
+
+echo "cp \"\$tmp\" '$file'"
+
+# shellcheck disable=SC2016
+echo 'rm -f "$tmp"'
+
+echo 'change' >> "$__messages_out"
+
+if [ -f "$__object/parameter/onchange" ]
+then
+    cat "$__object/parameter/onchange"
+fi

cdist/conf/type/__sed/man.rst

@@ -0,0 +1,57 @@
+cdist-type__sed(7)
+==================
+
+NAME
+----
+cdist-type__sed - Transform text files with ``sed``
+
+
+DESCRIPTION
+-----------
+Transform text files with ``sed``.
+
+
+REQUIRED MULTIPLE PARAMETERS
+----------------------------
+script
+   ``sed`` script.
+   If ``-`` then the script is read from ``stdin``.
+
+
+OPTIONAL PARAMETERS
+-------------------
+file
+   Path to the file. Defaults to ``$__object_id``.
+
+onchange
+   Execute this command if ``sed`` changes file.
+
+
+BOOLEAN PARAMETERS
+------------------
+regexp-extended
+   Use extended regular expressions in the script.
+   Might not be supported with every ``sed`` version.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+   __sed /tmp/foobar --script 's/foo/bar/'
+
+   echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script -
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.

cdist/conf/type/__sed/parameter/boolean

@@ -0,0 +1 @@
+regexp-extended

cdist/conf/type/__sed/parameter/optional

@@ -0,0 +1,2 @@
+file
+onchange

cdist/conf/type/__sed/parameter/required_multiple

@@ -0,0 +1 @@
+script

cdist/integration.py

@@ -84,7 +84,7 @@ def _process_hosts_simple(action, host, manifest, verbose,
     """
     if isinstance(host, str):
         hosts = [host, ]
-    elif isinstance(host, collections.Iterable):
+    elif isinstance(host, collections.abc.Iterable):
         hosts = host
     else:
         raise cdist.Error('Invalid host argument: {}'.format(host))

cdist/util/fsproperty.py

@@ -33,7 +33,7 @@ class AbsolutePathRequiredError(cdist.Error):
         return 'Absolute path required, got: {}'.format(self.path)
 
 
-class FileList(collections.MutableSequence):
+class FileList(collections.abc.MutableSequence):
     """A list that stores it's state in a file.
 
     """
@@ -102,7 +102,7 @@ class FileList(collections.MutableSequence):
         self.__write(lines)
 
 
-class DirectoryDict(collections.MutableMapping):
+class DirectoryDict(collections.abc.MutableMapping):
     """A dict that stores it's items as files in a directory.
 
     """

docs/changelog

@@ -3,6 +3,15 @@ Changelog
 
 next:
 	* Explorer machine_type: Rewrite (Dennis Camera)
+	* New type: __sed (Ander Punnar)
+	* New type: __haproxy_dualstack (Evilham and ungleich)
+	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_upgrade_all: Add new --apt-with-new-pkgs argument (Evilham)
+	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
+	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
+	* Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)