++changelog

Even if deprecated, the parameter *must* be supported, which isn't the case
right now.

This was due to a misunderstanding of how deprecating parameters work, see:
https://www.cdi.st/manual/latest/cdist-type.html#deprecated-parameters

cdist/conf/explorer/memory

@@ -27,19 +27,18 @@
 str2bytes() {
 	awk -F' ' '
 	$2 ==   "B" || !$2 { print $1 }
-	$2 ==  "kB" { print $1 * 1000 }
-	$2 ==  "MB" { print $1 * 1000 * 1000 }
-	$2 ==  "GB" { print $1 * 1000 * 1000 * 1000 }
-	$2 ==  "TB" { print $1 * 1000 * 1000 * 1000 * 1000 }
-	$2 == "kiB" { print $1 * 1024 }
-	$2 == "MiB" { print $1 * 1024 * 1024 }
-	$2 == "GiB" { print $1 * 1024 * 1024 * 1024 }
-	$2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }'
+	$2 ==  "kB" { printf "%.f\n", ($1 * 1000) }
+	$2 ==  "MB" { printf "%.f\n", ($1 * 1000 * 1000) }
+	$2 ==  "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) }
+	$2 ==  "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) }
+	$2 == "kiB" { printf "%.f\n", ($1 * 1024) }
+	$2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) }
+	$2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) }
+	$2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }'
 }
 
 bytes2kib() {
-	set -- "$(cat)"
-	test "$1" -gt 0 && echo $(($1 / 1024))
+	awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }'
 }
 
 

cdist/conf/explorer/os_version

@@ -21,6 +21,17 @@
 # All os variables are lower case
 #
 
+rc_getvar() {
+   awk -F= -v varname="$2" '
+      function unquote(s) {
+         if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/)
+            return substr(s, 2, length(s) - 2)
+         else
+            return s
+      }
+      $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1"
+}
+
 case $("${__explorer:?}/os")
 in
    amazon)
@@ -43,6 +54,8 @@ in
               # sid versions don't have a number, so we decode by codename:
               case $(expr "$debian_version" : '\([a-z]\{1,\}\)/')
               in
+                  trixie) echo 12.99 ;;
+                  bookworm) echo 11.99 ;;
                   bullseye) echo 10.99 ;;
                   buster) echo 9.99 ;;
                   stretch) echo 8.99 ;;
@@ -50,7 +63,7 @@ in
                   wheezy) echo 6.99 ;;
                   squeeze) echo 5.99 ;;
                   lenny) echo 4.99 ;;
-                  *) exit 1
+                  *) echo 99.99 ;;
               esac
               ;;
           *)
@@ -84,12 +97,20 @@ in
       cat /etc/gentoo-release
    ;;
    macosx)
-      sw_vers -productVersion
+      # NOTE: Legacy versions (< 10.3) do not support options
+      sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }'
    ;;
    freebsd)
       # Apparently uname -r is not a reliable way to get the patch level.
       # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
-      freebsd-version
+      if command -v freebsd-version >/dev/null 2>&1
+      then
+         # get userland version
+         freebsd-version -u
+      else
+         # fallback to kernel release for FreeBSD < 10.0
+         uname -r
+      fi
    ;;
    *bsd|solaris)
       uname -r
@@ -114,7 +135,20 @@ in
       fi
    ;;
    ubuntu)
-      lsb_release -sr
+      if command -v lsb_release >/dev/null 2>&1
+      then
+         lsb_release -sr
+      elif test -r /usr/lib/os-release
+      then
+         # fallback to /usr/lib/os-release if lsb_release is not present (like
+         # on minimized Ubuntu installations)
+         rc_getvar /usr/lib/os-release VERSION_ID
+      elif test -r /etc/lsb-release
+      then
+         # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old
+         # versions without /usr/lib/os-release.
+         rc_getvar /etc/lsb-release DISTRIB_RELEASE
+      fi
    ;;
    alpine)
        cat /etc/alpine-release

cdist/conf/type/__apt_backports/manifest

@@ -28,6 +28,7 @@
 #  lsb_release may not be given in all installations
 codename_os_release() {
     # shellcheck disable=SC1090
+    # shellcheck disable=SC1091
     . "$__global/explorer/os_release"
     printf "%s" "$VERSION_CODENAME"
 }

cdist/conf/type/__apt_source/gencode-remote

@@ -22,7 +22,21 @@
 name="$__object_id"
 destination="/etc/apt/sources.list.d/${name}.list"
 
+# There are special arguments to apt(8) to prevent aborts if apt woudn't been
+# updated after the 19th April 2021 till the bullseye release. The additional
+# arguments acknoledge the happend suite change (the apt(8) update does the
+# same by itself).
+#
+# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+# allows backward compatablility to pre-buster Debian versions.
+#
+# See more: ticket #861
+# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
+# run 'apt-get update' only if something changed with our sources.list file
+#   it will be run a second time on error as a redundancy messure to success
 if grep -q "^__file${destination}" "$__messages_in"; then
-   printf 'apt-get update || apt-get update\n'
+   printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts"
 fi
 

cdist/conf/type/__apt_update_index/gencode-remote

@@ -18,9 +18,23 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+
+# There are special arguments to apt(8) to prevent aborts if apt woudn't been
+# updated after the 19th April 2021 till the bullseye release. The additional
+# arguments acknoledge the happend suite change (the apt(8) update does the
+# same by itself).
+#
+# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+# allows backward compatablility to pre-buster Debian versions.
+#
+# See more: ticket #861
+# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
 # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists
+#   it will be run a second time on error as a redundancy messure to success
 cat << DONE
 if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then
-   apt-get update || apt-get update
+   apt-get $apt_opts update || apt-get $apt_opts update
 fi
 DONE

cdist/conf/type/__debconf_set_selections/parameter/deprecated/file

@@ -0,0 +1 @@
+'file' has been deprecated in favour of 'line' in order to provide idempotency.

cdist/conf/type/__filesystem/explorer/lsblk

@@ -27,7 +27,7 @@ else
 fi
 
 case "$os" in
-    alpine|centos|fedora|redhat|suse|gentoo)
+    alpine|centos|fedora|gentoo|redhat|suse|ubuntu)
         if [ ! -x "$(command -v lsblk)" ]; then
             echo "lsblk is required for __filesystem type" >&2
             exit 1

cdist/conf/type/__grafana_dashboard/manifest

@@ -15,7 +15,7 @@ case $os in
                 # Differntation not needed anymore
                 apt_source_distribution=stable
                 ;;
-            10*)
+            10*|11*)
                 # Differntation not needed anymore
                 apt_source_distribution=stable
                 ;;

cdist/conf/type/__letsencrypt_cert/manifest

@@ -41,7 +41,7 @@ if [ -z "${certbot_fullpath}" ]; then
 					require="__apt_source/stretch-backports" __package_apt certbot \
 						--target-release stretch-backports
 				;;
-				10*)
+				10*|11*)
 					__package_apt certbot
 				;;
 

cdist/conf/type/__package_apt/gencode-remote

@@ -81,12 +81,24 @@ aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::=
 
 case "$state_should" in
     present)
+        # There are special arguments to apt(8) to prevent aborts if apt woudn't been
+        # updated after the 19th April 2021 till the bullseye release. The additional
+        # arguments acknoledge the happend suite change (the apt(8) update does the
+        # same by itself).
+        #
+        # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+        # allows backward compatablility to pre-buster Debian versions.
+        #
+        # See more: ticket #861
+        # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+        apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
         # following is bit ugly, but important hack.
         # due to how cdist config run works, there isn't
         # currently better way to do it :(
         cat << EOF
 if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ]
-then echo apt-get update > /dev/null 2>&1 || true
+then echo apt-get $apt_opts update > /dev/null 2>&1 || true
 fi
 EOF
         if [ -n "$version" ]; then

cdist/conf/type/__package_pkg_freebsd/gencode-remote

@@ -37,6 +37,7 @@ assert ()                 #  If condition false,
 	then
 		echo "Assertion failed:  \"$1\""
 		# shellcheck disable=SC2039
+		# shellcheck disable=SC3044
 		echo "File \"$0\", line $lineno, called by $(caller 0)"
 		exit $E_ASSERT_FAILED
 	fi

cdist/conf/type/__package_update_index/gencode-remote

@@ -41,7 +41,19 @@ fi
 case "$type" in
     yum) ;;
     apt)
-        echo "apt-get --quiet update"
+        # There are special arguments to apt(8) to prevent aborts if apt woudn't been
+        # updated after the 19th April 2021 till the bullseye release. The additional
+        # arguments acknoledge the happend suite change (the apt(8) update does the
+        # same by itself).
+        #
+        # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+        # allows backward compatablility to pre-buster Debian versions.
+        #
+        # See more: ticket #861
+        # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+        apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
+        echo "apt-get --quiet $apt_opts update"
         echo "apt-cache updated (age was: $currage)" >> "$__messages_out"
         ;;
     pacman)

cdist/conf/type/__sed/explorer/file

@@ -0,0 +1,16 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/file" ]
+then
+    file="$( cat "$__object/parameter/file" )"
+else
+    file="/$__object_id"
+fi
+
+if [ ! -e "$file" ]
+then
+    echo "$file does not exist" >&2
+    exit 1
+fi
+
+cat "$file"

cdist/conf/type/__sed/gencode-remote

@@ -0,0 +1,58 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/file" ]
+then
+    file="$( cat "$__object/parameter/file" )"
+else
+    file="/$__object_id"
+fi
+
+script="$( cat "$__object/parameter/script" )"
+
+if [ "$script" = '-' ]
+then
+    script="$( cat "$__object/stdin" )"
+fi
+
+# since stdin is not available in explorer, we pull file from target with explorer
+
+file_from_target="$__object/explorer/file"
+
+sed_cmd='sed'
+
+if [ -f "$__object/parameter/regexp-extended" ]
+then
+    sed_cmd="$sed_cmd -E"
+fi
+
+# do sed dry run, diff result and if no change, then there's nothing to do
+# also redirect diff's output to stderr for debugging purposes
+
+if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2
+then
+    exit 0
+fi
+
+# we can't use -i, because it's not posix, so we fly with tempfile and cp
+# and we use cp because we want to preserve destination file's attributes
+
+# shellcheck disable=SC2016
+echo 'tmp="$__object/tempfile"'
+
+echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
+
+echo "$script"
+
+echo 'EOF'
+
+echo "cp \"\$tmp\" '$file'"
+
+# shellcheck disable=SC2016
+echo 'rm -f "$tmp"'
+
+echo 'change' >> "$__messages_out"
+
+if [ -f "$__object/parameter/onchange" ]
+then
+    cat "$__object/parameter/onchange"
+fi

cdist/conf/type/__sed/man.rst

@@ -0,0 +1,57 @@
+cdist-type__sed(7)
+==================
+
+NAME
+----
+cdist-type__sed - Transform text files with ``sed``
+
+
+DESCRIPTION
+-----------
+Transform text files with ``sed``.
+
+
+REQUIRED MULTIPLE PARAMETERS
+----------------------------
+script
+   ``sed`` script.
+   If ``-`` then the script is read from ``stdin``.
+
+
+OPTIONAL PARAMETERS
+-------------------
+file
+   Path to the file. Defaults to ``$__object_id``.
+
+onchange
+   Execute this command if ``sed`` changes file.
+
+
+BOOLEAN PARAMETERS
+------------------
+regexp-extended
+   Use extended regular expressions in the script.
+   Might not be supported with every ``sed`` version.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+   __sed /tmp/foobar --script 's/foo/bar/'
+
+   echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script -
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.

cdist/conf/type/__sed/parameter/boolean

@@ -0,0 +1 @@
+regexp-extended

cdist/conf/type/__sed/parameter/optional

@@ -0,0 +1,2 @@
+file
+onchange

cdist/conf/type/__sed/parameter/required_multiple

@@ -0,0 +1 @@
+script

cdist/conf/type/__ssh_authorized_keys/explorer/keys

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 
 # shellcheck disable=SC1090
+# shellcheck disable=SC1091
 file="$( . "$__type_explorer/file" )"
 
 if [ -f "$file" ]

cdist/conf/type/__update_alternatives/explorer/alternatives

@@ -1,4 +1,4 @@
 #!/bin/sh -e
 
-update-alternatives --display "$__object_id" 2>/dev/null \
-    | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
+LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \
+| awk -F ' - ' '/priority [0-9]+$/ { print $1 }'

cdist/conf/type/__update_alternatives/explorer/link

@@ -18,12 +18,12 @@ for altdir in \
     /var/lib/dpkg/alternatives \
     /var/lib/alternatives
 do
-    if [ ! -f "$altdir/$__object_id" ]
+    if [ ! -f "$altdir/${__object_id:?}" ]
     then
         continue
     fi
 
-    link="$( awk 'NR==2' "$altdir/$__object_id" )"
+    link="$( awk 'NR==2' "$altdir/${__object_id:?}" )"
 
     if [ -n "$link" ]
     then
@@ -31,9 +31,12 @@ do
     fi
 done
 
-if [ -z "$link" ]
+if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then
-    echo "unable to get link for $__object_id" >&2
+    # NOTE: ignore error for dry-runs because a package providing the link
+    #       might be managed by another cdist object (which wasn't executed,
+    #       because dry run…).
+    echo "unable to get link for ${__object_id:?}" >&2
     exit 1
 fi
 

cdist/conf/type/__update_alternatives/explorer/path_is

@@ -1,11 +1,15 @@
 #!/bin/sh -e
 
-path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \
-    | awk '/link currently points to/ {print $5}' )"
+path_is=$(
+    LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \
+    | awk '/link currently points to/ { print $5 }')
 
-if [ -z "$path_is" ]
+if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then
-    echo "unable to get current path for $__object_id" >&2
+    # NOTE: ignore error for dry-runs because a package providing the
+    #       alternative might be managed by another cdist object (which
+    #       wasn't executed, because dry run…).
+    echo "unable to get current path for ${__object_id:?}" >&2
     exit 1
 fi
 

cdist/conf/type/__update_alternatives/explorer/path_should_state

@@ -1,6 +1,6 @@
 #!/bin/sh -e
 
-if [ -f "$( cat "$__object/parameter/path" )" ]
+if [ -f "$( cat "${__object:?}/parameter/path" )" ]
 then
     echo 'present'
 else

cdist/conf/type/__update_alternatives/gencode-remote

@@ -18,37 +18,39 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 
-path_is="$( cat "$__object/explorer/path_is" )"
+path_is="$( cat "${__object:?}/explorer/path_is" )"
 
-path_should="$( cat "$__object/parameter/path" )"
+path_should="$( cat "${__object:?}/parameter/path" )"
 
 if [ "$path_is" = "$path_should" ]
 then
     exit 0
 fi
 
-if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ]
+if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \
+    && [ -z "${__cdist_dry_run+dry run}" ]
 then
     echo "$path_should does not exist in target" >&2
     exit 1
 fi
 
-name="$__object_id"
+name=${__object_id:?}
 
-alternatives="$( cat "$__object/explorer/alternatives" )"
-
-if ! echo "$alternatives" | grep -Fxq "$path_should"
+if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives"
 then
-    if [ ! -f "$__object/parameter/install" ]
+    if [ -f "${__object:?}/parameter/install" ]
     then
+        link="$( cat "${__object:?}/explorer/link" )"
+        echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
+    elif [ -z "${__cdist_dry_run+dry run}" ]
+    then
+        # NOTE: ignore error for dry-runs because a package providing the link
+        #       to be installed might be managed by another cdist object (which
+        #       wasn't executed, because dry run…).
         echo "$path_should is not in $name alternatives." >&2
         echo 'Please install missing packages or use --install to add path to alternatives.' >&2
         exit 1
     fi
-
-    link="$( cat "$__object/explorer/link" )"
-
-    echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
 fi
 
 echo "update-alternatives --set '$name' '$path_should'"

docs/changelog

@@ -2,10 +2,27 @@ Changelog
 ---------
 
 next:
+	* Explorer machine_type: Rewrite (Dennis Camera)
+	* New type: __sed (Ander Punnar)
+	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
+	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
+	* Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham)
+
+6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
 	* Core: Fix logging bug (Dennis Camera)
+	* Build: Improve Makefile compatibility (Evilham)
+	* Type __filesystem: Support ubuntu (Joachim Desroches)
+	* Explorer os_version: Fall back to os-release/lsb-release file on Ubuntu (Dennis Camera)
+	* Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera)
+	* Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera)
+	* Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera)
+	* Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)