++changelog

Even if deprecated, the parameter *must* be supported, which isn't the case
right now.

This was due to a misunderstanding of how deprecating parameters work, see:
https://www.cdi.st/manual/latest/cdist-type.html#deprecated-parameters

Makefile

@@ -35,9 +35,9 @@ DOCS_SRC_DIR=./docs/src
 SPEECHDIR=./docs/speeches
 TYPEDIR=./cdist/conf/type
 
-SPHINXM=make -C $(DOCS_SRC_DIR) man
+SPHINXM=$(MAKE) -C $(DOCS_SRC_DIR) man
-SPHINXH=make -C $(DOCS_SRC_DIR) html
+SPHINXH=$(MAKE) -C $(DOCS_SRC_DIR) html
-SPHINXC=make -C $(DOCS_SRC_DIR) clean
+SPHINXC=$(MAKE) -C $(DOCS_SRC_DIR) clean
 
 ################################################################################
 # Manpages

README.md

@@ -24,8 +24,8 @@ For community-maintained types there is
 
 ## Participating
 
-IRC: ``#cdist`` @ freenode
+IRC: ``#cdist`` @ [libera](https://libera.chat)
 
 Matrix: ``#cdist:ungleich.ch``
 
-Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist
+Matrix and IRC are bridged.

cdist/argparse.py

@@ -485,19 +485,31 @@ def get_parsers():
     parser['scan'].add_argument(
         '-m', '--mode', help='Which modes should run',
         action='append', default=[],
-        choices=['scan', 'trigger'])
+        choices=['scan', 'trigger', 'config'])
+    parser['scan'].add_argument(
+        '--list',
+        action='store_true',
+        help='List the known hosts and exit')
     parser['scan'].add_argument(
         '--config',
         action='store_true',
         help='Try to configure detected hosts')
     parser['scan'].add_argument(
-        '-I', '--interfaces',
+        '-I', '--interface',
-        action='append',  default=[],
+        action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
     parser['scan'].add_argument(
-        '-d', '--delay',
+        '--name-mapper',
-        action='store',  default=3600,
+        action='store',  default=None,
-        help='How long to wait before reconfiguring after last try')
+        help='Map addresses to names, required for config mode')
+    parser['scan'].add_argument(
+        '-d', '--config-delay',
+        action='store',  default=3600, type=int,
+        help='How long (seconds) to wait before reconfiguring after last try')
+    parser['scan'].add_argument(
+        '-t', '--trigger-delay',
+        action='store',  default=5, type=int,
+        help='How long (seconds) to wait between ICMPv6 echo requests')
     parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
 
     for p in parser:

cdist/conf/explorer/memory

@@ -27,19 +27,18 @@
 str2bytes() {
 	awk -F' ' '
 	$2 ==   "B" || !$2 { print $1 }
-	$2 ==  "kB" { print $1 * 1000 }
+	$2 ==  "kB" { printf "%.f\n", ($1 * 1000) }
-	$2 ==  "MB" { print $1 * 1000 * 1000 }
+	$2 ==  "MB" { printf "%.f\n", ($1 * 1000 * 1000) }
-	$2 ==  "GB" { print $1 * 1000 * 1000 * 1000 }
+	$2 ==  "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) }
-	$2 ==  "TB" { print $1 * 1000 * 1000 * 1000 * 1000 }
+	$2 ==  "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) }
-	$2 == "kiB" { print $1 * 1024 }
+	$2 == "kiB" { printf "%.f\n", ($1 * 1024) }
-	$2 == "MiB" { print $1 * 1024 * 1024 }
+	$2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) }
-	$2 == "GiB" { print $1 * 1024 * 1024 * 1024 }
+	$2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) }
-	$2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }'
+	$2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }'
 }
 
 bytes2kib() {
-	set -- "$(cat)"
+	awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }'
-	test "$1" -gt 0 && echo $(($1 / 1024))
 }
 
 

cdist/conf/explorer/os_version

@@ -1,6 +1,7 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,12 +18,22 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
 # All os variables are lower case
 #
-#
 
-case "$("$__explorer/os")" in
+rc_getvar() {
+   awk -F= -v varname="$2" '
+      function unquote(s) {
+         if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/)
+            return substr(s, 2, length(s) - 2)
+         else
+            return s
+      }
+      $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1"
+}
+
+case $("${__explorer:?}/os")
+in
    amazon)
       cat /etc/system-release
    ;;
@@ -43,6 +54,8 @@ case "$("$__explorer/os")" in
               # sid versions don't have a number, so we decode by codename:
               case $(expr "$debian_version" : '\([a-z]\{1,\}\)/')
               in
+                  trixie) echo 12.99 ;;
+                  bookworm) echo 11.99 ;;
                   bullseye) echo 10.99 ;;
                   buster) echo 9.99 ;;
                   stretch) echo 8.99 ;;
@@ -50,7 +63,7 @@ case "$("$__explorer/os")" in
                   wheezy) echo 6.99 ;;
                   squeeze) echo 5.99 ;;
                   lenny) echo 4.99 ;;
-                  *) exit 1
+                  *) echo 99.99 ;;
               esac
               ;;
           *)
@@ -59,7 +72,23 @@ case "$("$__explorer/os")" in
       esac
    ;;
    devuan)
-      cat /etc/devuan_version
+      devuan_version=$(cat /etc/devuan_version)
+      case ${devuan_version}
+      in
+         (*/ceres)
+            # ceres versions don't have a number, so we decode by codename:
+            case ${devuan_version}
+            in
+               (chimaera/ceres) echo 3.99 ;;
+               (beowulf/ceres) echo 2.99 ;;
+               (ascii/ceres) echo 1.99 ;;
+               (*) exit 1
+            esac
+            ;;
+         (*)
+            echo "${devuan_version}"
+            ;;
+      esac
    ;;
    fedora)
       cat /etc/fedora-release
@@ -68,12 +97,20 @@ case "$("$__explorer/os")" in
       cat /etc/gentoo-release
    ;;
    macosx)
-      sw_vers -productVersion
+      # NOTE: Legacy versions (< 10.3) do not support options
+      sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }'
    ;;
    freebsd)
       # Apparently uname -r is not a reliable way to get the patch level.
       # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
-      freebsd-version
+      if command -v freebsd-version >/dev/null 2>&1
+      then
+         # get userland version
+         freebsd-version -u
+      else
+         # fallback to kernel release for FreeBSD < 10.0
+         uname -r
+      fi
    ;;
    *bsd|solaris)
       uname -r
@@ -98,7 +135,20 @@ case "$("$__explorer/os")" in
       fi
    ;;
    ubuntu)
-      lsb_release -sr
+      if command -v lsb_release >/dev/null 2>&1
+      then
+         lsb_release -sr
+      elif test -r /usr/lib/os-release
+      then
+         # fallback to /usr/lib/os-release if lsb_release is not present (like
+         # on minimized Ubuntu installations)
+         rc_getvar /usr/lib/os-release VERSION_ID
+      elif test -r /etc/lsb-release
+      then
+         # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old
+         # versions without /usr/lib/os-release.
+         rc_getvar /etc/lsb-release DISTRIB_RELEASE
+      fi
    ;;
    alpine)
        cat /etc/alpine-release

cdist/conf/type/__apt_backports/manifest

@@ -28,6 +28,7 @@
 #  lsb_release may not be given in all installations
 codename_os_release() {
     # shellcheck disable=SC1090
+    # shellcheck disable=SC1091
     . "$__global/explorer/os_release"
     printf "%s" "$VERSION_CODENAME"
 }

cdist/conf/type/__apt_pin/man.rst

@@ -0,0 +1,79 @@
+cdist-type__apt_pin(7)
+======================
+
+NAME
+----
+cdist-type__apt_pin - Manage apt pinning rules
+
+
+DESCRIPTION
+-----------
+Adds/removes/edits rules to pin some packages to a specific distribution. Useful if using multiple debian repositories at the same time. (Useful, if one wants to use a few specific packages from backports or perhaps Debain testing... or even sid.)
+
+
+REQUIRED PARAMETERS
+-------------------
+distribution
+   Specifies what distribution the package should be pinned to. Accepts both codenames (buster/bullseye/sid) and suite names (stable/testing/...).
+
+
+OPTIONAL PARAMETERS
+-------------------
+package
+   Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used.
+
+priority
+   The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority)
+
+state
+   Will be passed to underlying `__file` type; see there for valid values and defaults.
+
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+   # Add the bullseye repo to buster, but do not install any packages by default,
+   # only if explicitely asked for (-1 means "never" for apt)
+    __apt_pin bullseye-default \
+       --package "*" \
+       --distribution bullseye \
+       --priority -1
+
+    require="__apt_pin/bullseye-default" __apt_source bullseye \
+       --uri http://deb.debian.org/debian/ \
+       --distribution bullseye \
+       --component main
+
+    __apt_pin foo --package "foo foo-*" --distribution bullseye
+
+    __foo # Assuming, this installs the `foo` package internally
+
+    __package foo-plugin-extras # Assuming we also need some extra stuff
+
+
+SEE ALSO
+--------
+:strong:`apt_preferences`\ (5)
+:strong:`cdist-type__apt_source`\ (7)
+:strong:`cdist-type__apt_backports`\ (7)
+:strong:`cdist-type__file`\ (7)
+
+AUTHORS
+-------
+Daniel Fancsali <fancsali@gmail.com>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Daniel Fancsali. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__apt_pin/manifest

@@ -0,0 +1,63 @@
+#!/bin/sh -e
+#
+# 2021 Daniel Fancsali (fancsali@gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+name="$__object_id"
+
+os=$(cat "$__global/explorer/os")
+state="$(cat "$__object/parameter/state")"
+
+if [ -f "$__object/parameter/package" ]; then
+    package="$(cat "$__object/parameter/package")"
+else
+    package=$name
+fi
+
+distribution="$(cat "$__object/parameter/distribution")"
+priority="$(cat "$__object/parameter/priority")"
+
+
+case "$os" in
+   debian|ubuntu|devuan)
+   ;;
+   *)
+      printf "This type is specific to Debian and it's derivatives" >&2
+      exit 1
+   ;;
+esac
+
+case $distribution in
+    stable|testing|unstable|experimental)
+        pin="release a=$distribution"
+        ;;
+    *)
+        pin="release n=$distribution"
+        ;;
+esac
+
+
+__file "/etc/apt/preferences.d/$name" \
+    --owner root --group root --mode 0644 \
+    --state "$state" \
+    --source - << EOF
+Package: $package
+Pin: $pin
+Pin-Priority: $priority
+EOF

cdist/conf/type/__apt_pin/parameter/default/state

@@ -0,0 +1 @@
+present

cdist/conf/type/__apt_pin/parameter/optional

@@ -0,0 +1,2 @@
+state
+package

cdist/conf/type/__apt_pin/parameter/required

@@ -0,0 +1,2 @@
+distribution
+priority

cdist/conf/type/__apt_source/gencode-remote

@@ -22,7 +22,21 @@
 name="$__object_id"
 destination="/etc/apt/sources.list.d/${name}.list"
 
+# There are special arguments to apt(8) to prevent aborts if apt woudn't been
+# updated after the 19th April 2021 till the bullseye release. The additional
+# arguments acknoledge the happend suite change (the apt(8) update does the
+# same by itself).
+#
+# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+# allows backward compatablility to pre-buster Debian versions.
+#
+# See more: ticket #861
+# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
+# run 'apt-get update' only if something changed with our sources.list file
+#   it will be run a second time on error as a redundancy messure to success
 if grep -q "^__file${destination}" "$__messages_in"; then
-   printf 'apt-get update || apt-get update\n'
+   printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts"
 fi
 

cdist/conf/type/__apt_update_index/gencode-remote

@@ -18,9 +18,23 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+
+# There are special arguments to apt(8) to prevent aborts if apt woudn't been
+# updated after the 19th April 2021 till the bullseye release. The additional
+# arguments acknoledge the happend suite change (the apt(8) update does the
+# same by itself).
+#
+# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+# allows backward compatablility to pre-buster Debian versions.
+#
+# See more: ticket #861
+# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
 # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists
+#   it will be run a second time on error as a redundancy messure to success
 cat << DONE
 if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then
-   apt-get update || apt-get update
+   apt-get $apt_opts update || apt-get $apt_opts update
 fi
 DONE

cdist/conf/type/__debconf_set_selections/explorer/state

@@ -0,0 +1,142 @@
+#!/bin/sh -e
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Determine current debconf selections' state.
+# Prints one of:
+#   present: all selections are already set as they should.
+#   different: one or more of the selections have a different value.
+#   absent: one or more of the selections are not (currently) defined.
+#
+
+test -x /usr/bin/perl || {
+	# cannot find perl (no perl ~ no debconf)
+	echo 'absent'
+	exit 0
+}
+
+linesfile="${__object:?}/parameter/line"
+test -s "${linesfile}" || {
+	if test -s "${__object:?}/parameter/file"
+	then
+		echo absent
+	else
+		echo present
+	fi
+	exit 0
+}
+
+# assert __type_explorer is set (because it is used by the Perl script)
+: "${__type_explorer:?}"
+
+/usr/bin/perl -- - "${linesfile}" <<'EOF'
+use strict;
+use warnings "all";
+
+use Fcntl qw(:DEFAULT :flock);
+
+use Debconf::Db;
+use Debconf::Question;
+
+# Extract @known... arrays from debconf-set-selections
+# These values are required to distinguish flags and values in the given lines.
+# DC: I couldn't think of a more ugly solution to the problem…
+my @knownflags;
+my @knowntypes;
+my $debconf_set_selections = '/usr/bin/debconf-set-selections';
+if (-e $debconf_set_selections) {
+	my $sed_known = 's/^my \(@known\(flags\|types\) = qw([a-z ]*);\).*$/\1/p';
+	eval `sed -n '$sed_known' '$debconf_set_selections'`;
+}
+
+sub mungeline ($) {
+	my $line = shift;
+	chomp $line;
+	$line =~ s/\r$//;
+	return $line;
+}
+
+sub fatal { printf STDERR @_; exit 1; }
+
+my $state = 'present';
+
+sub state {
+	my $new = shift;
+	if ($state eq 'present'
+	    or ($state eq 'different' and $new eq 'absent')) {
+		$state = $new;
+	}
+}
+
+
+# Load Debconf DB but manually lock on the state explorer script,
+# because Debconf aborts immediately if executed concurrently.
+# This is not really an ideal solution because the Debconf DB could be locked by
+# another process (e.g. apt-get), but no way to achieve this could be found.
+# If you know how to, please provide a patch.
+my $lockfile = "%ENV{'__type_explorer'}/state";
+if (open my $lock_fh, '+<', $lockfile) {
+   flock $lock_fh, LOCK_EX or die "Cannot lock $lockfile";
+}
+{
+	Debconf::Db->load(readonly => 'true');
+}
+
+
+while (<>) {
+	# Read and process lines (taken from debconf-set-selections)
+	$_ = mungeline($_);
+	while (/\\$/ && ! eof) {
+		s/\\$//;
+		$_ .= mungeline(<>);
+	}
+	next if /^\s*$/ || /^\s*\#/;
+
+	my ($owner, $label, $type, $content) = /^\s*(\S+)\s+(\S+)\s+(\S+)(?:\s(.*))?/
+		or fatal "invalid line: %s\n", $_;
+	$content = '' unless defined $content;
+
+
+	# Compare is and should state
+	my $q = Debconf::Question->get($label);
+
+	unless (defined $q) {
+		# probably a preseed
+		state 'absent';
+		next;
+	}
+
+	if (grep { $_ eq $q->type } @knownflags) {
+		# This line wants to set a flag, presumably.
+		if ($q->flag($q->type) ne $content) {
+			state 'different';
+		}
+	} else {
+		# Otherwise, it's probably a value…
+		if ($q->value ne $content) {
+			state 'different';
+		}
+
+		unless (grep { $_ eq $owner } (split /, /, $q->owners)) {
+			state 'different';
+		}
+	}
+}
+
+printf "%s\n", $state;
+EOF

cdist/conf/type/__debconf_set_selections/gencode-remote

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,16 +18,37 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# Setup selections
-#
 
-filename="$(cat "$__object/parameter/file")"
+if test -f "${__object:?}/parameter/line"
-
+then
-if [ "$filename" = "-" ]; then
+	filename="${__object:?}/parameter/line"
-    filename="$__object/stdin"
+elif test -s "${__object:?}/parameter/file"
+then
+	filename=$(cat "${__object:?}/parameter/file")
+	if test "${filename}" = '-'
+	then
+		filename="${__object:?}/stdin"
+	fi
+else
+	printf 'Neither --line nor --file set.\n' >&2
+	exit 1
 fi
 
-echo "debconf-set-selections << __file-eof"
+# setting no lines makes no sense
-cat "$filename"
+test -s "${filename}" || exit 0
-echo "__file-eof"
+
+state_is=$(cat "${__object:?}/explorer/state")
+
+if test "${state_is}" != 'present'
+then
+	cat <<-CODE
+	debconf-set-selections <<'EOF'
+	$(cat "${filename}")
+	EOF
+	CODE
+
+	awk '
+		{
+			printf "set %s %s %s %s\n", $1, $2, $3, $4
+		}' "${filename}" >>"${__messages_out:?}"
+fi

cdist/conf/type/__debconf_set_selections/man.rst

@@ -8,15 +8,33 @@ cdist-type__debconf_set_selections - Setup debconf selections
 
 DESCRIPTION
 -----------
-On Debian and alike systems debconf-set-selections(1) can be used
+On Debian and alike systems :strong:`debconf-set-selections`\ (1) can be used
 to setup configuration parameters.
 
 
 REQUIRED PARAMETERS
 -------------------
+cf. ``--line``.
+
+
+OPTIONAL PARAMETERS
+-------------------
 file
-   Use the given filename as input for debconf-set-selections(1)
+   Use the given filename as input for :strong:`debconf-set-selections`\ (1)
-   If filename is "-", read from stdin.
+   If filename is ``-``, read from stdin.
+
+   **This parameter is deprecated, because it doesn't work with state detection.**
+line
+   A line in :strong:`debconf-set-selections`\ (1) compatible format.
+   This parameter can be used multiple times to set multiple options.
+
+   (This parameter is actually required, but marked optional because the
+   deprecated ``--file`` is still accepted.)
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
 
 
 EXAMPLES
@@ -24,30 +42,29 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # Setup configuration for nslcd
+    # Setup gitolite's gituser
-    __debconf_set_selections nslcd --file /path/to/file
+    __debconf_set_selections nslcd --line 'gitolite gitolite/gituser string git'
 
-    # Setup configuration for nslcd from another type
+    # Setup configuration for nslcd from a file.
-    __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd"
+    # NB: Multiple lines can be passed to --line, although this can be considered a hack.
-
+    __debconf_set_selections nslcd --line "$(cat "${__files:?}/preseed/nslcd.debconf")"
-    __debconf_set_selections nslcd --file - << eof
-    gitolite gitolite/gituser string git
-    eof
 
 
 SEE ALSO
 --------
-:strong:`debconf-set-selections`\ (1), :strong:`cdist-type__update_alternatives`\ (7)
+- :strong:`cdist-type__update_alternatives`\ (7)
+- :strong:`debconf-set-selections`\ (1)
 
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2014 Nico Schottelius. You can redistribute it
+Copyright \(C) 2011-2014 Nico Schottelius, 2021 Dennis Camera.
-and/or modify it under the terms of the GNU General Public License as
+You can redistribute it and/or modify it under the terms of the GNU General
-published by the Free Software Foundation, either version 3 of the
+Public License as published by the Free Software Foundation, either version 3 of
-License, or (at your option) any later version.
+the License, or (at your option) any later version.

cdist/conf/type/__debconf_set_selections/manifest

@@ -1,6 +1,6 @@
 #!/bin/sh -e
 #
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,20 +18,4 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-if [ -f "$__object/parameter/destination" ]; then
+__package_apt debconf
-    destination=$(cat "$__object/parameter/destination")
-else
-    destination="/$__object_id"
-fi
-
-ownergroup=""
-if [ -f "$__object/parameter/owner" ]; then
-    ownergroup=$(cat "$__object/parameter/owner")
-fi
-if [ -f "$__object/parameter/group" ]; then
-    ownergroup="${ownergroup}:$(cat "$__object/parameter/group")"
-fi
-
-if [ "$ownergroup" ]; then
-    echo chown -R "$ownergroup" "$destination"
-fi

cdist/conf/type/__debconf_set_selections/parameter/deprecated/file

@@ -0,0 +1 @@
+'file' has been deprecated in favour of 'line' in order to provide idempotency.

cdist/conf/type/__debconf_set_selections/parameter/optional_multiple

@@ -0,0 +1 @@
+line

cdist/conf/type/__download/explorer/remote_cmd

@@ -1,19 +0,0 @@
-#!/bin/sh -e
-
-if [ -f "$__object/parameter/cmd-get" ]
-then
-    cmd="$( cat "$__object/parameter/cmd-get" )"
-
-elif command -v curl > /dev/null
-then
-    cmd="curl -L -o - '%s'"
-
-elif command -v fetch > /dev/null
-then
-    cmd="fetch -o - '%s'"
-
-else
-    cmd="wget -O - '%s'"
-fi
-
-echo "$cmd"

cdist/conf/type/__download/explorer/remote_cmd_get

@@ -0,0 +1,16 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cat "$__object/parameter/cmd-get"
+elif
+    command -v curl > /dev/null
+then
+    echo "curl -sSL -o - '%s'"
+elif
+    command -v fetch > /dev/null
+then
+    echo "fetch -o - '%s'"
+else
+    echo "wget -O - '%s'"
+fi

cdist/conf/type/__download/explorer/remote_cmd_sum

@@ -0,0 +1,82 @@
+#!/bin/sh -e
+
+if [ ! -f "$__object/parameter/sum" ]
+then
+    exit 0
+fi
+
+if [ -f "$__object/parameter/cmd-sum" ]
+then
+    cat "$__object/parameter/cmd-sum"
+    exit 0
+fi
+
+sum_should="$( cat "$__object/parameter/sum" )"
+
+if echo "$sum_should" | grep -Fq ':'
+then
+    sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
+else
+    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+    then
+        sum_hash='cksum'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
+    then
+        sum_hash='md5'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
+    then
+        sum_hash='sha1'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
+    then
+        sum_hash='sha256'
+    else
+        echo 'hash format detection failed' >&2
+        exit 1
+    fi
+fi
+
+os="$( "$__explorer/os" )"
+
+case "$sum_hash" in
+    cksum)
+        echo "cksum %s | awk '{print \$1\" \"\$2}'"
+    ;;
+    md5)
+        case "$os" in
+            freebsd)
+                echo "md5 -q %s"
+            ;;
+            *)
+                echo "md5sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    sha1)
+        case "$os" in
+            freebsd)
+                echo "sha1 -q %s"
+            ;;
+            *)
+                echo "sha1sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    sha256)
+        case "$os" in
+            freebsd)
+                echo "sha256 -q %s"
+            ;;
+            *)
+                echo "sha256sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    *)
+        # we arrive here only if --sum is given with unknown format prefix
+        echo "unknown hash format: $sum_hash" >&2
+        exit 1
+    ;;
+esac

cdist/conf/type/__download/explorer/state

@@ -1,6 +1,11 @@
 #!/bin/sh -e
 
-dst="/$__object_id"
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
+else
+    dst="/$__object_id"
+fi
 
 if [ ! -f "$dst" ]
 then
@@ -16,57 +21,19 @@ fi
 
 sum_should="$( cat "$__object/parameter/sum" )"
 
-if [ -f "$__object/parameter/cmd-sum" ]
+if echo "$sum_should" | grep -Fq ':'
 then
-    # shellcheck disable=SC2059
+    sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
-    sum_is="$( eval "$( printf \
-        "$( cat "$__object/parameter/cmd-sum" )" \
-        "$dst" )" )"
-else
-    os="$( "$__explorer/os" )"
-
-    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
-    then
-        sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
-
-    elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="md5:$( md5 -q "$dst" )"
-            ;;
-            *)
-                sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-
-    elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="sha1:$( sha1 -q "$dst" )"
-            ;;
-            *)
-                sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-
-    elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="sha256:$( sha256 -q "$dst" )"
-            ;;
-            *)
-                sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-    fi
 fi
 
+sum_cmd="$( "$__type_explorer/remote_cmd_sum" )"
+
+# shellcheck disable=SC2059
+sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )"
+
 if [ -z "$sum_is" ]
 then
-    echo 'no checksum from target' >&2
+    echo 'existing destination checksum failed' >&2
     exit 1
 fi
 

cdist/conf/type/__download/gencode-local

@@ -11,34 +11,133 @@ fi
 
 url="$( cat "$__object/parameter/url" )"
 
-tmp="$( mktemp )"
+if [ -f "$__object/parameter/destination" ]
-
+then
-dst="/$__object_id"
+    dst="$( cat "$__object/parameter/destination" )"
+else
+    dst="/$__object_id"
+fi
 
 if [ -f "$__object/parameter/cmd-get" ]
 then
     cmd="$( cat "$__object/parameter/cmd-get" )"
 
-elif command -v wget > /dev/null
-then
-    cmd="wget -O - '%s'"
-
 elif command -v curl > /dev/null
 then
-    cmd="curl -L -o - '%s'"
+    cmd="curl -sSL -o - '%s'"
 
 elif command -v fetch > /dev/null
 then
     cmd="fetch -o - '%s'"
 
+elif command -v wget > /dev/null
+then
+    cmd="wget -O - '%s'"
+
 else
-    echo 'no usable locally installed utility for downloading' >&2
+    echo 'local download failed, no usable utility' >&2
     exit 1
 fi
 
-printf "$cmd > %s\n" \
+echo "download_tmp=\"\$( mktemp )\""
-    "$url" \
+
-    "$tmp"
+# shellcheck disable=SC2059
+printf "$cmd > \"\$download_tmp\"\n" "$url"
+
+if [ -f "$__object/parameter/sum" ]
+then
+    sum_should="$( cat "$__object/parameter/sum" )"
+
+    if [ -f "$__object/parameter/cmd-sum" ]
+    then
+        local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
+    else
+        if echo "$sum_should" | grep -Fq ':'
+        then
+            sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
+
+            sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
+        else
+            if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+            then
+                sum_hash='cksum'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
+            then
+                sum_hash='md5'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
+            then
+                sum_hash='sha1'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
+            then
+                sum_hash='sha256'
+            else
+                echo 'hash format detection failed' >&2
+                exit 1
+            fi
+        fi
+
+        case "$sum_hash" in
+            cksum)
+                local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'"
+            ;;
+            md5)
+                if command -v md5 > /dev/null
+                then
+                    local_cmd_sum="md5 -q %s"
+                elif
+                    command -v md5sum > /dev/null
+                then
+                    local_cmd_sum="md5sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            sha1)
+                if command -v sha1 > /dev/null
+                then
+                    local_cmd_sum="sha1 -q %s"
+                elif
+                    command -v sha1sum > /dev/null
+                then
+                    local_cmd_sum="sha1sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            sha256)
+                if command -v sha256 > /dev/null
+                then
+                    local_cmd_sum="sha256 -q %s"
+                elif
+                    command -v sha256sum > /dev/null
+                then
+                    local_cmd_sum="sha256sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            *)
+                # we arrive here only if --sum is given with unknown format prefix
+                echo "unknown hash format: $sum_hash" >&2
+                exit 1
+            ;;
+        esac
+
+        if [ -z "$local_cmd_sum" ]
+        then
+            echo 'local checksum verification failed, no usable utility' >&2
+            exit 1
+        fi
+    fi
+
+    # shellcheck disable=SC2059
+    echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\""
+
+    echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
+
+    echo "echo 'local download checksum mismatch' >&2"
+
+    echo "rm -f \"\$download_tmp\""
+
+    echo 'exit 1; fi'
+fi
 
 if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$'
 then
@@ -47,12 +146,10 @@ else
     target_host="$__target_host"
 fi
 
-printf '%s %s %s:%s\n' \
+# shellcheck disable=SC2016
+printf '%s "$download_tmp" %s:%s\n' \
     "$__remote_copy" \
-    "$tmp" \
     "$target_host" \
     "$dst"
 
-echo "rm -f '$tmp'"
+echo "rm -f \"\$download_tmp\""
-
-echo 'downloaded' > "$__messages_out"

cdist/conf/type/__download/gencode-remote

@@ -6,17 +6,51 @@ state_is="$( cat "$__object/explorer/state" )"
 
 if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
 then
-    cmd="$( cat "$__object/explorer/remote_cmd" )"
+    cmd_get="$( cat "$__object/explorer/remote_cmd_get" )"
 
     url="$( cat "$__object/parameter/url" )"
 
-    dst="/$__object_id"
+    if [ -f "$__object/parameter/destination" ]
+    then
+        dst="$( cat "$__object/parameter/destination" )"
+    else
+        dst="/$__object_id"
+    fi
 
-    printf "$cmd > %s\n" \
+    echo "download_tmp=\"\$( mktemp )\""
-        "$url" \
-        "$dst"
 
-    echo 'downloaded' > "$__messages_out"
+    # shellcheck disable=SC2059
+    printf "$cmd_get > \"\$download_tmp\"\n" "$url"
+
+    if [ -f "$__object/parameter/sum" ]
+    then
+        sum_should="$( cat "$__object/parameter/sum" )"
+
+        if [ -f "$__object/parameter/cmd-sum" ]
+        then
+            remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
+        else
+            remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )"
+
+            if echo "$sum_should" | grep -Fq ':'
+            then
+                sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
+            fi
+        fi
+
+        # shellcheck disable=SC2059
+        echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\""
+
+        echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
+
+        echo "echo 'remote download checksum mismatch' >&2"
+
+        echo "rm -f \"\$download_tmp\""
+
+        echo 'exit 1; fi'
+    fi
+
+    echo "mv \"\$download_tmp\" '$dst'"
 fi
 
 if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ]

cdist/conf/type/__download/man.rst

@@ -8,7 +8,7 @@ cdist-type__download - Download a file
 
 DESCRIPTION
 -----------
-By default type will try to use ``wget``, ``curl`` or ``fetch``.
+By default type will try to use ``curl``, ``fetch`` or ``wget``.
 If download happens in target (see ``--download``) then type will
 fallback to (and install) ``wget``.
 
@@ -16,6 +16,8 @@ If download happens in local machine, then environment variables like
 ``{http,https,ftp}_proxy`` etc can be used on cdist execution
 (``http_proxy=foo cdist config ...``).
 
+To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -25,14 +27,29 @@ url
 
 OPTIONAL PARAMETERS
 -------------------
+destination
+   Downloaded file's destination in target. If unset, ``$__object_id`` is used.
+
 sum
-   Checksum is used to decide if existing destination file must be redownloaded.
+   Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256.
-   By default output of ``cksum`` without filename is expected.
+
-   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
+   Type tries to detect hash format with regexes, but prefixes
+   ``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported.
+
+   Checksum have two purposes - state check and post-download verification.
+   In state check, if destination checksum mismatches, then content of URL
+   will be downloaded to temporary file. If downloaded temporary file's
+   checksum matches, then it will be moved to destination (overwritten).
+
+   For local downloads it is expected that usable utilities for checksum
+   calculation exist in the system.
 
 download
-   If ``local`` (default), then download file to local storage and copy
+   If ``local`` (default), then file is downloaded to local storage and copied
-   it to target host. If ``remote``, then download happens in target.
+   to target host. If ``remote``, then download happens in target.
+
+   For local downloads it is expected that usable utilities for downloading
+   exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``.
 
 cmd-get
    Command used for downloading.
@@ -62,7 +79,7 @@ EXAMPLES
     require='__directory/opt/cpma' \
         __download /opt/cpma/cnq3.zip \
             --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
-            --sum md5:46da3021ca9eace277115ec9106c5b46
+            --sum 46da3021ca9eace277115ec9106c5b46
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \

cdist/conf/type/__download/manifest

@@ -1,6 +1,6 @@
 #!/bin/sh -e
 
-if grep -Eq '^wget' "$__object/explorer/remote_cmd"
+if grep -Eq '^wget' "$__object/explorer/remote_cmd_get"
 then
     __package wget
 fi

cdist/conf/type/__download/parameter/optional

@@ -1,5 +1,6 @@
-sum
 cmd-get
 cmd-sum
+destination
 download
 onchange
+sum

cdist/conf/type/__filesystem/explorer/lsblk

@@ -27,7 +27,7 @@ else
 fi
 
 case "$os" in
-    alpine|centos|fedora|redhat|suse|gentoo)
+    alpine|centos|fedora|gentoo|redhat|suse|ubuntu)
         if [ ! -x "$(command -v lsblk)" ]; then
             echo "lsblk is required for __filesystem type" >&2
             exit 1

cdist/conf/type/__grafana_dashboard/manifest

@@ -15,7 +15,7 @@ case $os in
                 # Differntation not needed anymore
                 apt_source_distribution=stable
                 ;;
-            10*)
+            10*|11*)
                 # Differntation not needed anymore
                 apt_source_distribution=stable
                 ;;

cdist/conf/type/__letsencrypt_cert/manifest

@@ -41,7 +41,7 @@ if [ -z "${certbot_fullpath}" ]; then
 					require="__apt_source/stretch-backports" __package_apt certbot \
 						--target-release stretch-backports
 				;;
-				10*)
+				10*|11*)
 					__package_apt certbot
 				;;
 

cdist/conf/type/__package_apt/gencode-remote

@@ -81,12 +81,24 @@ aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::=
 
 case "$state_should" in
     present)
+        # There are special arguments to apt(8) to prevent aborts if apt woudn't been
+        # updated after the 19th April 2021 till the bullseye release. The additional
+        # arguments acknoledge the happend suite change (the apt(8) update does the
+        # same by itself).
+        #
+        # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+        # allows backward compatablility to pre-buster Debian versions.
+        #
+        # See more: ticket #861
+        # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+        apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
         # following is bit ugly, but important hack.
         # due to how cdist config run works, there isn't
         # currently better way to do it :(
         cat << EOF
 if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ]
-then echo apt-get update > /dev/null 2>&1 || true
+then echo apt-get $apt_opts update > /dev/null 2>&1 || true
 fi
 EOF
         if [ -n "$version" ]; then

cdist/conf/type/__package_pkg_freebsd/gencode-remote

@@ -37,6 +37,7 @@ assert ()                 #  If condition false,
 	then
 		echo "Assertion failed:  \"$1\""
 		# shellcheck disable=SC2039
+		# shellcheck disable=SC3044
 		echo "File \"$0\", line $lineno, called by $(caller 0)"
 		exit $E_ASSERT_FAILED
 	fi

cdist/conf/type/__package_update_index/gencode-remote

@@ -41,7 +41,19 @@ fi
 case "$type" in
     yum) ;;
     apt)
-        echo "apt-get --quiet update"
+        # There are special arguments to apt(8) to prevent aborts if apt woudn't been
+        # updated after the 19th April 2021 till the bullseye release. The additional
+        # arguments acknoledge the happend suite change (the apt(8) update does the
+        # same by itself).
+        #
+        # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+        # allows backward compatablility to pre-buster Debian versions.
+        #
+        # See more: ticket #861
+        # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+        apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
+        echo "apt-get --quiet $apt_opts update"
         echo "apt-cache updated (age was: $currage)" >> "$__messages_out"
         ;;
     pacman)

cdist/conf/type/__rsync/gencode-local

@@ -1,39 +1,104 @@
 #!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
-source=$(cat "$__object/parameter/source")
+if ! command -v rsync > /dev/null
-remote_user=$(cat "$__object/parameter/remote-user")
+then
+    echo 'rsync is missing in local machine' >&2
+    exit 1
+fi
 
-if [ -f "$__object/parameter/destination" ]; then
+src="$( cat "$__object/parameter/source" )"
-    destination=$(cat "$__object/parameter/destination")
+
+if [ ! -e "$src" ]
+then
+    echo "$src not found" >&2
+    exit 1
+fi
+
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
 else
-    destination="/$__object_id"
+    dst="/$__object_id"
 fi
 
-set --
+# if source is directory, then make sure that
-if [ -f "$__object/parameter/rsync-opts" ]; then
+# source and destination are ending with slash,
-    while read -r opts; do
+# because this is what you almost always want when
-        set -- "$@" "--$opts"
+# rsyncing two directories.
-    done < "$__object/parameter/rsync-opts"
+
+if [ -d "$src" ]
+then
+    if ! echo "$src" | grep -Eq '/$'
+    then
+        src="$src/"
+    fi
+
+    if ! echo "$dst" | grep -Eq '/$'
+    then
+        dst="$dst/"
+    fi
 fi
 
-echo rsync -a \
+remote_user="$( cat "$__object/parameter/remote-user" )"
-    --no-owner --no-group \
+
-    -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"
+options="$( cat "$__object/parameter/options" )"
+
+if [ -f "$__object/parameter/option" ]
+then
+    while read -r l
+    do
+        # there's a limitation in argparse: value can't begin with '-'.
+        # to workaround this, let's prefix opts with '\' in manifest and remove here.
+        # read more about argparse issue: https://bugs.python.org/issue9334
+
+        options="$options $( echo "$l" | sed 's/\\//g' )"
+    done \
+        < "$__object/parameter/option"
+fi
+
+if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ]
+then
+    options="$options --chown="
+
+    if [ -f "$__object/parameter/owner" ]
+    then
+        owner="$( cat "$__object/parameter/owner" )"
+        options="$options$owner"
+    fi
+
+    if [ -f "$__object/parameter/group" ]
+    then
+        group="$( cat "$__object/parameter/group" )"
+        options="$options:$group"
+    fi
+fi
+
+if [ -f "$__object/parameter/mode" ]
+then
+    mode="$( cat "$__object/parameter/mode" )"
+    options="$options --chmod=$mode"
+fi
+
+# IMPORTANT
+#
+# 1. we first dry-run rsync with change summary to find out
+#    if there are any changes and code generation is needed.
+# 2. normally, to get current state or target host, we run
+#    such operations in type explorers, but that's not
+#    possible due to how rsync works.
+# 3. redirecting output of dry-run to stderr to ease debugging.
+# 4. to understand how that cryptic regex works, please
+#    open rsync manpage and read about --itemize-changes.
+
+export RSYNC_RSH="$__remote_exec"
+
+# shellcheck disable=SC2086
+if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \
+    | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2
+then
+    exit 0
+fi
+
+echo "export RSYNC_RSH='$__remote_exec'"
+
+echo "rsync $options $src $remote_user@$__target_host:$dst"

cdist/conf/type/__rsync/man.rst

@@ -3,112 +3,73 @@ cdist-type__rsync(7)
 
 NAME
 ----
-cdist-type__rsync - Mirror directories using rsync
+cdist-type__rsync - Mirror directories using ``rsync``
 
 
 DESCRIPTION
 -----------
-WARNING: This type is of BETA quality:
+The purpose of this type is to bring power of ``rsync`` into ``cdist``.
-
-- it has not been tested widely
-- interfaces *may* change
-- if there is a better approach to solve the problem -> the type may even vanish
-
-If you are fine with these constraints, please read on.
-
-
-This cdist type allows you to mirror local directories to the
-target host using rsync. Rsync will be installed in the manifest of the type.
-If group or owner are giveng, a recursive chown will be executed on the 
-target host.
-
-A slash will be appended to the source directory so that only the contents
-of the directory are taken and not the directory name itself.
 
 
 REQUIRED PARAMETERS
 -------------------
 source
-    Where to take files from
+   Source directory in local machine.
+   If source is directory, slash (``/``) will be added to source and destination paths.
 
 
 OPTIONAL PARAMETERS
 -------------------
-group
+destination
-   Group to chgrp to.
+   Destination directory. Defaults to ``$__object_id``.
 
 owner
-   User to chown to.
+   Will be passed to ``rsync`` as ``--chown=OWNER``.
+   Read ``rsync(1)`` for more details.
 
-destination
+group
-    Use this as the base destination instead of the object id
+   Will be passed to ``rsync`` as ``--chown=:GROUP``.
+   Read ``rsync(1)`` for more details.
+
+mode
+   Will be passed to ``rsync`` as ``--chmod=MODE``.
+   Read ``rsync(1)`` for more details.
+
+options
+   Defaults to ``--recursive --links --perms --times``.
+   Due to `bug in Python's argparse<https://bugs.python.org/issue9334>`_, value must be prefixed with ``\``.
 
 remote-user
-    Use this user instead of the default "root" for rsync operations.
+   Defaults to ``root``.
 
 
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
-rsync-opts
+option
-    Use this option to give rsync options with.
+   Pass additional options to ``rsync``.
-    See rsync(1) for available options.
+   See ``rsync(1)`` for all possible options.
-    Only "--" options are supported.
+   Due to `bug in Python's argparse<https://bugs.python.org/issue9334>`_, value must be prefixed with ``\``.
-    Write the options without the beginning "--"
-    Can be specified multiple times.
-
-
-MESSAGES
---------
-NONE
 
 
 EXAMPLES
 --------
-
 .. code-block:: sh
 
-    # You can use any source directory
+    __rsync /var/www/example.com \
-    __rsync /tmp/testdir \
+        --owner root \
-        --source /etc
+        --group www-data \
-
+        --mode 'D750,F640' \
-    # Use source from type
+        --source "$__files/example.com/www"
-    __rsync /etc \
-        --source "$__type/files/package"
-
-    # Allow multiple __rsync objects to write to the same dir
-    __rsync mystuff \
-        --destination /usr/local/bin \
-        --source "$__type/files/package"
-
-    __rsync otherstuff \
-        --destination /usr/local/bin \
-        --source "$__type/files/package2"
-
-    # Use rsync option --exclude
-    __rsync /tmp/testdir \
-        --source /etc \
-        --rsync-opts exclude=sshd_conf
-
-    # Use rsync with multiple options --exclude --dry-run
-    __rsync /tmp/testing \
-        --source /home/tester \
-        --rsync-opts exclude=id_rsa \
-        --rsync-opts dry-run
-
-
-SEE ALSO
---------
-:strong:`rsync`\ (1)
 
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+Ander Punnar <ander-at-kvlt-dot-ee>
 
 
 COPYING
 -------
-Copyright \(C) 2015 Nico Schottelius. You can redistribute it
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
-and/or modify it under the terms of the GNU General Public License as
+under the terms of the GNU General Public License as published by the Free
-published by the Free Software Foundation, either version 3 of the
+Software Foundation, either version 3 of the License, or (at your option)
-License, or (at your option) any later version.
+any later version.

cdist/conf/type/__rsync/manifest

@@ -1,21 +1,3 @@
 #!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
 __package rsync

cdist/conf/type/__rsync/parameter/default/options

@@ -0,0 +1 @@
+--recursive --links --perms --times

cdist/conf/type/__rsync/parameter/optional

@@ -1,4 +1,6 @@
 destination
-owner
 group
+mode
+options
+owner
 remote-user

cdist/conf/type/__rsync/parameter/optional_multiple

@@ -1 +1 @@
-rsync-opts
+option

cdist/conf/type/__sed/explorer/file

@@ -0,0 +1,16 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/file" ]
+then
+    file="$( cat "$__object/parameter/file" )"
+else
+    file="/$__object_id"
+fi
+
+if [ ! -e "$file" ]
+then
+    echo "$file does not exist" >&2
+    exit 1
+fi
+
+cat "$file"

cdist/conf/type/__sed/gencode-remote

@@ -0,0 +1,58 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/file" ]
+then
+    file="$( cat "$__object/parameter/file" )"
+else
+    file="/$__object_id"
+fi
+
+script="$( cat "$__object/parameter/script" )"
+
+if [ "$script" = '-' ]
+then
+    script="$( cat "$__object/stdin" )"
+fi
+
+# since stdin is not available in explorer, we pull file from target with explorer
+
+file_from_target="$__object/explorer/file"
+
+sed_cmd='sed'
+
+if [ -f "$__object/parameter/regexp-extended" ]
+then
+    sed_cmd="$sed_cmd -E"
+fi
+
+# do sed dry run, diff result and if no change, then there's nothing to do
+# also redirect diff's output to stderr for debugging purposes
+
+if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2
+then
+    exit 0
+fi
+
+# we can't use -i, because it's not posix, so we fly with tempfile and cp
+# and we use cp because we want to preserve destination file's attributes
+
+# shellcheck disable=SC2016
+echo 'tmp="$__object/tempfile"'
+
+echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
+
+echo "$script"
+
+echo 'EOF'
+
+echo "cp \"\$tmp\" '$file'"
+
+# shellcheck disable=SC2016
+echo 'rm -f "$tmp"'
+
+echo 'change' >> "$__messages_out"
+
+if [ -f "$__object/parameter/onchange" ]
+then
+    cat "$__object/parameter/onchange"
+fi

cdist/conf/type/__sed/man.rst

@@ -0,0 +1,57 @@
+cdist-type__sed(7)
+==================
+
+NAME
+----
+cdist-type__sed - Transform text files with ``sed``
+
+
+DESCRIPTION
+-----------
+Transform text files with ``sed``.
+
+
+REQUIRED MULTIPLE PARAMETERS
+----------------------------
+script
+   ``sed`` script.
+   If ``-`` then the script is read from ``stdin``.
+
+
+OPTIONAL PARAMETERS
+-------------------
+file
+   Path to the file. Defaults to ``$__object_id``.
+
+onchange
+   Execute this command if ``sed`` changes file.
+
+
+BOOLEAN PARAMETERS
+------------------
+regexp-extended
+   Use extended regular expressions in the script.
+   Might not be supported with every ``sed`` version.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+   __sed /tmp/foobar --script 's/foo/bar/'
+
+   echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script -
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.

cdist/conf/type/__sed/parameter/boolean

@@ -0,0 +1 @@
+regexp-extended

cdist/conf/type/__sed/parameter/optional

@@ -0,0 +1,2 @@
+file
+onchange

cdist/conf/type/__sed/parameter/required_multiple

@@ -0,0 +1 @@
+script

cdist/conf/type/__snakeoil_cert/man.rst

@@ -38,6 +38,7 @@ cert-path
 EXAMPLES
 --------
 .. code-block:: sh
+
 	__snakeoil_cert localhost-rsa \
 	    --common-name localhost \
 	    --key-type rsa:4096

cdist/conf/type/__ssh_authorized_keys/explorer/keys

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 
 # shellcheck disable=SC1090
+# shellcheck disable=SC1091
 file="$( . "$__type_explorer/file" )"
 
 if [ -f "$file" ]

cdist/conf/type/__update_alternatives/explorer/alternatives

@@ -1,4 +1,4 @@
 #!/bin/sh -e
 
-update-alternatives --display "$__object_id" 2>/dev/null \
+LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \
-    | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
+| awk -F ' - ' '/priority [0-9]+$/ { print $1 }'

cdist/conf/type/__update_alternatives/explorer/link

@@ -18,12 +18,12 @@ for altdir in \
     /var/lib/dpkg/alternatives \
     /var/lib/alternatives
 do
-    if [ ! -f "$altdir/$__object_id" ]
+    if [ ! -f "$altdir/${__object_id:?}" ]
     then
         continue
     fi
 
-    link="$( awk 'NR==2' "$altdir/$__object_id" )"
+    link="$( awk 'NR==2' "$altdir/${__object_id:?}" )"
 
     if [ -n "$link" ]
     then
@@ -31,9 +31,12 @@ do
     fi
 done
 
-if [ -z "$link" ]
+if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then
-    echo "unable to get link for $__object_id" >&2
+    # NOTE: ignore error for dry-runs because a package providing the link
+    #       might be managed by another cdist object (which wasn't executed,
+    #       because dry run…).
+    echo "unable to get link for ${__object_id:?}" >&2
     exit 1
 fi
 

cdist/conf/type/__update_alternatives/explorer/path_is

@@ -1,11 +1,15 @@
 #!/bin/sh -e
 
-path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \
+path_is=$(
-    | awk '/link currently points to/ {print $5}' )"
+    LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \
+    | awk '/link currently points to/ { print $5 }')
 
-if [ -z "$path_is" ]
+if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then
-    echo "unable to get current path for $__object_id" >&2
+    # NOTE: ignore error for dry-runs because a package providing the
+    #       alternative might be managed by another cdist object (which
+    #       wasn't executed, because dry run…).
+    echo "unable to get current path for ${__object_id:?}" >&2
     exit 1
 fi
 

cdist/conf/type/__update_alternatives/explorer/path_should_state

@@ -1,6 +1,6 @@
 #!/bin/sh -e
 
-if [ -f "$( cat "$__object/parameter/path" )" ]
+if [ -f "$( cat "${__object:?}/parameter/path" )" ]
 then
     echo 'present'
 else

cdist/conf/type/__update_alternatives/gencode-remote

@@ -18,37 +18,39 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 
-path_is="$( cat "$__object/explorer/path_is" )"
+path_is="$( cat "${__object:?}/explorer/path_is" )"
 
-path_should="$( cat "$__object/parameter/path" )"
+path_should="$( cat "${__object:?}/parameter/path" )"
 
 if [ "$path_is" = "$path_should" ]
 then
     exit 0
 fi
 
-if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ]
+if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \
+    && [ -z "${__cdist_dry_run+dry run}" ]
 then
     echo "$path_should does not exist in target" >&2
     exit 1
 fi
 
-name="$__object_id"
+name=${__object_id:?}
 
-alternatives="$( cat "$__object/explorer/alternatives" )"
+if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives"
-
-if ! echo "$alternatives" | grep -Fxq "$path_should"
 then
-    if [ ! -f "$__object/parameter/install" ]
+    if [ -f "${__object:?}/parameter/install" ]
     then
+        link="$( cat "${__object:?}/explorer/link" )"
+        echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
+    elif [ -z "${__cdist_dry_run+dry run}" ]
+    then
+        # NOTE: ignore error for dry-runs because a package providing the link
+        #       to be installed might be managed by another cdist object (which
+        #       wasn't executed, because dry run…).
         echo "$path_should is not in $name alternatives." >&2
         echo 'Please install missing packages or use --install to add path to alternatives.' >&2
         exit 1
     fi
-
-    link="$( cat "$__object/explorer/link" )"
-
-    echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
 fi
 
 echo "update-alternatives --set '$name' '$path_should'"

cdist/log.py

@@ -36,25 +36,27 @@ import threading
 logging.OFF = logging.CRITICAL + 10  # disable logging
 logging.addLevelName(logging.OFF, 'OFF')
 
+
 logging.VERBOSE = logging.INFO - 5
 logging.addLevelName(logging.VERBOSE, 'VERBOSE')
 
 
-def _verbose(msg, *args, **kwargs):
+def _verbose(self, msg, *args, **kwargs):
-    logging.log(logging.VERBOSE, msg, *args, **kwargs)
+    self.log(logging.VERBOSE, msg, args, **kwargs)
 
 
-logging.verbose = _verbose
+logging.Logger.verbose = _verbose
+
 
 logging.TRACE = logging.DEBUG - 5
 logging.addLevelName(logging.TRACE, 'TRACE')
 
 
-def _trace(msg, *args, **kwargs):
+def _trace(self, msg, *args, **kwargs):
-    logging.log(logging.TRACE, msg, *args, **kwargs)
+    self.log(logging.TRACE, msg, *args, **kwargs)
 
 
-logging.trace = _trace
+logging.Logger.trace = _trace
 
 
 class CdistFormatter(logging.Formatter):

cdist/scan/commandline.py

@@ -20,36 +20,98 @@
 #
 
 import logging
+import sys
+from datetime import datetime
 
 log = logging.getLogger("scan")
 
 
-# define this outside of the class to not handle scapy import errors by default
+def run(scan, args):
-def commandline(args):
+    # We run each component in a separate process since they
-    log.debug(args)
+    # must not block on each other.
-
-    try:
-        import cdist.scan.scan as scan
-    except ModuleNotFoundError:
-        print('cdist scan requires scapy to be installed')
-
     processes = []
 
-    if not args.mode:
-        # By default scan and trigger, but do not call any action
-        args.mode = ['scan', 'trigger', ]
-
     if 'trigger' in args.mode:
-        t = scan.Trigger(interfaces=args.interfaces)
+        t = scan.Trigger(interfaces=args.interface,
+                         sleeptime=args.trigger_delay)
         t.start()
         processes.append(t)
         log.debug("Trigger started")
 
     if 'scan' in args.mode:
-        s = scan.Scanner(interfaces=args.interfaces, args=args)
+        s = scan.Scanner(
+                autoconfigure='config' in args.mode,
+                interfaces=args.interface,
+                name_mapper=args.name_mapper)
         s.start()
         processes.append(s)
         log.debug("Scanner started")
 
     for process in processes:
         process.join()
+
+
+def list(scan, args):
+    s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper)
+    hosts = s.list()
+
+    # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
+    # 6 colons.
+    ipv6_max_size = 8 * 4 + 10
+    date_max_size = len(datetime.now().strftime(scan.datetime_format))
+    name_max_size = 25
+
+    print("{} | {} | {} | {}".format(
+        'name'.ljust(name_max_size),
+        'address'.ljust(ipv6_max_size),
+        'last seen'.ljust(date_max_size),
+        'last configured'.ljust(date_max_size)))
+    print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
+    for host in hosts:
+        last_seen = host.last_seen()
+        if last_seen:
+            last_seen = last_seen.strftime(scan.datetime_format)
+        else:
+            last_seen = '-'
+
+        last_configured = host.last_configured()
+        if last_configured is not None:
+            last_configured = last_configured.strftime(scan.datetime_format)
+        else:
+            last_configured = '-'
+
+        print("{} | {} | {} | {}".format(
+            host.name(default='-').ljust(name_max_size),
+            host.address().ljust(ipv6_max_size),
+            last_seen.ljust(date_max_size),
+            last_configured.ljust(date_max_size)))
+
+
+# CLI processing is defined outside of the main scan class to handle
+# non-available optional scapy dependency (instead of crashing mid-flight).
+def commandline(args):
+    log.debug(args)
+
+    # Check if we have the optional scapy dependency available.
+    try:
+        import cdist.scan.scan as scan
+    except ModuleNotFoundError:
+        log.error('cdist scan requires scapy to be installed. Exiting.')
+        sys.exit(1)
+
+    # Set default operation mode.
+    if not args.mode:
+        # By default scan and trigger, but do not call any action.
+        args.mode = ['scan', 'trigger', ]
+
+    if 'config' in args.mode and args.name_mapper is None:
+        print('--name-mapper must be specified for scanner config mode.',
+              file=sys.stderr)
+        sys.exit(1)
+
+    # Print known hosts and exit is --list is specified - do not start
+    # the scanner.
+    if args.list:
+        list(scan, args)
+    else:
+        run(scan, args)

cdist/scan/scan.py

@@ -19,38 +19,6 @@
 #
 #
 
-#
-# Interface to be implemented:
-# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated
-#   scan: scan / listen for icmp6 replies
-#   trigger: send trigger to multicast
-#   config: configure newly detected hosts
-#   install: install newly detected hosts
-#
-# Scanner logic
-#  - save results to configdir:
-#     basedir = ~/.cdist/scan/<ipv6-address>
-#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time
-#           or similar
-#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record
-#           unix time or similar
-#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record
-#           unix time or similar
-#
-#
-#
-#
-# cdist scan --list
-#       Show all known hosts including last seen flag
-#
-# Logic for reconfiguration:
-#
-#  - record when configured last time
-#  - introduce a parameter --reconfigure-after that takes time argument
-#  - reconfigure if a) host alive and b) reconfigure-after time passed
-#
-
-
 from multiprocessing import Process
 import os
 import logging
@@ -61,7 +29,84 @@ import datetime
 
 import cdist.config
 
+logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("scan")
+datetime_format = '%Y-%m-%d %H:%M:%S'
+
+
+class Host(object):
+    def __init__(self, addr, outdir, name_mapper=None):
+        self.addr = addr
+        self.workdir = os.path.join(outdir, addr)
+        self.name_mapper = name_mapper
+
+        os.makedirs(self.workdir, exist_ok=True)
+
+    def __get(self, key, default=None):
+        fname = os.path.join(self.workdir, key)
+        value = default
+        if os.path.isfile(fname):
+            with open(fname, "r") as fd:
+                value = fd.readline()
+        return value
+
+    def __set(self, key, value):
+        fname = os.path.join(self.workdir, key)
+        with open(fname, "w") as fd:
+            fd.write(f"{value}")
+
+    def name(self, default=None):
+        if self.name_mapper is None:
+            return default
+
+        fpath = os.path.join(os.getcwd(), self.name_mapper)
+        if os.path.isfile(fpath) and os.access(fpath, os.X_OK):
+            out = subprocess.run([fpath, self.addr], capture_output=True)
+            if out.returncode != 0:
+                return default
+            else:
+                value = out.stdout.decode()
+                return (default if len(value) == 0 else value)
+        else:
+            return default
+
+    def address(self):
+        return self.addr
+
+    def last_seen(self, default=None):
+        raw = self.__get('last_seen')
+        if raw:
+            return datetime.datetime.strptime(raw, datetime_format)
+        else:
+            return default
+
+    def last_configured(self, default=None):
+        raw = self.__get('last_configured')
+        if raw:
+            return datetime.datetime.strptime(raw, datetime_format)
+        else:
+            return default
+
+    def seen(self):
+        now = datetime.datetime.now().strftime(datetime_format)
+        self.__set('last_seen', now)
+
+    # XXX: There's no easy way to use the config module without feeding it with
+    # CLI args. Might as well call everything from scratch!
+    def configure(self):
+        target = self.name() or self.address()
+        cmd = ['cdist', 'config', '-v', target]
+
+        fname = os.path.join(self.workdir, 'last_configuration_log')
+        with open(fname, "w") as fd:
+            log.debug("Executing: %s", cmd)
+            completed_process = subprocess.run(cmd, stdout=fd, stderr=fd)
+            if completed_process.returncode != 0:
+                log.error("%s return with non-zero code %i - see %s for \
+                        details.", cmd, completed_process.returncode, fname)
+
+        now = datetime.datetime.now().strftime(datetime_format)
+        self.__set('last_configured', now)
 
 
 class Trigger(object):
@@ -69,12 +114,14 @@ class Trigger(object):
     Trigger an ICMPv6EchoReply from all hosts that are alive
     """
 
-    def __init__(self, interfaces=None, verbose=False):
+    def __init__(self, interfaces, sleeptime, verbose=False):
         self.interfaces = interfaces
+
+        # Used by scapy / send in trigger/2.
         self.verbose = verbose
 
-        # Wait 5 seconds before triggering again - FIXME: add parameter
+        # Delay in seconds between sent ICMPv6EchoRequests.
-        self.sleeptime = 5
+        self.sleeptime = sleeptime
 
     def start(self):
         self.processes = []
@@ -93,9 +140,14 @@ class Trigger(object):
             time.sleep(self.sleeptime)
 
     def trigger(self, interface):
-        packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest()
+        try:
-        log.debug("Sending request on %s", interface)
+            log.debug("Sending ICMPv6EchoRequest on %s", interface)
-        send(packet, verbose=self.verbose)
+            packet = IPv6(
+                    dst="ff02::1%{}".format(interface)
+                    ) / ICMPv6EchoRequest()
+            send(packet, verbose=self.verbose)
+        except Exception as e:
+            log.error("Could not send ICMPv6EchoRequest: %s", e)
 
 
 class Scanner(object):
@@ -103,41 +155,62 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces=None, args=None, outdir=None):
+    def __init__(self, interfaces, autoconfigure=False, outdir=None,
+                 name_mapper=None):
         self.interfaces = interfaces
+        self.autoconfigure = autoconfigure
+        self.name_mapper = name_mapper
+        self.config_delay = datetime.timedelta(seconds=3600)
 
         if outdir:
             self.outdir = outdir
         else:
             self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan')
+        os.makedirs(self.outdir, exist_ok=True)
+
+        self.running_configs = {}
 
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
-            host = pkg['IPv6'].src
+            host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper)
-            log.verbose("Host %s is alive", host)
+            if host.name():
+                log.verbose("Host %s (%s) is alive", host.name(),
+                            host.address())
+            else:
+                log.verbose("Host %s is alive", host.address())
 
-            dir = os.path.join(self.outdir, host)
+            host.seen()
-            fname = os.path.join(dir, "last_seen")
 
-            now = datetime.datetime.now()
+            # Configure if needed.
+            if self.autoconfigure and \
+                    host.last_configured(default=datetime.datetime.min) + \
+                    self.config_delay < datetime.datetime.now():
+                self.config(host)
 
-            os.makedirs(dir, exist_ok=True)
+    def list(self):
+        hosts = []
+        for addr in os.listdir(self.outdir):
+            hosts.append(Host(addr, self.outdir, self.name_mapper))
 
-            # FIXME: maybe adjust the format so we can easily parse again
+        return hosts
-            with open(fname, "w") as fd:
-                fd.write(f"{now}\n")
 
-    def config(self):
+    def config(self, host):
-        """
+        if host.name() is None:
-        Configure a host
+            log.debug("config - could not resolve name for %s, aborting.",
+                      host.address())
+            return
 
-        - Assume we are only called if necessary
+        previous_config_process = self.running_configs.get(host.name())
-        - However we need to ensure to not run in parallel
+        if previous_config_process is not None and \
-        - Maybe keep dict storing per host processes
+                previous_config_process.is_alive():
-        - Save the result
+            log.debug("config - is already running for %s, aborting.",
-        - Save the output -> probably aligned to config mode
+                      host.name())
 
-        """
+        log.info("config - running against host %s (%s).", host.name(),
+                 host.address())
+        p = Process(target=host.configure())
+        p.start()
+        self.running_configs[host.name()] = p
 
     def start(self):
         self.process = Process(target=self.scan)
@@ -148,47 +221,9 @@ class Scanner(object):
 
     def scan(self):
         log.debug("Scanning - zzzzz")
-        sniff(iface=self.interfaces,
+        try:
-              filter="icmp6",
+            sniff(iface=self.interfaces,
-              prn=self.handle_pkg)
+                  filter="icmp6",
-
+                  prn=self.handle_pkg)
-
+        except Exception as e:
-if __name__ == '__main__':
+            log.error("Could not start listener: %s", e)
-    t = Trigger(interfaces=["wlan0"])
-    t.start()
-
-    # Scanner can listen on many interfaces at the same time
-    s = Scanner(interfaces=["wlan0"])
-    s.scan()
-
-    # Join back the trigger processes
-    t.join()
-
-    # Test in my lan shows:
-    # [18:48] bridge:cdist% ls -1d fe80::*
-    # fe80::142d:f0a5:725b:1103
-    # fe80::20d:b9ff:fe49:ac11
-    # fe80::20d:b9ff:fe4c:547d
-    # fe80::219:d2ff:feb2:2e12
-    # fe80::21b:fcff:feee:f446
-    # fe80::21b:fcff:feee:f45c
-    # fe80::21b:fcff:feee:f4b1
-    # fe80::21b:fcff:feee:f4ba
-    # fe80::21b:fcff:feee:f4bc
-    # fe80::21b:fcff:feee:f4c1
-    # fe80::21d:72ff:fe86:46b
-    # fe80::42b0:34ff:fe6f:f6f0
-    # fe80::42b0:34ff:fe6f:f863
-    # fe80::42b0:34ff:fe6f:f9b2
-    # fe80::4a5d:60ff:fea1:e55f
-    # fe80::77a3:5e3f:82cc:f2e5
-    # fe80::9e93:4eff:fe6c:c1f4
-    # fe80::ba69:f4ff:fec5:6041
-    # fe80::ba69:f4ff:fec5:8db7
-    # fe80::bad8:12ff:fe65:313d
-    # fe80::bad8:12ff:fe65:d9b1
-    # fe80::ce2d:e0ff:fed4:2611
-    # fe80::ce32:e5ff:fe79:7ea7
-    # fe80::d66d:6dff:fe33:e00
-    # fe80::e2ff:f7ff:fe00:20e6
-    # fe80::f29f:c2ff:fe7c:275e

docs/changelog

@@ -2,6 +2,29 @@ Changelog
 ---------
 
 next:
+	* Explorer machine_type: Rewrite (Dennis Camera)
+	* New type: __sed (Ander Punnar)
+	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
+	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
+	* Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham)
+
+6.9.8: 2021-08-24
+	* Type __rsync: Rewrite (Ander Punnar)
+	* New type: __apt_pin (Daniel Fancsali)
+	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
+	* Core: Fix logging bug (Dennis Camera)
+	* Build: Improve Makefile compatibility (Evilham)
+	* Type __filesystem: Support ubuntu (Joachim Desroches)
+	* Explorer os_version: Fall back to os-release/lsb-release file on Ubuntu (Dennis Camera)
+	* Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera)
+	* Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera)
+	* Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera)
+	* Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar)
+
+6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)
@@ -9,6 +32,11 @@ next:
 	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham)
 	* Type __git: Fix group explorer (Ander Punnar)
 	* Type __pyvenv: Fix group explorer (Dennis Camera)
+	* Type __download: Improve checksum verification, add optional --destination (Ander Punnar)
+	* Type __debconf_set_selections: Add state explorer (Dennis Camera)
+	* Core: Implement usable cdist scan (Timothée Floure)
+	* New type: __snakeoil_cert (Ander Punnar)
+	* Type __rsync: Honour $__remote_exec env var (Daniel Fancsali)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
@@ -137,7 +165,7 @@ next:
 	* Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham)
 	* Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham)
 	* Configuration: Add notes to cdist.cfg.skeleton (Evil Ham)
-	* Explorers cpu_cores, memory: Improve *BSD support (Evil Ham)
+	* Explorers cpu_cores, memory: Improve BSD support (Evil Ham)
 	* Core: Remove debug logging noise (Evil Ham)
 
 6.5.4: 2020-04-11
@@ -202,7 +230,7 @@ next:
 	* Documentation: PreOS english nitpicking (Evil Ham)
 	* Documentation: Add installing from source with signature verification (Darko Poljak)
 	* Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak)
-	* Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham)
+	* Type __start_on_boot: Docs: remove unsupported BSD claim (Evil Ham)
 	* New type: __openldap_server (Evil Ham)
 
 6.2.0: 2019-11-30
@@ -1061,9 +1089,9 @@ next:
 	* Removed type __removeline (replaced by __line) (Nico Schottelius)
 	* Type __directory: Parameter --parents and --recursive are now boolean (Nico Schottelius)
 	* Type __package_apt, __package_luarocks, __package_opkg,
-		__package_pacman, __package_pkg_freebsd, __package_pkg_openbsd,
+	  __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd,
-		__package_rubygem, __package_yum, __process:
+	  __package_rubygem, __package_yum, __process:
-			Parameter state accepts only "present" and "absent" (Nico Schottelius)
+	  Parameter state accepts only "present" and "absent" (Nico Schottelius)
 	* Dist: Initial support for pypi packaging (Nico Schottelius)
 
 2.0.15: 2012-11-02

docs/src/cdist-install.rst

@@ -12,7 +12,7 @@ This is the machine from which you will configure target hosts.
  * /bin/sh: A POSIX like shell (for instance bash, dash, zsh)
  * Python >= 3.5
  * SSH client
- * sphinx (for building html docs and/or the man pages)
+ * sphinx with the rtd theme (for building html docs and/or the man pages)
 
 Target Hosts
 ~~~~~~~~~~~~

docs/src/cdist-scan.rst

@@ -0,0 +1,84 @@
+Scan
+=====
+
+Description
+-----------
+Runs cdist as a daemon that discover/watch on hosts and reconfigure them
+periodically. It is especially useful in netboot-based environment where hosts
+boot unconfigured, and to ensure your infrastructure stays in sync with your
+configuration.
+
+This feature is still consider to be in **beta** stage, and only operate on
+IPv6 (including link-local).
+
+Usage (Examples)
+----------------
+
+Discover hosts on local network and configure those whose name is resolved by
+the name mapper script.
+
+.. code-block:: sh
+
+    $ cdist scan --beta --interface eth0 \
+      --mode scan --name-mapper path/to/script \
+      --mode trigger --mode config
+
+List known hosts and exit.
+
+.. code-block:: sh
+
+    $ cdist scan --beta --list --name-mapper path/to/script
+
+Please refer to `cdist(1)` for a detailed list of parameters.
+
+Modes
+-----
+
+The scanner has 3 modes that can be independently toggled. If the `--mode`
+parameter is not specified, only `tigger` and `scan` are enabled (= hosts are
+not configured).
+
+trigger
+  Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to
+  trigger detection by the `scan` module.
+
+scan
+  Watch for incoming ICMPv6 replies and optionally configure detected hosts.
+
+config
+  Enable configuration of hosts detected by `scan`.
+
+Name Mapper Script
+------------------
+
+The name mapper script takes an IPv6 address as first argument and writes the
+resolved name to stdout - if any. The script must be executable.
+
+Simplest script:
+
+.. code-block:: sh
+
+  #!/bin/sh
+
+  case "$1" in
+  	"fe80::20d:b9ff:fe57:3524")
+  		printf "my-host-01"
+  		;;
+  	"fe80::7603:bdff:fe05:89bb")
+  		printf "my-host-02"
+  		;;
+  esac
+
+Resolving name from `PTR` DNS record:
+
+.. code-block:: sh
+
+  #!/bin/sh
+
+  for cmd in dig sed; do
+  	if ! command -v $cmd > /dev/null; then
+  		exit 1
+  	fi
+  done
+
+  dig +short -x "$1" | sed -e 's/.$//'

docs/src/conf.py

@@ -56,7 +56,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'cdist'
-copyright = 'ungleich GmbH 2020'
+copyright = 'ungleich GmbH 2021'
 # author = 'Darko Poljak'
 
 # The version info for the project you're documenting, acts as replacement for

docs/src/index.rst

@@ -34,6 +34,7 @@ It natively supports IPv6 since the first release.
    cdist-parallelization
    cdist-inventory
    cdist-preos
+   cdist-scan
    cdist-integration
    cdist-reference
    cdist-best-practice

docs/src/man1/cdist.rst

@@ -88,6 +88,9 @@ SYNOPSIS
     cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t]
                [pattern]
 
+    cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list]
+               [-d CONFIG_DELAY] [-t TRIGGER_DELAY]
+
 
 DESCRIPTION
 -----------
@@ -641,6 +644,31 @@ Display information for cdist (global explorers, types).
 **-t, --types**
     Display info for types.
 
+SCAN
+----
+
+Runs cdist as a daemon that discover/watch on hosts and reconfigure them
+periodically.
+
+**-I INTERFACE, --interfaces INTERFACE**
+    Interface to listen on. Can be specified multiple times.
+
+**-m MODE, --mode MODE**
+    Scanner components to enable. Can be specified multiple time to enable more
+    than one component. Supported modes are: scan, trigger and config. Defaults
+    to tiggger and scan.
+
+**--name-mapper PATH_TO_SCRIPT**
+    Path to script used to resolve a remote host name from an IPv6 address.
+
+**--list**
+    List known hosts and exit.
+
+**-d CONFIG_DELAY, --config-delay CONFIG_DELAY**
+    How long (seconds) to wait before reconfiguring after last try (config mode only).
+
+**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY**
+    How long (seconds) to wait between ICMPv6 echo requests (trigger mode only).
 
 CONFIGURATION
 -------------