From decc0ad54dffbafcd7d3644c28ae4d149926e612 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 18 Sep 2020 19:33:36 +0300
Subject: [PATCH 01/63] [__package_pip] detect pip binary

---
 cdist/conf/type/__package_pip/explorer/pip   | 10 ++++++++++
 cdist/conf/type/__package_pip/explorer/state |  2 +-
 cdist/conf/type/__package_pip/gencode-remote |  7 ++++++-
 3 files changed, 17 insertions(+), 2 deletions(-)
 create mode 100755 cdist/conf/type/__package_pip/explorer/pip

diff --git a/cdist/conf/type/__package_pip/explorer/pip b/cdist/conf/type/__package_pip/explorer/pip
new file mode 100755
index 00000000..cf9fae89
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/pip
@@ -0,0 +1,10 @@
+#!/bin/sh -e
+
+for bin in pip3 pip
+do
+    if check="$( command -v "$bin" )"
+    then
+        echo "$check"
+        break
+    fi
+done
diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state
index 5be07280..3cc98ab9 100644
--- a/cdist/conf/type/__package_pip/explorer/state
+++ b/cdist/conf/type/__package_pip/explorer/state
@@ -32,7 +32,7 @@ pipparam="$__object/parameter/pip"
 if [ -f "$pipparam" ]; then
     pip=$(cat "$pipparam")
 else
-    pip="pip"
+    pip="$( "$__type_explorer/pip" )"
 fi
 
 # If there is no pip, it may get created from somebody else.
diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index dcc4fdf9..9c5eb0d1 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -38,7 +38,12 @@ pipparam="$__object/parameter/pip"
 if [ -f "$pipparam" ]; then
     pip=$(cat "$pipparam")
 else
-    pip="pip"
+    pip="$( cat "$__object/explorer/pip" )"
+    if [ -z "$pip" ]
+    then
+        echo 'pip not found in path' >&2
+        exit 1
+    fi
 fi
 
 runasparam="$__object/parameter/runas"

From 89b621511561105cfcfcf12bbf820b74ab03396d Mon Sep 17 00:00:00 2001
From: Darko Poljak <darko.poljak@ungleich.ch>
Date: Mon, 21 Sep 2020 09:04:05 +0200
Subject: [PATCH 02/63] Clarify stdin input

Resolve #836.
---
 cdist/argparse.py                | 36 ++++++--------------------
 cdist/config.py                  |  6 +++--
 cdist/inventory.py               |  4 +--
 cdist/test/inventory/__init__.py |  3 +--
 docs/src/man1/cdist.rst          | 44 ++++++++++----------------------
 5 files changed, 29 insertions(+), 64 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 77303591..1d16bb25 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -273,8 +273,7 @@ def get_parsers():
             '-f', '--file',
             help=('Read specified file for a list of additional hosts to '
                   'operate on or if \'-\' is given, read stdin (one host per '
-                  'line). If no host or host file is specified then, by '
-                  'default, read hosts from stdin.'),
+                  'line).'),
             dest='hostfile', required=False)
     parser['config_args'].add_argument(
            '-p', '--parallel', nargs='?', metavar='HOST_MAX',
@@ -326,9 +325,7 @@ def get_parsers():
     parser['add-host'].add_argument(
            '-f', '--file',
            help=('Read additional hosts to add from specified file '
-                 'or from stdin if \'-\' (each host on separate line). '
-                 'If no host or host file is specified then, by default, '
-                 'read from stdin.'),
+                 'or from stdin if \'-\' (each host on separate line). '),
            dest='hostfile', required=False)
 
     parser['add-tag'] = parser['invsub'].add_parser(
@@ -342,20 +339,12 @@ def get_parsers():
     parser['add-tag'].add_argument(
            '-f', '--file',
            help=('Read additional hosts to add tags from specified file '
-                 'or from stdin if \'-\' (each host on separate line). '
-                 'If no host or host file is specified then, by default, '
-                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                 ' are specified then tags are read from stdin and are'
-                 ' added to all hosts.'),
+                 'or from stdin if \'-\' (each host on separate line). '),
            dest='hostfile', required=False)
     parser['add-tag'].add_argument(
            '-T', '--tag-file',
            help=('Read additional tags to add from specified file '
-                 'or from stdin if \'-\' (each tag on separate line). '
-                 'If no tag or tag file is specified then, by default, '
-                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                 ' are specified then tags are read from stdin and are'
-                 ' added to all hosts.'),
+                 'or from stdin if \'-\' (each tag on separate line). '),
            dest='tagfile', required=False)
     parser['add-tag'].add_argument(
            '-t', '--taglist',
@@ -376,9 +365,7 @@ def get_parsers():
     parser['del-host'].add_argument(
             '-f', '--file',
             help=('Read additional hosts to delete from specified file '
-                  'or from stdin if \'-\' (each host on separate line). '
-                  'If no host or host file is specified then, by default, '
-                  'read from stdin.'),
+                  'or from stdin if \'-\' (each host on separate line). '),
             dest='hostfile', required=False)
 
     parser['del-tag'] = parser['invsub'].add_parser(
@@ -396,20 +383,13 @@ def get_parsers():
     parser['del-tag'].add_argument(
             '-f', '--file',
             help=('Read additional hosts to delete tags for from specified '
-                  'file or from stdin if \'-\' (each host on separate line). '
-                  'If no host or host file is specified then, by default, '
-                  'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                  ' are specified then tags are read from stdin and are'
-                  ' deleted from all hosts.'),
+                  'file or from stdin if \'-\' (each host on separate '
+                  'line). '),
             dest='hostfile', required=False)
     parser['del-tag'].add_argument(
             '-T', '--tag-file',
             help=('Read additional tags from specified file '
-                  'or from stdin if \'-\' (each tag on separate line). '
-                  'If no tag or tag file is specified then, by default, '
-                  'read from stdin. If no tags/tagfile nor'
-                  ' hosts/hostfile are specified then tags are read from'
-                  ' stdin and are added to all hosts.'),
+                  'or from stdin if \'-\' (each tag on separate line). '),
             dest='tagfile', required=False)
     parser['del-tag'].add_argument(
             '-t', '--taglist',
diff --git a/cdist/config.py b/cdist/config.py
index 30416008..e84f6f84 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -175,9 +175,11 @@ class Config:
             raise cdist.Error(("Cannot read both, manifest and host file, "
                                "from stdin"))
 
-        # if no host source is specified then read hosts from stdin
         if not (args.hostfile or args.host):
-            args.hostfile = '-'
+            if args.tag or args.all_tagged_hosts:
+                raise cdist.Error(("Target host tag(s) missing"))
+            else:
+                raise cdist.Error(("Target host(s) missing"))
 
         if args.manifest == '-':
             # read initial manifest from stdin
diff --git a/cdist/inventory.py b/cdist/inventory.py
index 6ab20fa7..0387f326 100644
--- a/cdist/inventory.py
+++ b/cdist/inventory.py
@@ -299,7 +299,7 @@ class InventoryHost(Inventory):
         self.all = all
 
         if not self.hosts and not self.hostfile:
-            self.hostfile = "-"
+            raise cdist.Error("Host(s) missing")
 
     def _new_hostpath(self, hostpath):
         # create empty file
@@ -355,7 +355,7 @@ class InventoryTag(Inventory):
         else:
             self.allhosts = False
         if not self.tags and not self.tagfile:
-            self.tagfile = "-"
+            raise cdist.Error("Tag(s) missing")
 
         if self.hostfile == "-" and self.tagfile == "-":
             raise cdist.Error("Cannot read both, hosts and tags, from stdin")
diff --git a/cdist/test/inventory/__init__.py b/cdist/test/inventory/__init__.py
index 287c855c..a8cd8bf8 100644
--- a/cdist/test/inventory/__init__.py
+++ b/cdist/test/inventory/__init__.py
@@ -307,11 +307,10 @@ class InventoryTestCase(test.CdistTestCase):
             raise e
 
     # InventoryTag
+    @unittest.expectedFailure
     def test_inventory_tag_init(self):
         invTag = inventory.InventoryTag(db_basedir=inventory_dir,
                                         action="add")
-        self.assertTrue(invTag.allhosts)
-        self.assertEqual(invTag.tagfile, "-")
 
     def test_inventory_tag_stdin_multiple_hosts(self):
         try:
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index aa2607f8..0ecb4a61 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -177,10 +177,8 @@ Install command is currently in beta.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read specified file for a list of additional hosts to operate on
-    or if '-' is given, read stdin (one host per line).
-    If no host or host file is specified then, by default,
-    read hosts from stdin. For the file format see
-    :strong:`HOSTFILE FORMAT` below.
+    or if '-' is given, read stdin (one host per line). For the file
+    format see :strong:`HOSTFILE FORMAT` below.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -299,9 +297,8 @@ Add host(s) to inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to add from specified file or
-    from stdin if '-' (each host on separate line). If no
-    host or host file is specified then, by default, read
-    from stdin. Hostfile format is the same as config hostfile format.
+    from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -327,11 +324,8 @@ Add tag(s) to inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to add tags from specified file
-    or from stdin if '-' (each host on separate line). If
-    no host or host file is specified then, by default,
-    read from stdin. If no tags/tagfile nor hosts/hostfile
-    are specified then tags are read from stdin and are
-    added to all hosts. Hostfile format is the same as config hostfile format.
+    or from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -346,11 +340,8 @@ Add tag(s) to inventory database.
 
 **-T TAGFILE, --tag-file TAGFILE**
     Read additional tags to add from specified file or
-    from stdin if '-' (each tag on separate line). If no
-    tag or tag file is specified then, by default, read
-    from stdin. If no tags/tagfile nor hosts/hostfile are
-    specified then tags are read from stdin and are added
-    to all hosts. Tagfile format is the same as config hostfile format.
+    from stdin if '-' (each tag on separate line).
+    Tagfile format is the same as config hostfile format.
 
 **-t TAGLIST, --taglist TAGLIST**
     Tag list to be added for specified host(s), comma
@@ -372,9 +363,8 @@ Delete host(s) from inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to delete from specified file or
-    from stdin if '-' (each host on separate line). If no
-    host or host file is specified then, by default, read
-    from stdin. Hostfile format is the same as config hostfile format.
+    from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -404,11 +394,8 @@ Delete tag(s) from inventory database.
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to delete tags for from
     specified file or from stdin if '-' (each host on
-    separate line). If no host or host file is specified
-    then, by default, read from stdin. If no tags/tagfile
-    nor hosts/hostfile are specified then tags are read
-    from stdin and are deleted from all hosts. Hostfile
-    format is the same as config hostfile format.
+    separate line). Hostfile format is the same as
+    config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -423,11 +410,8 @@ Delete tag(s) from inventory database.
 
 **-T TAGFILE, --tag-file TAGFILE**
     Read additional tags from specified file or from stdin
-    if '-' (each tag on separate line). If no tag or tag
-    file is specified then, by default, read from stdin.
-    If no tags/tagfile nor hosts/hostfile are specified
-    then tags are read from stdin and are added to all
-    hosts. Tagfile format is the same as config hostfile format.
+    if '-' (each tag on separate line).
+    Tagfile format is the same as config hostfile format.
 
 **-t TAGLIST, --taglist TAGLIST**
     Tag list to be deleted for specified host(s), comma

From 89a0080e133052b82ce19e7fdbaf045dcc833da7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:09:26 +0200
Subject: [PATCH 03/63] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 7a0d050d..260b4fe4 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* Core: Clarify stdin input (Darko Poljak)
+	* Type __package_pip: Detect pip binary (Ander Punnar)
+
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)
 	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)

From 0fc10749edc698d1f70f826603d009fba943d252 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:11:35 +0200
Subject: [PATCH 04/63] Fix shellcheck

---
 cdist/conf/type/__package_pip/gencode-remote | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 9c5eb0d1..a1375c2d 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -60,7 +60,7 @@ case "$state_should" in
         then
             echo "su -c '$pip install -q $name' $runas"
         else
-            echo $pip install -q "$name"
+            echo "$pip" install -q "$name"
         fi
         echo "installed" >> "$__messages_out"
     ;;
@@ -69,7 +69,7 @@ case "$state_should" in
         then
             echo "su -c '$pip uninstall -q -y $name' $runas"
         else
-            echo $pip uninstall -q -y "$name"
+            echo "$pip" uninstall -q -y "$name"
         fi
         echo "removed" >> "$__messages_out"
     ;;

From b6922508b9e07834009a6bfae2d3933b5ea62fbf Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:17:34 +0200
Subject: [PATCH 05/63] Update helper script

---
 bin/build-helper | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/build-helper b/bin/build-helper
index ed41e438..d4d603ed 100755
--- a/bin/build-helper
+++ b/bin/build-helper
@@ -371,7 +371,6 @@ eof
 Manual steps post release:
     - cdist-web
     - send generated mailinglist.tmp mail
-    - twitter
 eof
     ;;
 

From 84a7818121047cf6bb23d8bd3c055d09bb033d32 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 23 Sep 2020 20:29:47 +0200
Subject: [PATCH 06/63] docs: make varaibles environment-aware

There are all overwriting the environment, even the comment states
otherwise. Fixes it.
---
 docs/src/Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/src/Makefile b/docs/src/Makefile
index 2ecf7a32..ba87d170 100644
--- a/docs/src/Makefile
+++ b/docs/src/Makefile
@@ -2,10 +2,10 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-PAPER         =
-BUILDDIR      = ../dist
+SPHINXOPTS   ?=
+SPHINXBUILD  ?= sphinx-build
+PAPER        ?=
+BUILDDIR     ?= ../dist
 # for cache, etc.
 _BUILDDIR     = _build
 

From 73d6c9d469d1429999ea2e980b3ac4c63a3e0dd4 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 27 Sep 2020 10:17:35 +0200
Subject: [PATCH 07/63] Add custom remote copy/exec examples

---
 docs/src/cdist-remote-exec-copy.rst | 389 +++++++++++++++++++++++++++-
 1 file changed, 388 insertions(+), 1 deletion(-)

diff --git a/docs/src/cdist-remote-exec-copy.rst b/docs/src/cdist-remote-exec-copy.rst
index bb818310..e7b7b226 100644
--- a/docs/src/cdist-remote-exec-copy.rst
+++ b/docs/src/cdist-remote-exec-copy.rst
@@ -10,7 +10,7 @@ By default this is accomplished with ssh and scp respectively.
 The default implementations used by cdist are::
 
     __remote_exec: ssh -o User=root
-    __remote_copy: scp -o User=root
+    __remote_copy: scp -o User=root -q
 
 The user can override these defaults by providing custom implementations and
 passing them to cdist with the --remote-exec and/or --remote-copy arguments.
@@ -26,3 +26,390 @@ specified by enclosed in square brackets (see :strong:`ssh`\ (1) and
 With this simple interface the user can take total control of how cdist
 interacts with the target when required, while the default implementation 
 remains as simple as possible.
+
+
+Examples
+--------
+
+Here are examples of using alternative __remote_copy and __remote_exec scripts.
+
+All scripts from below are present in cdist sources in `other/examples/remote`
+directory.
+
+ssh
+~~~
+
+Same as cdist default.
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    #echo "$@" | logger -t "cdist-ssh-copy"
+    scp -o User=root -q $@
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    #echo "$@" | logger -t "cdist-ssh-exec"
+    ssh -o User=root $@
+
+local
+~~~~~
+
+This effectively turns remote calling into local calling. Probably most useful
+for the unit testing.
+
+**copy**
+
+.. code-block:: sh
+
+    code="$(echo "$@" | sed "s|\([[:space:]]\)$__target_host:|\1|g")"
+    cp -L $code
+
+**exec**
+
+.. code-block:: sh
+
+    target_host=$1; shift
+    echo "$@" | /bin/sh 
+
+chroot
+~~~~~~
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script /path/to/your/chroot" target-id
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-chroot-copy"
+       :
+    }
+
+    chroot="$1"; shift
+    target_host="$__target_host"
+
+    # replace target_host with chroot location
+    code="$(echo "$@" | sed "s|$target_host:|$chroot|g")"
+
+    log "target_host: $target_host"
+    log "chroot: $chroot"
+    log "$@"
+    log "$code"
+
+    # copy files into chroot
+    cp $code
+
+    log "-----"
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script /path/to/your/chroot" target-id
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-chroot-exec"
+       :
+    }
+
+    chroot="$1"; shift
+    target_host="$1"; shift
+
+    script=$(mktemp "${chroot}/tmp/chroot-${0##*/}.XXXXXXXXXX")
+    trap cleanup INT TERM EXIT
+    cleanup() {
+       [ $__cdist_debug ] || rm "$script"
+    }
+
+    log "target_host: $target_host"
+    log "script: $script"
+    log "@: $@"
+    echo "#!/bin/sh -l" > "$script"
+    echo "$@" >> "$script"
+    chmod +x "$script"
+
+    relative_script="${script#$chroot}"
+    log "relative_script: $relative_script"
+
+    # run in chroot
+    chroot "$chroot" "$relative_script"
+
+    log "-----"
+
+rsync
+~~~~~
+
+**copy**
+
+Usage: cdist config --remote-copy /path/to/this/script target_host
+
+.. code-block:: sh
+
+    # For rsync to do the right thing, the source has to end with "/" if it is
+    # a directory. The below preprocessor loop takes care of that.
+
+    # second last argument is the source
+    source_index=$(($#-1))
+    index=0
+    for arg in $@; do
+       if [ $index -eq 0 ]; then
+          # reset $@
+          set --
+       fi
+       index=$((index+=1))
+       if [ $index -eq $source_index -a -d "$arg" ]; then
+          arg="${arg%/}/"
+       fi
+       set -- "$@" "$arg"
+    done
+
+    rsync --backup --suffix=~cdist -e 'ssh -o User=root' $@
+
+schroot
+~~~~~~~
+
+__remote_copy and __remote_exec scripts to run cdist against a chroot on the
+target host over ssh.
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script schroot-chroot-name" target_host
+
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-schroot-copy"
+       :
+    }
+
+    chroot_name="$1"; shift
+    target_host="$__target_host"
+
+    # get directory for given chroot_name
+    chroot="$(ssh -o User=root -q $target_host schroot -c $chroot_name --config | awk -F = '/directory=/ {print $2}')"
+
+    # prefix destination with chroot
+    code="$(echo "$@" | sed "s|$target_host:|$target_host:$chroot|g")"
+
+    log "target_host: $target_host"
+    log "chroot_name: $chroot_name"
+    log "chroot: $chroot"
+    log "@: $@"
+    log "code: $code"
+
+    # copy files into remote chroot
+    scp -o User=root -q $code
+
+    log "-----"
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script schroot-chroot-name" target_host
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-schroot-exec"
+       :
+    }
+
+    chroot_name="$1"; shift
+    target_host="$1"; shift
+
+    code="ssh -o User=root -q $target_host schroot -c $chroot_name -- $@"
+
+    log "target_host: $target_host"
+    log "chroot_name: $chroot_name"
+    log "@: $@"
+    log "code: $code"
+
+    # run in remote chroot
+    $code
+
+    log "-----"
+
+schroot-uri
+~~~~~~~~~~~
+
+__remote_exec/__remote_copy script to run cdist against a schroot target URI.
+
+Usage::
+
+    cdist config \
+        --remote-exec "/path/to/this/script exec" \
+        --remote-copy "/path/to/this/script copy" \
+        target_uri
+
+    # target_uri examples:
+    schroot:///chroot-name
+    schroot://foo.ethz.ch/chroot-name
+    schroot://user-name@foo.ethz.ch/chroot-name
+
+    # and how to match them in .../manifest/init
+    case "$target_host" in
+    schroot://*)
+        # any schroot
+    ;;
+    schroot://foo.ethz.ch/*)
+        # any schroot on specific host
+    ;;
+    schroot://foo.ethz.ch/chroot-name)
+        # specific schroot on specific host
+    ;;
+    schroot:///chroot-name)
+        # specific schroot on localhost
+    ;;
+    esac
+
+**copy/exec**
+
+.. code-block:: sh
+
+    my_name="${0##*/}"
+    mode="$1"; shift
+
+    log() {
+       # uncomment me for debugging
+       #echo "$@" | logger -t "cdist-$my_name-$mode"
+       :
+    }
+
+    die() {
+       echo "$@" >&2
+       exit 1
+    }
+
+
+    uri="$__target_host"
+
+    scheme="${uri%%:*}"; rest="${uri#$scheme:}"; rest="${rest#//}"
+    authority="${rest%%/*}"; rest="${rest#$authority}"
+    path="${rest%\?*}"; rest="${rest#$path}"
+    schroot_name="${path#/}"
+
+    [ "$scheme" = "schroot" ] || die "Failed to parse scheme from __target_host ($__target_host). Expected 'schroot', got '$scheme'"
+    [ -n "$schroot_name" ] || die "Failed to parse schroot name from __target_host: $__target_host"
+
+    case "$authority" in
+       '')
+          # authority is empty, neither user nor host given
+          user=""
+          host=""
+       ;; 
+       *@*)
+          # authority contains @, take user from authority
+          user="${authority%@*}"
+          host="${authority#*@}"
+       ;; 
+       *) 
+          # no user in authority, default to root
+          user="root"
+          host="$authority"
+       ;;
+    esac
+
+    log "mode: $mode"
+    log "@: $@"
+    log "uri: $uri"
+    log "scheme: $scheme"
+    log "authority: $authority"
+    log "user: $user"
+    log "host: $host"
+    log "path: $path"
+    log "schroot_name: $schroot_name"
+
+    exec_prefix=""
+    copy_prefix=""
+    if [ -n "$host" ]; then
+       # we are working on a remote host
+       exec_prefix="ssh -o User=$user -q $host"
+       copy_prefix="scp -o User=$user -q"
+       copy_destination_prefix="$host:"
+    else
+       # working on local machine
+       copy_prefix="cp"
+       copy_destination_prefix=""
+    fi
+    log "exec_prefix: $exec_prefix"
+    log "copy_prefix: $copy_prefix"
+    log "copy_destination_prefix: $copy_destination_prefix"
+
+    case "$mode" in
+       exec)
+          # In exec mode the first argument is the __target_host which we already got from env. Get rid of it.
+          shift
+          code="$exec_prefix schroot -c $schroot_name -- sh -c '$@'"
+       ;;
+       copy)
+          # get directory for given chroot_name
+          schroot_directory="$($exec_prefix schroot -c $schroot_name --config | awk -F = '/directory=/ {print $2}')"
+          [ -n "$schroot_directory" ] || die "Failed to retreive schroot directory for schroot: $schroot_name"
+          log "schroot_directory: $schroot_directory"
+          # prefix destination with chroot
+          code="$copy_prefix $(echo "$@" | sed "s|$uri:|${copy_destination_prefix}${schroot_directory}|g")"
+       ;;
+       *) die "Unknown mode: $mode";;
+    esac
+
+    log "code: $code"
+
+    # Run the code
+    $code
+
+    log "-----"
+
+sudo
+~~~~
+
+**copy**
+
+Use rsync over ssh to copy files. Uses the "--rsync-path" option
+to run the remote rsync instance with sudo.
+
+This command assumes your ssh configuration is already set up in ~/.ssh/config.
+
+Usage: cdist config --remote-copy /path/to/this/script target_host
+
+.. code-block:: sh
+
+    # For rsync to do the right thing, the source has to end with "/" if it is
+    # a directory. The below preprocessor loop takes care of that.
+
+    # second last argument is the source
+    source_index=$(($#-1))
+    index=0
+    for arg in $@; do
+       if [ $index -eq 0 ]; then
+          # reset $@
+          set --
+       fi
+       index=$((index+=1))
+       if [ $index -eq $source_index -a -d "$arg" ]; then
+          arg="${arg%/}/"
+       fi
+       set -- "$@" "$arg"
+    done
+
+    rsync --copy-links --rsync-path="sudo rsync" -e 'ssh' "$@"
+
+**exec**
+
+Prefixes all remote commands with sudo.
+
+This command assumes your ssh configuration is already set up in ~/.ssh/config.
+
+Usage: cdist config --remote-exec "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    host="$1"; shift
+    ssh -q "$host" sudo sh -c \""$@"\"

From 652c89185816526e5165ff9d5686a0d245ec3e5f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 29 Sep 2020 05:57:54 +0200
Subject: [PATCH 08/63] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 260b4fe4..8d380524 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
+	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From f994226d0e23b667309fd75116c0c8a14d15f34d Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 29 Sep 2020 19:44:47 +0200
Subject: [PATCH 09/63] [__package_pkgng_freebsd] Bootstrap pkg if necessary

In a pristine FreeBSD base installation, pkg is really a bootstrapper utility,
in such cases the type used to fail instead of automatically bootstrapping pkg.
---
 .../type/__package_pkgng_freebsd/explorer/pkg_bootstrapped   | 4 ++++
 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version | 5 +++++
 cdist/conf/type/__package_pkgng_freebsd/gencode-remote       | 5 +++++
 3 files changed, 14 insertions(+)
 create mode 100755 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped

diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped
new file mode 100755
index 00000000..429f15d3
--- /dev/null
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+if pkg -N >/dev/null 2>&1; then
+	echo "YES"
+fi
diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
index 92ce0623..f0fb9127 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
@@ -21,6 +21,11 @@
 # Retrieve the status of a package - parsed dpkg output
 #
 
+if ! pkg -N >/dev/null 2>&1; then
+   # Nothing to do if pkg is not bootstrapped
+   exit
+fi
+
 if [ -f "$__object/parameter/name" ]; then
    name="$(cat "$__object/parameter/name")"
 else
diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
index dd36efda..b5944177 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
+++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
@@ -43,6 +43,7 @@ fi
 repo="$(cat "$__object/parameter/repo")"
 state="$(cat "$__object/parameter/state")"
 curr_version="$(cat "$__object/explorer/pkg_version")"
+pkg_bootstrapped="$(cat "$__object/explorer/pkg_bootstrapped")"
 add_cmd="pkg install -y"
 rm_cmd="pkg delete -y"
 upg_cmd="pkg upgrade -y"
@@ -73,6 +74,10 @@ execcmd(){
          ;;
    esac
 
+   if [ -z "${pkg_bootstrapped}" ]; then
+       echo "pkg bootstrap -y >/dev/null 2>&1"
+   fi
+
    echo "$_cmd >/dev/null 2>&1"   # Silence the output of the command
    echo "status=\$?"
    echo "if [ \"\$status\" -ne \"0\" ]; then"

From 52b5f05163416196403634cb9a52f746cdb96391 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 30 Sep 2020 08:56:31 +0200
Subject: [PATCH 10/63] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8d380524..34c97d67 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
+	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 3fa74b454a9512319e3559a53dd477f38724fd9b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 30 Sep 2020 15:43:32 +0200
Subject: [PATCH 11/63] Fix typo

---
 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
index f0fb9127..1c6ba5e5 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Retrieve the status of a package - parsed dpkg output
+# Retrieve the status of a package - parsed pkgng output
 #
 
 if ! pkg -N >/dev/null 2>&1; then

From 5aeed14b1b8b022f952680cceb12a5099a099558 Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Thu, 8 Oct 2020 16:15:20 +0200
Subject: [PATCH 12/63] Fixed calling of __systemd_service type with correct
 arguments.

---
 cdist/conf/type/__service/manifest | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__service/manifest b/cdist/conf/type/__service/manifest
index cb5af234..beb0713c 100644
--- a/cdist/conf/type/__service/manifest
+++ b/cdist/conf/type/__service/manifest
@@ -7,7 +7,9 @@ action="$(cat "$__object/parameter/action")"
 
 case "$manager" in
 	systemd)
-		__systemd_service "$name" --action "$action"
+		test "$action" = "start" && action="running"
+		test "$action" = "stop"  && action="stopped"
+		__systemd_service "$name" --state "$action"
 	;;
 	*)
 		# Unknown: handled by `service $NAME $action` in gencode-remote.

From c030deea3dbccfe73637b730f0d545db7f8d5f35 Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Fri, 9 Oct 2020 06:51:44 +0200
Subject: [PATCH 13/63] [__line] Add support for '--state replace'

It is currently counter-intuitive that something like:

    # File '/thing' contents
    #SomeSetting WrongValue

    # Manifest
    __line '/thing' \
           --line 'SomeSeting GoodValue' \
           --regex '^(#[[:space:]]*)?SomeSetting[[:space:]]'

Produces:

    # Resulting '/thing' contents
    #SomeSetting WrongValue

This makes sense given the implementation, but it masks a very common use-case.

Changing the default behaviour for such a base type is not really an option, so
instead we add a `replace` as a valid value for `--state`, which would result
in:

    # Resulting '/thing' contents with: --state replace
    SomeSetting GoodValue

For compatibility, if the regex is missing, `--state replace` behaves just as
`--state present`.
---
 cdist/conf/type/__line/explorer/state | 12 +++++++++++-
 cdist/conf/type/__line/gencode-remote | 10 +++++++---
 cdist/conf/type/__line/man.rst        | 13 +++++++++++--
 3 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state
index e8fc3630..9d480b19 100755
--- a/cdist/conf/type/__line/explorer/state
+++ b/cdist/conf/type/__line/explorer/state
@@ -53,8 +53,10 @@ function _find(_text, _pattern) {
 BEGIN {
    getline anchor < (ENVIRON["__object"] "/parameter/" position)
    getline pattern < (ENVIRON["__object"] "/parameter/" needle)
+   getline line < (ENVIRON["__object"] "/parameter/line")
 
    found_line = 0
+   correct_line = 0
    correct_pos = (position != "after" && position != "before")
 }
 {
@@ -63,15 +65,18 @@ BEGIN {
          getline
          if (_find($0, pattern)) {
             found_line++
+	    if (index($0, line) == 1) { correct_line++ }
             correct_pos = 1
             exit 0
          }
       } else if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
       }
    } else if (position == "before") {
       if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
          getline
          if (match($0, anchor)) {
             correct_pos = 1
@@ -81,13 +86,18 @@ BEGIN {
    } else {
       if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
          exit 0
       }
    }
 }
 END {
    if (found_line && correct_pos) {
-      print "present"
+      if (correct_line) {
+        print "present"
+      } else {
+        print "matching"
+      }
    } else if (found_line) {
       print "wrongposition"
    } else {
diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote
index 88cae68b..a89886da 100755
--- a/cdist/conf/type/__line/gencode-remote
+++ b/cdist/conf/type/__line/gencode-remote
@@ -38,7 +38,11 @@ if [ -z "$state_is" ]; then
    exit 1
 fi
 
-if [ "$state_should" = "$state_is" ]; then
+if [ "$state_should" = "$state_is" ] || \
+	{ [ "$state_should" = "present" ] && [ "$state_is" = "matching" ] ;} || \
+	{ [ "$state_should" = "replace" ] && [ "$state_is" = "present" ] ;} ; then
+   # If state matches already, or 'present' is used and regex matches
+   # or 'replace' is used and the exact line is present, then there is
    # nothing to do
    exit 0
 fi
@@ -61,8 +65,8 @@ fi
 add=0
 remove=0
 case "$state_should" in
-   present)
-      if [ "$state_is" = "wrongposition" ]; then
+   present|replace)
+      if [ "$state_is" = "wrongposition" ] || [ "$state_is" = "matching" ]; then
          echo updated >> "$__messages_out"
          remove=1
       else
diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst
index f76cab64..70490f68 100644
--- a/cdist/conf/type/__line/man.rst
+++ b/cdist/conf/type/__line/man.rst
@@ -31,7 +31,7 @@ file
 line
     Specifies the line which should be absent or present.
 
-    Must be present, if state is 'present'.
+    Must be present, if state is 'present' or 'replace'.
     Ignored if regex is given and state is 'absent'.
 
 regex
@@ -41,10 +41,13 @@ regex
     If state is 'absent', ensure all lines matching the regular expression
     are absent.
 
+    If state is 'replace', ensure all lines matching the regular expression
+    are exactly 'line'.
+
     The regular expression is interpreted by awk's match function.
 
 state
-    'present' or 'absent', defaults to 'present'
+    'present', 'absent' or 'replace', defaults to 'present'.
 
 onchange
     The code to run if line is added, removed or updated.
@@ -99,6 +102,12 @@ EXAMPLES
         --line '-session required pam_exec.so debug log=/tmp/classify.log /usr/local/libexec/classify' \
         --after '^session[[:space:]]+include[[:space:]]+password-auth-ac$'
 
+    # Uncomment as needed and set a value in a configuration file.
+    __line /etc/example.conf \
+        --line 'SomeSetting SomeValue' \
+        --regex '^(#[[:space:]]*)?SomeSetting[[:space:]]' \
+        --state replace
+
 
 SEE ALSO
 --------

From 4df5c91912edf9f31a6716d7cbe0adaedbc2b0e3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 9 Oct 2020 06:52:52 +0200
Subject: [PATCH 14/63] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 34c97d67..6d237449 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,8 @@ next:
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
 	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
+	* Type __service: Fix calling __systemd_service (Mark Verboom)
+	* Type __line: Add 'replace' state (Evil Ham)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 507fa6fa93210959b52bb639a9394c8ac52b40e5 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 17 Oct 2020 17:05:09 +0200
Subject: [PATCH 15/63] __download: fix non-existent parameter of __unpack

Probably happened due to renaming .. guess it's correct now.
---
 cdist/conf/type/__download/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index eb3ac971..d8814683 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -69,7 +69,7 @@ EXAMPLES
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
-            --move-existing-destination \
+            --backup-destination \
             --destination /opt/cpma/server
 
 

From b2e6afb57e6799b934a2eaa387cea337d65d8aac Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 17 Oct 2020 23:01:36 +0200
Subject: [PATCH 16/63] __download: adapt download+unpack example in manpage

---
 cdist/conf/type/__download/man.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index d8814683..54503470 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -70,6 +70,7 @@ EXAMPLES
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
             --backup-destination \
+            --preserve-archive \
             --destination /opt/cpma/server
 
 

From 955b84727611caa5aa05e4521af326c94c649e89 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 18 Oct 2020 15:55:14 +0200
Subject: [PATCH 17/63] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 6d237449..8ad2c624 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
 	* Type __service: Fix calling __systemd_service (Mark Verboom)
 	* Type __line: Add 'replace' state (Evil Ham)
+	* Type __download: Fix man page (Matthias Stecher)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From e3d906a85fe5e0c9f254d3620e8568437c93d97c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 15 Sep 2020 00:55:26 +0300
Subject: [PATCH 18/63] [__acl] remove deprecated parameters, fix some bugs and
 improve manual

---
 cdist/conf/type/__acl/explorer/checks         | 39 -------------------
 cdist/conf/type/__acl/explorer/getent         |  4 ++
 cdist/conf/type/__acl/gencode-remote          | 36 ++++++++---------
 cdist/conf/type/__acl/man.rst                 | 12 +++---
 .../conf/type/__acl/parameter/deprecated/acl  |  1 -
 .../type/__acl/parameter/deprecated/group     |  1 -
 .../conf/type/__acl/parameter/deprecated/mask |  1 -
 .../type/__acl/parameter/deprecated/other     |  1 -
 .../conf/type/__acl/parameter/deprecated/user |  1 -
 cdist/conf/type/__acl/parameter/optional      |  2 -
 .../type/__acl/parameter/optional_multiple    |  3 --
 11 files changed, 26 insertions(+), 75 deletions(-)
 delete mode 100755 cdist/conf/type/__acl/explorer/checks
 create mode 100755 cdist/conf/type/__acl/explorer/getent
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/acl
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/group
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/mask
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/other
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/user

diff --git a/cdist/conf/type/__acl/explorer/checks b/cdist/conf/type/__acl/explorer/checks
deleted file mode 100755
index 70bb0412..00000000
--- a/cdist/conf/type/__acl/explorer/checks
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/sh -e
-#
-# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-# TODO check if filesystem has ACL turned on etc
-
-if [ -f "$__object/parameter/acl" ]
-then
-    grep -E '^(default:)?(user|group):' "$__object/parameter/acl" \
-    | while read -r acl
-    do
-        param="$( echo "$acl" | awk -F: '{print $(NF-2)}' )"
-        check="$( echo "$acl" | awk -F: '{print $(NF-1)}' )"
-
-        [ "$param" = 'user' ] && db=passwd || db="$param"
-
-        if ! getent "$db" "$check" > /dev/null
-        then
-            echo "missing $param '$check'" >&2
-            exit 1
-        fi
-    done
-fi
diff --git a/cdist/conf/type/__acl/explorer/getent b/cdist/conf/type/__acl/explorer/getent
new file mode 100755
index 00000000..7e6c2c30
--- /dev/null
+++ b/cdist/conf/type/__acl/explorer/getent
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+getent passwd | awk -F: '{print "user:"$1}'
+getent group | awk -F: '{print "group:"$1}'
diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote
index e5404a9d..32318e91 100755
--- a/cdist/conf/type/__acl/gencode-remote
+++ b/cdist/conf/type/__acl/gencode-remote
@@ -22,8 +22,8 @@ file_is="$( cat "$__object/explorer/file_is" )"
 
 if [ "$file_is" = 'missing' ] \
     && [ -z "$__cdist_dry_run" ] \
-    && \( [ ! -f "$__object/parameter/file" ] \
-        || [ ! -f "$__object/parameter/directory" ] \)
+    && [ ! -f "$__object/parameter/file" ] \
+    && [ ! -f "$__object/parameter/directory" ]
 then
     exit 0
 fi
@@ -47,28 +47,26 @@ then
 elif [ -f "$__object/parameter/entry" ]
 then
     acl_should="$( cat "$__object/parameter/entry" )"
-elif [ -f "$__object/parameter/acl" ]
-then
-    acl_should="$( cat "$__object/parameter/acl" )"
-elif
-    [ -f "$__object/parameter/user" ] \
-        || [ -f "$__object/parameter/group" ] \
-        || [ -f "$__object/parameter/mask" ] \
-        || [ -f "$__object/parameter/other" ]
-then
-    acl_should="$( for param in user group mask other
-    do
-        [ ! -f "$__object/parameter/$param" ] && continue
-
-        echo "$param" | grep -Eq 'mask|other' && sep=:: || sep=:
-
-        echo "$param$sep$( cat "$__object/parameter/$param" )"
-    done )"
 else
     echo 'no parameters set' >&2
     exit 1
 fi
 
+# instead of setfacl's non-helpful message "Option -m: Invalid argument near character X"
+# let's check if target has necessary users and groups, since mistyped or missing
+# users/groups in target is most common reason.
+echo "$acl_should" \
+    | grep -Po '(user|group):[^:]+' \
+    | sort -u \
+    | while read -r l
+    do
+        if ! grep "$l" -Fxq "$__object/explorer/getent"
+        then
+            echo "no $l' in target" | sed "s/:/ '/" >&2
+            exit 1
+        fi
+    done
+
 if [ -f "$__object/parameter/default" ]
 then
     acl_should="$( echo "$acl_should" \
diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst
index 28412871..307be72b 100644
--- a/cdist/conf/type/__acl/man.rst
+++ b/cdist/conf/type/__acl/man.rst
@@ -12,11 +12,14 @@ Fully supported and tested on Linux (ext4 filesystem), partial support for FreeB
 
 See ``setfacl`` and ``acl`` manpages for more details.
 
+One of ``--entry`` or ``--source`` must be used.
 
-REQUIRED MULTIPLE PARAMETERS
+
+OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
 entry
    Set ACL entry following ``getfacl`` output syntax.
+   Must be used if ``--source`` is not used.
 
 
 OPTIONAL PARAMETERS
@@ -25,6 +28,7 @@ source
    Read ACL entries from stdin or file.
    Ordering of entries is not important.
    When reading from file, comments and empty lines are ignored.
+   Must be used if ``--entry`` is not used.
 
 file
    Create/change file with ``__file`` using ``user:group:mode`` pattern.
@@ -48,12 +52,6 @@ remove
    ``mask`` and ``other`` entries can't be removed, but only changed.
 
 
-DEPRECATED PARAMETERS
----------------------
-Parameters ``acl``, ``user``, ``group``, ``mask`` and ``other`` are deprecated and they
-will be removed in future versions. Please use ``entry`` parameter instead.
-
-
 EXAMPLES
 --------
 
diff --git a/cdist/conf/type/__acl/parameter/deprecated/acl b/cdist/conf/type/__acl/parameter/deprecated/acl
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/acl
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/group b/cdist/conf/type/__acl/parameter/deprecated/group
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/group
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/mask b/cdist/conf/type/__acl/parameter/deprecated/mask
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/mask
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/other b/cdist/conf/type/__acl/parameter/deprecated/other
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/other
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/user b/cdist/conf/type/__acl/parameter/deprecated/user
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/user
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/optional b/cdist/conf/type/__acl/parameter/optional
index cdcbc0b8..5a0c29a3 100644
--- a/cdist/conf/type/__acl/parameter/optional
+++ b/cdist/conf/type/__acl/parameter/optional
@@ -1,5 +1,3 @@
-mask
-other
 source
 file
 directory
diff --git a/cdist/conf/type/__acl/parameter/optional_multiple b/cdist/conf/type/__acl/parameter/optional_multiple
index c615d507..4c884f03 100644
--- a/cdist/conf/type/__acl/parameter/optional_multiple
+++ b/cdist/conf/type/__acl/parameter/optional_multiple
@@ -1,4 +1 @@
 entry
-acl
-user
-group

From 716cd37281903aafd7d65df539265a6722e809b1 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 21 Sep 2020 11:18:39 +0300
Subject: [PATCH 19/63] [__update_alternatives] rewrite and support --install

---
 .../explorer/alternatives                     |  4 ++
 .../type/__update_alternatives/explorer/link  | 40 +++++++++++++++++++
 .../__update_alternatives/explorer/path_is    | 12 ++++++
 .../explorer/path_should_state                |  8 ++++
 .../type/__update_alternatives/explorer/state |  8 ----
 .../type/__update_alternatives/gencode-remote | 37 ++++++++++++++---
 cdist/conf/type/__update_alternatives/man.rst | 13 ++++--
 .../__update_alternatives/parameter/boolean   |  1 +
 8 files changed, 107 insertions(+), 16 deletions(-)
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/alternatives
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/link
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/path_is
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/path_should_state
 delete mode 100755 cdist/conf/type/__update_alternatives/explorer/state
 create mode 100644 cdist/conf/type/__update_alternatives/parameter/boolean

diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives
new file mode 100755
index 00000000..34aaca56
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/alternatives
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+update-alternatives --display "$__object_id" 2>/dev/null \
+    | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link
new file mode 100755
index 00000000..6519e7c2
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/link
@@ -0,0 +1,40 @@
+#!/bin/sh -e
+
+# fedora's (update-)alternatives --display output doesn't have
+# "link <name> is <path>" line, but debian does. so, let's find
+# out how they store this information.
+#
+# debian and friends:
+#   https://salsa.debian.org/dpkg-team/dpkg/-/blob/master/utils/update-alternatives.c
+#   see calls to altdb_print_line function
+#
+# fedora and friends:
+#   https://github.com/fedora-sysv/chkconfig/blob/master/alternatives.c
+#   see calls to parseLine function
+#
+# conclusion: it is safe to assume that (master) link is on second line
+
+for altdir in \
+    /var/lib/dpkg/alternatives \
+    /var/lib/alternatives
+do
+    if [ ! -f "$altdir/$__object_id" ]
+    then
+        continue
+    fi
+
+    link="$( awk 'NR==2' "$altdir/$__object_id" )"
+
+    if [ -n "$link" ]
+    then
+        break
+    fi
+done
+
+if [ -z "$link" ]
+then
+    echo "unable to get link for $__object_id" >&2
+    exit 1
+fi
+
+echo "$link"
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
new file mode 100755
index 00000000..fc304d5d
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -0,0 +1,12 @@
+#!/bin/sh -e
+
+path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \
+    | awk '/link currently points to/ {print $5}' )"
+
+if [ -z "$path_is" ]
+then
+    echo "unable to get current path for $__object_id" >&2
+    exit 1
+fi
+
+echo "$path_is"
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state
new file mode 100755
index 00000000..59e015c5
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+if [ -f "$( cat "$__object/parameter/path" )" ]
+then
+    echo 'present'
+else
+    echo 'absent'
+fi
diff --git a/cdist/conf/type/__update_alternatives/explorer/state b/cdist/conf/type/__update_alternatives/explorer/state
deleted file mode 100755
index 04a78aaa..00000000
--- a/cdist/conf/type/__update_alternatives/explorer/state
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh -e
-path="$(cat "$__object/parameter/path")"
-name="$__object_id"
-link="$(readlink "/etc/alternatives/$name")"
-if [ "$path" = "$link" ]
-then echo present
-else echo absent
-fi
diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index c0b49814..e393cdef 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Ander Punnar (ander@kvlt.ee)
 #
 # This file is part of cdist.
 #
@@ -16,12 +17,38 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
-if [ "$(cat "$__object/explorer/state")" = 'present' ]
-then exit 0
+path_is="$( cat "$__object/explorer/path_is" )"
+
+path_should="$( cat "$__object/parameter/path" )"
+
+if [ "$path_is" = "$path_should" ]
+then
+    exit 0
+fi
+
+if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ]
+then
+    echo "$path_should does not exist in target" >&2
+    exit 1
 fi
 
-path="$(cat "$__object/parameter/path")"
 name="$__object_id"
-echo "update-alternatives --quiet --set '$name' '$path'"
+
+alternatives="$( cat "$__object/explorer/alternatives" )"
+
+if ! echo "$alternatives" | grep -Fxq "$path_should"
+then
+    if [ ! -f "$__object/parameter/install" ]
+    then
+        echo "$path_should is not in $name alternatives." >&2
+        echo 'Please install missing packages or use --install to add path to alternatives.' >&2
+        exit 1
+    fi
+
+    link="$( cat "$__object/explorer/link" )"
+
+    echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
+fi
+
+echo "update-alternatives --set '$name' '$path_should'"
diff --git a/cdist/conf/type/__update_alternatives/man.rst b/cdist/conf/type/__update_alternatives/man.rst
index 73d82d11..0dc973f2 100644
--- a/cdist/conf/type/__update_alternatives/man.rst
+++ b/cdist/conf/type/__update_alternatives/man.rst
@@ -19,6 +19,12 @@ path
    Use this path for the given alternative
 
 
+BOOLEAN PARAMETERS
+------------------
+install
+   Add (``update-alternatives --install``) missing path to alternatives.
+
+
 EXAMPLES
 --------
 
@@ -36,11 +42,12 @@ SEE ALSO
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Ander Punnar <ander@kvlt.ee>
 
 
 COPYING
 -------
-Copyright \(C) 2013 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
+Copyright \(C) 2013 Nico Schottelius and 2020 Ander Punnar. You can
+redistribute it and/or modify it under the terms of the GNU General Public
+License as published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__update_alternatives/parameter/boolean b/cdist/conf/type/__update_alternatives/parameter/boolean
new file mode 100644
index 00000000..7c32f559
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/parameter/boolean
@@ -0,0 +1 @@
+install

From 687c1d2dd9bed4130a65885d19ce31c87c9d79de Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 19 Oct 2020 06:57:00 +0200
Subject: [PATCH 20/63] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 8ad2c624..a0f1ead2 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,8 @@ next:
 	* Type __service: Fix calling __systemd_service (Mark Verboom)
 	* Type __line: Add 'replace' state (Evil Ham)
 	* Type __download: Fix man page (Matthias Stecher)
+	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
+	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 367da4b77e79b82fe84c2ce1a7f75cfad92db81e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 28 Oct 2020 18:18:24 +0100
Subject: [PATCH 21/63] [type/__file] Fix --state pre-exists

---
 cdist/conf/type/__file/gencode-remote | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index 35356b13..2675b03a 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -87,11 +87,6 @@ case "$state_should" in
         fi
     ;;
 
-    pre-exists)
-        # pre-exists should never reach gencode-remote…
-        exit 1
-   ;;
-
     absent)
         if [ "$type" = "file" ]; then
             echo "rm -f '$destination'"

From 9277e0ba19c346189658712acf17a772967dfa32 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 29 Oct 2020 09:30:58 +0100
Subject: [PATCH 22/63] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index a0f1ead2..5977d80e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,7 @@ next:
 	* Type __download: Fix man page (Matthias Stecher)
 	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
+	* Type __file: Fix state pre-exists (Dennis Camera)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 82a9aa79020ff2369b1f4d530a3a6d5ca41915f8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 29 Oct 2020 10:39:42 +0100
Subject: [PATCH 23/63] [type/__apt_norecommends] Use 00InstallRecommends file
 as debian-installer does

debian-installer can be preseeded with `base-installer/install-recommends` to
disable installation of recommended packages already during OS installation.
d-i will then create the file `/etc/apt/apt.conf.d/00InstallRecommends`
(cf. https://salsa.debian.org/installer-team/base-installer/-/blob/master/library.sh).

__apt_norecommends should use the same file to avoid having two config files
effectively doing the same thing.
---
 cdist/conf/type/__apt_norecommends/man.rst  |  9 +++--
 cdist/conf/type/__apt_norecommends/manifest | 41 +++++++++++----------
 2 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/cdist/conf/type/__apt_norecommends/man.rst b/cdist/conf/type/__apt_norecommends/man.rst
index 001fffe4..9297b518 100644
--- a/cdist/conf/type/__apt_norecommends/man.rst
+++ b/cdist/conf/type/__apt_norecommends/man.rst
@@ -32,11 +32,12 @@ EXAMPLES
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
+Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Copyright \(C) 2014 Steven Armstrong. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2014 Steven Armstrong, 2020 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__apt_norecommends/manifest b/cdist/conf/type/__apt_norecommends/manifest
index e737df89..fc187784 100755
--- a/cdist/conf/type/__apt_norecommends/manifest
+++ b/cdist/conf/type/__apt_norecommends/manifest
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,26 +20,28 @@
 #
 
 
-os=$(cat "$__global/explorer/os")
+os=$(cat "${__global:?}/explorer/os")
 
-case "$os" in
-   ubuntu|debian|devuan)
-      # No stinking recommends thank you very much.
-      # If I want something installed I will do so myself.
-      __file /etc/apt/apt.conf.d/99-no-recommends \
-         --owner root --group root --mode 644 \
-         --source - << DONE
-APT::Install-Recommends "0";
-APT::Install-Suggests "0";
-APT::AutoRemove::RecommendsImportant "0";
-APT::AutoRemove::SuggestsImportant "0";
-DONE
-   ;;
-   *)
-      cat >&2 << DONE
+case ${os}
+in
+	(ubuntu|debian|devuan)
+		__file /etc/apt/apt.conf.d/00InstallRecommends --state present \
+			--owner root --group root --mode 0644 --source - <<-'EOF'
+		APT::Install-Recommends "false";
+		APT::Install-Suggests "false";
+		APT::AutoRemove::RecommendsImportant "false";
+		APT::AutoRemove::SuggestsImportant "false";
+		EOF
+
+		# TODO: Remove the following object after some time
+		require=__file/etc/apt/apt.conf.d/00InstallRecommends \
+		__file /etc/apt/apt.conf.d/99-no-recommends --state absent
+		;;
+	(*)
+		cat >&2 <<EOF
 The developer of this type (${__type##*/}) did not think your operating system
 ($os) would have any use for it. If you think otherwise please submit a patch.
-DONE
-      exit 1
-   ;;
+EOF
+		exit 1
+		;;
 esac

From b9ad22595fdb75c17e88e33dc028c33dbd5a7de9 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 18:03:27 +0100
Subject: [PATCH 24/63] [scanner] begin scanner implementation - non invasive

---
 cdist/scan.py | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 cdist/scan.py

diff --git a/cdist/scan.py b/cdist/scan.py
new file mode 100644
index 00000000..fa1bf5de
--- /dev/null
+++ b/cdist/scan.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+from scapy.all import *
+from scapy.data import ETHER_TYPES
+
+
+class Scanner(object):
+    def recv_msg_cpu(self, pkg):
+#        print(pkg.__repr__())
+        if ICMPv6EchoReply in pkg:
+            host = pkg['IPv6'].src
+            print(f"Host {host} is alive")
+
+
+    def scan(self):
+        sniff(iface="wlan0",
+              filter="icmp6",
+              prn=self.recv_msg_cpu)
+
+
+if __name__ == '__main__':
+    s = Scanner()
+    s.scan()

From 87b46a622411362f441b62199972e71d85d7d2d6 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 18:49:20 +0100
Subject: [PATCH 25/63] [scanner] finish prototype

ping @poljakowski - it's your turn now
---
 cdist/scan.py | 138 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 131 insertions(+), 7 deletions(-)

diff --git a/cdist/scan.py b/cdist/scan.py
index fa1bf5de..e2100499 100644
--- a/cdist/scan.py
+++ b/cdist/scan.py
@@ -19,24 +19,148 @@
 #
 #
 
-from scapy.all import *
-from scapy.data import ETHER_TYPES
+#
+# Interface to be implemented:
+# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated
+#   scan: scan / listen for icmp6 replies
+#   trigger: send trigger to multicast
+#   config: configure newly detected hosts
+#   install: install newly detected hosts
+#
+# Scanner logic
+#  - save results to configdir:
+#     basedir = ~/.cdist/scan/<ipv6-address>
+#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time or similar
+#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#
+#
+#
+#
+# cdist scan --list
+#       Show all known hosts including last seen flag
+#
+# Logic for reconfiguration:
+#
+#  - record when configured last time
+#  - introduce a parameter --reconfigure-after that takes time argument
+#  - reconfigure if a) host alive and b) reconfigure-after time passed
+#
 
 
+from multiprocessing import Process
+import os
+
+# FIXME: fail gracefully if non existent - i.e. "scapy required for scanner - please install python3-scapy"
+from scapy.all import *
+
+# Datetime overwrites scapy.all.datetime - needs to be imported AFTER
+import datetime
+
+class Trigger(object):
+    """
+    Trigger an ICMPv6EchoReply from all hosts that are alive
+    """
+
+    def __init__(self, interfaces=None, verbose=False):
+        self.interfaces = interfaces
+        self.verbose = verbose
+
+        # Wait 5 seconds before triggering again - FIXME: add parameter
+        self.sleeptime = 5
+
+    def start(self):
+        self.processes = []
+        for interface in self.interfaces:
+            p = Process(target=self.run_interface, args=(interface,))
+            self.processes.append(p)
+            p.start()
+
+    def join(self):
+        for process in self.processes:
+            process.join()
+
+    def run_interface(self, interface):
+        while True:
+            self.trigger(interface)
+            time.sleep(self.sleeptime)
+
+    def trigger(self, interface):
+        packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
+        send(packet, verbose=self.verbose)
+
 class Scanner(object):
-    def recv_msg_cpu(self, pkg):
-#        print(pkg.__repr__())
+    """
+    Scan for replies of hosts, maintain the up-to-date database
+    """
+
+    def __init__(self, interfaces=None, outdir=None):
+        self.interfaces = interfaces
+
+        if outdir:
+            self.outdir = outdir
+        else:
+            self.outdir = "."
+
+    def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
             host = pkg['IPv6'].src
             print(f"Host {host} is alive")
 
+            dir = os.path.join(self.outdir, host)
+            fname = os.path.join(dir, "last_seen")
+
+            now = datetime.datetime.now()
+
+            os.makedirs(dir, exist_ok=True)
+
+            # FIXME: maybe adjust the format so we can easily parse again
+            with open(fname, "w") as fd:
+                fd.write(f"{now}\n")
+
 
     def scan(self):
-        sniff(iface="wlan0",
+        sniff(iface=self.interfaces,
               filter="icmp6",
-              prn=self.recv_msg_cpu)
+              prn=self.handle_pkg)
 
 
 if __name__ == '__main__':
-    s = Scanner()
+    t = Trigger(interfaces=["wlan0"])
+    t.start()
+
+    # Scanner can listen on many interfaces at the same time
+    s = Scanner(interfaces=["wlan0"])
     s.scan()
+
+    # Join back the trigger processes
+    t.join()
+
+    # Test in my lan shows:
+    # [18:48] bridge:cdist% ls -1d fe80::*
+    # fe80::142d:f0a5:725b:1103
+    # fe80::20d:b9ff:fe49:ac11
+    # fe80::20d:b9ff:fe4c:547d
+    # fe80::219:d2ff:feb2:2e12
+    # fe80::21b:fcff:feee:f446
+    # fe80::21b:fcff:feee:f45c
+    # fe80::21b:fcff:feee:f4b1
+    # fe80::21b:fcff:feee:f4ba
+    # fe80::21b:fcff:feee:f4bc
+    # fe80::21b:fcff:feee:f4c1
+    # fe80::21d:72ff:fe86:46b
+    # fe80::42b0:34ff:fe6f:f6f0
+    # fe80::42b0:34ff:fe6f:f863
+    # fe80::42b0:34ff:fe6f:f9b2
+    # fe80::4a5d:60ff:fea1:e55f
+    # fe80::77a3:5e3f:82cc:f2e5
+    # fe80::9e93:4eff:fe6c:c1f4
+    # fe80::ba69:f4ff:fec5:6041
+    # fe80::ba69:f4ff:fec5:8db7
+    # fe80::bad8:12ff:fe65:313d
+    # fe80::bad8:12ff:fe65:d9b1
+    # fe80::ce2d:e0ff:fed4:2611
+    # fe80::ce32:e5ff:fe79:7ea7
+    # fe80::d66d:6dff:fe33:e00
+    # fe80::e2ff:f7ff:fe00:20e6
+    # fe80::f29f:c2ff:fe7c:275e

From 91d99bf08accb3bde0da420294fc9fb68d3d4068 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 21:22:36 +0100
Subject: [PATCH 26/63] [RFC] scanner documentation

---
 docs/dev/logs/2020-10-29.org | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 docs/dev/logs/2020-10-29.org

diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org
new file mode 100644
index 00000000..718fd68c
--- /dev/null
+++ b/docs/dev/logs/2020-10-29.org
@@ -0,0 +1,34 @@
+* The scanner, 2020-10-29, Hacking Villa Diesbach
+** Motivation
+   - The purpose of cdist is to ensure systems are in a configured state
+   - If systems reboot into a clean (think: netboot) state they are
+     stuck in an unconfigured mode
+   - We can either trigger *from* those machines
+     - this is what cdist trigger is for
+   - Or we can regulary *scan* for machines
+     - This method does not need any modification to standard OS
+** How it works
+   - cdist scan uses the all nodes multicast group ff02::1
+   - It sends a ping packet there in regular intervals
+   - This even works in non-IPv6 networks, as all operating systems
+     are IPv6 capable and usually IPv6 enabled by default
+     - Link local is always accessible!
+   - cdist scan receives an answer from all alive hosts
+     - These results are stored in ~/.cdist/scan/${hostip}
+     - We record the last_seen date  ~/.cdist/scan/${hostip}/last_seen
+   - After a host is detected, cdist *can* try to configure it
+     - It saves the result (+/- logging needs to be defined) in
+       ~/.cdist/scan/${hostip}/{config, install}_result
+     - If logging is saved: maybe in ~/.cdist/scan/${hostip}/{config, install}_log
+     - Final naming TBD
+** Benefits from the scanning approach
+   - We know when a host is alive/dead
+   - We can use standard OS w/o trigger customisation
+     - Only requirement: we can ssh into it
+     - Can make use f.i. of Alpine Linux w/ ssh keys feeding in
+   - We can trigger regular reconfiguration
+     - If alive && last_config_time > 1d -> reconfigure
+   - Data can be exported to f.i. prometheus
+     - Record when configured (successfully)
+     - Record when seen
+   - Enables configurations in stateless environments

From 09dfcfe81e0d9e6520d9bf98598037f86c84641e Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 23:16:08 +0100
Subject: [PATCH 27/63] [scanner] add to beta commands

---
 cdist/argparse.py            | 32 ++++++++++++++++++++-
 cdist/scan/commandline.py    | 55 ++++++++++++++++++++++++++++++++++++
 cdist/{ => scan}/scan.py     | 20 +++++++++----
 docs/dev/logs/2020-10-29.org | 23 +++++++++++++++
 4 files changed, 124 insertions(+), 6 deletions(-)
 create mode 100644 cdist/scan/commandline.py
 rename cdist/{ => scan}/scan.py (90%)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 1d16bb25..ff195e8c 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -8,10 +8,11 @@ import cdist.configuration
 import cdist.log
 import cdist.preos
 import cdist.info
+import cdist.scan.commandline
 
 
 # set of beta sub-commands
-BETA_COMMANDS = set(('install', 'inventory', ))
+BETA_COMMANDS = set(('install', 'inventory', 'scan', ))
 # set of beta arguments for sub-commands
 BETA_ARGS = {
     'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
@@ -470,6 +471,35 @@ def get_parsers():
             'pattern', nargs='?', help='Glob pattern.')
     parser['info'].set_defaults(func=cdist.info.Info.commandline)
 
+    # Scan = config + further
+    parser['scan'] = parser['sub'].add_parser('scan', add_help=False,
+                                                 parents=[parser['config']])
+
+    parser['scan'] = parser['sub'].add_parser(
+            'scan', parents=[parser['loglevel'],
+                             parser['beta'],
+                             parser['colored_output'],
+                             parser['common'],
+                             parser['config_main']])
+
+    parser['scan'].add_argument(
+        '-m', '--mode', help='Which modes should run',
+        action='append', default=[],
+        choices=['scan', 'trigger'])
+    parser['scan'].add_argument(
+        '--config',
+        action='store_true',
+        help='Try to configure detected hosts')
+    parser['scan'].add_argument(
+        '-I', '--interfaces',
+        action='append',  default=[],
+        help='On which interfaces to scan/trigger')
+    parser['scan'].add_argument(
+        '-d', '--delay',
+        action='store',  default=3600,
+        help='How long to wait before reconfiguring after last try')
+    parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
+
     for p in parser:
         parser[p].epilog = EPILOG
 
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
new file mode 100644
index 00000000..0fb718e5
--- /dev/null
+++ b/cdist/scan/commandline.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import logging
+
+log = logging.getLogger("scan")
+
+
+# define this outside of the class to not handle scapy import errors by default
+def commandline(args):
+    log.debug(args)
+
+    try:
+        import cdist.scan.scan as scan
+    except ModuleNotFoundError:
+        print('cdist scan requires scapy to be installed')
+
+    processes = []
+
+    if not args.mode:
+        # By default scan and trigger, but do not call any action
+        args.mode = ['scan', 'trigger' ]
+
+    if 'trigger' in args.mode:
+        t = scan.Trigger(interfaces=args.interfaces)
+        t.start()
+        processes.append(t)
+        log.debug("Trigger started")
+
+    if 'scan' in args.mode:
+        s = scan.Scanner(interfaces=args.interfaces, args=args)
+        s.start()
+        processes.append(s)
+        log.debug("Scanner started")
+
+    for process in processes:
+        process.join()
diff --git a/cdist/scan.py b/cdist/scan/scan.py
similarity index 90%
rename from cdist/scan.py
rename to cdist/scan/scan.py
index e2100499..fcbf1899 100644
--- a/cdist/scan.py
+++ b/cdist/scan/scan.py
@@ -50,13 +50,14 @@
 
 from multiprocessing import Process
 import os
-
-# FIXME: fail gracefully if non existent - i.e. "scapy required for scanner - please install python3-scapy"
+import logging
 from scapy.all import *
 
 # Datetime overwrites scapy.all.datetime - needs to be imported AFTER
 import datetime
 
+log = logging.getLogger("scan")
+
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
@@ -87,6 +88,7 @@ class Trigger(object):
 
     def trigger(self, interface):
         packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
+        log.debug(f"Sending request on {interface}")
         send(packet, verbose=self.verbose)
 
 class Scanner(object):
@@ -94,18 +96,18 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces=None, outdir=None):
+    def __init__(self, interfaces=None, args=None, outdir=None):
         self.interfaces = interfaces
 
         if outdir:
             self.outdir = outdir
         else:
-            self.outdir = "."
+            self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan')
 
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
             host = pkg['IPv6'].src
-            print(f"Host {host} is alive")
+            log.verbose(f"Host {host} is alive")
 
             dir = os.path.join(self.outdir, host)
             fname = os.path.join(dir, "last_seen")
@@ -118,13 +120,21 @@ class Scanner(object):
             with open(fname, "w") as fd:
                 fd.write(f"{now}\n")
 
+    def start(self):
+        self.process = Process(target=self.scan)
+        self.process.start()
+
+    def join(self):
+        self.process.join()
 
     def scan(self):
+        log.debug("Scanning - zzzzz")
         sniff(iface=self.interfaces,
               filter="icmp6",
               prn=self.handle_pkg)
 
 
+
 if __name__ == '__main__':
     t = Trigger(interfaces=["wlan0"])
     t.start()
diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org
index 718fd68c..4461be8c 100644
--- a/docs/dev/logs/2020-10-29.org
+++ b/docs/dev/logs/2020-10-29.org
@@ -32,3 +32,26 @@
      - Record when configured (successfully)
      - Record when seen
    - Enables configurations in stateless environments
+** Sample output v2020-10-29
+23:14] bridge:~% sudo  cdist scan -b -I wlan0 -vv
+VERBOSE: cdist: version 6.8.0-36-g91d99bf0
+VERBOSE: scan: Host fe80::21d:72ff:fe86:46b is alive
+VERBOSE: scan: Host fe80::ce2d:e0ff:fed4:2611 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4c1 is alive
+VERBOSE: scan: Host fe80::e2ff:f7ff:fe00:20e6 is alive
+VERBOSE: scan: Host fe80::20d:b9ff:fe49:ac11 is alive
+VERBOSE: scan: Host fe80::9e93:4eff:fe6c:c1f4 is alive
+VERBOSE: scan: Host fe80::ce32:e5ff:fe79:7ea7 is alive
+VERBOSE: scan: Host fe80::219:d2ff:feb2:2e12 is alive
+VERBOSE: scan: Host fe80::d66d:6dff:fe33:e00 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f446 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4b1 is alive
+VERBOSE: scan: Host fe80::20d:b9ff:fe4c:547d is alive
+VERBOSE: scan: Host fe80::bad8:12ff:fe65:313d is alive
+VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f6f0 is alive
+VERBOSE: scan: Host fe80::ba69:f4ff:fec5:6041 is alive
+VERBOSE: scan: Host fe80::f29f:c2ff:fe7c:275e is alive
+VERBOSE: scan: Host fe80::ba69:f4ff:fec5:8db7 is alive
+VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f863 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4bc is alive
+...

From e30ecdda535200ac75a6e774590d2e1b5e1b5b28 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 13:05:40 +0200
Subject: [PATCH 28/63] Add __uci and __uci_commit types

---
 cdist/conf/type/__uci/explorer/state          | 89 +++++++++++++++++++
 cdist/conf/type/__uci/gencode-remote          | 67 ++++++++++++++
 cdist/conf/type/__uci/man.rst                 | 77 ++++++++++++++++
 cdist/conf/type/__uci/manifest                | 40 +++++++++
 cdist/conf/type/__uci/parameter/default/state |  1 +
 .../type/__uci/parameter/default/transaction  |  1 +
 cdist/conf/type/__uci/parameter/optional      |  2 +
 .../type/__uci/parameter/required_multiple    |  1 +
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++
 cdist/conf/type/__uci_commit/man.rst          | 58 ++++++++++++
 cdist/conf/type/__uci_commit/nonparallel      |  0
 11 files changed, 357 insertions(+)
 create mode 100644 cdist/conf/type/__uci/explorer/state
 create mode 100755 cdist/conf/type/__uci/gencode-remote
 create mode 100644 cdist/conf/type/__uci/man.rst
 create mode 100755 cdist/conf/type/__uci/manifest
 create mode 100644 cdist/conf/type/__uci/parameter/default/state
 create mode 100644 cdist/conf/type/__uci/parameter/default/transaction
 create mode 100644 cdist/conf/type/__uci/parameter/optional
 create mode 100644 cdist/conf/type/__uci/parameter/required_multiple
 create mode 100755 cdist/conf/type/__uci_commit/gencode-remote
 create mode 100644 cdist/conf/type/__uci_commit/man.rst
 create mode 100644 cdist/conf/type/__uci_commit/nonparallel

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
new file mode 100644
index 00000000..2e98b606
--- /dev/null
+++ b/cdist/conf/type/__uci/explorer/state
@@ -0,0 +1,89 @@
+#!/bin/sh
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current state of the configuration option
+# The output of this explorer is one of these values:
+#   present
+#     The configuration option is present and has the value of the
+#     parameter --value.
+#   absent
+#     The configuration option is not defined.
+#   different
+#     The configuration option is present but has a different value than the
+#     parameter --value.
+#   rearranged
+#     The configuration option is present (a list) and has the same values as
+#     the parameter --value, but in a different order.
+
+RS=$(printf '\036')
+
+option=${__object_id:?}
+
+values_is=$(uci -s -N -d "${RS}" get "${option}" 2>/dev/null) || {
+	echo absent
+	exit 0
+}
+
+# strip off trailing newline
+printf '%s' "${values_is}" \
+| awk '
+BEGIN {
+	state = "present"  # assume all is fine
+}
+NR == FNR {
+	# memoize "should" state
+	should[FNR] = $0
+
+	# go to next line (important!)
+	next
+}
+
+# compare "is" state
+$0 == should[FNR] { next }
+
+FNR > length(should) {
+	# there are more "is" records than "should" -> definitely different
+	state = "different"
+	exit
+}
+
+{
+	# see if we can find the value somewhere in should
+	for (i in should) {
+		if ($0 == should[i]) {
+			# ... value found -> rearranged
+			# FIXME: Duplicate values are not properly handled here. Do they matter?
+			state = "rearranged"
+			next
+		}
+	}
+
+	state = "different"
+	exit
+}
+
+END {
+	if (FNR < length(should)) {
+		# "is" was shorter than "should" -> different
+		state = "different"
+	}
+
+	print state
+}
+' "${__object:?}/parameter/value" RS="${RS}" -
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
new file mode 100755
index 00000000..48f114fe
--- /dev/null
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -0,0 +1,67 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
+
+config=${__object_id:?}
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+case ${state_should}
+in
+	(present)
+		if in_list "${state_is}" 'present' 'rearranged'
+		then
+			# NOTE: order is ignored so rearranged is also fine.
+			exit 0
+		fi
+
+		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
+		then
+			# "should" is a list
+			if test "${state_is}" != 'absent'
+			then
+				printf "uci delete '%s'\n" "${config}"
+			fi
+
+			while read -r value
+			do
+				printf "uci add_list '%s'='%s'\n" "${config}" "${value}"
+			done <"${__object:?}/parameter/value"
+		else
+			# "should" is a scalar
+			value=$(cat "${__object:?}/parameter/value")
+			printf "uci set '%s'='%s'\n" "${config}" "${value}"
+		fi
+		;;
+	(absent)
+		if in_list "${state_is}" 'absent'
+		then
+			exit 0
+		fi
+
+		printf "uci delete '%s'\n" "${config}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
new file mode 100644
index 00000000..d23d8b2b
--- /dev/null
+++ b/cdist/conf/type/__uci/man.rst
@@ -0,0 +1,77 @@
+cdist-type__uci(7)
+==================
+
+NAME
+----
+cdist-type__uci - Manage configuration values in OpenWrt's
+Unified Configuration Interface (UCI)
+
+
+DESCRIPTION
+-----------
+This cdist type can be used to alter configuration options in OpenWrt's UCI
+system.
+
+Options can be applied in batches if the `--transaction` parameter is used.
+It is important to ensure that the `__uci_commit` object is executed before a
+new transaction is started.
+
+REQUIRED PARAMETERS
+-------------------
+value
+    The value to be set. Can be used multiple times.
+
+    Due to the way cdist handles arguments, values **must not** contain newline
+    characters.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    `present` or `absent`, defaults to `present`.
+transaction
+    The name of the transaction this option belongs to.
+    If none is given: "default" is used.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Set the system hostname
+    __uci system.@system[0].hostname --value 'OpenWrt'
+
+    # Enable NTP and NTPd (in one transaction)
+    __uci system.ntp.enabled --value 1 --transaction ntp
+    __uci system.ntp.enable_server --value 1 --transaction ntp
+    __uci system.ntp.server --transaction ntp \
+        --value '0.openwrt.pool.ntp.org' \
+        --value '1.openwrt.pool.ntp.org' \
+        --value '2.openwrt.pool.ntp.org' \
+        --value '3.openwrt.pool.ntp.org'
+    export require=__uci_commit/ntp
+
+
+SEE ALSO
+--------
+- https://openwrt.org/docs/guide-user/base-system/uci
+- :strong:`cdist-type__uci_commit`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
new file mode 100755
index 00000000..e5b0fb30
--- /dev/null
+++ b/cdist/conf/type/__uci/manifest
@@ -0,0 +1,40 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+os=$(cat "${__global:?}/explorer/os")
+
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+case ${os}
+in
+	(openwrt)
+		# okay
+		;;
+	(*)
+		printf "Your operating system (%s) is currently not supported by this type (%s)\n" "${os}" "${__type##*/}" >&2
+		printf "Please contribute an implementation for it if you can.\n" >&2
+		exit 1
+		;;
+esac
+
+
+# Make sure the changes are being commited
+require=${__object_name:?} __uci_commit "${transaction_name}"
diff --git a/cdist/conf/type/__uci/parameter/default/state b/cdist/conf/type/__uci/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__uci/parameter/default/transaction b/cdist/conf/type/__uci/parameter/default/transaction
new file mode 100644
index 00000000..4ad96d51
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/default/transaction
@@ -0,0 +1 @@
+default
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
new file mode 100644
index 00000000..ddbbba16
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -0,0 +1,2 @@
+state
+transaction
diff --git a/cdist/conf/type/__uci/parameter/required_multiple b/cdist/conf/type/__uci/parameter/required_multiple
new file mode 100644
index 00000000..6d4e1507
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/required_multiple
@@ -0,0 +1 @@
+value
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
new file mode 100755
index 00000000..bed0eefb
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -0,0 +1,21 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+echo 'uci commit'
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
new file mode 100644
index 00000000..c55d06e5
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/man.rst
@@ -0,0 +1,58 @@
+cdist-type__uci_commit(7)
+=========================
+
+NAME
+----
+cdist-type__uci_commit - Commit a UCI transaction.
+
+
+DESCRIPTION
+-----------
+This type executes the "uci commit" command on the target.
+It is usually not required to use this type. Use the `--transaction` parameter
+of `cdist-type__uci`\ (7) instead.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Commit the default transaction
+    __uci_commit default
+
+    # Commit another transaction
+    __uci_commit my_transaction
+
+
+SEE ALSO
+--------
+:strong:`cdist-type__uci`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_commit/nonparallel b/cdist/conf/type/__uci_commit/nonparallel
new file mode 100644
index 00000000..e69de29b

From 55e7b32449c9a86e34054c321b20a49795c6652f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 14:00:29 +0200
Subject: [PATCH 29/63] [type/__uci] Only generate __uci_commit if changes are
 required

---
 cdist/conf/type/__uci/manifest | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index e5b0fb30..74524513 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,9 +18,12 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; } }
 
 os=$(cat "${__global:?}/explorer/os")
 
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case ${os}
@@ -35,6 +38,25 @@ in
 		;;
 esac
 
+changes_required=false
 
-# Make sure the changes are being commited
-require=${__object_name:?} __uci_commit "${transaction_name}"
+case ${state_should}
+in
+	(present)
+		# NOTE: order is ignored so rearranged is also fine.
+		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
+		;;
+	(absent)
+		in_list "${state_is}" 'absent' || changes_required=true
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
+
+if ${changes_required}
+then
+	# Make sure the changes are being committed
+	require=${__object_name:?} __uci_commit "${transaction_name}"
+fi

From a09120977f231fe3dda1c7c07073fc7e03fc5ea0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 14:07:10 +0200
Subject: [PATCH 30/63] [type/__uci] Allow omission of --value parameter if
 --state absent

---
 cdist/conf/type/__uci/explorer/state                   | 10 +++++++++-
 cdist/conf/type/__uci/man.rst                          |  1 +
 cdist/conf/type/__uci/manifest                         |  5 +++++
 .../parameter/{required_multiple => optional_multiple} |  0
 4 files changed, 15 insertions(+), 1 deletion(-)
 rename cdist/conf/type/__uci/parameter/{required_multiple => optional_multiple} (100%)

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
index 2e98b606..6fb4b173 100644
--- a/cdist/conf/type/__uci/explorer/state
+++ b/cdist/conf/type/__uci/explorer/state
@@ -40,6 +40,14 @@ values_is=$(uci -s -N -d "${RS}" get "${option}" 2>/dev/null) || {
 	exit 0
 }
 
+if test -f "${__object:?}/parameter/value"
+then
+	should_file="${__object:?}/parameter/value"
+else
+	should_file='/dev/null'
+fi
+
+
 # strip off trailing newline
 printf '%s' "${values_is}" \
 | awk '
@@ -86,4 +94,4 @@ END {
 
 	print state
 }
-' "${__object:?}/parameter/value" RS="${RS}" -
+' "${should_file}" RS="${RS}" -
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index d23d8b2b..c6bb81ab 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -20,6 +20,7 @@ REQUIRED PARAMETERS
 -------------------
 value
     The value to be set. Can be used multiple times.
+    This parameter is allowed to be omitted if `--state` is `absent`.
 
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index 74524513..e56462e5 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -43,6 +43,11 @@ changes_required=false
 case ${state_should}
 in
 	(present)
+		test -s "${__object:?}/parameter/value" || {
+			echo 'The parameter --value is required.' >&2
+			exit 1
+		}
+
 		# NOTE: order is ignored so rearranged is also fine.
 		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
 		;;
diff --git a/cdist/conf/type/__uci/parameter/required_multiple b/cdist/conf/type/__uci/parameter/optional_multiple
similarity index 100%
rename from cdist/conf/type/__uci/parameter/required_multiple
rename to cdist/conf/type/__uci/parameter/optional_multiple

From d8f20a6a204142360e7fb7afd6603c69ae613e2d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 12 Jun 2020 19:39:19 +0200
Subject: [PATCH 31/63] [type/__uci] Implement "real" transactions using batch
 files

---
 .../__uci/{gencode-remote => gencode-local}   | 22 ++++++++++++++++---
 cdist/conf/type/__uci/man.rst                 |  3 +--
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++++++++++++++-
 3 files changed, 40 insertions(+), 6 deletions(-)
 rename cdist/conf/type/__uci/{gencode-remote => gencode-local} (79%)

diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-local
similarity index 79%
rename from cdist/conf/type/__uci/gencode-remote
rename to cdist/conf/type/__uci/gencode-local
index 48f114fe..bba5944d 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-local
@@ -20,11 +20,27 @@
 
 in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
 
+uci_cmd() {
+	printf 'printf "%s\n"' "$1"
+	shift
+	printf " '%s'" "$@"
+	printf " >>'%s'\n" "${tmpfile}"
+}
+
 config=${__object_id:?}
 
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+tmpdir="${__global:?}/tmp/__uci"
+# HACK
+mkdir -p "${tmpdir}"
+
+tmpfile="${tmpdir}/${transaction_name}.txt"
+
+
 case ${state_should}
 in
 	(present)
@@ -44,12 +60,12 @@ in
 
 			while read -r value
 			do
-				printf "uci add_list '%s'='%s'\n" "${config}" "${value}"
+				uci_cmd "add_list '%s'='%s'" "${config}" "${value}"
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
 			value=$(cat "${__object:?}/parameter/value")
-			printf "uci set '%s'='%s'\n" "${config}" "${value}"
+			uci_cmd "set '%s'='%s'" "${config}" "${value}"
 		fi
 		;;
 	(absent)
@@ -58,7 +74,7 @@ in
 			exit 0
 		fi
 
-		printf "uci delete '%s'\n" "${config}"
+		uci_cmd "delete '%s'" "${config}"
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index c6bb81ab..2f64355b 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -13,8 +13,7 @@ This cdist type can be used to alter configuration options in OpenWrt's UCI
 system.
 
 Options can be applied in batches if the `--transaction` parameter is used.
-It is important to ensure that the `__uci_commit` object is executed before a
-new transaction is started.
+
 
 REQUIRED PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index bed0eefb..ac74e5e4 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -18,4 +18,23 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-echo 'uci commit'
+transaction_name=${__object_id:?}
+batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
+
+test -s "${batchfile}" || exit 0
+
+cat <<'EOF'
+rollback() {
+	uci changes \
+	| sed -e 's/\..*$//' -e 's/^-//' \
+	| while read -r package
+	  do
+		  uci revert "${package}"
+	  done
+}
+
+EOF
+
+echo "uci batch <<'EOF' && uci commit || rollback"
+cat "${batchfile}"
+echo 'EOF'

From d3574b2d3e807ae436afd90aa1ed9d01b78ca6a3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 16:43:16 +0200
Subject: [PATCH 32/63] [type/__uci] Send messages when options are set to be
 altered

---
 cdist/conf/type/__uci/gencode-local | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index bba5944d..ba27dc5e 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -53,6 +53,8 @@ in
 		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
 		then
 			# "should" is a list
+			printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+
 			if test "${state_is}" != 'absent'
 			then
 				printf "uci delete '%s'\n" "${config}"
@@ -64,6 +66,8 @@ in
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
+			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
 			value=$(cat "${__object:?}/parameter/value")
 			uci_cmd "set '%s'='%s'" "${config}" "${value}"
 		fi
@@ -74,6 +78,7 @@ in
 			exit 0
 		fi
 
+		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
 		uci_cmd "delete '%s'" "${config}"
 		;;
 	(*)

From 3a3be3631010b1bf17d084c775e306f11429f5d9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 12 Jun 2020 19:53:26 +0200
Subject: [PATCH 33/63] [type/__uci_commit] Send message on commit of a
 transaction

---
 cdist/conf/type/__uci_commit/gencode-remote | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index ac74e5e4..0332d7e0 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,6 +23,8 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
+printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
+
 cat <<'EOF'
 rollback() {
 	uci changes \

From e7369a1f9957ac31830a69de8101c434666da5da Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 17 Jun 2020 13:00:24 +0200
Subject: [PATCH 34/63] [type/__uci_commit] Abort if uncommited changes are
 present on the target

---
 cdist/conf/type/__uci_commit/explorer/changes | 22 ++++++++++++++++
 cdist/conf/type/__uci_commit/gencode-remote   | 25 +++++++++++++------
 2 files changed, 39 insertions(+), 8 deletions(-)
 create mode 100644 cdist/conf/type/__uci_commit/explorer/changes

diff --git a/cdist/conf/type/__uci_commit/explorer/changes b/cdist/conf/type/__uci_commit/explorer/changes
new file mode 100644
index 00000000..2690def7
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/explorer/changes
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer outputs the uncommited UCI changes on the target.
+
+uci changes
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index 0332d7e0..dd1d839c 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,20 +23,29 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
+if test -s "${__object:?}/explorer/changes"
+then
+	echo 'Uncommited UCI changes were found on the target:'
+	cat "${__object:?}/explorer/changes"
+	echo
+	echo 'This can be caused by manual changes or due to a previous failed run.'
+	echo 'Please investigate the situation, revert or commit the changes, and try again.'
+	exit 1
+fi >&2
+
 printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
 
-cat <<'EOF'
+cat <<CODE
 rollback() {
-	uci changes \
-	| sed -e 's/\..*$//' -e 's/^-//' \
+	uci changes \\
+	| sed -e 's/\..*\$//' -e 's/^-//' \\
 	| while read -r package
 	  do
-		  uci revert "${package}"
+		  uci revert "\${package}"
 	  done
 }
 
+uci batch <<'EOF' && uci commit || rollback
+$(cat "${batchfile}")
 EOF
-
-echo "uci batch <<'EOF' && uci commit || rollback"
-cat "${batchfile}"
-echo 'EOF'
+CODE

From cc599dab15ed5c8761b3bcdfb113403fa6d4bc77 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 5 Jul 2020 10:25:36 +0200
Subject: [PATCH 35/63] [type/__uci_commit] Move uncommited changes check from
 explorer to code-remote

This is done to prevent false positives/negatives (see NOTE in code)
---
 cdist/conf/type/__uci_commit/explorer/changes | 22 -------------------
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++++++++++-----
 2 files changed, 15 insertions(+), 28 deletions(-)
 delete mode 100644 cdist/conf/type/__uci_commit/explorer/changes

diff --git a/cdist/conf/type/__uci_commit/explorer/changes b/cdist/conf/type/__uci_commit/explorer/changes
deleted file mode 100644
index 2690def7..00000000
--- a/cdist/conf/type/__uci_commit/explorer/changes
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/sh
-#
-# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-# This explorer outputs the uncommited UCI changes on the target.
-
-uci changes
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index dd1d839c..504f7c3b 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,19 +23,28 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
-if test -s "${__object:?}/explorer/changes"
+printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
+
+# NOTE: Uncommited changes are checked in code-remote instead of in an explorer
+#       because in cdist there is no interlocking between explorers and code
+#       execution.
+#       Checking for uncommited changes in an explorer leaves a time slot in
+#       which changes made are not detected by the code.
+#       Furthermore an explorer running concurrently with another transactions
+#       code-remote could lead to a false positive.
+
+cat <<CODE
+changes=\$(uci changes)
+
+if test -n "\${changes}"
 then
 	echo 'Uncommited UCI changes were found on the target:'
-	cat "${__object:?}/explorer/changes"
-	echo
+	printf '%s\n\n' "\${changes}"
 	echo 'This can be caused by manual changes or due to a previous failed run.'
 	echo 'Please investigate the situation, revert or commit the changes, and try again.'
 	exit 1
 fi >&2
 
-printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
-
-cat <<CODE
 rollback() {
 	uci changes \\
 	| sed -e 's/\..*\$//' -e 's/^-//' \\

From 3ef638a61123069953a9e24044b6c94eba0b3b6b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 20 Jul 2020 12:12:15 +0200
Subject: [PATCH 36/63] [type/__uci_commit] Fail when uci(1) reports errors

---
 cdist/conf/type/__uci_commit/gencode-remote | 37 ++++++++++++++++-----
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index 504f7c3b..e5c93966 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -45,16 +45,37 @@ then
 	exit 1
 fi >&2
 
-rollback() {
-	uci changes \\
-	| sed -e 's/\..*\$//' -e 's/^-//' \\
-	| while read -r package
-	  do
-		  uci revert "\${package}"
-	  done
+check_errors() {
+	# reads stdin and forwards non-empty lines to stderr.
+	# returns 0 if stdin is empty, else 1.
+	! grep -e . >&2
 }
 
-uci batch <<'EOF' && uci commit || rollback
+commit() {
+	uci commit
+}
+
+rollback() {
+	echo >&2
+	echo 'An error occurred when trying to commit transaction ${transaction_name}!' >&2
+
+	uci changes \\
+	| sed -e 's/^-//' -e 's/\..*\$//' \\
+	| sort -u \\
+	| while read -r _package
+	  do
+		  uci revert "\${_package}"
+		  echo "\${_package}"  # for logging
+	  done \\
+	| awk '
+	  BEGIN { printf "Reverted changes in: " }
+	  { printf "%s%s", (FNR > 1 ? ", " : ""), \$0 }
+	  END { printf "\\n" }' >&2
+
+	return 1
+}
+
+uci batch <<'EOF' 2>&1 | check_errors && commit || rollback
 $(cat "${batchfile}")
 EOF
 CODE

From 4da39681188493981ec3c55ea918787195655972 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 20:04:04 +0200
Subject: [PATCH 37/63] [type/__uci_section] Add type

---
 cdist/conf/type/__uci_section/explorer/match  |  68 ++++++++++
 .../conf/type/__uci_section/explorer/options  |  28 ++++
 cdist/conf/type/__uci_section/explorer/type   |  25 ++++
 cdist/conf/type/__uci_section/man.rst         |  76 +++++++++++
 cdist/conf/type/__uci_section/manifest        | 122 ++++++++++++++++++
 .../__uci_section/parameter/default/state     |   1 +
 .../parameter/default/transaction             |   1 +
 .../type/__uci_section/parameter/optional     |   4 +
 .../__uci_section/parameter/optional_multiple |   1 +
 9 files changed, 326 insertions(+)
 create mode 100644 cdist/conf/type/__uci_section/explorer/match
 create mode 100644 cdist/conf/type/__uci_section/explorer/options
 create mode 100644 cdist/conf/type/__uci_section/explorer/type
 create mode 100644 cdist/conf/type/__uci_section/man.rst
 create mode 100755 cdist/conf/type/__uci_section/manifest
 create mode 100644 cdist/conf/type/__uci_section/parameter/default/state
 create mode 100644 cdist/conf/type/__uci_section/parameter/default/transaction
 create mode 100644 cdist/conf/type/__uci_section/parameter/optional
 create mode 100644 cdist/conf/type/__uci_section/parameter/optional_multiple

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
new file mode 100644
index 00000000..9cf1ea3e
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -0,0 +1,68 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer the "prefix" of the section matching --match.
+
+squote_values() {
+	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
+	    -e "/='.*'$/"'!{s/=/='\''/;s/$/'\''/}'
+}
+
+RS=$(printf '\036')
+
+if ! test -e "${__object:?}/parameter/match"
+then
+	if echo "${__object_id:?}" | grep -qvE '^[^.]+\.[^.]+$'
+	then
+		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
+		exit 1
+	fi
+
+	# If no --match is given, we take the __object_id as the section identifier.
+	echo "${__object_id:?}"
+	exit 0
+fi
+
+test -s "${__object:?}/parameter/match" \
+&& test -s "${__object:?}/parameter/type" \
+|| {
+	echo 'Parameters --match and --type must be used together.' >&2
+	exit 1
+}
+
+# Find by match
+match=$(cat "${__object:?}/parameter/match")
+sect_type_filter=$(cat "${__object:?}/parameter/type")
+
+package_filter=${sect_type_filter%%.*}
+section_filter=${sect_type_filter##*.}
+regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
+
+matched_sections=$(
+	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
+	| grep -e "${regex}" \
+	| sed -e 's/\.[^.]*=.*$//')
+
+if test "$(echo "${matched_sections}" | wc -l)" -gt 1
+then
+	printf 'Found multiple matching sections:\n%s\n' "${matched_sections}" >&2
+	exit 1
+fi
+
+echo "${matched_sections}"
diff --git a/cdist/conf/type/__uci_section/explorer/options b/cdist/conf/type/__uci_section/explorer/options
new file mode 100644
index 00000000..67b4f83f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/options
@@ -0,0 +1,28 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current options of the configuration section.
+
+RS=$(printf '\036')
+
+section=$("${__type_explorer:?}/match")
+test -n "${section}" || exit 0
+
+uci -s -N -d "${RS}" show "${section}" 2>/dev/null \
+| grep -v -e "^${section}=" || true
diff --git a/cdist/conf/type/__uci_section/explorer/type b/cdist/conf/type/__uci_section/explorer/type
new file mode 100644
index 00000000..1675c2e0
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/type
@@ -0,0 +1,25 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current section type.
+
+section=$("${__type_explorer:?}/match")
+test -n "${section}" || exit 0
+
+uci -s -N get "${section}" 2>/dev/null || true
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
new file mode 100644
index 00000000..3468bfc3
--- /dev/null
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -0,0 +1,76 @@
+cdist-type__uci_section(7)
+==========================
+
+NAME
+----
+cdist-type__uci_section - Manage configuration sections in OpenWrt's
+Unified Configuration Interface (UCI)
+
+
+DESCRIPTION
+-----------
+This cdist type can be used to replace whole configuration sections in OpenWrt's
+UCI system.
+It can be thought of as syntactic sugar for `cdist-type__uci`\ (7), as this type
+will generate the required `__uci` objects to make the section contain exactly
+the options specified via ``--option``.
+
+Since many default UCI sections are unnamed, this type allows to find the
+matching section by one of its options using the ``--match`` parameter.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+match
+    Allows to find a section to "replace" through one of its parameters.
+    The value to this parameter is a ``<option>=<string>`` string.
+option
+    An option that should be present in the section.
+    This parameter can be used multiple times to specify multiple options.
+    The value to this parameter is a ``<option>=<string>`` string.
+
+    Lists can be expressed by repeatedly using the same key.
+state
+    `present` or `absent`, defaults to `present`.
+transaction
+    The name of the transaction this option belongs to.
+    The value will be forwarded to `cdist-type__uci`\ (7).
+type
+    The type of the section in the format: ``<config>.<section-type>``
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # TODO
+    __uci_section ...
+
+
+SEE ALSO
+--------
+:strong:`cdist-type__uci`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
new file mode 100755
index 00000000..e16277ba
--- /dev/null
+++ b/cdist/conf/type/__uci_section/manifest
@@ -0,0 +1,122 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+grep_line() { echo "$2" | grep -qxF "$1"; }
+unquote_lines() {
+	sed -e '/^".*"$/{s/^"//;s/"$//}' \
+	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
+}
+
+append_values() {
+	while read -r _value
+	do
+		set -- "$@" --value "${_value}"
+	done
+	unset _value
+	"$@" </dev/null
+}
+
+section=$(cat "${__object:?}/explorer/match")
+
+state_should=$(cat "${__object:?}/parameter/state")
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+case $state_should
+in
+	(present)
+		test -f "${__object:?}/parameter/type" || {
+			echo 'Parameter --type is required.' >&2
+			exit 1
+		}
+		type_is=$(cat "${__object:?}/explorer/type")
+		type_should=$(cat "${__object:?}/parameter/type")
+
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
+		if test -n "${type_is}"
+		then
+			if test "${type_is}" != "${type_should##*.}"
+			then
+				# Check if section type matches (section exists and --type provided)
+				printf 'Section type "%s" does not match --type "%s".\n' \
+					"${type_is}" "${type_should}" >&2
+				exit 1
+			fi
+			sect_type=${type_is}
+		else
+			sect_type=${type_should##*.}
+		fi
+
+		if test -z "${section}"
+		then
+			# No section exists and --match was used.
+			# So we generate a new section identifier from $__object_id.
+			case ${__object_id:?}
+			in
+				(*.*) section=${__object_id:?} ;;
+				(*) section="${type_should%%.*}.${__object_id:?}" ;;
+			esac
+		fi
+
+		# Make sure the section itself is present
+		__uci "${section}" --state present --transaction "${transaction_name}" \
+			--value "${sect_type}"
+		export require=__uci/"${section}"
+
+		# Delete options not in "should"
+		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		| while read -r optname
+		  do
+			  if ! grep_line "${optname}" "${optnames_should}"
+			  then
+				  __uci "${section}.${optname}" --state absent \
+					  --transaction "${transaction_name}" </dev/null
+			  fi
+		  done
+
+		# Set "should" options
+		echo "${optnames_should}" \
+		| while read -r optname
+		  do
+			  test -n "${optname}" || continue  # ignore empty lines
+
+			  grep "^${optname}=" <"${__object:?}/parameter/option" \
+			  | sed -e 's/^.*=//' \
+			  | unquote_lines \
+			  | append_values \
+				    __uci "${section}.${optname}" --state present \
+					    --transaction "${transaction_name}"
+		  done
+		;;
+	(absent)
+		# if explorer found no section there is nothing to delete
+		test -n "${section}" || exit 0
+
+		__uci "${section}" --state absent --transaction "${transaction_name}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__uci_section/parameter/default/state b/cdist/conf/type/__uci_section/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__uci_section/parameter/default/transaction b/cdist/conf/type/__uci_section/parameter/default/transaction
new file mode 100644
index 00000000..4ad96d51
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/default/transaction
@@ -0,0 +1 @@
+default
diff --git a/cdist/conf/type/__uci_section/parameter/optional b/cdist/conf/type/__uci_section/parameter/optional
new file mode 100644
index 00000000..8b956bae
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/optional
@@ -0,0 +1,4 @@
+match
+state
+transaction
+type
diff --git a/cdist/conf/type/__uci_section/parameter/optional_multiple b/cdist/conf/type/__uci_section/parameter/optional_multiple
new file mode 100644
index 00000000..01925a15
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/optional_multiple
@@ -0,0 +1 @@
+option

From 179815b5e93bd51b6757d92d14470808c5bee746 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 21 Jun 2020 18:22:54 +0200
Subject: [PATCH 38/63] [type/__uci_section] Ignore SC2015 error (notabug)

---
 cdist/conf/type/__uci_section/explorer/match | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index 9cf1ea3e..a3832bff 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -39,8 +39,8 @@ then
 	exit 0
 fi
 
-test -s "${__object:?}/parameter/match" \
-&& test -s "${__object:?}/parameter/type" \
+# shellcheck disable=SC2015
+test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type" \
 || {
 	echo 'Parameters --match and --type must be used together.' >&2
 	exit 1

From d453d964e1e0827dad217e1dd424dbaf82e434fc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 20 Jul 2020 11:21:06 +0200
Subject: [PATCH 39/63] [type/__uci_section] Fix in section matching

---
 cdist/conf/type/__uci_section/explorer/match | 23 +++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index a3832bff..a5619d4e 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -17,7 +17,8 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-# This explorer the "prefix" of the section matching --match.
+# This explorer determines the "prefix" of the --type section matching --match
+# if set, or __object_id otherwise.
 
 squote_values() {
 	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
@@ -28,7 +29,7 @@ RS=$(printf '\036')
 
 if ! test -e "${__object:?}/parameter/match"
 then
-	if echo "${__object_id:?}" | grep -qvE '^[^.]+\.[^.]+$'
+	if ! echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$'
 	then
 		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
 		exit 1
@@ -47,16 +48,28 @@ test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type
 }
 
 # Find by match
-match=$(cat "${__object:?}/parameter/match")
-sect_type_filter=$(cat "${__object:?}/parameter/type")
+# NOTE: Apart from section types all values are printed in single quotes by UCI.
+match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
 
+if ! grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' "${__object:?}/parameter/type"
+then
+	echo 'Section types are a package name and section type separated by a "." (period).' >&2
+	exit 1
+fi
+
+sect_type_filter=$(cat "${__object:?}/parameter/type")
 package_filter=${sect_type_filter%%.*}
-section_filter=${sect_type_filter##*.}
+section_filter=${sect_type_filter#*.}
+# FIXME: Does not work with named sections
 regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
 
 matched_sections=$(
 	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
 	| grep -e "${regex}" \
+	| while read -r _line
+	  do
+		  if test "${_line#*=}" = "${match#*=}"; then echo "${_line}"; fi
+	  done \
 	| sed -e 's/\.[^.]*=.*$//')
 
 if test "$(echo "${matched_sections}" | wc -l)" -gt 1

From f782a5a370a956995f444a677ecffe7f1564764d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 23 Oct 2020 11:45:37 +0200
Subject: [PATCH 40/63] [type/__uci] Refactor to do proper quoting of UCI
 commands

---
 cdist/conf/type/__uci/gencode-local | 40 ++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index ba27dc5e..44bf9c4f 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -18,15 +18,37 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
+in_list() {
+	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
+}
+
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
 
 uci_cmd() {
-	printf 'printf "%s\n"' "$1"
-	shift
-	printf " '%s'" "$@"
-	printf " >>'%s'\n" "${tmpfile}"
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	# Will output a shell command to be executed locally to add the given
+	# command to $tmpfile.
+	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
+	printf ' >>%s\n' "$(quote "${tmpfile}")"
 }
 
+
 config=${__object_id:?}
 
 state_is=$(cat "${__object:?}/explorer/state")
@@ -57,19 +79,19 @@ in
 
 			if test "${state_is}" != 'absent'
 			then
-				printf "uci delete '%s'\n" "${config}"
+				uci_cmd delete "${config}"
 			fi
 
 			while read -r value
 			do
-				uci_cmd "add_list '%s'='%s'" "${config}" "${value}"
+				uci_cmd add_list "${config}"="${value}"
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
 			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
 
 			value=$(cat "${__object:?}/parameter/value")
-			uci_cmd "set '%s'='%s'" "${config}" "${value}"
+			uci_cmd set "${config}"="${value}"
 		fi
 		;;
 	(absent)
@@ -79,7 +101,7 @@ in
 		fi
 
 		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
-		uci_cmd "delete '%s'" "${config}"
+		uci_cmd delete "${config}"
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2

From 3a6b0851459f08ad56f1c6a138b168fedfba993e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 23 Oct 2020 22:03:25 +0200
Subject: [PATCH 41/63] [type/__uci] Check __object_id for syntax errors

---
 cdist/conf/type/__uci/gencode-local | 45 +++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index 44bf9c4f..af91a698 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -48,8 +48,53 @@ uci_cmd() {
 	printf ' >>%s\n' "$(quote "${tmpfile}")"
 }
 
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+uci_validate_tuple() (
+	tok=${1:?}
+	case $tok
+	in
+		(*.*.*)
+			# check option
+			option=${tok##*.}
+			uci_validate_name "${option}" || {
+				printf 'Invalid option: %s\n' "${option}" >&2
+				return 1
+			}
+			tok=${tok%.*}
+			;;
+		(*.*)
+			# no option (section definition)
+			;;
+		(*)
+			printf 'Invalid tuple: %s\n' "$1" >&2
+			return 1
+			;;
+	esac
+
+	case ${tok#*.}
+	in
+		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
+		(*)  section=${tok#*.} ;;
+	esac
+	uci_validate_name "${section}" || {
+		printf 'Invalid section: %s\n' "${1#*.}" >&2
+		return 1
+	}
+
+	config=${tok%%.*}
+	uci_validate_name "${config}" || {
+		printf 'Invalid config: %s\n' "${config}" >&2
+		return 1
+	}
+)
+
 
 config=${__object_id:?}
+uci_validate_tuple "${config}"
 
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")

From c37253b852477a0a22568643bad461d20b58706f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 22 Oct 2020 20:32:48 +0200
Subject: [PATCH 42/63] [type/__uci_section] Check __object_id for syntax
 errors

---
 cdist/conf/type/__uci_section/manifest | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index e16277ba..1a59b970 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -33,6 +33,32 @@ append_values() {
 	"$@" </dev/null
 }
 
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+## Check section name and error if invalid!
+case ${__object_id:?}
+in
+	(*.*)
+		uci_validate_name "${__object_id%%.*}" || {
+			printf 'Invalid package name: %s\n' "${__object_id%%.*}" >&2
+			exit 1
+		}
+		uci_validate_name "${__object_id#*.}" || {
+			printf 'Invalid section name: %s\n' "${__object_id#*.}" >&2
+			exit 1
+		}
+		;;
+	(*)
+		uci_validate_name "${__object_id:?}" || {
+			printf 'Invalid section name: %s\n' "${__object_id:?}" >&2
+			exit 1
+		}
+		;;
+esac
+
 section=$(cat "${__object:?}/explorer/match")
 
 state_should=$(cat "${__object:?}/parameter/state")

From fe26c119b52c3822f009fe9aeb1b52b60139e33a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 10:01:49 +0200
Subject: [PATCH 43/63] [type/__uci*] Update man pages

---
 cdist/conf/type/__uci/man.rst         | 15 ++++----
 cdist/conf/type/__uci_commit/man.rst  | 12 ++++---
 cdist/conf/type/__uci_section/man.rst | 51 ++++++++++++++++++++-------
 3 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 2f64355b..cf184f9d 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -3,23 +3,22 @@ cdist-type__uci(7)
 
 NAME
 ----
-cdist-type__uci - Manage configuration values in OpenWrt's
-Unified Configuration Interface (UCI)
+cdist-type__uci - Manage configuration values in UCI
 
 
 DESCRIPTION
 -----------
-This cdist type can be used to alter configuration options in OpenWrt's UCI
-system.
+This cdist type can be used to alter configuration options in OpenWrt's
+Unified Configuration Interface (UCI) system.
 
-Options can be applied in batches if the `--transaction` parameter is used.
+Options can be applied in batches if the ``--transaction`` parameter is used.
 
 
 REQUIRED PARAMETERS
 -------------------
 value
     The value to be set. Can be used multiple times.
-    This parameter is allowed to be omitted if `--state` is `absent`.
+    This parameter is ignored if ``--state`` is ``absent``.
 
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
@@ -28,10 +27,10 @@ value
 OPTIONAL PARAMETERS
 -------------------
 state
-    `present` or `absent`, defaults to `present`.
+    ``present`` or ``absent``, defaults to ``present``.
 transaction
     The name of the transaction this option belongs to.
-    If none is given: "default" is used.
+    If none is given: ``default`` is used.
 
 
 BOOLEAN PARAMETERS
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
index c55d06e5..ea6611eb 100644
--- a/cdist/conf/type/__uci_commit/man.rst
+++ b/cdist/conf/type/__uci_commit/man.rst
@@ -3,14 +3,16 @@ cdist-type__uci_commit(7)
 
 NAME
 ----
-cdist-type__uci_commit - Commit a UCI transaction.
+cdist-type__uci_commit - Commit UCI transactions
 
 
 DESCRIPTION
 -----------
-This type executes the "uci commit" command on the target.
-It is usually not required to use this type. Use the `--transaction` parameter
-of `cdist-type__uci`\ (7) instead.
+This type executes the ``uci commit`` command on the target with the commands
+queued in a transaction.
+It is usually not required to use this type. Use the ``--transaction`` parameter
+of :strong:`cdist-type__uci`\ (7) and :strong:`cdist-type__uci_section`\ (7)
+instead.
 
 
 REQUIRED PARAMETERS
@@ -42,7 +44,7 @@ EXAMPLES
 
 SEE ALSO
 --------
-:strong:`cdist-type__uci`\ (7)
+:strong:`cdist-type__uci`\ (7), :strong:`cdist-type__uci_section`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 3468bfc3..86729316 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -3,21 +3,23 @@ cdist-type__uci_section(7)
 
 NAME
 ----
-cdist-type__uci_section - Manage configuration sections in OpenWrt's
-Unified Configuration Interface (UCI)
+cdist-type__uci_section - Manage configuration sections in UCI
 
 
 DESCRIPTION
 -----------
 This cdist type can be used to replace whole configuration sections in OpenWrt's
-UCI system.
-It can be thought of as syntactic sugar for `cdist-type__uci`\ (7), as this type
-will generate the required `__uci` objects to make the section contain exactly
-the options specified via ``--option``.
+Unified Configuration Interface (UCI) system.
+It can be thought of as syntactic sugar for :strong:`cdist-type__uci`\ (7),
+as this type will generate the required `__uci` objects to make the section
+contain exactly the options specified using ``--option``.
 
 Since many default UCI sections are unnamed, this type allows to find the
 matching section by one of its options using the ``--match`` parameter.
 
+**NOTE:** Options already present on the target and not listed in ``--option``
+or ``--list`` will be deleted.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -28,18 +30,18 @@ OPTIONAL PARAMETERS
 -------------------
 match
     Allows to find a section to "replace" through one of its parameters.
-    The value to this parameter is a ``<option>=<string>`` string.
+    The value to this parameter is a ``<option>=<value>`` string.
 option
     An option that should be present in the section.
     This parameter can be used multiple times to specify multiple options.
-    The value to this parameter is a ``<option>=<string>`` string.
+    The value to this parameter is a ``<option>=<value>`` string.
 
     Lists can be expressed by repeatedly using the same key.
 state
-    `present` or `absent`, defaults to `present`.
+    ``present`` or ``absent``, defaults to ``present``.
 transaction
     The name of the transaction this option belongs to.
-    The value will be forwarded to `cdist-type__uci`\ (7).
+    The value will be forwarded to :strong:`cdist-type__uci`\ (7).
 type
     The type of the section in the format: ``<config>.<section-type>``
 
@@ -54,13 +56,36 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # TODO
-    __uci_section ...
+    # Configure the dropbear daemon
+    __uci_section dropbear --type dropbear.dropbear --transaction sshd \
+        --match Port=22 --option Port=22 \
+        --option PasswordAuth=off \
+        --option RootPasswordAuth=off
+
+    # Block SSH access from the guest network
+    __uci_section firewall.block_ssh_from_guest --type firewall.rule \
+        --transaction fwrules \
+        --option name='Block-SSH-Access-from-Guest' \
+        --option src='guest' \
+        --option proto='tcp' \
+        --option dest_port='22' \
+        --option target='REJECT'
+
+    # Configure a Wi-Fi access point
+    __uci_section wireless.default_radio0 --type wireless.wifi-iface \
+        --transaction wifi \
+        --option device='radio0' \
+        --option mode='ap' \
+        --option network='wlan' \
+        --option ssid='mywifi' \
+        --option encryption="psk2' \
+        --option key='hunter2'
 
 
 SEE ALSO
 --------
-:strong:`cdist-type__uci`\ (7)
+- https://openwrt.org/docs/guide-user/base-system/uci
+- :strong:`cdist-type__uci`\ (7)
 
 
 AUTHORS

From 0840afce03c50824278d64cd204b86e79077b48e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 12:57:40 +0200
Subject: [PATCH 44/63] [type/__uci] Add --type parameter

---
 cdist/conf/type/__uci/gencode-local      |  54 ++++++++----
 cdist/conf/type/__uci/gencode-remote     | 101 +++++++++++++++++++++++
 cdist/conf/type/__uci/man.rst            |   9 +-
 cdist/conf/type/__uci/parameter/optional |   1 +
 4 files changed, 146 insertions(+), 19 deletions(-)
 create mode 100644 cdist/conf/type/__uci/gencode-remote

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index af91a698..34482a01 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -117,27 +117,45 @@ in
 			exit 0
 		fi
 
-		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
-		then
-			# "should" is a list
-			printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+		# Determine type
+		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
+		case ${type}
+		in
+			(option|list) ;;
+			('')
+				# Guess type by the number of values
+				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
+					 && type=list \
+					 || type=option
+				;;
+			(*)
+				printf 'Invalid --type: %s\n' "${type}" >&2
+				exit 1
+				;;
+		esac
 
-			if test "${state_is}" != 'absent'
-			then
-				uci_cmd delete "${config}"
-			fi
+		case ${type}
+		in
+			(list)
+				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
 
-			while read -r value
-			do
-				uci_cmd add_list "${config}"="${value}"
-			done <"${__object:?}/parameter/value"
-		else
-			# "should" is a scalar
-			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+				if test "${state_is}" != 'absent'
+				then
+					uci_cmd delete "${config}"
+				fi
 
-			value=$(cat "${__object:?}/parameter/value")
-			uci_cmd set "${config}"="${value}"
-		fi
+				while read -r value
+				do
+					uci_cmd add_list "${config}"="${value}"
+				done <"${__object:?}/parameter/value"
+				;;
+			(option)
+				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
+				value=$(cat "${__object:?}/parameter/value")
+				uci_cmd set "${config}"="${value}"
+				;;
+		esac
 		;;
 	(absent)
 		if in_list "${state_is}" 'absent'
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
new file mode 100644
index 00000000..348d15a1
--- /dev/null
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -0,0 +1,101 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+config=${__object_id:?}
+uci_validate_tuple "${config}"
+
+
+case ${state_should}
+in
+	(present)
+		if in_list "${state_is}" 'present' 'rearranged'
+		then
+			# NOTE: order is ignored so rearranged is also fine.
+			exit 0
+		fi
+
+		# Determine type
+		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
+		case ${type}
+		in
+			(option|list) ;;
+			('')
+				# Guess type by the number of values
+				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
+					 && type=list \
+					 || type=option
+				;;
+			(*)
+				printf 'Invalid --type: %s\n' "${type}" >&2
+				exit 1
+				;;
+		esac
+
+		case ${type}
+		in
+			(list)
+				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+
+				if test "${state_is}" != 'absent'
+				then
+					uci_cmd delete "${config}"
+				fi
+
+				while read -r value
+				do
+					uci_cmd add_list "${config}"="${value}"
+				done <"${__object:?}/parameter/value"
+				;;
+			(option)
+				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
+				value=$(cat "${__object:?}/parameter/value")
+				uci_cmd set "${config}"="${value}"
+				;;
+		esac
+		;;
+	(absent)
+		if in_list "${state_is}" 'absent'
+		then
+			exit 0
+		fi
+
+		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
+		uci_cmd delete "${config}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index cf184f9d..9d33d618 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -31,6 +31,10 @@ state
 transaction
     The name of the transaction this option belongs to.
     If none is given: ``default`` is used.
+type
+    If the type should generate an option or a list.
+    One of: ``option`` or ``list``.
+    Defaults to auto-detect based on the number of ``--value`` parameters.
 
 
 BOOLEAN PARAMETERS
@@ -46,10 +50,13 @@ EXAMPLES
     # Set the system hostname
     __uci system.@system[0].hostname --value 'OpenWrt'
 
+    # Set DHCP option 252: tell DHCP clients to not ask for proxy information.
+    __uci dhcp.lan.dhcp_option --type list --value '252,"\n"'
+
     # Enable NTP and NTPd (in one transaction)
     __uci system.ntp.enabled --value 1 --transaction ntp
     __uci system.ntp.enable_server --value 1 --transaction ntp
-    __uci system.ntp.server --transaction ntp \
+    __uci system.ntp.server --type list --transaction ntp \
         --value '0.openwrt.pool.ntp.org' \
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
index ddbbba16..2703b3df 100644
--- a/cdist/conf/type/__uci/parameter/optional
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -1,2 +1,3 @@
 state
 transaction
+type

From 49e867fab4c1bcb6549079e735abec2149e42cf6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 15:59:25 +0200
Subject: [PATCH 45/63] [type/__uci_section] Add more parameter checks

---
 cdist/conf/type/__uci_section/manifest | 42 +++++++++++++++++++-------
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 1a59b970..b5fa4a3b 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -19,11 +19,15 @@
 #
 
 grep_line() { echo "$2" | grep -qxF "$1"; }
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
+}
+validate_options() { grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"; }
 unquote_lines() {
 	sed -e '/^".*"$/{s/^"//;s/"$//}' \
 	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
 }
-
 append_values() {
 	while read -r _value
 	do
@@ -32,12 +36,23 @@ append_values() {
 	unset _value
 	"$@" </dev/null
 }
-
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+print_errors() {
+	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
+		BEGIN {
+			if (getline) {
+				print prefix
+				print
+				rc = 1
+			}
+		}
+		{ print }
+		END {
+			if (rc && suffix) print suffix
+			exit rc
+		}' >&2
 }
 
+
 ## Check section name and error if invalid!
 case ${__object_id:?}
 in
@@ -74,12 +89,6 @@ in
 		type_is=$(cat "${__object:?}/explorer/type")
 		type_should=$(cat "${__object:?}/parameter/type")
 
-		if test -f "${__object:?}/parameter/option"
-		then
-			optnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
-		fi
-
 		if test -n "${type_is}"
 		then
 			if test "${type_is}" != "${type_should##*.}"
@@ -105,6 +114,17 @@ in
 			esac
 		fi
 
+		# Check options for syntax errors
+		validate_options "${__object:?}/parameter/object" \
+		| print_errors 'Found erroneous options in arguments:'
+
+		# Collect option names
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
 		# Make sure the section itself is present
 		__uci "${section}" --state present --transaction "${transaction_name}" \
 			--value "${sect_type}"

From b99ca3cbdf7c761db05598e31e9788e5e4e81cc6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 16:01:02 +0200
Subject: [PATCH 46/63] [type/__uci_section] Split up --option and --list

---
 cdist/conf/type/__uci_section/man.rst         | 19 +++++++-
 cdist/conf/type/__uci_section/manifest        | 48 ++++++++++++-------
 .../__uci_section/parameter/optional_multiple |  1 +
 3 files changed, 50 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 86729316..b8829433 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -28,15 +28,21 @@ None.
 
 OPTIONAL PARAMETERS
 -------------------
+list
+    An option that is part of a list and should be present in the section (as
+    part of a list).  Lists with multiple options can be expressed by using the
+    same ``<option>`` repeatedly.
+
+    The value to this parameter is a ``<option>=<value>`` string.
 match
     Allows to find a section to "replace" through one of its parameters.
+
     The value to this parameter is a ``<option>=<value>`` string.
 option
     An option that should be present in the section.
     This parameter can be used multiple times to specify multiple options.
-    The value to this parameter is a ``<option>=<value>`` string.
 
-    Lists can be expressed by repeatedly using the same key.
+    The value to this parameter is a ``<option>=<value>`` string.
 state
     ``present`` or ``absent``, defaults to ``present``.
 transaction
@@ -62,6 +68,15 @@ EXAMPLES
         --option PasswordAuth=off \
         --option RootPasswordAuth=off
 
+    # Define a firewall zone comprised of lan and wlan networks
+    __uci_section firewall.internal --type firewall.zone --transaction fw \
+        --option name='internal' \
+        --list network='lan' \
+        --list network='wlan' \
+        --option input='ACCEPT' \
+        --option output='ACCEPT' \
+        --option forward='ACCEPT'
+
     # Block SSH access from the guest network
     __uci_section firewall.block_ssh_from_guest --type firewall.rule \
         --transaction fwrules \
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index b5fa4a3b..a7865315 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,7 +18,14 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-grep_line() { echo "$2" | grep -qxF "$1"; }
+grep_line() { { shift; printf '%s\n' "$@"; } | grep -qxF "$1"; }
+prefix_lines() {
+	while test $# -gt 0
+	do
+		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
+		shift; shift
+	done
+}
 uci_validate_name() {
 	# like util.c uci_validate_name()
 	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
@@ -115,14 +122,25 @@ in
 		fi
 
 		# Check options for syntax errors
-		validate_options "${__object:?}/parameter/object" \
+		validate_options "${__object:?}/parameter/list" "${__object:?}/parameter/object" \
 		| print_errors 'Found erroneous options in arguments:'
 
 		# Collect option names
+		if test -f "${__object:?}/parameter/list"
+		then
+			listnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
+		fi
+
 		if test -f "${__object:?}/parameter/option"
 		then
-			optnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+			optnames_should=$(sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort)
+
+			echo "${optnames_should}" \
+			| uniq -d \
+			| print_errors \
+				  'Found duplicate --options:' \
+				  "$(printf '\nUse --list for lists, instead.')"
 		fi
 
 		# Make sure the section itself is present
@@ -130,29 +148,27 @@ in
 			--value "${sect_type}"
 		export require=__uci/"${section}"
 
-		# Delete options not in "should"
+		# Delete options/lists not in "should"
 		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
 		| while read -r optname
 		  do
-			  if ! grep_line "${optname}" "${optnames_should}"
-			  then
-				  __uci "${section}.${optname}" --state absent \
-					  --transaction "${transaction_name}" </dev/null
-			  fi
+			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
+				  __uci "${section}.${optname}" --state absent --transaction "${transaction_name}"
+			  } </dev/null
 		  done
 
 		# Set "should" options
-		echo "${optnames_should}" \
-		| while read -r optname
+		prefix_lines option "${optnames_should}" list "${listnames_should}" \
+		| while read -r _type _optname
 		  do
-			  test -n "${optname}" || continue  # ignore empty lines
+			  test -n "${_type}${_optname}" || continue  # ignore empty lines
 
-			  grep "^${optname}=" <"${__object:?}/parameter/option" \
+			  grep "^${_optname}=" <"${__object:?}/parameter/${_type}" \
 			  | sed -e 's/^.*=//' \
 			  | unquote_lines \
 			  | append_values \
-				    __uci "${section}.${optname}" --state present \
-					    --transaction "${transaction_name}"
+				    __uci "${section}.${_optname}" --state present \
+					    --type "${_type}" --transaction "${transaction_name}"
 		  done
 		;;
 	(absent)
diff --git a/cdist/conf/type/__uci_section/parameter/optional_multiple b/cdist/conf/type/__uci_section/parameter/optional_multiple
index 01925a15..98af35e6 100644
--- a/cdist/conf/type/__uci_section/parameter/optional_multiple
+++ b/cdist/conf/type/__uci_section/parameter/optional_multiple
@@ -1 +1,2 @@
+list
 option

From 8728817af6ae1fa801a39456df3f84d420f87744 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 21:11:50 +0200
Subject: [PATCH 47/63] [type/__uci] Unquote UCI reported values

Without unquoting values printed in single quotes by UCI would always lead to
the state explorer reporting "different".
---
 cdist/conf/type/__uci/explorer/state | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
index 6fb4b173..d7363dbf 100644
--- a/cdist/conf/type/__uci/explorer/state
+++ b/cdist/conf/type/__uci/explorer/state
@@ -51,21 +51,34 @@ fi
 # strip off trailing newline
 printf '%s' "${values_is}" \
 | awk '
+function unquote(s) {
+	# simplified dequoting of single quoted strings
+	if (s ~ /^'\''.*'\''$/) {
+		s = substr(s, 2, length(s) - 2)
+		sub(/'"'\\\\''"'/, "'\''", s)
+	}
+	return s
+}
+
 BEGIN {
 	state = "present"  # assume all is fine
 }
 NR == FNR {
 	# memoize "should" state
 	should[FNR] = $0
+	should_count++
 
 	# go to next line (important!)
 	next
 }
 
 # compare "is" state
+
+{ $0 = unquote($0) }
+
 $0 == should[FNR] { next }
 
-FNR > length(should) {
+FNR > should_count {
 	# there are more "is" records than "should" -> definitely different
 	state = "different"
 	exit
@@ -87,7 +100,7 @@ FNR > length(should) {
 }
 
 END {
-	if (FNR < length(should)) {
+	if (FNR < should_count) {
 		# "is" was shorter than "should" -> different
 		state = "different"
 	}

From 4aebb1f12756593408d6f012adb923e4c9b81e1e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 21:18:22 +0200
Subject: [PATCH 48/63] [type/__uci*] Update man.rst regarding quoting
 requirements

---
 cdist/conf/type/__uci/man.rst         | 2 ++
 cdist/conf/type/__uci_section/man.rst | 8 ++++++++
 2 files changed, 10 insertions(+)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 9d33d618..c7f5f4f0 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -23,6 +23,8 @@ value
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
 
+    Values do not need special quoting for UCI. The only requirement is that the
+    value is passed to the type as a single shell argument.
 
 OPTIONAL PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index b8829433..753d00b3 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -34,6 +34,10 @@ list
     same ``<option>`` repeatedly.
 
     The value to this parameter is a ``<option>=<value>`` string.
+
+    ``<value>`` does not need special quoting for UCI.
+    The only requirement is that the value is passed to the type as a single
+    shell argument.
 match
     Allows to find a section to "replace" through one of its parameters.
 
@@ -43,6 +47,10 @@ option
     This parameter can be used multiple times to specify multiple options.
 
     The value to this parameter is a ``<option>=<value>`` string.
+
+    ``<value>`` does not need special quoting for UCI.
+    The only requirement is that the value is passed to the type as a single
+    shell argument.
 state
     ``present`` or ``absent``, defaults to ``present``.
 transaction

From 63d41a1053bd8b03ddda130d7129d233f97ac671 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 21:10:27 +0100
Subject: [PATCH 49/63] [type/__uci_section] Improve --match support with
 existing named sections

Use section if named section exists without --match option (e.g. empty section).
---
 cdist/conf/type/__uci_section/explorer/match | 80 +++++++++++++-------
 1 file changed, 51 insertions(+), 29 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index a5619d4e..0768e404 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -20,62 +20,84 @@
 # This explorer determines the "prefix" of the --type section matching --match
 # if set, or __object_id otherwise.
 
+RS=$(printf '\036')
+NL=$(printf '\n '); NL=${NL% }
+
 squote_values() {
 	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
 	    -e "/='.*'$/"'!{s/=/='\''/;s/$/'\''/}'
 }
+count_lines() (
+	IFS=${NL?}
+	# shellcheck disable=SC2048,SC2086
+	set -f -- $*; echo $#
+)
 
-RS=$(printf '\036')
-
-if ! test -e "${__object:?}/parameter/match"
-then
-	if ! echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$'
-	then
-		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
-		exit 1
-	fi
+echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' || {
+	echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
+	exit 1
+}
 
+test -s "${__object:?}/parameter/match" || {
 	# If no --match is given, we take the __object_id as the section identifier.
 	echo "${__object_id:?}"
 	exit 0
-fi
-
-# shellcheck disable=SC2015
-test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type" \
-|| {
+}
+test -s "${__object:?}/parameter/type" || {
 	echo 'Parameters --match and --type must be used together.' >&2
 	exit 1
 }
 
-# Find by match
-# NOTE: Apart from section types all values are printed in single quotes by UCI.
-match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
-
-if ! grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' "${__object:?}/parameter/type"
-then
+sect_type_param=$(cat "${__object:?}/parameter/type")
+expr "${sect_type_param}" : '[^.]\{1,\}\.[^.]\{1,\}$' >/dev/null 2>&1 || {
 	echo 'Section types are a package name and section type separated by a "." (period).' >&2
 	exit 1
+}
+package_filter=${sect_type_param%%.*}
+section_filter=${sect_type_param#*.}
+
+# Find by --match
+# NOTE: Apart from section types all values are printed in single quotes by uci show.
+match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
+
+if uci -s -N get "${__object_id:?}" >/dev/null 2>&1
+then
+	# Named section exists: ensure if --match applies to it
+	# if the "matched" option does not exist (e.g. empty section) we use the
+	# section unconditionally.
+	if match_value_is=$(uci -s -N get "${__object_id:?}.${match%%=*}" 2>/dev/null)
+	then
+		match_value_should=$(expr "${match}" : ".*='\\(.*\\)'$")
+
+		test "${match_value_is}" = "${match_value_should}" || {
+			printf 'Named section "%s" does not match --match "%s"\n' \
+				   "${__object_id:?}" "${match}" >&2
+			exit 1
+		}
+	fi
+
+	echo "${__object_id:?}"
+	exit 0
 fi
 
-sect_type_filter=$(cat "${__object:?}/parameter/type")
-package_filter=${sect_type_filter%%.*}
-section_filter=${sect_type_filter#*.}
-# FIXME: Does not work with named sections
-regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
+# No correctly named section exists already: find one to which --match applies
+regex="^${package_filter}\\.@${section_filter}\\[[0-9]\\{1,\\}\\]\\.${match%%=*}="
 
 matched_sections=$(
 	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
 	| grep -e "${regex}" \
 	| while read -r _line
 	  do
-		  if test "${_line#*=}" = "${match#*=}"; then echo "${_line}"; fi
+		  if test "${_line#*=}" = "${match#*=}"
+		  then
+			  echo "${_line}"
+		  fi
 	  done \
 	| sed -e 's/\.[^.]*=.*$//')
 
-if test "$(echo "${matched_sections}" | wc -l)" -gt 1
-then
+test "$(count_lines "${matched_sections}")" -le 1 || {
 	printf 'Found multiple matching sections:\n%s\n' "${matched_sections}" >&2
 	exit 1
-fi
+}
 
 echo "${matched_sections}"

From 7b301195040c4861dbd15c5f2514c4e9eeadbe61 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Oct 2020 18:52:43 +0100
Subject: [PATCH 50/63] [type/__uci] Externalise functions to separate file

---
 cdist/conf/type/__uci/files/functions.sh | 75 +++++++++++++++++++++
 cdist/conf/type/__uci/gencode-local      | 85 ++----------------------
 cdist/conf/type/__uci/manifest           |  4 +-
 3 files changed, 85 insertions(+), 79 deletions(-)
 create mode 100644 cdist/conf/type/__uci/files/functions.sh

diff --git a/cdist/conf/type/__uci/files/functions.sh b/cdist/conf/type/__uci/files/functions.sh
new file mode 100644
index 00000000..4ddb82e0
--- /dev/null
+++ b/cdist/conf/type/__uci/files/functions.sh
@@ -0,0 +1,75 @@
+# -*- mode: sh; indent-tabs-mode: t -*-
+
+in_list() {
+	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
+}
+
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
+
+uci_cmd() {
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	# Will output a shell command to be executed locally to add the given
+	# command to $tmpfile.
+	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
+	printf ' >>%s\n' "$(quote "${tmpfile}")"
+}
+
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+uci_validate_tuple() (
+	tok=${1:?}
+	case $tok
+	in
+		(*.*.*)
+			# check option
+			option=${tok##*.}
+			uci_validate_name "${option}" || {
+				printf 'Invalid option: %s\n' "${option}" >&2
+				return 1
+			}
+			tok=${tok%.*}
+			;;
+		(*.*)
+			# no option (section definition)
+			;;
+		(*)
+			printf 'Invalid tuple: %s\n' "$1" >&2
+			return 1
+			;;
+	esac
+
+	case ${tok#*.}
+	in
+		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
+		(*)  section=${tok#*.} ;;
+	esac
+	uci_validate_name "${section}" || {
+		printf 'Invalid section: %s\n' "${1#*.}" >&2
+		return 1
+	}
+
+	config=${tok%%.*}
+	uci_validate_name "${config}" || {
+		printf 'Invalid config: %s\n' "${config}" >&2
+		return 1
+	}
+)
diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index 34482a01..b7664e85 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -18,84 +18,6 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() {
-	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
-}
-
-quote() {
-	for _arg
-	do
-		shift
-		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
-		then
-			# needs quoting
-			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
-		else
-			set -- "$@" "${_arg}"
-		fi
-	done
-	unset _arg
-
-	# NOTE: Use printf because POSIX echo interprets escape sequences
-	printf '%s' "$*"
-}
-
-uci_cmd() {
-	# Usage: uci_cmd [UCI ARGUMENTS]...
-	# Will output a shell command to be executed locally to add the given
-	# command to $tmpfile.
-	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
-	printf ' >>%s\n' "$(quote "${tmpfile}")"
-}
-
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
-}
-
-uci_validate_tuple() (
-	tok=${1:?}
-	case $tok
-	in
-		(*.*.*)
-			# check option
-			option=${tok##*.}
-			uci_validate_name "${option}" || {
-				printf 'Invalid option: %s\n' "${option}" >&2
-				return 1
-			}
-			tok=${tok%.*}
-			;;
-		(*.*)
-			# no option (section definition)
-			;;
-		(*)
-			printf 'Invalid tuple: %s\n' "$1" >&2
-			return 1
-			;;
-	esac
-
-	case ${tok#*.}
-	in
-		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
-		(*)  section=${tok#*.} ;;
-	esac
-	uci_validate_name "${section}" || {
-		printf 'Invalid section: %s\n' "${1#*.}" >&2
-		return 1
-	}
-
-	config=${tok%%.*}
-	uci_validate_name "${config}" || {
-		printf 'Invalid config: %s\n' "${config}" >&2
-		return 1
-	}
-)
-
-
-config=${__object_id:?}
-uci_validate_tuple "${config}"
-
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
@@ -107,6 +29,13 @@ mkdir -p "${tmpdir}"
 
 tmpfile="${tmpdir}/${transaction_name}.txt"
 
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+
+config=${__object_id:?}
+uci_validate_tuple "${config}"
+
 
 case ${state_should}
 in
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index e56462e5..d0c8239e 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,7 +18,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; } }
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
 
 os=$(cat "${__global:?}/explorer/os")
 

From a6c37095f1eea72acbebebfa88bdf84f65f61f51 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Oct 2020 18:50:34 +0100
Subject: [PATCH 51/63] [type/__uci_section] Externalise functions to separate
 file

---
 .../type/__uci_section/files/functions.sh     | 52 +++++++++++++++++++
 cdist/conf/type/__uci_section/manifest        | 42 +--------------
 2 files changed, 54 insertions(+), 40 deletions(-)
 create mode 100644 cdist/conf/type/__uci_section/files/functions.sh

diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
new file mode 100644
index 00000000..9f510958
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -0,0 +1,52 @@
+# -*- mode: sh; indent-tabs-mode: t -*-
+
+append_values() {
+	while read -r _value
+	do
+		set -- "$@" --value "${_value}"
+	done
+	unset _value
+	"$@" </dev/null
+}
+
+grep_line() {
+	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
+}
+
+prefix_lines() {
+	while test $# -gt 0
+	do
+		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
+		shift; shift
+	done
+}
+
+print_errors() {
+	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
+		BEGIN {
+			if (getline) {
+				print prefix
+				print
+				rc = 1
+			}
+		}
+		{ print }
+		END {
+			if (rc && suffix) print suffix
+			exit rc
+		}' >&2
+}
+
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
+}
+
+unquote_lines() {
+	sed -e '/^".*"$/{s/^"//;s/"$//}' \
+	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
+}
+
+validate_options() {
+	grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"
+}
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index a7865315..208a2c4e 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,46 +18,8 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-grep_line() { { shift; printf '%s\n' "$@"; } | grep -qxF "$1"; }
-prefix_lines() {
-	while test $# -gt 0
-	do
-		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
-		shift; shift
-	done
-}
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
-}
-validate_options() { grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"; }
-unquote_lines() {
-	sed -e '/^".*"$/{s/^"//;s/"$//}' \
-	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
-}
-append_values() {
-	while read -r _value
-	do
-		set -- "$@" --value "${_value}"
-	done
-	unset _value
-	"$@" </dev/null
-}
-print_errors() {
-	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
-		BEGIN {
-			if (getline) {
-				print prefix
-				print
-				rc = 1
-			}
-		}
-		{ print }
-		END {
-			if (rc && suffix) print suffix
-			exit rc
-		}' >&2
-}
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
 
 
 ## Check section name and error if invalid!

From c1ae3ccb2f725d0d94eaec9eeae7c8f5f98a5e45 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:13:06 +0100
Subject: [PATCH 52/63] [type/__uci*] Remove public-facing transaction
 "interface"

---
 cdist/conf/type/__uci/man.rst          | 13 ++++---------
 cdist/conf/type/__uci_section/man.rst  |  9 ++-------
 cdist/conf/type/__uci_section/manifest | 10 ++++------
 3 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index c7f5f4f0..8c3f79f1 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -11,8 +11,6 @@ DESCRIPTION
 This cdist type can be used to alter configuration options in OpenWrt's
 Unified Configuration Interface (UCI) system.
 
-Options can be applied in batches if the ``--transaction`` parameter is used.
-
 
 REQUIRED PARAMETERS
 -------------------
@@ -30,9 +28,6 @@ OPTIONAL PARAMETERS
 -------------------
 state
     ``present`` or ``absent``, defaults to ``present``.
-transaction
-    The name of the transaction this option belongs to.
-    If none is given: ``default`` is used.
 type
     If the type should generate an option or a list.
     One of: ``option`` or ``list``.
@@ -55,10 +50,10 @@ EXAMPLES
     # Set DHCP option 252: tell DHCP clients to not ask for proxy information.
     __uci dhcp.lan.dhcp_option --type list --value '252,"\n"'
 
-    # Enable NTP and NTPd (in one transaction)
-    __uci system.ntp.enabled --value 1 --transaction ntp
-    __uci system.ntp.enable_server --value 1 --transaction ntp
-    __uci system.ntp.server --type list --transaction ntp \
+    # Enable NTP and NTPd (each is applied individually)
+    __uci system.ntp.enabled --value 1
+    __uci system.ntp.enable_server --value 1
+    __uci system.ntp.server --type list \
         --value '0.openwrt.pool.ntp.org' \
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 753d00b3..a0ab78e8 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -53,9 +53,6 @@ option
     shell argument.
 state
     ``present`` or ``absent``, defaults to ``present``.
-transaction
-    The name of the transaction this option belongs to.
-    The value will be forwarded to :strong:`cdist-type__uci`\ (7).
 type
     The type of the section in the format: ``<config>.<section-type>``
 
@@ -71,13 +68,13 @@ EXAMPLES
 .. code-block:: sh
 
     # Configure the dropbear daemon
-    __uci_section dropbear --type dropbear.dropbear --transaction sshd \
+    __uci_section dropbear --type dropbear.dropbear \
         --match Port=22 --option Port=22 \
         --option PasswordAuth=off \
         --option RootPasswordAuth=off
 
     # Define a firewall zone comprised of lan and wlan networks
-    __uci_section firewall.internal --type firewall.zone --transaction fw \
+    __uci_section firewall.internal --type firewall.zone \
         --option name='internal' \
         --list network='lan' \
         --list network='wlan' \
@@ -87,7 +84,6 @@ EXAMPLES
 
     # Block SSH access from the guest network
     __uci_section firewall.block_ssh_from_guest --type firewall.rule \
-        --transaction fwrules \
         --option name='Block-SSH-Access-from-Guest' \
         --option src='guest' \
         --option proto='tcp' \
@@ -96,7 +92,6 @@ EXAMPLES
 
     # Configure a Wi-Fi access point
     __uci_section wireless.default_radio0 --type wireless.wifi-iface \
-        --transaction wifi \
         --option device='radio0' \
         --option mode='ap' \
         --option network='wlan' \
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 208a2c4e..41e30a07 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -46,7 +46,6 @@ esac
 section=$(cat "${__object:?}/explorer/match")
 
 state_should=$(cat "${__object:?}/parameter/state")
-transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case $state_should
 in
@@ -106,8 +105,7 @@ in
 		fi
 
 		# Make sure the section itself is present
-		__uci "${section}" --state present --transaction "${transaction_name}" \
-			--value "${sect_type}"
+		__uci "${section}" --state present --value "${sect_type}"
 		export require=__uci/"${section}"
 
 		# Delete options/lists not in "should"
@@ -115,7 +113,7 @@ in
 		| while read -r optname
 		  do
 			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
-				  __uci "${section}.${optname}" --state absent --transaction "${transaction_name}"
+				  __uci "${section}.${optname}" --state absent
 			  } </dev/null
 		  done
 
@@ -130,14 +128,14 @@ in
 			  | unquote_lines \
 			  | append_values \
 				    __uci "${section}.${_optname}" --state present \
-					    --type "${_type}" --transaction "${transaction_name}"
+					    --type "${_type}"
 		  done
 		;;
 	(absent)
 		# if explorer found no section there is nothing to delete
 		test -n "${section}" || exit 0
 
-		__uci "${section}" --state absent --transaction "${transaction_name}"
+		__uci "${section}" --state absent
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2

From e264fb004f797efc540f74191a0e9630c66dd819 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:27:07 +0100
Subject: [PATCH 53/63] [type/__uci] Convert to immediate remote execution

---
 cdist/conf/type/__uci/files/functions.sh |   6 +-
 cdist/conf/type/__uci/gencode-local      | 102 -----------------------
 cdist/conf/type/__uci/gencode-remote     |   8 --
 3 files changed, 2 insertions(+), 114 deletions(-)
 delete mode 100755 cdist/conf/type/__uci/gencode-local
 mode change 100644 => 100755 cdist/conf/type/__uci/gencode-remote

diff --git a/cdist/conf/type/__uci/files/functions.sh b/cdist/conf/type/__uci/files/functions.sh
index 4ddb82e0..277f648c 100644
--- a/cdist/conf/type/__uci/files/functions.sh
+++ b/cdist/conf/type/__uci/files/functions.sh
@@ -24,10 +24,8 @@ quote() {
 
 uci_cmd() {
 	# Usage: uci_cmd [UCI ARGUMENTS]...
-	# Will output a shell command to be executed locally to add the given
-	# command to $tmpfile.
-	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
-	printf ' >>%s\n' "$(quote "${tmpfile}")"
+	mkdir -p "${__object:?}/files"
+	printf '%s\n' "$(quote "$@")" >>"${__object:?}/files/uci_batch.txt"
 }
 
 uci_validate_name() {
diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
deleted file mode 100755
index b7664e85..00000000
--- a/cdist/conf/type/__uci/gencode-local
+++ /dev/null
@@ -1,102 +0,0 @@
-#!/bin/sh -e
-#
-# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-state_is=$(cat "${__object:?}/explorer/state")
-state_should=$(cat "${__object:?}/parameter/state")
-
-transaction_name=$(cat "${__object:?}/parameter/transaction")
-
-tmpdir="${__global:?}/tmp/__uci"
-# HACK
-mkdir -p "${tmpdir}"
-
-tmpfile="${tmpdir}/${transaction_name}.txt"
-
-# shellcheck source=files/functions.sh
-. "${__type:?}/files/functions.sh"
-
-
-config=${__object_id:?}
-uci_validate_tuple "${config}"
-
-
-case ${state_should}
-in
-	(present)
-		if in_list "${state_is}" 'present' 'rearranged'
-		then
-			# NOTE: order is ignored so rearranged is also fine.
-			exit 0
-		fi
-
-		# Determine type
-		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
-		case ${type}
-		in
-			(option|list) ;;
-			('')
-				# Guess type by the number of values
-				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
-					 && type=list \
-					 || type=option
-				;;
-			(*)
-				printf 'Invalid --type: %s\n' "${type}" >&2
-				exit 1
-				;;
-		esac
-
-		case ${type}
-		in
-			(list)
-				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
-
-				if test "${state_is}" != 'absent'
-				then
-					uci_cmd delete "${config}"
-				fi
-
-				while read -r value
-				do
-					uci_cmd add_list "${config}"="${value}"
-				done <"${__object:?}/parameter/value"
-				;;
-			(option)
-				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
-
-				value=$(cat "${__object:?}/parameter/value")
-				uci_cmd set "${config}"="${value}"
-				;;
-		esac
-		;;
-	(absent)
-		if in_list "${state_is}" 'absent'
-		then
-			exit 0
-		fi
-
-		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
-		uci_cmd delete "${config}"
-		;;
-	(*)
-		printf 'Invalid --state: %s\n' "${state_should}" >&2
-		exit 1
-		;;
-esac
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
old mode 100644
new mode 100755
index 348d15a1..a15f50a6
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -91,11 +91,3 @@ in
 		exit 1
 		;;
 esac
-
-if test -s "${__object:?}/files/uci_batch.txt"
-then
-	cat "${__type:?}/files/uci_apply.sh"
-	printf "uci_apply <<'EOF'\n"
-	cat "${__object:?}/files/uci_batch.txt"
-	printf '\nEOF\n'
-fi

From dfe9e08c280a2a3bc0ded15069b7d60df5a757f8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:06:05 +0100
Subject: [PATCH 54/63] [type/__uci_commit] Delete type

---
 cdist/conf/type/__uci/man.rst               |  2 -
 cdist/conf/type/__uci/manifest              | 18 -----
 cdist/conf/type/__uci_commit/gencode-remote | 81 ---------------------
 cdist/conf/type/__uci_commit/man.rst        | 60 ---------------
 cdist/conf/type/__uci_commit/nonparallel    |  0
 5 files changed, 161 deletions(-)
 delete mode 100755 cdist/conf/type/__uci_commit/gencode-remote
 delete mode 100644 cdist/conf/type/__uci_commit/man.rst
 delete mode 100644 cdist/conf/type/__uci_commit/nonparallel

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 8c3f79f1..81a53473 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -58,13 +58,11 @@ EXAMPLES
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
         --value '3.openwrt.pool.ntp.org'
-    export require=__uci_commit/ntp
 
 
 SEE ALSO
 --------
 - https://openwrt.org/docs/guide-user/base-system/uci
-- :strong:`cdist-type__uci_commit`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index d0c8239e..26920011 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,15 +18,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
-. "${__type:?}/files/functions.sh"
-
-
 os=$(cat "${__global:?}/explorer/os")
 
-state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
-transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case ${os}
 in
@@ -40,8 +34,6 @@ in
 		;;
 esac
 
-changes_required=false
-
 case ${state_should}
 in
 	(present)
@@ -49,21 +41,11 @@ in
 			echo 'The parameter --value is required.' >&2
 			exit 1
 		}
-
-		# NOTE: order is ignored so rearranged is also fine.
-		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
 		;;
 	(absent)
-		in_list "${state_is}" 'absent' || changes_required=true
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
 		exit 1
 		;;
 esac
-
-if ${changes_required}
-then
-	# Make sure the changes are being committed
-	require=${__object_name:?} __uci_commit "${transaction_name}"
-fi
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
deleted file mode 100755
index e5c93966..00000000
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/bin/sh -e
-#
-# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-transaction_name=${__object_id:?}
-batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
-
-test -s "${batchfile}" || exit 0
-
-printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
-
-# NOTE: Uncommited changes are checked in code-remote instead of in an explorer
-#       because in cdist there is no interlocking between explorers and code
-#       execution.
-#       Checking for uncommited changes in an explorer leaves a time slot in
-#       which changes made are not detected by the code.
-#       Furthermore an explorer running concurrently with another transactions
-#       code-remote could lead to a false positive.
-
-cat <<CODE
-changes=\$(uci changes)
-
-if test -n "\${changes}"
-then
-	echo 'Uncommited UCI changes were found on the target:'
-	printf '%s\n\n' "\${changes}"
-	echo 'This can be caused by manual changes or due to a previous failed run.'
-	echo 'Please investigate the situation, revert or commit the changes, and try again.'
-	exit 1
-fi >&2
-
-check_errors() {
-	# reads stdin and forwards non-empty lines to stderr.
-	# returns 0 if stdin is empty, else 1.
-	! grep -e . >&2
-}
-
-commit() {
-	uci commit
-}
-
-rollback() {
-	echo >&2
-	echo 'An error occurred when trying to commit transaction ${transaction_name}!' >&2
-
-	uci changes \\
-	| sed -e 's/^-//' -e 's/\..*\$//' \\
-	| sort -u \\
-	| while read -r _package
-	  do
-		  uci revert "\${_package}"
-		  echo "\${_package}"  # for logging
-	  done \\
-	| awk '
-	  BEGIN { printf "Reverted changes in: " }
-	  { printf "%s%s", (FNR > 1 ? ", " : ""), \$0 }
-	  END { printf "\\n" }' >&2
-
-	return 1
-}
-
-uci batch <<'EOF' 2>&1 | check_errors && commit || rollback
-$(cat "${batchfile}")
-EOF
-CODE
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
deleted file mode 100644
index ea6611eb..00000000
--- a/cdist/conf/type/__uci_commit/man.rst
+++ /dev/null
@@ -1,60 +0,0 @@
-cdist-type__uci_commit(7)
-=========================
-
-NAME
-----
-cdist-type__uci_commit - Commit UCI transactions
-
-
-DESCRIPTION
------------
-This type executes the ``uci commit`` command on the target with the commands
-queued in a transaction.
-It is usually not required to use this type. Use the ``--transaction`` parameter
-of :strong:`cdist-type__uci`\ (7) and :strong:`cdist-type__uci_section`\ (7)
-instead.
-
-
-REQUIRED PARAMETERS
--------------------
-None.
-
-
-OPTIONAL PARAMETERS
--------------------
-None.
-
-
-BOOLEAN PARAMETERS
-------------------
-None.
-
-
-EXAMPLES
---------
-
-.. code-block:: sh
-
-    # Commit the default transaction
-    __uci_commit default
-
-    # Commit another transaction
-    __uci_commit my_transaction
-
-
-SEE ALSO
---------
-:strong:`cdist-type__uci`\ (7), :strong:`cdist-type__uci_section`\ (7)
-
-
-AUTHORS
--------
-Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
-
-
-COPYING
--------
-Copyright \(C) 2020 Dennis Camera. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_commit/nonparallel b/cdist/conf/type/__uci_commit/nonparallel
deleted file mode 100644
index e69de29b..00000000

From ec984f81b5a1db6b7f3e54255b570e07cf5282f9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:36:10 +0100
Subject: [PATCH 55/63] [type/__uci] Delete --transaction parameter

---
 cdist/conf/type/__uci/parameter/default/transaction | 1 -
 cdist/conf/type/__uci/parameter/optional            | 1 -
 2 files changed, 2 deletions(-)
 delete mode 100644 cdist/conf/type/__uci/parameter/default/transaction

diff --git a/cdist/conf/type/__uci/parameter/default/transaction b/cdist/conf/type/__uci/parameter/default/transaction
deleted file mode 100644
index 4ad96d51..00000000
--- a/cdist/conf/type/__uci/parameter/default/transaction
+++ /dev/null
@@ -1 +0,0 @@
-default
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
index 2703b3df..d9080e3a 100644
--- a/cdist/conf/type/__uci/parameter/optional
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -1,3 +1,2 @@
 state
-transaction
 type

From 3e5f18d409e448f69e4f259fc451d4a0e5a7a548 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 18:42:50 +0100
Subject: [PATCH 56/63] [type/__uci] Apply all commands in a single batch

---
 cdist/conf/type/__uci/files/uci_apply.sh | 43 ++++++++++++++++++++++++
 cdist/conf/type/__uci/gencode-remote     |  8 +++++
 cdist/conf/type/__uci/nonparallel        |  0
 3 files changed, 51 insertions(+)
 create mode 100644 cdist/conf/type/__uci/files/uci_apply.sh
 create mode 100644 cdist/conf/type/__uci/nonparallel

diff --git a/cdist/conf/type/__uci/files/uci_apply.sh b/cdist/conf/type/__uci/files/uci_apply.sh
new file mode 100644
index 00000000..63f94290
--- /dev/null
+++ b/cdist/conf/type/__uci/files/uci_apply.sh
@@ -0,0 +1,43 @@
+changes=$(uci changes)
+
+if test -n "${changes}"
+then
+	echo 'Uncommited UCI changes were found on the target:'
+	printf '%s\n\n' "${changes}"
+	echo 'This can be caused by manual changes or due to a previous failed run.'
+	echo 'Please investigate the situation, revert or commit the changes, and try again.'
+	exit 1
+fi >&2
+
+check_errors() {
+	# reads stdin and forwards non-empty lines to stderr.
+	# returns 0 if stdin is empty, else 1.
+	! grep -e . >&2
+}
+
+commit() {
+	uci commit
+}
+
+rollback() {
+	printf '\nAn error occurred when trying to commit UCI transaction!\n' >&2
+
+	uci changes \
+	| sed -e 's/^-//' -e 's/\..*\$//' \
+	| sort -u \
+	| while read -r _package
+	  do
+		  uci revert "${_package}"
+		  echo "${_package}"  # for logging
+	  done \
+	| awk '
+	  BEGIN { printf "Reverted changes in: " }
+	  { printf "%s%s", (FNR > 1 ? ", " : ""), $0 }
+	  END { printf "\n" }' >&2
+
+	return 1
+}
+
+uci_apply() {
+	uci batch 2>&1 | check_errors && commit || rollback
+}
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
index a15f50a6..348d15a1 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -91,3 +91,11 @@ in
 		exit 1
 		;;
 esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci/nonparallel b/cdist/conf/type/__uci/nonparallel
new file mode 100644
index 00000000..e69de29b

From 9d40500570875f29aa48784c6637683aa31afe4e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 20:40:29 +0100
Subject: [PATCH 57/63] [type/__uci_section] Apply all commands in a single
 batch

---
 .../type/__uci_section/files/functions.sh     |  31 +++-
 .../type/__uci_section/files/uci_apply.sh     |   1 +
 cdist/conf/type/__uci_section/gencode-remote  | 146 ++++++++++++++++++
 cdist/conf/type/__uci_section/manifest        |  76 ++-------
 cdist/conf/type/__uci_section/nonparallel     |   0
 5 files changed, 180 insertions(+), 74 deletions(-)
 create mode 120000 cdist/conf/type/__uci_section/files/uci_apply.sh
 create mode 100755 cdist/conf/type/__uci_section/gencode-remote
 create mode 100644 cdist/conf/type/__uci_section/nonparallel

diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
index 9f510958..17c64bbc 100644
--- a/cdist/conf/type/__uci_section/files/functions.sh
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -1,13 +1,6 @@
 # -*- mode: sh; indent-tabs-mode: t -*-
 
-append_values() {
-	while read -r _value
-	do
-		set -- "$@" --value "${_value}"
-	done
-	unset _value
-	"$@" </dev/null
-}
+NL=$(printf '\n '); NL=${NL% }
 
 grep_line() {
 	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
@@ -37,6 +30,28 @@ print_errors() {
 		}' >&2
 }
 
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+	printf '%s' "$*"
+}
+
+uci_cmd() {
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	mkdir -p "${__object:?}/files"
+	printf '%s\n' "$(quote "$@")" >>"${__object:?}/files/uci_batch.txt"
+}
+
 uci_validate_name() {
 	# like util.c uci_validate_name()
 	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
diff --git a/cdist/conf/type/__uci_section/files/uci_apply.sh b/cdist/conf/type/__uci_section/files/uci_apply.sh
new file mode 120000
index 00000000..4209151f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/uci_apply.sh
@@ -0,0 +1 @@
+../../__uci/files/uci_apply.sh
\ No newline at end of file
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
new file mode 100755
index 00000000..05bca0a7
--- /dev/null
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -0,0 +1,146 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+
+section=$(cat "${__object:?}/explorer/match")
+
+state_is=$(test -s "${__object:?}/explorer/type" && echo present || echo absent)
+state_should=$(cat "${__object:?}/parameter/state")
+
+case $state_should
+in
+	(present)
+		test -f "${__object:?}/parameter/type" || {
+			echo 'Parameter --type is required.' >&2
+			exit 1
+		}
+
+		type_is=$(cat "${__object:?}/explorer/type")
+		type_should=$(cat "${__object:?}/parameter/type")
+
+		if test -n "${type_is}"
+		then
+			sect_type=${type_is}
+		else
+			sect_type=${type_should##*.}
+		fi
+
+		if test -z "${section}"
+		then
+			# No section exists and --match was used.
+			# So we generate a new section identifier from $__object_id.
+			case ${__object_id:?}
+			in
+				(*.*) section=${__object_id:?} ;;
+				(*) section="${type_should%%.*}.${__object_id:?}" ;;
+			esac
+		fi
+
+		# Collect option names
+		if test -f "${__object:?}/parameter/list"
+		then
+			listnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
+		fi
+
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
+		# Make sure the section itself is present
+		if test "${state_is}" = absent
+		then
+			printf 'set %s\n' "${section}" >>"${__messages_out:?}"
+			# shellcheck disable=SC2140
+			uci_cmd set "${section}"="${sect_type}"
+		fi
+
+		# Delete options/lists not in "should"
+		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		| while read -r _optname
+		  do
+			  grep_line "${_optname}" "${listnames_should}" "${optnames_should}" || {
+				  printf 'delete %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+				  uci_cmd delete "${section}.${_optname}"
+			  } </dev/null
+		  done
+
+		# Set "should" options
+		prefix_lines option "${optnames_should}" list "${listnames_should}" \
+		| while read -r _type _optname
+		  do
+			  test -n "${_type}${_optname}" || continue  # ignore empty lines
+
+			  case $_type
+			  in
+				  (list)
+					  printf 'set_list %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  if grep -q -e "^${_optname}=" "${__object:?}/explorer/options"
+					  then
+						  uci_cmd delete "${section}.${_optname}"
+					  fi
+
+					  grep "^${_optname}=" "${__object:?}/parameter/list" \
+					  | sed -e 's/^.*=//' \
+					  | unquote_lines \
+					  | while read -r _value
+					    do
+						    # shellcheck disable=SC2140
+						    uci_cmd add_list "${section}.${_optname}"="${_value}"
+					    done
+					  ;;
+				  (option)
+					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  # shellcheck disable=SC2140
+					  uci_cmd set "${section}.${_optname}"="$(
+						  grep "^${_optname}=" "${__object:?}/parameter/option" \
+						  | sed -e 's/^.*=//' \
+						  | unquote_lines \
+						  | head -n 1)"
+					  ;;
+			  esac
+		  done
+		;;
+	(absent)
+		if test "${state_is}" = absent
+		then
+			# if explorer found no section there is nothing to delete
+			exit 0
+		fi
+
+		printf 'delete %s\n' "${section}" >>"${__messages_out:?}"
+		uci_cmd delete "${section}"
+		;;
+esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 41e30a07..7e23e443 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -43,8 +43,6 @@ in
 		;;
 esac
 
-section=$(cat "${__object:?}/explorer/match")
-
 state_should=$(cat "${__object:?}/parameter/state")
 
 case $state_should
@@ -57,85 +55,31 @@ in
 		type_is=$(cat "${__object:?}/explorer/type")
 		type_should=$(cat "${__object:?}/parameter/type")
 
-		if test -n "${type_is}"
+		if test -n "${type_is}" && test "${type_is}" != "${type_should##*.}"
 		then
-			if test "${type_is}" != "${type_should##*.}"
-			then
-				# Check if section type matches (section exists and --type provided)
-				printf 'Section type "%s" does not match --type "%s".\n' \
-					"${type_is}" "${type_should}" >&2
-				exit 1
-			fi
-			sect_type=${type_is}
-		else
-			sect_type=${type_should##*.}
-		fi
-
-		if test -z "${section}"
-		then
-			# No section exists and --match was used.
-			# So we generate a new section identifier from $__object_id.
-			case ${__object_id:?}
-			in
-				(*.*) section=${__object_id:?} ;;
-				(*) section="${type_should%%.*}.${__object_id:?}" ;;
-			esac
+			# Check if section type matches (section exists and --type provided)
+			printf 'Section type "%s" does not match --type "%s".\n' \
+				"${type_is}" "${type_should}" >&2
+			exit 1
 		fi
 
 		# Check options for syntax errors
 		validate_options "${__object:?}/parameter/list" "${__object:?}/parameter/object" \
 		| print_errors 'Found erroneous options in arguments:'
 
-		# Collect option names
-		if test -f "${__object:?}/parameter/list"
+		# Check for duplicate option names
+		if test -s "${__object:?}/parameter/option"
 		then
-			listnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
-		fi
-
-		if test -f "${__object:?}/parameter/option"
-		then
-			optnames_should=$(sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort)
-
-			echo "${optnames_should}" \
+			sed -e 's/=.*$//' "${__object:?}/parameter/option" \
+			| sort \
 			| uniq -d \
 			| print_errors \
 				  'Found duplicate --options:' \
 				  "$(printf '\nUse --list for lists, instead.')"
 		fi
-
-		# Make sure the section itself is present
-		__uci "${section}" --state present --value "${sect_type}"
-		export require=__uci/"${section}"
-
-		# Delete options/lists not in "should"
-		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
-		| while read -r optname
-		  do
-			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
-				  __uci "${section}.${optname}" --state absent
-			  } </dev/null
-		  done
-
-		# Set "should" options
-		prefix_lines option "${optnames_should}" list "${listnames_should}" \
-		| while read -r _type _optname
-		  do
-			  test -n "${_type}${_optname}" || continue  # ignore empty lines
-
-			  grep "^${_optname}=" <"${__object:?}/parameter/${_type}" \
-			  | sed -e 's/^.*=//' \
-			  | unquote_lines \
-			  | append_values \
-				    __uci "${section}.${_optname}" --state present \
-					    --type "${_type}"
-		  done
 		;;
 	(absent)
-		# if explorer found no section there is nothing to delete
-		test -n "${section}" || exit 0
-
-		__uci "${section}" --state absent
+		:
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
diff --git a/cdist/conf/type/__uci_section/nonparallel b/cdist/conf/type/__uci_section/nonparallel
new file mode 100644
index 00000000..e69de29b

From ade69729ddd64938ef69c08a522a85ec825628e9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 31 Oct 2020 18:18:15 +0100
Subject: [PATCH 58/63] [type/__uci_section] Only generate UCI commands if
 state differs

---
 .../conf/type/__uci_section/explorer/options  | 22 ++++-
 .../type/__uci_section/files/functions.sh     |  8 --
 .../type/__uci_section/files/option_state.awk | 91 +++++++++++++++++++
 cdist/conf/type/__uci_section/gencode-remote  | 68 ++++++++++----
 4 files changed, 160 insertions(+), 29 deletions(-)
 create mode 100644 cdist/conf/type/__uci_section/files/option_state.awk

diff --git a/cdist/conf/type/__uci_section/explorer/options b/cdist/conf/type/__uci_section/explorer/options
index 67b4f83f..e1e60668 100644
--- a/cdist/conf/type/__uci_section/explorer/options
+++ b/cdist/conf/type/__uci_section/explorer/options
@@ -25,4 +25,24 @@ section=$("${__type_explorer:?}/match")
 test -n "${section}" || exit 0
 
 uci -s -N -d "${RS}" show "${section}" 2>/dev/null \
-| grep -v -e "^${section}=" || true
+| awk -v VSEP="${RS}" '
+  {
+	  # Strip off the config and section parts
+	  is_opt = sub(/^([^.]*\.){2}/, "")
+
+	  if (!is_opt) {
+		  # this line represents the section -> skip
+		  next
+	  }
+
+	  if (index($0, VSEP)) {
+		  # Put values each on a line, like --option and --list parameters
+		  opt = substr($0, 1, index($0, "=") - 1)
+		  split(substr($0, length(opt) + 2), values, VSEP)
+		  for (i in values) {
+			  printf "%s=%s\n", opt, values[i]
+		  }
+	  } else {
+		  print
+	  }
+  }'
diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
index 17c64bbc..60cb9148 100644
--- a/cdist/conf/type/__uci_section/files/functions.sh
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -6,14 +6,6 @@ grep_line() {
 	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
 }
 
-prefix_lines() {
-	while test $# -gt 0
-	do
-		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
-		shift; shift
-	done
-}
-
 print_errors() {
 	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
 		BEGIN {
diff --git a/cdist/conf/type/__uci_section/files/option_state.awk b/cdist/conf/type/__uci_section/files/option_state.awk
new file mode 100644
index 00000000..97cd94fb
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/option_state.awk
@@ -0,0 +1,91 @@
+# -*- mode: awk; indent-tabs-mode:t -*-
+# Usage: awk -f option_state.awk option_type option_name
+# e.g. awk -f option_state.awk option title
+#      awk -f option_state.awk list entry
+
+function unquote(s) {
+	# simplified dequoting of single quoted strings
+	if (s ~ /^'.*'$/) {
+		s = substr(s, 2, length(s) - 2)
+		sub(/'\\''/, "'", s)
+	}
+	return s
+}
+
+function valueof(line) {
+	if (line !~ /^[[:alpha:]_]+=/) return 0
+	return unquote(substr(line, index(line, "=") + 1))
+}
+
+BEGIN {
+	__object = ENVIRON["__object"]
+	if (!__object) exit 1
+
+	opttype = ARGV[1]
+	optname = ARGV[2]
+
+	if (opttype !~ /^(option|list)/ || !optname) {
+		print "invalid"
+		exit (e=1)
+	}
+
+	ARGV[1] = __object "/parameter/" opttype
+	ARGV[2] = __object "/explorer/options"
+
+	state = "present"
+}
+
+NR == FNR {
+	# memoize "should" state
+	if (index($0, optname "=") == 1) {
+		should[++should_count] = valueof($0)
+	}
+
+	# go to next line (important!)
+	next
+}
+
+{
+	# compare "is" state
+	if (index($0, optname "=") != 1)
+		next
+	++is_count
+
+	v = valueof($0)
+
+	if (v == should[is_count]) {
+		# looks good, but can't say definitely just from this line
+	} else if (is_count > should_count) {
+		# there are more "is" records than "should" -> definitely different
+		state = "different"
+		exit
+	} else {
+		# see if we can find the "is" value somewhere in "should"
+		for (i in should) {
+			if (v == should[i]) {
+				# value found -> could be rearranged
+				# FIXME: Duplicate values are not properly handled here. Do they matter?
+				state = "rearranged"
+				next
+			}
+		}
+
+		# "is" value could not be found in "should" -> definitely different
+		state = "different"
+		exit
+	}
+}
+
+END {
+	if (e) exit
+
+	if (!is_count) {
+		# no "is" values -> absent
+		state = "absent"
+	} else if (is_count < should_count) {
+		# "is" was shorter than "should" -> different
+		state = "different"
+	}
+
+	print state
+}
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
index 05bca0a7..285659b0 100755
--- a/cdist/conf/type/__uci_section/gencode-remote
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -70,7 +70,8 @@ in
 		fi
 
 		# Make sure the section itself is present
-		if test "${state_is}" = absent
+		if test "${state_is}" = absent \
+			|| test "${type_is}" != "${type_should#*.}"
 		then
 			printf 'set %s\n' "${section}" >>"${__messages_out:?}"
 			# shellcheck disable=SC2140
@@ -78,7 +79,7 @@ in
 		fi
 
 		# Delete options/lists not in "should"
-		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		sed -e 's/=.*$//' "${__object:?}/explorer/options" \
 		| while read -r _optname
 		  do
 			  grep_line "${_optname}" "${listnames_should}" "${optnames_should}" || {
@@ -87,18 +88,55 @@ in
 			  } </dev/null
 		  done
 
-		# Set "should" options
-		prefix_lines option "${optnames_should}" list "${listnames_should}" \
-		| while read -r _type _optname
-		  do
-			  test -n "${_type}${_optname}" || continue  # ignore empty lines
+		opt_proc_error() {
+			printf 'An error occurred during processing of option %s\n' "${1:?}" >&2
+			exit 1
+		}
 
-			  case $_type
+		# Set "should" options
+		echo "${optnames_should}" \
+		| grep -e . \
+		| while read -r _optname
+		  do
+			  _opt_state=$(awk -f "${__type:?}/files/option_state.awk" option "${_optname}") \
+				  || opt_proc_error "${_optname}"
+			  case ${_opt_state}
 			  in
-				  (list)
+				  (invalid)
+					  opt_proc_error "${_optname}"
+					  ;;
+				  (present)
+					  ;;
+				  (*)
+					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  # shellcheck disable=SC2140
+					  uci_cmd set "${section}.${_optname}"="$(
+						  grep -e "^${_optname}=" "${__object:?}/parameter/option" \
+						  | sed -e 's/^.*=//' \
+						  | unquote_lines \
+						  | head -n 1)"
+					  ;;
+			  esac
+		  done
+
+		echo "${listnames_should}" \
+		| grep -e . \
+		| while read -r _optname
+		  do
+			  _list_state=$(awk -f "${__type:?}/files/option_state.awk" list "${_optname}") \
+				  || opt_proc_error "${_optname}"
+			  case ${_list_state}
+			  in
+				  (invalid)
+					  opt_proc_error "${_optname}"
+					  ;;
+				  (present)
+					  ;;
+				  (*)
 					  printf 'set_list %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
 
-					  if grep -q -e "^${_optname}=" "${__object:?}/explorer/options"
+					  if test "${_list_state}" != absent
 					  then
 						  uci_cmd delete "${section}.${_optname}"
 					  fi
@@ -112,16 +150,6 @@ in
 						    uci_cmd add_list "${section}.${_optname}"="${_value}"
 					    done
 					  ;;
-				  (option)
-					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
-
-					  # shellcheck disable=SC2140
-					  uci_cmd set "${section}.${_optname}"="$(
-						  grep "^${_optname}=" "${__object:?}/parameter/option" \
-						  | sed -e 's/^.*=//' \
-						  | unquote_lines \
-						  | head -n 1)"
-					  ;;
 			  esac
 		  done
 		;;

From 2be8c634584561158960fc5cf2f363f3f5c4cc4c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 3 Nov 2020 06:43:57 +0100
Subject: [PATCH 59/63] pycodestyle fixes

---
 cdist/argparse.py         |  2 +-
 cdist/scan/commandline.py |  2 +-
 cdist/scan/scan.py        | 12 ++++++++----
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index ff195e8c..88759d7b 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -473,7 +473,7 @@ def get_parsers():
 
     # Scan = config + further
     parser['scan'] = parser['sub'].add_parser('scan', add_help=False,
-                                                 parents=[parser['config']])
+                                              parents=[parser['config']])
 
     parser['scan'] = parser['sub'].add_parser(
             'scan', parents=[parser['loglevel'],
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 0fb718e5..eca4cf13 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -37,7 +37,7 @@ def commandline(args):
 
     if not args.mode:
         # By default scan and trigger, but do not call any action
-        args.mode = ['scan', 'trigger' ]
+        args.mode = ['scan', 'trigger', ]
 
     if 'trigger' in args.mode:
         t = scan.Trigger(interfaces=args.interfaces)
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index fcbf1899..0ce4dff3 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -30,9 +30,12 @@
 # Scanner logic
 #  - save results to configdir:
 #     basedir = ~/.cdist/scan/<ipv6-address>
-#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time or similar
-#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
-#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time
+#           or similar
+#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record
+#           unix time or similar
+#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record
+#           unix time or similar
 #
 #
 #
@@ -58,6 +61,7 @@ import datetime
 
 log = logging.getLogger("scan")
 
+
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
@@ -91,6 +95,7 @@ class Trigger(object):
         log.debug(f"Sending request on {interface}")
         send(packet, verbose=self.verbose)
 
+
 class Scanner(object):
     """
     Scan for replies of hosts, maintain the up-to-date database
@@ -134,7 +139,6 @@ class Scanner(object):
               prn=self.handle_pkg)
 
 
-
 if __name__ == '__main__':
     t = Trigger(interfaces=["wlan0"])
     t.start()

From d28a70a73c2affcf3268623a51df2b7a6cdd534a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 6 Nov 2020 08:32:40 +0100
Subject: [PATCH 60/63] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 5977d80e..335a0f9f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,8 @@ next:
 	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 	* Type __file: Fix state pre-exists (Dennis Camera)
+	* Type __apt_norecommends: Use 00InstallRecommends file as debian-installer does (Dennis Camera)
+	* Core: Introduce scanner (noninvasive, beta) (Nico Schottelius)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 0f1df5ef68a60c5b8077f437bf0d07f1f79788e7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:07:58 +0100
Subject: [PATCH 61/63] Fix shellcheck source directives

---
 cdist/conf/type/__uci/gencode-remote         | 2 +-
 cdist/conf/type/__uci_section/gencode-remote | 2 +-
 cdist/conf/type/__uci_section/manifest       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
index 348d15a1..70a3d3e0 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 state_is=$(cat "${__object:?}/explorer/state")
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
index 285659b0..50fdfa4e 100755
--- a/cdist/conf/type/__uci_section/gencode-remote
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci_section/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 7e23e443..0325b92c 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci_section/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 

From c7c3075f629191ed7e05a76395b77dce01e2928f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:10:14 +0100
Subject: [PATCH 62/63] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 335a0f9f..40b816a6 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -13,6 +13,7 @@ next:
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 	* Type __file: Fix state pre-exists (Dennis Camera)
 	* Type __apt_norecommends: Use 00InstallRecommends file as debian-installer does (Dennis Camera)
+	* New types: __uci, __uci_section (Dennis Camera)
 	* Core: Introduce scanner (noninvasive, beta) (Nico Schottelius)
 
 6.8.0: 2020-09-11

From 348c6eedc903ecd28b02c5dda3821e71ef255a4f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:12:20 +0100
Subject: [PATCH 63/63] Release 6.9.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 40b816a6..c47123d9 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.0: 2020-11-07
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)