From 9a2e5758f59b27fa17799bd67ba5accf82168f38 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 26 Jan 2019 17:00:03 +0100
Subject: [PATCH 001/366] Implement triggering functionality

---
 cdist/argparse.py                             |  24 +-
 .../type/__cdist_preos_trigger/gencode-remote |  12 +
 cdist/conf/type/__cdist_preos_trigger/man.rst |  45 ++++
 .../conf/type/__cdist_preos_trigger/manifest  |  67 ++++++
 .../__cdist_preos_trigger/parameter/required  |   1 +
 cdist/preos/debootstrap/debootstrap.py        |   6 +
 cdist/preos/debootstrap/files/code            |   9 +-
 cdist/trigger.py                              | 225 ++++++++++++++++++
 completions/bash/cdist-completion.bash        |  10 +-
 completions/zsh/_cdist                        |   2 +-
 docs/changelog                                |   3 +
 docs/src/cdist-preos.rst                      |  55 ++++-
 docs/src/cdist-trigger.rst                    |  33 +++
 docs/src/index.rst                            |   1 +
 docs/src/man1/cdist.rst                       | 121 +++++++++-
 15 files changed, 593 insertions(+), 21 deletions(-)
 create mode 100644 cdist/conf/type/__cdist_preos_trigger/gencode-remote
 create mode 100644 cdist/conf/type/__cdist_preos_trigger/man.rst
 create mode 100644 cdist/conf/type/__cdist_preos_trigger/manifest
 create mode 100644 cdist/conf/type/__cdist_preos_trigger/parameter/required
 create mode 100644 cdist/trigger.py
 create mode 100644 docs/src/cdist-trigger.rst

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 611c484a..47320e01 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -5,12 +5,13 @@ import logging
 import collections
 import functools
 import cdist.configuration
+import cdist.trigger
 import cdist.preos
 import cdist.info
 
 
 # set of beta sub-commands
-BETA_COMMANDS = set(('install', 'inventory', ))
+BETA_COMMANDS = set(('install', 'inventory', 'preos', 'trigger', ))
 # set of beta arguments for sub-commands
 BETA_ARGS = {
     'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
@@ -468,6 +469,27 @@ def get_parsers():
             'pattern', nargs='?', help='Glob pattern.')
     parser['info'].set_defaults(func=cdist.info.Info.commandline)
 
+    # Trigger
+    parser['trigger'] = parser['sub'].add_parser(
+            'trigger', parents=[parser['loglevel'],
+                                parser['beta'],
+                                parser['config_main']])
+    parser['trigger'].add_argument(
+            '-6', '--ipv6', default=False,
+            help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'),
+            action='store_true')
+    parser['trigger'].add_argument(
+            '-D', '--directory', action='store', required=False,
+            help=('Where to create local files'))
+    parser['trigger'].add_argument(
+            '-H', '--http-port', action='store', default=3000, required=False,
+            help=('Create trigger listener via http on specified port'))
+    parser['trigger'].add_argument(
+            '-S', '--source', action='store', required=False,
+            help=('Which file to copy for creation'))
+
+    parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline)
+
     for p in parser:
         parser[p].epilog = EPILOG
 
diff --git a/cdist/conf/type/__cdist_preos_trigger/gencode-remote b/cdist/conf/type/__cdist_preos_trigger/gencode-remote
new file mode 100644
index 00000000..d5e9fe5c
--- /dev/null
+++ b/cdist/conf/type/__cdist_preos_trigger/gencode-remote
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+os=$(cat "$__global/explorer/os")
+
+case "$os" in
+    devuan)
+        echo "update-rc.d cdist-preos-trigger defaults > /dev/null"
+    ;;
+    *)
+    ;;
+esac
+
diff --git a/cdist/conf/type/__cdist_preos_trigger/man.rst b/cdist/conf/type/__cdist_preos_trigger/man.rst
new file mode 100644
index 00000000..abbd553b
--- /dev/null
+++ b/cdist/conf/type/__cdist_preos_trigger/man.rst
@@ -0,0 +1,45 @@
+cdist-type__cdist_preos_trigger(7)
+==================================
+
+NAME
+----
+cdist-type__cdist_preos_trigger - configure cdist preos trigger
+
+
+DESCRIPTION
+-----------
+Create cdist PreOS trigger by creating systemd unit file that will be started
+at boot and will execute trigger command - connect to specified host and port.
+
+
+REQUIRED PARAMETERS
+-------------------
+trigger-command
+    Command that will be executed as a PreOS cdist trigger.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Configure default curl trigger for host cdist.ungleich.ch at port 80.
+    __cdist_preos_trigger http --trigger-command '/usr/bin/curl cdist.ungleich.ch:80'
+
+
+AUTHORS
+-------
+Darko Poljak <darko.poljak--@--ungleich.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2016 Darko Poljak. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__cdist_preos_trigger/manifest b/cdist/conf/type/__cdist_preos_trigger/manifest
new file mode 100644
index 00000000..750a0548
--- /dev/null
+++ b/cdist/conf/type/__cdist_preos_trigger/manifest
@@ -0,0 +1,67 @@
+#!/bin/sh
+
+os="$(cat "$__global/explorer/os")"
+trigger_command=$(cat "$__object/parameter/trigger-command")
+
+case "$os" in
+    devuan)
+        __file /etc/init.d/cdist-preos-trigger --owner root \
+            --group root \
+            --mode 755 \
+            --source - << EOF
+#!/bin/sh
+# /etc/init.d/cdist-preos-trigger
+
+### BEGIN INIT INFO
+# Provides:          cdist-preos-trigger
+# Required-Start:    \$all
+# Required-Stop:
+# Default-Start:     2 3 4 5 S
+# Default-Stop:      0 1 6
+# Short-Description: Execute cdist preos trigger command
+# Description:       Execute cdist preos trigger commnad.
+### END INIT INFO
+
+case "\$1" in
+    start)
+        echo "Starting cdist-preos-trigger command"
+        ${trigger_command} &
+        ;;
+    stop)
+        # no-op
+        ;;
+    *)
+        echo "Usage: /etc/init.d/cdist-preos-trigger {start|stop}"
+        exit 1
+        ;;
+esac
+
+exit 0
+EOF
+    ;;
+    *)
+        __file /etc/systemd/system/cdist-preos-trigger.service --owner root \
+            --group root \
+            --mode 644 \
+            --source - << EOF
+[Unit]
+Description=preos trigger
+Wants=network-online.target
+After=network.target network-online.target
+
+[Service]
+Type=simple
+Restart=no
+# Broken systemd
+ExecStartPre=/bin/sleep 5
+ExecStart=${trigger_command}
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+        require="__file/etc/systemd/system/cdist-preos-trigger.service" \
+            __start_on_boot cdist-preos-trigger
+    ;;
+esac
+
diff --git a/cdist/conf/type/__cdist_preos_trigger/parameter/required b/cdist/conf/type/__cdist_preos_trigger/parameter/required
new file mode 100644
index 00000000..3407a482
--- /dev/null
+++ b/cdist/conf/type/__cdist_preos_trigger/parameter/required
@@ -0,0 +1 @@
+trigger-command
diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py
index f53dd4a7..7b56008d 100644
--- a/cdist/preos/debootstrap/debootstrap.py
+++ b/cdist/preos/debootstrap/debootstrap.py
@@ -127,6 +127,12 @@ class Debian(object):
                             help="suite used for debootstrap, "
                                  "by default '{}'".format(defargs.suite),
                             dest='suite', default=defargs.suite)
+        parser.add_argument(
+            '-t', '--trigger-command',
+            help=("trigger command that will be added to cdist config; "
+                  "'__cdist_preos_trigger http ...' type is appended to "
+                  "initial manifest"),
+            dest='trigger_command')
         parser.add_argument(
             '-y', '--remote-copy',
             help=("remote copy that cdist config will use, by default "
diff --git a/cdist/preos/debootstrap/files/code b/cdist/preos/debootstrap/files/code
index 9e37003b..50e73972 100755
--- a/cdist/preos/debootstrap/files/code
+++ b/cdist/preos/debootstrap/files/code
@@ -127,6 +127,13 @@ then
         exit 1
     fi
 
+    if [ "${trigger_command}" ]
+    then
+        trigger_line="__cdist_preos_trigger http --trigger-command '${trigger_command}'\n"
+    else
+        trigger_line=""
+    fi
+
     if [ "${keyfile_cnt}" -a "${keyfile_cnt}" -gt 0 ]
     then
         i="$((keyfile_cnt - 1))"
@@ -174,7 +181,7 @@ then
     fi
     grub_lines="${grub_manifest_line}${grub_kern_params_line}"
 
-    printf "${ssh_auth_keys_line}${grub_lines}" \
+    printf "${trigger_line}${ssh_auth_keys_line}${grub_lines}" \
         | cat "${manifest}" - |\
         cdist config \
             ${cdist_params} -i - \
diff --git a/cdist/trigger.py b/cdist/trigger.py
new file mode 100644
index 00000000..4e66ac6b
--- /dev/null
+++ b/cdist/trigger.py
@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2016 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import ipaddress
+import logging
+import re
+import socket
+import http.server
+import os
+import socketserver
+import shutil
+
+import cdist.config
+import cdist.log
+import cdist.util.ipaddr as ipaddr
+
+
+class Trigger():
+    """cdist trigger handling"""
+
+    # Arguments that are only trigger specific
+    triggers_args = ["http_port", "ipv6", "directory", "source", ]
+
+    def __init__(self, http_port=None, dry_run=False, ipv6=False,
+                 directory=None, source=None, cdistargs=None):
+        self.dry_run = dry_run
+        self.http_port = int(http_port)
+        self.ipv6 = ipv6
+        self.args = cdistargs
+
+        self.directory = directory
+        self.source = source
+
+        log.debug("IPv6: %s", self.ipv6)
+
+    def run_httpd(self):
+        server_address = ('', self.http_port)
+
+        if self.ipv6:
+            httpdcls = HTTPServerV6
+        else:
+            httpdcls = HTTPServerV4
+        httpd = httpdcls(self.args, self.directory, self.source,
+                         server_address, TriggerHttp)
+
+        log.debug("Starting server at port %d", self.http_port)
+        if self.dry_run:
+            log.debug("Running in dry run mode")
+        httpd.serve_forever()
+
+    def run(self):
+        if self.http_port:
+            self.run_httpd()
+
+    @classmethod
+    def commandline(cls, args):
+        global log
+
+        # remove root logger default cdist handler and configure trigger's own
+        logging.getLogger().handlers = []
+        logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s')
+
+        log = logging.getLogger("trigger")
+        ownargs = {}
+        for targ in cls.triggers_args:
+            arg = getattr(args, targ)
+            ownargs[targ] = arg
+
+            del arg
+
+        t = cls(dry_run=args.dry_run, cdistargs=args, **ownargs)
+        t.run()
+
+
+class TriggerHttp(http.server.BaseHTTPRequestHandler):
+    actions = {
+        "cdist": ["config", "install", ],
+        "file":  ["present", "absent", ],
+    }
+
+    def do_HEAD(self):
+        self.dispatch_request()
+
+    def do_POST(self):
+        self.dispatch_request()
+
+    def do_GET(self):
+        self.dispatch_request()
+
+    def _actions_regex(self):
+        regex = ["^/(?P<subsystem>", ]
+        regex.extend("|".join(self.actions.keys()))
+        regex.append(")/(?P<action>")
+        regex.extend("|".join("|".join(self.actions[x]) for x in self.actions))
+        regex.append(")/")
+
+        return "".join(regex)
+
+    def dispatch_request(self):
+        host = self.client_address[0]
+        code = 200
+        message = None
+
+        self.cdistargs = self.server.cdistargs
+
+        actions_regex = self._actions_regex()
+        m = re.match(actions_regex, self.path)
+
+        if m:
+            subsystem = m.group('subsystem')
+            action = m.group('action')
+            handler = getattr(self, "handler_" + subsystem)
+
+            if action not in self.actions[subsystem]:
+                code = 404
+        else:
+            code = 404
+
+        if code == 200:
+            log.debug("Calling {} -> {}".format(subsystem, action))
+            try:
+                handler(action, host)
+            except cdist.Error as e:
+                # cdist is not broken, cdist run is broken
+                code = 599  # use arbitrary unassigned error code
+                message = str(e)
+            except Exception as e:
+                # cdist/trigger server is broken
+                code = 500
+
+        self.send_response(code=code, message=message)
+        self.end_headers()
+
+    def handler_file(self, action, host):
+        if not self.server.directory or not self.server.source:
+            log.info("Cannot serve file request: directory or source "
+                     "not setup")
+            return
+
+        try:
+            ipaddress.ip_address(host)
+        except ValueError:
+            log.error("Host is not a valid IP address - aborting")
+            return
+
+        dst = os.path.join(self.server.directory, host)
+
+        if action == "present":
+            shutil.copyfile(self.server.source, dst)
+        if action == "absent":
+            if os.path.exists(dst):
+                os.remove(dst)
+
+    def handler_cdist(self, action, host):
+        log.debug("Running cdist action %s for %s", action, host)
+
+        if self.server.dry_run:
+            log.info("Dry run, skipping cdist execution")
+            return
+
+        cname = action.title()
+        module = getattr(cdist, action)
+        theclass = getattr(module, cname)
+
+        if hasattr(self.cdistargs, 'out_path'):
+            out_path = self.cdistargs.out_path
+        else:
+            out_path = None
+        host_base_path, hostdir = theclass.create_host_base_dirs(
+            host, theclass.create_base_root_path(out_path))
+        theclass.construct_remote_exec_copy_patterns(self.cdistargs)
+        host_tags = None
+        host_name = ipaddr.resolve_target_host_name(host)
+        log.debug('Resolved target host name: %s', host_name)
+        if host_name:
+            target_host = host_name
+        else:
+            target_host = host
+        log.debug('Using target_host: %s', target_host)
+        log.debug("Executing cdist onehost with params: %s, %s, %s, %s, %s, ",
+                  target_host, host_tags, host_base_path, hostdir,
+                  self.cdistargs)
+        theclass.onehost(target_host, host_tags, host_base_path, hostdir,
+                         self.cdistargs, parallel=False)
+
+
+class HTTPServerV6(socketserver.ForkingMixIn, http.server.HTTPServer):
+    """
+    Server that listens to both IPv4 and IPv6 requests.
+    """
+    address_family = socket.AF_INET6
+
+    def __init__(self, cdistargs, directory, source, *args, **kwargs):
+        self.cdistargs = cdistargs
+        self.dry_run = cdistargs.dry_run
+        self.directory = directory
+        self.source = source
+
+        http.server.HTTPServer.__init__(self, *args, **kwargs)
+
+
+class HTTPServerV4(HTTPServerV6):
+    """
+    Server that listens to IPv4 requests.
+    """
+    address_family = socket.AF_INET
diff --git a/completions/bash/cdist-completion.bash b/completions/bash/cdist-completion.bash
index cdac7f29..ed5c6a55 100644
--- a/completions/bash/cdist-completion.bash
+++ b/completions/bash/cdist-completion.bash
@@ -6,7 +6,7 @@ _cdist()
     prev="${COMP_WORDS[COMP_CWORD-1]}"
     prevprev="${COMP_WORDS[COMP_CWORD-2]}"
     opts="-h --help -q --quiet -v --verbose -V --version"
-    cmds="banner config install inventory preos shell"
+    cmds="banner config install inventory preos shell trigger"
 
     case "${prevprev}" in
         shell)
@@ -80,6 +80,14 @@ _cdist()
             COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
             return 0
             ;;
+        trigger)
+            opts="-h --help -d --debug -v --verbose -b --beta \
+                -C --cache-path-pattern -c --conf-dir -i --initial-manifest \
+                -j --jobs -n --dry-run -o --out-dir --remote-copy \
+                --remote-exec -6 --ipv6 -H --http-port -D --directory -S --source"
+            COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+            return 0
+            ;;
         inventory)
             cmds="list add-host del-host add-tag del-tag"
             opts="-h --help -q --quiet -v --verbose"
diff --git a/completions/zsh/_cdist b/completions/zsh/_cdist
index 1bd275ef..5d653095 100644
--- a/completions/zsh/_cdist
+++ b/completions/zsh/_cdist
@@ -11,7 +11,7 @@ _cdist()
 
     case $state in
         opts_cmds)
-            _arguments '1:Options and commands:(banner config install inventory preos shell -h --help -q --quiet -v --verbose -V --version)'
+            _arguments '1:Options and commands:(banner config install inventory preos shell trigger -h --help -q --quiet -v --verbose -V --version)'
             ;;
         *)
             case $words[2] in
diff --git a/docs/changelog b/docs/changelog
index 9f1e6245..25566aa8 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Core: Add trigger functionality (Nico Schottelius, Darko Poljak)
+
 6.5.1: 2020-02-15
 	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)
 	* Explorer os_release: Add fallbacks (Dennis Camera)
diff --git a/docs/src/cdist-preos.rst b/docs/src/cdist-preos.rst
index 9570bcfc..59d0d13e 100644
--- a/docs/src/cdist-preos.rst
+++ b/docs/src/cdist-preos.rst
@@ -25,15 +25,16 @@ For example, to create an ubuntu PreOS:
 .. code-block:: sh
 
     $ cdist preos ubuntu /preos/preos-ubuntu -B -C \
-        -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu
+        -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \
+        -t "/usr/bin/curl 192.168.111.5:3000/install/"
 
 For more info about the available options see the cdist manual page.
 
-This will bootstrap (``-B``) ubuntu PreOS in the ``/preos/preos-ubuntu``
-directory, it will be configured (``-C``) using default the built-in initial
-manifest and with specified ssh authorized key (``-k``).
-After bootstrapping and configuration, the PXE boot directory will be
-created (``-p``) in ``/preos/pxe-ubuntu``.
+This will bootstrap (``-B``) ubuntu PreOS in ``/preos/preos-ubuntu`` directory, it
+will be configured (``-C``) using default built-in initial manifest and with
+specified ssh authorized key (``-k``) and with specified trigger command (``-t``).
+After bootstrapping and configuration PXE
+boot directory will be created (``-p``) in ``/preos/pxe-ubuntu``.
 
 After PreOS is created, new machines can be booted using the created PXE
 (after proper dhcp and tftp settings).
@@ -41,8 +42,17 @@ After PreOS is created, new machines can be booted using the created PXE
 Since PreOS is configured with ssh authorized key it can be accessed through
 ssh, i.e. it can be further installed and configured with cdist.
 
-Implementing a new PreOS sub-command
-------------------------------------
+When installing and configuring new machines using cdist's PreOS concept
+cdist can use triggering for host installation/configuration, which is described
+in the previous chapter.
+
+When new machine is booted with PreOS then trigger command is executed.
+Machine will connect to cdist trigger server. If the request is, for example,
+for installation then cdist trigger server will start install command for the
+client host using parameters specified at trigger server startup.
+
+Implementing new PreOS sub-command
+----------------------------------
 preos command is implemented as a plugin system. This plugin system scans for
 preos subcommands in the ``cdist/preos/`` distribution directory and also in
 ``~/.cdist/preos/`` directory if it exists.
@@ -127,3 +137,32 @@ When you try to run this new preos you will get:
 
 In the ``commandline`` function/method you have all the freedom to actually create
 a PreOS.
+
+Simple tipical use case for using PreOS and trigger
+---------------------------------------------------
+Tipical use case for using PreOS and trigger command include the following steps.
+
+#. Create PreOS PXE with ssh key and trigger command for installation.
+
+    .. code-block:: sh
+
+        $ cdist preos ubuntu /preos/ubuntu -b -C \
+            -k ~/.ssh/id_rsa.pub -p /preos/pxe \
+            -t "/usr/bin/curl 192.168.111.5:3000/install/"
+
+#. Configure dhcp server and tftp server.
+
+#. On cdist host (192.168.111.5 from above) start trigger command (it will use
+   default init manifest for installation).
+
+    .. code-block:: sh
+
+        $ cdist trigger -b -v
+
+#. After all is set up start new machines (PXE boot).
+
+#. New machine boots and executes trigger command, i.e. triggers installation.
+
+#. Cdist trigger server starts installing host that has triggered it.
+
+#. After cdist install is finished new host is installed.
diff --git a/docs/src/cdist-trigger.rst b/docs/src/cdist-trigger.rst
new file mode 100644
index 00000000..94477c29
--- /dev/null
+++ b/docs/src/cdist-trigger.rst
@@ -0,0 +1,33 @@
+Trigger
+=======
+
+Description
+-----------
+cdist supports triggering for host installation/configuration using trigger command.
+This command starts trigger server at management node, for example:
+
+.. code-block:: sh
+
+    $ cdist trigger -b -v
+
+This will start cdist trigger server in verbose mode. cdist trigger server accepts
+simple requests for configuration and for installation:
+
+* :strong:`/cdist/install/.*` for installation
+* :strong:`/cdist/config/.*` for configuration.
+
+Machines can then trigger cdist trigger server with appropriate requests.
+If the request is, for example, for installation (:strong:`/cdist/install/`)
+then cdist trigger server will start install command for the client host using
+parameters specified at trigger server startup. For the above example that means
+that client will be installed using default initial manifest.
+
+When triggered cdist will try to reverse DNS lookup for host name and if
+host name is dervied then it is used for running cdist config. If no
+host name is resolved then IP address is used.
+
+This command returns the following response codes to client requests:
+
+* 200 for success
+* 599 for cdist run errors
+* 500 for cdist/server errors.
diff --git a/docs/src/index.rst b/docs/src/index.rst
index 31c044dc..652a018a 100644
--- a/docs/src/index.rst
+++ b/docs/src/index.rst
@@ -33,6 +33,7 @@ It natively supports IPv6 since the first release.
    cdist-messaging
    cdist-parallelization
    cdist-inventory
+   cdist-trigger
    cdist-preos
    cdist-integration
    cdist-reference
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 38248821..fba2b322 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -11,7 +11,7 @@ SYNOPSIS
 
 ::
 
-    cdist [-h] [-V] {banner,config,install,inventory,preos,shell,info} ...
+    cdist [-h] [-V] {banner,config,install,inventory,preos,shell,info,trigger} ...
 
     cdist banner [-h] [-l LOGLEVEL] [-q] [-v]
 
@@ -67,27 +67,37 @@ SYNOPSIS
                                        [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
                                        [-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
                                        [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
-                                       [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
+                                       [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND]
+                                       [-y REMOTE_COPY]
                                        target_dir
 
     cdist preos [preos-options] devuan [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
                                        [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
                                        [-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
                                        [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
-                                       [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
+                                       [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND]
+                                       [-y REMOTE_COPY]
                                        target_dir
 
     cdist preos [preos-options] ubuntu [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
                                        [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
                                        [-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
                                        [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
-                                       [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
+                                       [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND]
+                                       [-y REMOTE_COPY]
                                        target_dir
 
     cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL]
 
     cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern]
 
+    cdist trigger [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN]
+                  [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n]
+                  [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]]
+                  [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY]
+                  [--remote-exec REMOTE_EXEC] [-6] [-D DIRECTORY]
+                  [-H HTTP_PORT] [-S SOURCE]
+
 
 DESCRIPTION
 -----------
@@ -534,6 +544,10 @@ PREOS DEBIAN/DEVUAN
 **-s SUITE, --suite SUITE**
     suite used for debootstrap, by default 'stable'
 
+**-t TRIGGER_COMMAND, --trigger-command TRIGGER_COMMAND**
+    trigger command that will be added to cdist config;
+    '``__cdist_preos_trigger http ...``' type is appended to initial manifest
+
 **-y REMOTE_COPY, --remote-copy REMOTE_COPY**
     remote copy that cdist config will use, by default
     internal script is used
@@ -594,6 +608,10 @@ PREOS UBUNTU
 **-s SUITE, --suite SUITE**
     suite used for debootstrap, by default 'xenial'
 
+**-t TRIGGER_COMMAND, --trigger-command TRIGGER_COMMAND**
+    trigger command that will be added to cdist config;
+    '``__cdist_preos_trigger http ...``' type is appended to initial manifest
+
 **-y REMOTE_COPY, --remote-copy REMOTE_COPY**
     remote copy that cdist config will use, by default
     internal script is used
@@ -643,6 +661,83 @@ Display information for cdist (global explorers, types).
 **-t, --types**
     Display info for types.
 
+TRIGGER
+-------
+Start trigger (simple http server) that waits for connections. When host
+connects then it triggers config or install command and then cdist
+config/install is executed which configures/installs host.
+When triggered cdist will try to reverse DNS lookup for host name and if
+host name is dervied then it is used for running cdist config. If no
+host name is resolved then IP address is used.
+Request path recognizes following requests:
+
+* :strong:`/cdist/config/.*` for config
+* :strong:`/cdist/install/.*` for install.
+
+This command returns the following response codes to client requests:
+
+* 200 for success
+* 599 for cdist run errors
+* 500 for cdist/server errors.
+
+
+**-6, --ipv6**
+
+    Listen to both IPv4 and IPv6 (instead of only IPv4)
+
+**-b, --beta**
+
+    Enable beta functionality.
+
+**-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN**
+
+    Sepcify custom cache path pattern. It can also be set by
+    CDIST_CACHE_PATH_PATTERN environment variable. If it is not set then
+    default hostdir is used. For more info on format see
+    :strong:`CACHE PATH PATTERN FORMAT` below.
+
+**-c CONF_DIR, --conf-dir CONF_DIR**
+
+    Add configuration directory (can be repeated, last one wins)
+
+**-D DIRECTORY, --directory DIRECTORY**
+    Where to create local files
+
+**-H HTTP_PORT, --http-port HTTP_PORT**
+
+    Create trigger listener via http on specified port
+
+**-i MANIFEST, --initial-manifest MANIFEST**
+
+    path to a cdist manifest or '-' to read from stdin.
+
+**-j [JOBS], --jobs [JOBS]**
+
+    Specify the maximum number of parallel jobs, currently
+    only global explorers are supported
+
+**-n, --dry-run**
+
+    do not execute code
+
+**-o OUT_PATH, --out-dir OUT_PATH**
+
+    directory to save cdist output in
+
+**-r REMOTE_OUT_PATH, --remote-out-dir REMOTE_OUT_PATH**
+
+    Directory to save cdist output in on the target host
+
+**--remote-copy REMOTE_COPY**
+
+    Command to use for remote copy (should behave like scp)
+
+**--remote-exec REMOTE_EXEC**
+
+    Command to use for remote execution (should behave like ssh)
+
+**-S SOURCE, --source SOURCE**
+    Which file to copy for creation
 
 CONFIGURATION
 -------------
@@ -838,20 +933,28 @@ EXAMPLES
     # Configure all hosts from inventory db
     $ cdist config -b -A
 
-    # Create default debian PreOS in debug mode
+    # Create default debian PreOS in debug mode with config
+    # trigger command
     $ cdist preos debian /preos/preos-debian -vvvv -C \
-        -k ~/.ssh/id_rsa.pub -p /preos/pxe-debian
+        -k ~/.ssh/id_rsa.pub -p /preos/pxe-debian \
+        -t "/usr/bin/curl 192.168.111.5:3000/config/"
 
-    # Create ubuntu PreOS
+    # Create ubuntu PreOS with install trigger command
     $ cdist preos ubuntu /preos/preos-ubuntu -C \
-        -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu
+        -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \
+        -t "/usr/bin/curl 192.168.111.5:3000/install/"
 
-    # Create ubuntu PreOS on drive /dev/sdb
+    # Create ubuntu PreOS on drive /dev/sdb with install trigger command
     # and set root password to 'password'.
     $ cdist preos ubuntu /mnt -B -C \
         -k ~/.ssh/id_rsa.pub -D /dev/sdb \
+        -t "/usr/bin/curl 192.168.111.5:3000/install/" \
         -P password
 
+    # Start trigger in verbose mode that will configure host using specified
+    # init manifest
+    % cdist trigger -v -i ~/.cdist/manifest/init-for-triggered
+
 
 ENVIRONMENT
 -----------

From 9d030bddc15f32177854ca4a54c8018b6f492497 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 20 Sep 2019 20:08:55 +0200
Subject: [PATCH 002/366] ++

---
 cdist/argparse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 47320e01..323383ae 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -11,7 +11,7 @@ import cdist.info
 
 
 # set of beta sub-commands
-BETA_COMMANDS = set(('install', 'inventory', 'preos', 'trigger', ))
+BETA_COMMANDS = set(('install', 'inventory', 'trigger', ))
 # set of beta arguments for sub-commands
 BETA_ARGS = {
     'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),

From 912c6759034f7034d2299fc4e63e09e342b49b17 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 28 Jan 2019 16:06:36 +0100
Subject: [PATCH 003/366] Log trigger server error

---
 cdist/trigger.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/trigger.py b/cdist/trigger.py
index 4e66ac6b..1b9a384b 100644
--- a/cdist/trigger.py
+++ b/cdist/trigger.py
@@ -145,6 +145,7 @@ class TriggerHttp(http.server.BaseHTTPRequestHandler):
                 message = str(e)
             except Exception as e:
                 # cdist/trigger server is broken
+                log.exception(e)
                 code = 500
 
         self.send_response(code=code, message=message)

From a09467917374edc91fb3c6df3d5ea5abd2a585d1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 28 Jan 2019 16:46:40 +0100
Subject: [PATCH 004/366] Update trigger to config

---
 cdist/argparse.py       | 11 +++---
 cdist/config.py         | 13 ++++---
 docs/src/man1/cdist.rst | 80 +++++++++++++++++++++++------------------
 3 files changed, 60 insertions(+), 44 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 323383ae..ece7ff10 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -473,11 +473,8 @@ def get_parsers():
     parser['trigger'] = parser['sub'].add_parser(
             'trigger', parents=[parser['loglevel'],
                                 parser['beta'],
+                                parser['common'],
                                 parser['config_main']])
-    parser['trigger'].add_argument(
-            '-6', '--ipv6', default=False,
-            help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'),
-            action='store_true')
     parser['trigger'].add_argument(
             '-D', '--directory', action='store', required=False,
             help=('Where to create local files'))
@@ -485,7 +482,11 @@ def get_parsers():
             '-H', '--http-port', action='store', default=3000, required=False,
             help=('Create trigger listener via http on specified port'))
     parser['trigger'].add_argument(
-            '-S', '--source', action='store', required=False,
+            '--ipv6', default=False,
+            help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'),
+            action='store_true')
+    parser['trigger'].add_argument(
+            '-O', '--source', action='store', required=False,
             help=('Which file to copy for creation'))
 
     parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline)
diff --git a/cdist/config.py b/cdist/config.py
index 97cc1da6..44afe7d5 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -144,7 +144,7 @@ class Config(object):
 
         # Determine forcing IPv4/IPv6 options if any, only for
         # default remote commands.
-        if args.force_ipv:
+        if hasattr(args, 'force_ipv') and args.force_ipv:
             force_addr_opt = " -{}".format(args.force_ipv)
         else:
             force_addr_opt = ""
@@ -357,10 +357,13 @@ class Config(object):
 
     @staticmethod
     def _address_family(args):
-        if args.force_ipv == 4:
-            family = socket.AF_INET
-        elif args.force_ipv == 6:
-            family = socket.AF_INET6
+        if hasattr(args, 'force_ipv'):
+            if args.force_ipv == 4:
+                family = socket.AF_INET
+            elif args.force_ipv == 6:
+                family = socket.AF_INET6
+            else:
+                family = 0
         else:
             family = 0
         return family
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index fba2b322..01e25c4e 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -91,12 +91,12 @@ SYNOPSIS
 
     cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern]
 
-    cdist trigger [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN]
-                  [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n]
-                  [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]]
-                  [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY]
-                  [--remote-exec REMOTE_EXEC] [-6] [-D DIRECTORY]
-                  [-H HTTP_PORT] [-S SOURCE]
+    cdist trigger [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4]
+                  [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
+                  [-j [JOBS]] [-n] [-o OUT_PATH] [-P]
+                  [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH]
+                  [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
+                  [-S] [-D DIRECTORY] [-H HTTP_PORT] [--ipv6] [-O SOURCE]
 
 
 DESCRIPTION
@@ -680,64 +680,76 @@ This command returns the following response codes to client requests:
 * 599 for cdist run errors
 * 500 for cdist/server errors.
 
+**-4, -force-ipv4**
+    Force to use IPv4 addresses only. No influence for
+    custom remote commands.
 
-**-6, --ipv6**
-
-    Listen to both IPv4 and IPv6 (instead of only IPv4)
-
-**-b, --beta**
-
-    Enable beta functionality.
+**-6, --force-ipv6**
+    Force to use IPv6 addresses only. No influence for
+    custom remote commands.
 
 **-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN**
-
-    Sepcify custom cache path pattern. It can also be set by
-    CDIST_CACHE_PATH_PATTERN environment variable. If it is not set then
-    default hostdir is used. For more info on format see
-    :strong:`CACHE PATH PATTERN FORMAT` below.
+    Specify custom cache path pattern. If it is not set
+    then default hostdir is used.
 
 **-c CONF_DIR, --conf-dir CONF_DIR**
-
-    Add configuration directory (can be repeated, last one wins)
+    Add configuration directory (can be repeated, last one
+    wins).
 
 **-D DIRECTORY, --directory DIRECTORY**
     Where to create local files
 
-**-H HTTP_PORT, --http-port HTTP_PORT**
+**-g CONFIG_FILE, --config-file CONFIG_FILE**
+    Use specified custom configuration file.
 
+**-H HTTP_PORT, --http-port HTTP_PORT**
     Create trigger listener via http on specified port
 
 **-i MANIFEST, --initial-manifest MANIFEST**
+    Path to a cdist manifest or '-' to read from stdin.
 
-    path to a cdist manifest or '-' to read from stdin.
+**--ipv6**
+    Listen to both IPv4 and IPv6 (instead of only IPv4)
 
 **-j [JOBS], --jobs [JOBS]**
-
-    Specify the maximum number of parallel jobs, currently
-    only global explorers are supported
+    Operate in parallel in specified maximum number of
+    jobs. Global explorers, object prepare and object run
+    are supported. Without argument CPU count is used by
+    default. Currently in beta.
 
 **-n, --dry-run**
+    Do not execute code.
 
-    do not execute code
+**-O SOURCE, --source SOURCE**
+    Which file to copy for creation
 
 **-o OUT_PATH, --out-dir OUT_PATH**
+    Directory to save cdist output in.
 
-    directory to save cdist output in
+**-P, --timestamp**
+    Timestamp log messages with the current local date and
+    time in the format: YYYYMMDDHHMMSS.us.
+
+**-R [{tar,tgz,tbz2,txz}], --use-archiving [{tar,tgz,tbz2,txz}]**
+    Operate by using archiving with compression where
+    appropriate. Supported values are: tar - tar archive,
+    tgz - gzip tar archive (the default), tbz2 - bzip2 tar
+    archive and txz - lzma tar archive. Currently in beta.
 
 **-r REMOTE_OUT_PATH, --remote-out-dir REMOTE_OUT_PATH**
-
-    Directory to save cdist output in on the target host
+    Directory to save cdist output in on the target host.
 
 **--remote-copy REMOTE_COPY**
-
-    Command to use for remote copy (should behave like scp)
+    Command to use for remote copy (should behave like
+    scp).
 
 **--remote-exec REMOTE_EXEC**
+    Command to use for remote execution (should behave
+    like ssh).
 
-    Command to use for remote execution (should behave like ssh)
+**-S, --disable-saving-output-streams**
+    Disable saving output streams.
 
-**-S SOURCE, --source SOURCE**
-    Which file to copy for creation
 
 CONFIGURATION
 -------------

From 1a5cc67cc8f16262806697978313f0855872eab6 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Mon, 28 Jan 2019 16:35:02 +0100
Subject: [PATCH 005/366] Add missing configuration arg

---
 cdist/config.py  | 13 +++++--------
 cdist/trigger.py |  5 ++++-
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index 44afe7d5..97cc1da6 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -144,7 +144,7 @@ class Config(object):
 
         # Determine forcing IPv4/IPv6 options if any, only for
         # default remote commands.
-        if hasattr(args, 'force_ipv') and args.force_ipv:
+        if args.force_ipv:
             force_addr_opt = " -{}".format(args.force_ipv)
         else:
             force_addr_opt = ""
@@ -357,13 +357,10 @@ class Config(object):
 
     @staticmethod
     def _address_family(args):
-        if hasattr(args, 'force_ipv'):
-            if args.force_ipv == 4:
-                family = socket.AF_INET
-            elif args.force_ipv == 6:
-                family = socket.AF_INET6
-            else:
-                family = 0
+        if args.force_ipv == 4:
+            family = socket.AF_INET
+        elif args.force_ipv == 6:
+            family = socket.AF_INET6
         else:
             family = 0
         return family
diff --git a/cdist/trigger.py b/cdist/trigger.py
index 1b9a384b..58f98dca 100644
--- a/cdist/trigger.py
+++ b/cdist/trigger.py
@@ -200,8 +200,11 @@ class TriggerHttp(http.server.BaseHTTPRequestHandler):
         log.debug("Executing cdist onehost with params: %s, %s, %s, %s, %s, ",
                   target_host, host_tags, host_base_path, hostdir,
                   self.cdistargs)
+        cfg = cdist.configuration.Configuration(self.cdistargs)
+        configuration = cfg.get_config(section='GLOBAL')
         theclass.onehost(target_host, host_tags, host_base_path, hostdir,
-                         self.cdistargs, parallel=False)
+                         self.cdistargs, parallel=False,
+                         configuration=configuration)
 
 
 class HTTPServerV6(socketserver.ForkingMixIn, http.server.HTTPServer):

From 4a5a475e3e79d4a1b8d50b0b96b5291840ff5bda Mon Sep 17 00:00:00 2001
From: Dominique Roux <dominique.roux4@gmail.com>
Date: Fri, 1 Feb 2019 17:49:14 +0100
Subject: [PATCH 006/366] Updated the man pages for the cdist trigger and preos

---
 docs/src/cdist-preos.rst   | 4 ++--
 docs/src/cdist-trigger.rst | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/docs/src/cdist-preos.rst b/docs/src/cdist-preos.rst
index 59d0d13e..ad88b277 100644
--- a/docs/src/cdist-preos.rst
+++ b/docs/src/cdist-preos.rst
@@ -26,7 +26,7 @@ For example, to create an ubuntu PreOS:
 
     $ cdist preos ubuntu /preos/preos-ubuntu -B -C \
         -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \
-        -t "/usr/bin/curl 192.168.111.5:3000/install/"
+        -t "/usr/bin/curl 192.168.111.5:3000/cdist/install/"
 
 For more info about the available options see the cdist manual page.
 
@@ -148,7 +148,7 @@ Tipical use case for using PreOS and trigger command include the following steps
 
         $ cdist preos ubuntu /preos/ubuntu -b -C \
             -k ~/.ssh/id_rsa.pub -p /preos/pxe \
-            -t "/usr/bin/curl 192.168.111.5:3000/install/"
+            -t "/usr/bin/curl 192.168.111.5:3000/cdist/install/"
 
 #. Configure dhcp server and tftp server.
 
diff --git a/docs/src/cdist-trigger.rst b/docs/src/cdist-trigger.rst
index 94477c29..93e9d920 100644
--- a/docs/src/cdist-trigger.rst
+++ b/docs/src/cdist-trigger.rst
@@ -13,8 +13,10 @@ This command starts trigger server at management node, for example:
 This will start cdist trigger server in verbose mode. cdist trigger server accepts
 simple requests for configuration and for installation:
 
-* :strong:`/cdist/install/.*` for installation
-* :strong:`/cdist/config/.*` for configuration.
+* :strong:`/cdist/install/` for installation
+* :strong:`/cdist/config/` for configuration.
+
+Other configuration parameters are the same as in like cdist config (See `cdist <man1/cdist.html>`_).
 
 Machines can then trigger cdist trigger server with appropriate requests.
 If the request is, for example, for installation (:strong:`/cdist/install/`)

From 3dcd27015811e1e6659c3d13e885f28815a7e9a8 Mon Sep 17 00:00:00 2001
From: Darko Poljak <darko.poljak@gmail.com>
Date: Thu, 1 Nov 2018 17:56:40 +0100
Subject: [PATCH 007/366] Implement python types

---
 cdist/conf/type/__file_py/__init__.py         | 103 +++++
 cdist/conf/type/__file_py/explorer/cksum      |  34 ++
 cdist/conf/type/__file_py/explorer/stat       |  56 +++
 cdist/conf/type/__file_py/explorer/type       |  33 ++
 .../type/__file_py/parameter/default/state    |   1 +
 cdist/conf/type/__file_py/parameter/optional  |   5 +
 cdist/config.py                               |  65 ++-
 cdist/core/__init__.py                        |   1 +
 cdist/core/code.py                            |  42 ++
 cdist/core/manifest.py                        |  75 +++-
 cdist/core/pytypes.py                         | 162 ++++++++
 cdist/exec/local.py                           |  16 +
 cdist/util/python_type_util.py                |  27 ++
 docs/changelog                                |   1 +
 docs/dev/python-types/benchmark               | 376 ++++++++++++++++++
 docs/dev/python-types/benchmark.sh            |  43 ++
 docs/dev/python-types/conf/manifest/pyinit    |   7 +
 docs/dev/python-types/conf/manifest/shinit    |   7 +
 .../conf/type/__dummy_config_py/__init__.py   |  22 +
 .../type/__dummy_config_py/files/dummy.conf   |   1 +
 .../type/__dummy_config_sh/files/dummy.conf   |   1 +
 .../conf/type/__dummy_config_sh/manifest      |   6 +
 docs/dev/python-types/test.sh                 |  27 ++
 docs/dev/python-types/timeit.sh               |  36 ++
 docs/src/cdist-type.rst                       |  86 ++++
 scripts/cdist                                 |   2 +-
 26 files changed, 1224 insertions(+), 11 deletions(-)
 create mode 100644 cdist/conf/type/__file_py/__init__.py
 create mode 100755 cdist/conf/type/__file_py/explorer/cksum
 create mode 100755 cdist/conf/type/__file_py/explorer/stat
 create mode 100755 cdist/conf/type/__file_py/explorer/type
 create mode 100644 cdist/conf/type/__file_py/parameter/default/state
 create mode 100644 cdist/conf/type/__file_py/parameter/optional
 create mode 100644 cdist/core/pytypes.py
 create mode 100644 cdist/util/python_type_util.py
 create mode 100644 docs/dev/python-types/benchmark
 create mode 100755 docs/dev/python-types/benchmark.sh
 create mode 100644 docs/dev/python-types/conf/manifest/pyinit
 create mode 100644 docs/dev/python-types/conf/manifest/shinit
 create mode 100644 docs/dev/python-types/conf/type/__dummy_config_py/__init__.py
 create mode 100644 docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf
 create mode 100644 docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf
 create mode 100644 docs/dev/python-types/conf/type/__dummy_config_sh/manifest
 create mode 100755 docs/dev/python-types/test.sh
 create mode 100755 docs/dev/python-types/timeit.sh

diff --git a/cdist/conf/type/__file_py/__init__.py b/cdist/conf/type/__file_py/__init__.py
new file mode 100644
index 00000000..1212f8fd
--- /dev/null
+++ b/cdist/conf/type/__file_py/__init__.py
@@ -0,0 +1,103 @@
+import os
+import re
+import sys
+from cdist.core.pytypes import *
+
+
+class FileType(PythonType):
+    def get_attribute(self, stat_file, attribute, value_should):
+        if os.path.exists(stat_file):
+            if re.match('[0-9]', value_should):
+                index = 1
+            else:
+                index = 2
+            with open(stat_file, 'r') as f:
+                for line in f:
+                    if re.match(attribute + ":", line):
+                        fields = line.split()
+                        return fields[index]
+            return None
+
+    def set_attribute(self, attribute, value_should, destination):
+        cmd = {
+            'group': 'chgrp',
+            'owner': 'chown',
+            'mode': 'chmod',
+        }
+        self.send_message("{} '{}'".format(cmd[attribute], value_should))
+        return "{} '{}' '{}'".format(cmd[attribute], value_should, destination)
+
+    def type_manifest(self):
+        yield from ()
+
+    def type_gencode(self):
+        typeis = self.get_explorer('type')
+        state_should = self.get_parameter('state')
+
+        if state_should == 'exists' and typeis == 'file':
+            return
+
+        source = self.get_parameter('source')
+        if source == '-':
+            source = self.stdin_path
+        destination = '/' + self.object_id
+        if state_should == 'pre-exists':
+            if source is not None:
+                self.die('--source cannot be used with --state pre-exists')
+            if typeis == 'file':
+                return None
+            else:
+                self.die('File {} does not exist'.format(destination))
+
+        create_file = False
+        upload_file = False
+        set_attributes = False
+        code = []
+        if state_should == 'present' or state_should == 'exists':
+            if source is None:
+                remote_stat = self.get_explorer('stat')
+                if not remote_stat:
+                    create_file = True
+            else:
+                if os.path.exists(source):
+                    if typeis == 'file':
+                        local_cksum = self.run_local(['cksum', source, ])
+                        local_cksum = local_cksum.split()[0]
+                        remote_cksum = self.get_explorer('cksum')
+                        remote_cksum = remote_cksum.split()[0]
+                        upload_file = local_cksum != remote_cksum
+                    else:
+                        upload_file = True
+                else:
+                    self.die('Source {} does not exist'.format(source))
+            if create_file or upload_file:
+                set_attributes = True
+                tempfile_template = '{}.cdist.XXXXXXXXXX'.format(destination)
+                destination_upload = self.run_remote(
+                    ["mktemp", tempfile_template, ])
+                if upload_file:
+                    self.transfer(source, destination_upload)
+                code.append('rm -rf {}'.format(destination))
+                code.append('mv {} {}'.format(destination_upload, destination))
+
+        if state_should in ('present', 'exists', 'pre-exists', ):
+            for attribute in ('group', 'owner', 'mode', ):
+                if attribute in self.parameters:
+                    value_should = self.get_parameter(attribute)
+                    if attribute == 'mode':
+                        value_should = re.sub('^0', '', value_should)
+                    stat_file = self.get_explorer_file('stat')
+                    value_is = self.get_attribute(stat_file, attribute,
+                                                  value_should)
+                    if set_attributes or value_should != value_is:
+                        code.append(self.set_attribute(attribute,
+                                                       value_should,
+                                                       destination))
+        elif state_should == 'absent':
+            if typeis == 'file':
+                code.append('rm -f {}'.format(destination))
+                self.send_message('remove')
+        else:
+            self.die('Unknown state {}'.format(state_should))
+
+        return "\n".join(code)
diff --git a/cdist/conf/type/__file_py/explorer/cksum b/cdist/conf/type/__file_py/explorer/cksum
new file mode 100755
index 00000000..335e4e7a
--- /dev/null
+++ b/cdist/conf/type/__file_py/explorer/cksum
@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Retrieve the md5sum of a file to be created, if it is already existing.
+#
+
+destination="/$__object_id"
+
+if [ -e "$destination" ]; then
+    if [ -f  "$destination" ]; then
+        cksum < "$destination"
+    else
+        echo "NO REGULAR FILE"
+    fi
+else
+    echo "NO FILE FOUND, NO CHECKSUM CALCULATED."
+fi
diff --git a/cdist/conf/type/__file_py/explorer/stat b/cdist/conf/type/__file_py/explorer/stat
new file mode 100755
index 00000000..8a917556
--- /dev/null
+++ b/cdist/conf/type/__file_py/explorer/stat
@@ -0,0 +1,56 @@
+#!/bin/sh
+#
+# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+destination="/$__object_id"
+
+# nothing to work with, nothing we could do
+[ -e "$destination" ] || exit 0
+
+os=$("$__explorer/os")
+case "$os" in
+   "freebsd"|"netbsd"|"openbsd")
+      # FIXME: should be something like this based on man page, but can not test
+      stat -f "type: %ST
+owner: %Du %Su
+group: %Dg %Sg
+mode: %Op %Sp
+size: %Dz
+links: %Dl
+" "$destination"
+   ;;
+   "macosx")
+     stat -f "type: %HT
+owner: %Du %Su
+group: %Dg %Sg
+mode: %Lp %Sp
+size: %Dz
+links: %Dl
+" "$destination"
+   ;;
+   *)
+      stat --printf="type: %F
+owner: %u %U
+group: %g %G
+mode: %a %A
+size: %s
+links: %h
+" "$destination"
+   ;;
+esac
diff --git a/cdist/conf/type/__file_py/explorer/type b/cdist/conf/type/__file_py/explorer/type
new file mode 100755
index 00000000..e723047c
--- /dev/null
+++ b/cdist/conf/type/__file_py/explorer/type
@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+destination="/$__object_id"
+
+if [ ! -e "$destination" ]; then
+   echo none
+elif [ -h "$destination" ]; then
+   echo symlink
+elif [ -f "$destination" ]; then
+   echo file
+elif [ -d "$destination" ]; then
+   echo directory
+else
+   echo unknown
+fi
diff --git a/cdist/conf/type/__file_py/parameter/default/state b/cdist/conf/type/__file_py/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__file_py/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__file_py/parameter/optional b/cdist/conf/type/__file_py/parameter/optional
new file mode 100644
index 00000000..c696d592
--- /dev/null
+++ b/cdist/conf/type/__file_py/parameter/optional
@@ -0,0 +1,5 @@
+state
+group
+mode
+owner
+source
diff --git a/cdist/config.py b/cdist/config.py
index 97cc1da6..74cd1ae2 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -38,6 +38,7 @@ import cdist.hostsource
 import cdist.exec.local
 import cdist.exec.remote
 import cdist.util.ipaddr as ipaddr
+import cdist.util.python_type_util as pytype_util
 import cdist.configuration
 from cdist import core, inventory
 from cdist.util.remoteutil import inspect_ssh_mux_opts
@@ -90,13 +91,15 @@ class Config(object):
                 shutil.rmtree(path)
 
     def __init__(self, local, remote, dry_run=False, jobs=None,
-                 cleanup_cmds=None, remove_remote_files_dirs=False):
+                 cleanup_cmds=None, remove_remote_files_dirs=False,
+                 timestamp=False):
 
         self.local = local
         self.remote = remote
         self._open_logger()
         self.dry_run = dry_run
         self.jobs = jobs
+        self.timestamp = timestamp
         if cleanup_cmds:
             self.cleanup_cmds = cleanup_cmds
         else:
@@ -428,7 +431,8 @@ class Config(object):
                 cleanup_cmds.append(cleanup_cmd)
             c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs,
                     cleanup_cmds=cleanup_cmds,
-                    remove_remote_files_dirs=remove_remote_files_dirs)
+                    remove_remote_files_dirs=remove_remote_files_dirs,
+                    timestamp=args.timestamp)
             c.run()
             cls._remove_paths()
 
@@ -466,6 +470,7 @@ class Config(object):
             'dry' if self.dry_run else 'configuration'))
 
         self._init_files_dirs()
+        self.local.collect_python_types()
 
         self.explorer.run_global_explorers(self.local.global_explorer_out_path)
         try:
@@ -779,15 +784,41 @@ class Config(object):
                     args = [param, cdist_type.name]
                 self.log.warning(format, *args)
 
+    def _timeit(self, func, msg_prefix):
+        def wrapper_func(*args, **kwargs):
+            loglevel = self.log.getEffectiveLevel()
+            if loglevel >= logging.VERBOSE and self.timestamp:
+                start_time = time.time()
+                rv = func(*args, **kwargs)
+                end_time = time.time()
+                duration = end_time - start_time
+                self.log.verbose("%s duration: %.6f seconds",
+                                 msg_prefix, duration)
+            else:
+                rv = func(*args, **kwargs)
+            return rv
+        return wrapper_func
+
     def object_prepare(self, cdist_object, transfer_type_explorers=True):
         """Prepare object: Run type explorer + manifest"""
         self._handle_deprecation(cdist_object)
         self.log.verbose("Preparing object {}".format(cdist_object.name))
         self.log.verbose(
             "Running manifest and explorers for " + cdist_object.name)
-        self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
         try:
-            self.manifest.run_type_manifest(cdist_object)
+            self.log.verbose("Preparing object {}".format(cdist_object.name))
+            self.log.verbose(
+                    "Running manifest and explorers for " + cdist_object.name)
+            self.explorer.run_type_explorers(cdist_object,
+                                             transfer_type_explorers)
+            if pytype_util.is_python_type(cdist_object.cdist_type):
+                self._timeit(self.manifest.run_py_type_manifest,
+                             "Python type manifest for {}".format(
+                                 cdist_object.name))(cdist_object)
+            else:
+                self._timeit(self.manifest.run_type_manifest,
+                             "Type manifest for {}".format(
+                                 cdist_object.name))(cdist_object)
             self.log.trace("[ORDER_DEP] Removing order dep files for %s",
                            cdist_object)
             cdist_object.cleanup()
@@ -805,9 +836,21 @@ class Config(object):
 
             # Generate
             self.log.debug("Generating code for %s" % (cdist_object.name))
-            cdist_object.code_local = self.code.run_gencode_local(cdist_object)
-            cdist_object.code_remote = self.code.run_gencode_remote(
-                cdist_object)
+            if pytype_util.is_python_type(cdist_object.cdist_type):
+                cdist_object.code_local = ''
+                cdist_object.code_remote = self._timeit(
+                    self.code.run_py,
+                    "Python type generate code for {}".format(
+                        cdist_object.name))(cdist_object)
+            else:
+                cdist_object.code_local = self._timeit(
+                    self.code.run_gencode_local,
+                    "Type generate code local for {}".format(
+                        cdist_object.name))(cdist_object)
+                cdist_object.code_remote = self._timeit(
+                    self.code.run_gencode_remote,
+                    "Type generate code remote for {}".format(
+                        cdist_object.name))(cdist_object)
             if cdist_object.code_local or cdist_object.code_remote:
                 cdist_object.changed = True
 
@@ -818,12 +861,16 @@ class Config(object):
                 if cdist_object.code_local:
                     self.log.trace("Executing local code for %s"
                                    % (cdist_object.name))
-                    self.code.run_code_local(cdist_object)
+                    self._timeit(self.code.run_code_local,
+                                 "Type run code local for {}".format(
+                                     cdist_object.name))(cdist_object)
                 if cdist_object.code_remote:
                     self.log.trace("Executing remote code for %s"
                                    % (cdist_object.name))
                     self.code.transfer_code_remote(cdist_object)
-                    self.code.run_code_remote(cdist_object)
+                    self._timeit(self.code.run_code_remote,
+                                 "Type run code remote for {}".format(
+                                     cdist_object.name))(cdist_object)
 
             # Mark this object as done
             self.log.trace("Finishing run of " + cdist_object.name)
diff --git a/cdist/core/__init__.py b/cdist/core/__init__.py
index b79cdb21..de2fbbbf 100644
--- a/cdist/core/__init__.py
+++ b/cdist/core/__init__.py
@@ -29,3 +29,4 @@ from cdist.core.manifest import Manifest
 from cdist.core.code import Code
 from cdist.core.util import listdir
 from cdist.core.util import log_level_env_var_val, log_level_name_env_var_val
+import cdist.core.pytypes
diff --git a/cdist/core/code.py b/cdist/core/code.py
index 1550880a..34d5bd97 100644
--- a/cdist/core/code.py
+++ b/cdist/core/code.py
@@ -22,6 +22,10 @@
 #
 
 import os
+import importlib.util
+import inspect
+import cdist
+from cdist.core.pytypes import PythonType
 from . import util
 
 
@@ -116,6 +120,44 @@ class Code(object):
         if dry_run:
             self.env['__cdist_dry_run'] = '1'
 
+    def run_py(self, cdist_object):
+        cdist_type = cdist_object.cdist_type
+        module_name = cdist_type.name
+        file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
+
+        if os.path.isfile(file_path):
+            spec = importlib.util.spec_from_file_location(module_name,
+                                                          file_path)
+            m = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(m)
+            classes = inspect.getmembers(m, inspect.isclass)
+            type_class = None
+            for _, cl in classes:
+                if cl != PythonType and issubclass(cl, PythonType):
+                    if type_class:
+                        raise cdist.Error("Only one python type class is "
+                                          "supported, but at least two "
+                                          "found: {}".format((type_class,
+                                                              cl, )))
+                    else:
+                        type_class = cl
+            env = os.environ.copy()
+            env.update(self.env)
+            message_prefix = cdist_object.name
+            type_obj = type_class(env=env, cdist_object=cdist_object,
+                                  local=self.local, remote=self.remote,
+                                  message_prefix=message_prefix)
+            if hasattr(type_obj, 'run') and inspect.ismethod(type_obj.run):
+                if self.local.save_output_streams:
+                    which = 'gencode-py'
+                    stderr_path = os.path.join(cdist_object.stderr_path, which)
+                    stdout_path = os.path.join(cdist_object.stdout_path, which)
+                    with open(stderr_path, 'a+') as stderr, \
+                            open(stdout_path, 'a+') as stdout:
+                        return type_obj.run(stdout=stdout, stderr=stderr)
+                else:
+                    return type_obj.run()
+
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.cdist_type
         script = os.path.join(self.local.type_path,
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 8aeaf860..546fba25 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -22,9 +22,13 @@
 
 import logging
 import os
-
+import importlib.util
+import inspect
 import cdist
+import cdist.emulator
 from . import util
+from cdist.core.pytypes import PythonType, Command
+
 
 '''
 common:
@@ -226,3 +230,72 @@ class Manifest(object):
                 pass
         _rm_file(Manifest.ORDER_DEP_STATE_NAME)
         _rm_file(Manifest.TYPEORDER_DEP_NAME)
+
+    def env_py_type_manifest(self, cdist_object):
+        env = os.environ.copy()
+        env.update(self.env)
+        env.update({
+            '__cdist_object_marker': self.local.object_marker_name,
+            '__cdist_manifest': cdist_object.cdist_type,
+            '__manifest': self.local.manifest_path,
+            '__object': cdist_object.absolute_path,
+            '__object_id': cdist_object.object_id,
+            '__object_name': cdist_object.name,
+            '__type': cdist_object.cdist_type.absolute_path,
+        })
+
+        return env
+
+    def run_py_type_manifest(self, cdist_object):
+        cdist_type = cdist_object.cdist_type
+        module_name = cdist_type.name
+        file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
+        message_prefix = cdist_object.name
+        if os.path.isfile(file_path):
+            self.log.verbose("Running python type manifest for object %s",
+                             cdist_object.name)
+            spec = importlib.util.spec_from_file_location(module_name,
+                                                          file_path)
+            m = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(m)
+            classes = inspect.getmembers(m, inspect.isclass)
+            type_class = None
+            for _, cl in classes:
+                if cl != PythonType and issubclass(cl, PythonType):
+                    if type_class:
+                        raise cdist.Error("Only one python type class is "
+                                          "supported, but at least two "
+                                          "found: {}".format((type_class,
+                                                              cl, )))
+                    else:
+                        type_class = cl
+            env = self.env_py_type_manifest(cdist_object)
+            type_obj = type_class(env=env, cdist_object=cdist_object,
+                                  local=self.local, remote=None,
+                                  message_prefix=message_prefix)
+            if self.local.save_output_streams:
+                which = 'manifest'
+                stderr_path = os.path.join(cdist_object.stderr_path, which)
+                stdout_path = os.path.join(cdist_object.stdout_path, which)
+                with open(stderr_path, 'a+') as stderr, \
+                        open(stdout_path, 'a+') as stdout:
+                    self._process_py_type_manifest_entries(
+                        type_obj, env, stdout=stdout, stderr=stderr)
+            else:
+                self._process_py_type_manifest_entries(type_obj, env)
+
+    def _process_py_type_manifest_entries(self, type_obj, env, stdout=None,
+                                          stderr=None):
+        if hasattr(type_obj, 'manifest') and \
+                inspect.ismethod(type_obj.manifest):
+            for cmd in type_obj.manifest(stdout=stdout, stderr=stderr):
+                if not isinstance(cmd, Command):
+                    raise TypeError("Manifest command must be of type Command")
+                kwargs = {
+                    'argv': cmd.cmd_line(),
+                    'env': env,
+                }
+                if cmd.stdin:
+                    kwargs['stdin'] = cmd.stdin
+                emulator = cdist.emulator.Emulator(**kwargs)
+                emulator.run()
diff --git a/cdist/core/pytypes.py b/cdist/core/pytypes.py
new file mode 100644
index 00000000..ae4164ec
--- /dev/null
+++ b/cdist/core/pytypes.py
@@ -0,0 +1,162 @@
+import logging
+import os
+import io
+import sys
+import re
+from cdist import message, Error
+
+
+__all__ = ["PythonType", "Command", "command"]
+
+
+class PythonType:
+    def __init__(self, env, cdist_object, local, remote, message_prefix=None):
+        self.env = env
+        self.cdist_object = cdist_object
+        self.object_id = cdist_object.object_id
+        self.object_name = cdist_object.name
+        self.cdist_type = cdist_object.cdist_type
+        self.local = local
+        self.remote = remote
+        self.object_path = cdist_object.absolute_path
+        self.type_path = cdist_object.cdist_type.absolute_path
+        self.explorer_path = os.path.join(self.object_path, 'explorer')
+        self.parameters = cdist_object.parameters
+        self.stdin_path = os.path.join(self.object_path, 'stdin')
+        self.log = logging.getLogger(
+            self.local.target_host[0] + ':' + self.object_name)
+
+        self.message_prefix = message_prefix
+        self.message = None
+
+    def get_parameter(self, name):
+        return self.parameters.get(name)
+
+    def get_explorer_file(self, name):
+        path = os.path.join(self.explorer_path, name)
+        return path
+
+    def get_explorer(self, name):
+        path = self.get_explorer_file(name)
+        with open(path, 'r') as f:
+            value = f.read()
+            if value:
+                value = value.strip()
+            return value
+
+    def run_local(self, command, env=None):
+        rv = self.local.run(command, env=env, return_output=True)
+        if rv:
+            rv = rv.rstrip('\n')
+        return rv
+
+    def run_remote(self, command, env=None):
+        rv = self.remote.run(command, env=env, return_output=True)
+        if rv:
+            rv = rv.rstrip('\n')
+        return rv
+
+    def transfer(self, source, destination):
+        self.remote.transfer(source, destination)
+
+    def die(self, msg):
+        raise Error("{}: {}".format(self.cdist_object, msg))
+
+    def type_manifest(self):
+        pass
+
+    def type_gencode(self):
+        pass
+
+    def manifest(self, stdout=None, stderr=None):
+        try:
+            if self.message_prefix:
+                self.message = message.Message(self.message_prefix,
+                                               self.local.messages_path)
+                self.env.update(self.message.env)
+            if stdout is not None:
+                stdout_save = sys.stdout
+                sys.stdout = stdout
+            if stderr is not None:
+                stderr_save = sys.stderr
+                sys.stderr = stderr
+            yield from self.type_manifest()
+        finally:
+            if self.message:
+                self.message.merge_messages()
+            if stdout is not None:
+                sys.stdout = stdout_save
+            if stderr is not None:
+                sys.stderr = stderr_save
+
+    def run(self, stdout=None, stderr=None):
+        try:
+            if self.message_prefix:
+                self.message = message.Message(self.message_prefix,
+                                               self.local.messages_path)
+            if stdout is not None:
+                stdout_save = sys.stdout
+                sys.stdout = stdout
+            if stderr is not None:
+                stderr_save = sys.stderr
+                sys.stderr = stderr
+            return self.type_gencode()
+        finally:
+            if self.message:
+                self.message.merge_messages()
+            if stdout is not None:
+                sys.stdout = stdout_save
+            if stderr is not None:
+                sys.stderr = stderr_save
+
+    def send_message(self, msg):
+        if self.message:
+            with open(self.message.messages_out, 'a') as f:
+                print(msg, file=f)
+
+    def receive_message(self, pattern):
+        if self.message:
+            with open(self.message.messages_in, 'r') as f:
+                for line in f:
+                    match = re.search(pattern, line)
+                    if match:
+                        return match
+        return None
+
+
+class Command:
+    def __init__(self, name, *args, **kwargs):
+        self.name = name
+        self.args = args
+        self.kwargs = kwargs
+        self.stdin = None
+
+    def feed_stdin(self, value):
+        # If file-like object then read its value.
+        if value is not None and isinstance(value, io.IOBase):
+            value = value.read()
+
+        # Convert to bytes file-like object.
+        if value is None:
+            self.stdin = None
+        elif isinstance(value, str):
+            self.stdin = io.BytesIO(value.encode('utf-8'))
+        elif isinstance(value, bytes) or isinstance(value, bytearray):
+            self.stdin = io.BytesIO(value)
+        else:
+            raise TypeError("value must be str, bytes, bytearray, file-like "
+                            "object or None")
+        return self
+
+    def cmd_line(self):
+        argv = [self.name, ]
+        for param in self.args:
+            argv.append(param)
+        for key, value in self.kwargs.items():
+            argv.append("--{}".format(key))
+            argv.append(value)
+        return argv
+
+
+def command(name, *args, **kwargs):
+    return Command(name, *args, **kwargs)
diff --git a/cdist/exec/local.py b/cdist/exec/local.py
index ad6c6e36..c1877c90 100644
--- a/cdist/exec/local.py
+++ b/cdist/exec/local.py
@@ -35,6 +35,9 @@ import cdist
 import cdist.message
 from cdist import core
 import cdist.exec.util as util
+import cdist.util.python_type_util as pytype_util
+from cdist.core import pytypes
+import functools
 
 CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ]
 
@@ -370,3 +373,16 @@ class Local(object):
                 raise cdist.Error(
                         "Linking emulator from %s to %s failed: %s" % (
                             src, dst, e.__str__()))
+
+    def collect_python_types(self):
+        for cdist_type in core.CdistType.list_types(self.type_path):
+            if pytype_util.is_python_type(cdist_type):
+                self.log.trace("Detected python type %s, collecting it".format(
+                    cdist_type.name))
+                f = functools.partial(pytypes.command, cdist_type.name)
+                if cdist_type.name.startswith('__'):
+                    attr_name = cdist_type.name.replace('__', '', 1)
+                else:
+                    attr_name = cdist_type.name
+                setattr(pytypes, attr_name, f)
+                pytypes.__all__.append(attr_name)
diff --git a/cdist/util/python_type_util.py b/cdist/util/python_type_util.py
new file mode 100644
index 00000000..fd97684e
--- /dev/null
+++ b/cdist/util/python_type_util.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+#
+# 2019 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import os
+
+
+def is_python_type(cdist_type):
+    init_path = os.path.join(cdist_type.absolute_path, '__init__.py')
+    return os.path.exists(init_path)
diff --git a/docs/changelog b/docs/changelog
index 25566aa8..9d3b94a1 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Core: Add trigger functionality (Nico Schottelius, Darko Poljak)
+	* Core: Implement core support for python types (Darko Poljak)
 
 6.5.1: 2020-02-15
 	* Type __consul_agent: Add Debian 10 support (Nico Schottelius)
diff --git a/docs/dev/python-types/benchmark b/docs/dev/python-types/benchmark
new file mode 100644
index 00000000..8fcf69f3
--- /dev/null
+++ b/docs/dev/python-types/benchmark
@@ -0,0 +1,376 @@
+# sh type, no file at remote
+echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121125154.045799] 185.203.112.26: Starting configuration run
+INFO: [20181121125237.029892] 185.203.112.26: Processing __file/root/foo0.bin
+INFO: [20181121125239.881529] 185.203.112.26: Processing __file/root/foo1.bin
+INFO: [20181121125243.265672] 185.203.112.26: Processing __file/root/foo2.bin
+INFO: [20181121125246.929903] 185.203.112.26: Processing __file/root/foo3.bin
+INFO: [20181121125251.811258] 185.203.112.26: Processing __file/root/foo4.bin
+INFO: [20181121125257.784416] 185.203.112.26: Processing __file/root/foo5.bin
+INFO: [20181121125302.686275] 185.203.112.26: Processing __file/root/foo6.bin
+INFO: [20181121125306.394030] 185.203.112.26: Processing __file/root/foo7.bin
+INFO: [20181121125308.610023] 185.203.112.26: Processing __file/root/foo8.bin
+INFO: [20181121125310.868538] 185.203.112.26: Processing __file/root/foo9.bin
+INFO: [20181121125313.017972] 185.203.112.26: Processing __file/root/foo10.bin
+INFO: [20181121125315.201342] 185.203.112.26: Processing __file/root/foo11.bin
+INFO: [20181121125317.333055] 185.203.112.26: Processing __file/root/foo12.bin
+INFO: [20181121125319.463929] 185.203.112.26: Processing __file/root/foo13.bin
+INFO: [20181121125321.595410] 185.203.112.26: Processing __file/root/foo14.bin
+INFO: [20181121125323.689697] 185.203.112.26: Processing __file/root/foo15.bin
+INFO: [20181121125325.768283] 185.203.112.26: Processing __file/root/foo16.bin
+INFO: [20181121125327.814793] 185.203.112.26: Processing __file/root/foo17.bin
+INFO: [20181121125329.873073] 185.203.112.26: Processing __file/root/foo18.bin
+INFO: [20181121125331.953886] 185.203.112.26: Processing __file/root/foo19.bin
+INFO: [20181121125334.118290] 185.203.112.26: Processing __file/root/foo20.bin
+INFO: [20181121125336.390849] 185.203.112.26: Processing __file/root/foo21.bin
+INFO: [20181121125338.576698] 185.203.112.26: Processing __file/root/foo22.bin
+INFO: [20181121125340.819044] 185.203.112.26: Processing __file/root/foo23.bin
+INFO: [20181121125343.680419] 185.203.112.26: Processing __file/root/foo24.bin
+INFO: [20181121125346.044907] 185.203.112.26: Processing __file/root/foo25.bin
+INFO: [20181121125348.179574] 185.203.112.26: Processing __file/root/foo26.bin
+INFO: [20181121125350.314970] 185.203.112.26: Processing __file/root/foo27.bin
+INFO: [20181121125352.447394] 185.203.112.26: Processing __file/root/foo28.bin
+INFO: [20181121125354.586637] 185.203.112.26: Processing __file/root/foo29.bin
+INFO: [20181121125356.722699] 185.203.112.26: Processing __file/root/foo30.bin
+INFO: [20181121125358.883538] 185.203.112.26: Processing __file/root/foo31.bin
+INFO: [20181121125401.020967] 185.203.112.26: Processing __file/root/foo32.bin
+INFO: [20181121125403.160146] 185.203.112.26: Processing __file/root/foo33.bin
+INFO: [20181121125405.289048] 185.203.112.26: Processing __file/root/foo34.bin
+INFO: [20181121125407.423994] 185.203.112.26: Processing __file/root/foo35.bin
+INFO: [20181121125409.530135] 185.203.112.26: Processing __file/root/foo36.bin
+INFO: [20181121125411.659683] 185.203.112.26: Processing __file/root/foo37.bin
+INFO: [20181121125413.786177] 185.203.112.26: Processing __file/root/foo38.bin
+INFO: [20181121125415.919152] 185.203.112.26: Processing __file/root/foo39.bin
+INFO: [20181121125418.051496] 185.203.112.26: Processing __file/root/foo40.bin
+INFO: [20181121125420.204577] 185.203.112.26: Processing __file/root/foo41.bin
+INFO: [20181121125422.339697] 185.203.112.26: Processing __file/root/foo42.bin
+INFO: [20181121125424.450966] 185.203.112.26: Processing __file/root/foo43.bin
+INFO: [20181121125426.487831] 185.203.112.26: Processing __file/root/foo44.bin
+INFO: [20181121125428.585516] 185.203.112.26: Processing __file/root/foo45.bin
+INFO: [20181121125430.749002] 185.203.112.26: Processing __file/root/foo46.bin
+INFO: [20181121125432.865290] 185.203.112.26: Processing __file/root/foo47.bin
+INFO: [20181121125435.004009] 185.203.112.26: Processing __file/root/foo48.bin
+INFO: [20181121125437.228566] 185.203.112.26: Processing __file/root/foo49.bin
+INFO: [20181121125439.429440] 185.203.112.26: Finished successful run in 165.38 seconds
+
+# sh type, files exist at remote but content changes
+echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121125529.952800] 185.203.112.26: Starting configuration run
+INFO: [20181121125541.175180] 185.203.112.26: Processing __file/root/foo0.bin
+INFO: [20181121125543.219561] 185.203.112.26: Processing __file/root/foo1.bin
+INFO: [20181121125545.116073] 185.203.112.26: Processing __file/root/foo2.bin
+INFO: [20181121125547.011359] 185.203.112.26: Processing __file/root/foo3.bin
+INFO: [20181121125548.916288] 185.203.112.26: Processing __file/root/foo4.bin
+INFO: [20181121125550.821351] 185.203.112.26: Processing __file/root/foo5.bin
+INFO: [20181121125552.723887] 185.203.112.26: Processing __file/root/foo6.bin
+INFO: [20181121125554.635662] 185.203.112.26: Processing __file/root/foo7.bin
+INFO: [20181121125556.568639] 185.203.112.26: Processing __file/root/foo8.bin
+INFO: [20181121125558.508852] 185.203.112.26: Processing __file/root/foo9.bin
+INFO: [20181121125600.464475] 185.203.112.26: Processing __file/root/foo10.bin
+INFO: [20181121125602.429261] 185.203.112.26: Processing __file/root/foo11.bin
+INFO: [20181121125604.428942] 185.203.112.26: Processing __file/root/foo12.bin
+INFO: [20181121125606.442193] 185.203.112.26: Processing __file/root/foo13.bin
+INFO: [20181121125608.474473] 185.203.112.26: Processing __file/root/foo14.bin
+INFO: [20181121125610.535252] 185.203.112.26: Processing __file/root/foo15.bin
+INFO: [20181121125612.609560] 185.203.112.26: Processing __file/root/foo16.bin
+INFO: [20181121125614.708507] 185.203.112.26: Processing __file/root/foo17.bin
+INFO: [20181121125616.824721] 185.203.112.26: Processing __file/root/foo18.bin
+INFO: [20181121125618.924521] 185.203.112.26: Processing __file/root/foo19.bin
+INFO: [20181121125621.007543] 185.203.112.26: Processing __file/root/foo20.bin
+INFO: [20181121125623.133204] 185.203.112.26: Processing __file/root/foo21.bin
+INFO: [20181121125625.333471] 185.203.112.26: Processing __file/root/foo22.bin
+INFO: [20181121125627.396334] 185.203.112.26: Processing __file/root/foo23.bin
+INFO: [20181121125629.526492] 185.203.112.26: Processing __file/root/foo24.bin
+INFO: [20181121125631.628454] 185.203.112.26: Processing __file/root/foo25.bin
+INFO: [20181121125633.743142] 185.203.112.26: Processing __file/root/foo26.bin
+INFO: [20181121125635.952547] 185.203.112.26: Processing __file/root/foo27.bin
+INFO: [20181121125637.986746] 185.203.112.26: Processing __file/root/foo28.bin
+INFO: [20181121125640.020415] 185.203.112.26: Processing __file/root/foo29.bin
+INFO: [20181121125642.081373] 185.203.112.26: Processing __file/root/foo30.bin
+INFO: [20181121125644.174744] 185.203.112.26: Processing __file/root/foo31.bin
+INFO: [20181121125646.286532] 185.203.112.26: Processing __file/root/foo32.bin
+INFO: [20181121125648.396447] 185.203.112.26: Processing __file/root/foo33.bin
+INFO: [20181121125650.460107] 185.203.112.26: Processing __file/root/foo34.bin
+INFO: [20181121125652.557125] 185.203.112.26: Processing __file/root/foo35.bin
+INFO: [20181121125654.667456] 185.203.112.26: Processing __file/root/foo36.bin
+INFO: [20181121125656.746960] 185.203.112.26: Processing __file/root/foo37.bin
+INFO: [20181121125658.854229] 185.203.112.26: Processing __file/root/foo38.bin
+INFO: [20181121125700.968145] 185.203.112.26: Processing __file/root/foo39.bin
+INFO: [20181121125703.109376] 185.203.112.26: Processing __file/root/foo40.bin
+INFO: [20181121125705.318163] 185.203.112.26: Processing __file/root/foo41.bin
+INFO: [20181121125707.440575] 185.203.112.26: Processing __file/root/foo42.bin
+INFO: [20181121125709.551261] 185.203.112.26: Processing __file/root/foo43.bin
+INFO: [20181121125711.657753] 185.203.112.26: Processing __file/root/foo44.bin
+INFO: [20181121125713.774819] 185.203.112.26: Processing __file/root/foo45.bin
+INFO: [20181121125715.887428] 185.203.112.26: Processing __file/root/foo46.bin
+INFO: [20181121125717.995104] 185.203.112.26: Processing __file/root/foo47.bin
+INFO: [20181121125720.110196] 185.203.112.26: Processing __file/root/foo48.bin
+INFO: [20181121125722.232932] 185.203.112.26: Processing __file/root/foo49.bin
+INFO: [20181121125724.451523] 185.203.112.26: Finished successful run in 114.50 seconds
+
+# py type, no file at remote
+echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file_py /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121125812.034197] 185.203.112.26: Starting configuration run
+INFO: [20181121125823.927353] 185.203.112.26: Processing __file_py/root/foo0.bin
+INFO: [20181121125825.715361] 185.203.112.26: Processing __file_py/root/foo1.bin
+INFO: [20181121125827.511296] 185.203.112.26: Processing __file_py/root/foo2.bin
+INFO: [20181121125829.293455] 185.203.112.26: Processing __file_py/root/foo3.bin
+INFO: [20181121125831.086696] 185.203.112.26: Processing __file_py/root/foo4.bin
+INFO: [20181121125832.867582] 185.203.112.26: Processing __file_py/root/foo5.bin
+INFO: [20181121125834.652511] 185.203.112.26: Processing __file_py/root/foo6.bin
+INFO: [20181121125836.450393] 185.203.112.26: Processing __file_py/root/foo7.bin
+INFO: [20181121125838.255152] 185.203.112.26: Processing __file_py/root/foo8.bin
+INFO: [20181121125840.065808] 185.203.112.26: Processing __file_py/root/foo9.bin
+INFO: [20181121125841.889049] 185.203.112.26: Processing __file_py/root/foo10.bin
+INFO: [20181121125843.719280] 185.203.112.26: Processing __file_py/root/foo11.bin
+INFO: [20181121125845.560165] 185.203.112.26: Processing __file_py/root/foo12.bin
+INFO: [20181121125847.416138] 185.203.112.26: Processing __file_py/root/foo13.bin
+INFO: [20181121125849.289851] 185.203.112.26: Processing __file_py/root/foo14.bin
+INFO: [20181121125851.180203] 185.203.112.26: Processing __file_py/root/foo15.bin
+INFO: [20181121125853.074978] 185.203.112.26: Processing __file_py/root/foo16.bin
+INFO: [20181121125855.086107] 185.203.112.26: Processing __file_py/root/foo17.bin
+INFO: [20181121125857.041100] 185.203.112.26: Processing __file_py/root/foo18.bin
+INFO: [20181121125859.025581] 185.203.112.26: Processing __file_py/root/foo19.bin
+INFO: [20181121125901.072067] 185.203.112.26: Processing __file_py/root/foo20.bin
+INFO: [20181121125903.026711] 185.203.112.26: Processing __file_py/root/foo21.bin
+INFO: [20181121125904.994824] 185.203.112.26: Processing __file_py/root/foo22.bin
+INFO: [20181121125906.956296] 185.203.112.26: Processing __file_py/root/foo23.bin
+INFO: [20181121125908.929231] 185.203.112.26: Processing __file_py/root/foo24.bin
+INFO: [20181121125910.882672] 185.203.112.26: Processing __file_py/root/foo25.bin
+INFO: [20181121125912.839834] 185.203.112.26: Processing __file_py/root/foo26.bin
+INFO: [20181121125914.789904] 185.203.112.26: Processing __file_py/root/foo27.bin
+INFO: [20181121125916.743930] 185.203.112.26: Processing __file_py/root/foo28.bin
+INFO: [20181121125918.698258] 185.203.112.26: Processing __file_py/root/foo29.bin
+INFO: [20181121125920.657118] 185.203.112.26: Processing __file_py/root/foo30.bin
+INFO: [20181121125922.618898] 185.203.112.26: Processing __file_py/root/foo31.bin
+INFO: [20181121125924.567847] 185.203.112.26: Processing __file_py/root/foo32.bin
+INFO: [20181121125926.524617] 185.203.112.26: Processing __file_py/root/foo33.bin
+INFO: [20181121125928.396400] 185.203.112.26: Processing __file_py/root/foo34.bin
+INFO: [20181121125930.209237] 185.203.112.26: Processing __file_py/root/foo35.bin
+INFO: [20181121125931.998377] 185.203.112.26: Processing __file_py/root/foo36.bin
+INFO: [20181121125933.786883] 185.203.112.26: Processing __file_py/root/foo37.bin
+INFO: [20181121125935.579348] 185.203.112.26: Processing __file_py/root/foo38.bin
+INFO: [20181121125937.366197] 185.203.112.26: Processing __file_py/root/foo39.bin
+INFO: [20181121125939.155643] 185.203.112.26: Processing __file_py/root/foo40.bin
+INFO: [20181121125941.052837] 185.203.112.26: Processing __file_py/root/foo41.bin
+INFO: [20181121125942.953670] 185.203.112.26: Processing __file_py/root/foo42.bin
+INFO: [20181121125944.781567] 185.203.112.26: Processing __file_py/root/foo43.bin
+INFO: [20181121125946.622485] 185.203.112.26: Processing __file_py/root/foo44.bin
+INFO: [20181121125948.470701] 185.203.112.26: Processing __file_py/root/foo45.bin
+INFO: [20181121125950.356949] 185.203.112.26: Processing __file_py/root/foo46.bin
+INFO: [20181121125952.232014] 185.203.112.26: Processing __file_py/root/foo47.bin
+INFO: [20181121125954.128887] 185.203.112.26: Processing __file_py/root/foo48.bin
+INFO: [20181121125956.037541] 185.203.112.26: Processing __file_py/root/foo49.bin
+INFO: [20181121125957.514738] 185.203.112.26: Finished successful run in 105.48 seconds
+
+# py type, files exist at remote but content changes
+echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file_py /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121130056.484643] 185.203.112.26: Starting configuration run
+INFO: [20181121130108.545059] 185.203.112.26: Processing __file_py/root/foo0.bin
+INFO: [20181121130110.339217] 185.203.112.26: Processing __file_py/root/foo1.bin
+INFO: [20181121130112.136448] 185.203.112.26: Processing __file_py/root/foo2.bin
+INFO: [20181121130113.923820] 185.203.112.26: Processing __file_py/root/foo3.bin
+INFO: [20181121130115.715667] 185.203.112.26: Processing __file_py/root/foo4.bin
+INFO: [20181121130117.508696] 185.203.112.26: Processing __file_py/root/foo5.bin
+INFO: [20181121130119.300839] 185.203.112.26: Processing __file_py/root/foo6.bin
+INFO: [20181121130124.296312] 185.203.112.26: Processing __file_py/root/foo7.bin
+INFO: [20181121130131.109195] 185.203.112.26: Processing __file_py/root/foo8.bin
+INFO: [20181121130133.303817] 185.203.112.26: Processing __file_py/root/foo9.bin
+INFO: [20181121130136.396440] 185.203.112.26: Processing __file_py/root/foo10.bin
+INFO: [20181121130138.443128] 185.203.112.26: Processing __file_py/root/foo11.bin
+INFO: [20181121130140.462868] 185.203.112.26: Processing __file_py/root/foo12.bin
+INFO: [20181121130142.476196] 185.203.112.26: Processing __file_py/root/foo13.bin
+INFO: [20181121130145.937900] 185.203.112.26: Processing __file_py/root/foo14.bin
+INFO: [20181121130148.013672] 185.203.112.26: Processing __file_py/root/foo15.bin
+INFO: [20181121130150.042588] 185.203.112.26: Processing __file_py/root/foo16.bin
+INFO: [20181121130152.050793] 185.203.112.26: Processing __file_py/root/foo17.bin
+INFO: [20181121130154.083089] 185.203.112.26: Processing __file_py/root/foo18.bin
+INFO: [20181121130156.100091] 185.203.112.26: Processing __file_py/root/foo19.bin
+INFO: [20181121130158.103005] 185.203.112.26: Processing __file_py/root/foo20.bin
+INFO: [20181121130200.188390] 185.203.112.26: Processing __file_py/root/foo21.bin
+INFO: [20181121130202.197574] 185.203.112.26: Processing __file_py/root/foo22.bin
+INFO: [20181121130205.269102] 185.203.112.26: Processing __file_py/root/foo23.bin
+INFO: [20181121130208.457011] 185.203.112.26: Processing __file_py/root/foo24.bin
+INFO: [20181121130211.574321] 185.203.112.26: Processing __file_py/root/foo25.bin
+INFO: [20181121130213.719894] 185.203.112.26: Processing __file_py/root/foo26.bin
+INFO: [20181121130215.762977] 185.203.112.26: Processing __file_py/root/foo27.bin
+INFO: [20181121130217.778624] 185.203.112.26: Processing __file_py/root/foo28.bin
+INFO: [20181121130219.840477] 185.203.112.26: Processing __file_py/root/foo29.bin
+INFO: [20181121130221.852389] 185.203.112.26: Processing __file_py/root/foo30.bin
+INFO: [20181121130223.850898] 185.203.112.26: Processing __file_py/root/foo31.bin
+INFO: [20181121130225.858812] 185.203.112.26: Processing __file_py/root/foo32.bin
+INFO: [20181121130227.855295] 185.203.112.26: Processing __file_py/root/foo33.bin
+INFO: [20181121130229.952673] 185.203.112.26: Processing __file_py/root/foo34.bin
+INFO: [20181121130231.956904] 185.203.112.26: Processing __file_py/root/foo35.bin
+INFO: [20181121130233.961954] 185.203.112.26: Processing __file_py/root/foo36.bin
+INFO: [20181121130236.012158] 185.203.112.26: Processing __file_py/root/foo37.bin
+INFO: [20181121130238.024422] 185.203.112.26: Processing __file_py/root/foo38.bin
+INFO: [20181121130241.238800] 185.203.112.26: Processing __file_py/root/foo39.bin
+INFO: [20181121130243.463237] 185.203.112.26: Processing __file_py/root/foo40.bin
+INFO: [20181121130245.610314] 185.203.112.26: Processing __file_py/root/foo41.bin
+INFO: [20181121130247.661385] 185.203.112.26: Processing __file_py/root/foo42.bin
+INFO: [20181121130250.399845] 185.203.112.26: Processing __file_py/root/foo43.bin
+INFO: [20181121130252.832133] 185.203.112.26: Processing __file_py/root/foo44.bin
+INFO: [20181121130254.955658] 185.203.112.26: Processing __file_py/root/foo45.bin
+INFO: [20181121130257.039587] 185.203.112.26: Processing __file_py/root/foo46.bin
+INFO: [20181121130259.178847] 185.203.112.26: Processing __file_py/root/foo47.bin
+INFO: [20181121130301.357922] 185.203.112.26: Processing __file_py/root/foo48.bin
+INFO: [20181121130303.356299] 185.203.112.26: Processing __file_py/root/foo49.bin
+INFO: [20181121130305.144393] 185.203.112.26: Finished successful run in 128.66 seconds
+
+
+
+# init test file content
+head -c 102400 /dev/random > /tmp/test.file
+
+# sh type, no file at remote
+echo 'x=0; while [ $x -lt 50 ]; do __file /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121130612.519698] 185.203.112.26: Starting configuration run
+INFO: [20181121130624.219344] 185.203.112.26: Processing __file/root/foo0.bin
+INFO: [20181121130626.980052] 185.203.112.26: Processing __file/root/foo1.bin
+INFO: [20181121130631.200669] 185.203.112.26: Processing __file/root/foo2.bin
+INFO: [20181121130642.790229] 185.203.112.26: Processing __file/root/foo3.bin
+INFO: [20181121130646.565599] 185.203.112.26: Processing __file/root/foo4.bin
+INFO: [20181121130648.724875] 185.203.112.26: Processing __file/root/foo5.bin
+INFO: [20181121130651.464686] 185.203.112.26: Processing __file/root/foo6.bin
+INFO: [20181121130653.639581] 185.203.112.26: Processing __file/root/foo7.bin
+INFO: [20181121130655.773987] 185.203.112.26: Processing __file/root/foo8.bin
+INFO: [20181121130657.933136] 185.203.112.26: Processing __file/root/foo9.bin
+INFO: [20181121130700.065158] 185.203.112.26: Processing __file/root/foo10.bin
+INFO: [20181121130702.216456] 185.203.112.26: Processing __file/root/foo11.bin
+INFO: [20181121130704.429030] 185.203.112.26: Processing __file/root/foo12.bin
+INFO: [20181121130706.562114] 185.203.112.26: Processing __file/root/foo13.bin
+INFO: [20181121130708.696584] 185.203.112.26: Processing __file/root/foo14.bin
+INFO: [20181121130710.830002] 185.203.112.26: Processing __file/root/foo15.bin
+INFO: [20181121130712.966631] 185.203.112.26: Processing __file/root/foo16.bin
+INFO: [20181121130715.151833] 185.203.112.26: Processing __file/root/foo17.bin
+INFO: [20181121130717.355196] 185.203.112.26: Processing __file/root/foo18.bin
+INFO: [20181121130719.486316] 185.203.112.26: Processing __file/root/foo19.bin
+INFO: [20181121130721.619933] 185.203.112.26: Processing __file/root/foo20.bin
+INFO: [20181121130723.786670] 185.203.112.26: Processing __file/root/foo21.bin
+INFO: [20181121130725.924736] 185.203.112.26: Processing __file/root/foo22.bin
+INFO: [20181121130728.060224] 185.203.112.26: Processing __file/root/foo23.bin
+INFO: [20181121130730.178729] 185.203.112.26: Processing __file/root/foo24.bin
+INFO: [20181121130732.309264] 185.203.112.26: Processing __file/root/foo25.bin
+INFO: [20181121130734.479895] 185.203.112.26: Processing __file/root/foo26.bin
+INFO: [20181121130736.653085] 185.203.112.26: Processing __file/root/foo27.bin
+INFO: [20181121130738.814291] 185.203.112.26: Processing __file/root/foo28.bin
+INFO: [20181121130741.029646] 185.203.112.26: Processing __file/root/foo29.bin
+INFO: [20181121130743.128717] 185.203.112.26: Processing __file/root/foo30.bin
+INFO: [20181121130745.233272] 185.203.112.26: Processing __file/root/foo31.bin
+INFO: [20181121130747.364681] 185.203.112.26: Processing __file/root/foo32.bin
+INFO: [20181121130749.491793] 185.203.112.26: Processing __file/root/foo33.bin
+INFO: [20181121130751.620492] 185.203.112.26: Processing __file/root/foo34.bin
+INFO: [20181121130753.743519] 185.203.112.26: Processing __file/root/foo35.bin
+INFO: [20181121130755.862169] 185.203.112.26: Processing __file/root/foo36.bin
+INFO: [20181121130758.000172] 185.203.112.26: Processing __file/root/foo37.bin
+INFO: [20181121130800.090405] 185.203.112.26: Processing __file/root/foo38.bin
+INFO: [20181121130802.211849] 185.203.112.26: Processing __file/root/foo39.bin
+INFO: [20181121130804.356363] 185.203.112.26: Processing __file/root/foo40.bin
+INFO: [20181121130806.548412] 185.203.112.26: Processing __file/root/foo41.bin
+INFO: [20181121130808.671279] 185.203.112.26: Processing __file/root/foo42.bin
+INFO: [20181121130810.752813] 185.203.112.26: Processing __file/root/foo43.bin
+INFO: [20181121130812.844502] 185.203.112.26: Processing __file/root/foo44.bin
+INFO: [20181121130814.950501] 185.203.112.26: Processing __file/root/foo45.bin
+INFO: [20181121130817.040587] 185.203.112.26: Processing __file/root/foo46.bin
+INFO: [20181121130819.175850] 185.203.112.26: Processing __file/root/foo47.bin
+INFO: [20181121130821.332900] 185.203.112.26: Processing __file/root/foo48.bin
+INFO: [20181121130823.543119] 185.203.112.26: Processing __file/root/foo49.bin
+INFO: [20181121130825.833163] 185.203.112.26: Finished successful run in 133.31 seconds
+
+# sh type, files exist at remote
+echo 'x=0; while [ $x -lt 50 ]; do __file /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121130854.980007] 185.203.112.26: Starting configuration run
+INFO: [20181121130957.927705] 185.203.112.26: Finished successful run in 62.95 seconds
+
+# py type, no file at remote
+echo 'x=0; while [ $x -lt 50 ]; do __file_py /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121131110.179480] 185.203.112.26: Starting configuration run
+INFO: [20181121131122.086849] 185.203.112.26: Processing __file_py/root/foo0.bin
+INFO: [20181121131123.876029] 185.203.112.26: Processing __file_py/root/foo1.bin
+INFO: [20181121131125.668664] 185.203.112.26: Processing __file_py/root/foo2.bin
+INFO: [20181121131127.460721] 185.203.112.26: Processing __file_py/root/foo3.bin
+INFO: [20181121131129.591229] 185.203.112.26: Processing __file_py/root/foo4.bin
+INFO: [20181121131131.390379] 185.203.112.26: Processing __file_py/root/foo5.bin
+INFO: [20181121131133.195275] 185.203.112.26: Processing __file_py/root/foo6.bin
+INFO: [20181121131135.006282] 185.203.112.26: Processing __file_py/root/foo7.bin
+INFO: [20181121131136.834448] 185.203.112.26: Processing __file_py/root/foo8.bin
+INFO: [20181121131138.659301] 185.203.112.26: Processing __file_py/root/foo9.bin
+INFO: [20181121131140.496856] 185.203.112.26: Processing __file_py/root/foo10.bin
+INFO: [20181121131142.367813] 185.203.112.26: Processing __file_py/root/foo11.bin
+INFO: [20181121131144.239817] 185.203.112.26: Processing __file_py/root/foo12.bin
+INFO: [20181121131146.133314] 185.203.112.26: Processing __file_py/root/foo13.bin
+INFO: [20181121131148.049380] 185.203.112.26: Processing __file_py/root/foo14.bin
+INFO: [20181121131149.974696] 185.203.112.26: Processing __file_py/root/foo15.bin
+INFO: [20181121131151.929083] 185.203.112.26: Processing __file_py/root/foo16.bin
+INFO: [20181121131153.923590] 185.203.112.26: Processing __file_py/root/foo17.bin
+INFO: [20181121131155.874910] 185.203.112.26: Processing __file_py/root/foo18.bin
+INFO: [20181121131157.857904] 185.203.112.26: Processing __file_py/root/foo19.bin
+INFO: [20181121131159.902006] 185.203.112.26: Processing __file_py/root/foo20.bin
+INFO: [20181121131201.859840] 185.203.112.26: Processing __file_py/root/foo21.bin
+INFO: [20181121131203.810875] 185.203.112.26: Processing __file_py/root/foo22.bin
+INFO: [20181121131205.763291] 185.203.112.26: Processing __file_py/root/foo23.bin
+INFO: [20181121131207.710932] 185.203.112.26: Processing __file_py/root/foo24.bin
+INFO: [20181121131209.658154] 185.203.112.26: Processing __file_py/root/foo25.bin
+INFO: [20181121131211.615374] 185.203.112.26: Processing __file_py/root/foo26.bin
+INFO: [20181121131213.569721] 185.203.112.26: Processing __file_py/root/foo27.bin
+INFO: [20181121131215.522624] 185.203.112.26: Processing __file_py/root/foo28.bin
+INFO: [20181121131217.471128] 185.203.112.26: Processing __file_py/root/foo29.bin
+INFO: [20181121131219.421712] 185.203.112.26: Processing __file_py/root/foo30.bin
+INFO: [20181121131221.375699] 185.203.112.26: Processing __file_py/root/foo31.bin
+INFO: [20181121131223.327672] 185.203.112.26: Processing __file_py/root/foo32.bin
+INFO: [20181121131225.281373] 185.203.112.26: Processing __file_py/root/foo33.bin
+INFO: [20181121131227.256711] 185.203.112.26: Processing __file_py/root/foo34.bin
+INFO: [20181121131229.209255] 185.203.112.26: Processing __file_py/root/foo35.bin
+INFO: [20181121131231.170170] 185.203.112.26: Processing __file_py/root/foo36.bin
+INFO: [20181121131233.123407] 185.203.112.26: Processing __file_py/root/foo37.bin
+INFO: [20181121131235.077713] 185.203.112.26: Processing __file_py/root/foo38.bin
+INFO: [20181121131237.017138] 185.203.112.26: Processing __file_py/root/foo39.bin
+INFO: [20181121131238.988189] 185.203.112.26: Processing __file_py/root/foo40.bin
+INFO: [20181121131241.026849] 185.203.112.26: Processing __file_py/root/foo41.bin
+INFO: [20181121131242.978335] 185.203.112.26: Processing __file_py/root/foo42.bin
+INFO: [20181121131244.934562] 185.203.112.26: Processing __file_py/root/foo43.bin
+INFO: [20181121131246.885320] 185.203.112.26: Processing __file_py/root/foo44.bin
+INFO: [20181121131248.835008] 185.203.112.26: Processing __file_py/root/foo45.bin
+INFO: [20181121131250.789727] 185.203.112.26: Processing __file_py/root/foo46.bin
+INFO: [20181121131252.738686] 185.203.112.26: Processing __file_py/root/foo47.bin
+INFO: [20181121131254.691465] 185.203.112.26: Processing __file_py/root/foo48.bin
+INFO: [20181121131256.640896] 185.203.112.26: Processing __file_py/root/foo49.bin
+INFO: [20181121131258.194372] 185.203.112.26: Finished successful run in 108.01 seconds
+
+# py type, files exist at remote
+echo 'x=0; while [ $x -lt 50 ]; do __file_py /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121131327.054523] 185.203.112.26: Starting configuration run
+INFO: [20181121131428.031761] 185.203.112.26: Finished successful run in 60.98 seconds
+
+
+# Summary
+
+# sh type, no file at remote
+INFO: [20181121125439.429440] 185.203.112.26: Finished successful run in 165.38 seconds
+# py type, no file at remote
+INFO: [20181121125957.514738] 185.203.112.26: Finished successful run in 105.48 seconds
+
+# sh type, files exist at remote but content changes
+INFO: [20181121125724.451523] 185.203.112.26: Finished successful run in 114.50 seconds
+# py type, files exist at remote but content changes
+INFO: [20181121130305.144393] 185.203.112.26: Finished successful run in 128.66 seconds
+
+
+# sh type, no file at remote
+INFO: [20181121130825.833163] 185.203.112.26: Finished successful run in 133.31 seconds
+# py type, no file at remote
+INFO: [20181121131258.194372] 185.203.112.26: Finished successful run in 108.01 seconds
+
+# sh type, files exist at remote
+INFO: [20181121130957.927705] 185.203.112.26: Finished successful run in 62.95 seconds
+# py type, files exist at remote
+INFO: [20181121131428.031761] 185.203.112.26: Finished successful run in 60.98 seconds
diff --git a/docs/dev/python-types/benchmark.sh b/docs/dev/python-types/benchmark.sh
new file mode 100755
index 00000000..3e01941f
--- /dev/null
+++ b/docs/dev/python-types/benchmark.sh
@@ -0,0 +1,43 @@
+#!/bin/sh
+
+# Addapt to your env.
+CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf"
+export CDIST_PATH
+TARGET_HOST=185.203.112.26
+
+if [ $# -eq 0 ]
+then
+    N=1
+else
+    N=$1
+fi
+
+manifest() {
+    bytes=$(echo "$1 * 1024" | bc)
+    echo "head -c ${bytes} /dev/random | __file$2 /root/foo$3.bin --source - --mode 0640 --owner root --group root"
+}
+
+verbosity="-vv" #"-vvv"
+i=0
+while [ "$i" -lt "$N" ]
+do
+    if [ "$N" -ne 1 ]
+    then
+        printf "iteration %d\\n" "$i"
+    fi
+    printf "shinit clean state...\\n"
+    ssh root@${TARGET_HOST} 'rm foo$i.bin;'
+    manifest 50 "" $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
+
+    printf "pyinit clean state...\\n"
+    ssh root@${TARGET_HOST} 'rm foo$i.bin;'
+    manifest 50 '_py' $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
+
+    printf "shinit present state...\\n"
+    manifest 50 "" $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
+
+    printf "pyinit present state...\\n"
+    manifest 50 '_py' $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
+
+    i=$((i + 1))
+done
diff --git a/docs/dev/python-types/conf/manifest/pyinit b/docs/dev/python-types/conf/manifest/pyinit
new file mode 100644
index 00000000..53f15a97
--- /dev/null
+++ b/docs/dev/python-types/conf/manifest/pyinit
@@ -0,0 +1,7 @@
+#for x in 1; do
+#    echo xxx${x} | __file_py /root/foobar${x} --source - --mode 0640 --owner root --group root;
+#done
+#__dummy_config_py test1
+
+echo xxx | __file_py /root/foobar --source - --mode 0640 --owner root --group root
+__dummy_config_py test1
diff --git a/docs/dev/python-types/conf/manifest/shinit b/docs/dev/python-types/conf/manifest/shinit
new file mode 100644
index 00000000..44129546
--- /dev/null
+++ b/docs/dev/python-types/conf/manifest/shinit
@@ -0,0 +1,7 @@
+#for x in 1; do
+#    echo xxx${x} | __file /root/foobar${x} --source - --mode 0640 --owner root --group root;
+#done
+#__dummy_config_sh test1
+
+echo xxx | __file /root/foobar --source - --mode 0640 --owner root --group root
+__dummy_config_sh test1
diff --git a/docs/dev/python-types/conf/type/__dummy_config_py/__init__.py b/docs/dev/python-types/conf/type/__dummy_config_py/__init__.py
new file mode 100644
index 00000000..0b5fa692
--- /dev/null
+++ b/docs/dev/python-types/conf/type/__dummy_config_py/__init__.py
@@ -0,0 +1,22 @@
+import os
+import sys
+from cdist.core.pytypes import *
+
+
+class DummyConfig(PythonType):
+    def type_manifest(self):
+        print('dummy manifest stdout')
+        print('dummy manifest stderr\n', file=sys.stderr)
+        yield file_py('/root/dummy1.conf',
+                      mode='0640',
+                      owner='root',
+                      group='root',
+                      source='-').feed_stdin('dummy=1\n')
+
+        self_path = os.path.dirname(os.path.realpath(__file__))
+        conf_path = os.path.join(self_path, 'files', 'dummy.conf')
+        yield file_py('/root/dummy2.conf',
+                      mode='0640',
+                      owner='root',
+                      group='root',
+                      source=conf_path)
diff --git a/docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf b/docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf
new file mode 100644
index 00000000..972d11ca
--- /dev/null
+++ b/docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf
@@ -0,0 +1 @@
+dummy=2
diff --git a/docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf b/docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf
new file mode 100644
index 00000000..972d11ca
--- /dev/null
+++ b/docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf
@@ -0,0 +1 @@
+dummy=2
diff --git a/docs/dev/python-types/conf/type/__dummy_config_sh/manifest b/docs/dev/python-types/conf/type/__dummy_config_sh/manifest
new file mode 100644
index 00000000..d675d6a3
--- /dev/null
+++ b/docs/dev/python-types/conf/type/__dummy_config_sh/manifest
@@ -0,0 +1,6 @@
+printf 'dummy manifest stdout\n'
+printf 'dummy manifest stderr\n' >&2
+
+printf "dummy=1\\n" | __file /root/dummy1.conf --mode 0640 --owner root --group root --source -
+
+__file /root/dummy2.conf --mode 0600 --owner root --group root --source  "$__type/files/dummy.conf"
diff --git a/docs/dev/python-types/test.sh b/docs/dev/python-types/test.sh
new file mode 100755
index 00000000..941ea264
--- /dev/null
+++ b/docs/dev/python-types/test.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# Addapt to your env.
+CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf"
+export CDIST_PATH
+TARGET_HOST=185.203.112.26
+env | grep CDIST_PATH
+
+for streams in ' ' '-S'
+do
+    for x in sh py
+    do
+        printf "[%s] Removing old foobar* files\\n" "$x"
+        printf -- "----------------\\n"
+        ssh root@${TARGET_HOST} 'rm foobar*; rm dummy*'
+        printf "[%s] Listing foobar* files\\n" "$x"
+        printf -- "----------------\\n"
+        ssh root@${TARGET_HOST} 'ls foobar* dummy*'
+        printf "[%s] Running cdist config, streams: %s\\n" "$x" "$streams"
+        printf -- "----------------\\n"
+        ./bin/cdist config -P ${streams} -v -i ./docs/dev/python-types/conf/manifest/${x}init  -- ${TARGET_HOST}
+        printf "[%s] Listing foobar* files\\n" "$x"
+        printf -- "----------------\\n"
+        ssh root@${TARGET_HOST} 'ls foobar* dummy*'
+        ./bin/cdist config -P ${streams} -v -i ./docs/dev/python-types/conf/manifest/${x}init  -- ${TARGET_HOST}
+    done
+done
diff --git a/docs/dev/python-types/timeit.sh b/docs/dev/python-types/timeit.sh
new file mode 100755
index 00000000..f1b6c9fb
--- /dev/null
+++ b/docs/dev/python-types/timeit.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+# Addapt to your env.
+CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf"
+export CDIST_PATH
+TARGET_HOST=185.203.112.26
+
+if [ $# -eq 0 ]
+then
+    N=1
+else
+    N=$1
+fi
+
+i=0
+while [ "$i" -lt "$N" ]
+do
+    if [ "$N" -ne 1 ]
+    then
+        printf "iteration %d\\n" "$i"
+    fi
+    printf "shinit clean state...\\n"
+    ssh root@${TARGET_HOST} 'rm foobar*; rm dummy*;'
+
+    time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/shinit ${TARGET_HOST}
+    printf "pyinit clean state...\\n"
+    ssh root@$${TARGET_HOST} 'rm foobar*; rm dummy*;'
+    time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/pyinit ${TARGET_HOST}
+
+    printf "shinit present state...\\n"
+    time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/shinit ${TARGET_HOST}
+
+    printf "pyinit present state...\\n"
+    time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/pyinit ${TARGET_HOST}
+    i=$((i + 1))
+done
diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst
index 582c0938..3a4a0b13 100644
--- a/docs/src/cdist-type.rst
+++ b/docs/src/cdist-type.rst
@@ -522,3 +522,89 @@ How to include a type into upstream cdist
 If you think your type may be useful for others, ensure it works with the
 current master branch of cdist and have a look at `cdist hacking <cdist-hacker.html>`_ on
 how to submit it.
+
+
+Python types
+------------
+From version/branch **beta** cdist support python types, types that are written
+in python language with cdist's core support. cdist detects such type if type is
+detectable as a python package, i.e. if **__init__.py** file is present in type's
+root directory. Upon that detection cdist will try to run such type as core python
+type.
+
+Note that this differs from plain cdist type where scripts are written in pure
+python and have a proper shebang.
+
+Core python types replace manifest and gencode scripts. Parameters, singleton,
+nonparallel are still defined as for common types. Explorer code is also written
+in shell, since this is the code that is directly executed at target host.
+
+When writing python type you can extend **cdist.core.pytypes.PythonType** class.
+You need to implement the following methods:
+
+* **type_manifest**: implementation should yield **cdist.core.pytypes.<type-name>**
+  attribute function call result, or **yield from ()** if type does not use other types
+* **type_gencode**: implementation should return a string consisting of lines
+  of shell code that will be executed at target host.
+
+**cdist.core.pytypes.<type-name>** attributes correspond to detected python types.
+**Note** that double underscore ('__') at the beginning of type name is removed.
+
+Example:
+
+.. code-block:: sh
+
+    import os
+    import sys
+    from cdist.core.pytypes import *
+
+
+    class DummyConfig(PythonType):
+        def type_manifest(self):
+            print('dummy manifest stdout')
+            print('dummy manifest stderr\n', file=sys.stderr)
+            yield file_py('/root/dummy1.conf',
+                          mode='0640',
+                          owner='root',
+                          group='root',
+                          source='-').feed_stdin('dummy=1\n')
+
+            self_path = os.path.dirname(os.path.realpath(__file__))
+            conf_path = os.path.join(self_path, 'files', 'dummy.conf')
+            yield file_py('/root/dummy2.conf',
+                          mode='0640',
+                          owner='root',
+                          group='root',
+                          source=conf_path)
+
+**cdist.core.PythonType** class provides the following methods:
+
+* **get_parameter**: get type parameter
+* **get_explorer_file**: get path to file for specified explorer
+* **get_explorer**: get value for specified explorer
+* **run_local**: run specified command locally
+* **run_remote**: run specified command remotely
+* **transfer**: transfer specified source to the remote
+* **die**: raise error
+* **send_message**: send message
+* **receive_message**: get message.
+
+When running python type, cdist will save output streams to **gencode-py**,
+stdout and stderr output files.
+
+As a reference implementation you can take a look at **__file_py** type,
+which is re-implementation of **__file** type.
+
+Furthermore, under **docs/dev/python-types** there are sample cdist conf directory,
+init manifests and scripts for running and measuring duration of samples.
+There, under **conf/type/__dummy_config** you can find another example of
+python type, which (unlike **__file_py** type) also uses new manifest implementation
+that yields **cdist.core.pytypes.<type-name>** attribute function call results.
+
+**NOTE** that python types implementation is under the beta, not directly controled by
+the **-b/--beta** option. It is controled by the explicit usage of python types in
+your config.
+
+Also, this documenation is only an introduction, and not a complete guide to python
+types. Currently, it is just a short introduction so one can start to write and use
+python types.
diff --git a/scripts/cdist b/scripts/cdist
index 7bf12c01..664504a0 100755
--- a/scripts/cdist
+++ b/scripts/cdist
@@ -60,7 +60,7 @@ def commandline():
 
 
 if __name__ == "__main__":
-    cdistpythonversion = '3.2'
+    cdistpythonversion = '3.5'
     if sys.version < cdistpythonversion:
         print('Python >= {} is required on the source host.'.format(
                 cdistpythonversion), file=sys.stderr)

From a2243cf59edb145bb3341b9ad56b14428bc90197 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 22 Jun 2019 12:55:56 +0200
Subject: [PATCH 008/366] Add support for python type defined argument parser

---
 cdist/conf/type/__file_py/__init__.py | 14 ++++++-
 cdist/core/code.py                    | 25 ++---------
 cdist/core/manifest.py                | 25 ++---------
 cdist/core/pytypes.py                 | 60 ++++++++++++++++++++-------
 cdist/emulator.py                     | 34 ++++++++++++---
 docs/src/cdist-type.rst               |  3 ++
 6 files changed, 96 insertions(+), 65 deletions(-)

diff --git a/cdist/conf/type/__file_py/__init__.py b/cdist/conf/type/__file_py/__init__.py
index 1212f8fd..3397b417 100644
--- a/cdist/conf/type/__file_py/__init__.py
+++ b/cdist/conf/type/__file_py/__init__.py
@@ -1,7 +1,7 @@
 import os
 import re
-import sys
 from cdist.core.pytypes import *
+import argparse
 
 
 class FileType(PythonType):
@@ -101,3 +101,15 @@ class FileType(PythonType):
             self.die('Unknown state {}'.format(state_should))
 
         return "\n".join(code)
+
+    def get_args_parser(self):
+        parser = argparse.ArgumentParser(add_help=False,
+                                         argument_default=argparse.SUPPRESS)
+        parser.add_argument('--state', dest='state', action='store',
+                            required=False, default='present')
+        for param in ('group', 'mode', 'owner', 'source'):
+            parser.add_argument('--' + param, dest=param, action='store',
+                                required=False, default=None)
+
+        parser.add_argument("object_id", nargs=1)
+        return parser
diff --git a/cdist/core/code.py b/cdist/core/code.py
index 34d5bd97..ae1fddf2 100644
--- a/cdist/core/code.py
+++ b/cdist/core/code.py
@@ -22,10 +22,8 @@
 #
 
 import os
-import importlib.util
 import inspect
-import cdist
-from cdist.core.pytypes import PythonType
+from cdist.core.pytypes import get_pytype_class
 from . import util
 
 
@@ -122,25 +120,8 @@ class Code(object):
 
     def run_py(self, cdist_object):
         cdist_type = cdist_object.cdist_type
-        module_name = cdist_type.name
-        file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
-
-        if os.path.isfile(file_path):
-            spec = importlib.util.spec_from_file_location(module_name,
-                                                          file_path)
-            m = importlib.util.module_from_spec(spec)
-            spec.loader.exec_module(m)
-            classes = inspect.getmembers(m, inspect.isclass)
-            type_class = None
-            for _, cl in classes:
-                if cl != PythonType and issubclass(cl, PythonType):
-                    if type_class:
-                        raise cdist.Error("Only one python type class is "
-                                          "supported, but at least two "
-                                          "found: {}".format((type_class,
-                                                              cl, )))
-                    else:
-                        type_class = cl
+        type_class = get_pytype_class(cdist_type)
+        if type_class is not None:
             env = os.environ.copy()
             env.update(self.env)
             message_prefix = cdist_object.name
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 546fba25..16dec465 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -22,12 +22,11 @@
 
 import logging
 import os
-import importlib.util
 import inspect
 import cdist
 import cdist.emulator
 from . import util
-from cdist.core.pytypes import PythonType, Command
+from cdist.core.pytypes import Command, get_pytype_class
 
 
 '''
@@ -248,27 +247,11 @@ class Manifest(object):
 
     def run_py_type_manifest(self, cdist_object):
         cdist_type = cdist_object.cdist_type
-        module_name = cdist_type.name
-        file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
-        message_prefix = cdist_object.name
-        if os.path.isfile(file_path):
+        type_class = get_pytype_class(cdist_type)
+        if type_class is not None:
             self.log.verbose("Running python type manifest for object %s",
                              cdist_object.name)
-            spec = importlib.util.spec_from_file_location(module_name,
-                                                          file_path)
-            m = importlib.util.module_from_spec(spec)
-            spec.loader.exec_module(m)
-            classes = inspect.getmembers(m, inspect.isclass)
-            type_class = None
-            for _, cl in classes:
-                if cl != PythonType and issubclass(cl, PythonType):
-                    if type_class:
-                        raise cdist.Error("Only one python type class is "
-                                          "supported, but at least two "
-                                          "found: {}".format((type_class,
-                                                              cl, )))
-                    else:
-                        type_class = cl
+            message_prefix = cdist_object.name
             env = self.env_py_type_manifest(cdist_object)
             type_obj = type_class(env=env, cdist_object=cdist_object,
                                   local=self.local, remote=None,
diff --git a/cdist/core/pytypes.py b/cdist/core/pytypes.py
index ae4164ec..9dfc8df9 100644
--- a/cdist/core/pytypes.py
+++ b/cdist/core/pytypes.py
@@ -4,6 +4,9 @@ import io
 import sys
 import re
 from cdist import message, Error
+import importlib.util
+import inspect
+import cdist
 
 
 __all__ = ["PythonType", "Command", "command"]
@@ -13,18 +16,20 @@ class PythonType:
     def __init__(self, env, cdist_object, local, remote, message_prefix=None):
         self.env = env
         self.cdist_object = cdist_object
-        self.object_id = cdist_object.object_id
-        self.object_name = cdist_object.name
-        self.cdist_type = cdist_object.cdist_type
         self.local = local
         self.remote = remote
-        self.object_path = cdist_object.absolute_path
-        self.type_path = cdist_object.cdist_type.absolute_path
-        self.explorer_path = os.path.join(self.object_path, 'explorer')
-        self.parameters = cdist_object.parameters
-        self.stdin_path = os.path.join(self.object_path, 'stdin')
-        self.log = logging.getLogger(
-            self.local.target_host[0] + ':' + self.object_name)
+        if self.cdist_object:
+            self.object_id = cdist_object.object_id
+            self.object_name = cdist_object.name
+            self.cdist_type = cdist_object.cdist_type
+            self.object_path = cdist_object.absolute_path
+            self.explorer_path = os.path.join(self.object_path, 'explorer')
+            self.type_path = cdist_object.cdist_type.absolute_path
+            self.parameters = cdist_object.parameters
+            self.stdin_path = os.path.join(self.object_path, 'stdin')
+        if self.local:
+            self.log = logging.getLogger(
+                self.local.target_host[0] + ':' + self.object_name)
 
         self.message_prefix = message_prefix
         self.message = None
@@ -62,12 +67,6 @@ class PythonType:
     def die(self, msg):
         raise Error("{}: {}".format(self.cdist_object, msg))
 
-    def type_manifest(self):
-        pass
-
-    def type_gencode(self):
-        pass
-
     def manifest(self, stdout=None, stderr=None):
         try:
             if self.message_prefix:
@@ -123,6 +122,15 @@ class PythonType:
                         return match
         return None
 
+    def get_args_parser(self):
+        pass
+
+    def type_manifest(self):
+        pass
+
+    def type_gencode(self):
+        pass
+
 
 class Command:
     def __init__(self, name, *args, **kwargs):
@@ -160,3 +168,23 @@ class Command:
 
 def command(name, *args, **kwargs):
     return Command(name, *args, **kwargs)
+
+
+def get_pytype_class(cdist_type):
+    module_name = cdist_type.name
+    file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
+    type_class = None
+    if os.path.isfile(file_path):
+        spec = importlib.util.spec_from_file_location(module_name, file_path)
+        m = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(m)
+        classes = inspect.getmembers(m, inspect.isclass)
+        for _, cl in classes:
+            if cl != PythonType and issubclass(cl, PythonType):
+                if type_class:
+                    raise cdist.Error(
+                        "Only one python type class is supported, but at least"
+                        " two found: {}".format((type_class, cl, )))
+                else:
+                    type_class = cl
+    return type_class
diff --git a/cdist/emulator.py b/cdist/emulator.py
index 4800e2a3..7dc24a83 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -30,6 +30,9 @@ import cdist
 from cdist import core
 from cdist import flock
 from cdist.core.manifest import Manifest
+import cdist.util.python_type_util as pytype_util
+from cdist.core.pytypes import get_pytype_class
+import inspect
 
 
 class MissingRequiredEnvironmentVariableError(cdist.Error):
@@ -100,7 +103,28 @@ class Emulator(object):
     def run(self):
         """Emulate type commands (i.e. __file and co)"""
 
-        self.commandline()
+        args_parser = None
+        if pytype_util.is_python_type(self.cdist_type):
+            type_class = get_pytype_class(self.cdist_type)
+            if type_class is not None:
+                # We only need to call parse_args so we need plain instance.
+                type_obj = type_class(env=None, cdist_object=None, local=None,
+                                      remote=None)
+                if (hasattr(type_obj, 'get_args_parser') and
+                        inspect.ismethod(type_obj.get_args_parser)):
+                    args_parser = type_obj.get_args_parser()
+                    self.log.trace("Using python type argument parser")
+                    print("Using python type argument parser")
+            if args_parser is None:
+                # Fallback to classic way.
+                args_parser = self.get_args_parser()
+                self.log.trace("Fallback to classic argument parser")
+                print("Fallback to classic argument parser")
+        else:
+            args_parser = self.get_args_parser()
+            self.log.trace("Using emulator classic argument parser")
+            print("Using emulator classic argument parser")
+        self.commandline(args_parser)
         self.init_object()
 
         # locking for parallel execution
@@ -131,9 +155,7 @@ class Emulator(object):
 
         self.log = logging.getLogger(self.target_host[0])
 
-    def commandline(self):
-        """Parse command line"""
-
+    def get_args_parser(self):
         parser = argparse.ArgumentParser(add_help=False,
                                          argument_default=argparse.SUPPRESS)
 
@@ -165,8 +187,10 @@ class Emulator(object):
         # If not singleton support one positional parameter
         if not self.cdist_type.is_singleton:
             parser.add_argument("object_id", nargs=1)
+        return parser
 
-        # And finally parse/verify parameter
+    def commandline(self, parser):
+        """Parse command line"""
         self.args = parser.parse_args(self.argv[1:])
         self.log.trace('Args: %s' % self.args)
 
diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst
index 3a4a0b13..388c3caf 100644
--- a/docs/src/cdist-type.rst
+++ b/docs/src/cdist-type.rst
@@ -542,6 +542,9 @@ in shell, since this is the code that is directly executed at target host.
 When writing python type you can extend **cdist.core.pytypes.PythonType** class.
 You need to implement the following methods:
 
+* **get_args_parser**: implementation should return **argparse.ArgumentParser** and if
+  it is undefined or returned None then cdist falls back to classic type parameter
+  definition and argument parsing
 * **type_manifest**: implementation should yield **cdist.core.pytypes.<type-name>**
   attribute function call result, or **yield from ()** if type does not use other types
 * **type_gencode**: implementation should return a string consisting of lines

From 420aaaee164e8ed22ed03002db64a048a20615c5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 20 Sep 2019 20:16:55 +0200
Subject: [PATCH 009/366] Make python file type default, mv old to __file_old

---
 .../type/{__file_py => __file}/__init__.py    |  0
 cdist/conf/type/__file/explorer/stat          | 35 +-------
 cdist/conf/type/__file/parameter/optional     |  1 -
 .../{__file_py => __file_old}/explorer/cksum  |  0
 cdist/conf/type/__file_old/explorer/stat      | 88 +++++++++++++++++++
 .../{__file_py => __file_old}/explorer/type   |  0
 .../type/{__file => __file_old}/gencode-local |  0
 .../{__file => __file_old}/gencode-remote     |  0
 .../conf/type/{__file => __file_old}/man.rst  |  0
 .../parameter/default/state                   |  0
 .../parameter/optional                        |  1 +
 cdist/conf/type/__file_py/explorer/stat       | 56 ------------
 12 files changed, 90 insertions(+), 91 deletions(-)
 rename cdist/conf/type/{__file_py => __file}/__init__.py (100%)
 rename cdist/conf/type/{__file_py => __file_old}/explorer/cksum (100%)
 create mode 100755 cdist/conf/type/__file_old/explorer/stat
 rename cdist/conf/type/{__file_py => __file_old}/explorer/type (100%)
 rename cdist/conf/type/{__file => __file_old}/gencode-local (100%)
 rename cdist/conf/type/{__file => __file_old}/gencode-remote (100%)
 rename cdist/conf/type/{__file => __file_old}/man.rst (100%)
 rename cdist/conf/type/{__file_py => __file_old}/parameter/default/state (100%)
 rename cdist/conf/type/{__file_py => __file_old}/parameter/optional (76%)
 delete mode 100755 cdist/conf/type/__file_py/explorer/stat

diff --git a/cdist/conf/type/__file_py/__init__.py b/cdist/conf/type/__file/__init__.py
similarity index 100%
rename from cdist/conf/type/__file_py/__init__.py
rename to cdist/conf/type/__file/__init__.py
diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat
index 91c8cc84..c772abcf 100755
--- a/cdist/conf/type/__file/explorer/stat
+++ b/cdist/conf/type/__file/explorer/stat
@@ -52,7 +52,6 @@ fallback() {
 # nothing to work with, nothing we could do
 [ -e "$destination" ] || exit 0
 
-
 if ! command -v stat >/dev/null
 then
    fallback
@@ -69,39 +68,7 @@ group: %Dg %Sg
 mode: %Lp %Sp
 size: %Dz
 links: %Dl
-" "$destination" | awk '/^type/ { print tolower($0); next } { print }'
-      ;;
-    solaris)
-        ls1="$( ls -ld "$destination" )"
-        ls2="$( ls -ldn "$destination" )"
-
-        if [ -f "$__object/parameter/mode" ]
-        then mode_should="$( cat "$__object/parameter/mode" )"
-        fi
-
-        # yes, it is ugly hack, but if you know better way...
-        if [ -z "$( find "$destination" -perm "$mode_should" )" ]
-        then octets=888
-        else octets="$( echo "$mode_should" | sed 's/^0//' )"
-        fi
-
-        case "$( echo "$ls1" | cut -c1-1 )" in
-            -) echo 'type: regular file' ;;
-            d) echo 'type: directory' ;;
-        esac
-
-        echo "owner: $( echo "$ls2" \
-            | awk '{print $3}' ) $( echo "$ls1" \
-                | awk '{print $3}' )"
-
-        echo "group: $( echo "$ls2" \
-            | awk '{print $4}' ) $( echo "$ls1" \
-                | awk '{print $4}' )"
-
-        echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )"
-        echo "size: $( echo "$ls1" | awk '{print $5}' )"
-        echo "links: $( echo "$ls1" | awk '{print $2}' )"
-    ;;
+   ;;
    *)
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
diff --git a/cdist/conf/type/__file/parameter/optional b/cdist/conf/type/__file/parameter/optional
index 9b98352c..c696d592 100644
--- a/cdist/conf/type/__file/parameter/optional
+++ b/cdist/conf/type/__file/parameter/optional
@@ -3,4 +3,3 @@ group
 mode
 owner
 source
-onchange
diff --git a/cdist/conf/type/__file_py/explorer/cksum b/cdist/conf/type/__file_old/explorer/cksum
similarity index 100%
rename from cdist/conf/type/__file_py/explorer/cksum
rename to cdist/conf/type/__file_old/explorer/cksum
diff --git a/cdist/conf/type/__file_old/explorer/stat b/cdist/conf/type/__file_old/explorer/stat
new file mode 100755
index 00000000..13c1c208
--- /dev/null
+++ b/cdist/conf/type/__file_old/explorer/stat
@@ -0,0 +1,88 @@
+#!/bin/sh
+#
+# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+# 2019 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+destination="/$__object_id"
+
+# nothing to work with, nothing we could do
+[ -e "$destination" ] || exit 0
+
+os=$("$__explorer/os")
+case "$os" in
+   "freebsd"|"netbsd"|"openbsd"|"macosx")
+      stat -f "type: %HT
+owner: %Du %Su
+group: %Dg %Sg
+mode: %Lp %Sp
+size: %Dz
+links: %Dl
+" "$destination" | awk '/^type/ { print tolower($0); next; } { print; }'
+      ;;
+   alpine)
+       # busybox stat
+      stat -c "type: %F
+owner: %u %U
+group: %g %G
+mode: %a %A
+size: %s
+links: %h
+" "$destination"
+      ;;
+    solaris)
+        ls1="$( ls -ld "$destination" )"
+        ls2="$( ls -ldn "$destination" )"
+
+        if [ -f "$__object/parameter/mode" ]
+        then mode_should="$( cat "$__object/parameter/mode" )"
+        fi
+
+        # yes, it is ugly hack, but if you know better way...
+        if [ -z "$( find "$destination" -perm "$mode_should" )" ]
+        then octets=888
+        else octets="$( echo "$mode_should" | sed 's/^0//' )"
+        fi
+
+        case "$( echo "$ls1" | cut -c1-1 )" in
+            -) echo 'type: regular file' ;;
+            d) echo 'type: directory' ;;
+        esac
+
+        echo "owner: $( echo "$ls2" \
+            | awk '{print $3}' ) $( echo "$ls1" \
+                | awk '{print $3}' )"
+
+        echo "group: $( echo "$ls2" \
+            | awk '{print $4}' ) $( echo "$ls1" \
+                | awk '{print $4}' )"
+
+        echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )"
+        echo "size: $( echo "$ls1" | awk '{print $5}' )"
+        echo "links: $( echo "$ls1" | awk '{print $2}' )"
+    ;;
+   *)
+      stat --printf="type: %F
+owner: %u %U
+group: %g %G
+mode: %a %A
+size: %s
+links: %h
+" "$destination"
+      ;;
+esac
diff --git a/cdist/conf/type/__file_py/explorer/type b/cdist/conf/type/__file_old/explorer/type
similarity index 100%
rename from cdist/conf/type/__file_py/explorer/type
rename to cdist/conf/type/__file_old/explorer/type
diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file_old/gencode-local
similarity index 100%
rename from cdist/conf/type/__file/gencode-local
rename to cdist/conf/type/__file_old/gencode-local
diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file_old/gencode-remote
similarity index 100%
rename from cdist/conf/type/__file/gencode-remote
rename to cdist/conf/type/__file_old/gencode-remote
diff --git a/cdist/conf/type/__file/man.rst b/cdist/conf/type/__file_old/man.rst
similarity index 100%
rename from cdist/conf/type/__file/man.rst
rename to cdist/conf/type/__file_old/man.rst
diff --git a/cdist/conf/type/__file_py/parameter/default/state b/cdist/conf/type/__file_old/parameter/default/state
similarity index 100%
rename from cdist/conf/type/__file_py/parameter/default/state
rename to cdist/conf/type/__file_old/parameter/default/state
diff --git a/cdist/conf/type/__file_py/parameter/optional b/cdist/conf/type/__file_old/parameter/optional
similarity index 76%
rename from cdist/conf/type/__file_py/parameter/optional
rename to cdist/conf/type/__file_old/parameter/optional
index c696d592..9b98352c 100644
--- a/cdist/conf/type/__file_py/parameter/optional
+++ b/cdist/conf/type/__file_old/parameter/optional
@@ -3,3 +3,4 @@ group
 mode
 owner
 source
+onchange
diff --git a/cdist/conf/type/__file_py/explorer/stat b/cdist/conf/type/__file_py/explorer/stat
deleted file mode 100755
index 8a917556..00000000
--- a/cdist/conf/type/__file_py/explorer/stat
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/sh
-#
-# 2013 Steven Armstrong (steven-cdist armstrong.cc)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-destination="/$__object_id"
-
-# nothing to work with, nothing we could do
-[ -e "$destination" ] || exit 0
-
-os=$("$__explorer/os")
-case "$os" in
-   "freebsd"|"netbsd"|"openbsd")
-      # FIXME: should be something like this based on man page, but can not test
-      stat -f "type: %ST
-owner: %Du %Su
-group: %Dg %Sg
-mode: %Op %Sp
-size: %Dz
-links: %Dl
-" "$destination"
-   ;;
-   "macosx")
-     stat -f "type: %HT
-owner: %Du %Su
-group: %Dg %Sg
-mode: %Lp %Sp
-size: %Dz
-links: %Dl
-" "$destination"
-   ;;
-   *)
-      stat --printf="type: %F
-owner: %u %U
-group: %g %G
-mode: %a %A
-size: %s
-links: %h
-" "$destination"
-   ;;
-esac

From 1588476a91fb620a0ac80c1254bf703623a7225c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 1 Oct 2019 08:14:24 +0200
Subject: [PATCH 010/366] Remove unnecessary files

---
 cdist/conf/type/__file/parameter/default/state | 1 -
 cdist/conf/type/__file/parameter/optional      | 5 -----
 2 files changed, 6 deletions(-)
 delete mode 100644 cdist/conf/type/__file/parameter/default/state
 delete mode 100644 cdist/conf/type/__file/parameter/optional

diff --git a/cdist/conf/type/__file/parameter/default/state b/cdist/conf/type/__file/parameter/default/state
deleted file mode 100644
index e7f6134f..00000000
--- a/cdist/conf/type/__file/parameter/default/state
+++ /dev/null
@@ -1 +0,0 @@
-present
diff --git a/cdist/conf/type/__file/parameter/optional b/cdist/conf/type/__file/parameter/optional
deleted file mode 100644
index c696d592..00000000
--- a/cdist/conf/type/__file/parameter/optional
+++ /dev/null
@@ -1,5 +0,0 @@
-state
-group
-mode
-owner
-source

From b7af305636aaba6f1b3872b36382c8fbb802a605 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 1 Oct 2019 08:15:01 +0200
Subject: [PATCH 011/366] Align with shell type: implement onchange

---
 cdist/conf/type/__file/__init__.py | 11 ++++++++++-
 cdist/conf/type/__file_py          |  1 +
 2 files changed, 11 insertions(+), 1 deletion(-)
 create mode 120000 cdist/conf/type/__file_py

diff --git a/cdist/conf/type/__file/__init__.py b/cdist/conf/type/__file/__init__.py
index 3397b417..bcb525fc 100644
--- a/cdist/conf/type/__file/__init__.py
+++ b/cdist/conf/type/__file/__init__.py
@@ -52,6 +52,7 @@ class FileType(PythonType):
         create_file = False
         upload_file = False
         set_attributes = False
+        fire_onchange = False
         code = []
         if state_should == 'present' or state_should == 'exists':
             if source is None:
@@ -72,6 +73,7 @@ class FileType(PythonType):
                     self.die('Source {} does not exist'.format(source))
             if create_file or upload_file:
                 set_attributes = True
+                fire_onchange = True
                 tempfile_template = '{}.cdist.XXXXXXXXXX'.format(destination)
                 destination_upload = self.run_remote(
                     ["mktemp", tempfile_template, ])
@@ -90,6 +92,7 @@ class FileType(PythonType):
                     value_is = self.get_attribute(stat_file, attribute,
                                                   value_should)
                     if set_attributes or value_should != value_is:
+                        fire_onchange = True
                         code.append(self.set_attribute(attribute,
                                                        value_should,
                                                        destination))
@@ -97,9 +100,15 @@ class FileType(PythonType):
             if typeis == 'file':
                 code.append('rm -f {}'.format(destination))
                 self.send_message('remove')
+                fire_onchange = True
         else:
             self.die('Unknown state {}'.format(state_should))
 
+        if fire_onchange:
+            onchange = self.get_parameter('onchange')
+            if onchange:
+                code.append(onchange)
+
         return "\n".join(code)
 
     def get_args_parser(self):
@@ -107,7 +116,7 @@ class FileType(PythonType):
                                          argument_default=argparse.SUPPRESS)
         parser.add_argument('--state', dest='state', action='store',
                             required=False, default='present')
-        for param in ('group', 'mode', 'owner', 'source'):
+        for param in ('group', 'mode', 'owner', 'source', 'onchange'):
             parser.add_argument('--' + param, dest=param, action='store',
                                 required=False, default=None)
 
diff --git a/cdist/conf/type/__file_py b/cdist/conf/type/__file_py
new file mode 120000
index 00000000..efa910bd
--- /dev/null
+++ b/cdist/conf/type/__file_py
@@ -0,0 +1 @@
+__file
\ No newline at end of file

From 595cfec57233728feeafbdb3131daeffdb95b778 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 30 Nov 2019 23:29:16 +0100
Subject: [PATCH 012/366] Adapt order dependency impl to python types impl

---
 cdist/__init__.py      | 3 +++
 cdist/core/manifest.py | 7 ++-----
 cdist/emulator.py      | 5 ++---
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index c673b3ba..279fd502 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -47,6 +47,9 @@ REMOTE_COPY = "scp -o User=root -q"
 REMOTE_EXEC = "ssh -o User=root"
 REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
 
+ORDER_DEP_STATE_NAME = 'order_dep_state'
+TYPEORDER_DEP_NAME = 'typeorder_dep'
+
 
 class Error(Exception):
     """Base exception class for this project"""
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 16dec465..84bc813f 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -100,9 +100,6 @@ class Manifest(object):
 
     """
 
-    ORDER_DEP_STATE_NAME = 'order_dep_state'
-    TYPEORDER_DEP_NAME = 'typeorder_dep'
-
     def __init__(self, target_host, local, dry_run=False):
         self.target_host = target_host
         self.local = local
@@ -227,8 +224,8 @@ class Manifest(object):
                 os.remove(os.path.join(self.local.base_path, fname))
             except FileNotFoundError:
                 pass
-        _rm_file(Manifest.ORDER_DEP_STATE_NAME)
-        _rm_file(Manifest.TYPEORDER_DEP_NAME)
+        _rm_file(cdist.ORDER_DEP_STATE_NAME)
+        _rm_file(cdist.TYPEORDER_DEP_NAME)
 
     def env_py_type_manifest(self, cdist_object):
         env = os.environ.copy()
diff --git a/cdist/emulator.py b/cdist/emulator.py
index 7dc24a83..00060a10 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -29,7 +29,6 @@ import sys
 import cdist
 from cdist import core
 from cdist import flock
-from cdist.core.manifest import Manifest
 import cdist.util.python_type_util as pytype_util
 from cdist.core.pytypes import get_pytype_class
 import inspect
@@ -87,9 +86,9 @@ class Emulator(object):
         self.typeorder_path = os.path.join(self.global_path, "typeorder")
 
         self.typeorder_dep_path = os.path.join(self.global_path,
-                                               Manifest.TYPEORDER_DEP_NAME)
+                                               cdist.TYPEORDER_DEP_NAME)
         self.order_dep_state_path = os.path.join(self.global_path,
-                                                 Manifest.ORDER_DEP_STATE_NAME)
+                                                 cdist.ORDER_DEP_STATE_NAME)
 
         self.type_name = os.path.basename(argv[0])
         self.cdist_type = core.CdistType(self.type_base_path, self.type_name)

From 77d704113ce7c1065a77b9b01ab3f9954e8b80ca Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 15 Feb 2020 14:52:10 +0100
Subject: [PATCH 013/366] Update file stat from master

---
 cdist/conf/type/__file/explorer/stat     | 35 ++++++++++++-
 cdist/conf/type/__file_old/explorer/stat | 64 +++++++++++++++++-------
 2 files changed, 80 insertions(+), 19 deletions(-)

diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat
index c772abcf..91c8cc84 100755
--- a/cdist/conf/type/__file/explorer/stat
+++ b/cdist/conf/type/__file/explorer/stat
@@ -52,6 +52,7 @@ fallback() {
 # nothing to work with, nothing we could do
 [ -e "$destination" ] || exit 0
 
+
 if ! command -v stat >/dev/null
 then
    fallback
@@ -68,7 +69,39 @@ group: %Dg %Sg
 mode: %Lp %Sp
 size: %Dz
 links: %Dl
-   ;;
+" "$destination" | awk '/^type/ { print tolower($0); next } { print }'
+      ;;
+    solaris)
+        ls1="$( ls -ld "$destination" )"
+        ls2="$( ls -ldn "$destination" )"
+
+        if [ -f "$__object/parameter/mode" ]
+        then mode_should="$( cat "$__object/parameter/mode" )"
+        fi
+
+        # yes, it is ugly hack, but if you know better way...
+        if [ -z "$( find "$destination" -perm "$mode_should" )" ]
+        then octets=888
+        else octets="$( echo "$mode_should" | sed 's/^0//' )"
+        fi
+
+        case "$( echo "$ls1" | cut -c1-1 )" in
+            -) echo 'type: regular file' ;;
+            d) echo 'type: directory' ;;
+        esac
+
+        echo "owner: $( echo "$ls2" \
+            | awk '{print $3}' ) $( echo "$ls1" \
+                | awk '{print $3}' )"
+
+        echo "group: $( echo "$ls2" \
+            | awk '{print $4}' ) $( echo "$ls1" \
+                | awk '{print $4}' )"
+
+        echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )"
+        echo "size: $( echo "$ls1" | awk '{print $5}' )"
+        echo "links: $( echo "$ls1" | awk '{print $2}' )"
+    ;;
    *)
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
diff --git a/cdist/conf/type/__file_old/explorer/stat b/cdist/conf/type/__file_old/explorer/stat
index 13c1c208..91c8cc84 100755
--- a/cdist/conf/type/__file_old/explorer/stat
+++ b/cdist/conf/type/__file_old/explorer/stat
@@ -2,6 +2,7 @@
 #
 # 2013 Steven Armstrong (steven-cdist armstrong.cc)
 # 2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -21,29 +22,54 @@
 
 destination="/$__object_id"
 
+fallback() {
+   # Fallback: Patch the output together, manually.
+
+   ls_line=$(ls -ldn "$destination")
+
+   uid=$(echo "$ls_line" | awk '{ print $3 }')
+   gid=$(echo "$ls_line" | awk '{ print $4 }')
+
+   owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd)
+   group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
+
+   mode_text=$(echo "$ls_line" | awk '{ print $1 }')
+   mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }')
+
+   size=$(echo "$ls_line" | awk '{ print $5 }')
+   links=$(echo "$ls_line" | awk '{ print $2 }')
+
+   printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\nsize: %d\nlinks: %d\n' \
+      "$("$__type_explorer/type")" \
+      "$uid" "$owner" \
+      "$gid" "$group" \
+      "$mode" "$mode_text" \
+      "$size" \
+      "$links"
+}
+
+
 # nothing to work with, nothing we could do
 [ -e "$destination" ] || exit 0
 
-os=$("$__explorer/os")
-case "$os" in
-   "freebsd"|"netbsd"|"openbsd"|"macosx")
+
+if ! command -v stat >/dev/null
+then
+   fallback
+   exit
+fi
+
+
+case $("$__explorer/os")
+in
+   freebsd|netbsd|openbsd|macosx)
       stat -f "type: %HT
 owner: %Du %Su
 group: %Dg %Sg
 mode: %Lp %Sp
 size: %Dz
 links: %Dl
-" "$destination" | awk '/^type/ { print tolower($0); next; } { print; }'
-      ;;
-   alpine)
-       # busybox stat
-      stat -c "type: %F
-owner: %u %U
-group: %g %G
-mode: %a %A
-size: %s
-links: %h
-" "$destination"
+" "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
     solaris)
         ls1="$( ls -ld "$destination" )"
@@ -77,12 +103,14 @@ links: %h
         echo "links: $( echo "$ls1" | awk '{print $2}' )"
     ;;
    *)
-      stat --printf="type: %F
+      # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
+      # NOTE: BusyBox's stat might not support the "-c" option, in which case
+      #       we fall through to the shell fallback.
+      stat -c "type: %F
 owner: %u %U
 group: %g %G
 mode: %a %A
 size: %s
-links: %h
-" "$destination"
-      ;;
+links: %h" "$destination" 2>/dev/null || fallback
+         ;;
 esac

From d26c36914a4fa9889af57f549cb2854968663a0b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 27 Jul 2020 10:55:28 +0200
Subject: [PATCH 014/366] [__timezone] Make type singleton

---
 cdist/conf/type/__timezone/gencode-remote     |  2 +-
 cdist/conf/type/__timezone/man.rst            | 22 ++++++++++++-------
 cdist/conf/type/__timezone/manifest           |  2 +-
 cdist/conf/type/__timezone/parameter/required |  1 +
 cdist/conf/type/__timezone/singleton          |  0
 5 files changed, 17 insertions(+), 10 deletions(-)
 create mode 100644 cdist/conf/type/__timezone/parameter/required
 create mode 100644 cdist/conf/type/__timezone/singleton

diff --git a/cdist/conf/type/__timezone/gencode-remote b/cdist/conf/type/__timezone/gencode-remote
index 5299f548..b685c990 100755
--- a/cdist/conf/type/__timezone/gencode-remote
+++ b/cdist/conf/type/__timezone/gencode-remote
@@ -22,7 +22,7 @@
 # This type allows to configure the desired localtime timezone.
 
 timezone_is=$(cat "$__object/explorer/timezone_is")
-timezone_should="$__object_id"
+timezone_should=$(cat "$__object/parameter/tz")
 os=$(cat "$__global/explorer/os")
 
 if [ "$timezone_is" = "$timezone_should" ]; then
diff --git a/cdist/conf/type/__timezone/man.rst b/cdist/conf/type/__timezone/man.rst
index 8a945c16..6012c552 100644
--- a/cdist/conf/type/__timezone/man.rst
+++ b/cdist/conf/type/__timezone/man.rst
@@ -14,7 +14,8 @@ This type creates a symlink (/etc/localtime) to the selected timezone
 
 REQUIRED PARAMETERS
 -------------------
-None.
+tz
+    The name of timezone to set.
 
 
 OPTIONAL PARAMETERS
@@ -27,19 +28,24 @@ EXAMPLES
 
 .. code-block:: sh
 
-    #Set up Europe/Andorra as our timezone.
-    __timezone Europe/Andorra
+    # Set up Europe/Andorra as our timezone.
+    __timezone --tz Europe/Andorra
 
-    #Set up US/Central as our timezone.
-    __timezone US/Central
+    # Set up US/Central as our timezone.
+    __timezone --tz US/Central
 
 
 AUTHORS
 -------
-Ramon Salvadó <rsalvado--@--gnuine--dot--com>
+| Steven Armstrong <steven-cdist--@--armstrong.cc>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Ramon Salvadó <rsalvado--@--gnuine--dot--com>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Free use of this software is
-granted under the terms of the GNU General Public License version 3 (GPLv3).
+Copyright \(C) 2012-2020 the `AUTHORS`_. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__timezone/manifest b/cdist/conf/type/__timezone/manifest
index 3d28ccba..0eb7fb9c 100755
--- a/cdist/conf/type/__timezone/manifest
+++ b/cdist/conf/type/__timezone/manifest
@@ -22,7 +22,7 @@
 #
 # This type allows to configure the desired localtime timezone.
 
-timezone="$__object_id"
+timezone=$(cat "$__object/parameter/tz")
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
diff --git a/cdist/conf/type/__timezone/parameter/required b/cdist/conf/type/__timezone/parameter/required
new file mode 100644
index 00000000..975445e4
--- /dev/null
+++ b/cdist/conf/type/__timezone/parameter/required
@@ -0,0 +1 @@
+tz
diff --git a/cdist/conf/type/__timezone/singleton b/cdist/conf/type/__timezone/singleton
new file mode 100644
index 00000000..e69de29b

From c053a2c4a000b0c15df186ef2e067ea68ec9499c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Wed, 29 Jul 2020 11:31:12 +0200
Subject: [PATCH 015/366] Fix building man pages

Resolves #830.
---
 Makefile                                  | 4 ++--
 cdist/conf/type/__openldap_server/man.rst | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index f89ac1e7..3712511c 100644
--- a/Makefile
+++ b/Makefile
@@ -81,7 +81,7 @@ version:
 	}
 
 # Manpages #3: generic part
-man: version $(MANTYPES) $(DOCSREF)
+man: version configskel $(MANTYPES) $(DOCSREF) $(DOCSTYPESREF)
 	$(SPHINXM)
 
 html: version configskel $(MANTYPES) $(DOCSREF) $(DOCSTYPESREF)
@@ -104,7 +104,7 @@ DOTMANTYPES=$(subst /man.rst,.rst,$(DOTMANTYPEPREFIX))
 $(DOTMAN7DSTDIR)/cdist-type%.rst: $(DOTTYPEDIR)/%/man.rst
 	ln -sf "$^" $@
 
-dotman: version $(DOTMANTYPES)
+dotman: version configskel $(DOTMANTYPES) $(DOCSREF) $(DOCSTYPESREF)
 	$(SPHINXM)
 
 ################################################################################
diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst
index a96c7dad..fa714ec0 100644
--- a/cdist/conf/type/__openldap_server/man.rst
+++ b/cdist/conf/type/__openldap_server/man.rst
@@ -31,8 +31,8 @@ manager-password-hash
     Generate e.g. with: `slappasswd -s weneedgoodsecurity`.
     See `slappasswd(8C)`, `slapd.conf(5)`.
     TODO: implement this: http://blog.adamsbros.org/2015/06/09/openldap-ssha-salted-hashes-by-hand/
-      to derive from the manager-password parameter and ensure idempotency (care with salts).
-      At that point, manager-password-hash should be deprecated and ignored.
+    to derive from the manager-password parameter and ensure idempotency (care with salts).
+    At that point, manager-password-hash should be deprecated and ignored.
 
 serverid
     The server for the directory.

From d37d2dc307d066f4d5f1bbbb93e0480965dfe3b8 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 2 Aug 2020 13:53:38 +0300
Subject: [PATCH 016/366] [__unpack] add parameter --tar-extra-args

---
 cdist/conf/type/__unpack/gencode-remote     |  7 +++++++
 cdist/conf/type/__unpack/man.rst            | 11 +++++++++++
 cdist/conf/type/__unpack/parameter/optional |  1 +
 3 files changed, 19 insertions(+)

diff --git a/cdist/conf/type/__unpack/gencode-remote b/cdist/conf/type/__unpack/gencode-remote
index 45c7173a..3b7a19a7 100755
--- a/cdist/conf/type/__unpack/gencode-remote
+++ b/cdist/conf/type/__unpack/gencode-remote
@@ -23,6 +23,13 @@ case "$src" in
 
             cmd="$cmd --strip-components=$tar_strip"
         fi
+
+        if [ -f "$__object/parameter/tar-extra-args" ]
+        then
+            tar_extra_args="$( cat "$__object/parameter/tar-extra-args" )"
+
+            cmd="$cmd $tar_extra_args"
+        fi
     ;;
     *.7z)
         case "$os" in
diff --git a/cdist/conf/type/__unpack/man.rst b/cdist/conf/type/__unpack/man.rst
index 8fe96e43..bd0603cf 100644
--- a/cdist/conf/type/__unpack/man.rst
+++ b/cdist/conf/type/__unpack/man.rst
@@ -33,6 +33,10 @@ sum-file
 tar-strip
     Tarball specific. See ``man tar`` for ``--strip-components``.
 
+tar-extra-args
+    Tarball sepcific. Append additional arguments to ``tar`` command.
+    See ``man tar`` for possible arguments.
+
 
 OPTIONAL BOOLEAN PARAMETERS
 ---------------------------
@@ -65,6 +69,13 @@ EXAMPLES
             --preserve-archive \
             --destination /opt/cpma/server
 
+    # example usecase for --tar-* args
+    __unpack /root/strelaysrv.tar.gz \
+        --preserve-archive \
+        --destination /usr/local/bin \
+        --tar-strip 1 \
+        --tar-extra-args '--wildcards "*/strelaysrv"'
+
 
 AUTHORS
 -------
diff --git a/cdist/conf/type/__unpack/parameter/optional b/cdist/conf/type/__unpack/parameter/optional
index d136dd0c..da26ca63 100644
--- a/cdist/conf/type/__unpack/parameter/optional
+++ b/cdist/conf/type/__unpack/parameter/optional
@@ -1,2 +1,3 @@
 sum-file
 tar-strip
+tar-extra-args

From 935f2395bc8f02d373f06d1e22acad813403ead2 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 2 Aug 2020 13:54:30 +0300
Subject: [PATCH 017/366] [__locale_system] fix for debian and ubuntu

ubuntu 6.10 and debian etch are 10+ years old and EOL. rather than
preserving compatibility I'll just remove it. while /etc/environment
works too, correct place is /etc/default/locale (as it was before
breaking change). also /etc/debian_version (os_version explorer) may
contain minor version with dot (10.5) or string (bullseye/sid).
---
 cdist/conf/type/__locale_system/manifest | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index e4286ef6..f7d75387 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -40,27 +40,13 @@ os=$(cat "$__global/explorer/os")
 case $os
 in
     debian)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 4 >/dev/null
-        then
-            # Debian 4 (etch) and later
-            locale_conf="/etc/default/locale"
-        else
-            locale_conf="/etc/environment"
-        fi
+        locale_conf="/etc/default/locale"
         ;;
     devuan)
         locale_conf="/etc/default/locale"
         ;;
     ubuntu)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 6.10 >/dev/null
-        then
-            # Ubuntu 6.10 (edgy) and later
-            locale_conf="/etc/default/locale"
-        else
-            locale_conf="/etc/environment"
-        fi
+        locale_conf="/etc/default/locale"
         ;;
     archlinux)
         locale_conf="/etc/locale.conf"

From 885d5a58f40e5adfd6aa616712e4c7246eca7880 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 16:49:01 +0200
Subject: [PATCH 018/366] [type/__locale_system] Fix floating point version
 comparison

---
 cdist/conf/type/__locale_system/manifest | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index e4286ef6..521f1007 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -24,6 +24,9 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
+bccmp() { test "$(bc)" -gt 0; }
+
+
 key=$__object_id
 onchange_cmd=  # none, by default
 quote_value=false
@@ -41,7 +44,7 @@ case $os
 in
     debian)
         os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 4 >/dev/null
+        if printf '%f >= 4\n' "${os_version}" | bccmp
         then
             # Debian 4 (etch) and later
             locale_conf="/etc/default/locale"
@@ -54,7 +57,7 @@ in
         ;;
     ubuntu)
         os_version=$(cat "${__global}/explorer/os_version")
-        if expr "${os_version}" '>=' 6.10 >/dev/null
+        if printf '%f >= 6.10\n' "${os_version}" | bccmp
         then
             # Ubuntu 6.10 (edgy) and later
             locale_conf="/etc/default/locale"
@@ -68,7 +71,7 @@ in
     centos|redhat|scientific)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if expr "${version_id}" '>=' 7 >/dev/null
+        if printf '%f >= 7\n' "${version_id}" | bccmp
         then
             locale_conf="/etc/locale.conf"
         else
@@ -78,7 +81,7 @@ in
     fedora)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if expr "${version_id}" '>=' 18 >/dev/null
+        if printf '%f >= 18\n' "${version_id}" | bccmp
         then
             locale_conf="/etc/locale.conf"
             quote_value=false
@@ -113,7 +116,8 @@ in
         locale_conf="/etc/default/init"
         locale_conf_group="sys"
 
-        if expr "$(cat "${__global}/explorer/os_version")" '>=' 5.11 >/dev/null
+        os_version=$(cat "${__global}/explorer/os_version")
+        if printf '%f >= 5.11\n' "${os_version}" | bccmp
         then
             # mode on Oracle Solaris 11 is actually 0444,
             # but the write bit makes sense, IMO

From 71710fa00a86e3dcda6be82b959956cb88ba9902 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 20:17:00 +0200
Subject: [PATCH 019/366] [type/__locale_system] Implement "proper" version
 comparison

Proper in the sense that it can handle all numeric version numbers even if they
are not floating point (e.g. 16.04.6).
---
 cdist/conf/type/__locale_system/manifest | 28 ++++++++++++++++--------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 521f1007..88eadb31 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -24,7 +24,20 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
-bccmp() { test "$(bc)" -gt 0; }
+version_ge() {
+    awk -F '[^0-9.]' -v target="${1:?}" '
+    function max(x, y) { return x > y ? x : y }
+    BEGIN {
+        getline
+        nx = split($1, x, ".")
+        ny = split(target, y, ".")
+        for (i = 1; i <= max(nx, ny); ++i) {
+            diff = int(x[i]) - int(y[i])
+            if (diff == 0) continue
+            exit (diff < 0)
+        }
+    }'
+}
 
 
 key=$__object_id
@@ -43,8 +56,7 @@ os=$(cat "$__global/explorer/os")
 case $os
 in
     debian)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if printf '%f >= 4\n' "${os_version}" | bccmp
+        if version_ge 4 <"${__global}/explorer/os_version"
         then
             # Debian 4 (etch) and later
             locale_conf="/etc/default/locale"
@@ -56,8 +68,7 @@ in
         locale_conf="/etc/default/locale"
         ;;
     ubuntu)
-        os_version=$(cat "${__global}/explorer/os_version")
-        if printf '%f >= 6.10\n' "${os_version}" | bccmp
+        if version_ge 6.10 <"${__global}/explorer/os_version"
         then
             # Ubuntu 6.10 (edgy) and later
             locale_conf="/etc/default/locale"
@@ -71,7 +82,7 @@ in
     centos|redhat|scientific)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if printf '%f >= 7\n' "${version_id}" | bccmp
+        if echo "${version_id}" | version_ge 7
         then
             locale_conf="/etc/locale.conf"
         else
@@ -81,7 +92,7 @@ in
     fedora)
         # shellcheck source=/dev/null
         version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
-        if printf '%f >= 18\n' "${version_id}" | bccmp
+        if echo "${version_id}" | version_ge 18
         then
             locale_conf="/etc/locale.conf"
             quote_value=false
@@ -116,8 +127,7 @@ in
         locale_conf="/etc/default/init"
         locale_conf_group="sys"
 
-        os_version=$(cat "${__global}/explorer/os_version")
-        if printf '%f >= 5.11\n' "${os_version}" | bccmp
+        if version_ge 5.11 <"${__global}/explorer/os_version"
         then
             # mode on Oracle Solaris 11 is actually 0444,
             # but the write bit makes sense, IMO

From 7b480f4293b84a97850430a9436e28c66ccf5736 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 22:08:14 +0200
Subject: [PATCH 020/366] [type/__locale_system] Fix version extraction for
 SuSE

---
 cdist/conf/type/__locale_system/manifest | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest
index 88eadb31..4b996ebc 100755
--- a/cdist/conf/type/__locale_system/manifest
+++ b/cdist/conf/type/__locale_system/manifest
@@ -163,7 +163,13 @@ in
         key="export ${__object_id}"
         ;;
     suse)
-        os_version=$(cat "${__global}/explorer/os_version")
+        if test -s "${__global}/explorer/os_release"
+        then
+            # shellcheck source=/dev/null
+            os_version=$(. "${__global}/explorer/os_release" && echo "${VERSION}")
+        else
+            os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global}/explorer/os_version")
+        fi
         os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
 
         # https://documentation.suse.com/sles/15-SP2/html/SLES-all/cha-suse.html#sec-suse-l10n

From b370b70ff4ccc1c6efca97f61f220a5ce0b191da Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 2 Aug 2020 22:13:41 +0200
Subject: [PATCH 021/366] [explorer/os] Fix OS detection for openSUSE

All distros with ID_LIKE suse should be treated as "suse".
My openSUSE Leap 15.1 installation has:
ID_LIKE="suse opensuse"

This patch doesn't require a strict "suse" value but only the word suse to be in
the list.
---
 cdist/conf/explorer/os | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os
index 2d2aede6..8f54d9c8 100755
--- a/cdist/conf/explorer/os
+++ b/cdist/conf/explorer/os
@@ -144,7 +144,8 @@ esac
 
 if [ -f /etc/os-release ]; then
    # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse
-   if grep -q ^ID_LIKE=\"suse\" /etc/os-release 2>/dev/null; then
+   if (. /etc/os-release && echo "${ID_LIKE}" | grep -q '\(^\|\ \)suse\($\|\ \)')
+   then
       echo suse
       exit 0
    fi

From 17ab4bd80c9c9dda2fa4f8ccde5ee454f7b5ba2d Mon Sep 17 00:00:00 2001
From: Joachim Desroches <joachim.desroches@epfl.ch>
Date: Thu, 6 Aug 2020 11:45:05 +0200
Subject: [PATCH 022/366] Add Alpine Linux as supported for __filesystem.

---
 cdist/conf/type/__filesystem/explorer/lsblk | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk
index 9ae544ac..9be3c575 100644
--- a/cdist/conf/type/__filesystem/explorer/lsblk
+++ b/cdist/conf/type/__filesystem/explorer/lsblk
@@ -18,16 +18,16 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$("$__explorer/os")
+os=$("${__explorer:?}/os")
 
-if [ -f "$__object/parameter/device" ]; then
+if [ -f "${__object:?}/parameter/device" ]; then
     blkdev="$(cat "$__object/parameter/device")"
 else
-    blkdev="$__object_id"
+    blkdev="${__object_id:?}"
 fi
 
 case "$os" in
-    centos|fedora|redhat|suse|gentoo)
+    alpine|centos|fedora|redhat|suse|gentoo)
         if [ ! -x "$(command -v lsblk)" ]; then
             echo "lsblk is required for __filesystem type" >&2
             exit 1

From 74dd47c8c352cc3de3b5684bbe90520f61d95f37 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 15 Aug 2020 21:11:43 +0200
Subject: [PATCH 023/366] ++changelog

---
 docs/changelog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 269a2049..2c09bfea 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,12 @@
 Changelog
 ---------
 
+next:
+	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)
+	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)
+	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)
+	* Type __filesystem: Support Alpine Linux (Joachim Desroches)
+
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)
 	* New type: __download (Ander Punnar)

From 8f94a226c7f0b875987e25e57682414204380fa2 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 15 Aug 2020 21:54:07 +0200
Subject: [PATCH 024/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 2c09bfea..9e44cc4d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)
 	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)
 	* Type __filesystem: Support Alpine Linux (Joachim Desroches)
+	* Type __locale_system: Fix version comparison (Dennis Camera)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From 6fed1785298eb506e7ea6cd177efe7d96658769f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 17 Aug 2020 09:27:48 +0200
Subject: [PATCH 025/366] [explorer/os_version] Convert Debian sid to version
 number.

Conversion of Debian sid to versions is done based on Debian codenames.
The version number is the version number of the final release - 0.01.

It is unknown if Debian < 4.0 has any sort of version information
available (apart from maybe checking base-files package version).
But I don't think any of these systems are still alive,
so I think going with 3.99 is fine for those.
---
 cdist/conf/explorer/os_version | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 1d54ea60..a7b1d3bc 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -31,7 +31,32 @@ case "$("$__explorer/os")" in
       cat /etc/arch-release
    ;;
    debian)
-      cat /etc/debian_version
+      debian_version=$(cat /etc/debian_version)
+      case $debian_version
+      in
+          testing/unstable)
+              # previous to Debian 4.0 testing/unstable was used
+              # cf. https://metadata.ftp-master.debian.org/changelogs/main/b/base-files/base-files_11_changelog
+              echo 3.99
+              ;;
+          */sid)
+              # sid versions don't have a number, so we decode by codename:
+              case $(expr "$debian_version" : '\([a-z]\{1,\}\)/')
+              in
+                  bullseye) echo 10.99 ;;
+                  buster) echo 9.99 ;;
+                  stretch) echo 8.99 ;;
+                  jessie) echo 7.99 ;;
+                  wheezy) echo 6.99 ;;
+                  squeeze) echo 5.99 ;;
+                  lenny) echo 4.99 ;;
+                  *) exit 1
+              esac
+              ;;
+          *)
+              echo "$debian_version"
+              ;;
+      esac
    ;;
    devuan)
       cat /etc/devuan_version
@@ -73,4 +98,4 @@ case "$("$__explorer/os")" in
    alpine)
        cat /etc/alpine-release
    ;;
-esac
\ No newline at end of file
+esac

From 502d75304707704d3ba73374701394b98d259506 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 18 Aug 2020 00:46:07 +0300
Subject: [PATCH 026/366] [__unpack] add --onchange

---
 cdist/conf/type/__unpack/gencode-remote     | 5 +++++
 cdist/conf/type/__unpack/man.rst            | 3 +++
 cdist/conf/type/__unpack/parameter/optional | 1 +
 3 files changed, 9 insertions(+)

diff --git a/cdist/conf/type/__unpack/gencode-remote b/cdist/conf/type/__unpack/gencode-remote
index 3b7a19a7..c4451f73 100755
--- a/cdist/conf/type/__unpack/gencode-remote
+++ b/cdist/conf/type/__unpack/gencode-remote
@@ -80,3 +80,8 @@ if [ ! -f "$__object/parameter/preserve-archive" ]
 then
     echo "rm -f '$src'"
 fi
+
+if [ -f "$__object/parameter/onchange" ]
+then
+    cat "$__object/parameter/onchange"
+fi
diff --git a/cdist/conf/type/__unpack/man.rst b/cdist/conf/type/__unpack/man.rst
index bd0603cf..daa03814 100644
--- a/cdist/conf/type/__unpack/man.rst
+++ b/cdist/conf/type/__unpack/man.rst
@@ -50,6 +50,9 @@ backup-destination
 preserve-archive
     Don't delete archive after unpacking.
 
+onchange
+    Execute this command after unpack.
+
 
 EXAMPLES
 --------
diff --git a/cdist/conf/type/__unpack/parameter/optional b/cdist/conf/type/__unpack/parameter/optional
index da26ca63..d846ac75 100644
--- a/cdist/conf/type/__unpack/parameter/optional
+++ b/cdist/conf/type/__unpack/parameter/optional
@@ -1,3 +1,4 @@
 sum-file
 tar-strip
 tar-extra-args
+onchange

From d239169c4f91a4ce33be170ed29b08299c51234f Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 18 Aug 2020 00:48:58 +0300
Subject: [PATCH 027/366] [__download] fix manual: onchange parameter in wrong
 section

---
 cdist/conf/type/__download/man.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index 6ec0b19a..eb3ac971 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -30,9 +30,6 @@ sum
    By default output of ``cksum`` without filename is expected.
    Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
 
-onchange
-   Execute this command after download.
-
 
 OPTIONAL PARAMETERS
 -------------------
@@ -54,6 +51,9 @@ cmd-sum
    format specification ``%s`` which will become destination.
    For example: ``md5sum '%s' | awk '{print $1}'``.
 
+onchange
+   Execute this command after download.
+
 
 EXAMPLES
 --------

From ba26a437be4d8034037e7a6eb42382aee01f829e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Tue, 18 Aug 2020 11:06:19 +0200
Subject: [PATCH 028/366] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 9e44cc4d..a3e4167a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,9 @@ next:
 	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)
 	* Type __filesystem: Support Alpine Linux (Joachim Desroches)
 	* Type __locale_system: Fix version comparison (Dennis Camera)
+	* Type __unpack: Add --onchange parameter (Ander Punnar)
+	* Type __download: Fix manual (Ander Punnar)
+	* Explorer os_version: Convert Debian sid to version number (Dennis Camera)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From c17541f24cf118b0989a5899cab3a439ed534a46 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Sat, 22 Aug 2020 19:53:42 +0200
Subject: [PATCH 029/366] Expand and split by consecutive require delimiters

Resolves #832.
---
 cdist/emulator.py               |  6 +++++-
 cdist/test/emulator/__init__.py | 10 ++++++++++
 docs/src/cdist-manifest.rst     |  3 ++-
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/cdist/emulator.py b/cdist/emulator.py
index 9fe84056..60b94880 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -25,6 +25,7 @@ import argparse
 import logging
 import os
 import sys
+import re
 
 import cdist
 from cdist import core
@@ -389,12 +390,15 @@ class Emulator:
         if "require" in self.env:
             requirements = self.env['require']
             self.log.debug("reqs = " + requirements)
-            for requirement in requirements.split(" "):
+            for requirement in self._parse_require(requirements):
                 # Ignore empty fields - probably the only field anyway
                 if len(requirement) == 0:
                     continue
                 self.record_requirement(requirement)
 
+    def _parse_require(self, require):
+        return re.split(r'[ \t\n]+', require)
+
     def record_auto_requirements(self):
         """An object shall automatically depend on all objects that it
            defined in it's type manifest.
diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py
index e375676c..befd7b57 100644
--- a/cdist/test/emulator/__init__.py
+++ b/cdist/test/emulator/__init__.py
@@ -685,6 +685,16 @@ class EmulatorAlreadyExistingRequirementsWarnTestCase(test.CdistTestCase):
         self.env['require'] = '__directory/spam'
         emu = emulator.Emulator(argv, env=self.env)
 
+    def test_parse_require(self):
+        require = " \t \n  \t\t\n\t\na\tb\nc d \te\t\nf\ng\t "
+        expected = ['', 'a', 'b', 'c', 'd', 'e', 'f', 'g', '', ]
+
+        argv = ['__directory', 'spam']
+        emu = emulator.Emulator(argv, env=self.env)
+        requirements = emu._parse_require(require)
+
+        self.assertEqual(expected, requirements)
+
 
 if __name__ == '__main__':
     import unittest
diff --git a/docs/src/cdist-manifest.rst b/docs/src/cdist-manifest.rst
index 5dbca479..2e49a721 100644
--- a/docs/src/cdist-manifest.rst
+++ b/docs/src/cdist-manifest.rst
@@ -95,7 +95,8 @@ Dependencies
 ------------
 If you want to describe that something requires something else, just
 setup the variable "require" to contain the requirements. Multiple
-requirements can be added white space separated.
+requirements can be added separated with (optionally consecutive)
+delimiters including space, tab and newline.
 
 ::
 

From b5a40eb0d1c8990f0feb8e918a95e54e4fd35d98 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 27 Aug 2020 12:25:11 +0200
Subject: [PATCH 030/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index a3e4167a..eda77a12 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -10,6 +10,7 @@ next:
 	* Type __unpack: Add --onchange parameter (Ander Punnar)
 	* Type __download: Fix manual (Ander Punnar)
 	* Explorer os_version: Convert Debian sid to version number (Dennis Camera)
+	* Core: Expand require delimiter characters, split by consecutive delimiters (Darko Poljak)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From b1375464cc7507b38dd02f2df52c75489837d129 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 6 Sep 2020 13:24:58 +0200
Subject: [PATCH 031/366] __systemd_service: fix manpage typos

---
 cdist/conf/type/__systemd_service/man.rst | 25 +++++++++++++++--------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__systemd_service/man.rst b/cdist/conf/type/__systemd_service/man.rst
index 7eca398b..cd14c985 100644
--- a/cdist/conf/type/__systemd_service/man.rst
+++ b/cdist/conf/type/__systemd_service/man.rst
@@ -1,9 +1,10 @@
-cdist-type__systemd-service(7)
+cdist-type__systemd_service(7)
 ==============================
 
 NAME
 ----
-cdist-type__systemd-service - Controls a systemd service state
+cdist-type__systemd_service - Controls a systemd service state
+
 
 DESCRIPTION
 -----------
@@ -14,11 +15,12 @@ service after configuration applied or shutdown one service.
 The activation or deactivation is out of scope. Look for the
 :strong:`cdist-type__systemd_util`\ (7) type instead.
 
+
 REQUIRED PARAMETERS
 -------------------
-
 None.
 
+
 OPTIONAL PARAMETERS
 -------------------
 
@@ -31,12 +33,12 @@ state
     running
         Service should run (default)
 
-    stoppend
-        Service should stopped
+    stopped
+        Service should be stopped
 
 action
     Executes an action on on the service. It will only execute it if the
-    service keeps the state **running**. There are following actions, where:
+    service keeps the state ``running``. There are following actions, where:
 
     reload
         Reloads the service
@@ -48,11 +50,12 @@ BOOLEAN PARAMETERS
 ------------------
 
 if-required
-    Only execute the action if minimum one required type outputs a message to
-    **$__messages_out**. Through this, the action should only executed if a
+    Only execute the action if at minimum one required type outputs a message
+    to ``$__messages_out``. Through this, the action should only executed if a
     dependency did something. The action will not executed if no dependencies
     given.
 
+
 MESSAGES
 --------
 
@@ -68,12 +71,14 @@ restart
 reload
     Reloaded the service
 
+
 ABORTS
 ------
 Aborts in following cases:
 
 systemd or the service does not exist
 
+
 EXAMPLES
 --------
 .. code-block:: sh
@@ -95,13 +100,15 @@ EXAMPLES
 
     # reload the service for a modified configuration file
     # only reloads the service if the file really changed
-    require="__config_file/etc/foo.conf" __systemd_service foo \
+    require="__file/etc/foo.conf" __systemd_service foo \
         --action reload --if-required
 
+
 AUTHORS
 -------
 Matthias Stecher <matthiasstecher at gmx.de>
 
+
 COPYRIGHT
 ---------
 Copyright \(C) 2020 Matthias Stecher. You can redistribute it

From 6b262a61c19234f6450cef11f715020dcd5f40d5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 10 Sep 2020 13:24:58 +0200
Subject: [PATCH 032/366] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index eda77a12..e4ec3fdd 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,8 @@ next:
 	* Type __download: Fix manual (Ander Punnar)
 	* Explorer os_version: Convert Debian sid to version number (Dennis Camera)
 	* Core: Expand require delimiter characters, split by consecutive delimiters (Darko Poljak)
+	* Type __timezone: Make singleton (Dennis Camera)
+	* Type __systemd_service: Fix manpage typos (Matthias Stecher)
 
 6.7.0: 2020-07-28
 	* Delete deprecated type: __pf_apply (Darko Poljak)

From 53b91adbd8be4d858f1f4da3389b63babc85db0c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 11 Sep 2020 14:19:45 +0200
Subject: [PATCH 033/366] Fix shellcheck

---
 cdist/conf/explorer/os | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os
index 8f54d9c8..46d87f3e 100755
--- a/cdist/conf/explorer/os
+++ b/cdist/conf/explorer/os
@@ -144,6 +144,7 @@ esac
 
 if [ -f /etc/os-release ]; then
    # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse
+   # shellcheck disable=SC1091
    if (. /etc/os-release && echo "${ID_LIKE}" | grep -q '\(^\|\ \)suse\($\|\ \)')
    then
       echo suse

From 2885c6a24838956167ae6c33ebf87067e8790083 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Fri, 11 Sep 2020 14:15:26 +0200
Subject: [PATCH 034/366] Release 6.8.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index e4ec3fdd..7a0d050d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)
 	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)
 	* Explorer os: Fix OS detection for openSUSE (Dennis Camera)

From decc0ad54dffbafcd7d3644c28ae4d149926e612 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 18 Sep 2020 19:33:36 +0300
Subject: [PATCH 035/366] [__package_pip] detect pip binary

---
 cdist/conf/type/__package_pip/explorer/pip   | 10 ++++++++++
 cdist/conf/type/__package_pip/explorer/state |  2 +-
 cdist/conf/type/__package_pip/gencode-remote |  7 ++++++-
 3 files changed, 17 insertions(+), 2 deletions(-)
 create mode 100755 cdist/conf/type/__package_pip/explorer/pip

diff --git a/cdist/conf/type/__package_pip/explorer/pip b/cdist/conf/type/__package_pip/explorer/pip
new file mode 100755
index 00000000..cf9fae89
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/pip
@@ -0,0 +1,10 @@
+#!/bin/sh -e
+
+for bin in pip3 pip
+do
+    if check="$( command -v "$bin" )"
+    then
+        echo "$check"
+        break
+    fi
+done
diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state
index 5be07280..3cc98ab9 100644
--- a/cdist/conf/type/__package_pip/explorer/state
+++ b/cdist/conf/type/__package_pip/explorer/state
@@ -32,7 +32,7 @@ pipparam="$__object/parameter/pip"
 if [ -f "$pipparam" ]; then
     pip=$(cat "$pipparam")
 else
-    pip="pip"
+    pip="$( "$__type_explorer/pip" )"
 fi
 
 # If there is no pip, it may get created from somebody else.
diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index dcc4fdf9..9c5eb0d1 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -38,7 +38,12 @@ pipparam="$__object/parameter/pip"
 if [ -f "$pipparam" ]; then
     pip=$(cat "$pipparam")
 else
-    pip="pip"
+    pip="$( cat "$__object/explorer/pip" )"
+    if [ -z "$pip" ]
+    then
+        echo 'pip not found in path' >&2
+        exit 1
+    fi
 fi
 
 runasparam="$__object/parameter/runas"

From 89b621511561105cfcfcf12bbf820b74ab03396d Mon Sep 17 00:00:00 2001
From: Darko Poljak <darko.poljak@ungleich.ch>
Date: Mon, 21 Sep 2020 09:04:05 +0200
Subject: [PATCH 036/366] Clarify stdin input

Resolve #836.
---
 cdist/argparse.py                | 36 ++++++--------------------
 cdist/config.py                  |  6 +++--
 cdist/inventory.py               |  4 +--
 cdist/test/inventory/__init__.py |  3 +--
 docs/src/man1/cdist.rst          | 44 ++++++++++----------------------
 5 files changed, 29 insertions(+), 64 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 77303591..1d16bb25 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -273,8 +273,7 @@ def get_parsers():
             '-f', '--file',
             help=('Read specified file for a list of additional hosts to '
                   'operate on or if \'-\' is given, read stdin (one host per '
-                  'line). If no host or host file is specified then, by '
-                  'default, read hosts from stdin.'),
+                  'line).'),
             dest='hostfile', required=False)
     parser['config_args'].add_argument(
            '-p', '--parallel', nargs='?', metavar='HOST_MAX',
@@ -326,9 +325,7 @@ def get_parsers():
     parser['add-host'].add_argument(
            '-f', '--file',
            help=('Read additional hosts to add from specified file '
-                 'or from stdin if \'-\' (each host on separate line). '
-                 'If no host or host file is specified then, by default, '
-                 'read from stdin.'),
+                 'or from stdin if \'-\' (each host on separate line). '),
            dest='hostfile', required=False)
 
     parser['add-tag'] = parser['invsub'].add_parser(
@@ -342,20 +339,12 @@ def get_parsers():
     parser['add-tag'].add_argument(
            '-f', '--file',
            help=('Read additional hosts to add tags from specified file '
-                 'or from stdin if \'-\' (each host on separate line). '
-                 'If no host or host file is specified then, by default, '
-                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                 ' are specified then tags are read from stdin and are'
-                 ' added to all hosts.'),
+                 'or from stdin if \'-\' (each host on separate line). '),
            dest='hostfile', required=False)
     parser['add-tag'].add_argument(
            '-T', '--tag-file',
            help=('Read additional tags to add from specified file '
-                 'or from stdin if \'-\' (each tag on separate line). '
-                 'If no tag or tag file is specified then, by default, '
-                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                 ' are specified then tags are read from stdin and are'
-                 ' added to all hosts.'),
+                 'or from stdin if \'-\' (each tag on separate line). '),
            dest='tagfile', required=False)
     parser['add-tag'].add_argument(
            '-t', '--taglist',
@@ -376,9 +365,7 @@ def get_parsers():
     parser['del-host'].add_argument(
             '-f', '--file',
             help=('Read additional hosts to delete from specified file '
-                  'or from stdin if \'-\' (each host on separate line). '
-                  'If no host or host file is specified then, by default, '
-                  'read from stdin.'),
+                  'or from stdin if \'-\' (each host on separate line). '),
             dest='hostfile', required=False)
 
     parser['del-tag'] = parser['invsub'].add_parser(
@@ -396,20 +383,13 @@ def get_parsers():
     parser['del-tag'].add_argument(
             '-f', '--file',
             help=('Read additional hosts to delete tags for from specified '
-                  'file or from stdin if \'-\' (each host on separate line). '
-                  'If no host or host file is specified then, by default, '
-                  'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                  ' are specified then tags are read from stdin and are'
-                  ' deleted from all hosts.'),
+                  'file or from stdin if \'-\' (each host on separate '
+                  'line). '),
             dest='hostfile', required=False)
     parser['del-tag'].add_argument(
             '-T', '--tag-file',
             help=('Read additional tags from specified file '
-                  'or from stdin if \'-\' (each tag on separate line). '
-                  'If no tag or tag file is specified then, by default, '
-                  'read from stdin. If no tags/tagfile nor'
-                  ' hosts/hostfile are specified then tags are read from'
-                  ' stdin and are added to all hosts.'),
+                  'or from stdin if \'-\' (each tag on separate line). '),
             dest='tagfile', required=False)
     parser['del-tag'].add_argument(
             '-t', '--taglist',
diff --git a/cdist/config.py b/cdist/config.py
index 30416008..e84f6f84 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -175,9 +175,11 @@ class Config:
             raise cdist.Error(("Cannot read both, manifest and host file, "
                                "from stdin"))
 
-        # if no host source is specified then read hosts from stdin
         if not (args.hostfile or args.host):
-            args.hostfile = '-'
+            if args.tag or args.all_tagged_hosts:
+                raise cdist.Error(("Target host tag(s) missing"))
+            else:
+                raise cdist.Error(("Target host(s) missing"))
 
         if args.manifest == '-':
             # read initial manifest from stdin
diff --git a/cdist/inventory.py b/cdist/inventory.py
index 6ab20fa7..0387f326 100644
--- a/cdist/inventory.py
+++ b/cdist/inventory.py
@@ -299,7 +299,7 @@ class InventoryHost(Inventory):
         self.all = all
 
         if not self.hosts and not self.hostfile:
-            self.hostfile = "-"
+            raise cdist.Error("Host(s) missing")
 
     def _new_hostpath(self, hostpath):
         # create empty file
@@ -355,7 +355,7 @@ class InventoryTag(Inventory):
         else:
             self.allhosts = False
         if not self.tags and not self.tagfile:
-            self.tagfile = "-"
+            raise cdist.Error("Tag(s) missing")
 
         if self.hostfile == "-" and self.tagfile == "-":
             raise cdist.Error("Cannot read both, hosts and tags, from stdin")
diff --git a/cdist/test/inventory/__init__.py b/cdist/test/inventory/__init__.py
index 287c855c..a8cd8bf8 100644
--- a/cdist/test/inventory/__init__.py
+++ b/cdist/test/inventory/__init__.py
@@ -307,11 +307,10 @@ class InventoryTestCase(test.CdistTestCase):
             raise e
 
     # InventoryTag
+    @unittest.expectedFailure
     def test_inventory_tag_init(self):
         invTag = inventory.InventoryTag(db_basedir=inventory_dir,
                                         action="add")
-        self.assertTrue(invTag.allhosts)
-        self.assertEqual(invTag.tagfile, "-")
 
     def test_inventory_tag_stdin_multiple_hosts(self):
         try:
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index aa2607f8..0ecb4a61 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -177,10 +177,8 @@ Install command is currently in beta.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read specified file for a list of additional hosts to operate on
-    or if '-' is given, read stdin (one host per line).
-    If no host or host file is specified then, by default,
-    read hosts from stdin. For the file format see
-    :strong:`HOSTFILE FORMAT` below.
+    or if '-' is given, read stdin (one host per line). For the file
+    format see :strong:`HOSTFILE FORMAT` below.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -299,9 +297,8 @@ Add host(s) to inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to add from specified file or
-    from stdin if '-' (each host on separate line). If no
-    host or host file is specified then, by default, read
-    from stdin. Hostfile format is the same as config hostfile format.
+    from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -327,11 +324,8 @@ Add tag(s) to inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to add tags from specified file
-    or from stdin if '-' (each host on separate line). If
-    no host or host file is specified then, by default,
-    read from stdin. If no tags/tagfile nor hosts/hostfile
-    are specified then tags are read from stdin and are
-    added to all hosts. Hostfile format is the same as config hostfile format.
+    or from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -346,11 +340,8 @@ Add tag(s) to inventory database.
 
 **-T TAGFILE, --tag-file TAGFILE**
     Read additional tags to add from specified file or
-    from stdin if '-' (each tag on separate line). If no
-    tag or tag file is specified then, by default, read
-    from stdin. If no tags/tagfile nor hosts/hostfile are
-    specified then tags are read from stdin and are added
-    to all hosts. Tagfile format is the same as config hostfile format.
+    from stdin if '-' (each tag on separate line).
+    Tagfile format is the same as config hostfile format.
 
 **-t TAGLIST, --taglist TAGLIST**
     Tag list to be added for specified host(s), comma
@@ -372,9 +363,8 @@ Delete host(s) from inventory database.
 
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to delete from specified file or
-    from stdin if '-' (each host on separate line). If no
-    host or host file is specified then, by default, read
-    from stdin. Hostfile format is the same as config hostfile format.
+    from stdin if '-' (each host on separate line).
+    Hostfile format is the same as config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -404,11 +394,8 @@ Delete tag(s) from inventory database.
 **-f HOSTFILE, --file HOSTFILE**
     Read additional hosts to delete tags for from
     specified file or from stdin if '-' (each host on
-    separate line). If no host or host file is specified
-    then, by default, read from stdin. If no tags/tagfile
-    nor hosts/hostfile are specified then tags are read
-    from stdin and are deleted from all hosts. Hostfile
-    format is the same as config hostfile format.
+    separate line). Hostfile format is the same as
+    config hostfile format.
 
 **-g CONFIG_FILE, --config-file CONFIG_FILE**
     Use specified custom configuration file.
@@ -423,11 +410,8 @@ Delete tag(s) from inventory database.
 
 **-T TAGFILE, --tag-file TAGFILE**
     Read additional tags from specified file or from stdin
-    if '-' (each tag on separate line). If no tag or tag
-    file is specified then, by default, read from stdin.
-    If no tags/tagfile nor hosts/hostfile are specified
-    then tags are read from stdin and are added to all
-    hosts. Tagfile format is the same as config hostfile format.
+    if '-' (each tag on separate line).
+    Tagfile format is the same as config hostfile format.
 
 **-t TAGLIST, --taglist TAGLIST**
     Tag list to be deleted for specified host(s), comma

From 89a0080e133052b82ce19e7fdbaf045dcc833da7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:09:26 +0200
Subject: [PATCH 037/366] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 7a0d050d..260b4fe4 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* Core: Clarify stdin input (Darko Poljak)
+	* Type __package_pip: Detect pip binary (Ander Punnar)
+
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)
 	* Type __unpack: Add --tar-extra-args parameter (Ander Punnar)

From 0fc10749edc698d1f70f826603d009fba943d252 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:11:35 +0200
Subject: [PATCH 038/366] Fix shellcheck

---
 cdist/conf/type/__package_pip/gencode-remote | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 9c5eb0d1..a1375c2d 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -60,7 +60,7 @@ case "$state_should" in
         then
             echo "su -c '$pip install -q $name' $runas"
         else
-            echo $pip install -q "$name"
+            echo "$pip" install -q "$name"
         fi
         echo "installed" >> "$__messages_out"
     ;;
@@ -69,7 +69,7 @@ case "$state_should" in
         then
             echo "su -c '$pip uninstall -q -y $name' $runas"
         else
-            echo $pip uninstall -q -y "$name"
+            echo "$pip" uninstall -q -y "$name"
         fi
         echo "removed" >> "$__messages_out"
     ;;

From b6922508b9e07834009a6bfae2d3933b5ea62fbf Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Sep 2020 09:17:34 +0200
Subject: [PATCH 039/366] Update helper script

---
 bin/build-helper | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/build-helper b/bin/build-helper
index ed41e438..d4d603ed 100755
--- a/bin/build-helper
+++ b/bin/build-helper
@@ -371,7 +371,6 @@ eof
 Manual steps post release:
     - cdist-web
     - send generated mailinglist.tmp mail
-    - twitter
 eof
     ;;
 

From 84a7818121047cf6bb23d8bd3c055d09bb033d32 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 23 Sep 2020 20:29:47 +0200
Subject: [PATCH 040/366] docs: make varaibles environment-aware

There are all overwriting the environment, even the comment states
otherwise. Fixes it.
---
 docs/src/Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/src/Makefile b/docs/src/Makefile
index 2ecf7a32..ba87d170 100644
--- a/docs/src/Makefile
+++ b/docs/src/Makefile
@@ -2,10 +2,10 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-PAPER         =
-BUILDDIR      = ../dist
+SPHINXOPTS   ?=
+SPHINXBUILD  ?= sphinx-build
+PAPER        ?=
+BUILDDIR     ?= ../dist
 # for cache, etc.
 _BUILDDIR     = _build
 

From 73d6c9d469d1429999ea2e980b3ac4c63a3e0dd4 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 27 Sep 2020 10:17:35 +0200
Subject: [PATCH 041/366] Add custom remote copy/exec examples

---
 docs/src/cdist-remote-exec-copy.rst | 389 +++++++++++++++++++++++++++-
 1 file changed, 388 insertions(+), 1 deletion(-)

diff --git a/docs/src/cdist-remote-exec-copy.rst b/docs/src/cdist-remote-exec-copy.rst
index bb818310..e7b7b226 100644
--- a/docs/src/cdist-remote-exec-copy.rst
+++ b/docs/src/cdist-remote-exec-copy.rst
@@ -10,7 +10,7 @@ By default this is accomplished with ssh and scp respectively.
 The default implementations used by cdist are::
 
     __remote_exec: ssh -o User=root
-    __remote_copy: scp -o User=root
+    __remote_copy: scp -o User=root -q
 
 The user can override these defaults by providing custom implementations and
 passing them to cdist with the --remote-exec and/or --remote-copy arguments.
@@ -26,3 +26,390 @@ specified by enclosed in square brackets (see :strong:`ssh`\ (1) and
 With this simple interface the user can take total control of how cdist
 interacts with the target when required, while the default implementation 
 remains as simple as possible.
+
+
+Examples
+--------
+
+Here are examples of using alternative __remote_copy and __remote_exec scripts.
+
+All scripts from below are present in cdist sources in `other/examples/remote`
+directory.
+
+ssh
+~~~
+
+Same as cdist default.
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    #echo "$@" | logger -t "cdist-ssh-copy"
+    scp -o User=root -q $@
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    #echo "$@" | logger -t "cdist-ssh-exec"
+    ssh -o User=root $@
+
+local
+~~~~~
+
+This effectively turns remote calling into local calling. Probably most useful
+for the unit testing.
+
+**copy**
+
+.. code-block:: sh
+
+    code="$(echo "$@" | sed "s|\([[:space:]]\)$__target_host:|\1|g")"
+    cp -L $code
+
+**exec**
+
+.. code-block:: sh
+
+    target_host=$1; shift
+    echo "$@" | /bin/sh 
+
+chroot
+~~~~~~
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script /path/to/your/chroot" target-id
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-chroot-copy"
+       :
+    }
+
+    chroot="$1"; shift
+    target_host="$__target_host"
+
+    # replace target_host with chroot location
+    code="$(echo "$@" | sed "s|$target_host:|$chroot|g")"
+
+    log "target_host: $target_host"
+    log "chroot: $chroot"
+    log "$@"
+    log "$code"
+
+    # copy files into chroot
+    cp $code
+
+    log "-----"
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script /path/to/your/chroot" target-id
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-chroot-exec"
+       :
+    }
+
+    chroot="$1"; shift
+    target_host="$1"; shift
+
+    script=$(mktemp "${chroot}/tmp/chroot-${0##*/}.XXXXXXXXXX")
+    trap cleanup INT TERM EXIT
+    cleanup() {
+       [ $__cdist_debug ] || rm "$script"
+    }
+
+    log "target_host: $target_host"
+    log "script: $script"
+    log "@: $@"
+    echo "#!/bin/sh -l" > "$script"
+    echo "$@" >> "$script"
+    chmod +x "$script"
+
+    relative_script="${script#$chroot}"
+    log "relative_script: $relative_script"
+
+    # run in chroot
+    chroot "$chroot" "$relative_script"
+
+    log "-----"
+
+rsync
+~~~~~
+
+**copy**
+
+Usage: cdist config --remote-copy /path/to/this/script target_host
+
+.. code-block:: sh
+
+    # For rsync to do the right thing, the source has to end with "/" if it is
+    # a directory. The below preprocessor loop takes care of that.
+
+    # second last argument is the source
+    source_index=$(($#-1))
+    index=0
+    for arg in $@; do
+       if [ $index -eq 0 ]; then
+          # reset $@
+          set --
+       fi
+       index=$((index+=1))
+       if [ $index -eq $source_index -a -d "$arg" ]; then
+          arg="${arg%/}/"
+       fi
+       set -- "$@" "$arg"
+    done
+
+    rsync --backup --suffix=~cdist -e 'ssh -o User=root' $@
+
+schroot
+~~~~~~~
+
+__remote_copy and __remote_exec scripts to run cdist against a chroot on the
+target host over ssh.
+
+**copy**
+
+Usage: cdist config --remote-copy "/path/to/this/script schroot-chroot-name" target_host
+
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-schroot-copy"
+       :
+    }
+
+    chroot_name="$1"; shift
+    target_host="$__target_host"
+
+    # get directory for given chroot_name
+    chroot="$(ssh -o User=root -q $target_host schroot -c $chroot_name --config | awk -F = '/directory=/ {print $2}')"
+
+    # prefix destination with chroot
+    code="$(echo "$@" | sed "s|$target_host:|$target_host:$chroot|g")"
+
+    log "target_host: $target_host"
+    log "chroot_name: $chroot_name"
+    log "chroot: $chroot"
+    log "@: $@"
+    log "code: $code"
+
+    # copy files into remote chroot
+    scp -o User=root -q $code
+
+    log "-----"
+
+**exec**
+
+Usage: cdist config --remote-exec "/path/to/this/script schroot-chroot-name" target_host
+
+.. code-block:: sh
+
+    log() {
+       #echo "$@" | logger -t "cdist-schroot-exec"
+       :
+    }
+
+    chroot_name="$1"; shift
+    target_host="$1"; shift
+
+    code="ssh -o User=root -q $target_host schroot -c $chroot_name -- $@"
+
+    log "target_host: $target_host"
+    log "chroot_name: $chroot_name"
+    log "@: $@"
+    log "code: $code"
+
+    # run in remote chroot
+    $code
+
+    log "-----"
+
+schroot-uri
+~~~~~~~~~~~
+
+__remote_exec/__remote_copy script to run cdist against a schroot target URI.
+
+Usage::
+
+    cdist config \
+        --remote-exec "/path/to/this/script exec" \
+        --remote-copy "/path/to/this/script copy" \
+        target_uri
+
+    # target_uri examples:
+    schroot:///chroot-name
+    schroot://foo.ethz.ch/chroot-name
+    schroot://user-name@foo.ethz.ch/chroot-name
+
+    # and how to match them in .../manifest/init
+    case "$target_host" in
+    schroot://*)
+        # any schroot
+    ;;
+    schroot://foo.ethz.ch/*)
+        # any schroot on specific host
+    ;;
+    schroot://foo.ethz.ch/chroot-name)
+        # specific schroot on specific host
+    ;;
+    schroot:///chroot-name)
+        # specific schroot on localhost
+    ;;
+    esac
+
+**copy/exec**
+
+.. code-block:: sh
+
+    my_name="${0##*/}"
+    mode="$1"; shift
+
+    log() {
+       # uncomment me for debugging
+       #echo "$@" | logger -t "cdist-$my_name-$mode"
+       :
+    }
+
+    die() {
+       echo "$@" >&2
+       exit 1
+    }
+
+
+    uri="$__target_host"
+
+    scheme="${uri%%:*}"; rest="${uri#$scheme:}"; rest="${rest#//}"
+    authority="${rest%%/*}"; rest="${rest#$authority}"
+    path="${rest%\?*}"; rest="${rest#$path}"
+    schroot_name="${path#/}"
+
+    [ "$scheme" = "schroot" ] || die "Failed to parse scheme from __target_host ($__target_host). Expected 'schroot', got '$scheme'"
+    [ -n "$schroot_name" ] || die "Failed to parse schroot name from __target_host: $__target_host"
+
+    case "$authority" in
+       '')
+          # authority is empty, neither user nor host given
+          user=""
+          host=""
+       ;; 
+       *@*)
+          # authority contains @, take user from authority
+          user="${authority%@*}"
+          host="${authority#*@}"
+       ;; 
+       *) 
+          # no user in authority, default to root
+          user="root"
+          host="$authority"
+       ;;
+    esac
+
+    log "mode: $mode"
+    log "@: $@"
+    log "uri: $uri"
+    log "scheme: $scheme"
+    log "authority: $authority"
+    log "user: $user"
+    log "host: $host"
+    log "path: $path"
+    log "schroot_name: $schroot_name"
+
+    exec_prefix=""
+    copy_prefix=""
+    if [ -n "$host" ]; then
+       # we are working on a remote host
+       exec_prefix="ssh -o User=$user -q $host"
+       copy_prefix="scp -o User=$user -q"
+       copy_destination_prefix="$host:"
+    else
+       # working on local machine
+       copy_prefix="cp"
+       copy_destination_prefix=""
+    fi
+    log "exec_prefix: $exec_prefix"
+    log "copy_prefix: $copy_prefix"
+    log "copy_destination_prefix: $copy_destination_prefix"
+
+    case "$mode" in
+       exec)
+          # In exec mode the first argument is the __target_host which we already got from env. Get rid of it.
+          shift
+          code="$exec_prefix schroot -c $schroot_name -- sh -c '$@'"
+       ;;
+       copy)
+          # get directory for given chroot_name
+          schroot_directory="$($exec_prefix schroot -c $schroot_name --config | awk -F = '/directory=/ {print $2}')"
+          [ -n "$schroot_directory" ] || die "Failed to retreive schroot directory for schroot: $schroot_name"
+          log "schroot_directory: $schroot_directory"
+          # prefix destination with chroot
+          code="$copy_prefix $(echo "$@" | sed "s|$uri:|${copy_destination_prefix}${schroot_directory}|g")"
+       ;;
+       *) die "Unknown mode: $mode";;
+    esac
+
+    log "code: $code"
+
+    # Run the code
+    $code
+
+    log "-----"
+
+sudo
+~~~~
+
+**copy**
+
+Use rsync over ssh to copy files. Uses the "--rsync-path" option
+to run the remote rsync instance with sudo.
+
+This command assumes your ssh configuration is already set up in ~/.ssh/config.
+
+Usage: cdist config --remote-copy /path/to/this/script target_host
+
+.. code-block:: sh
+
+    # For rsync to do the right thing, the source has to end with "/" if it is
+    # a directory. The below preprocessor loop takes care of that.
+
+    # second last argument is the source
+    source_index=$(($#-1))
+    index=0
+    for arg in $@; do
+       if [ $index -eq 0 ]; then
+          # reset $@
+          set --
+       fi
+       index=$((index+=1))
+       if [ $index -eq $source_index -a -d "$arg" ]; then
+          arg="${arg%/}/"
+       fi
+       set -- "$@" "$arg"
+    done
+
+    rsync --copy-links --rsync-path="sudo rsync" -e 'ssh' "$@"
+
+**exec**
+
+Prefixes all remote commands with sudo.
+
+This command assumes your ssh configuration is already set up in ~/.ssh/config.
+
+Usage: cdist config --remote-exec "/path/to/this/script" target_host
+
+.. code-block:: sh
+
+    host="$1"; shift
+    ssh -q "$host" sudo sh -c \""$@"\"

From 652c89185816526e5165ff9d5686a0d245ec3e5f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 29 Sep 2020 05:57:54 +0200
Subject: [PATCH 042/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 260b4fe4..8d380524 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
+	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From f994226d0e23b667309fd75116c0c8a14d15f34d Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 29 Sep 2020 19:44:47 +0200
Subject: [PATCH 043/366] [__package_pkgng_freebsd] Bootstrap pkg if necessary

In a pristine FreeBSD base installation, pkg is really a bootstrapper utility,
in such cases the type used to fail instead of automatically bootstrapping pkg.
---
 .../type/__package_pkgng_freebsd/explorer/pkg_bootstrapped   | 4 ++++
 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version | 5 +++++
 cdist/conf/type/__package_pkgng_freebsd/gencode-remote       | 5 +++++
 3 files changed, 14 insertions(+)
 create mode 100755 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped

diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped
new file mode 100755
index 00000000..429f15d3
--- /dev/null
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+if pkg -N >/dev/null 2>&1; then
+	echo "YES"
+fi
diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
index 92ce0623..f0fb9127 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
@@ -21,6 +21,11 @@
 # Retrieve the status of a package - parsed dpkg output
 #
 
+if ! pkg -N >/dev/null 2>&1; then
+   # Nothing to do if pkg is not bootstrapped
+   exit
+fi
+
 if [ -f "$__object/parameter/name" ]; then
    name="$(cat "$__object/parameter/name")"
 else
diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
index dd36efda..b5944177 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
+++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
@@ -43,6 +43,7 @@ fi
 repo="$(cat "$__object/parameter/repo")"
 state="$(cat "$__object/parameter/state")"
 curr_version="$(cat "$__object/explorer/pkg_version")"
+pkg_bootstrapped="$(cat "$__object/explorer/pkg_bootstrapped")"
 add_cmd="pkg install -y"
 rm_cmd="pkg delete -y"
 upg_cmd="pkg upgrade -y"
@@ -73,6 +74,10 @@ execcmd(){
          ;;
    esac
 
+   if [ -z "${pkg_bootstrapped}" ]; then
+       echo "pkg bootstrap -y >/dev/null 2>&1"
+   fi
+
    echo "$_cmd >/dev/null 2>&1"   # Silence the output of the command
    echo "status=\$?"
    echo "if [ \"\$status\" -ne \"0\" ]; then"

From 52b5f05163416196403634cb9a52f746cdb96391 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 30 Sep 2020 08:56:31 +0200
Subject: [PATCH 044/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8d380524..34c97d67 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
+	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 3fa74b454a9512319e3559a53dd477f38724fd9b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 30 Sep 2020 15:43:32 +0200
Subject: [PATCH 045/366] Fix typo

---
 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
index f0fb9127..1c6ba5e5 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
+++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Retrieve the status of a package - parsed dpkg output
+# Retrieve the status of a package - parsed pkgng output
 #
 
 if ! pkg -N >/dev/null 2>&1; then

From 5aeed14b1b8b022f952680cceb12a5099a099558 Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Thu, 8 Oct 2020 16:15:20 +0200
Subject: [PATCH 046/366] Fixed calling of __systemd_service type with correct
 arguments.

---
 cdist/conf/type/__service/manifest | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__service/manifest b/cdist/conf/type/__service/manifest
index cb5af234..beb0713c 100644
--- a/cdist/conf/type/__service/manifest
+++ b/cdist/conf/type/__service/manifest
@@ -7,7 +7,9 @@ action="$(cat "$__object/parameter/action")"
 
 case "$manager" in
 	systemd)
-		__systemd_service "$name" --action "$action"
+		test "$action" = "start" && action="running"
+		test "$action" = "stop"  && action="stopped"
+		__systemd_service "$name" --state "$action"
 	;;
 	*)
 		# Unknown: handled by `service $NAME $action` in gencode-remote.

From c030deea3dbccfe73637b730f0d545db7f8d5f35 Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Fri, 9 Oct 2020 06:51:44 +0200
Subject: [PATCH 047/366] [__line] Add support for '--state replace'

It is currently counter-intuitive that something like:

    # File '/thing' contents
    #SomeSetting WrongValue

    # Manifest
    __line '/thing' \
           --line 'SomeSeting GoodValue' \
           --regex '^(#[[:space:]]*)?SomeSetting[[:space:]]'

Produces:

    # Resulting '/thing' contents
    #SomeSetting WrongValue

This makes sense given the implementation, but it masks a very common use-case.

Changing the default behaviour for such a base type is not really an option, so
instead we add a `replace` as a valid value for `--state`, which would result
in:

    # Resulting '/thing' contents with: --state replace
    SomeSetting GoodValue

For compatibility, if the regex is missing, `--state replace` behaves just as
`--state present`.
---
 cdist/conf/type/__line/explorer/state | 12 +++++++++++-
 cdist/conf/type/__line/gencode-remote | 10 +++++++---
 cdist/conf/type/__line/man.rst        | 13 +++++++++++--
 3 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state
index e8fc3630..9d480b19 100755
--- a/cdist/conf/type/__line/explorer/state
+++ b/cdist/conf/type/__line/explorer/state
@@ -53,8 +53,10 @@ function _find(_text, _pattern) {
 BEGIN {
    getline anchor < (ENVIRON["__object"] "/parameter/" position)
    getline pattern < (ENVIRON["__object"] "/parameter/" needle)
+   getline line < (ENVIRON["__object"] "/parameter/line")
 
    found_line = 0
+   correct_line = 0
    correct_pos = (position != "after" && position != "before")
 }
 {
@@ -63,15 +65,18 @@ BEGIN {
          getline
          if (_find($0, pattern)) {
             found_line++
+	    if (index($0, line) == 1) { correct_line++ }
             correct_pos = 1
             exit 0
          }
       } else if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
       }
    } else if (position == "before") {
       if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
          getline
          if (match($0, anchor)) {
             correct_pos = 1
@@ -81,13 +86,18 @@ BEGIN {
    } else {
       if (_find($0, pattern)) {
          found_line++
+         if (index($0, line) == 1) { correct_line++ }
          exit 0
       }
    }
 }
 END {
    if (found_line && correct_pos) {
-      print "present"
+      if (correct_line) {
+        print "present"
+      } else {
+        print "matching"
+      }
    } else if (found_line) {
       print "wrongposition"
    } else {
diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote
index 88cae68b..a89886da 100755
--- a/cdist/conf/type/__line/gencode-remote
+++ b/cdist/conf/type/__line/gencode-remote
@@ -38,7 +38,11 @@ if [ -z "$state_is" ]; then
    exit 1
 fi
 
-if [ "$state_should" = "$state_is" ]; then
+if [ "$state_should" = "$state_is" ] || \
+	{ [ "$state_should" = "present" ] && [ "$state_is" = "matching" ] ;} || \
+	{ [ "$state_should" = "replace" ] && [ "$state_is" = "present" ] ;} ; then
+   # If state matches already, or 'present' is used and regex matches
+   # or 'replace' is used and the exact line is present, then there is
    # nothing to do
    exit 0
 fi
@@ -61,8 +65,8 @@ fi
 add=0
 remove=0
 case "$state_should" in
-   present)
-      if [ "$state_is" = "wrongposition" ]; then
+   present|replace)
+      if [ "$state_is" = "wrongposition" ] || [ "$state_is" = "matching" ]; then
          echo updated >> "$__messages_out"
          remove=1
       else
diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst
index f76cab64..70490f68 100644
--- a/cdist/conf/type/__line/man.rst
+++ b/cdist/conf/type/__line/man.rst
@@ -31,7 +31,7 @@ file
 line
     Specifies the line which should be absent or present.
 
-    Must be present, if state is 'present'.
+    Must be present, if state is 'present' or 'replace'.
     Ignored if regex is given and state is 'absent'.
 
 regex
@@ -41,10 +41,13 @@ regex
     If state is 'absent', ensure all lines matching the regular expression
     are absent.
 
+    If state is 'replace', ensure all lines matching the regular expression
+    are exactly 'line'.
+
     The regular expression is interpreted by awk's match function.
 
 state
-    'present' or 'absent', defaults to 'present'
+    'present', 'absent' or 'replace', defaults to 'present'.
 
 onchange
     The code to run if line is added, removed or updated.
@@ -99,6 +102,12 @@ EXAMPLES
         --line '-session required pam_exec.so debug log=/tmp/classify.log /usr/local/libexec/classify' \
         --after '^session[[:space:]]+include[[:space:]]+password-auth-ac$'
 
+    # Uncomment as needed and set a value in a configuration file.
+    __line /etc/example.conf \
+        --line 'SomeSetting SomeValue' \
+        --regex '^(#[[:space:]]*)?SomeSetting[[:space:]]' \
+        --state replace
+
 
 SEE ALSO
 --------

From 4df5c91912edf9f31a6716d7cbe0adaedbc2b0e3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 9 Oct 2020 06:52:52 +0200
Subject: [PATCH 048/366] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 34c97d67..6d237449 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,8 @@ next:
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)
 	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
+	* Type __service: Fix calling __systemd_service (Mark Verboom)
+	* Type __line: Add 'replace' state (Evil Ham)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 8ecae42199f27693eec771513174ce8143c8ca83 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:00:11 +0300
Subject: [PATCH 049/366] remove bin/cdist script

---
 bin/cdist | 33 ---------------------------------
 1 file changed, 33 deletions(-)
 delete mode 100755 bin/cdist

diff --git a/bin/cdist b/bin/cdist
deleted file mode 100755
index 645020a1..00000000
--- a/bin/cdist
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/sh
-# -*- coding: utf-8 -*-
-#
-# 2012 Nico Schottelius (nico-cdist at schottelius.org)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-# Wrapper for real script to allow execution from checkout
-dir=${0%/*}
-
-# Ensure version is present - the bundled/shipped version contains a static version,
-# the git version contains a dynamic version
-"$dir/build-helper" version
-
-libdir=$(cd "${dir}/../" && pwd -P)
-export PYTHONPATH="${libdir}"
-
-"$dir/../scripts/cdist" "$@"

From 45d51c0e1553ce12fcea09b1b9e0d2fb9c0304c5 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:00:44 +0300
Subject: [PATCH 050/366] rename build-helper -> cdist-build-helper

---
 bin/{build-helper => cdist-build-helper} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename bin/{build-helper => cdist-build-helper} (100%)

diff --git a/bin/build-helper b/bin/cdist-build-helper
similarity index 100%
rename from bin/build-helper
rename to bin/cdist-build-helper

From 3f1939716f4c4c2cdf538a22170a67fe181c675c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:02:45 +0300
Subject: [PATCH 051/366] enable running scripts/cdist directly and symlinked

---
 scripts/cdist | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scripts/cdist b/scripts/cdist
index b1d782ab..2ce40ae0 100755
--- a/scripts/cdist
+++ b/scripts/cdist
@@ -22,7 +22,15 @@
 #
 
 import logging
+import os
 import sys
+
+cdist_bin = os.path.abspath(__file__)
+if os.path.islink(cdist_bin):
+    cdist_bin = os.readlink(cdist_bin)
+cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
+sys.path.insert(0, cdist_dir)
+
 import cdist
 import cdist.argparse
 import cdist.banner

From fdc1ab93e968575b1d1f6f3b2a6cefa6ed6bb850 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:03:11 +0300
Subject: [PATCH 052/366] move scripts/* to bin/

---
 {scripts => bin}/cdist          | 0
 {scripts => bin}/cdist-dump     | 0
 {scripts => bin}/cdist-new-type | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 rename {scripts => bin}/cdist (100%)
 rename {scripts => bin}/cdist-dump (100%)
 rename {scripts => bin}/cdist-new-type (100%)

diff --git a/scripts/cdist b/bin/cdist
similarity index 100%
rename from scripts/cdist
rename to bin/cdist
diff --git a/scripts/cdist-dump b/bin/cdist-dump
similarity index 100%
rename from scripts/cdist-dump
rename to bin/cdist-dump
diff --git a/scripts/cdist-new-type b/bin/cdist-new-type
similarity index 100%
rename from scripts/cdist-new-type
rename to bin/cdist-new-type

From 86057cef19b10859cd66ceb4e3ebade5986aff89 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 14 Oct 2020 02:05:17 +0300
Subject: [PATCH 053/366] don't die if there is no version.py

---
 cdist/__init__.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index be573170..26e7d071 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -24,10 +24,13 @@ import os
 import hashlib
 
 import cdist.log
-import cdist.version
 
 
-VERSION = cdist.version.VERSION
+try:
+    import cdist.version
+    VERSION = cdist.version.VERSION
+except ModuleNotFoundError:
+    VERSION = 'from git'
 
 BANNER = """
              ..          .       .x+=:.        s

From fd04c036139f150be79801e5fd3b49086e5d7263 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 13:42:16 +0300
Subject: [PATCH 054/366] add parent dir to module search path only when
 importing fails

---
 bin/cdist | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index 2ce40ae0..66ccbb5a 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -25,13 +25,19 @@ import logging
 import os
 import sys
 
-cdist_bin = os.path.abspath(__file__)
-if os.path.islink(cdist_bin):
-    cdist_bin = os.readlink(cdist_bin)
-cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
-sys.path.insert(0, cdist_dir)
+# try to import cdist and if that fails, then add this file's parent dir to
+# module search path and try again. additionally check if this file is
+# symlinked, so user can symlink this file to, for example, ~/.local/bin.
+try:
+    import cdist
+except ModuleNotFoundError:
+    cdist_bin = os.path.abspath(__file__)
+    if os.path.islink(cdist_bin):
+        cdist_bin = os.readlink(cdist_bin)
+    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
+    sys.path.insert(0, cdist_dir)
+    import cdist
 
-import cdist
 import cdist.argparse
 import cdist.banner
 import cdist.config

From 1614b62f702a82731b3b9c636894268b4310821b Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 13:48:28 +0300
Subject: [PATCH 055/366] fallback VERSION to "unknown version"

---
 cdist/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index 26e7d071..350a2765 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -30,7 +30,7 @@ try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
-    VERSION = 'from git'
+    VERSION = 'unknown version'
 
 BANNER = """
              ..          .       .x+=:.        s

From 174aa7728065da611dc8887d9f33ead7e27c473a Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:11:00 +0300
Subject: [PATCH 056/366] __file__ already is absolute

---
 bin/cdist | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/cdist b/bin/cdist
index 66ccbb5a..d7eae312 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -31,7 +31,7 @@ import sys
 try:
     import cdist
 except ModuleNotFoundError:
-    cdist_bin = os.path.abspath(__file__)
+    cdist_bin = __file__
     if os.path.islink(cdist_bin):
         cdist_bin = os.readlink(cdist_bin)
     cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))

From 65c8af4ba3751dddbfe07fd5077e5a99b5d240b8 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:11:12 +0300
Subject: [PATCH 057/366] overengineered version discovery

---
 cdist/__init__.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index 350a2765..f659506f 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -22,6 +22,7 @@
 
 import os
 import hashlib
+import subprocess
 
 import cdist.log
 
@@ -30,7 +31,19 @@ try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
-    VERSION = 'unknown version'
+    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+    if os.path.isdir(os.path.join(cdist_dir, '.git')):
+        run_git = subprocess.run(
+            ['git', 'describe', '--always'],
+            cwd=cdist_dir,
+            capture_output=True,
+            text=True)
+        if run_git.returncode == 0:
+            VERSION = str(run_git.stdout)
+        else:
+            VERSION = 'from git'
+    else:
+        VERSION = 'unknown version'
 
 BANNER = """
              ..          .       .x+=:.        s

From 42d5d6c3e29f6500c3f6321b01c3b268e6886ec6 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:12:39 +0300
Subject: [PATCH 058/366] redundant str()

---
 cdist/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index f659506f..c16562e7 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -39,7 +39,7 @@ except ModuleNotFoundError:
             capture_output=True,
             text=True)
         if run_git.returncode == 0:
-            VERSION = str(run_git.stdout)
+            VERSION = run_git.stdout
         else:
             VERSION = 'from git'
     else:

From b41d80075a616a1c7ac3a361079c748424409995 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 14:16:04 +0300
Subject: [PATCH 059/366] update paths in setup.py

---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 7b000041..002be19c 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ import subprocess
 
 
 # We have it only if it is a git cloned repo.
-build_helper = os.path.join('bin', 'build-helper')
+build_helper = os.path.join('bin', 'cdist-build-helper')
 # Version file path.
 version_file = os.path.join('cdist', 'version.py')
 # If we have build-helper we could be a git repo.
@@ -56,7 +56,7 @@ setup(
     name="cdist",
     packages=["cdist", "cdist.core", "cdist.exec", "cdist.util", ],
     package_data={'cdist': package_data},
-    scripts=["scripts/cdist", "scripts/cdist-dump", "scripts/cdist-new-type"],
+    scripts=["bin/cdist", "bin/cdist-dump", "bin/cdist-new-type"],
     version=cdist.version.VERSION,
     description="A Usable Configuration Management System",
     author="Nico Schottelius",

From e55db1b427fcc7f750adfcb3646a3eeb446113de Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 15:41:38 +0300
Subject: [PATCH 060/366] use check_output for git describe execution and
 define fallback VERSION earlier

---
 cdist/__init__.py | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index c16562e7..1c60ae0f 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -27,23 +27,21 @@ import subprocess
 import cdist.log
 
 
+VERSION = 'unknown version'
+
 try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
     cdist_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
     if os.path.isdir(os.path.join(cdist_dir, '.git')):
-        run_git = subprocess.run(
-            ['git', 'describe', '--always'],
-            cwd=cdist_dir,
-            capture_output=True,
-            text=True)
-        if run_git.returncode == 0:
-            VERSION = run_git.stdout
-        else:
-            VERSION = 'from git'
-    else:
-        VERSION = 'unknown version'
+        try:
+            VERSION = subprocess.check_output(
+                ['git', 'describe', '--always'],
+                cwd=cdist_dir,
+                universal_newlines=True)
+        except:
+            pass
 
 BANNER = """
              ..          .       .x+=:.        s

From 54d83a62118ed3f0c6328c4b25a3b9ee56adf27a Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Fri, 16 Oct 2020 15:50:50 +0300
Subject: [PATCH 061/366] there is no single author anymore, also remove www.

---
 setup.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 002be19c..858c2c17 100644
--- a/setup.py
+++ b/setup.py
@@ -59,9 +59,8 @@ setup(
     scripts=["bin/cdist", "bin/cdist-dump", "bin/cdist-new-type"],
     version=cdist.version.VERSION,
     description="A Usable Configuration Management System",
-    author="Nico Schottelius",
-    author_email="nico-cdist-pypi@schottelius.org",
-    url="https://www.cdi.st/",
+    author="cdist contributors",
+    url="https://cdi.st",
     classifiers=[
         "Development Status :: 6 - Mature",
         "Environment :: Console",

From 507fa6fa93210959b52bb639a9394c8ac52b40e5 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 17 Oct 2020 17:05:09 +0200
Subject: [PATCH 062/366] __download: fix non-existent parameter of __unpack

Probably happened due to renaming .. guess it's correct now.
---
 cdist/conf/type/__download/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index eb3ac971..d8814683 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -69,7 +69,7 @@ EXAMPLES
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
-            --move-existing-destination \
+            --backup-destination \
             --destination /opt/cpma/server
 
 

From d20fb743243f0341820a5b41a5725fb705eb5aae Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sat, 17 Oct 2020 23:16:42 +0300
Subject: [PATCH 063/366] use os.path.realpath instead, because it eliminates
 any symbolic links encountered in the path

---
 bin/cdist | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index d7eae312..1f92f157 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -25,16 +25,16 @@ import logging
 import os
 import sys
 
-# try to import cdist and if that fails, then add this file's parent dir to
-# module search path and try again. additionally check if this file is
-# symlinked, so user can symlink this file to, for example, ~/.local/bin.
+# try to import cdist and if that fails,
+# then add this file's parent dir to
+# module search path and try again.
 try:
     import cdist
 except ModuleNotFoundError:
-    cdist_bin = __file__
-    if os.path.islink(cdist_bin):
-        cdist_bin = os.readlink(cdist_bin)
-    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(cdist_bin), os.pardir))
+    cdist_dir = os.path.realpath(
+        os.path.join(
+            os.path.dirname(os.path.realpath(__file__)),
+            os.pardir))
     sys.path.insert(0, cdist_dir)
     import cdist
 

From b2e6afb57e6799b934a2eaa387cea337d65d8aac Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 17 Oct 2020 23:01:36 +0200
Subject: [PATCH 064/366] __download: adapt download+unpack example in manpage

---
 cdist/conf/type/__download/man.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index d8814683..54503470 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -70,6 +70,7 @@ EXAMPLES
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
             --backup-destination \
+            --preserve-archive \
             --destination /opt/cpma/server
 
 

From 955b84727611caa5aa05e4521af326c94c649e89 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 18 Oct 2020 15:55:14 +0200
Subject: [PATCH 065/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 6d237449..8ad2c624 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __package_pkgng_freebsd: Bootstrap pkg if necessary (Evil Ham)
 	* Type __service: Fix calling __systemd_service (Mark Verboom)
 	* Type __line: Add 'replace' state (Evil Ham)
+	* Type __download: Fix man page (Matthias Stecher)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 6964070282a6c5d09a458017623012fc17e3008d Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 18 Oct 2020 17:13:22 +0300
Subject: [PATCH 066/366] s/build-helper/cdist-build-helper/

---
 .gitlab-ci.yml             | 8 ++++----
 README-maintainers         | 2 +-
 bin/cdist-build-helper     | 2 +-
 docs/src/cdist-install.rst | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e215652c..e48355ea 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,15 +6,15 @@ image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
 unit_tests:
     stage: test
     script:
-        - ./bin/build-helper version
-        - ./bin/build-helper test
+        - ./bin/cdist-build-helper version
+        - ./bin/cdist-build-helper test
 
 pycodestyle:
     stage: test
     script:
-        - ./bin/build-helper pycodestyle
+        - ./bin/cdist-build-helper pycodestyle
 
 shellcheck:
     stage: test
     script:
-        - ./bin/build-helper shellcheck
+        - ./bin/cdist-build-helper shellcheck
diff --git a/README-maintainers b/README-maintainers
index af57f475..5766dd7d 100644
--- a/README-maintainers
+++ b/README-maintainers
@@ -1,4 +1,4 @@
-Maintainers should use ./bin/build-helper script.
+Maintainers should use ./bin/cdist-build-helper script.
 
 Makefile is intended for end users. It can be used for non-maintaining
 targets that can be run from pure source (without git repository).
diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper
index d4d603ed..bdef0dbb 100755
--- a/bin/cdist-build-helper
+++ b/bin/cdist-build-helper
@@ -495,7 +495,7 @@ eof
     ;;
 
     shellcheck-build-helper)
-        ${SHELLCHECKCMD} ./bin/build-helper
+        ${SHELLCHECKCMD} ./bin/cdist-build-helper
     ;;
 
     check-shellcheck)
diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst
index 6f4f14d7..18863145 100644
--- a/docs/src/cdist-install.rst
+++ b/docs/src/cdist-install.rst
@@ -49,7 +49,7 @@ create version.py:
 
 .. code-block:: sh
 
-    ./bin/build-helper version
+    ./bin/cdist-build-helper version
 
 Then you install it with:
 
@@ -70,7 +70,7 @@ Or directly with distutils:
 
     python setup.py install
 
-Note that `bin/build-helper` script is intended for cdist maintainers.
+Note that `bin/cdist-build-helper` script is intended for cdist maintainers.
 
 
 Available versions in git

From e3d906a85fe5e0c9f254d3620e8568437c93d97c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 15 Sep 2020 00:55:26 +0300
Subject: [PATCH 067/366] [__acl] remove deprecated parameters, fix some bugs
 and improve manual

---
 cdist/conf/type/__acl/explorer/checks         | 39 -------------------
 cdist/conf/type/__acl/explorer/getent         |  4 ++
 cdist/conf/type/__acl/gencode-remote          | 36 ++++++++---------
 cdist/conf/type/__acl/man.rst                 | 12 +++---
 .../conf/type/__acl/parameter/deprecated/acl  |  1 -
 .../type/__acl/parameter/deprecated/group     |  1 -
 .../conf/type/__acl/parameter/deprecated/mask |  1 -
 .../type/__acl/parameter/deprecated/other     |  1 -
 .../conf/type/__acl/parameter/deprecated/user |  1 -
 cdist/conf/type/__acl/parameter/optional      |  2 -
 .../type/__acl/parameter/optional_multiple    |  3 --
 11 files changed, 26 insertions(+), 75 deletions(-)
 delete mode 100755 cdist/conf/type/__acl/explorer/checks
 create mode 100755 cdist/conf/type/__acl/explorer/getent
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/acl
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/group
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/mask
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/other
 delete mode 100644 cdist/conf/type/__acl/parameter/deprecated/user

diff --git a/cdist/conf/type/__acl/explorer/checks b/cdist/conf/type/__acl/explorer/checks
deleted file mode 100755
index 70bb0412..00000000
--- a/cdist/conf/type/__acl/explorer/checks
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/sh -e
-#
-# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-# TODO check if filesystem has ACL turned on etc
-
-if [ -f "$__object/parameter/acl" ]
-then
-    grep -E '^(default:)?(user|group):' "$__object/parameter/acl" \
-    | while read -r acl
-    do
-        param="$( echo "$acl" | awk -F: '{print $(NF-2)}' )"
-        check="$( echo "$acl" | awk -F: '{print $(NF-1)}' )"
-
-        [ "$param" = 'user' ] && db=passwd || db="$param"
-
-        if ! getent "$db" "$check" > /dev/null
-        then
-            echo "missing $param '$check'" >&2
-            exit 1
-        fi
-    done
-fi
diff --git a/cdist/conf/type/__acl/explorer/getent b/cdist/conf/type/__acl/explorer/getent
new file mode 100755
index 00000000..7e6c2c30
--- /dev/null
+++ b/cdist/conf/type/__acl/explorer/getent
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+getent passwd | awk -F: '{print "user:"$1}'
+getent group | awk -F: '{print "group:"$1}'
diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote
index e5404a9d..32318e91 100755
--- a/cdist/conf/type/__acl/gencode-remote
+++ b/cdist/conf/type/__acl/gencode-remote
@@ -22,8 +22,8 @@ file_is="$( cat "$__object/explorer/file_is" )"
 
 if [ "$file_is" = 'missing' ] \
     && [ -z "$__cdist_dry_run" ] \
-    && \( [ ! -f "$__object/parameter/file" ] \
-        || [ ! -f "$__object/parameter/directory" ] \)
+    && [ ! -f "$__object/parameter/file" ] \
+    && [ ! -f "$__object/parameter/directory" ]
 then
     exit 0
 fi
@@ -47,28 +47,26 @@ then
 elif [ -f "$__object/parameter/entry" ]
 then
     acl_should="$( cat "$__object/parameter/entry" )"
-elif [ -f "$__object/parameter/acl" ]
-then
-    acl_should="$( cat "$__object/parameter/acl" )"
-elif
-    [ -f "$__object/parameter/user" ] \
-        || [ -f "$__object/parameter/group" ] \
-        || [ -f "$__object/parameter/mask" ] \
-        || [ -f "$__object/parameter/other" ]
-then
-    acl_should="$( for param in user group mask other
-    do
-        [ ! -f "$__object/parameter/$param" ] && continue
-
-        echo "$param" | grep -Eq 'mask|other' && sep=:: || sep=:
-
-        echo "$param$sep$( cat "$__object/parameter/$param" )"
-    done )"
 else
     echo 'no parameters set' >&2
     exit 1
 fi
 
+# instead of setfacl's non-helpful message "Option -m: Invalid argument near character X"
+# let's check if target has necessary users and groups, since mistyped or missing
+# users/groups in target is most common reason.
+echo "$acl_should" \
+    | grep -Po '(user|group):[^:]+' \
+    | sort -u \
+    | while read -r l
+    do
+        if ! grep "$l" -Fxq "$__object/explorer/getent"
+        then
+            echo "no $l' in target" | sed "s/:/ '/" >&2
+            exit 1
+        fi
+    done
+
 if [ -f "$__object/parameter/default" ]
 then
     acl_should="$( echo "$acl_should" \
diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst
index 28412871..307be72b 100644
--- a/cdist/conf/type/__acl/man.rst
+++ b/cdist/conf/type/__acl/man.rst
@@ -12,11 +12,14 @@ Fully supported and tested on Linux (ext4 filesystem), partial support for FreeB
 
 See ``setfacl`` and ``acl`` manpages for more details.
 
+One of ``--entry`` or ``--source`` must be used.
 
-REQUIRED MULTIPLE PARAMETERS
+
+OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
 entry
    Set ACL entry following ``getfacl`` output syntax.
+   Must be used if ``--source`` is not used.
 
 
 OPTIONAL PARAMETERS
@@ -25,6 +28,7 @@ source
    Read ACL entries from stdin or file.
    Ordering of entries is not important.
    When reading from file, comments and empty lines are ignored.
+   Must be used if ``--entry`` is not used.
 
 file
    Create/change file with ``__file`` using ``user:group:mode`` pattern.
@@ -48,12 +52,6 @@ remove
    ``mask`` and ``other`` entries can't be removed, but only changed.
 
 
-DEPRECATED PARAMETERS
----------------------
-Parameters ``acl``, ``user``, ``group``, ``mask`` and ``other`` are deprecated and they
-will be removed in future versions. Please use ``entry`` parameter instead.
-
-
 EXAMPLES
 --------
 
diff --git a/cdist/conf/type/__acl/parameter/deprecated/acl b/cdist/conf/type/__acl/parameter/deprecated/acl
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/acl
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/group b/cdist/conf/type/__acl/parameter/deprecated/group
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/group
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/mask b/cdist/conf/type/__acl/parameter/deprecated/mask
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/mask
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/other b/cdist/conf/type/__acl/parameter/deprecated/other
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/other
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/deprecated/user b/cdist/conf/type/__acl/parameter/deprecated/user
deleted file mode 100644
index 94e14159..00000000
--- a/cdist/conf/type/__acl/parameter/deprecated/user
+++ /dev/null
@@ -1 +0,0 @@
-see manual for details
diff --git a/cdist/conf/type/__acl/parameter/optional b/cdist/conf/type/__acl/parameter/optional
index cdcbc0b8..5a0c29a3 100644
--- a/cdist/conf/type/__acl/parameter/optional
+++ b/cdist/conf/type/__acl/parameter/optional
@@ -1,5 +1,3 @@
-mask
-other
 source
 file
 directory
diff --git a/cdist/conf/type/__acl/parameter/optional_multiple b/cdist/conf/type/__acl/parameter/optional_multiple
index c615d507..4c884f03 100644
--- a/cdist/conf/type/__acl/parameter/optional_multiple
+++ b/cdist/conf/type/__acl/parameter/optional_multiple
@@ -1,4 +1 @@
 entry
-acl
-user
-group

From 716cd37281903aafd7d65df539265a6722e809b1 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 21 Sep 2020 11:18:39 +0300
Subject: [PATCH 068/366] [__update_alternatives] rewrite and support --install

---
 .../explorer/alternatives                     |  4 ++
 .../type/__update_alternatives/explorer/link  | 40 +++++++++++++++++++
 .../__update_alternatives/explorer/path_is    | 12 ++++++
 .../explorer/path_should_state                |  8 ++++
 .../type/__update_alternatives/explorer/state |  8 ----
 .../type/__update_alternatives/gencode-remote | 37 ++++++++++++++---
 cdist/conf/type/__update_alternatives/man.rst | 13 ++++--
 .../__update_alternatives/parameter/boolean   |  1 +
 8 files changed, 107 insertions(+), 16 deletions(-)
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/alternatives
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/link
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/path_is
 create mode 100755 cdist/conf/type/__update_alternatives/explorer/path_should_state
 delete mode 100755 cdist/conf/type/__update_alternatives/explorer/state
 create mode 100644 cdist/conf/type/__update_alternatives/parameter/boolean

diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives
new file mode 100755
index 00000000..34aaca56
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/alternatives
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+update-alternatives --display "$__object_id" 2>/dev/null \
+    | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link
new file mode 100755
index 00000000..6519e7c2
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/link
@@ -0,0 +1,40 @@
+#!/bin/sh -e
+
+# fedora's (update-)alternatives --display output doesn't have
+# "link <name> is <path>" line, but debian does. so, let's find
+# out how they store this information.
+#
+# debian and friends:
+#   https://salsa.debian.org/dpkg-team/dpkg/-/blob/master/utils/update-alternatives.c
+#   see calls to altdb_print_line function
+#
+# fedora and friends:
+#   https://github.com/fedora-sysv/chkconfig/blob/master/alternatives.c
+#   see calls to parseLine function
+#
+# conclusion: it is safe to assume that (master) link is on second line
+
+for altdir in \
+    /var/lib/dpkg/alternatives \
+    /var/lib/alternatives
+do
+    if [ ! -f "$altdir/$__object_id" ]
+    then
+        continue
+    fi
+
+    link="$( awk 'NR==2' "$altdir/$__object_id" )"
+
+    if [ -n "$link" ]
+    then
+        break
+    fi
+done
+
+if [ -z "$link" ]
+then
+    echo "unable to get link for $__object_id" >&2
+    exit 1
+fi
+
+echo "$link"
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
new file mode 100755
index 00000000..fc304d5d
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -0,0 +1,12 @@
+#!/bin/sh -e
+
+path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \
+    | awk '/link currently points to/ {print $5}' )"
+
+if [ -z "$path_is" ]
+then
+    echo "unable to get current path for $__object_id" >&2
+    exit 1
+fi
+
+echo "$path_is"
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state
new file mode 100755
index 00000000..59e015c5
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+if [ -f "$( cat "$__object/parameter/path" )" ]
+then
+    echo 'present'
+else
+    echo 'absent'
+fi
diff --git a/cdist/conf/type/__update_alternatives/explorer/state b/cdist/conf/type/__update_alternatives/explorer/state
deleted file mode 100755
index 04a78aaa..00000000
--- a/cdist/conf/type/__update_alternatives/explorer/state
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh -e
-path="$(cat "$__object/parameter/path")"
-name="$__object_id"
-link="$(readlink "/etc/alternatives/$name")"
-if [ "$path" = "$link" ]
-then echo present
-else echo absent
-fi
diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index c0b49814..e393cdef 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Ander Punnar (ander@kvlt.ee)
 #
 # This file is part of cdist.
 #
@@ -16,12 +17,38 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
-if [ "$(cat "$__object/explorer/state")" = 'present' ]
-then exit 0
+path_is="$( cat "$__object/explorer/path_is" )"
+
+path_should="$( cat "$__object/parameter/path" )"
+
+if [ "$path_is" = "$path_should" ]
+then
+    exit 0
+fi
+
+if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ]
+then
+    echo "$path_should does not exist in target" >&2
+    exit 1
 fi
 
-path="$(cat "$__object/parameter/path")"
 name="$__object_id"
-echo "update-alternatives --quiet --set '$name' '$path'"
+
+alternatives="$( cat "$__object/explorer/alternatives" )"
+
+if ! echo "$alternatives" | grep -Fxq "$path_should"
+then
+    if [ ! -f "$__object/parameter/install" ]
+    then
+        echo "$path_should is not in $name alternatives." >&2
+        echo 'Please install missing packages or use --install to add path to alternatives.' >&2
+        exit 1
+    fi
+
+    link="$( cat "$__object/explorer/link" )"
+
+    echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
+fi
+
+echo "update-alternatives --set '$name' '$path_should'"
diff --git a/cdist/conf/type/__update_alternatives/man.rst b/cdist/conf/type/__update_alternatives/man.rst
index 73d82d11..0dc973f2 100644
--- a/cdist/conf/type/__update_alternatives/man.rst
+++ b/cdist/conf/type/__update_alternatives/man.rst
@@ -19,6 +19,12 @@ path
    Use this path for the given alternative
 
 
+BOOLEAN PARAMETERS
+------------------
+install
+   Add (``update-alternatives --install``) missing path to alternatives.
+
+
 EXAMPLES
 --------
 
@@ -36,11 +42,12 @@ SEE ALSO
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Ander Punnar <ander@kvlt.ee>
 
 
 COPYING
 -------
-Copyright \(C) 2013 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
+Copyright \(C) 2013 Nico Schottelius and 2020 Ander Punnar. You can
+redistribute it and/or modify it under the terms of the GNU General Public
+License as published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__update_alternatives/parameter/boolean b/cdist/conf/type/__update_alternatives/parameter/boolean
new file mode 100644
index 00000000..7c32f559
--- /dev/null
+++ b/cdist/conf/type/__update_alternatives/parameter/boolean
@@ -0,0 +1 @@
+install

From 687c1d2dd9bed4130a65885d19ce31c87c9d79de Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 19 Oct 2020 06:57:00 +0200
Subject: [PATCH 069/366] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 8ad2c624..a0f1ead2 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,8 @@ next:
 	* Type __service: Fix calling __systemd_service (Mark Verboom)
 	* Type __line: Add 'replace' state (Evil Ham)
 	* Type __download: Fix man page (Matthias Stecher)
+	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
+	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 367da4b77e79b82fe84c2ce1a7f75cfad92db81e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 28 Oct 2020 18:18:24 +0100
Subject: [PATCH 070/366] [type/__file] Fix --state pre-exists

---
 cdist/conf/type/__file/gencode-remote | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index 35356b13..2675b03a 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -87,11 +87,6 @@ case "$state_should" in
         fi
     ;;
 
-    pre-exists)
-        # pre-exists should never reach gencode-remote…
-        exit 1
-   ;;
-
     absent)
         if [ "$type" = "file" ]; then
             echo "rm -f '$destination'"

From 9277e0ba19c346189658712acf17a772967dfa32 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 29 Oct 2020 09:30:58 +0100
Subject: [PATCH 071/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index a0f1ead2..5977d80e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,7 @@ next:
 	* Type __download: Fix man page (Matthias Stecher)
 	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
+	* Type __file: Fix state pre-exists (Dennis Camera)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 82a9aa79020ff2369b1f4d530a3a6d5ca41915f8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 29 Oct 2020 10:39:42 +0100
Subject: [PATCH 072/366] [type/__apt_norecommends] Use 00InstallRecommends
 file as debian-installer does

debian-installer can be preseeded with `base-installer/install-recommends` to
disable installation of recommended packages already during OS installation.
d-i will then create the file `/etc/apt/apt.conf.d/00InstallRecommends`
(cf. https://salsa.debian.org/installer-team/base-installer/-/blob/master/library.sh).

__apt_norecommends should use the same file to avoid having two config files
effectively doing the same thing.
---
 cdist/conf/type/__apt_norecommends/man.rst  |  9 +++--
 cdist/conf/type/__apt_norecommends/manifest | 41 +++++++++++----------
 2 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/cdist/conf/type/__apt_norecommends/man.rst b/cdist/conf/type/__apt_norecommends/man.rst
index 001fffe4..9297b518 100644
--- a/cdist/conf/type/__apt_norecommends/man.rst
+++ b/cdist/conf/type/__apt_norecommends/man.rst
@@ -32,11 +32,12 @@ EXAMPLES
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
+Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Copyright \(C) 2014 Steven Armstrong. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2014 Steven Armstrong, 2020 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__apt_norecommends/manifest b/cdist/conf/type/__apt_norecommends/manifest
index e737df89..fc187784 100755
--- a/cdist/conf/type/__apt_norecommends/manifest
+++ b/cdist/conf/type/__apt_norecommends/manifest
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,26 +20,28 @@
 #
 
 
-os=$(cat "$__global/explorer/os")
+os=$(cat "${__global:?}/explorer/os")
 
-case "$os" in
-   ubuntu|debian|devuan)
-      # No stinking recommends thank you very much.
-      # If I want something installed I will do so myself.
-      __file /etc/apt/apt.conf.d/99-no-recommends \
-         --owner root --group root --mode 644 \
-         --source - << DONE
-APT::Install-Recommends "0";
-APT::Install-Suggests "0";
-APT::AutoRemove::RecommendsImportant "0";
-APT::AutoRemove::SuggestsImportant "0";
-DONE
-   ;;
-   *)
-      cat >&2 << DONE
+case ${os}
+in
+	(ubuntu|debian|devuan)
+		__file /etc/apt/apt.conf.d/00InstallRecommends --state present \
+			--owner root --group root --mode 0644 --source - <<-'EOF'
+		APT::Install-Recommends "false";
+		APT::Install-Suggests "false";
+		APT::AutoRemove::RecommendsImportant "false";
+		APT::AutoRemove::SuggestsImportant "false";
+		EOF
+
+		# TODO: Remove the following object after some time
+		require=__file/etc/apt/apt.conf.d/00InstallRecommends \
+		__file /etc/apt/apt.conf.d/99-no-recommends --state absent
+		;;
+	(*)
+		cat >&2 <<EOF
 The developer of this type (${__type##*/}) did not think your operating system
 ($os) would have any use for it. If you think otherwise please submit a patch.
-DONE
-      exit 1
-   ;;
+EOF
+		exit 1
+		;;
 esac

From b9ad22595fdb75c17e88e33dc028c33dbd5a7de9 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 18:03:27 +0100
Subject: [PATCH 073/366] [scanner] begin scanner implementation - non invasive

---
 cdist/scan.py | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 cdist/scan.py

diff --git a/cdist/scan.py b/cdist/scan.py
new file mode 100644
index 00000000..fa1bf5de
--- /dev/null
+++ b/cdist/scan.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+from scapy.all import *
+from scapy.data import ETHER_TYPES
+
+
+class Scanner(object):
+    def recv_msg_cpu(self, pkg):
+#        print(pkg.__repr__())
+        if ICMPv6EchoReply in pkg:
+            host = pkg['IPv6'].src
+            print(f"Host {host} is alive")
+
+
+    def scan(self):
+        sniff(iface="wlan0",
+              filter="icmp6",
+              prn=self.recv_msg_cpu)
+
+
+if __name__ == '__main__':
+    s = Scanner()
+    s.scan()

From 87b46a622411362f441b62199972e71d85d7d2d6 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 18:49:20 +0100
Subject: [PATCH 074/366] [scanner] finish prototype

ping @poljakowski - it's your turn now
---
 cdist/scan.py | 138 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 131 insertions(+), 7 deletions(-)

diff --git a/cdist/scan.py b/cdist/scan.py
index fa1bf5de..e2100499 100644
--- a/cdist/scan.py
+++ b/cdist/scan.py
@@ -19,24 +19,148 @@
 #
 #
 
-from scapy.all import *
-from scapy.data import ETHER_TYPES
+#
+# Interface to be implemented:
+# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated
+#   scan: scan / listen for icmp6 replies
+#   trigger: send trigger to multicast
+#   config: configure newly detected hosts
+#   install: install newly detected hosts
+#
+# Scanner logic
+#  - save results to configdir:
+#     basedir = ~/.cdist/scan/<ipv6-address>
+#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time or similar
+#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#
+#
+#
+#
+# cdist scan --list
+#       Show all known hosts including last seen flag
+#
+# Logic for reconfiguration:
+#
+#  - record when configured last time
+#  - introduce a parameter --reconfigure-after that takes time argument
+#  - reconfigure if a) host alive and b) reconfigure-after time passed
+#
 
 
+from multiprocessing import Process
+import os
+
+# FIXME: fail gracefully if non existent - i.e. "scapy required for scanner - please install python3-scapy"
+from scapy.all import *
+
+# Datetime overwrites scapy.all.datetime - needs to be imported AFTER
+import datetime
+
+class Trigger(object):
+    """
+    Trigger an ICMPv6EchoReply from all hosts that are alive
+    """
+
+    def __init__(self, interfaces=None, verbose=False):
+        self.interfaces = interfaces
+        self.verbose = verbose
+
+        # Wait 5 seconds before triggering again - FIXME: add parameter
+        self.sleeptime = 5
+
+    def start(self):
+        self.processes = []
+        for interface in self.interfaces:
+            p = Process(target=self.run_interface, args=(interface,))
+            self.processes.append(p)
+            p.start()
+
+    def join(self):
+        for process in self.processes:
+            process.join()
+
+    def run_interface(self, interface):
+        while True:
+            self.trigger(interface)
+            time.sleep(self.sleeptime)
+
+    def trigger(self, interface):
+        packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
+        send(packet, verbose=self.verbose)
+
 class Scanner(object):
-    def recv_msg_cpu(self, pkg):
-#        print(pkg.__repr__())
+    """
+    Scan for replies of hosts, maintain the up-to-date database
+    """
+
+    def __init__(self, interfaces=None, outdir=None):
+        self.interfaces = interfaces
+
+        if outdir:
+            self.outdir = outdir
+        else:
+            self.outdir = "."
+
+    def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
             host = pkg['IPv6'].src
             print(f"Host {host} is alive")
 
+            dir = os.path.join(self.outdir, host)
+            fname = os.path.join(dir, "last_seen")
+
+            now = datetime.datetime.now()
+
+            os.makedirs(dir, exist_ok=True)
+
+            # FIXME: maybe adjust the format so we can easily parse again
+            with open(fname, "w") as fd:
+                fd.write(f"{now}\n")
+
 
     def scan(self):
-        sniff(iface="wlan0",
+        sniff(iface=self.interfaces,
               filter="icmp6",
-              prn=self.recv_msg_cpu)
+              prn=self.handle_pkg)
 
 
 if __name__ == '__main__':
-    s = Scanner()
+    t = Trigger(interfaces=["wlan0"])
+    t.start()
+
+    # Scanner can listen on many interfaces at the same time
+    s = Scanner(interfaces=["wlan0"])
     s.scan()
+
+    # Join back the trigger processes
+    t.join()
+
+    # Test in my lan shows:
+    # [18:48] bridge:cdist% ls -1d fe80::*
+    # fe80::142d:f0a5:725b:1103
+    # fe80::20d:b9ff:fe49:ac11
+    # fe80::20d:b9ff:fe4c:547d
+    # fe80::219:d2ff:feb2:2e12
+    # fe80::21b:fcff:feee:f446
+    # fe80::21b:fcff:feee:f45c
+    # fe80::21b:fcff:feee:f4b1
+    # fe80::21b:fcff:feee:f4ba
+    # fe80::21b:fcff:feee:f4bc
+    # fe80::21b:fcff:feee:f4c1
+    # fe80::21d:72ff:fe86:46b
+    # fe80::42b0:34ff:fe6f:f6f0
+    # fe80::42b0:34ff:fe6f:f863
+    # fe80::42b0:34ff:fe6f:f9b2
+    # fe80::4a5d:60ff:fea1:e55f
+    # fe80::77a3:5e3f:82cc:f2e5
+    # fe80::9e93:4eff:fe6c:c1f4
+    # fe80::ba69:f4ff:fec5:6041
+    # fe80::ba69:f4ff:fec5:8db7
+    # fe80::bad8:12ff:fe65:313d
+    # fe80::bad8:12ff:fe65:d9b1
+    # fe80::ce2d:e0ff:fed4:2611
+    # fe80::ce32:e5ff:fe79:7ea7
+    # fe80::d66d:6dff:fe33:e00
+    # fe80::e2ff:f7ff:fe00:20e6
+    # fe80::f29f:c2ff:fe7c:275e

From 91d99bf08accb3bde0da420294fc9fb68d3d4068 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 21:22:36 +0100
Subject: [PATCH 075/366] [RFC] scanner documentation

---
 docs/dev/logs/2020-10-29.org | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 docs/dev/logs/2020-10-29.org

diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org
new file mode 100644
index 00000000..718fd68c
--- /dev/null
+++ b/docs/dev/logs/2020-10-29.org
@@ -0,0 +1,34 @@
+* The scanner, 2020-10-29, Hacking Villa Diesbach
+** Motivation
+   - The purpose of cdist is to ensure systems are in a configured state
+   - If systems reboot into a clean (think: netboot) state they are
+     stuck in an unconfigured mode
+   - We can either trigger *from* those machines
+     - this is what cdist trigger is for
+   - Or we can regulary *scan* for machines
+     - This method does not need any modification to standard OS
+** How it works
+   - cdist scan uses the all nodes multicast group ff02::1
+   - It sends a ping packet there in regular intervals
+   - This even works in non-IPv6 networks, as all operating systems
+     are IPv6 capable and usually IPv6 enabled by default
+     - Link local is always accessible!
+   - cdist scan receives an answer from all alive hosts
+     - These results are stored in ~/.cdist/scan/${hostip}
+     - We record the last_seen date  ~/.cdist/scan/${hostip}/last_seen
+   - After a host is detected, cdist *can* try to configure it
+     - It saves the result (+/- logging needs to be defined) in
+       ~/.cdist/scan/${hostip}/{config, install}_result
+     - If logging is saved: maybe in ~/.cdist/scan/${hostip}/{config, install}_log
+     - Final naming TBD
+** Benefits from the scanning approach
+   - We know when a host is alive/dead
+   - We can use standard OS w/o trigger customisation
+     - Only requirement: we can ssh into it
+     - Can make use f.i. of Alpine Linux w/ ssh keys feeding in
+   - We can trigger regular reconfiguration
+     - If alive && last_config_time > 1d -> reconfigure
+   - Data can be exported to f.i. prometheus
+     - Record when configured (successfully)
+     - Record when seen
+   - Enables configurations in stateless environments

From 09dfcfe81e0d9e6520d9bf98598037f86c84641e Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Thu, 29 Oct 2020 23:16:08 +0100
Subject: [PATCH 076/366] [scanner] add to beta commands

---
 cdist/argparse.py            | 32 ++++++++++++++++++++-
 cdist/scan/commandline.py    | 55 ++++++++++++++++++++++++++++++++++++
 cdist/{ => scan}/scan.py     | 20 +++++++++----
 docs/dev/logs/2020-10-29.org | 23 +++++++++++++++
 4 files changed, 124 insertions(+), 6 deletions(-)
 create mode 100644 cdist/scan/commandline.py
 rename cdist/{ => scan}/scan.py (90%)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 1d16bb25..ff195e8c 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -8,10 +8,11 @@ import cdist.configuration
 import cdist.log
 import cdist.preos
 import cdist.info
+import cdist.scan.commandline
 
 
 # set of beta sub-commands
-BETA_COMMANDS = set(('install', 'inventory', ))
+BETA_COMMANDS = set(('install', 'inventory', 'scan', ))
 # set of beta arguments for sub-commands
 BETA_ARGS = {
     'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
@@ -470,6 +471,35 @@ def get_parsers():
             'pattern', nargs='?', help='Glob pattern.')
     parser['info'].set_defaults(func=cdist.info.Info.commandline)
 
+    # Scan = config + further
+    parser['scan'] = parser['sub'].add_parser('scan', add_help=False,
+                                                 parents=[parser['config']])
+
+    parser['scan'] = parser['sub'].add_parser(
+            'scan', parents=[parser['loglevel'],
+                             parser['beta'],
+                             parser['colored_output'],
+                             parser['common'],
+                             parser['config_main']])
+
+    parser['scan'].add_argument(
+        '-m', '--mode', help='Which modes should run',
+        action='append', default=[],
+        choices=['scan', 'trigger'])
+    parser['scan'].add_argument(
+        '--config',
+        action='store_true',
+        help='Try to configure detected hosts')
+    parser['scan'].add_argument(
+        '-I', '--interfaces',
+        action='append',  default=[],
+        help='On which interfaces to scan/trigger')
+    parser['scan'].add_argument(
+        '-d', '--delay',
+        action='store',  default=3600,
+        help='How long to wait before reconfiguring after last try')
+    parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
+
     for p in parser:
         parser[p].epilog = EPILOG
 
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
new file mode 100644
index 00000000..0fb718e5
--- /dev/null
+++ b/cdist/scan/commandline.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import logging
+
+log = logging.getLogger("scan")
+
+
+# define this outside of the class to not handle scapy import errors by default
+def commandline(args):
+    log.debug(args)
+
+    try:
+        import cdist.scan.scan as scan
+    except ModuleNotFoundError:
+        print('cdist scan requires scapy to be installed')
+
+    processes = []
+
+    if not args.mode:
+        # By default scan and trigger, but do not call any action
+        args.mode = ['scan', 'trigger' ]
+
+    if 'trigger' in args.mode:
+        t = scan.Trigger(interfaces=args.interfaces)
+        t.start()
+        processes.append(t)
+        log.debug("Trigger started")
+
+    if 'scan' in args.mode:
+        s = scan.Scanner(interfaces=args.interfaces, args=args)
+        s.start()
+        processes.append(s)
+        log.debug("Scanner started")
+
+    for process in processes:
+        process.join()
diff --git a/cdist/scan.py b/cdist/scan/scan.py
similarity index 90%
rename from cdist/scan.py
rename to cdist/scan/scan.py
index e2100499..fcbf1899 100644
--- a/cdist/scan.py
+++ b/cdist/scan/scan.py
@@ -50,13 +50,14 @@
 
 from multiprocessing import Process
 import os
-
-# FIXME: fail gracefully if non existent - i.e. "scapy required for scanner - please install python3-scapy"
+import logging
 from scapy.all import *
 
 # Datetime overwrites scapy.all.datetime - needs to be imported AFTER
 import datetime
 
+log = logging.getLogger("scan")
+
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
@@ -87,6 +88,7 @@ class Trigger(object):
 
     def trigger(self, interface):
         packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
+        log.debug(f"Sending request on {interface}")
         send(packet, verbose=self.verbose)
 
 class Scanner(object):
@@ -94,18 +96,18 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces=None, outdir=None):
+    def __init__(self, interfaces=None, args=None, outdir=None):
         self.interfaces = interfaces
 
         if outdir:
             self.outdir = outdir
         else:
-            self.outdir = "."
+            self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan')
 
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
             host = pkg['IPv6'].src
-            print(f"Host {host} is alive")
+            log.verbose(f"Host {host} is alive")
 
             dir = os.path.join(self.outdir, host)
             fname = os.path.join(dir, "last_seen")
@@ -118,13 +120,21 @@ class Scanner(object):
             with open(fname, "w") as fd:
                 fd.write(f"{now}\n")
 
+    def start(self):
+        self.process = Process(target=self.scan)
+        self.process.start()
+
+    def join(self):
+        self.process.join()
 
     def scan(self):
+        log.debug("Scanning - zzzzz")
         sniff(iface=self.interfaces,
               filter="icmp6",
               prn=self.handle_pkg)
 
 
+
 if __name__ == '__main__':
     t = Trigger(interfaces=["wlan0"])
     t.start()
diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org
index 718fd68c..4461be8c 100644
--- a/docs/dev/logs/2020-10-29.org
+++ b/docs/dev/logs/2020-10-29.org
@@ -32,3 +32,26 @@
      - Record when configured (successfully)
      - Record when seen
    - Enables configurations in stateless environments
+** Sample output v2020-10-29
+23:14] bridge:~% sudo  cdist scan -b -I wlan0 -vv
+VERBOSE: cdist: version 6.8.0-36-g91d99bf0
+VERBOSE: scan: Host fe80::21d:72ff:fe86:46b is alive
+VERBOSE: scan: Host fe80::ce2d:e0ff:fed4:2611 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4c1 is alive
+VERBOSE: scan: Host fe80::e2ff:f7ff:fe00:20e6 is alive
+VERBOSE: scan: Host fe80::20d:b9ff:fe49:ac11 is alive
+VERBOSE: scan: Host fe80::9e93:4eff:fe6c:c1f4 is alive
+VERBOSE: scan: Host fe80::ce32:e5ff:fe79:7ea7 is alive
+VERBOSE: scan: Host fe80::219:d2ff:feb2:2e12 is alive
+VERBOSE: scan: Host fe80::d66d:6dff:fe33:e00 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f446 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4b1 is alive
+VERBOSE: scan: Host fe80::20d:b9ff:fe4c:547d is alive
+VERBOSE: scan: Host fe80::bad8:12ff:fe65:313d is alive
+VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f6f0 is alive
+VERBOSE: scan: Host fe80::ba69:f4ff:fec5:6041 is alive
+VERBOSE: scan: Host fe80::f29f:c2ff:fe7c:275e is alive
+VERBOSE: scan: Host fe80::ba69:f4ff:fec5:8db7 is alive
+VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f863 is alive
+VERBOSE: scan: Host fe80::21b:fcff:feee:f4bc is alive
+...

From e30ecdda535200ac75a6e774590d2e1b5e1b5b28 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 13:05:40 +0200
Subject: [PATCH 077/366] Add __uci and __uci_commit types

---
 cdist/conf/type/__uci/explorer/state          | 89 +++++++++++++++++++
 cdist/conf/type/__uci/gencode-remote          | 67 ++++++++++++++
 cdist/conf/type/__uci/man.rst                 | 77 ++++++++++++++++
 cdist/conf/type/__uci/manifest                | 40 +++++++++
 cdist/conf/type/__uci/parameter/default/state |  1 +
 .../type/__uci/parameter/default/transaction  |  1 +
 cdist/conf/type/__uci/parameter/optional      |  2 +
 .../type/__uci/parameter/required_multiple    |  1 +
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++
 cdist/conf/type/__uci_commit/man.rst          | 58 ++++++++++++
 cdist/conf/type/__uci_commit/nonparallel      |  0
 11 files changed, 357 insertions(+)
 create mode 100644 cdist/conf/type/__uci/explorer/state
 create mode 100755 cdist/conf/type/__uci/gencode-remote
 create mode 100644 cdist/conf/type/__uci/man.rst
 create mode 100755 cdist/conf/type/__uci/manifest
 create mode 100644 cdist/conf/type/__uci/parameter/default/state
 create mode 100644 cdist/conf/type/__uci/parameter/default/transaction
 create mode 100644 cdist/conf/type/__uci/parameter/optional
 create mode 100644 cdist/conf/type/__uci/parameter/required_multiple
 create mode 100755 cdist/conf/type/__uci_commit/gencode-remote
 create mode 100644 cdist/conf/type/__uci_commit/man.rst
 create mode 100644 cdist/conf/type/__uci_commit/nonparallel

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
new file mode 100644
index 00000000..2e98b606
--- /dev/null
+++ b/cdist/conf/type/__uci/explorer/state
@@ -0,0 +1,89 @@
+#!/bin/sh
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current state of the configuration option
+# The output of this explorer is one of these values:
+#   present
+#     The configuration option is present and has the value of the
+#     parameter --value.
+#   absent
+#     The configuration option is not defined.
+#   different
+#     The configuration option is present but has a different value than the
+#     parameter --value.
+#   rearranged
+#     The configuration option is present (a list) and has the same values as
+#     the parameter --value, but in a different order.
+
+RS=$(printf '\036')
+
+option=${__object_id:?}
+
+values_is=$(uci -s -N -d "${RS}" get "${option}" 2>/dev/null) || {
+	echo absent
+	exit 0
+}
+
+# strip off trailing newline
+printf '%s' "${values_is}" \
+| awk '
+BEGIN {
+	state = "present"  # assume all is fine
+}
+NR == FNR {
+	# memoize "should" state
+	should[FNR] = $0
+
+	# go to next line (important!)
+	next
+}
+
+# compare "is" state
+$0 == should[FNR] { next }
+
+FNR > length(should) {
+	# there are more "is" records than "should" -> definitely different
+	state = "different"
+	exit
+}
+
+{
+	# see if we can find the value somewhere in should
+	for (i in should) {
+		if ($0 == should[i]) {
+			# ... value found -> rearranged
+			# FIXME: Duplicate values are not properly handled here. Do they matter?
+			state = "rearranged"
+			next
+		}
+	}
+
+	state = "different"
+	exit
+}
+
+END {
+	if (FNR < length(should)) {
+		# "is" was shorter than "should" -> different
+		state = "different"
+	}
+
+	print state
+}
+' "${__object:?}/parameter/value" RS="${RS}" -
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
new file mode 100755
index 00000000..48f114fe
--- /dev/null
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -0,0 +1,67 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
+
+config=${__object_id:?}
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+case ${state_should}
+in
+	(present)
+		if in_list "${state_is}" 'present' 'rearranged'
+		then
+			# NOTE: order is ignored so rearranged is also fine.
+			exit 0
+		fi
+
+		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
+		then
+			# "should" is a list
+			if test "${state_is}" != 'absent'
+			then
+				printf "uci delete '%s'\n" "${config}"
+			fi
+
+			while read -r value
+			do
+				printf "uci add_list '%s'='%s'\n" "${config}" "${value}"
+			done <"${__object:?}/parameter/value"
+		else
+			# "should" is a scalar
+			value=$(cat "${__object:?}/parameter/value")
+			printf "uci set '%s'='%s'\n" "${config}" "${value}"
+		fi
+		;;
+	(absent)
+		if in_list "${state_is}" 'absent'
+		then
+			exit 0
+		fi
+
+		printf "uci delete '%s'\n" "${config}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
new file mode 100644
index 00000000..d23d8b2b
--- /dev/null
+++ b/cdist/conf/type/__uci/man.rst
@@ -0,0 +1,77 @@
+cdist-type__uci(7)
+==================
+
+NAME
+----
+cdist-type__uci - Manage configuration values in OpenWrt's
+Unified Configuration Interface (UCI)
+
+
+DESCRIPTION
+-----------
+This cdist type can be used to alter configuration options in OpenWrt's UCI
+system.
+
+Options can be applied in batches if the `--transaction` parameter is used.
+It is important to ensure that the `__uci_commit` object is executed before a
+new transaction is started.
+
+REQUIRED PARAMETERS
+-------------------
+value
+    The value to be set. Can be used multiple times.
+
+    Due to the way cdist handles arguments, values **must not** contain newline
+    characters.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    `present` or `absent`, defaults to `present`.
+transaction
+    The name of the transaction this option belongs to.
+    If none is given: "default" is used.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Set the system hostname
+    __uci system.@system[0].hostname --value 'OpenWrt'
+
+    # Enable NTP and NTPd (in one transaction)
+    __uci system.ntp.enabled --value 1 --transaction ntp
+    __uci system.ntp.enable_server --value 1 --transaction ntp
+    __uci system.ntp.server --transaction ntp \
+        --value '0.openwrt.pool.ntp.org' \
+        --value '1.openwrt.pool.ntp.org' \
+        --value '2.openwrt.pool.ntp.org' \
+        --value '3.openwrt.pool.ntp.org'
+    export require=__uci_commit/ntp
+
+
+SEE ALSO
+--------
+- https://openwrt.org/docs/guide-user/base-system/uci
+- :strong:`cdist-type__uci_commit`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
new file mode 100755
index 00000000..e5b0fb30
--- /dev/null
+++ b/cdist/conf/type/__uci/manifest
@@ -0,0 +1,40 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+os=$(cat "${__global:?}/explorer/os")
+
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+case ${os}
+in
+	(openwrt)
+		# okay
+		;;
+	(*)
+		printf "Your operating system (%s) is currently not supported by this type (%s)\n" "${os}" "${__type##*/}" >&2
+		printf "Please contribute an implementation for it if you can.\n" >&2
+		exit 1
+		;;
+esac
+
+
+# Make sure the changes are being commited
+require=${__object_name:?} __uci_commit "${transaction_name}"
diff --git a/cdist/conf/type/__uci/parameter/default/state b/cdist/conf/type/__uci/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__uci/parameter/default/transaction b/cdist/conf/type/__uci/parameter/default/transaction
new file mode 100644
index 00000000..4ad96d51
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/default/transaction
@@ -0,0 +1 @@
+default
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
new file mode 100644
index 00000000..ddbbba16
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -0,0 +1,2 @@
+state
+transaction
diff --git a/cdist/conf/type/__uci/parameter/required_multiple b/cdist/conf/type/__uci/parameter/required_multiple
new file mode 100644
index 00000000..6d4e1507
--- /dev/null
+++ b/cdist/conf/type/__uci/parameter/required_multiple
@@ -0,0 +1 @@
+value
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
new file mode 100755
index 00000000..bed0eefb
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -0,0 +1,21 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+echo 'uci commit'
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
new file mode 100644
index 00000000..c55d06e5
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/man.rst
@@ -0,0 +1,58 @@
+cdist-type__uci_commit(7)
+=========================
+
+NAME
+----
+cdist-type__uci_commit - Commit a UCI transaction.
+
+
+DESCRIPTION
+-----------
+This type executes the "uci commit" command on the target.
+It is usually not required to use this type. Use the `--transaction` parameter
+of `cdist-type__uci`\ (7) instead.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Commit the default transaction
+    __uci_commit default
+
+    # Commit another transaction
+    __uci_commit my_transaction
+
+
+SEE ALSO
+--------
+:strong:`cdist-type__uci`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_commit/nonparallel b/cdist/conf/type/__uci_commit/nonparallel
new file mode 100644
index 00000000..e69de29b

From 55e7b32449c9a86e34054c321b20a49795c6652f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 14:00:29 +0200
Subject: [PATCH 078/366] [type/__uci] Only generate __uci_commit if changes
 are required

---
 cdist/conf/type/__uci/manifest | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index e5b0fb30..74524513 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,9 +18,12 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; } }
 
 os=$(cat "${__global:?}/explorer/os")
 
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case ${os}
@@ -35,6 +38,25 @@ in
 		;;
 esac
 
+changes_required=false
 
-# Make sure the changes are being commited
-require=${__object_name:?} __uci_commit "${transaction_name}"
+case ${state_should}
+in
+	(present)
+		# NOTE: order is ignored so rearranged is also fine.
+		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
+		;;
+	(absent)
+		in_list "${state_is}" 'absent' || changes_required=true
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
+
+if ${changes_required}
+then
+	# Make sure the changes are being committed
+	require=${__object_name:?} __uci_commit "${transaction_name}"
+fi

From a09120977f231fe3dda1c7c07073fc7e03fc5ea0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 3 Jun 2020 14:07:10 +0200
Subject: [PATCH 079/366] [type/__uci] Allow omission of --value parameter if
 --state absent

---
 cdist/conf/type/__uci/explorer/state                   | 10 +++++++++-
 cdist/conf/type/__uci/man.rst                          |  1 +
 cdist/conf/type/__uci/manifest                         |  5 +++++
 .../parameter/{required_multiple => optional_multiple} |  0
 4 files changed, 15 insertions(+), 1 deletion(-)
 rename cdist/conf/type/__uci/parameter/{required_multiple => optional_multiple} (100%)

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
index 2e98b606..6fb4b173 100644
--- a/cdist/conf/type/__uci/explorer/state
+++ b/cdist/conf/type/__uci/explorer/state
@@ -40,6 +40,14 @@ values_is=$(uci -s -N -d "${RS}" get "${option}" 2>/dev/null) || {
 	exit 0
 }
 
+if test -f "${__object:?}/parameter/value"
+then
+	should_file="${__object:?}/parameter/value"
+else
+	should_file='/dev/null'
+fi
+
+
 # strip off trailing newline
 printf '%s' "${values_is}" \
 | awk '
@@ -86,4 +94,4 @@ END {
 
 	print state
 }
-' "${__object:?}/parameter/value" RS="${RS}" -
+' "${should_file}" RS="${RS}" -
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index d23d8b2b..c6bb81ab 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -20,6 +20,7 @@ REQUIRED PARAMETERS
 -------------------
 value
     The value to be set. Can be used multiple times.
+    This parameter is allowed to be omitted if `--state` is `absent`.
 
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index 74524513..e56462e5 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -43,6 +43,11 @@ changes_required=false
 case ${state_should}
 in
 	(present)
+		test -s "${__object:?}/parameter/value" || {
+			echo 'The parameter --value is required.' >&2
+			exit 1
+		}
+
 		# NOTE: order is ignored so rearranged is also fine.
 		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
 		;;
diff --git a/cdist/conf/type/__uci/parameter/required_multiple b/cdist/conf/type/__uci/parameter/optional_multiple
similarity index 100%
rename from cdist/conf/type/__uci/parameter/required_multiple
rename to cdist/conf/type/__uci/parameter/optional_multiple

From d8f20a6a204142360e7fb7afd6603c69ae613e2d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 12 Jun 2020 19:39:19 +0200
Subject: [PATCH 080/366] [type/__uci] Implement "real" transactions using
 batch files

---
 .../__uci/{gencode-remote => gencode-local}   | 22 ++++++++++++++++---
 cdist/conf/type/__uci/man.rst                 |  3 +--
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++++++++++++++-
 3 files changed, 40 insertions(+), 6 deletions(-)
 rename cdist/conf/type/__uci/{gencode-remote => gencode-local} (79%)

diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-local
similarity index 79%
rename from cdist/conf/type/__uci/gencode-remote
rename to cdist/conf/type/__uci/gencode-local
index 48f114fe..bba5944d 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-local
@@ -20,11 +20,27 @@
 
 in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
 
+uci_cmd() {
+	printf 'printf "%s\n"' "$1"
+	shift
+	printf " '%s'" "$@"
+	printf " >>'%s'\n" "${tmpfile}"
+}
+
 config=${__object_id:?}
 
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+tmpdir="${__global:?}/tmp/__uci"
+# HACK
+mkdir -p "${tmpdir}"
+
+tmpfile="${tmpdir}/${transaction_name}.txt"
+
+
 case ${state_should}
 in
 	(present)
@@ -44,12 +60,12 @@ in
 
 			while read -r value
 			do
-				printf "uci add_list '%s'='%s'\n" "${config}" "${value}"
+				uci_cmd "add_list '%s'='%s'" "${config}" "${value}"
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
 			value=$(cat "${__object:?}/parameter/value")
-			printf "uci set '%s'='%s'\n" "${config}" "${value}"
+			uci_cmd "set '%s'='%s'" "${config}" "${value}"
 		fi
 		;;
 	(absent)
@@ -58,7 +74,7 @@ in
 			exit 0
 		fi
 
-		printf "uci delete '%s'\n" "${config}"
+		uci_cmd "delete '%s'" "${config}"
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index c6bb81ab..2f64355b 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -13,8 +13,7 @@ This cdist type can be used to alter configuration options in OpenWrt's UCI
 system.
 
 Options can be applied in batches if the `--transaction` parameter is used.
-It is important to ensure that the `__uci_commit` object is executed before a
-new transaction is started.
+
 
 REQUIRED PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index bed0eefb..ac74e5e4 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -18,4 +18,23 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-echo 'uci commit'
+transaction_name=${__object_id:?}
+batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
+
+test -s "${batchfile}" || exit 0
+
+cat <<'EOF'
+rollback() {
+	uci changes \
+	| sed -e 's/\..*$//' -e 's/^-//' \
+	| while read -r package
+	  do
+		  uci revert "${package}"
+	  done
+}
+
+EOF
+
+echo "uci batch <<'EOF' && uci commit || rollback"
+cat "${batchfile}"
+echo 'EOF'

From d3574b2d3e807ae436afd90aa1ed9d01b78ca6a3 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 16:43:16 +0200
Subject: [PATCH 081/366] [type/__uci] Send messages when options are set to be
 altered

---
 cdist/conf/type/__uci/gencode-local | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index bba5944d..ba27dc5e 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -53,6 +53,8 @@ in
 		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
 		then
 			# "should" is a list
+			printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+
 			if test "${state_is}" != 'absent'
 			then
 				printf "uci delete '%s'\n" "${config}"
@@ -64,6 +66,8 @@ in
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
+			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
 			value=$(cat "${__object:?}/parameter/value")
 			uci_cmd "set '%s'='%s'" "${config}" "${value}"
 		fi
@@ -74,6 +78,7 @@ in
 			exit 0
 		fi
 
+		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
 		uci_cmd "delete '%s'" "${config}"
 		;;
 	(*)

From 3a3be3631010b1bf17d084c775e306f11429f5d9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 12 Jun 2020 19:53:26 +0200
Subject: [PATCH 082/366] [type/__uci_commit] Send message on commit of a
 transaction

---
 cdist/conf/type/__uci_commit/gencode-remote | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index ac74e5e4..0332d7e0 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,6 +23,8 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
+printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
+
 cat <<'EOF'
 rollback() {
 	uci changes \

From e7369a1f9957ac31830a69de8101c434666da5da Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 17 Jun 2020 13:00:24 +0200
Subject: [PATCH 083/366] [type/__uci_commit] Abort if uncommited changes are
 present on the target

---
 cdist/conf/type/__uci_commit/explorer/changes | 22 ++++++++++++++++
 cdist/conf/type/__uci_commit/gencode-remote   | 25 +++++++++++++------
 2 files changed, 39 insertions(+), 8 deletions(-)
 create mode 100644 cdist/conf/type/__uci_commit/explorer/changes

diff --git a/cdist/conf/type/__uci_commit/explorer/changes b/cdist/conf/type/__uci_commit/explorer/changes
new file mode 100644
index 00000000..2690def7
--- /dev/null
+++ b/cdist/conf/type/__uci_commit/explorer/changes
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer outputs the uncommited UCI changes on the target.
+
+uci changes
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index 0332d7e0..dd1d839c 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,20 +23,29 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
+if test -s "${__object:?}/explorer/changes"
+then
+	echo 'Uncommited UCI changes were found on the target:'
+	cat "${__object:?}/explorer/changes"
+	echo
+	echo 'This can be caused by manual changes or due to a previous failed run.'
+	echo 'Please investigate the situation, revert or commit the changes, and try again.'
+	exit 1
+fi >&2
+
 printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
 
-cat <<'EOF'
+cat <<CODE
 rollback() {
-	uci changes \
-	| sed -e 's/\..*$//' -e 's/^-//' \
+	uci changes \\
+	| sed -e 's/\..*\$//' -e 's/^-//' \\
 	| while read -r package
 	  do
-		  uci revert "${package}"
+		  uci revert "\${package}"
 	  done
 }
 
+uci batch <<'EOF' && uci commit || rollback
+$(cat "${batchfile}")
 EOF
-
-echo "uci batch <<'EOF' && uci commit || rollback"
-cat "${batchfile}"
-echo 'EOF'
+CODE

From cc599dab15ed5c8761b3bcdfb113403fa6d4bc77 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 5 Jul 2020 10:25:36 +0200
Subject: [PATCH 084/366] [type/__uci_commit] Move uncommited changes check
 from explorer to code-remote

This is done to prevent false positives/negatives (see NOTE in code)
---
 cdist/conf/type/__uci_commit/explorer/changes | 22 -------------------
 cdist/conf/type/__uci_commit/gencode-remote   | 21 +++++++++++++-----
 2 files changed, 15 insertions(+), 28 deletions(-)
 delete mode 100644 cdist/conf/type/__uci_commit/explorer/changes

diff --git a/cdist/conf/type/__uci_commit/explorer/changes b/cdist/conf/type/__uci_commit/explorer/changes
deleted file mode 100644
index 2690def7..00000000
--- a/cdist/conf/type/__uci_commit/explorer/changes
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/sh
-#
-# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-# This explorer outputs the uncommited UCI changes on the target.
-
-uci changes
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index dd1d839c..504f7c3b 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -23,19 +23,28 @@ batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
 
 test -s "${batchfile}" || exit 0
 
-if test -s "${__object:?}/explorer/changes"
+printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
+
+# NOTE: Uncommited changes are checked in code-remote instead of in an explorer
+#       because in cdist there is no interlocking between explorers and code
+#       execution.
+#       Checking for uncommited changes in an explorer leaves a time slot in
+#       which changes made are not detected by the code.
+#       Furthermore an explorer running concurrently with another transactions
+#       code-remote could lead to a false positive.
+
+cat <<CODE
+changes=\$(uci changes)
+
+if test -n "\${changes}"
 then
 	echo 'Uncommited UCI changes were found on the target:'
-	cat "${__object:?}/explorer/changes"
-	echo
+	printf '%s\n\n' "\${changes}"
 	echo 'This can be caused by manual changes or due to a previous failed run.'
 	echo 'Please investigate the situation, revert or commit the changes, and try again.'
 	exit 1
 fi >&2
 
-printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
-
-cat <<CODE
 rollback() {
 	uci changes \\
 	| sed -e 's/\..*\$//' -e 's/^-//' \\

From 3ef638a61123069953a9e24044b6c94eba0b3b6b Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 20 Jul 2020 12:12:15 +0200
Subject: [PATCH 085/366] [type/__uci_commit] Fail when uci(1) reports errors

---
 cdist/conf/type/__uci_commit/gencode-remote | 37 ++++++++++++++++-----
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
index 504f7c3b..e5c93966 100755
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ b/cdist/conf/type/__uci_commit/gencode-remote
@@ -45,16 +45,37 @@ then
 	exit 1
 fi >&2
 
-rollback() {
-	uci changes \\
-	| sed -e 's/\..*\$//' -e 's/^-//' \\
-	| while read -r package
-	  do
-		  uci revert "\${package}"
-	  done
+check_errors() {
+	# reads stdin and forwards non-empty lines to stderr.
+	# returns 0 if stdin is empty, else 1.
+	! grep -e . >&2
 }
 
-uci batch <<'EOF' && uci commit || rollback
+commit() {
+	uci commit
+}
+
+rollback() {
+	echo >&2
+	echo 'An error occurred when trying to commit transaction ${transaction_name}!' >&2
+
+	uci changes \\
+	| sed -e 's/^-//' -e 's/\..*\$//' \\
+	| sort -u \\
+	| while read -r _package
+	  do
+		  uci revert "\${_package}"
+		  echo "\${_package}"  # for logging
+	  done \\
+	| awk '
+	  BEGIN { printf "Reverted changes in: " }
+	  { printf "%s%s", (FNR > 1 ? ", " : ""), \$0 }
+	  END { printf "\\n" }' >&2
+
+	return 1
+}
+
+uci batch <<'EOF' 2>&1 | check_errors && commit || rollback
 $(cat "${batchfile}")
 EOF
 CODE

From 4da39681188493981ec3c55ea918787195655972 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 20 Jun 2020 20:04:04 +0200
Subject: [PATCH 086/366] [type/__uci_section] Add type

---
 cdist/conf/type/__uci_section/explorer/match  |  68 ++++++++++
 .../conf/type/__uci_section/explorer/options  |  28 ++++
 cdist/conf/type/__uci_section/explorer/type   |  25 ++++
 cdist/conf/type/__uci_section/man.rst         |  76 +++++++++++
 cdist/conf/type/__uci_section/manifest        | 122 ++++++++++++++++++
 .../__uci_section/parameter/default/state     |   1 +
 .../parameter/default/transaction             |   1 +
 .../type/__uci_section/parameter/optional     |   4 +
 .../__uci_section/parameter/optional_multiple |   1 +
 9 files changed, 326 insertions(+)
 create mode 100644 cdist/conf/type/__uci_section/explorer/match
 create mode 100644 cdist/conf/type/__uci_section/explorer/options
 create mode 100644 cdist/conf/type/__uci_section/explorer/type
 create mode 100644 cdist/conf/type/__uci_section/man.rst
 create mode 100755 cdist/conf/type/__uci_section/manifest
 create mode 100644 cdist/conf/type/__uci_section/parameter/default/state
 create mode 100644 cdist/conf/type/__uci_section/parameter/default/transaction
 create mode 100644 cdist/conf/type/__uci_section/parameter/optional
 create mode 100644 cdist/conf/type/__uci_section/parameter/optional_multiple

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
new file mode 100644
index 00000000..9cf1ea3e
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -0,0 +1,68 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer the "prefix" of the section matching --match.
+
+squote_values() {
+	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
+	    -e "/='.*'$/"'!{s/=/='\''/;s/$/'\''/}'
+}
+
+RS=$(printf '\036')
+
+if ! test -e "${__object:?}/parameter/match"
+then
+	if echo "${__object_id:?}" | grep -qvE '^[^.]+\.[^.]+$'
+	then
+		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
+		exit 1
+	fi
+
+	# If no --match is given, we take the __object_id as the section identifier.
+	echo "${__object_id:?}"
+	exit 0
+fi
+
+test -s "${__object:?}/parameter/match" \
+&& test -s "${__object:?}/parameter/type" \
+|| {
+	echo 'Parameters --match and --type must be used together.' >&2
+	exit 1
+}
+
+# Find by match
+match=$(cat "${__object:?}/parameter/match")
+sect_type_filter=$(cat "${__object:?}/parameter/type")
+
+package_filter=${sect_type_filter%%.*}
+section_filter=${sect_type_filter##*.}
+regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
+
+matched_sections=$(
+	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
+	| grep -e "${regex}" \
+	| sed -e 's/\.[^.]*=.*$//')
+
+if test "$(echo "${matched_sections}" | wc -l)" -gt 1
+then
+	printf 'Found multiple matching sections:\n%s\n' "${matched_sections}" >&2
+	exit 1
+fi
+
+echo "${matched_sections}"
diff --git a/cdist/conf/type/__uci_section/explorer/options b/cdist/conf/type/__uci_section/explorer/options
new file mode 100644
index 00000000..67b4f83f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/options
@@ -0,0 +1,28 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current options of the configuration section.
+
+RS=$(printf '\036')
+
+section=$("${__type_explorer:?}/match")
+test -n "${section}" || exit 0
+
+uci -s -N -d "${RS}" show "${section}" 2>/dev/null \
+| grep -v -e "^${section}=" || true
diff --git a/cdist/conf/type/__uci_section/explorer/type b/cdist/conf/type/__uci_section/explorer/type
new file mode 100644
index 00000000..1675c2e0
--- /dev/null
+++ b/cdist/conf/type/__uci_section/explorer/type
@@ -0,0 +1,25 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer retrieves the current section type.
+
+section=$("${__type_explorer:?}/match")
+test -n "${section}" || exit 0
+
+uci -s -N get "${section}" 2>/dev/null || true
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
new file mode 100644
index 00000000..3468bfc3
--- /dev/null
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -0,0 +1,76 @@
+cdist-type__uci_section(7)
+==========================
+
+NAME
+----
+cdist-type__uci_section - Manage configuration sections in OpenWrt's
+Unified Configuration Interface (UCI)
+
+
+DESCRIPTION
+-----------
+This cdist type can be used to replace whole configuration sections in OpenWrt's
+UCI system.
+It can be thought of as syntactic sugar for `cdist-type__uci`\ (7), as this type
+will generate the required `__uci` objects to make the section contain exactly
+the options specified via ``--option``.
+
+Since many default UCI sections are unnamed, this type allows to find the
+matching section by one of its options using the ``--match`` parameter.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+match
+    Allows to find a section to "replace" through one of its parameters.
+    The value to this parameter is a ``<option>=<string>`` string.
+option
+    An option that should be present in the section.
+    This parameter can be used multiple times to specify multiple options.
+    The value to this parameter is a ``<option>=<string>`` string.
+
+    Lists can be expressed by repeatedly using the same key.
+state
+    `present` or `absent`, defaults to `present`.
+transaction
+    The name of the transaction this option belongs to.
+    The value will be forwarded to `cdist-type__uci`\ (7).
+type
+    The type of the section in the format: ``<config>.<section-type>``
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # TODO
+    __uci_section ...
+
+
+SEE ALSO
+--------
+:strong:`cdist-type__uci`\ (7)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
new file mode 100755
index 00000000..e16277ba
--- /dev/null
+++ b/cdist/conf/type/__uci_section/manifest
@@ -0,0 +1,122 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+grep_line() { echo "$2" | grep -qxF "$1"; }
+unquote_lines() {
+	sed -e '/^".*"$/{s/^"//;s/"$//}' \
+	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
+}
+
+append_values() {
+	while read -r _value
+	do
+		set -- "$@" --value "${_value}"
+	done
+	unset _value
+	"$@" </dev/null
+}
+
+section=$(cat "${__object:?}/explorer/match")
+
+state_should=$(cat "${__object:?}/parameter/state")
+transaction_name=$(cat "${__object:?}/parameter/transaction")
+
+case $state_should
+in
+	(present)
+		test -f "${__object:?}/parameter/type" || {
+			echo 'Parameter --type is required.' >&2
+			exit 1
+		}
+		type_is=$(cat "${__object:?}/explorer/type")
+		type_should=$(cat "${__object:?}/parameter/type")
+
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
+		if test -n "${type_is}"
+		then
+			if test "${type_is}" != "${type_should##*.}"
+			then
+				# Check if section type matches (section exists and --type provided)
+				printf 'Section type "%s" does not match --type "%s".\n' \
+					"${type_is}" "${type_should}" >&2
+				exit 1
+			fi
+			sect_type=${type_is}
+		else
+			sect_type=${type_should##*.}
+		fi
+
+		if test -z "${section}"
+		then
+			# No section exists and --match was used.
+			# So we generate a new section identifier from $__object_id.
+			case ${__object_id:?}
+			in
+				(*.*) section=${__object_id:?} ;;
+				(*) section="${type_should%%.*}.${__object_id:?}" ;;
+			esac
+		fi
+
+		# Make sure the section itself is present
+		__uci "${section}" --state present --transaction "${transaction_name}" \
+			--value "${sect_type}"
+		export require=__uci/"${section}"
+
+		# Delete options not in "should"
+		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		| while read -r optname
+		  do
+			  if ! grep_line "${optname}" "${optnames_should}"
+			  then
+				  __uci "${section}.${optname}" --state absent \
+					  --transaction "${transaction_name}" </dev/null
+			  fi
+		  done
+
+		# Set "should" options
+		echo "${optnames_should}" \
+		| while read -r optname
+		  do
+			  test -n "${optname}" || continue  # ignore empty lines
+
+			  grep "^${optname}=" <"${__object:?}/parameter/option" \
+			  | sed -e 's/^.*=//' \
+			  | unquote_lines \
+			  | append_values \
+				    __uci "${section}.${optname}" --state present \
+					    --transaction "${transaction_name}"
+		  done
+		;;
+	(absent)
+		# if explorer found no section there is nothing to delete
+		test -n "${section}" || exit 0
+
+		__uci "${section}" --state absent --transaction "${transaction_name}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__uci_section/parameter/default/state b/cdist/conf/type/__uci_section/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__uci_section/parameter/default/transaction b/cdist/conf/type/__uci_section/parameter/default/transaction
new file mode 100644
index 00000000..4ad96d51
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/default/transaction
@@ -0,0 +1 @@
+default
diff --git a/cdist/conf/type/__uci_section/parameter/optional b/cdist/conf/type/__uci_section/parameter/optional
new file mode 100644
index 00000000..8b956bae
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/optional
@@ -0,0 +1,4 @@
+match
+state
+transaction
+type
diff --git a/cdist/conf/type/__uci_section/parameter/optional_multiple b/cdist/conf/type/__uci_section/parameter/optional_multiple
new file mode 100644
index 00000000..01925a15
--- /dev/null
+++ b/cdist/conf/type/__uci_section/parameter/optional_multiple
@@ -0,0 +1 @@
+option

From 179815b5e93bd51b6757d92d14470808c5bee746 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 21 Jun 2020 18:22:54 +0200
Subject: [PATCH 087/366] [type/__uci_section] Ignore SC2015 error (notabug)

---
 cdist/conf/type/__uci_section/explorer/match | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index 9cf1ea3e..a3832bff 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -39,8 +39,8 @@ then
 	exit 0
 fi
 
-test -s "${__object:?}/parameter/match" \
-&& test -s "${__object:?}/parameter/type" \
+# shellcheck disable=SC2015
+test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type" \
 || {
 	echo 'Parameters --match and --type must be used together.' >&2
 	exit 1

From d453d964e1e0827dad217e1dd424dbaf82e434fc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 20 Jul 2020 11:21:06 +0200
Subject: [PATCH 088/366] [type/__uci_section] Fix in section matching

---
 cdist/conf/type/__uci_section/explorer/match | 23 +++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index a3832bff..a5619d4e 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -17,7 +17,8 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-# This explorer the "prefix" of the section matching --match.
+# This explorer determines the "prefix" of the --type section matching --match
+# if set, or __object_id otherwise.
 
 squote_values() {
 	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
@@ -28,7 +29,7 @@ RS=$(printf '\036')
 
 if ! test -e "${__object:?}/parameter/match"
 then
-	if echo "${__object_id:?}" | grep -qvE '^[^.]+\.[^.]+$'
+	if ! echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$'
 	then
 		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
 		exit 1
@@ -47,16 +48,28 @@ test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type
 }
 
 # Find by match
-match=$(cat "${__object:?}/parameter/match")
-sect_type_filter=$(cat "${__object:?}/parameter/type")
+# NOTE: Apart from section types all values are printed in single quotes by UCI.
+match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
 
+if ! grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' "${__object:?}/parameter/type"
+then
+	echo 'Section types are a package name and section type separated by a "." (period).' >&2
+	exit 1
+fi
+
+sect_type_filter=$(cat "${__object:?}/parameter/type")
 package_filter=${sect_type_filter%%.*}
-section_filter=${sect_type_filter##*.}
+section_filter=${sect_type_filter#*.}
+# FIXME: Does not work with named sections
 regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
 
 matched_sections=$(
 	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
 	| grep -e "${regex}" \
+	| while read -r _line
+	  do
+		  if test "${_line#*=}" = "${match#*=}"; then echo "${_line}"; fi
+	  done \
 	| sed -e 's/\.[^.]*=.*$//')
 
 if test "$(echo "${matched_sections}" | wc -l)" -gt 1

From f782a5a370a956995f444a677ecffe7f1564764d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 23 Oct 2020 11:45:37 +0200
Subject: [PATCH 089/366] [type/__uci] Refactor to do proper quoting of UCI
 commands

---
 cdist/conf/type/__uci/gencode-local | 40 ++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index ba27dc5e..44bf9c4f 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -18,15 +18,37 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r NDL; echo "${NDL}")"; } }
+in_list() {
+	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
+}
+
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
 
 uci_cmd() {
-	printf 'printf "%s\n"' "$1"
-	shift
-	printf " '%s'" "$@"
-	printf " >>'%s'\n" "${tmpfile}"
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	# Will output a shell command to be executed locally to add the given
+	# command to $tmpfile.
+	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
+	printf ' >>%s\n' "$(quote "${tmpfile}")"
 }
 
+
 config=${__object_id:?}
 
 state_is=$(cat "${__object:?}/explorer/state")
@@ -57,19 +79,19 @@ in
 
 			if test "${state_is}" != 'absent'
 			then
-				printf "uci delete '%s'\n" "${config}"
+				uci_cmd delete "${config}"
 			fi
 
 			while read -r value
 			do
-				uci_cmd "add_list '%s'='%s'" "${config}" "${value}"
+				uci_cmd add_list "${config}"="${value}"
 			done <"${__object:?}/parameter/value"
 		else
 			# "should" is a scalar
 			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
 
 			value=$(cat "${__object:?}/parameter/value")
-			uci_cmd "set '%s'='%s'" "${config}" "${value}"
+			uci_cmd set "${config}"="${value}"
 		fi
 		;;
 	(absent)
@@ -79,7 +101,7 @@ in
 		fi
 
 		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
-		uci_cmd "delete '%s'" "${config}"
+		uci_cmd delete "${config}"
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2

From 3a6b0851459f08ad56f1c6a138b168fedfba993e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 23 Oct 2020 22:03:25 +0200
Subject: [PATCH 090/366] [type/__uci] Check __object_id for syntax errors

---
 cdist/conf/type/__uci/gencode-local | 45 +++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index 44bf9c4f..af91a698 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -48,8 +48,53 @@ uci_cmd() {
 	printf ' >>%s\n' "$(quote "${tmpfile}")"
 }
 
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+uci_validate_tuple() (
+	tok=${1:?}
+	case $tok
+	in
+		(*.*.*)
+			# check option
+			option=${tok##*.}
+			uci_validate_name "${option}" || {
+				printf 'Invalid option: %s\n' "${option}" >&2
+				return 1
+			}
+			tok=${tok%.*}
+			;;
+		(*.*)
+			# no option (section definition)
+			;;
+		(*)
+			printf 'Invalid tuple: %s\n' "$1" >&2
+			return 1
+			;;
+	esac
+
+	case ${tok#*.}
+	in
+		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
+		(*)  section=${tok#*.} ;;
+	esac
+	uci_validate_name "${section}" || {
+		printf 'Invalid section: %s\n' "${1#*.}" >&2
+		return 1
+	}
+
+	config=${tok%%.*}
+	uci_validate_name "${config}" || {
+		printf 'Invalid config: %s\n' "${config}" >&2
+		return 1
+	}
+)
+
 
 config=${__object_id:?}
+uci_validate_tuple "${config}"
 
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")

From c37253b852477a0a22568643bad461d20b58706f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 22 Oct 2020 20:32:48 +0200
Subject: [PATCH 091/366] [type/__uci_section] Check __object_id for syntax
 errors

---
 cdist/conf/type/__uci_section/manifest | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index e16277ba..1a59b970 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -33,6 +33,32 @@ append_values() {
 	"$@" </dev/null
 }
 
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+## Check section name and error if invalid!
+case ${__object_id:?}
+in
+	(*.*)
+		uci_validate_name "${__object_id%%.*}" || {
+			printf 'Invalid package name: %s\n' "${__object_id%%.*}" >&2
+			exit 1
+		}
+		uci_validate_name "${__object_id#*.}" || {
+			printf 'Invalid section name: %s\n' "${__object_id#*.}" >&2
+			exit 1
+		}
+		;;
+	(*)
+		uci_validate_name "${__object_id:?}" || {
+			printf 'Invalid section name: %s\n' "${__object_id:?}" >&2
+			exit 1
+		}
+		;;
+esac
+
 section=$(cat "${__object:?}/explorer/match")
 
 state_should=$(cat "${__object:?}/parameter/state")

From fe26c119b52c3822f009fe9aeb1b52b60139e33a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 10:01:49 +0200
Subject: [PATCH 092/366] [type/__uci*] Update man pages

---
 cdist/conf/type/__uci/man.rst         | 15 ++++----
 cdist/conf/type/__uci_commit/man.rst  | 12 ++++---
 cdist/conf/type/__uci_section/man.rst | 51 ++++++++++++++++++++-------
 3 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 2f64355b..cf184f9d 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -3,23 +3,22 @@ cdist-type__uci(7)
 
 NAME
 ----
-cdist-type__uci - Manage configuration values in OpenWrt's
-Unified Configuration Interface (UCI)
+cdist-type__uci - Manage configuration values in UCI
 
 
 DESCRIPTION
 -----------
-This cdist type can be used to alter configuration options in OpenWrt's UCI
-system.
+This cdist type can be used to alter configuration options in OpenWrt's
+Unified Configuration Interface (UCI) system.
 
-Options can be applied in batches if the `--transaction` parameter is used.
+Options can be applied in batches if the ``--transaction`` parameter is used.
 
 
 REQUIRED PARAMETERS
 -------------------
 value
     The value to be set. Can be used multiple times.
-    This parameter is allowed to be omitted if `--state` is `absent`.
+    This parameter is ignored if ``--state`` is ``absent``.
 
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
@@ -28,10 +27,10 @@ value
 OPTIONAL PARAMETERS
 -------------------
 state
-    `present` or `absent`, defaults to `present`.
+    ``present`` or ``absent``, defaults to ``present``.
 transaction
     The name of the transaction this option belongs to.
-    If none is given: "default" is used.
+    If none is given: ``default`` is used.
 
 
 BOOLEAN PARAMETERS
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
index c55d06e5..ea6611eb 100644
--- a/cdist/conf/type/__uci_commit/man.rst
+++ b/cdist/conf/type/__uci_commit/man.rst
@@ -3,14 +3,16 @@ cdist-type__uci_commit(7)
 
 NAME
 ----
-cdist-type__uci_commit - Commit a UCI transaction.
+cdist-type__uci_commit - Commit UCI transactions
 
 
 DESCRIPTION
 -----------
-This type executes the "uci commit" command on the target.
-It is usually not required to use this type. Use the `--transaction` parameter
-of `cdist-type__uci`\ (7) instead.
+This type executes the ``uci commit`` command on the target with the commands
+queued in a transaction.
+It is usually not required to use this type. Use the ``--transaction`` parameter
+of :strong:`cdist-type__uci`\ (7) and :strong:`cdist-type__uci_section`\ (7)
+instead.
 
 
 REQUIRED PARAMETERS
@@ -42,7 +44,7 @@ EXAMPLES
 
 SEE ALSO
 --------
-:strong:`cdist-type__uci`\ (7)
+:strong:`cdist-type__uci`\ (7), :strong:`cdist-type__uci_section`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 3468bfc3..86729316 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -3,21 +3,23 @@ cdist-type__uci_section(7)
 
 NAME
 ----
-cdist-type__uci_section - Manage configuration sections in OpenWrt's
-Unified Configuration Interface (UCI)
+cdist-type__uci_section - Manage configuration sections in UCI
 
 
 DESCRIPTION
 -----------
 This cdist type can be used to replace whole configuration sections in OpenWrt's
-UCI system.
-It can be thought of as syntactic sugar for `cdist-type__uci`\ (7), as this type
-will generate the required `__uci` objects to make the section contain exactly
-the options specified via ``--option``.
+Unified Configuration Interface (UCI) system.
+It can be thought of as syntactic sugar for :strong:`cdist-type__uci`\ (7),
+as this type will generate the required `__uci` objects to make the section
+contain exactly the options specified using ``--option``.
 
 Since many default UCI sections are unnamed, this type allows to find the
 matching section by one of its options using the ``--match`` parameter.
 
+**NOTE:** Options already present on the target and not listed in ``--option``
+or ``--list`` will be deleted.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -28,18 +30,18 @@ OPTIONAL PARAMETERS
 -------------------
 match
     Allows to find a section to "replace" through one of its parameters.
-    The value to this parameter is a ``<option>=<string>`` string.
+    The value to this parameter is a ``<option>=<value>`` string.
 option
     An option that should be present in the section.
     This parameter can be used multiple times to specify multiple options.
-    The value to this parameter is a ``<option>=<string>`` string.
+    The value to this parameter is a ``<option>=<value>`` string.
 
     Lists can be expressed by repeatedly using the same key.
 state
-    `present` or `absent`, defaults to `present`.
+    ``present`` or ``absent``, defaults to ``present``.
 transaction
     The name of the transaction this option belongs to.
-    The value will be forwarded to `cdist-type__uci`\ (7).
+    The value will be forwarded to :strong:`cdist-type__uci`\ (7).
 type
     The type of the section in the format: ``<config>.<section-type>``
 
@@ -54,13 +56,36 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # TODO
-    __uci_section ...
+    # Configure the dropbear daemon
+    __uci_section dropbear --type dropbear.dropbear --transaction sshd \
+        --match Port=22 --option Port=22 \
+        --option PasswordAuth=off \
+        --option RootPasswordAuth=off
+
+    # Block SSH access from the guest network
+    __uci_section firewall.block_ssh_from_guest --type firewall.rule \
+        --transaction fwrules \
+        --option name='Block-SSH-Access-from-Guest' \
+        --option src='guest' \
+        --option proto='tcp' \
+        --option dest_port='22' \
+        --option target='REJECT'
+
+    # Configure a Wi-Fi access point
+    __uci_section wireless.default_radio0 --type wireless.wifi-iface \
+        --transaction wifi \
+        --option device='radio0' \
+        --option mode='ap' \
+        --option network='wlan' \
+        --option ssid='mywifi' \
+        --option encryption="psk2' \
+        --option key='hunter2'
 
 
 SEE ALSO
 --------
-:strong:`cdist-type__uci`\ (7)
+- https://openwrt.org/docs/guide-user/base-system/uci
+- :strong:`cdist-type__uci`\ (7)
 
 
 AUTHORS

From 0840afce03c50824278d64cd204b86e79077b48e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 12:57:40 +0200
Subject: [PATCH 093/366] [type/__uci] Add --type parameter

---
 cdist/conf/type/__uci/gencode-local      |  54 ++++++++----
 cdist/conf/type/__uci/gencode-remote     | 101 +++++++++++++++++++++++
 cdist/conf/type/__uci/man.rst            |   9 +-
 cdist/conf/type/__uci/parameter/optional |   1 +
 4 files changed, 146 insertions(+), 19 deletions(-)
 create mode 100644 cdist/conf/type/__uci/gencode-remote

diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index af91a698..34482a01 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -117,27 +117,45 @@ in
 			exit 0
 		fi
 
-		if test "$(wc -l "${__object:?}/parameter/value")" -gt 1
-		then
-			# "should" is a list
-			printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+		# Determine type
+		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
+		case ${type}
+		in
+			(option|list) ;;
+			('')
+				# Guess type by the number of values
+				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
+					 && type=list \
+					 || type=option
+				;;
+			(*)
+				printf 'Invalid --type: %s\n' "${type}" >&2
+				exit 1
+				;;
+		esac
 
-			if test "${state_is}" != 'absent'
-			then
-				uci_cmd delete "${config}"
-			fi
+		case ${type}
+		in
+			(list)
+				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
 
-			while read -r value
-			do
-				uci_cmd add_list "${config}"="${value}"
-			done <"${__object:?}/parameter/value"
-		else
-			# "should" is a scalar
-			printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+				if test "${state_is}" != 'absent'
+				then
+					uci_cmd delete "${config}"
+				fi
 
-			value=$(cat "${__object:?}/parameter/value")
-			uci_cmd set "${config}"="${value}"
-		fi
+				while read -r value
+				do
+					uci_cmd add_list "${config}"="${value}"
+				done <"${__object:?}/parameter/value"
+				;;
+			(option)
+				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
+				value=$(cat "${__object:?}/parameter/value")
+				uci_cmd set "${config}"="${value}"
+				;;
+		esac
 		;;
 	(absent)
 		if in_list "${state_is}" 'absent'
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
new file mode 100644
index 00000000..348d15a1
--- /dev/null
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -0,0 +1,101 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+config=${__object_id:?}
+uci_validate_tuple "${config}"
+
+
+case ${state_should}
+in
+	(present)
+		if in_list "${state_is}" 'present' 'rearranged'
+		then
+			# NOTE: order is ignored so rearranged is also fine.
+			exit 0
+		fi
+
+		# Determine type
+		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
+		case ${type}
+		in
+			(option|list) ;;
+			('')
+				# Guess type by the number of values
+				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
+					 && type=list \
+					 || type=option
+				;;
+			(*)
+				printf 'Invalid --type: %s\n' "${type}" >&2
+				exit 1
+				;;
+		esac
+
+		case ${type}
+		in
+			(list)
+				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
+
+				if test "${state_is}" != 'absent'
+				then
+					uci_cmd delete "${config}"
+				fi
+
+				while read -r value
+				do
+					uci_cmd add_list "${config}"="${value}"
+				done <"${__object:?}/parameter/value"
+				;;
+			(option)
+				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
+
+				value=$(cat "${__object:?}/parameter/value")
+				uci_cmd set "${config}"="${value}"
+				;;
+		esac
+		;;
+	(absent)
+		if in_list "${state_is}" 'absent'
+		then
+			exit 0
+		fi
+
+		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
+		uci_cmd delete "${config}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index cf184f9d..9d33d618 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -31,6 +31,10 @@ state
 transaction
     The name of the transaction this option belongs to.
     If none is given: ``default`` is used.
+type
+    If the type should generate an option or a list.
+    One of: ``option`` or ``list``.
+    Defaults to auto-detect based on the number of ``--value`` parameters.
 
 
 BOOLEAN PARAMETERS
@@ -46,10 +50,13 @@ EXAMPLES
     # Set the system hostname
     __uci system.@system[0].hostname --value 'OpenWrt'
 
+    # Set DHCP option 252: tell DHCP clients to not ask for proxy information.
+    __uci dhcp.lan.dhcp_option --type list --value '252,"\n"'
+
     # Enable NTP and NTPd (in one transaction)
     __uci system.ntp.enabled --value 1 --transaction ntp
     __uci system.ntp.enable_server --value 1 --transaction ntp
-    __uci system.ntp.server --transaction ntp \
+    __uci system.ntp.server --type list --transaction ntp \
         --value '0.openwrt.pool.ntp.org' \
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
index ddbbba16..2703b3df 100644
--- a/cdist/conf/type/__uci/parameter/optional
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -1,2 +1,3 @@
 state
 transaction
+type

From 49e867fab4c1bcb6549079e735abec2149e42cf6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 15:59:25 +0200
Subject: [PATCH 094/366] [type/__uci_section] Add more parameter checks

---
 cdist/conf/type/__uci_section/manifest | 42 +++++++++++++++++++-------
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 1a59b970..b5fa4a3b 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -19,11 +19,15 @@
 #
 
 grep_line() { echo "$2" | grep -qxF "$1"; }
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
+}
+validate_options() { grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"; }
 unquote_lines() {
 	sed -e '/^".*"$/{s/^"//;s/"$//}' \
 	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
 }
-
 append_values() {
 	while read -r _value
 	do
@@ -32,12 +36,23 @@ append_values() {
 	unset _value
 	"$@" </dev/null
 }
-
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+print_errors() {
+	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
+		BEGIN {
+			if (getline) {
+				print prefix
+				print
+				rc = 1
+			}
+		}
+		{ print }
+		END {
+			if (rc && suffix) print suffix
+			exit rc
+		}' >&2
 }
 
+
 ## Check section name and error if invalid!
 case ${__object_id:?}
 in
@@ -74,12 +89,6 @@ in
 		type_is=$(cat "${__object:?}/explorer/type")
 		type_should=$(cat "${__object:?}/parameter/type")
 
-		if test -f "${__object:?}/parameter/option"
-		then
-			optnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
-		fi
-
 		if test -n "${type_is}"
 		then
 			if test "${type_is}" != "${type_should##*.}"
@@ -105,6 +114,17 @@ in
 			esac
 		fi
 
+		# Check options for syntax errors
+		validate_options "${__object:?}/parameter/object" \
+		| print_errors 'Found erroneous options in arguments:'
+
+		# Collect option names
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
 		# Make sure the section itself is present
 		__uci "${section}" --state present --transaction "${transaction_name}" \
 			--value "${sect_type}"

From b99ca3cbdf7c761db05598e31e9788e5e4e81cc6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 16:01:02 +0200
Subject: [PATCH 095/366] [type/__uci_section] Split up --option and --list

---
 cdist/conf/type/__uci_section/man.rst         | 19 +++++++-
 cdist/conf/type/__uci_section/manifest        | 48 ++++++++++++-------
 .../__uci_section/parameter/optional_multiple |  1 +
 3 files changed, 50 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 86729316..b8829433 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -28,15 +28,21 @@ None.
 
 OPTIONAL PARAMETERS
 -------------------
+list
+    An option that is part of a list and should be present in the section (as
+    part of a list).  Lists with multiple options can be expressed by using the
+    same ``<option>`` repeatedly.
+
+    The value to this parameter is a ``<option>=<value>`` string.
 match
     Allows to find a section to "replace" through one of its parameters.
+
     The value to this parameter is a ``<option>=<value>`` string.
 option
     An option that should be present in the section.
     This parameter can be used multiple times to specify multiple options.
-    The value to this parameter is a ``<option>=<value>`` string.
 
-    Lists can be expressed by repeatedly using the same key.
+    The value to this parameter is a ``<option>=<value>`` string.
 state
     ``present`` or ``absent``, defaults to ``present``.
 transaction
@@ -62,6 +68,15 @@ EXAMPLES
         --option PasswordAuth=off \
         --option RootPasswordAuth=off
 
+    # Define a firewall zone comprised of lan and wlan networks
+    __uci_section firewall.internal --type firewall.zone --transaction fw \
+        --option name='internal' \
+        --list network='lan' \
+        --list network='wlan' \
+        --option input='ACCEPT' \
+        --option output='ACCEPT' \
+        --option forward='ACCEPT'
+
     # Block SSH access from the guest network
     __uci_section firewall.block_ssh_from_guest --type firewall.rule \
         --transaction fwrules \
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index b5fa4a3b..a7865315 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,7 +18,14 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-grep_line() { echo "$2" | grep -qxF "$1"; }
+grep_line() { { shift; printf '%s\n' "$@"; } | grep -qxF "$1"; }
+prefix_lines() {
+	while test $# -gt 0
+	do
+		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
+		shift; shift
+	done
+}
 uci_validate_name() {
 	# like util.c uci_validate_name()
 	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
@@ -115,14 +122,25 @@ in
 		fi
 
 		# Check options for syntax errors
-		validate_options "${__object:?}/parameter/object" \
+		validate_options "${__object:?}/parameter/list" "${__object:?}/parameter/object" \
 		| print_errors 'Found erroneous options in arguments:'
 
 		# Collect option names
+		if test -f "${__object:?}/parameter/list"
+		then
+			listnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
+		fi
+
 		if test -f "${__object:?}/parameter/option"
 		then
-			optnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+			optnames_should=$(sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort)
+
+			echo "${optnames_should}" \
+			| uniq -d \
+			| print_errors \
+				  'Found duplicate --options:' \
+				  "$(printf '\nUse --list for lists, instead.')"
 		fi
 
 		# Make sure the section itself is present
@@ -130,29 +148,27 @@ in
 			--value "${sect_type}"
 		export require=__uci/"${section}"
 
-		# Delete options not in "should"
+		# Delete options/lists not in "should"
 		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
 		| while read -r optname
 		  do
-			  if ! grep_line "${optname}" "${optnames_should}"
-			  then
-				  __uci "${section}.${optname}" --state absent \
-					  --transaction "${transaction_name}" </dev/null
-			  fi
+			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
+				  __uci "${section}.${optname}" --state absent --transaction "${transaction_name}"
+			  } </dev/null
 		  done
 
 		# Set "should" options
-		echo "${optnames_should}" \
-		| while read -r optname
+		prefix_lines option "${optnames_should}" list "${listnames_should}" \
+		| while read -r _type _optname
 		  do
-			  test -n "${optname}" || continue  # ignore empty lines
+			  test -n "${_type}${_optname}" || continue  # ignore empty lines
 
-			  grep "^${optname}=" <"${__object:?}/parameter/option" \
+			  grep "^${_optname}=" <"${__object:?}/parameter/${_type}" \
 			  | sed -e 's/^.*=//' \
 			  | unquote_lines \
 			  | append_values \
-				    __uci "${section}.${optname}" --state present \
-					    --transaction "${transaction_name}"
+				    __uci "${section}.${_optname}" --state present \
+					    --type "${_type}" --transaction "${transaction_name}"
 		  done
 		;;
 	(absent)
diff --git a/cdist/conf/type/__uci_section/parameter/optional_multiple b/cdist/conf/type/__uci_section/parameter/optional_multiple
index 01925a15..98af35e6 100644
--- a/cdist/conf/type/__uci_section/parameter/optional_multiple
+++ b/cdist/conf/type/__uci_section/parameter/optional_multiple
@@ -1 +1,2 @@
+list
 option

From 8728817af6ae1fa801a39456df3f84d420f87744 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 21:11:50 +0200
Subject: [PATCH 096/366] [type/__uci] Unquote UCI reported values

Without unquoting values printed in single quotes by UCI would always lead to
the state explorer reporting "different".
---
 cdist/conf/type/__uci/explorer/state | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__uci/explorer/state b/cdist/conf/type/__uci/explorer/state
index 6fb4b173..d7363dbf 100644
--- a/cdist/conf/type/__uci/explorer/state
+++ b/cdist/conf/type/__uci/explorer/state
@@ -51,21 +51,34 @@ fi
 # strip off trailing newline
 printf '%s' "${values_is}" \
 | awk '
+function unquote(s) {
+	# simplified dequoting of single quoted strings
+	if (s ~ /^'\''.*'\''$/) {
+		s = substr(s, 2, length(s) - 2)
+		sub(/'"'\\\\''"'/, "'\''", s)
+	}
+	return s
+}
+
 BEGIN {
 	state = "present"  # assume all is fine
 }
 NR == FNR {
 	# memoize "should" state
 	should[FNR] = $0
+	should_count++
 
 	# go to next line (important!)
 	next
 }
 
 # compare "is" state
+
+{ $0 = unquote($0) }
+
 $0 == should[FNR] { next }
 
-FNR > length(should) {
+FNR > should_count {
 	# there are more "is" records than "should" -> definitely different
 	state = "different"
 	exit
@@ -87,7 +100,7 @@ FNR > length(should) {
 }
 
 END {
-	if (FNR < length(should)) {
+	if (FNR < should_count) {
 		# "is" was shorter than "should" -> different
 		state = "different"
 	}

From 4aebb1f12756593408d6f012adb923e4c9b81e1e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 24 Oct 2020 21:18:22 +0200
Subject: [PATCH 097/366] [type/__uci*] Update man.rst regarding quoting
 requirements

---
 cdist/conf/type/__uci/man.rst         | 2 ++
 cdist/conf/type/__uci_section/man.rst | 8 ++++++++
 2 files changed, 10 insertions(+)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 9d33d618..c7f5f4f0 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -23,6 +23,8 @@ value
     Due to the way cdist handles arguments, values **must not** contain newline
     characters.
 
+    Values do not need special quoting for UCI. The only requirement is that the
+    value is passed to the type as a single shell argument.
 
 OPTIONAL PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index b8829433..753d00b3 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -34,6 +34,10 @@ list
     same ``<option>`` repeatedly.
 
     The value to this parameter is a ``<option>=<value>`` string.
+
+    ``<value>`` does not need special quoting for UCI.
+    The only requirement is that the value is passed to the type as a single
+    shell argument.
 match
     Allows to find a section to "replace" through one of its parameters.
 
@@ -43,6 +47,10 @@ option
     This parameter can be used multiple times to specify multiple options.
 
     The value to this parameter is a ``<option>=<value>`` string.
+
+    ``<value>`` does not need special quoting for UCI.
+    The only requirement is that the value is passed to the type as a single
+    shell argument.
 state
     ``present`` or ``absent``, defaults to ``present``.
 transaction

From 63d41a1053bd8b03ddda130d7129d233f97ac671 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 21:10:27 +0100
Subject: [PATCH 098/366] [type/__uci_section] Improve --match support with
 existing named sections

Use section if named section exists without --match option (e.g. empty section).
---
 cdist/conf/type/__uci_section/explorer/match | 80 +++++++++++++-------
 1 file changed, 51 insertions(+), 29 deletions(-)

diff --git a/cdist/conf/type/__uci_section/explorer/match b/cdist/conf/type/__uci_section/explorer/match
index a5619d4e..0768e404 100644
--- a/cdist/conf/type/__uci_section/explorer/match
+++ b/cdist/conf/type/__uci_section/explorer/match
@@ -20,62 +20,84 @@
 # This explorer determines the "prefix" of the --type section matching --match
 # if set, or __object_id otherwise.
 
+RS=$(printf '\036')
+NL=$(printf '\n '); NL=${NL% }
+
 squote_values() {
 	sed -e '/=".*"$/{s/="/='\''/;s/"$/'\''/}' \
 	    -e "/='.*'$/"'!{s/=/='\''/;s/$/'\''/}'
 }
+count_lines() (
+	IFS=${NL?}
+	# shellcheck disable=SC2048,SC2086
+	set -f -- $*; echo $#
+)
 
-RS=$(printf '\036')
-
-if ! test -e "${__object:?}/parameter/match"
-then
-	if ! echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$'
-	then
-		echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
-		exit 1
-	fi
+echo "${__object_id:?}" | grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' || {
+	echo 'Section identifiers are a package and section name separated by a "." (period).' >&2
+	exit 1
+}
 
+test -s "${__object:?}/parameter/match" || {
 	# If no --match is given, we take the __object_id as the section identifier.
 	echo "${__object_id:?}"
 	exit 0
-fi
-
-# shellcheck disable=SC2015
-test -s "${__object:?}/parameter/match" && test -s "${__object:?}/parameter/type" \
-|| {
+}
+test -s "${__object:?}/parameter/type" || {
 	echo 'Parameters --match and --type must be used together.' >&2
 	exit 1
 }
 
-# Find by match
-# NOTE: Apart from section types all values are printed in single quotes by UCI.
-match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
-
-if ! grep -q -e '^[^.]\{1,\}\.[^.]\{1,\}$' "${__object:?}/parameter/type"
-then
+sect_type_param=$(cat "${__object:?}/parameter/type")
+expr "${sect_type_param}" : '[^.]\{1,\}\.[^.]\{1,\}$' >/dev/null 2>&1 || {
 	echo 'Section types are a package name and section type separated by a "." (period).' >&2
 	exit 1
+}
+package_filter=${sect_type_param%%.*}
+section_filter=${sect_type_param#*.}
+
+# Find by --match
+# NOTE: Apart from section types all values are printed in single quotes by uci show.
+match=$(head -n 1 "${__object:?}/parameter/match" | squote_values)
+
+if uci -s -N get "${__object_id:?}" >/dev/null 2>&1
+then
+	# Named section exists: ensure if --match applies to it
+	# if the "matched" option does not exist (e.g. empty section) we use the
+	# section unconditionally.
+	if match_value_is=$(uci -s -N get "${__object_id:?}.${match%%=*}" 2>/dev/null)
+	then
+		match_value_should=$(expr "${match}" : ".*='\\(.*\\)'$")
+
+		test "${match_value_is}" = "${match_value_should}" || {
+			printf 'Named section "%s" does not match --match "%s"\n' \
+				   "${__object_id:?}" "${match}" >&2
+			exit 1
+		}
+	fi
+
+	echo "${__object_id:?}"
+	exit 0
 fi
 
-sect_type_filter=$(cat "${__object:?}/parameter/type")
-package_filter=${sect_type_filter%%.*}
-section_filter=${sect_type_filter#*.}
-# FIXME: Does not work with named sections
-regex="^${package_filter}\.@${section_filter}\[[0-9]\{1,\}\]\.${match%%=*}="
+# No correctly named section exists already: find one to which --match applies
+regex="^${package_filter}\\.@${section_filter}\\[[0-9]\\{1,\\}\\]\\.${match%%=*}="
 
 matched_sections=$(
 	uci -s -N -d "${RS}" show "${package_filter}" 2>/dev/null \
 	| grep -e "${regex}" \
 	| while read -r _line
 	  do
-		  if test "${_line#*=}" = "${match#*=}"; then echo "${_line}"; fi
+		  if test "${_line#*=}" = "${match#*=}"
+		  then
+			  echo "${_line}"
+		  fi
 	  done \
 	| sed -e 's/\.[^.]*=.*$//')
 
-if test "$(echo "${matched_sections}" | wc -l)" -gt 1
-then
+test "$(count_lines "${matched_sections}")" -le 1 || {
 	printf 'Found multiple matching sections:\n%s\n' "${matched_sections}" >&2
 	exit 1
-fi
+}
 
 echo "${matched_sections}"

From 7b301195040c4861dbd15c5f2514c4e9eeadbe61 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Oct 2020 18:52:43 +0100
Subject: [PATCH 099/366] [type/__uci] Externalise functions to separate file

---
 cdist/conf/type/__uci/files/functions.sh | 75 +++++++++++++++++++++
 cdist/conf/type/__uci/gencode-local      | 85 ++----------------------
 cdist/conf/type/__uci/manifest           |  4 +-
 3 files changed, 85 insertions(+), 79 deletions(-)
 create mode 100644 cdist/conf/type/__uci/files/functions.sh

diff --git a/cdist/conf/type/__uci/files/functions.sh b/cdist/conf/type/__uci/files/functions.sh
new file mode 100644
index 00000000..4ddb82e0
--- /dev/null
+++ b/cdist/conf/type/__uci/files/functions.sh
@@ -0,0 +1,75 @@
+# -*- mode: sh; indent-tabs-mode: t -*-
+
+in_list() {
+	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
+}
+
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
+
+uci_cmd() {
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	# Will output a shell command to be executed locally to add the given
+	# command to $tmpfile.
+	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
+	printf ' >>%s\n' "$(quote "${tmpfile}")"
+}
+
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
+}
+
+uci_validate_tuple() (
+	tok=${1:?}
+	case $tok
+	in
+		(*.*.*)
+			# check option
+			option=${tok##*.}
+			uci_validate_name "${option}" || {
+				printf 'Invalid option: %s\n' "${option}" >&2
+				return 1
+			}
+			tok=${tok%.*}
+			;;
+		(*.*)
+			# no option (section definition)
+			;;
+		(*)
+			printf 'Invalid tuple: %s\n' "$1" >&2
+			return 1
+			;;
+	esac
+
+	case ${tok#*.}
+	in
+		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
+		(*)  section=${tok#*.} ;;
+	esac
+	uci_validate_name "${section}" || {
+		printf 'Invalid section: %s\n' "${1#*.}" >&2
+		return 1
+	}
+
+	config=${tok%%.*}
+	uci_validate_name "${config}" || {
+		printf 'Invalid config: %s\n' "${config}" >&2
+		return 1
+	}
+)
diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
index 34482a01..b7664e85 100755
--- a/cdist/conf/type/__uci/gencode-local
+++ b/cdist/conf/type/__uci/gencode-local
@@ -18,84 +18,6 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() {
-	printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; }
-}
-
-quote() {
-	for _arg
-	do
-		shift
-		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
-		then
-			# needs quoting
-			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
-		else
-			set -- "$@" "${_arg}"
-		fi
-	done
-	unset _arg
-
-	# NOTE: Use printf because POSIX echo interprets escape sequences
-	printf '%s' "$*"
-}
-
-uci_cmd() {
-	# Usage: uci_cmd [UCI ARGUMENTS]...
-	# Will output a shell command to be executed locally to add the given
-	# command to $tmpfile.
-	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
-	printf ' >>%s\n' "$(quote "${tmpfile}")"
-}
-
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(echo "$*" | tr -d '[:alnum:]_')"
-}
-
-uci_validate_tuple() (
-	tok=${1:?}
-	case $tok
-	in
-		(*.*.*)
-			# check option
-			option=${tok##*.}
-			uci_validate_name "${option}" || {
-				printf 'Invalid option: %s\n' "${option}" >&2
-				return 1
-			}
-			tok=${tok%.*}
-			;;
-		(*.*)
-			# no option (section definition)
-			;;
-		(*)
-			printf 'Invalid tuple: %s\n' "$1" >&2
-			return 1
-			;;
-	esac
-
-	case ${tok#*.}
-	in
-		(@*) section=$(expr "${tok#*.}" : '@\(.*\)\[-*[0-9]*\]$') ;;
-		(*)  section=${tok#*.} ;;
-	esac
-	uci_validate_name "${section}" || {
-		printf 'Invalid section: %s\n' "${1#*.}" >&2
-		return 1
-	}
-
-	config=${tok%%.*}
-	uci_validate_name "${config}" || {
-		printf 'Invalid config: %s\n' "${config}" >&2
-		return 1
-	}
-)
-
-
-config=${__object_id:?}
-uci_validate_tuple "${config}"
-
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
@@ -107,6 +29,13 @@ mkdir -p "${tmpdir}"
 
 tmpfile="${tmpdir}/${transaction_name}.txt"
 
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+
+config=${__object_id:?}
+uci_validate_tuple "${config}"
+
 
 case ${state_should}
 in
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index e56462e5..d0c8239e 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,7 +18,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-in_list() { printf '%s\n' "$@" | { grep -qxF "$(read -r ndl; echo "${ndl}")"; } }
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
 
 os=$(cat "${__global:?}/explorer/os")
 

From a6c37095f1eea72acbebebfa88bdf84f65f61f51 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Oct 2020 18:50:34 +0100
Subject: [PATCH 100/366] [type/__uci_section] Externalise functions to
 separate file

---
 .../type/__uci_section/files/functions.sh     | 52 +++++++++++++++++++
 cdist/conf/type/__uci_section/manifest        | 42 +--------------
 2 files changed, 54 insertions(+), 40 deletions(-)
 create mode 100644 cdist/conf/type/__uci_section/files/functions.sh

diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
new file mode 100644
index 00000000..9f510958
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -0,0 +1,52 @@
+# -*- mode: sh; indent-tabs-mode: t -*-
+
+append_values() {
+	while read -r _value
+	do
+		set -- "$@" --value "${_value}"
+	done
+	unset _value
+	"$@" </dev/null
+}
+
+grep_line() {
+	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
+}
+
+prefix_lines() {
+	while test $# -gt 0
+	do
+		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
+		shift; shift
+	done
+}
+
+print_errors() {
+	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
+		BEGIN {
+			if (getline) {
+				print prefix
+				print
+				rc = 1
+			}
+		}
+		{ print }
+		END {
+			if (rc && suffix) print suffix
+			exit rc
+		}' >&2
+}
+
+uci_validate_name() {
+	# like util.c uci_validate_name()
+	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
+}
+
+unquote_lines() {
+	sed -e '/^".*"$/{s/^"//;s/"$//}' \
+	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
+}
+
+validate_options() {
+	grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"
+}
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index a7865315..208a2c4e 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,46 +18,8 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-grep_line() { { shift; printf '%s\n' "$@"; } | grep -qxF "$1"; }
-prefix_lines() {
-	while test $# -gt 0
-	do
-		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
-		shift; shift
-	done
-}
-uci_validate_name() {
-	# like util.c uci_validate_name()
-	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
-}
-validate_options() { grep -shv -e '^[[:alnum:]_]\{1,\}=' "$@"; }
-unquote_lines() {
-	sed -e '/^".*"$/{s/^"//;s/"$//}' \
-	    -e '/'"^'.*'"'$/{s/'"^'"'//;s/'"'$"'//}'
-}
-append_values() {
-	while read -r _value
-	do
-		set -- "$@" --value "${_value}"
-	done
-	unset _value
-	"$@" </dev/null
-}
-print_errors() {
-	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
-		BEGIN {
-			if (getline) {
-				print prefix
-				print
-				rc = 1
-			}
-		}
-		{ print }
-		END {
-			if (rc && suffix) print suffix
-			exit rc
-		}' >&2
-}
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
 
 
 ## Check section name and error if invalid!

From c1ae3ccb2f725d0d94eaec9eeae7c8f5f98a5e45 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:13:06 +0100
Subject: [PATCH 101/366] [type/__uci*] Remove public-facing transaction
 "interface"

---
 cdist/conf/type/__uci/man.rst          | 13 ++++---------
 cdist/conf/type/__uci_section/man.rst  |  9 ++-------
 cdist/conf/type/__uci_section/manifest | 10 ++++------
 3 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index c7f5f4f0..8c3f79f1 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -11,8 +11,6 @@ DESCRIPTION
 This cdist type can be used to alter configuration options in OpenWrt's
 Unified Configuration Interface (UCI) system.
 
-Options can be applied in batches if the ``--transaction`` parameter is used.
-
 
 REQUIRED PARAMETERS
 -------------------
@@ -30,9 +28,6 @@ OPTIONAL PARAMETERS
 -------------------
 state
     ``present`` or ``absent``, defaults to ``present``.
-transaction
-    The name of the transaction this option belongs to.
-    If none is given: ``default`` is used.
 type
     If the type should generate an option or a list.
     One of: ``option`` or ``list``.
@@ -55,10 +50,10 @@ EXAMPLES
     # Set DHCP option 252: tell DHCP clients to not ask for proxy information.
     __uci dhcp.lan.dhcp_option --type list --value '252,"\n"'
 
-    # Enable NTP and NTPd (in one transaction)
-    __uci system.ntp.enabled --value 1 --transaction ntp
-    __uci system.ntp.enable_server --value 1 --transaction ntp
-    __uci system.ntp.server --type list --transaction ntp \
+    # Enable NTP and NTPd (each is applied individually)
+    __uci system.ntp.enabled --value 1
+    __uci system.ntp.enable_server --value 1
+    __uci system.ntp.server --type list \
         --value '0.openwrt.pool.ntp.org' \
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
diff --git a/cdist/conf/type/__uci_section/man.rst b/cdist/conf/type/__uci_section/man.rst
index 753d00b3..a0ab78e8 100644
--- a/cdist/conf/type/__uci_section/man.rst
+++ b/cdist/conf/type/__uci_section/man.rst
@@ -53,9 +53,6 @@ option
     shell argument.
 state
     ``present`` or ``absent``, defaults to ``present``.
-transaction
-    The name of the transaction this option belongs to.
-    The value will be forwarded to :strong:`cdist-type__uci`\ (7).
 type
     The type of the section in the format: ``<config>.<section-type>``
 
@@ -71,13 +68,13 @@ EXAMPLES
 .. code-block:: sh
 
     # Configure the dropbear daemon
-    __uci_section dropbear --type dropbear.dropbear --transaction sshd \
+    __uci_section dropbear --type dropbear.dropbear \
         --match Port=22 --option Port=22 \
         --option PasswordAuth=off \
         --option RootPasswordAuth=off
 
     # Define a firewall zone comprised of lan and wlan networks
-    __uci_section firewall.internal --type firewall.zone --transaction fw \
+    __uci_section firewall.internal --type firewall.zone \
         --option name='internal' \
         --list network='lan' \
         --list network='wlan' \
@@ -87,7 +84,6 @@ EXAMPLES
 
     # Block SSH access from the guest network
     __uci_section firewall.block_ssh_from_guest --type firewall.rule \
-        --transaction fwrules \
         --option name='Block-SSH-Access-from-Guest' \
         --option src='guest' \
         --option proto='tcp' \
@@ -96,7 +92,6 @@ EXAMPLES
 
     # Configure a Wi-Fi access point
     __uci_section wireless.default_radio0 --type wireless.wifi-iface \
-        --transaction wifi \
         --option device='radio0' \
         --option mode='ap' \
         --option network='wlan' \
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 208a2c4e..41e30a07 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -46,7 +46,6 @@ esac
 section=$(cat "${__object:?}/explorer/match")
 
 state_should=$(cat "${__object:?}/parameter/state")
-transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case $state_should
 in
@@ -106,8 +105,7 @@ in
 		fi
 
 		# Make sure the section itself is present
-		__uci "${section}" --state present --transaction "${transaction_name}" \
-			--value "${sect_type}"
+		__uci "${section}" --state present --value "${sect_type}"
 		export require=__uci/"${section}"
 
 		# Delete options/lists not in "should"
@@ -115,7 +113,7 @@ in
 		| while read -r optname
 		  do
 			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
-				  __uci "${section}.${optname}" --state absent --transaction "${transaction_name}"
+				  __uci "${section}.${optname}" --state absent
 			  } </dev/null
 		  done
 
@@ -130,14 +128,14 @@ in
 			  | unquote_lines \
 			  | append_values \
 				    __uci "${section}.${_optname}" --state present \
-					    --type "${_type}" --transaction "${transaction_name}"
+					    --type "${_type}"
 		  done
 		;;
 	(absent)
 		# if explorer found no section there is nothing to delete
 		test -n "${section}" || exit 0
 
-		__uci "${section}" --state absent --transaction "${transaction_name}"
+		__uci "${section}" --state absent
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2

From e264fb004f797efc540f74191a0e9630c66dd819 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:27:07 +0100
Subject: [PATCH 102/366] [type/__uci] Convert to immediate remote execution

---
 cdist/conf/type/__uci/files/functions.sh |   6 +-
 cdist/conf/type/__uci/gencode-local      | 102 -----------------------
 cdist/conf/type/__uci/gencode-remote     |   8 --
 3 files changed, 2 insertions(+), 114 deletions(-)
 delete mode 100755 cdist/conf/type/__uci/gencode-local
 mode change 100644 => 100755 cdist/conf/type/__uci/gencode-remote

diff --git a/cdist/conf/type/__uci/files/functions.sh b/cdist/conf/type/__uci/files/functions.sh
index 4ddb82e0..277f648c 100644
--- a/cdist/conf/type/__uci/files/functions.sh
+++ b/cdist/conf/type/__uci/files/functions.sh
@@ -24,10 +24,8 @@ quote() {
 
 uci_cmd() {
 	# Usage: uci_cmd [UCI ARGUMENTS]...
-	# Will output a shell command to be executed locally to add the given
-	# command to $tmpfile.
-	printf "printf '%%s\\\\n' %s" "$(quote "$(quote "$@")")"
-	printf ' >>%s\n' "$(quote "${tmpfile}")"
+	mkdir -p "${__object:?}/files"
+	printf '%s\n' "$(quote "$@")" >>"${__object:?}/files/uci_batch.txt"
 }
 
 uci_validate_name() {
diff --git a/cdist/conf/type/__uci/gencode-local b/cdist/conf/type/__uci/gencode-local
deleted file mode 100755
index b7664e85..00000000
--- a/cdist/conf/type/__uci/gencode-local
+++ /dev/null
@@ -1,102 +0,0 @@
-#!/bin/sh -e
-#
-# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-state_is=$(cat "${__object:?}/explorer/state")
-state_should=$(cat "${__object:?}/parameter/state")
-
-transaction_name=$(cat "${__object:?}/parameter/transaction")
-
-tmpdir="${__global:?}/tmp/__uci"
-# HACK
-mkdir -p "${tmpdir}"
-
-tmpfile="${tmpdir}/${transaction_name}.txt"
-
-# shellcheck source=files/functions.sh
-. "${__type:?}/files/functions.sh"
-
-
-config=${__object_id:?}
-uci_validate_tuple "${config}"
-
-
-case ${state_should}
-in
-	(present)
-		if in_list "${state_is}" 'present' 'rearranged'
-		then
-			# NOTE: order is ignored so rearranged is also fine.
-			exit 0
-		fi
-
-		# Determine type
-		type=$(cat "${__object:?}/parameter/type" 2>/dev/null || true)
-		case ${type}
-		in
-			(option|list) ;;
-			('')
-				# Guess type by the number of values
-				test "$(wc -l "${__object:?}/parameter/value")" -gt 1 \
-					 && type=list \
-					 || type=option
-				;;
-			(*)
-				printf 'Invalid --type: %s\n' "${type}" >&2
-				exit 1
-				;;
-		esac
-
-		case ${type}
-		in
-			(list)
-				printf 'set_list %s\n' "${config}" >>"${__messages_out:?}"
-
-				if test "${state_is}" != 'absent'
-				then
-					uci_cmd delete "${config}"
-				fi
-
-				while read -r value
-				do
-					uci_cmd add_list "${config}"="${value}"
-				done <"${__object:?}/parameter/value"
-				;;
-			(option)
-				printf 'set %s\n' "${config}" >>"${__messages_out:?}"
-
-				value=$(cat "${__object:?}/parameter/value")
-				uci_cmd set "${config}"="${value}"
-				;;
-		esac
-		;;
-	(absent)
-		if in_list "${state_is}" 'absent'
-		then
-			exit 0
-		fi
-
-		printf 'delete %s\n' "${config}" >>"${__messages_out:?}"
-		uci_cmd delete "${config}"
-		;;
-	(*)
-		printf 'Invalid --state: %s\n' "${state_should}" >&2
-		exit 1
-		;;
-esac
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
old mode 100644
new mode 100755
index 348d15a1..a15f50a6
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -91,11 +91,3 @@ in
 		exit 1
 		;;
 esac
-
-if test -s "${__object:?}/files/uci_batch.txt"
-then
-	cat "${__type:?}/files/uci_apply.sh"
-	printf "uci_apply <<'EOF'\n"
-	cat "${__object:?}/files/uci_batch.txt"
-	printf '\nEOF\n'
-fi

From dfe9e08c280a2a3bc0ded15069b7d60df5a757f8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:06:05 +0100
Subject: [PATCH 103/366] [type/__uci_commit] Delete type

---
 cdist/conf/type/__uci/man.rst               |  2 -
 cdist/conf/type/__uci/manifest              | 18 -----
 cdist/conf/type/__uci_commit/gencode-remote | 81 ---------------------
 cdist/conf/type/__uci_commit/man.rst        | 60 ---------------
 cdist/conf/type/__uci_commit/nonparallel    |  0
 5 files changed, 161 deletions(-)
 delete mode 100755 cdist/conf/type/__uci_commit/gencode-remote
 delete mode 100644 cdist/conf/type/__uci_commit/man.rst
 delete mode 100644 cdist/conf/type/__uci_commit/nonparallel

diff --git a/cdist/conf/type/__uci/man.rst b/cdist/conf/type/__uci/man.rst
index 8c3f79f1..81a53473 100644
--- a/cdist/conf/type/__uci/man.rst
+++ b/cdist/conf/type/__uci/man.rst
@@ -58,13 +58,11 @@ EXAMPLES
         --value '1.openwrt.pool.ntp.org' \
         --value '2.openwrt.pool.ntp.org' \
         --value '3.openwrt.pool.ntp.org'
-    export require=__uci_commit/ntp
 
 
 SEE ALSO
 --------
 - https://openwrt.org/docs/guide-user/base-system/uci
-- :strong:`cdist-type__uci_commit`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__uci/manifest b/cdist/conf/type/__uci/manifest
index d0c8239e..26920011 100755
--- a/cdist/conf/type/__uci/manifest
+++ b/cdist/conf/type/__uci/manifest
@@ -18,15 +18,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
-. "${__type:?}/files/functions.sh"
-
-
 os=$(cat "${__global:?}/explorer/os")
 
-state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
-transaction_name=$(cat "${__object:?}/parameter/transaction")
 
 case ${os}
 in
@@ -40,8 +34,6 @@ in
 		;;
 esac
 
-changes_required=false
-
 case ${state_should}
 in
 	(present)
@@ -49,21 +41,11 @@ in
 			echo 'The parameter --value is required.' >&2
 			exit 1
 		}
-
-		# NOTE: order is ignored so rearranged is also fine.
-		in_list "${state_is}" 'present' 'rearranged' || changes_required=true
 		;;
 	(absent)
-		in_list "${state_is}" 'absent' || changes_required=true
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
 		exit 1
 		;;
 esac
-
-if ${changes_required}
-then
-	# Make sure the changes are being committed
-	require=${__object_name:?} __uci_commit "${transaction_name}"
-fi
diff --git a/cdist/conf/type/__uci_commit/gencode-remote b/cdist/conf/type/__uci_commit/gencode-remote
deleted file mode 100755
index e5c93966..00000000
--- a/cdist/conf/type/__uci_commit/gencode-remote
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/bin/sh -e
-#
-# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-transaction_name=${__object_id:?}
-batchfile="${__global:?}/tmp/__uci/${transaction_name}.txt"
-
-test -s "${batchfile}" || exit 0
-
-printf 'commit transaction %s\n' "${transaction_name}" >>"${__messages_out:?}"
-
-# NOTE: Uncommited changes are checked in code-remote instead of in an explorer
-#       because in cdist there is no interlocking between explorers and code
-#       execution.
-#       Checking for uncommited changes in an explorer leaves a time slot in
-#       which changes made are not detected by the code.
-#       Furthermore an explorer running concurrently with another transactions
-#       code-remote could lead to a false positive.
-
-cat <<CODE
-changes=\$(uci changes)
-
-if test -n "\${changes}"
-then
-	echo 'Uncommited UCI changes were found on the target:'
-	printf '%s\n\n' "\${changes}"
-	echo 'This can be caused by manual changes or due to a previous failed run.'
-	echo 'Please investigate the situation, revert or commit the changes, and try again.'
-	exit 1
-fi >&2
-
-check_errors() {
-	# reads stdin and forwards non-empty lines to stderr.
-	# returns 0 if stdin is empty, else 1.
-	! grep -e . >&2
-}
-
-commit() {
-	uci commit
-}
-
-rollback() {
-	echo >&2
-	echo 'An error occurred when trying to commit transaction ${transaction_name}!' >&2
-
-	uci changes \\
-	| sed -e 's/^-//' -e 's/\..*\$//' \\
-	| sort -u \\
-	| while read -r _package
-	  do
-		  uci revert "\${_package}"
-		  echo "\${_package}"  # for logging
-	  done \\
-	| awk '
-	  BEGIN { printf "Reverted changes in: " }
-	  { printf "%s%s", (FNR > 1 ? ", " : ""), \$0 }
-	  END { printf "\\n" }' >&2
-
-	return 1
-}
-
-uci batch <<'EOF' 2>&1 | check_errors && commit || rollback
-$(cat "${batchfile}")
-EOF
-CODE
diff --git a/cdist/conf/type/__uci_commit/man.rst b/cdist/conf/type/__uci_commit/man.rst
deleted file mode 100644
index ea6611eb..00000000
--- a/cdist/conf/type/__uci_commit/man.rst
+++ /dev/null
@@ -1,60 +0,0 @@
-cdist-type__uci_commit(7)
-=========================
-
-NAME
-----
-cdist-type__uci_commit - Commit UCI transactions
-
-
-DESCRIPTION
------------
-This type executes the ``uci commit`` command on the target with the commands
-queued in a transaction.
-It is usually not required to use this type. Use the ``--transaction`` parameter
-of :strong:`cdist-type__uci`\ (7) and :strong:`cdist-type__uci_section`\ (7)
-instead.
-
-
-REQUIRED PARAMETERS
--------------------
-None.
-
-
-OPTIONAL PARAMETERS
--------------------
-None.
-
-
-BOOLEAN PARAMETERS
-------------------
-None.
-
-
-EXAMPLES
---------
-
-.. code-block:: sh
-
-    # Commit the default transaction
-    __uci_commit default
-
-    # Commit another transaction
-    __uci_commit my_transaction
-
-
-SEE ALSO
---------
-:strong:`cdist-type__uci`\ (7), :strong:`cdist-type__uci_section`\ (7)
-
-
-AUTHORS
--------
-Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
-
-
-COPYING
--------
-Copyright \(C) 2020 Dennis Camera. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
diff --git a/cdist/conf/type/__uci_commit/nonparallel b/cdist/conf/type/__uci_commit/nonparallel
deleted file mode 100644
index e69de29b..00000000

From ec984f81b5a1db6b7f3e54255b570e07cf5282f9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 11:36:10 +0100
Subject: [PATCH 104/366] [type/__uci] Delete --transaction parameter

---
 cdist/conf/type/__uci/parameter/default/transaction | 1 -
 cdist/conf/type/__uci/parameter/optional            | 1 -
 2 files changed, 2 deletions(-)
 delete mode 100644 cdist/conf/type/__uci/parameter/default/transaction

diff --git a/cdist/conf/type/__uci/parameter/default/transaction b/cdist/conf/type/__uci/parameter/default/transaction
deleted file mode 100644
index 4ad96d51..00000000
--- a/cdist/conf/type/__uci/parameter/default/transaction
+++ /dev/null
@@ -1 +0,0 @@
-default
diff --git a/cdist/conf/type/__uci/parameter/optional b/cdist/conf/type/__uci/parameter/optional
index 2703b3df..d9080e3a 100644
--- a/cdist/conf/type/__uci/parameter/optional
+++ b/cdist/conf/type/__uci/parameter/optional
@@ -1,3 +1,2 @@
 state
-transaction
 type

From 3e5f18d409e448f69e4f259fc451d4a0e5a7a548 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 18:42:50 +0100
Subject: [PATCH 105/366] [type/__uci] Apply all commands in a single batch

---
 cdist/conf/type/__uci/files/uci_apply.sh | 43 ++++++++++++++++++++++++
 cdist/conf/type/__uci/gencode-remote     |  8 +++++
 cdist/conf/type/__uci/nonparallel        |  0
 3 files changed, 51 insertions(+)
 create mode 100644 cdist/conf/type/__uci/files/uci_apply.sh
 create mode 100644 cdist/conf/type/__uci/nonparallel

diff --git a/cdist/conf/type/__uci/files/uci_apply.sh b/cdist/conf/type/__uci/files/uci_apply.sh
new file mode 100644
index 00000000..63f94290
--- /dev/null
+++ b/cdist/conf/type/__uci/files/uci_apply.sh
@@ -0,0 +1,43 @@
+changes=$(uci changes)
+
+if test -n "${changes}"
+then
+	echo 'Uncommited UCI changes were found on the target:'
+	printf '%s\n\n' "${changes}"
+	echo 'This can be caused by manual changes or due to a previous failed run.'
+	echo 'Please investigate the situation, revert or commit the changes, and try again.'
+	exit 1
+fi >&2
+
+check_errors() {
+	# reads stdin and forwards non-empty lines to stderr.
+	# returns 0 if stdin is empty, else 1.
+	! grep -e . >&2
+}
+
+commit() {
+	uci commit
+}
+
+rollback() {
+	printf '\nAn error occurred when trying to commit UCI transaction!\n' >&2
+
+	uci changes \
+	| sed -e 's/^-//' -e 's/\..*\$//' \
+	| sort -u \
+	| while read -r _package
+	  do
+		  uci revert "${_package}"
+		  echo "${_package}"  # for logging
+	  done \
+	| awk '
+	  BEGIN { printf "Reverted changes in: " }
+	  { printf "%s%s", (FNR > 1 ? ", " : ""), $0 }
+	  END { printf "\n" }' >&2
+
+	return 1
+}
+
+uci_apply() {
+	uci batch 2>&1 | check_errors && commit || rollback
+}
diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
index a15f50a6..348d15a1 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -91,3 +91,11 @@ in
 		exit 1
 		;;
 esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci/nonparallel b/cdist/conf/type/__uci/nonparallel
new file mode 100644
index 00000000..e69de29b

From 9d40500570875f29aa48784c6637683aa31afe4e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 25 Oct 2020 20:40:29 +0100
Subject: [PATCH 106/366] [type/__uci_section] Apply all commands in a single
 batch

---
 .../type/__uci_section/files/functions.sh     |  31 +++-
 .../type/__uci_section/files/uci_apply.sh     |   1 +
 cdist/conf/type/__uci_section/gencode-remote  | 146 ++++++++++++++++++
 cdist/conf/type/__uci_section/manifest        |  76 ++-------
 cdist/conf/type/__uci_section/nonparallel     |   0
 5 files changed, 180 insertions(+), 74 deletions(-)
 create mode 120000 cdist/conf/type/__uci_section/files/uci_apply.sh
 create mode 100755 cdist/conf/type/__uci_section/gencode-remote
 create mode 100644 cdist/conf/type/__uci_section/nonparallel

diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
index 9f510958..17c64bbc 100644
--- a/cdist/conf/type/__uci_section/files/functions.sh
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -1,13 +1,6 @@
 # -*- mode: sh; indent-tabs-mode: t -*-
 
-append_values() {
-	while read -r _value
-	do
-		set -- "$@" --value "${_value}"
-	done
-	unset _value
-	"$@" </dev/null
-}
+NL=$(printf '\n '); NL=${NL% }
 
 grep_line() {
 	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
@@ -37,6 +30,28 @@ print_errors() {
 		}' >&2
 }
 
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf %s "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "$(printf "'%s'" "$(printf %s "${_arg}" | sed -e "s/'/'\\\\''/g")")"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
+	printf '%s' "$*"
+}
+
+uci_cmd() {
+	# Usage: uci_cmd [UCI ARGUMENTS]...
+	mkdir -p "${__object:?}/files"
+	printf '%s\n' "$(quote "$@")" >>"${__object:?}/files/uci_batch.txt"
+}
+
 uci_validate_name() {
 	# like util.c uci_validate_name()
 	test -n "$*" && test -z "$(printf %s "$*" | tr -d '[:alnum:]_' | tr -c '' .)"
diff --git a/cdist/conf/type/__uci_section/files/uci_apply.sh b/cdist/conf/type/__uci_section/files/uci_apply.sh
new file mode 120000
index 00000000..4209151f
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/uci_apply.sh
@@ -0,0 +1 @@
+../../__uci/files/uci_apply.sh
\ No newline at end of file
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
new file mode 100755
index 00000000..05bca0a7
--- /dev/null
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -0,0 +1,146 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# shellcheck source=files/functions.sh
+. "${__type:?}/files/functions.sh"
+
+
+section=$(cat "${__object:?}/explorer/match")
+
+state_is=$(test -s "${__object:?}/explorer/type" && echo present || echo absent)
+state_should=$(cat "${__object:?}/parameter/state")
+
+case $state_should
+in
+	(present)
+		test -f "${__object:?}/parameter/type" || {
+			echo 'Parameter --type is required.' >&2
+			exit 1
+		}
+
+		type_is=$(cat "${__object:?}/explorer/type")
+		type_should=$(cat "${__object:?}/parameter/type")
+
+		if test -n "${type_is}"
+		then
+			sect_type=${type_is}
+		else
+			sect_type=${type_should##*.}
+		fi
+
+		if test -z "${section}"
+		then
+			# No section exists and --match was used.
+			# So we generate a new section identifier from $__object_id.
+			case ${__object_id:?}
+			in
+				(*.*) section=${__object_id:?} ;;
+				(*) section="${type_should%%.*}.${__object_id:?}" ;;
+			esac
+		fi
+
+		# Collect option names
+		if test -f "${__object:?}/parameter/list"
+		then
+			listnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
+		fi
+
+		if test -f "${__object:?}/parameter/option"
+		then
+			optnames_should=$(
+				sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort -u)
+		fi
+
+		# Make sure the section itself is present
+		if test "${state_is}" = absent
+		then
+			printf 'set %s\n' "${section}" >>"${__messages_out:?}"
+			# shellcheck disable=SC2140
+			uci_cmd set "${section}"="${sect_type}"
+		fi
+
+		# Delete options/lists not in "should"
+		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		| while read -r _optname
+		  do
+			  grep_line "${_optname}" "${listnames_should}" "${optnames_should}" || {
+				  printf 'delete %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+				  uci_cmd delete "${section}.${_optname}"
+			  } </dev/null
+		  done
+
+		# Set "should" options
+		prefix_lines option "${optnames_should}" list "${listnames_should}" \
+		| while read -r _type _optname
+		  do
+			  test -n "${_type}${_optname}" || continue  # ignore empty lines
+
+			  case $_type
+			  in
+				  (list)
+					  printf 'set_list %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  if grep -q -e "^${_optname}=" "${__object:?}/explorer/options"
+					  then
+						  uci_cmd delete "${section}.${_optname}"
+					  fi
+
+					  grep "^${_optname}=" "${__object:?}/parameter/list" \
+					  | sed -e 's/^.*=//' \
+					  | unquote_lines \
+					  | while read -r _value
+					    do
+						    # shellcheck disable=SC2140
+						    uci_cmd add_list "${section}.${_optname}"="${_value}"
+					    done
+					  ;;
+				  (option)
+					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  # shellcheck disable=SC2140
+					  uci_cmd set "${section}.${_optname}"="$(
+						  grep "^${_optname}=" "${__object:?}/parameter/option" \
+						  | sed -e 's/^.*=//' \
+						  | unquote_lines \
+						  | head -n 1)"
+					  ;;
+			  esac
+		  done
+		;;
+	(absent)
+		if test "${state_is}" = absent
+		then
+			# if explorer found no section there is nothing to delete
+			exit 0
+		fi
+
+		printf 'delete %s\n' "${section}" >>"${__messages_out:?}"
+		uci_cmd delete "${section}"
+		;;
+esac
+
+if test -s "${__object:?}/files/uci_batch.txt"
+then
+	cat "${__type:?}/files/uci_apply.sh"
+	printf "uci_apply <<'EOF'\n"
+	cat "${__object:?}/files/uci_batch.txt"
+	printf '\nEOF\n'
+fi
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 41e30a07..7e23e443 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -43,8 +43,6 @@ in
 		;;
 esac
 
-section=$(cat "${__object:?}/explorer/match")
-
 state_should=$(cat "${__object:?}/parameter/state")
 
 case $state_should
@@ -57,85 +55,31 @@ in
 		type_is=$(cat "${__object:?}/explorer/type")
 		type_should=$(cat "${__object:?}/parameter/type")
 
-		if test -n "${type_is}"
+		if test -n "${type_is}" && test "${type_is}" != "${type_should##*.}"
 		then
-			if test "${type_is}" != "${type_should##*.}"
-			then
-				# Check if section type matches (section exists and --type provided)
-				printf 'Section type "%s" does not match --type "%s".\n' \
-					"${type_is}" "${type_should}" >&2
-				exit 1
-			fi
-			sect_type=${type_is}
-		else
-			sect_type=${type_should##*.}
-		fi
-
-		if test -z "${section}"
-		then
-			# No section exists and --match was used.
-			# So we generate a new section identifier from $__object_id.
-			case ${__object_id:?}
-			in
-				(*.*) section=${__object_id:?} ;;
-				(*) section="${type_should%%.*}.${__object_id:?}" ;;
-			esac
+			# Check if section type matches (section exists and --type provided)
+			printf 'Section type "%s" does not match --type "%s".\n' \
+				"${type_is}" "${type_should}" >&2
+			exit 1
 		fi
 
 		# Check options for syntax errors
 		validate_options "${__object:?}/parameter/list" "${__object:?}/parameter/object" \
 		| print_errors 'Found erroneous options in arguments:'
 
-		# Collect option names
-		if test -f "${__object:?}/parameter/list"
+		# Check for duplicate option names
+		if test -s "${__object:?}/parameter/option"
 		then
-			listnames_should=$(
-				sed -e 's/=.*$//' "${__object:?}/parameter/list" | sort -u)
-		fi
-
-		if test -f "${__object:?}/parameter/option"
-		then
-			optnames_should=$(sed -e 's/=.*$//' "${__object:?}/parameter/option" | sort)
-
-			echo "${optnames_should}" \
+			sed -e 's/=.*$//' "${__object:?}/parameter/option" \
+			| sort \
 			| uniq -d \
 			| print_errors \
 				  'Found duplicate --options:' \
 				  "$(printf '\nUse --list for lists, instead.')"
 		fi
-
-		# Make sure the section itself is present
-		__uci "${section}" --state present --value "${sect_type}"
-		export require=__uci/"${section}"
-
-		# Delete options/lists not in "should"
-		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
-		| while read -r optname
-		  do
-			  grep_line "${optname}" "${listnames_should}" "${optnames_should}" || {
-				  __uci "${section}.${optname}" --state absent
-			  } </dev/null
-		  done
-
-		# Set "should" options
-		prefix_lines option "${optnames_should}" list "${listnames_should}" \
-		| while read -r _type _optname
-		  do
-			  test -n "${_type}${_optname}" || continue  # ignore empty lines
-
-			  grep "^${_optname}=" <"${__object:?}/parameter/${_type}" \
-			  | sed -e 's/^.*=//' \
-			  | unquote_lines \
-			  | append_values \
-				    __uci "${section}.${_optname}" --state present \
-					    --type "${_type}"
-		  done
 		;;
 	(absent)
-		# if explorer found no section there is nothing to delete
-		test -n "${section}" || exit 0
-
-		__uci "${section}" --state absent
+		:
 		;;
 	(*)
 		printf 'Invalid --state: %s\n' "${state_should}" >&2
diff --git a/cdist/conf/type/__uci_section/nonparallel b/cdist/conf/type/__uci_section/nonparallel
new file mode 100644
index 00000000..e69de29b

From ade69729ddd64938ef69c08a522a85ec825628e9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 31 Oct 2020 18:18:15 +0100
Subject: [PATCH 107/366] [type/__uci_section] Only generate UCI commands if
 state differs

---
 .../conf/type/__uci_section/explorer/options  | 22 ++++-
 .../type/__uci_section/files/functions.sh     |  8 --
 .../type/__uci_section/files/option_state.awk | 91 +++++++++++++++++++
 cdist/conf/type/__uci_section/gencode-remote  | 68 ++++++++++----
 4 files changed, 160 insertions(+), 29 deletions(-)
 create mode 100644 cdist/conf/type/__uci_section/files/option_state.awk

diff --git a/cdist/conf/type/__uci_section/explorer/options b/cdist/conf/type/__uci_section/explorer/options
index 67b4f83f..e1e60668 100644
--- a/cdist/conf/type/__uci_section/explorer/options
+++ b/cdist/conf/type/__uci_section/explorer/options
@@ -25,4 +25,24 @@ section=$("${__type_explorer:?}/match")
 test -n "${section}" || exit 0
 
 uci -s -N -d "${RS}" show "${section}" 2>/dev/null \
-| grep -v -e "^${section}=" || true
+| awk -v VSEP="${RS}" '
+  {
+	  # Strip off the config and section parts
+	  is_opt = sub(/^([^.]*\.){2}/, "")
+
+	  if (!is_opt) {
+		  # this line represents the section -> skip
+		  next
+	  }
+
+	  if (index($0, VSEP)) {
+		  # Put values each on a line, like --option and --list parameters
+		  opt = substr($0, 1, index($0, "=") - 1)
+		  split(substr($0, length(opt) + 2), values, VSEP)
+		  for (i in values) {
+			  printf "%s=%s\n", opt, values[i]
+		  }
+	  } else {
+		  print
+	  }
+  }'
diff --git a/cdist/conf/type/__uci_section/files/functions.sh b/cdist/conf/type/__uci_section/files/functions.sh
index 17c64bbc..60cb9148 100644
--- a/cdist/conf/type/__uci_section/files/functions.sh
+++ b/cdist/conf/type/__uci_section/files/functions.sh
@@ -6,14 +6,6 @@ grep_line() {
 	{ shift; printf '%s\n' "$@"; } | grep -qxF "$1"
 }
 
-prefix_lines() {
-	while test $# -gt 0
-	do
-		echo "$2" | awk -v prefix="$1" '$0 { printf "%s %s\n", prefix, $0 }'
-		shift; shift
-	done
-}
-
 print_errors() {
 	awk -v prefix="${1:-Found errors:}" -v suffix="${2-}" '
 		BEGIN {
diff --git a/cdist/conf/type/__uci_section/files/option_state.awk b/cdist/conf/type/__uci_section/files/option_state.awk
new file mode 100644
index 00000000..97cd94fb
--- /dev/null
+++ b/cdist/conf/type/__uci_section/files/option_state.awk
@@ -0,0 +1,91 @@
+# -*- mode: awk; indent-tabs-mode:t -*-
+# Usage: awk -f option_state.awk option_type option_name
+# e.g. awk -f option_state.awk option title
+#      awk -f option_state.awk list entry
+
+function unquote(s) {
+	# simplified dequoting of single quoted strings
+	if (s ~ /^'.*'$/) {
+		s = substr(s, 2, length(s) - 2)
+		sub(/'\\''/, "'", s)
+	}
+	return s
+}
+
+function valueof(line) {
+	if (line !~ /^[[:alpha:]_]+=/) return 0
+	return unquote(substr(line, index(line, "=") + 1))
+}
+
+BEGIN {
+	__object = ENVIRON["__object"]
+	if (!__object) exit 1
+
+	opttype = ARGV[1]
+	optname = ARGV[2]
+
+	if (opttype !~ /^(option|list)/ || !optname) {
+		print "invalid"
+		exit (e=1)
+	}
+
+	ARGV[1] = __object "/parameter/" opttype
+	ARGV[2] = __object "/explorer/options"
+
+	state = "present"
+}
+
+NR == FNR {
+	# memoize "should" state
+	if (index($0, optname "=") == 1) {
+		should[++should_count] = valueof($0)
+	}
+
+	# go to next line (important!)
+	next
+}
+
+{
+	# compare "is" state
+	if (index($0, optname "=") != 1)
+		next
+	++is_count
+
+	v = valueof($0)
+
+	if (v == should[is_count]) {
+		# looks good, but can't say definitely just from this line
+	} else if (is_count > should_count) {
+		# there are more "is" records than "should" -> definitely different
+		state = "different"
+		exit
+	} else {
+		# see if we can find the "is" value somewhere in "should"
+		for (i in should) {
+			if (v == should[i]) {
+				# value found -> could be rearranged
+				# FIXME: Duplicate values are not properly handled here. Do they matter?
+				state = "rearranged"
+				next
+			}
+		}
+
+		# "is" value could not be found in "should" -> definitely different
+		state = "different"
+		exit
+	}
+}
+
+END {
+	if (e) exit
+
+	if (!is_count) {
+		# no "is" values -> absent
+		state = "absent"
+	} else if (is_count < should_count) {
+		# "is" was shorter than "should" -> different
+		state = "different"
+	}
+
+	print state
+}
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
index 05bca0a7..285659b0 100755
--- a/cdist/conf/type/__uci_section/gencode-remote
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -70,7 +70,8 @@ in
 		fi
 
 		# Make sure the section itself is present
-		if test "${state_is}" = absent
+		if test "${state_is}" = absent \
+			|| test "${type_is}" != "${type_should#*.}"
 		then
 			printf 'set %s\n' "${section}" >>"${__messages_out:?}"
 			# shellcheck disable=SC2140
@@ -78,7 +79,7 @@ in
 		fi
 
 		# Delete options/lists not in "should"
-		sed -e 's/=.*$//;s/^.*\.//' "${__object:?}/explorer/options" \
+		sed -e 's/=.*$//' "${__object:?}/explorer/options" \
 		| while read -r _optname
 		  do
 			  grep_line "${_optname}" "${listnames_should}" "${optnames_should}" || {
@@ -87,18 +88,55 @@ in
 			  } </dev/null
 		  done
 
-		# Set "should" options
-		prefix_lines option "${optnames_should}" list "${listnames_should}" \
-		| while read -r _type _optname
-		  do
-			  test -n "${_type}${_optname}" || continue  # ignore empty lines
+		opt_proc_error() {
+			printf 'An error occurred during processing of option %s\n' "${1:?}" >&2
+			exit 1
+		}
 
-			  case $_type
+		# Set "should" options
+		echo "${optnames_should}" \
+		| grep -e . \
+		| while read -r _optname
+		  do
+			  _opt_state=$(awk -f "${__type:?}/files/option_state.awk" option "${_optname}") \
+				  || opt_proc_error "${_optname}"
+			  case ${_opt_state}
 			  in
-				  (list)
+				  (invalid)
+					  opt_proc_error "${_optname}"
+					  ;;
+				  (present)
+					  ;;
+				  (*)
+					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
+
+					  # shellcheck disable=SC2140
+					  uci_cmd set "${section}.${_optname}"="$(
+						  grep -e "^${_optname}=" "${__object:?}/parameter/option" \
+						  | sed -e 's/^.*=//' \
+						  | unquote_lines \
+						  | head -n 1)"
+					  ;;
+			  esac
+		  done
+
+		echo "${listnames_should}" \
+		| grep -e . \
+		| while read -r _optname
+		  do
+			  _list_state=$(awk -f "${__type:?}/files/option_state.awk" list "${_optname}") \
+				  || opt_proc_error "${_optname}"
+			  case ${_list_state}
+			  in
+				  (invalid)
+					  opt_proc_error "${_optname}"
+					  ;;
+				  (present)
+					  ;;
+				  (*)
 					  printf 'set_list %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
 
-					  if grep -q -e "^${_optname}=" "${__object:?}/explorer/options"
+					  if test "${_list_state}" != absent
 					  then
 						  uci_cmd delete "${section}.${_optname}"
 					  fi
@@ -112,16 +150,6 @@ in
 						    uci_cmd add_list "${section}.${_optname}"="${_value}"
 					    done
 					  ;;
-				  (option)
-					  printf 'set %s\n' "${section}.${_optname}" >>"${__messages_out:?}"
-
-					  # shellcheck disable=SC2140
-					  uci_cmd set "${section}.${_optname}"="$(
-						  grep "^${_optname}=" "${__object:?}/parameter/option" \
-						  | sed -e 's/^.*=//' \
-						  | unquote_lines \
-						  | head -n 1)"
-					  ;;
 			  esac
 		  done
 		;;

From 2be8c634584561158960fc5cf2f363f3f5c4cc4c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 3 Nov 2020 06:43:57 +0100
Subject: [PATCH 108/366] pycodestyle fixes

---
 cdist/argparse.py         |  2 +-
 cdist/scan/commandline.py |  2 +-
 cdist/scan/scan.py        | 12 ++++++++----
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index ff195e8c..88759d7b 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -473,7 +473,7 @@ def get_parsers():
 
     # Scan = config + further
     parser['scan'] = parser['sub'].add_parser('scan', add_help=False,
-                                                 parents=[parser['config']])
+                                              parents=[parser['config']])
 
     parser['scan'] = parser['sub'].add_parser(
             'scan', parents=[parser['loglevel'],
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 0fb718e5..eca4cf13 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -37,7 +37,7 @@ def commandline(args):
 
     if not args.mode:
         # By default scan and trigger, but do not call any action
-        args.mode = ['scan', 'trigger' ]
+        args.mode = ['scan', 'trigger', ]
 
     if 'trigger' in args.mode:
         t = scan.Trigger(interfaces=args.interfaces)
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index fcbf1899..0ce4dff3 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -30,9 +30,12 @@
 # Scanner logic
 #  - save results to configdir:
 #     basedir = ~/.cdist/scan/<ipv6-address>
-#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time or similar
-#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
-#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record unix time or similar
+#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time
+#           or similar
+#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record
+#           unix time or similar
+#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record
+#           unix time or similar
 #
 #
 #
@@ -58,6 +61,7 @@ import datetime
 
 log = logging.getLogger("scan")
 
+
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
@@ -91,6 +95,7 @@ class Trigger(object):
         log.debug(f"Sending request on {interface}")
         send(packet, verbose=self.verbose)
 
+
 class Scanner(object):
     """
     Scan for replies of hosts, maintain the up-to-date database
@@ -134,7 +139,6 @@ class Scanner(object):
               prn=self.handle_pkg)
 
 
-
 if __name__ == '__main__':
     t = Trigger(interfaces=["wlan0"])
     t.start()

From df881c0f9859759c10339e9d4a5ac24ff77fbe1a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 4 Nov 2020 08:34:17 +0100
Subject: [PATCH 109/366] [type/__file] Fix --state pre-exists also for
 non-dry-runs

---
 cdist/conf/type/__file/gencode-remote | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote
index 2675b03a..f7a528fd 100755
--- a/cdist/conf/type/__file/gencode-remote
+++ b/cdist/conf/type/__file/gencode-remote
@@ -95,6 +95,10 @@ case "$state_should" in
         fi
     ;;
 
+    pre-exists)
+        :
+        ;;
+
     *)
         echo "Unknown state: $state_should" >&2
         exit 1

From d28a70a73c2affcf3268623a51df2b7a6cdd534a Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 6 Nov 2020 08:32:40 +0100
Subject: [PATCH 110/366] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 5977d80e..335a0f9f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,8 @@ next:
 	* Type __acl: Remove deprecated parameters, fix bugs (Ander Punnar)
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 	* Type __file: Fix state pre-exists (Dennis Camera)
+	* Type __apt_norecommends: Use 00InstallRecommends file as debian-installer does (Dennis Camera)
+	* Core: Introduce scanner (noninvasive, beta) (Nico Schottelius)
 
 6.8.0: 2020-09-11
 	* Type __locale_system: Fix for debian and ubuntu (Ander Punnar)

From 0f1df5ef68a60c5b8077f437bf0d07f1f79788e7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:07:58 +0100
Subject: [PATCH 111/366] Fix shellcheck source directives

---
 cdist/conf/type/__uci/gencode-remote         | 2 +-
 cdist/conf/type/__uci_section/gencode-remote | 2 +-
 cdist/conf/type/__uci_section/manifest       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__uci/gencode-remote b/cdist/conf/type/__uci/gencode-remote
index 348d15a1..70a3d3e0 100755
--- a/cdist/conf/type/__uci/gencode-remote
+++ b/cdist/conf/type/__uci/gencode-remote
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 state_is=$(cat "${__object:?}/explorer/state")
diff --git a/cdist/conf/type/__uci_section/gencode-remote b/cdist/conf/type/__uci_section/gencode-remote
index 285659b0..50fdfa4e 100755
--- a/cdist/conf/type/__uci_section/gencode-remote
+++ b/cdist/conf/type/__uci_section/gencode-remote
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci_section/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 
diff --git a/cdist/conf/type/__uci_section/manifest b/cdist/conf/type/__uci_section/manifest
index 7e23e443..0325b92c 100755
--- a/cdist/conf/type/__uci_section/manifest
+++ b/cdist/conf/type/__uci_section/manifest
@@ -18,7 +18,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# shellcheck source=files/functions.sh
+# shellcheck source=cdist/conf/type/__uci_section/files/functions.sh
 . "${__type:?}/files/functions.sh"
 
 

From c7c3075f629191ed7e05a76395b77dce01e2928f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:10:14 +0100
Subject: [PATCH 112/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 335a0f9f..40b816a6 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -13,6 +13,7 @@ next:
 	* Type __update_alternatives: Rewrite, support --install (Ander Punnar)
 	* Type __file: Fix state pre-exists (Dennis Camera)
 	* Type __apt_norecommends: Use 00InstallRecommends file as debian-installer does (Dennis Camera)
+	* New types: __uci, __uci_section (Dennis Camera)
 	* Core: Introduce scanner (noninvasive, beta) (Nico Schottelius)
 
 6.8.0: 2020-09-11

From 348c6eedc903ecd28b02c5dda3821e71ef255a4f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 7 Nov 2020 12:12:20 +0100
Subject: [PATCH 113/366] Release 6.9.0

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 40b816a6..c47123d9 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.0: 2020-11-07
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)
 	* Documentation: Add custom remote copy/exec examples (Darko Poljak)

From 10abe514b8fb7e6d3ce3587a55773a21e78ce9e6 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 11 Jun 2020 21:27:53 +0200
Subject: [PATCH 114/366] [type/__hostname] Add support for OpenWrt

---
 cdist/conf/type/__hostname/gencode-remote | 3 +++
 cdist/conf/type/__hostname/manifest       | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index ae224611..02afcbfb 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -69,6 +69,9 @@ in
     centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void)
         echo "hostname '$name_should'"
     ;;
+    openwrt)
+        echo "echo '$name_should' >/proc/sys/kernel/hostname"
+    ;;
     macosx)
         echo "scutil --set HostName '$name_should'"
     ;;
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index e1e356a0..bf8a331c 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -149,6 +149,10 @@ in
     openbsd)
         echo "$name_should" | __file /etc/myname --source -
     ;;
+    openwrt)
+        __uci system.@system[0].hostname --value "$name_should"
+            # --transaction hostname
+    ;;
     slackware)
         # We write the FQDN into /etc/HOSTNAME.  But /etc/rc.d/rc.M will only
         # read the first component from this file and set it as the running

From b0f3bb3350be9575c1632c6e3f669e7ede431020 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 7 Nov 2020 16:49:09 +0100
Subject: [PATCH 115/366] New type __dpkg_architecture

This type handles foreign architectures added to dpkg.
---
 .../__dpkg_architecture/explorer/architecture |   8 ++
 .../explorer/foreign-architectures            |   8 ++
 .../type/__dpkg_architecture/gencode-remote   |  64 +++++++++++
 cdist/conf/type/__dpkg_architecture/man.rst   | 103 ++++++++++++++++++
 .../parameter/default/state                   |   1 +
 .../__dpkg_architecture/parameter/optional    |   1 +
 6 files changed, 185 insertions(+)
 create mode 100755 cdist/conf/type/__dpkg_architecture/explorer/architecture
 create mode 100755 cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
 create mode 100755 cdist/conf/type/__dpkg_architecture/gencode-remote
 create mode 100644 cdist/conf/type/__dpkg_architecture/man.rst
 create mode 100644 cdist/conf/type/__dpkg_architecture/parameter/default/state
 create mode 100644 cdist/conf/type/__dpkg_architecture/parameter/optional

diff --git a/cdist/conf/type/__dpkg_architecture/explorer/architecture b/cdist/conf/type/__dpkg_architecture/explorer/architecture
new file mode 100755
index 00000000..4478bc6e
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/explorer/architecture
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+# __dpkg_architecture/explorer/architecture
+
+# Get the main architecture of this machine
+
+
+# print or die in the gencode-remote
+dpkg --print-architecture || true
diff --git a/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
new file mode 100755
index 00000000..ff01672a
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+# __dpkg_architecture/explorer/foreign-architectures
+
+# Print all additional architectures
+
+
+# print or die in the gencode-remote
+dpkg --print-foreign-architectures || true
diff --git a/cdist/conf/type/__dpkg_architecture/gencode-remote b/cdist/conf/type/__dpkg_architecture/gencode-remote
new file mode 100755
index 00000000..adf9d655
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/gencode-remote
@@ -0,0 +1,64 @@
+#!/bin/sh -e
+# __dpkg_architecture/gencode-remote
+
+
+# Get parameter and explorer
+state_should="$(cat "$__object/parameter/state")"
+arch_wanted="$__object_id"
+main_arch="$(cat "$__object/explorer/architecture")"
+
+# Exit here if dpkg do not work (empty explorer)
+if [ -z "$main_arch" ]; then
+    echo "dpkg is not available or unable to detect a architecture!" >&2
+    exit 1
+fi
+
+
+# Check if requested architecture is the main one
+if [ "$arch_wanted" = "$main_arch" ]; then
+    # higher than present; we can not remove it
+    state_is="present"
+    caution="yes"
+
+# Check if the architecture not already used
+elif grep -qFx "$arch_wanted" "$__object/explorer/foreign-architectures"; then
+    state_is="present"
+
+# arch does not exist
+else
+    state_is="absent"
+fi
+
+
+# Check what to do
+if [ "$state_is" != "$state_should" ]; then
+    case "$state_should" in
+        present)
+            # print add code
+            printf "dpkg --add-architecture '%s'\n" "$arch_wanted"
+            # updating the index to make the new architecture available
+            echo "apt update"
+
+            echo added >> "$__messages_out"
+            ;;
+
+        absent)
+            if [ "$caution" ]; then
+                printf "can not remove the main arch '%s' of the system!\n" "$main_arch" >&2
+                exit 1
+            fi
+
+            # removing all existing packages for the architecture
+            printf "apt purge '.*:%s'\n" "$arch_wanted"
+            # print remove code
+            printf "dpkg --remove-architecture '%s'\n" "$arch_wanted"
+
+            echo removed >> "$__messages_out"
+            ;;
+
+        *)
+            printf "state '%s' is unknown!\n" "$state_should" >&2
+            exit 1
+            ;;
+    esac
+fi
diff --git a/cdist/conf/type/__dpkg_architecture/man.rst b/cdist/conf/type/__dpkg_architecture/man.rst
new file mode 100644
index 00000000..fa196229
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/man.rst
@@ -0,0 +1,103 @@
+cdist-type__dpkg_architecture(7)
+================================
+
+NAME
+----
+cdist-type__dpkg_architecture - Handles foreign architectures on debian-like
+systems managed by `dpkg`
+
+
+DESCRIPTION
+-----------
+This type handles foreign architectures on systems managed by
+:strong:`dpkg`\ (1). The object id is the name of the architecture accepted by
+`dpkg`, which should be added or removed.
+
+If the architecture is not setup on the system, it adds a new architecture as a
+new foreign architecture in `dpkg`. Then, it updates the apt package index to
+make packages from the new architecture available.
+
+If the architecture should be removed, it will remove it if it is not the base
+architecture on where the system was installed on. Before it, it will purge
+every package based on the "to be removed" architecture via `apt` to be able to
+remove the selected architecture.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    ``present`` or ``absent``. Defaults to ``present``.
+
+
+MESSAGES
+--------
+added
+   Added the specified architecture
+
+removed
+   Removed the specified architecture
+
+
+ABORTS
+------
+Aborts in the following cases:
+
+If :strong:`dpkg`\ (1) is not available. It will abort with a proper error
+message.
+
+If the architecture is the same as the base architecture the system is build
+upon it (returned by ``dpkg --print-architecture``) and it should be removed.
+
+It will fail if it can not execute :strong:`apt`\ (8). It is assumed that it is
+already installed.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+  # add i386 (32 bit) architecture
+  __dpkg_architecture i386
+
+  # remove it again :)
+  __dpkg_architecture i386 --state absent
+
+
+SEE ALSO
+--------
+`Multiarch on Debian systems <https://wiki.debian.org/Multiarch>`_
+
+`How to setup multiarch on Debian <https://wiki.debian.org/Multiarch/HOWTO>`_
+
+:strong:`dpkg`\ (1)
+:strong:`cdist-type__package_dpkg`\ (7)
+:strong:`cdist-type__package_apt`\ (7)
+
+Useful commands:
+
+.. code-block:: sh
+
+   # base architecture installed on this system
+   dpkg --print-architecture
+
+   # extra architectures added
+   dpkg --print-foreign-architectures
+
+
+AUTHORS
+-------
+Matthias Stecher <matthiasstecher at gmx.de>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Matthias Stecher. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+ublished by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__dpkg_architecture/parameter/default/state b/cdist/conf/type/__dpkg_architecture/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__dpkg_architecture/parameter/optional b/cdist/conf/type/__dpkg_architecture/parameter/optional
new file mode 100644
index 00000000..ff72b5c7
--- /dev/null
+++ b/cdist/conf/type/__dpkg_architecture/parameter/optional
@@ -0,0 +1 @@
+state

From 7777580d8ff94f2b0725dbbcef84d8957998acb6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 7 Nov 2020 20:56:17 +0100
Subject: [PATCH 116/366] __dpkg_architecture: add copyright headers

---
 .../__dpkg_architecture/explorer/architecture  | 18 ++++++++++++++++++
 .../explorer/foreign-architectures             | 18 ++++++++++++++++++
 .../type/__dpkg_architecture/gencode-remote    | 18 ++++++++++++++++++
 3 files changed, 54 insertions(+)

diff --git a/cdist/conf/type/__dpkg_architecture/explorer/architecture b/cdist/conf/type/__dpkg_architecture/explorer/architecture
index 4478bc6e..03e7e386 100755
--- a/cdist/conf/type/__dpkg_architecture/explorer/architecture
+++ b/cdist/conf/type/__dpkg_architecture/explorer/architecture
@@ -1,5 +1,23 @@
 #!/bin/sh -e
 # __dpkg_architecture/explorer/architecture
+#
+# 2020 Matthias Stecher <matthiasstecher at gmx.de>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 # Get the main architecture of this machine
 
diff --git a/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
index ff01672a..a150d307 100755
--- a/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
+++ b/cdist/conf/type/__dpkg_architecture/explorer/foreign-architectures
@@ -1,5 +1,23 @@
 #!/bin/sh -e
 # __dpkg_architecture/explorer/foreign-architectures
+#
+# 2020 Matthias Stecher <matthiasstecher at gmx.de>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 # Print all additional architectures
 
diff --git a/cdist/conf/type/__dpkg_architecture/gencode-remote b/cdist/conf/type/__dpkg_architecture/gencode-remote
index adf9d655..47fb24e7 100755
--- a/cdist/conf/type/__dpkg_architecture/gencode-remote
+++ b/cdist/conf/type/__dpkg_architecture/gencode-remote
@@ -1,5 +1,23 @@
 #!/bin/sh -e
 # __dpkg_architecture/gencode-remote
+#
+# 2020 Matthias Stecher <matthiasstecher at gmx.de>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
 
 
 # Get parameter and explorer

From 91bcc2a2939508b8722c2a148c89c92aaa4f830f Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 7 Nov 2020 21:03:38 +0100
Subject: [PATCH 117/366] __dpkg_architecture: make type nonparallel

I think it's not good that dpkg or apt is running in parallel.
---
 cdist/conf/type/__dpkg_architecture/nonparallel | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 cdist/conf/type/__dpkg_architecture/nonparallel

diff --git a/cdist/conf/type/__dpkg_architecture/nonparallel b/cdist/conf/type/__dpkg_architecture/nonparallel
new file mode 100644
index 00000000..e69de29b

From 9fc6ee0948347a0de20e633ce4a0ee358dc92c00 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 8 Nov 2020 13:14:13 +0100
Subject: [PATCH 118/366] __package_apt: add --install-recommends parameter

For a good reason, __package_apt doesn't install recommended packages as
default. But the option --install-recommends comes handy if you want to
install a package where you want to install all recommended packages
(and not to install all of them separately).

Also, the manpage now explains that the type won't install recommended
packages by default.
---
 cdist/conf/type/__package_apt/gencode-remote    | 17 ++++++++++++-----
 cdist/conf/type/__package_apt/man.rst           | 15 +++++++++++++--
 cdist/conf/type/__package_apt/parameter/boolean |  1 +
 3 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote
index f3d91566..fbfca330 100755
--- a/cdist/conf/type/__package_apt/gencode-remote
+++ b/cdist/conf/type/__package_apt/gencode-remote
@@ -42,6 +42,13 @@ else
    target_release=""
 fi
 
+if [ -f "$__object/parameter/install-recommends" ]; then
+    # required if __apt_norecommends is used
+    recommendsparam="-o APT::Install-Recommends=1"
+else
+    recommendsparam="-o APT::Install-Recommends=0"
+fi
+
 if [ -f "$__object/parameter/purge-if-absent" ]; then
 	purgeparam="--purge"
 else
@@ -62,16 +69,16 @@ case "$state_is" in
     ;;
 esac
 
-# Hint if we need to avoid questions at some point:
-# DEBIAN_PRIORITY=critical can reduce the number of questions
-aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o APT::Install-Recommends=0 -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
-
 if [ "$state_is" = "$state_should" ]; then
     if [ -z "$version" ] || [ "$version" = "$version_is" ]; then
         exit 0;
     fi
 fi
 
+# Hint if we need to avoid questions at some point:
+# DEBIAN_PRIORITY=critical can reduce the number of questions
+aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
+
 case "$state_should" in
     present)
         # following is bit ugly, but important hack.
@@ -85,7 +92,7 @@ EOF
         if [ -n "$version" ]; then
             name="${name}=${version}"
         fi
-        echo "$aptget install $target_release '$name'"
+        echo "$aptget $recommendsparam install $target_release '$name'"
         echo "installed" >> "$__messages_out"
     ;;
     absent)
diff --git a/cdist/conf/type/__package_apt/man.rst b/cdist/conf/type/__package_apt/man.rst
index a1691eac..4e6101a5 100644
--- a/cdist/conf/type/__package_apt/man.rst
+++ b/cdist/conf/type/__package_apt/man.rst
@@ -9,7 +9,9 @@ cdist-type__package_apt - Manage packages with apt-get
 DESCRIPTION
 -----------
 apt-get is usually used on Debian and variants (like Ubuntu) to
-manage packages.
+manage packages. The package will be installed without recommended
+or suggested packages. If such packages are required, install them
+separatly or use the parameter ``--install-recommends``.
 
 This type will also update package index, if it is older
 than one day, to avoid missing package error messages.
@@ -23,7 +25,7 @@ None
 OPTIONAL PARAMETERS
 -------------------
 name
-   If supplied, use the name and not the object id as the package name.
+    If supplied, use the name and not the object id as the package name.
 
 state
     Either "present" or "absent", defaults to "present"
@@ -39,6 +41,15 @@ version
 
 BOOLEAN PARAMETERS
 ------------------
+install-recommends
+    If the package will be installed, it also installs recommended packages
+    with it. It will not install recommended packages if the original package
+    is already installed.
+
+    In most cases, it is recommended to install recommended packages separatly
+    to control which additional packages will be installed to avoid useless
+    installed packages.
+
 purge-if-absent
     If this parameter is given when state is `absent`, the package is
     purged from the system (using `--purge`).
diff --git a/cdist/conf/type/__package_apt/parameter/boolean b/cdist/conf/type/__package_apt/parameter/boolean
index f9a0f6b0..a2e433f3 100644
--- a/cdist/conf/type/__package_apt/parameter/boolean
+++ b/cdist/conf/type/__package_apt/parameter/boolean
@@ -1 +1,2 @@
+install-recommends
 purge-if-absent

From fded60bd0f4ca7b6b9047c9f3b1470fdbd20c263 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 8 Nov 2020 13:27:01 +0100
Subject: [PATCH 119/366] ++changelog

---
 docs/changelog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index c47123d9..f271142a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,12 @@
 Changelog
 ---------
 
+next:
+	* Type __file: Fix state pre-exists (Dennis Camera)
+	* Type __hostname: Add support for OpenWrt (Dennis Camera)
+	* New type: __dpkg_architecture (Matthias Stecher)
+	* Type __package_apt: Add --install-recommends parameter (Matthias Stecher)
+
 6.9.0: 2020-11-07
 	* Core: Clarify stdin input (Darko Poljak)
 	* Type __package_pip: Detect pip binary (Ander Punnar)

From d2506ac04eac987caf0fe0c33f48866ee6e43c0d Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 8 Nov 2020 13:31:57 +0100
Subject: [PATCH 120/366] Release 6.9.1

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index f271142a..b6b8c22b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)
 	* Type __hostname: Add support for OpenWrt (Dennis Camera)
 	* New type: __dpkg_architecture (Matthias Stecher)

From a95eab77a5606f2552ac28c0b817945f72538c97 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 8 Nov 2020 15:18:04 +0100
Subject: [PATCH 121/366] __locale: add state explorer

.. so it doesn't execute code all the time.
---
 cdist/conf/type/__locale/explorer/state | 36 +++++++++++++++++++++++++
 cdist/conf/type/__locale/gencode-remote | 15 ++++++++---
 2 files changed, 47 insertions(+), 4 deletions(-)
 create mode 100755 cdist/conf/type/__locale/explorer/state

diff --git a/cdist/conf/type/__locale/explorer/state b/cdist/conf/type/__locale/explorer/state
new file mode 100755
index 00000000..4494fcbc
--- /dev/null
+++ b/cdist/conf/type/__locale/explorer/state
@@ -0,0 +1,36 @@
+#!/bin/sh -e
+# __locale/explorer/state
+#
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Check if the locale is already installed on the system.
+# Outputs 'present' or 'absent' depending if the locale exists.
+#
+
+
+# Get user-defined locale
+# locale name is echoed differently than the user propably set it (for UTF-8)
+locale="$(echo "$__object_id" | sed 's/UTF-8/utf8/')"
+
+# Check if the given locale exists on the system
+if localedef --list-archive | grep -qFx "$locale"; then
+    echo present
+else
+    echo absent
+fi
diff --git a/cdist/conf/type/__locale/gencode-remote b/cdist/conf/type/__locale/gencode-remote
index 1feb9884..4639cef8 100755
--- a/cdist/conf/type/__locale/gencode-remote
+++ b/cdist/conf/type/__locale/gencode-remote
@@ -23,6 +23,15 @@
 
 locale="$__object_id"
 
+state_is=$(cat "$__object/explorer/state")
+state_should=$(cat "$__object/parameter/state")
+
+# short circuit if there is nothing to do
+if [ "$state_is" = "$state_should" ]; then
+    exit 0
+fi
+
+
 # Hardcoded, create a pull request with
 # branching on $os in case it is at another location
 alias=/usr/share/locale/locale.alias
@@ -35,8 +44,6 @@ charmap=$(echo "$locale" | cut -d . -f 2)
 # W-T-F!
 locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
 
-state=$(cat "$__object/parameter/state")
-
 os=$(cat "$__global/explorer/os")
 
 # Nothing to be done on alpine
@@ -46,7 +53,7 @@ case "$os" in
         ;;
 esac
 
-case "$state" in
+case "$state_should" in
     present)
         echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
     ;;
@@ -54,7 +61,7 @@ case "$state" in
         echo localedef --delete-from-archive "$locale_remove"
     ;;
     *)
-        echo "Unsupported state: $state" >&2
+        echo "Unsupported state: $state_should" >&2
         exit 1
     ;;
 esac

From 792b4b10762c094d861282e78953dfd29b203a41 Mon Sep 17 00:00:00 2001
From: ssrq <ungleich@ssrq-sds-fds.ch>
Date: Mon, 9 Nov 2020 12:08:54 +0100
Subject: [PATCH 122/366] Add missing 'config' command

---
 docs/src/cdist-best-practice.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/src/cdist-best-practice.rst b/docs/src/cdist-best-practice.rst
index 39ec453e..f7677276 100644
--- a/docs/src/cdist-best-practice.rst
+++ b/docs/src/cdist-best-practice.rst
@@ -200,15 +200,15 @@ of cdist:
 .. code-block:: sh
 
     # Singleton type without parameter
-    echo __ungleich_munin_server | cdist --initial-manifest - munin.panter.ch
+    echo __ungleich_munin_server | cdist config --initial-manifest - munin.panter.ch
 
     # Singleton type with parameter
     echo __ungleich_munin_node --allow 1.2.3.4 | \
-        cdist --initial-manifest - rails-19.panter.ch
+        cdist config --initial-manifest - rails-19.panter.ch
 
     # Normal type
     echo __file /tmp/stdintest --mode 0644 | \
-        cdist --initial-manifest - cdist-dev-01.ungleich.ch
+        cdist config --initial-manifest - cdist-dev-01.ungleich.ch
 
 
 Other content in cdist repository

From ba906510529549446952bb3abfd8b49b05da7ca0 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 11 Nov 2020 07:49:32 +0100
Subject: [PATCH 123/366] ++changelog

---
 docs/changelog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index b6b8c22b..9fa0926c 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,11 @@
 Changelog
 ---------
 
+next:
+	* Documentation: Fix examples in best practice (Dennis Camera)
+	* Type __locale: Add state explorer (Matthias Stecher)
+	* Core: Reorganize scripts, version generation (Ander Punnar)
+
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)
 	* Type __hostname: Add support for OpenWrt (Dennis Camera)

From 3e48ef9e11795bf1c8a59ae202ed28f58cff1753 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:09:26 +0100
Subject: [PATCH 124/366] [type/__hostname] Lint

- Error if expected environment variables are unset
- Always wrap variable expansions in {}
---
 cdist/conf/type/__hostname/gencode-remote |  82 ++++++------
 cdist/conf/type/__hostname/manifest       | 147 +++++++++++-----------
 2 files changed, 114 insertions(+), 115 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 02afcbfb..9447daa9 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -20,26 +20,27 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$(cat "$__global/explorer/os")
-name_running=$(cat "$__global/explorer/hostname")
-has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
+os=$(cat "${__global:?}/explorer/os")
+name_running=$(cat "${__global:?}/explorer/hostname")
+has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl")
 
 
-if test -s "$__object/parameter/name"
+if test -s "${__object:?}/parameter/name"
 then
-    name_should=$(cat "$__object/parameter/name")
+    name_should=$(cat "${__object:?}/parameter/name")
 else
-    case $os
+    case ${os}
     in
         # RedHat-derivatives and BSDs
-        centos|fedora|redhat|scientific|freebsd|macosx|netbsd|openbsd)
+        (centos|fedora|redhat|scientific|freebsd|macosx|netbsd|openbsd)
             # Hostname is FQDN
-            name_should="${__target_host}"
-        ;;
-        *)
+            name_should=${__target_host:?}
+            ;;
+        (*)
             # Hostname is only first component of FQDN
-            name_should="${__target_host%%.*}"
-        ;;
+            name_should=${__target_host:?}
+            name_should=${name_should%%.*}
+            ;;
     esac
 fi
 
@@ -47,46 +48,46 @@ fi
 ################################################################################
 # Check if the (running) hostname is already correct
 #
-test "$name_running" != "$name_should" || exit 0
+test "${name_running}" != "${name_should}" || exit 0
 
 
 ################################################################################
 # Setup hostname
 #
-echo 'changed' >>"$__messages_out"
+echo 'changed' >>"${__messages_out:?}"
 
 # Use the good old way to set the hostname.
-case $os
+case ${os}
 in
-    alpine|debian|devuan|ubuntu)
+    (alpine|debian|devuan|ubuntu)
         echo 'hostname -F /etc/hostname'
-    ;;
-    archlinux)
+        ;;
+    (archlinux)
         echo 'command -v hostnamectl >/dev/null 2>&1' \
-            "&& hostnamectl set-hostname '$name_should'" \
-            "|| hostname '$name_should'"
-    ;;
-    centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void)
-        echo "hostname '$name_should'"
-    ;;
-    openwrt)
-        echo "echo '$name_should' >/proc/sys/kernel/hostname"
-    ;;
-    macosx)
-        echo "scutil --set HostName '$name_should'"
-    ;;
-    solaris)
-        echo "uname -S '$name_should'"
-    ;;
-    slackware|suse|opensuse-leap)
+            "&& hostnamectl set-hostname '${name_should}'" \
+            "|| hostname '${name_should}'"
+        ;;
+    (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void)
+        echo "hostname '${name_should}'"
+        ;;
+    (openwrt)
+        echo "echo '${name_should}' >/proc/sys/kernel/hostname"
+        ;;
+    (macosx)
+        echo "scutil --set HostName '${name_should}'"
+        ;;
+    (solaris)
+        echo "uname -S '${name_should}'"
+        ;;
+    (slackware|suse|opensuse-leap)
         # We do not read from /etc/HOSTNAME, because the running
         # hostname is the first component only while the file contains
         # the FQDN.
-        echo "hostname '$name_should'"
-    ;;
-    *)
+        echo "hostname '${name_should}'"
+        ;;
+    (*)
         # Fall back to set the hostname using hostnamectl, if available.
-        if test -n "$has_hostnamectl"
+        if test -n "${has_hostnamectl}"
         then
             # Don't use hostnamectl as the primary means to set the hostname for
             # systemd systems, because it cannot be trusted to work reliably and
@@ -97,7 +98,8 @@ in
             echo "test \"\$(hostname)\" = \"\$(cat /etc/hostname)\"" \
                  " || hostname -F /etc/hostname"
         else
-            printf "echo 'Unsupported OS: %s' >&2\nexit 1\n" "$os"
+            printf "echo 'Unsupported OS: %s' >&2\n" "${os}"
+            printf 'exit 1\n'
         fi
-    ;;
+        ;;
 esac
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index bf8a331c..125bb2fb 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -20,69 +20,65 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-not_supported() {
-    echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
-    echo "Please contribute an implementation for it if you can." >&2
-    exit 1
-}
-
 set_hostname_systemd() {
     echo "$1" | __file /etc/hostname --source -
 }
 
-os=$(cat "$__global/explorer/os")
-os_version=$(cat "$__global/explorer/os_version")
-os_major=$(echo "$os_version" | grep -o '^[0-9][0-9]*' || true)
+os=$(cat "${__global:?}/explorer/os")
+os_version=$(cat "${__global:?}/explorer/os_version")
+os_major=$(echo "${os_version}" | grep -o '^[0-9][0-9]*' || true)
 
-max_len=$(cat "$__object/explorer/max_len")
-has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl")
+max_len=$(cat "${__object:?}/explorer/max_len")
+has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl")
 
-if test -s "$__object/parameter/name"
+if test -s "${__object:?}/parameter/name"
 then
-    name_should=$(cat "$__object/parameter/name")
+    name_should=$(cat "${__object:?}/parameter/name")
 else
-    case $os
+    case ${os}
     in
         # RedHat-derivatives and BSDs
-        centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware)
+        (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware)
             # Hostname is FQDN
-            name_should="${__target_host}"
-        ;;
-        suse|opensuse-leap)
+            name_should=${__target_host:?}
+            ;;
+        (suse|opensuse-leap)
+            name_should=${__target_host:?}
+
             # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
             # systemd does not. The running hostname is the first
             # component in both cases.
             # In versions before 15.x, the FQDN is stored in /etc/hostname.
-            if test -n "$has_hostnamectl" && test "$os_major" -ge 15 \
-                    && test "$os_major" -ne 42
+            if test -n "${has_hostnamectl}" \
+                    && test "${os_major}" -ge 15 \
+                    && test "${os_major}" -ne 42
             then
-                name_should="${__target_host%%.*}"
-            else
-                name_should="${__target_host}"
+                name_should=${name_should%%.*}
             fi
-        ;;
+            ;;
         *)
             # Hostname is only first component of FQDN on all other systems.
-            name_should="${__target_host%%.*}"
-        ;;
+            name_should=${__target_host:?}
+            name_should=${name_should%%.*}
+            ;;
     esac
 fi
 
-if test -n "$max_len" && test "$(printf '%s' "$name_should" | wc -c)" -gt "$max_len"
+if test -n "${max_len}" && test "$(printf '%s' "${name_should}" | wc -c)" -gt "${max_len}"
 then
     printf "Host name too long. Up to %u characters allowed.\n" "${max_len}" >&2
     exit 1
 fi
 
-case $os
+case ${os}
 in
-    alpine|debian|devuan|ubuntu|void)
-        echo "$name_should" | __file /etc/hostname --source -
-    ;;
-    archlinux)
-        if test -n "$has_hostnamectl"
+    (alpine|debian|devuan|ubuntu|void)
+        echo "${name_should}" | __file /etc/hostname --source -
+        ;;
+    (archlinux)
+        if test -n "${has_hostnamectl}"
         then
-            set_hostname_systemd "$name_should"
+            set_hostname_systemd "${name_should}"
         else
             echo 'Ancient ArchLinux variants without hostnamectl are not supported.' >&2
             exit 1
@@ -97,8 +93,8 @@ in
             #     --value "\"$name_should\""
         fi
         ;;
-    centos|fedora|redhat|scientific)
-        if test -z "$has_hostnamectl"
+    (centos|fedora|redhat|scientific)
+        if test -z "${has_hostnamectl}"
         then
             # Only write to /etc/sysconfig/network on non-systemd versions.
             # On systemd-based versions this entry is ignored.
@@ -106,63 +102,62 @@ in
                 --file /etc/sysconfig/network \
                 --delimiter '=' --exact_delimiter \
                 --key HOSTNAME \
-                --value "\"$name_should\""
+                --value "\"${name_should}\""
         else
-            set_hostname_systemd "$name_should"
+            set_hostname_systemd "${name_should}"
         fi
-    ;;
-    gentoo)
+        ;;
+    (gentoo)
         # Only write to /etc/conf.d/hostname on OpenRC-based installations.
         # On systemd use hostnamectl(1) in gencode-remote.
-        if test -z "$has_hostnamectl"
+        if test -z "${has_hostnamectl}"
         then
             __key_value '/etc/conf.d/hostname:hostname' \
                 --file /etc/conf.d/hostname \
                 --delimiter '=' --exact_delimiter \
                 --key 'hostname' \
-                --value "\"$name_should\""
+                --value "\"${name_should}\""
         else
             set_hostname_systemd "$name_should"
         fi
-    ;;
-    freebsd)
+        ;;
+    (freebsd)
         __key_value '/etc/rc.conf:hostname' \
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
-            --value "\"$name_should\""
-    ;;
-    macosx)
+            --value "\"${name_should}\""
+        ;;
+    (macosx)
         # handled in gencode-remote
-        :
-    ;;
-    netbsd)
+        ;;
+    (netbsd)
         __key_value '/etc/rc.conf:hostname' \
             --file /etc/rc.conf \
             --delimiter '=' --exact_delimiter \
             --key 'hostname' \
-            --value "\"$name_should\""
+            --value "\"${name_should}\""
 
         # To avoid confusion, ensure that the hostname is only stored once.
         __file /etc/myname --state absent
-    ;;
-    openbsd)
-        echo "$name_should" | __file /etc/myname --source -
-    ;;
-    openwrt)
-        __uci system.@system[0].hostname --value "$name_should"
+        ;;
+    (openbsd)
+        echo "${name_should}" | __file /etc/myname --source -
+        ;;
+    (openwrt)
+        __uci system.@system[0].hostname --value "${name_should}"
             # --transaction hostname
-    ;;
-    slackware)
+        ;;
+    (slackware)
         # We write the FQDN into /etc/HOSTNAME.  But /etc/rc.d/rc.M will only
         # read the first component from this file and set it as the running
         # hostname on boot.
-        echo "$name_should" | __file /etc/HOSTNAME --source -
-    ;;
-    solaris)
-        echo "$name_should" | __file /etc/nodename --source -
-    ;;
-    suse|opensuse-leap)
+        echo "${name_should}" | __file /etc/HOSTNAME --source -
+        ;;
+    (solaris)
+        echo "${name_should}" | __file /etc/nodename --source -
+        ;;
+    (suse|opensuse-leap)
         # Modern SuSE provides /etc/HOSTNAME as a symlink for
         # backwards-compatibility. Unfortunately it cannot be used
         # here as __file does not follow the symlink.
@@ -171,23 +166,25 @@ in
         # not work correctly on openSUSE 12.x which provides
         # hostnamectl but not /etc/hostname.
 
-        if test -n "$has_hostnamectl" -a "$os_major" -gt 12
+        if test -n "${has_hostnamectl}" -a "${os_major}" -gt 12
         then
-            hostname_file='/etc/hostname'
+            hostname_file=/etc/hostname
         else
-            hostname_file='/etc/HOSTNAME'
+            hostname_file=/etc/HOSTNAME
         fi
 
-        echo "$name_should" | __file "$hostname_file" --source -
-    ;;
-    *)
+        echo "${name_should}" | __file "${hostname_file}" --source -
+        ;;
+    (*)
         # On other operating systems we fall back to systemd's
         # hostnamectl if available…
-        if test -n "$has_hostnamectl"
+        if test -n "${has_hostnamectl}"
         then
-            set_hostname_systemd "$name_should"
+            set_hostname_systemd "${name_should}"
         else
-            not_supported
+            echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
+            echo "Please contribute an implementation for it if you can." >&2
+            exit 1
         fi
-    ;;
+        ;;
 esac

From 702f3eba4f8f88b634a010b377d077d501a6f0e9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:15:02 +0100
Subject: [PATCH 125/366] [type/__hostname] Remove opensuse-leap OS string
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

everything should be suse now…
---
 cdist/conf/type/__hostname/gencode-remote | 2 +-
 cdist/conf/type/__hostname/manifest       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote
index 9447daa9..c1a97ac8 100755
--- a/cdist/conf/type/__hostname/gencode-remote
+++ b/cdist/conf/type/__hostname/gencode-remote
@@ -79,7 +79,7 @@ in
     (solaris)
         echo "uname -S '${name_should}'"
         ;;
-    (slackware|suse|opensuse-leap)
+    (slackware|suse)
         # We do not read from /etc/HOSTNAME, because the running
         # hostname is the first component only while the file contains
         # the FQDN.
diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index 125bb2fb..d044072b 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -42,7 +42,7 @@ else
             # Hostname is FQDN
             name_should=${__target_host:?}
             ;;
-        (suse|opensuse-leap)
+        (suse)
             name_should=${__target_host:?}
 
             # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
@@ -157,7 +157,7 @@ in
     (solaris)
         echo "${name_should}" | __file /etc/nodename --source -
         ;;
-    (suse|opensuse-leap)
+    (suse)
         # Modern SuSE provides /etc/HOSTNAME as a symlink for
         # backwards-compatibility. Unfortunately it cannot be used
         # here as __file does not follow the symlink.

From 87a0d91587568594a616c7f3588bc74106312761 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:20:18 +0100
Subject: [PATCH 126/366] [type/__hostname] Fix OS version detection for SuSE
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

everything should be suse now…
---
 cdist/conf/type/__hostname/manifest | 39 ++++++++++++++++-------------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest
index d044072b..b80aa2ef 100755
--- a/cdist/conf/type/__hostname/manifest
+++ b/cdist/conf/type/__hostname/manifest
@@ -25,8 +25,6 @@ set_hostname_systemd() {
 }
 
 os=$(cat "${__global:?}/explorer/os")
-os_version=$(cat "${__global:?}/explorer/os_version")
-os_major=$(echo "${os_version}" | grep -o '^[0-9][0-9]*' || true)
 
 max_len=$(cat "${__object:?}/explorer/max_len")
 has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl")
@@ -38,24 +36,10 @@ else
     case ${os}
     in
         # RedHat-derivatives and BSDs
-        (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware)
+        (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware|suse)
             # Hostname is FQDN
             name_should=${__target_host:?}
             ;;
-        (suse)
-            name_should=${__target_host:?}
-
-            # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
-            # systemd does not. The running hostname is the first
-            # component in both cases.
-            # In versions before 15.x, the FQDN is stored in /etc/hostname.
-            if test -n "${has_hostnamectl}" \
-                    && test "${os_major}" -ge 15 \
-                    && test "${os_major}" -ne 42
-            then
-                name_should=${name_should%%.*}
-            fi
-            ;;
         *)
             # Hostname is only first component of FQDN on all other systems.
             name_should=${__target_host:?}
@@ -158,6 +142,27 @@ in
         echo "${name_should}" | __file /etc/nodename --source -
         ;;
     (suse)
+        if test -s "${__global:?}/explorer/os_release"
+        then
+            # shellcheck source=/dev/null
+            os_version=$(. "${__global:?}/explorer/os_release" && echo "${VERSION}")
+        else
+            os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global:?}/explorer/os_version")
+        fi
+        os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
+
+        # Classic SuSE stores the FQDN in /etc/HOSTNAME, while
+        # systemd does not. The running hostname is the first
+        # component in both cases.
+        # In versions before 15.x, the FQDN is stored in /etc/hostname.
+        if test -n "${has_hostnamectl}" \
+                && test "${os_major}" -ge 15 \
+                && test "${os_major}" -ne 42
+        then
+            # strip away everything but the first part from $name_should
+            name_should=${name_should%%.*}
+        fi
+
         # Modern SuSE provides /etc/HOSTNAME as a symlink for
         # backwards-compatibility. Unfortunately it cannot be used
         # here as __file does not follow the symlink.

From 21dd500c05c91c4927b2a4166690da65b1c98f97 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:44:44 +0100
Subject: [PATCH 127/366] Make pycodestyle pipeline happy

---
 cdist/__init__.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cdist/__init__.py b/cdist/__init__.py
index 1c60ae0f..44366cd0 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -33,14 +33,15 @@ try:
     import cdist.version
     VERSION = cdist.version.VERSION
 except ModuleNotFoundError:
-    cdist_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+    cdist_dir = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), os.pardir))
     if os.path.isdir(os.path.join(cdist_dir, '.git')):
         try:
             VERSION = subprocess.check_output(
                 ['git', 'describe', '--always'],
                 cwd=cdist_dir,
                 universal_newlines=True)
-        except:
+        except Exception:
             pass
 
 BANNER = """

From e2d4f8037ab7d30187e4d153aa25ef36e8afc91c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:45:05 +0100
Subject: [PATCH 128/366] [bin/cdist-build-helper] Fix paths to ex scripts/
 scripts

---
 bin/cdist-build-helper | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper
index bdef0dbb..fb8a67a4 100755
--- a/bin/cdist-build-helper
+++ b/bin/cdist-build-helper
@@ -45,7 +45,7 @@ usage() {
         shellcheck-manifests
         shellcheck-local-gencodes
         shellcheck-remote-gencodes
-        shellcheck-scripts
+        shellcheck-bin
         shellcheck-gencodes
         shellcheck-types
         shellcheck
@@ -100,7 +100,7 @@ case "$option" in
                     if (\$0 ~ /^$end/) {
                         exit
                     } else {
-                        print \$0 
+                        print \$0
                     }
                 }
             }" "$basedir/docs/changelog"
@@ -135,7 +135,7 @@ case "$option" in
 
         version=$1; shift
 
-        ( 
+        (
         cat << eof
 Subject: cdist $version has been released
 
@@ -336,7 +336,7 @@ eof
         make docs-clean
         make docs
 
-        ############################################################# 
+        #############################################################
         # Everything green, let's do the release
 
         # Tag the current commit
@@ -405,7 +405,7 @@ eof
     ;;
 
     pycodestyle|pep8)
-        pycodestyle "${basedir}" "${basedir}/scripts/cdist"
+        pycodestyle "${basedir}" "${basedir}/bin/cdist"
     ;;
 
     check-pycodestyle)
@@ -460,9 +460,10 @@ eof
         test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
-    shellcheck-scripts)
+    # NOTE: shellcheck-scripts is kept for compatibility
+    shellcheck-bin|shellcheck-scripts)
         # shellcheck disable=SC2086
-        ${SHELLCHECKCMD} scripts/cdist-dump scripts/cdist-new-type > "${SHELLCHECKTMP}"
+        ${SHELLCHECKCMD} bin/cdist-dump bin/cdist-new-type > "${SHELLCHECKTMP}"
         test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
     ;;
 
@@ -480,7 +481,7 @@ eof
     shellcheck)
         "$0" shellcheck-global-explorers || exit 1
         "$0" shellcheck-types || exit 1
-        "$0" shellcheck-scripts || exit 1
+        "$0" shellcheck-bin || exit 1
     ;;
 
     shellcheck-type-files)

From f82e0167aaf295b8451a07fa931506a1290efcdc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 14:49:04 +0100
Subject: [PATCH 129/366] [.gitlab-ci.yml] Make version before other targets

---
 .gitlab-ci.yml | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e48355ea..a4bc67aa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,20 +1,23 @@
+---
+image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
+
 stages:
     - test
 
-image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
+before_script:
+    - ./bin/cdist-build-helper version
 
-unit_tests:
+shellcheck:
     stage: test
     script:
-        - ./bin/cdist-build-helper version
-        - ./bin/cdist-build-helper test
+        - ./bin/cdist-build-helper shellcheck
 
 pycodestyle:
     stage: test
     script:
         - ./bin/cdist-build-helper pycodestyle
 
-shellcheck:
+unit_tests:
     stage: test
     script:
-        - ./bin/cdist-build-helper shellcheck
+        - ./bin/cdist-build-helper test

From 0ee3fda94deff1461ac488d218a2af98057675c5 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 15:04:51 +0100
Subject: [PATCH 130/366] Fix paths to cdist executable

---
 cdist/integration.py          | 5 ++---
 cdist/test/__init__.py        | 2 +-
 cdist/test/config/__init__.py | 4 ++--
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/cdist/integration.py b/cdist/integration.py
index 03e4167d..17b65f09 100644
--- a/cdist/integration.py
+++ b/cdist/integration.py
@@ -54,13 +54,12 @@ _mydir = os.path.dirname(__file__)
 def find_cdist_exec():
     """Search cdist executable starting from local lib directory.
 
-    Detect if ../scripts/cdist (from local lib direcotry) exists and
+    Detect if ../bin/cdist (from local lib directory) exists and
     if it is executable. If not then try to find cdist exec path in
     os.get_exec_path() entries. If no cdist path is found rasie
     cdist.Error.
     """
-    cdist_path = os.path.abspath(os.path.join(_mydir, '..', 'scripts',
-                                              'cdist'))
+    cdist_path = os.path.abspath(os.path.join(_mydir, '..', 'bin', 'cdist'))
     if os.access(cdist_path, os.X_OK):
         return cdist_path
     cdist_path = find_cdist_exec_in_path()
diff --git a/cdist/test/__init__.py b/cdist/test/__init__.py
index faa3686a..16a311e4 100644
--- a/cdist/test/__init__.py
+++ b/cdist/test/__init__.py
@@ -26,7 +26,7 @@ import tempfile
 cdist_base_path = os.path.abspath(
     os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../"))
 
-cdist_exec_path = os.path.join(cdist_base_path, "scripts/cdist")
+cdist_exec_path = os.path.join(cdist_base_path, "bin/cdist")
 
 global_fixtures_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                    "fixtures"))
diff --git a/cdist/test/config/__init__.py b/cdist/test/config/__init__.py
index 0ed614b1..c405207c 100644
--- a/cdist/test/config/__init__.py
+++ b/cdist/test/config/__init__.py
@@ -202,7 +202,7 @@ class ConfigRunTestCase(test.CdistTestCase):
             host_dir_name=self.hostdir,
             # exec_path can not derivated from sys.argv in case of unittest
             exec_path=os.path.abspath(os.path.join(
-                my_dir, '../../../scripts/cdist')),
+                my_dir, '../../../bin/cdist')),
             initial_manifest=os.path.join(fixtures,
                                           'manifest/dryrun_manifest'),
             add_conf_dirs=[fixtures])
@@ -219,7 +219,7 @@ class ConfigRunTestCase(test.CdistTestCase):
             base_root_path=self.host_base_path,
             host_dir_name=self.hostdir,
             exec_path=os.path.abspath(os.path.join(
-                my_dir, '../../../scripts/cdist')),
+                my_dir, '../../../bin/cdist')),
             initial_manifest=os.path.join(
                 fixtures, 'manifest/init-deps-resolver'),
             add_conf_dirs=[fixtures])

From c39eb1dbce281ed6f64f804a16fb28e5c8dceccd Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 15:16:33 +0100
Subject: [PATCH 131/366] [cdist.emulator] Fix setting of log level (tests OK)

---
 cdist/emulator.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cdist/emulator.py b/cdist/emulator.py
index 60b94880..a2bdc3d4 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -120,17 +120,18 @@ class Emulator:
                 level = logging.WARNING
         else:
             level = logging.WARNING
+        self.log = logging.getLogger(self.target_host[0])
         try:
             logging.root.setLevel(level)
+            self.log.setLevel(level)
         except (ValueError, TypeError):
             # if invalid __cdist_log_level value
             logging.root.setLevel(logging.WARNING)
+            self.log.setLevel(logging.WARNING)
 
         colored_log = self.env.get('__cdist_colored_log', 'false')
         cdist.log.CdistFormatter.USE_COLORS = colored_log == 'true'
 
-        self.log = logging.getLogger(self.target_host[0])
-
     def commandline(self):
         """Parse command line"""
 

From 2f70a8b540f8d4c283029207e98f7883557338f0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 11 Nov 2020 15:25:46 +0100
Subject: [PATCH 132/366] [bin/cdist-build-helper] Keep going in shellcheck
 targets

---
 bin/cdist-build-helper | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper
index fb8a67a4..0380b3f8 100755
--- a/bin/cdist-build-helper
+++ b/bin/cdist-build-helper
@@ -468,20 +468,26 @@ eof
     ;;
 
     shellcheck-gencodes)
-        "$0" shellcheck-local-gencodes || exit 1
-        "$0" shellcheck-remote-gencodes || exit 1
+        errors=false
+        "$0" shellcheck-local-gencodes || errors=true
+        "$0" shellcheck-remote-gencodes || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck-types)
-        "$0" shellcheck-type-explorers || exit 1
-        "$0" shellcheck-manifests || exit 1
-        "$0" shellcheck-gencodes || exit 1
+        errors=false
+        "$0" shellcheck-type-explorers || errors=true
+        "$0" shellcheck-manifests || errors=true
+        "$0" shellcheck-gencodes || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck)
-        "$0" shellcheck-global-explorers || exit 1
-        "$0" shellcheck-types || exit 1
-        "$0" shellcheck-bin || exit 1
+        errors=false
+        "$0" shellcheck-global-explorers || errors=true
+        "$0" shellcheck-types || errors=true
+        "$0" shellcheck-bin || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck-type-files)
@@ -491,8 +497,10 @@ eof
     ;;
 
     shellcheck-with-files)
-        "$0" shellcheck || exit 1
-        "$0" shellcheck-type-files || exit 1
+        errors=false
+        "$0" shellcheck || errors=true
+        "$0" shellcheck-type-files || errors=true
+        ! $errors || exit 1
     ;;
 
     shellcheck-build-helper)

From ebf471e8d0216cc81928fcdff1fe159584187808 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 13 Nov 2020 02:32:45 +0100
Subject: [PATCH 133/366] [type/__hwclock] Add new type

---
 .../conf/type/__hwclock/explorer/adjtime_mode |  28 +++
 .../__hwclock/explorer/timedatectl_localrtc   |  27 +++
 cdist/conf/type/__hwclock/gencode-remote      |  62 +++++
 cdist/conf/type/__hwclock/man.rst             |  63 +++++
 cdist/conf/type/__hwclock/manifest            | 222 ++++++++++++++++++
 cdist/conf/type/__hwclock/parameter/required  |   1 +
 cdist/conf/type/__hwclock/singleton           |   0
 7 files changed, 403 insertions(+)
 create mode 100755 cdist/conf/type/__hwclock/explorer/adjtime_mode
 create mode 100755 cdist/conf/type/__hwclock/explorer/timedatectl_localrtc
 create mode 100755 cdist/conf/type/__hwclock/gencode-remote
 create mode 100644 cdist/conf/type/__hwclock/man.rst
 create mode 100755 cdist/conf/type/__hwclock/manifest
 create mode 100644 cdist/conf/type/__hwclock/parameter/required
 create mode 100644 cdist/conf/type/__hwclock/singleton

diff --git a/cdist/conf/type/__hwclock/explorer/adjtime_mode b/cdist/conf/type/__hwclock/explorer/adjtime_mode
new file mode 100755
index 00000000..2b27bedc
--- /dev/null
+++ b/cdist/conf/type/__hwclock/explorer/adjtime_mode
@@ -0,0 +1,28 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Prints the clock mode read from the /etc/adjtime file, if present.
+#
+
+# not all operating systems use an adjfile
+test -f /etc/adjtime || exit 0
+
+# 3rd line is clock mode
+# adjtime(5) https://man7.org/linux/man-pages/man5/adjtime.5.html
+sed -n 3p /etc/adjtime
diff --git a/cdist/conf/type/__hwclock/explorer/timedatectl_localrtc b/cdist/conf/type/__hwclock/explorer/timedatectl_localrtc
new file mode 100755
index 00000000..8239122e
--- /dev/null
+++ b/cdist/conf/type/__hwclock/explorer/timedatectl_localrtc
@@ -0,0 +1,27 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Prints the LocalRTC property using timedatectl on systemd-based systems.
+#
+
+command -v timedatectl >/dev/null 2>&1 || exit 0
+
+# NOTE: Older versions of timedatectl do not support `timedatectl show'
+timedatectl --no-pager status \
+| awk -F': ' '$1 ~ "RTC in local TZ$" { sub(/[ \t]*$/, "", $2); print $2 }'
diff --git a/cdist/conf/type/__hwclock/gencode-remote b/cdist/conf/type/__hwclock/gencode-remote
new file mode 100755
index 00000000..5995fb23
--- /dev/null
+++ b/cdist/conf/type/__hwclock/gencode-remote
@@ -0,0 +1,62 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+mode=$(cat "${__object:?}/parameter/mode")
+
+timedatectl_localrtc=$(cat "${__object:?}/explorer/timedatectl_localrtc")
+adjtime_mode=$(cat "${__object:?}/explorer/adjtime_mode")
+
+
+case ${mode}
+in
+	(localtime)
+		adjtime_str=LOCAL
+		local_rtc_str=yes
+		;;
+	(UTC|utc)
+		adjtime_str=UTC
+		local_rtc_str=no
+		;;
+	(*)
+		printf 'Invalid value for --mode: %s\n' "${mode}" >&2
+		printf 'Acceptable values are: localtime, utc.\n' >&2
+		exit 1
+esac
+
+
+if test -n "${timedatectl_localrtc}"
+then
+	# systemd
+	timedatectl_should=${local_rtc_str}
+	if test "${timedatectl_localrtc}" != "${timedatectl_should}"
+	then
+		printf 'timedatectl set-local-rtc %s\n' "${timedatectl_should}"
+	fi
+elif test -n "${adjtime_mode}"
+then
+	# others (update /etc/adjtime if present)
+	if test "${adjtime_mode}" != "${adjtime_str}"
+	then
+		# Update /etc/adjtime (3rd line is clock mode)
+		# adjtime(5) https://man7.org/linux/man-pages/man5/adjtime.5.html
+		# FIXME: Should maybe add third line if adjfile only contains two lines
+		printf "sed -i '3c\\\\\\n%s\\n' /etc/adjtime\\n" "${adjtime_str}"
+	fi
+fi
diff --git a/cdist/conf/type/__hwclock/man.rst b/cdist/conf/type/__hwclock/man.rst
new file mode 100644
index 00000000..65eb648f
--- /dev/null
+++ b/cdist/conf/type/__hwclock/man.rst
@@ -0,0 +1,63 @@
+cdist-type__hwclock(7)
+======================
+
+NAME
+----
+cdist-type__hwclock - Manage the hardware real time clock.
+
+
+DESCRIPTION
+-----------
+This type can be used to control how the hardware clock is used by the operating
+system.
+
+
+REQUIRED PARAMETERS
+-------------------
+mode
+    What mode the hardware clock is in.
+
+    Acceptable values:
+
+    localtime
+        The hardware clock is set to local time (common for systems also running
+        Windows.)
+    UTC
+        The hardware clock is set to UTC (common on UNIX systems.)
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Make the operating system treat the time read from the hwclock as UTC.
+    __hwclock --mode UTC
+
+
+SEE ALSO
+--------
+:strong:`hwclock`\ (8)
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__hwclock/manifest b/cdist/conf/type/__hwclock/manifest
new file mode 100755
index 00000000..7d9ab88f
--- /dev/null
+++ b/cdist/conf/type/__hwclock/manifest
@@ -0,0 +1,222 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# TODO: Consider supporting BADYEAR
+
+os=$(cat "${__global:?}/explorer/os")
+mode=$(cat "${__object:?}/parameter/mode")
+
+has_systemd_timedatectl=$(test -s "${__object:?}/explorer/timedatectl_localrtc" && echo true || echo false)
+
+
+case ${mode}
+in
+	(localtime)
+		local_clock=true
+		;;
+	(UTC|utc)
+		local_clock=false
+		;;
+	(*)
+		printf 'Invalid value for --mode: %s\n' "${mode}" >&2
+		printf 'Acceptable values are: UTC, localtime.\n' >&2
+		exit 1
+esac
+
+
+case ${os}
+in
+	(alpine|gentoo)
+		if ! $has_systemd_timedatectl
+		then
+			# NOTE: Gentoo also supports systemd, in which case /etc/conf.d is
+			#       not used. So we check for systemd presence here and only
+			#       update /etc/conf.d if systemd is not installed.
+			# https://wiki.gentoo.org/wiki/System_time#Hardware_clock
+
+			export CDIST_ORDER_DEPENDENCY=true
+			__file /etc/conf.d/hwclock --state present \
+				--owner root --group root --mode 0644
+			__key_value /etc/conf.d/hwclock:clock \
+				--file /etc/conf.d/hwclock \
+				--key clock \
+				--delimiter '=' --exact_delimiter \
+				--value "\"$($local_clock && echo local || echo UTC)\""
+			unset CDIST_ORDER_DEPENDENCY
+		fi
+		;;
+	(centos|fedora|redhat|scientific)
+		os_version=$(cat "${__global:?}/explorer/os_version")
+		os_major=$(expr "${os_version}" : '.* release \([0-9]*\)')
+		case ${os}
+		in
+			(centos|scientific)
+				update_sysconfig=$(test "${os_major}" -lt 6 && echo true || echo false)
+				;;
+			(fedora)
+				update_sysconfig=$(test "${os_major}" -lt 10 && echo true || echo false)
+				;;
+			(redhat|*)
+				case ${os_version}
+				in
+					('Red Hat Enterprise Linux'*)
+						update_sysconfig=$(test "${os_major}" -lt 6 && echo true || echo false)
+						;;
+					('Red Hat Linux'*)
+						update_sysconfig=true
+						;;
+					(*)
+						printf 'Could not determine Red Hat distribution.\n' >&2
+						printf "Please contribute an implementation for it if you can.\n" >&2
+						exit 1
+						;;
+				esac
+				;;
+		esac
+
+		if ${update_sysconfig:?}
+		then
+			export CDIST_ORDER_DEPENDENCY=true
+			__file /etc/sysconfig/clock --state present \
+				--owner root --group root --mode 0644
+			__key_value /etc/sysconfig/clock:UTC \
+				--file /etc/sysconfig/clock \
+				--key UTC \
+				--delimiter '=' --exact_delimiter \
+				--value "$($local_clock && echo false || echo true)"
+			unset CDIST_ORDER_DEPENDENCY
+		fi
+		;;
+	(debian|devuan|ubuntu)
+		os_major=$(sed 's/[^0-9].*$//' "${__global:?}/explorer/os_version")
+
+		case ${os}
+		in
+			(debian)
+				if test "${os_major}" -ge 7
+				then
+					update_rcS=false
+				elif test "${os_major}" -ge 3
+				then
+					update_rcS=true
+				else
+					# Debian 2.2 should be supportable using rcS.
+					# Debian 2.1 uses the ancient GMT key.
+					# Debian 1.3 does not have rcS.
+					printf "Your operating system (Debian %s) is currently not supported by this type (%s)\n" \
+						"$(cat "${__global:?}/explorer/os_version")" "${__type##*/}" >&2
+					printf "Please contribute an implementation for it if you can.\n" >&2
+					exit 1
+				fi
+				;;
+			(devuan)
+				update_rcS=false
+				;;
+			(ubuntu)
+				update_rcS=$(test "${os_major}" -lt 16 && echo true || echo false)
+				;;
+		esac
+
+		if ${update_rcS}
+		then
+			export CDIST_ORDER_DEPENDENCY=true
+			__file /etc/default/rcS --state present \
+				--owner root --group root --mode 0644
+			__key_value /etc/default/rcS:UTC \
+				--file /etc/default/rcS \
+				--key UTC \
+				--delimiter '=' --exact_delimiter \
+				--value "$($local_clock && echo no || echo yes)"
+			unset CDIST_ORDER_DEPENDENCY
+		fi		  
+		;;
+	(freebsd)
+		# cf. adjkerntz(8)
+		__file /etc/wall_cmos_clock \
+			--state "$($local_clock && echo present || echo absent)" \
+			--owner root --group wheel --mode 0444
+		;;
+	(netbsd)
+		# https://wiki.netbsd.org/guide/boot/#index9h2
+		__key_value /etc/rc.conf:rtclocaltime \
+			--file /etc/rc.conf \
+			--key rtclocaltime \
+			--delimiter '=' --exact_delimiter \
+			--value "$($local_clock && echo YES || echo NO)"
+		;;
+	(slackware)
+		__file /etc/hardwareclock --owner root --group root --mode 0644 \
+			--source - <<-EOF
+		# /etc/hardwareclock
+		#
+		# Tells how the hardware clock time is stored.
+		# This file is managed by cdist.
+
+		$($local_clock && echo localtime || echo UTC)
+		EOF
+		;;
+	(suse)
+		if test -s "${__global:?}/explorer/os_release"
+		then
+			# shellcheck source=/dev/null
+			os_version=$(. "${__global:?}/explorer/os_release" && echo "${VERSION}")
+		else
+			os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global:?}/explorer/os_version")
+		fi
+		os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
+
+		# TODO: Consider using `yast2 timezone set hwclock' instead
+		if expr "${os_major}" \< 12
+		then
+			# Starting with SuSE 12 (first systemd-based version)
+			# /etc/sysconfig/clock does not contain the HWCLOCK line
+			# anymore.
+			# With SuSE 13, it has been reduced to TIMEZONE configuration.
+			__key_value /etc/sysconfig/clock:HWCLOCK \
+				--file /etc/sysconfig/clock \
+				--delimiter '=' --exact_delimiter \
+				--key HWCLOCK \
+				--value "$($local_clock && echo '"--localtime"' || echo '"-u"')"
+		fi
+		;;
+	(void)
+		export CDIST_ORDER_DEPENDENCY=true
+		__file /etc/rc.conf \
+			--owner root --group root --mode 0644 \
+			--state present
+		__key_value /etc/rc.conf:HARDWARECLOCK \
+			--file /etc/rc.conf \
+			--delimiter '=' --exact_delimiter \
+			--key HARDWARECLOCK \
+			--value "\"$($local_clock && echo localtime || echo UTC)\""
+		unset CDIST_ORDER_DEPENDENCY
+		;;
+	(*)
+		if ! $has_systemd_timedatectl
+		then
+			printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2
+			printf "Please contribute an implementation for it if you can.\n" >&2
+			exit 1
+		fi
+		;;
+esac
+
+# NOTE: timedatectl set-local-rtc for systemd is in gencode-remote
+# NOTE: /etc/adjtime is also updated in gencode-remote
diff --git a/cdist/conf/type/__hwclock/parameter/required b/cdist/conf/type/__hwclock/parameter/required
new file mode 100644
index 00000000..17ab372f
--- /dev/null
+++ b/cdist/conf/type/__hwclock/parameter/required
@@ -0,0 +1 @@
+mode
diff --git a/cdist/conf/type/__hwclock/singleton b/cdist/conf/type/__hwclock/singleton
new file mode 100644
index 00000000..e69de29b

From a07a45887136105cc8be316307805ceaecdd4cf3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 13 Nov 2020 06:43:01 +0100
Subject: [PATCH 134/366] ++changelog

---
 docs/changelog | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 9fa0926c..ff411a46 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,7 +4,8 @@ Changelog
 next:
 	* Documentation: Fix examples in best practice (Dennis Camera)
 	* Type __locale: Add state explorer (Matthias Stecher)
-	* Core: Reorganize scripts, version generation (Ander Punnar)
+	* Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera)
+	* New type: __hwclock (Dennis Camera)
 
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)

From 50927527863fd382f5200287a808d8b36028ff87 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 14 Nov 2020 09:59:30 +0100
Subject: [PATCH 135/366] Update build helper script in .gitattributes

---
 .gitattributes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitattributes b/.gitattributes
index 45c10d7b..01d20f30 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -4,5 +4,5 @@
 docs/speeches export-ignore
 docs/video export-ignore
 docs/src/man7 export-ignore
-bin/build-helper export-ignore
+bin/cdist-build-helper export-ignore
 README-maintainers export-ignore

From 76aa00b12e9b675c797e997a06ec7597c53a85f3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 14 Nov 2020 10:13:58 +0100
Subject: [PATCH 136/366] Fix importing cdist module

Resolve #845.
---
 bin/cdist | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index 1f92f157..ddaffa7f 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -25,25 +25,23 @@ import logging
 import os
 import sys
 
-# try to import cdist and if that fails,
-# then add this file's parent dir to
-# module search path and try again.
-try:
-    import cdist
-except ModuleNotFoundError:
-    cdist_dir = os.path.realpath(
-        os.path.join(
-            os.path.dirname(os.path.realpath(__file__)),
-            os.pardir))
+# See if this file's parent is cdist module
+# and if so add it to module search path.
+cdist_dir = os.path.realpath(
+    os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        os.pardir))
+cdist_init_dir = os.path.join(cdist_dir, 'cdist', '__init__.py')
+if os.path.exists(cdist_init_dir):
     sys.path.insert(0, cdist_dir)
-    import cdist
 
-import cdist.argparse
-import cdist.banner
-import cdist.config
-import cdist.install
-import cdist.shell
-import cdist.inventory
+import cdist            # noqa 402
+import cdist.argparse   # noqa 402
+import cdist.banner     # noqa 402
+import cdist.config     # noqa 402
+import cdist.install    # noqa 402
+import cdist.shell      # noqa 402
+import cdist.inventory  # noqa 402
 
 
 def commandline():

From f75d4772090d3a9cfd798dd193cd324083a9b3a7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 13:45:12 +0100
Subject: [PATCH 137/366] Deprecate __locale and replace with __localedef

---
 cdist/conf/type/__locale/deprecated           |  1 +
 cdist/conf/type/__localedef/gencode-remote    | 60 +++++++++++++++++++
 cdist/conf/type/__localedef/man.rst           | 55 +++++++++++++++++
 cdist/conf/type/__localedef/manifest          | 41 +++++++++++++
 .../type/__localedef/parameter/default/state  |  1 +
 .../conf/type/__localedef/parameter/optional  |  1 +
 6 files changed, 159 insertions(+)
 create mode 100644 cdist/conf/type/__locale/deprecated
 create mode 100755 cdist/conf/type/__localedef/gencode-remote
 create mode 100644 cdist/conf/type/__localedef/man.rst
 create mode 100755 cdist/conf/type/__localedef/manifest
 create mode 100644 cdist/conf/type/__localedef/parameter/default/state
 create mode 100644 cdist/conf/type/__localedef/parameter/optional

diff --git a/cdist/conf/type/__locale/deprecated b/cdist/conf/type/__locale/deprecated
new file mode 100644
index 00000000..5a06b28e
--- /dev/null
+++ b/cdist/conf/type/__locale/deprecated
@@ -0,0 +1 @@
+This type is deprecated. Please use __localedef instead.
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
new file mode 100755
index 00000000..1feb9884
--- /dev/null
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -0,0 +1,60 @@
+#!/bin/sh -e
+#
+# 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Let localedef do the magic
+#
+
+locale="$__object_id"
+
+# Hardcoded, create a pull request with
+# branching on $os in case it is at another location
+alias=/usr/share/locale/locale.alias
+
+input=$(echo "$locale" | cut -d . -f 1)
+charmap=$(echo "$locale" | cut -d . -f 2)
+
+# Adding locale? The name is de_CH.UTF-8
+# Removing locale? The name is de_CH.utf8.
+# W-T-F!
+locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
+
+state=$(cat "$__object/parameter/state")
+
+os=$(cat "$__global/explorer/os")
+
+# Nothing to be done on alpine
+case "$os" in
+    alpine)
+        exit 0
+        ;;
+esac
+
+case "$state" in
+    present)
+        echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
+    ;;
+    absent)
+        echo localedef --delete-from-archive "$locale_remove"
+    ;;
+    *)
+        echo "Unsupported state: $state" >&2
+        exit 1
+    ;;
+esac
diff --git a/cdist/conf/type/__localedef/man.rst b/cdist/conf/type/__localedef/man.rst
new file mode 100644
index 00000000..0ccf484a
--- /dev/null
+++ b/cdist/conf/type/__localedef/man.rst
@@ -0,0 +1,55 @@
+cdist-type__localedef(7)
+========================
+
+NAME
+----
+cdist-type__localedef - Define and remove system locales
+
+
+DESCRIPTION
+-----------
+This cdist type allows you to define locales on the system using
+:strong:`localedef`\ (1) or remove them.
+On systems that don't support definition of new locales, the type will raise an
+error.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+   ``present`` or ``absent``. Defaults to ``present``.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Add locale de_CH.UTF-8
+    __localedef de_CH.UTF-8
+
+    # Same as above, but more explicit
+    __localedef de_CH.UTF-8 --state present
+
+    # Remove colourful British English
+    __localedef en_GB.UTF-8 --state absent
+
+
+SEE ALSO
+--------
+:strong:`locale`\ (1),
+:strong:`localedef`\ (1),
+:strong:`cdist-type__locale_system`\ (7)
+
+
+AUTHORS
+-------
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+
+
+COPYING
+-------
+Copyright \(C) 2013-2019 Nico Schottelius, 2020 Dennis Camera. Free use of this
+software is granted under the terms of the GNU General Public License version 3
+or later (GPLv3+).
diff --git a/cdist/conf/type/__localedef/manifest b/cdist/conf/type/__localedef/manifest
new file mode 100755
index 00000000..9f1e17ac
--- /dev/null
+++ b/cdist/conf/type/__localedef/manifest
@@ -0,0 +1,41 @@
+#!/bin/sh -e
+#
+# 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2015 David Hürlimann (david at ungleich.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Install required packages
+#
+
+os=$(cat "$__global/explorer/os")
+
+
+case "$os" in
+    debian|devuan)
+        # Debian needs a seperate package
+        __package locales --state present
+    ;;
+    archlinux|suse|ubuntu|scientific|centos|alpine)
+        :
+    ;;
+    *)
+        echo "Sorry, do not know how to handle os: $os" >&2
+        echo "Please edit the type ${__type##*/} to fix this." >&2
+        exit 1
+    ;;
+esac
diff --git a/cdist/conf/type/__localedef/parameter/default/state b/cdist/conf/type/__localedef/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__localedef/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__localedef/parameter/optional b/cdist/conf/type/__localedef/parameter/optional
new file mode 100644
index 00000000..ff72b5c7
--- /dev/null
+++ b/cdist/conf/type/__localedef/parameter/optional
@@ -0,0 +1 @@
+state

From 54e689f7c2b3bc25aedb2aaf9f5aece289ec5b66 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 13:49:04 +0100
Subject: [PATCH 138/366] [type/__localedef] Add state explorer

---
 cdist/conf/type/__localedef/explorer/state | 71 ++++++++++++++++++++++
 cdist/conf/type/__localedef/gencode-remote | 13 +++-
 2 files changed, 82 insertions(+), 2 deletions(-)
 create mode 100755 cdist/conf/type/__localedef/explorer/state

diff --git a/cdist/conf/type/__localedef/explorer/state b/cdist/conf/type/__localedef/explorer/state
new file mode 100755
index 00000000..d8db29c5
--- /dev/null
+++ b/cdist/conf/type/__localedef/explorer/state
@@ -0,0 +1,71 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# This explorer determines if the locale is defined on the target system.
+# Will print nothing on error.
+#
+# Possible output:
+#  present:
+#    the main locale (and possibly aliases) is present
+#  absent:
+#    neither the main locale nor any aliases are present
+#
+
+command -v locale >/dev/null 2>&1 || exit 0
+
+locales=$(locale -a)
+
+parse_locale() {
+	# This function will split locales into their parts. Locale strings are
+	# usually of the form: [language[_territory][.codeset][@modifier]]
+	# For simplicity, language and territory are not separated by this function.
+	# Old Linux systems were also using "english" or "german" as locale strings.
+	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
+	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
+	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
+	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
+}
+
+format_locale() {
+	# Usage: format_locale language codeset modifier
+	printf '%s' "$1"
+	test -z "$2" || printf '.%s' "$2"
+	test -z "$3" || printf '@%s' "$3"
+	printf '\n'
+}
+
+gnu_normalize_codeset() {
+	# reimplementation of glibc/locale/programs/localedef.c normalize_codeset()
+	echo "$*" | tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]'
+}
+
+locale_available() (
+	echo "${locales}" | grep -qxF "$1" || {
+		# glibc uses "normalized" locale names in archives.
+		# If a locale is stored in an archive, the normalized name will be
+		# printed by locale, so that needs to be checked, too.
+		localename=$(
+			parse_locale "$1" _lang _codeset _modifier \
+			&& format_locale "${_lang:?}" "$(gnu_normalize_codeset "${_codeset?}")" \
+			   "${_modifier?}")
+		echo "${locales}" | grep -qxF "${localename}"
+	}
+)
+
+locale_available "${__object_id:?}" && echo present || echo absent
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 1feb9884..9ea3f6f1 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -35,7 +35,8 @@ charmap=$(echo "$locale" | cut -d . -f 2)
 # W-T-F!
 locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
 
-state=$(cat "$__object/parameter/state")
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 
 os=$(cat "$__global/explorer/os")
 
@@ -46,7 +47,15 @@ case "$os" in
         ;;
 esac
 
-case "$state" in
+# NOTE: If state explorer fails (e.g. locale(1) missing), the following check
+#       will always fail and let definition/removal run.
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
+
+case ${state_should}
+in
     present)
         echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
     ;;

From cc29e54b851212568c3f6eddf4bf0a64c1eabb0d Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 23 Jul 2020 23:20:39 +0200
Subject: [PATCH 139/366] [type/__localedef] Differentiate between OSes and
 better handling of normalized locale names

---
 cdist/conf/type/__localedef/gencode-remote | 117 +++++++++++++++------
 1 file changed, 82 insertions(+), 35 deletions(-)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 9ea3f6f1..b9cc68e8 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,35 +18,16 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+# Manage system locales using localedef(1).
 #
-# Let localedef do the magic
-#
-
-locale="$__object_id"
-
-# Hardcoded, create a pull request with
-# branching on $os in case it is at another location
-alias=/usr/share/locale/locale.alias
-
-input=$(echo "$locale" | cut -d . -f 1)
-charmap=$(echo "$locale" | cut -d . -f 2)
-
-# Adding locale? The name is de_CH.UTF-8
-# Removing locale? The name is de_CH.utf8.
-# W-T-F!
-locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/')
 
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
-os=$(cat "$__global/explorer/os")
-
-# Nothing to be done on alpine
-case "$os" in
-    alpine)
-        exit 0
-        ;;
-esac
+test "${state_should}" = 'present' -o "${state_should}" = 'absent' || {
+	printf 'Invalid state: %s\n' "${state_should}" >&2
+	exit 1
+}
 
 # NOTE: If state explorer fails (e.g. locale(1) missing), the following check
 #       will always fail and let definition/removal run.
@@ -54,16 +36,81 @@ then
 	exit 0
 fi
 
-case ${state_should}
+locale=${__object_id:?}
+os=$(cat "${__global:?}/explorer/os")
+
+if expr "${locale}" : '.*/' >/dev/null
+then
+	printf 'Paths as locales are not supported.\n' >&2
+	printf '__object_id is: %s\n' "${locale}" >&2
+	exit 1
+fi
+
+parse_locale() {
+	# This function will split locales into their parts. Locale strings are
+	# usually of the form: [language[_territory][.codeset][@modifier]]
+	# For simplicity, language and territory are not separated by this function.
+	# Old Linux systems were also using "english" or "german" as locale strings.
+	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
+	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
+	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
+	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
+}
+
+format_locale() {
+	# Usage: format_locale language codeset modifier
+	printf '%s' "$1"
+	test -z "$2" || printf '.%s' "$2"
+	test -z "$3" || printf '@%s' "$3"
+	printf '\n'
+}
+
+gnu_normalize_codeset() {
+	echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]'
+}
+
+
+: "${lang=}" "${codeset=}" "${modifier=}"  # declare variables for shellcheck
+parse_locale "${locale}" lang codeset modifier
+
+
+case ${os}
 in
-    present)
-        echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale"
-    ;;
-    absent)
-        echo localedef --delete-from-archive "$locale_remove"
-    ;;
-    *)
-        echo "Unsupported state: $state" >&2
-        exit 1
-    ;;
+	(alpine|openwrt)
+		printf '%s does not support locales.\n' "${os}" >&2
+		exit 1
+		;;
+	(archlinux|debian|devuan|ubuntu|suse|centos|fedora|redhat|scientific)
+		# FIXME: The code below only works for glibc-based installations.
+
+		# NOTE: Hardcoded, create a pull request in case it is at another
+		#       location for some opther distro.
+		# NOTE: locale.alias can be symlinked (e.g. Debian)
+		aliasfile='/usr/share/locale/locale.alias'
+
+		case ${state_should}
+		in
+			(present)
+				input=$(format_locale "${lang}" '' "${modifier}")
+				cat <<-EOF
+				set --
+				if test -e '${aliasfile}'
+				then
+					set -- -A '${aliasfile}'
+				fi
+
+				localedef -i '${input}' -f '${codeset}' "\$@" '${locale}'
+				EOF
+				;;
+			(absent)
+				localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}")
+				printf "localedef --delete-from-archive '%s'\n" "${localename}"
+				;;
+		esac
+		;;
+	(*)
+		echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
+		echo "Please contribute an implementation for it if you can." >&2
+		exit 1
+		;;
 esac

From f44888f192d78d8ce1e3583f70f94b0c2039c1e2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 14:02:08 +0100
Subject: [PATCH 140/366] [type/__localedef] Only install dependencies in
 manifest. OS checking moved to gencode-remote

---
 cdist/conf/type/__localedef/manifest | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__localedef/manifest b/cdist/conf/type/__localedef/manifest
index 9f1e17ac..3ab3ad8c 100755
--- a/cdist/conf/type/__localedef/manifest
+++ b/cdist/conf/type/__localedef/manifest
@@ -2,6 +2,7 @@
 #
 # 2013-2019 Nico Schottelius (nico-cdist at schottelius.org)
 # 2015 David Hürlimann (david at ungleich.ch)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,24 +19,12 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# Install required packages
+# Install required packages.
 #
 
-os=$(cat "$__global/explorer/os")
-
-
-case "$os" in
-    debian|devuan)
-        # Debian needs a seperate package
-        __package locales --state present
-    ;;
-    archlinux|suse|ubuntu|scientific|centos|alpine)
-        :
-    ;;
-    *)
-        echo "Sorry, do not know how to handle os: $os" >&2
-        echo "Please edit the type ${__type##*/} to fix this." >&2
-        exit 1
-    ;;
+case $(cat "${__global:?}/explorer/os")
+in
+	(debian|devuan)
+		__package_apt locales --state present
+		;;
 esac

From dcef2c19f5b87e3a9594f01fefab66cc7488dce7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 14:07:42 +0100
Subject: [PATCH 141/366] [type/__localedef] Add support for FreeBSD

---
 cdist/conf/type/__localedef/gencode-remote | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index b9cc68e8..80e7eb1c 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -108,6 +108,25 @@ in
 				;;
 		esac
 		;;
+	(freebsd)
+		case ${state_should}
+		in
+			(present)
+				if expr "$(grep -oe '^[0-9]*' "${__global:?}/explorer/os_version")" '>=' 11 >/dev/null
+				then
+					# localedef(1) is available with FreeBSD >= 11
+					printf "localedef -i '%s' -f '%s' '%s'\n" "${input}" "${codeset}" "${locale}"
+				else
+					printf 'localedef(1) was added to FreeBSD starting with version 11.\n' >&2
+					printf 'Please upgrade your FreeBSD installation to use %s.\n' "${__type##*/}" >&2
+					exit 1
+				fi
+				;;
+			(absent)
+				printf "rm -R '/usr/share/locale/%s'\n" "${locale}"
+				;;
+		esac
+		;;
 	(*)
 		echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
 		echo "Please contribute an implementation for it if you can." >&2

From c1c60e3374e0cb93ea182c4cbcb813150756c3e4 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sun, 8 Nov 2020 15:24:46 +0100
Subject: [PATCH 142/366] [type/__localedef] Blacklist OpenBSD and NetBSD

---
 cdist/conf/type/__localedef/gencode-remote | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 80e7eb1c..af1a77f7 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -127,6 +127,12 @@ in
 				;;
 		esac
 		;;
+	(netbsd|openbsd)
+		# NetBSD/OpenBSD are missing localedef(1).
+		# We also do not delete defined locales because they can't be recreated.
+		echo "${os} is lacking localedef(1). Locale management unavailable." >&2
+		exit 1
+		;;
 	(*)
 		echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2
 		echo "Please contribute an implementation for it if you can." >&2

From 575bb62dc507fb8c294319f8db20ba45595e6869 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 13 Nov 2020 18:42:04 +0100
Subject: [PATCH 143/366] [type/__localedef] Externalise functions to separate
 files

---
 .../conf/type/__localedef/files/lib/glibc.sh  |  5 ++++
 .../conf/type/__localedef/files/lib/locale.sh | 20 +++++++++++++
 cdist/conf/type/__localedef/gencode-remote    | 29 ++++---------------
 3 files changed, 30 insertions(+), 24 deletions(-)
 create mode 100644 cdist/conf/type/__localedef/files/lib/glibc.sh
 create mode 100644 cdist/conf/type/__localedef/files/lib/locale.sh

diff --git a/cdist/conf/type/__localedef/files/lib/glibc.sh b/cdist/conf/type/__localedef/files/lib/glibc.sh
new file mode 100644
index 00000000..6ace80d4
--- /dev/null
+++ b/cdist/conf/type/__localedef/files/lib/glibc.sh
@@ -0,0 +1,5 @@
+# -*- mode: sh; indent-tabs-mode: t -*-
+
+gnu_normalize_codeset() {
+	echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]'
+}
diff --git a/cdist/conf/type/__localedef/files/lib/locale.sh b/cdist/conf/type/__localedef/files/lib/locale.sh
new file mode 100644
index 00000000..b5e61374
--- /dev/null
+++ b/cdist/conf/type/__localedef/files/lib/locale.sh
@@ -0,0 +1,20 @@
+# -*- mode: sh; indent-tabs-mode:t -*-
+
+parse_locale() {
+	# This function will split locales into their parts. Locale strings are
+	# usually of the form: [language[_territory][.codeset][@modifier]]
+	# For simplicity, language and territory are not separated by this function.
+	# Old Linux systems were also using "english" or "german" as locale strings.
+	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
+	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
+	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
+	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
+}
+
+format_locale() {
+	# Usage: format_locale language codeset modifier
+	printf '%s' "$1"
+	test -z "$2" || printf '.%s' "$2"
+	test -z "$3" || printf '@%s' "$3"
+	printf '\n'
+}
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index af1a77f7..17941c63 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -21,6 +21,11 @@
 # Manage system locales using localedef(1).
 #
 
+# shellcheck source=cdist/conf/type/__localedef/files/lib/locale.sh
+. "${__type:?}/files/lib/locale.sh"
+# shellcheck source=cdist/conf/type/__localedef/files/lib/glibc.sh
+. "${__type:?}/files/lib/glibc.sh"
+
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
@@ -46,30 +51,6 @@ then
 	exit 1
 fi
 
-parse_locale() {
-	# This function will split locales into their parts. Locale strings are
-	# usually of the form: [language[_territory][.codeset][@modifier]]
-	# For simplicity, language and territory are not separated by this function.
-	# Old Linux systems were also using "english" or "german" as locale strings.
-	# Usage: parse_locale locale_str lang_var codeset_var modifier_var
-	eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')"
-	eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')"
-	eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')"
-}
-
-format_locale() {
-	# Usage: format_locale language codeset modifier
-	printf '%s' "$1"
-	test -z "$2" || printf '.%s' "$2"
-	test -z "$3" || printf '@%s' "$3"
-	printf '\n'
-}
-
-gnu_normalize_codeset() {
-	echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]'
-}
-
-
 : "${lang=}" "${codeset=}" "${modifier=}"  # declare variables for shellcheck
 parse_locale "${locale}" lang codeset modifier
 

From eeb98719197a09691af60fa26c0b64b3aaf8960e Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 14 Nov 2020 09:54:01 +0100
Subject: [PATCH 144/366] [type/__localedef] glibc: Also delete aliases when
 removing a locale

---
 cdist/conf/type/__localedef/gencode-remote | 15 +++++++++++++--
 cdist/conf/type/__localedef/man.rst        |  5 +++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index 17941c63..d7fd6942 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -84,8 +84,19 @@ in
 				EOF
 				;;
 			(absent)
-				localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}")
-				printf "localedef --delete-from-archive '%s'\n" "${localename}"
+				main_localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}")
+
+				cat <<-EOF
+				while read -r _alias _localename
+				do
+				    if test "\${_localename}" = '$(format_locale "${lang}" "${codeset}")'
+				    then
+				        localedef --delete-from-archive "\${_alias}"
+				    fi
+				done <'${aliasfile}'
+
+				localedef --delete-from-archive '${main_localename}'
+				EOF
 				;;
 		esac
 		;;
diff --git a/cdist/conf/type/__localedef/man.rst b/cdist/conf/type/__localedef/man.rst
index 0ccf484a..454ce9d1 100644
--- a/cdist/conf/type/__localedef/man.rst
+++ b/cdist/conf/type/__localedef/man.rst
@@ -13,6 +13,11 @@ This cdist type allows you to define locales on the system using
 On systems that don't support definition of new locales, the type will raise an
 error.
 
+**NB:** This type respects the glibc ``locale.alias`` file,
+i.e. it defines alias locales or deletes aliases of a locale when it is removed.
+It is not possible, however, to use alias names to define locales or only remove
+certain aliases of a locale.
+
 
 OPTIONAL PARAMETERS
 -------------------

From 87faffd8750cac94190707c2b37233229c3fbde2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 14 Nov 2020 10:41:50 +0100
Subject: [PATCH 145/366] [type/__localdef] Also check for aliases in state
 explorer

---
 cdist/conf/type/__localedef/explorer/state | 31 +++++++++++++++++++++-
 cdist/conf/type/__localedef/gencode-remote |  7 +++--
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__localedef/explorer/state b/cdist/conf/type/__localedef/explorer/state
index d8db29c5..3ba57661 100755
--- a/cdist/conf/type/__localedef/explorer/state
+++ b/cdist/conf/type/__localedef/explorer/state
@@ -25,8 +25,14 @@
 #    the main locale (and possibly aliases) is present
 #  absent:
 #    neither the main locale nor any aliases are present
+#  alias-present:
+#    the main locale is absent, but at least one of its aliases is present
 #
 
+# Hardcoded, create a pull request in case it is at another location for
+# some other distro. (cf. gencode-remote)
+aliasfile='/usr/share/locale/locale.alias'
+
 command -v locale >/dev/null 2>&1 || exit 0
 
 locales=$(locale -a)
@@ -68,4 +74,27 @@ locale_available() (
 	}
 )
 
-locale_available "${__object_id:?}" && echo present || echo absent
+if locale_available "${__object_id:?}"
+then
+	echo present
+else
+	# NOTE: locale.alias can be symlinked.
+	if test -e "${aliasfile}"
+	then
+		# Check if one of the aliases of the locale is defined
+		baselocale=$(
+			parse_locale "${__object_id:?}" _lang _codeset _modifiers \
+			&& format_locale "${_lang}" "${_codeset}")
+		while read -r _alias _localename
+		do
+			if test "${_localename}" = "${baselocale}" \
+				&& echo "${locales}" | grep -qxF "${_alias}"
+			then
+				echo alias-present
+				exit 0
+			fi
+		done <"${aliasfile}"
+	fi
+
+	echo absent
+fi
diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote
index d7fd6942..4538151f 100755
--- a/cdist/conf/type/__localedef/gencode-remote
+++ b/cdist/conf/type/__localedef/gencode-remote
@@ -94,9 +94,12 @@ in
 				        localedef --delete-from-archive "\${_alias}"
 				    fi
 				done <'${aliasfile}'
-
-				localedef --delete-from-archive '${main_localename}'
 				EOF
+
+				if test "${state_is}" = present
+				then
+					printf "localedef --delete-from-archive '%s'\n" "${main_localename}"
+				fi
 				;;
 		esac
 		;;

From 9d4f69250ea0e6bb733cde1e42183480f48b5d8f Mon Sep 17 00:00:00 2001
From: ssrq <ungleich@ssrq-sds-fds.ch>
Date: Thu, 19 Nov 2020 19:33:47 +0100
Subject: [PATCH 146/366] __sshd config: New type

---
 cdist/conf/type/__sshd_config/explorer/state  | 121 ++++++++
 .../files/update_sshd_config.awk              | 293 ++++++++++++++++++
 cdist/conf/type/__sshd_config/gencode-remote  |  97 ++++++
 cdist/conf/type/__sshd_config/man.rst         |  94 ++++++
 cdist/conf/type/__sshd_config/manifest        |  48 +++
 .../type/__sshd_config/parameter/default/file |   1 +
 .../__sshd_config/parameter/default/state     |   1 +
 .../type/__sshd_config/parameter/optional     |   4 +
 .../__sshd_config/parameter/optional_multiple |   1 +
 9 files changed, 660 insertions(+)
 create mode 100644 cdist/conf/type/__sshd_config/explorer/state
 create mode 100644 cdist/conf/type/__sshd_config/files/update_sshd_config.awk
 create mode 100755 cdist/conf/type/__sshd_config/gencode-remote
 create mode 100644 cdist/conf/type/__sshd_config/man.rst
 create mode 100755 cdist/conf/type/__sshd_config/manifest
 create mode 100644 cdist/conf/type/__sshd_config/parameter/default/file
 create mode 100644 cdist/conf/type/__sshd_config/parameter/default/state
 create mode 100644 cdist/conf/type/__sshd_config/parameter/optional
 create mode 100644 cdist/conf/type/__sshd_config/parameter/optional_multiple

diff --git a/cdist/conf/type/__sshd_config/explorer/state b/cdist/conf/type/__sshd_config/explorer/state
new file mode 100644
index 00000000..75c68b8a
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/explorer/state
@@ -0,0 +1,121 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Determines the current state of the config option.
+# Possible output:
+#  - present: "should" option present in config file
+#  - default: the "should" option is the default -> don’t know if present
+#  - absent: no such option present in config file
+#
+
+joinlines() { sed -n -e H -e "\${x;s/^\\n//;s/\\n/${1:?}/g;p;}"; }
+trlower() { tr '[:upper:]' '[:lower:]'; }
+tolower() { printf '%s' "$*" | trlower; }
+
+default_value() {
+	sshd -T -f /dev/null -C "$(make_conn_spec)" \
+	| sed -n -e 's/^'"$(tolower "${1:?}")"'[[:blank:]]\{1,\}//p'
+}
+
+make_conn_spec() {
+	if test -s "${__object:?}/parameter/match"
+	then
+		_match_file="${__object:?}/parameter/match"
+	else
+		_match_file='/dev/null'
+	fi
+
+	for _kw in \
+		addr=Address \
+		user=User \
+		host=Host \
+		laddr=LocalAddress \
+		lport=LocalPort \
+		rdomain=RDomain
+	do
+		_specname=${_kw%%=*}
+		_confname=$(tolower "${_kw#*=}")
+		while read -r _k _v
+		do
+			if test "$(tolower "${_k}")" = "${_confname}"
+			then
+				printf '%s=%s\n' "${_specname}" "${_v}"
+				continue 2
+			fi
+		done <"${_match_file}"
+
+		# NOTE: Print test spec even for empty keys to suppress errors like:
+		#  'Match User' in configuration but 'user' not in connection test specification.
+		# except lport:
+		#  Invalid port '' in test mode specification lport=
+		test "${_specname}" = 'lport' || printf '%s=\n' "${_specname}"
+	done \
+	| joinlines ','
+	unset _match_file
+}
+
+sshd_config_file=$(cat "${__object:?}/parameter/file")
+state_should=$(cat "${__object:?}/parameter/state")
+
+if test -s "${__object:?}/parameter/option"
+then
+	option_name=$(cat "${__object:?}/parameter/option")
+else
+	option_name=${__object_id:?}
+fi
+
+value_should=$(cat "${__object:?}/parameter/value" 2>/dev/null) \
+|| test "${state_should}" = absent || exit 0  # param optional if --state absent
+
+command -v sshd >/dev/null 2>&1 || {
+	echo 'Cannot find sshd.' >&2
+	exit 1
+}
+
+test -e "${sshd_config_file}" || {
+	echo 'absent'
+	exit 0
+}
+
+value_is=$(
+	sshd -T -f "${sshd_config_file}" -C "$(make_conn_spec)" \
+	| sed -n -e 's/^'"$(tolower "${option_name}")"'[[:blank:]]\{1,\}//p')
+
+if printf '%s\n' "${value_is}" | {
+		if test -n "${value_should}"
+		then
+			grep -q -x -F "${value_should}"
+		else
+			# if no value provided, assume "any" value
+			grep -q -e .
+		fi
+	}
+then
+	if default_value "${option_name}" | grep -q -x -F "${value_is}"
+	then
+		# Might produce false positives for default values.
+		# TODO: Manual checking should be done, but for simplicity, this case is
+		#       currently ignored here.
+		echo default
+	else
+		echo present
+	fi
+else
+	echo absent
+fi
diff --git a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
new file mode 100644
index 00000000..d0bc2b4b
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
@@ -0,0 +1,293 @@
+# -*- mode: awk; indent-tabs-mode: t -*-
+
+function usage() {
+	print_err("Usage: awk -f update_sshd_config.awk -- -o set|unset [-m 'User git'] -l 'X11Forwarding no' /etc/ssh/sshd_config")
+}
+
+function print_err(s) { print s | "cat >&2" }
+
+function alength(a,    i) {
+	for (i = 0; (i + 1) in a; ++i);
+	return i
+}
+
+function join(sep, a, i,    s) {
+	for (i = i ? i : 1; i in a; i++)
+		s = s sep a[i]
+	return substr(s, 2)
+}
+
+function getopt(opts, argv, target, files,    i, c, lv, idx, nf) {
+	# trivial getopt(3) implementation; only basic functionality
+	if (argv[1] == "--") i++
+	for (i += 1; i in argv; i++) {
+		if (lv) { target[c] = argv[i]; lv = 0; continue }
+		if (argv[i] ~ /^-/) {
+			c = substr(argv[i], 2, 1)
+			idx = index(opts, c)
+			if (!idx) {
+				print_err(sprintf("invalid option -%c\n", c))
+				continue
+			}
+			if (substr(opts, idx + 1, 1) == ":") {
+				# option takes argument
+				if (length(argv[i]) > 2)
+					target[c] = substr(argv[i], 3)
+				else
+					lv = 1
+			} else {
+				target[c] = 1
+			}
+		} else
+			files[++nf] = argv[i]
+	}
+}
+
+# tokenise configuration line
+# this function mimics the counterpart in OpenSSH (misc.c)
+# but it returns two (next token SUBSEP rest) because I didn’t want to have to
+# simulate any pointer magic.
+function strdelim_internal(s, split_equals,    old) {
+	if (!s)
+		return ""
+
+	old = s
+
+	if (!match(s, WHITESPACE "|" QUOTE "" (split_equals ? "|" EQUALS : "")))
+		return s
+
+	s = substr(s, RSTART)
+	old = substr(old, 1, RSTART - 1)
+
+	if (s ~ "^" QUOTE) {
+		old = substr(old, 2)
+
+		# Find matching quote
+		if (match(s, QUOTE)) {
+			old = substr(old, 1, RSTART)
+			# s = substr()
+			if (match(s, "^" WHITESPACE "*"))
+				s = substr(s, RLENGTH)
+			return old
+		} else {
+			# no matching quote
+			return ""
+		}
+	}
+
+	if (match(s, "^" WHITESPACE "+")) {
+		sub("^" WHITESPACE "+", "", s)
+		if (split_equals)
+			sub(EQUALS WHITESPACE "*", "", s)
+	} else if (s ~ "^" EQUALS) {
+		s = substr(s, 2)
+	}
+
+	return old SUBSEP s
+}
+function strdelim(s) { return strdelim_internal(s, 1) }
+function strdelimw(s) { return strdelim_internal(s, 0) }
+
+function singleton_option(opt) {
+	return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|authenticationmethods|authorizedkeysfile|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/
+}
+
+function print_update() {
+	if (mode) {
+		if (match_only) printf "\t"
+		printf "%s\n", line_should
+		updated = 1
+	}
+}
+
+BEGIN {
+	FS = "\n"  # disable field splitting
+
+	WHITESPACE = "[ \t]"  # servconf.c, misc.c:strdelim_internal (without line breaks, cf. bugs)
+	QUOTE = "[\"]"  # misc.c:strdelim_internal
+	EQUALS = "[=]"
+
+	split("", opts)
+	split("", files)
+	getopt("ho:l:m:", ARGV, opts, files)
+
+	if (opts["h"]) { usage(); exit (e="0") }
+
+	line_should = opts["l"]
+	match_only = opts["m"]
+	num_files = alength(files)
+
+	if (num_files != 1 || !opts["o"] || !line_should) {
+		usage()
+		exit (e=126)
+	}
+
+	if (opts["o"] == "set") {
+		mode = 1
+	} else if (opts["o"] == "unset") {
+		mode = 0
+	} else {
+		print_err(sprintf("invalid mode %s\n", mode))
+		exit (e=1)
+	}
+
+	if (mode) {
+		# loop over sshd_config twice!
+		ARGV[2] = ARGV[1] = files[1]
+		ARGC = 3
+	} else {
+		# only loop once
+		ARGV[1] = files[1]
+		ARGC = 2
+	}
+
+	split(strdelim(line_should), should, SUBSEP)
+	option_should = tolower(should[1])
+	value_should = should[2]
+}
+
+{
+	line = $0
+
+	# Strip trailing whitespace. Allow \f (form feed) at EOL only
+	sub("(" WHITESPACE "|\f)*$", "", line)
+
+	# Strip leading whitespace
+	sub("^" WHITESPACE "*", "", line)
+
+	if (match(line, "^#" WHITESPACE "*")) {
+		prefix = substr(line, RSTART, RLENGTH)
+		line = substr(line, RSTART + RLENGTH)
+	} else {
+		prefix = ""
+	}
+
+	line_type = "invalid"
+	option_is = value_is = ""
+
+	if (line) {
+		split(strdelim(line), toks, SUBSEP)
+
+		if (tolower(toks[1]) == "match") {
+			MATCH = (prefix ~ /^#/ ? "#" : "") join(" ", toks, 2)
+			line_type = "match"
+		} else if (toks[1] ~ /^[A-Za-z][A-Za-z0-9]+$/) {
+			# This could be an option line
+			line_type = "option"
+			option_is = tolower(toks[1])
+			value_is = toks[2]
+		}
+	} else {
+		line_type = "empty"
+	}
+}
+
+# mode: unset
+
+!mode {
+	# delete matching config
+	if (prefix !~ /^#/)
+		if (MATCH == match_only && option_is == option_should)
+			if (!value_should || value_should == value_is)
+				next
+
+	print
+	next
+}
+
+
+# mode: set
+
+mode && NR == FNR {
+	if (line_type == "option") {
+		if (MATCH !~ /^#/) {
+			if (prefix ~ /^#/) {
+				# comment line
+				last_occ[MATCH, "#" option_is] = FNR
+			} else {
+				# option line
+				last_occ[MATCH, option_is] = FNR
+			}
+			last_occ[MATCH] = FNR
+		}
+	} else if (line_type == "invalid" && !prefix) {
+		# INVALID LINE
+		print_err(sprintf("%s: syntax error on line %u\n", ARGV[0], FNR))
+	}
+
+	next
+}
+
+# before second pass prepare hashes containing location information to be used
+# in the second pass.
+mode && NR > FNR && FNR == 1 {
+	# First we drop the locations of commented-out options if a non-commented
+	# option is available. If a non-commented option is available, we will
+	# append new config options there to have them all at one place.
+	for (k in last_occ) {
+		if (k ~ /^#/) {
+			# delete entries of commented out match blocks
+			delete last_occ[k]
+			continue
+		}
+
+		split(k, parts, SUBSEP)
+
+		if (parts[2] ~ /^#/ && ((parts[1], substr(parts[2], 2)) in last_occ))
+			delete last_occ[k]
+	}
+
+	# Reverse the option => line mapping. The line_map allows for easier lookups
+	# in the second pass.
+	# We only keep options, not top-level keywords, because we can only have
+	# one entry per line and there are conflicts with last lines of "sections".
+	for (k in last_occ) {
+		if (!index(k, SUBSEP)) continue
+		line_map[last_occ[k]] = k
+	}
+}
+
+# Second pass
+mode && line_map[FNR] == match_only SUBSEP option_should && !updated {
+	split(line_map[FNR], parts, SUBSEP)
+
+	# If option allows multiple values, print current value
+	if (!singleton_option(parts[2])) {
+		if (value_should != value_is)
+			print
+	}
+
+	print_update()
+
+	next
+}
+
+mode { print }
+
+# Is a comment option
+mode && line_map[FNR] == match_only SUBSEP "#" option_should && !updated {
+	print_update()
+}
+
+# Last line of the should match section
+mode && last_occ[match_only] == FNR && !updated {
+	# NOTE: Inserting empty lines is only cosmetic.  It is only done if
+	#       different options are next to each other and not in a match block
+	#       (match blocks are usually not in the default config and thus don’t
+	#       contain commented blocks.)
+	if (line && option_is != option_should && !MATCH)
+		print ""
+	print_update()
+}
+
+END {
+	if (e) exit e
+
+	if (mode && !updated) {
+		if (match_only && MATCH != match_only) {
+			printf "\nMatch %s\n", match_only
+		}
+
+		print_update()
+	}
+}
diff --git a/cdist/conf/type/__sshd_config/gencode-remote b/cdist/conf/type/__sshd_config/gencode-remote
new file mode 100755
index 00000000..0b44dfa7
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/gencode-remote
@@ -0,0 +1,97 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+joinlines() { sed -n -e H -e "\${x;s/^\\n//;s/\\n/${1:?}/g;p;}"; }
+
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+if test "${state_is}" = "${state_should}" -o "${state_is}" = 'default'
+then
+	# nothing to do (if the value is the default, ignore its state)
+	exit 0
+fi
+
+case ${state_should}
+in
+	(present)
+		mode='set'
+		;;
+	(absent)
+		mode='unset'
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
+esac
+
+sshd_config_file=$(cat "${__object:?}/parameter/file")
+
+quote() { printf "'%s'" "$(printf '%s' "$*" | sed -e "s/'/'\\\\''/g")"; }
+drop_awk_comments() { quote "$(sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@")"; }
+
+# Ensure the sshd_config file is there
+cat <<EOF
+test -e $(quote "${sshd_config_file}") || {
+	 : >$(quote "${sshd_config_file}")
+	 chown 0:0 $(quote "${sshd_config_file}")
+	 chmod 0644 $(quote "${sshd_config_file}")
+}
+
+EOF
+
+match_only=
+if test -s "${__object:?}/parameter/match"
+then
+	match_only=$(joinlines ' ' <"${__object:?}/parameter/match")
+fi
+
+if test -s "${__object:?}/parameter/option"
+then
+	option_line=$(cat "${__object:?}/parameter/option")
+else
+	option_line=${__object_id:?}
+fi
+
+if test -s "${__object:?}/parameter/value"
+then
+	option_line="${option_line} $(cat "${__object:?}/parameter/value")"
+fi
+
+# Send message on config update
+printf '%s%s %s\n' "${mode}" "${match_only:+ [${match_only}]}" \
+	"${option_line}" >>"${__messages_out:?}"
+
+# Update sshd_config (remote code)
+cat <<EOF
+awk $(drop_awk_comments "${__type:?}/files/update_sshd_config.awk") \\
+	-o ${mode} \\
+	-m $(quote "${match_only}") \\
+	-l $(quote "${option_line}") \\
+	$(quote "${sshd_config_file}") >$(quote "${sshd_config_file}.tmp") \\
+|| exit
+
+cmp -s $(quote "${sshd_config_file}") $(quote "${sshd_config_file}.tmp") || {
+	sshd -t -f $(quote "${sshd_config_file}.tmp") \\
+	&& cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}")
+}
+rm -f $(quote "${sshd_config_file}.tmp")
+EOF
diff --git a/cdist/conf/type/__sshd_config/man.rst b/cdist/conf/type/__sshd_config/man.rst
new file mode 100644
index 00000000..8b0069ac
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/man.rst
@@ -0,0 +1,94 @@
+cdist-type__sshd_config(7)
+==========================
+
+NAME
+----
+cdist-type__sshd_config - Manage options in sshd_config
+
+
+DESCRIPTION
+-----------
+This space intentionally left blank.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+file
+    The path to the sshd_config file to edit.
+    Defaults to ``/etc/ssh/sshd_config``.
+match
+    Restrict this option to apply only for certain connections.
+    Allowed values are what would be allowed to be written after a ``Match``
+    keyword in ``sshd_config``, e.g. ``--match 'User anoncvs'``.
+
+    Can be used multiple times. All of the values are ANDed together.
+option
+    The name of the option to manipulate. Defaults to ``__object_id``.
+state
+    Can be:
+
+    - ``present``: ensure a matching config line is present (or the default
+      value).
+    - ``absent``: ensure no matching config line is present.
+value
+    The option's value to be assigned to the option (if ``--state present``) or
+    removed (if ``--state absent``).
+
+    This option is required if ``--state present``. If not specified and
+    ``--state absent``, all values for the given option are removed.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Disallow root logins with password
+    __sshd_config PermitRootLogin --value without-password
+
+    # Disallow password-based authentication
+    __sshd_config PasswordAuthentication --value no
+
+    # Accept the EDITOR environment variable
+    __sshd_config AcceptEnv:EDITOR --option AcceptEnv --value EDITOR
+
+    # Force command for connections as git user
+    __sshd_config git@ForceCommand --match 'User git' --option ForceCommand \
+        --value 'cd ~git && exec git-shell ${SSH_ORIGINAL_COMMAND:+-c "${SSH_ORIGINAL_COMMAND}"}'
+
+
+SEE ALSO
+--------
+:strong:`sshd_config`\ (5)
+
+
+BUGS
+----
+- This type assumes a nicely formatted config file,
+  i.e. no config options spanning multiple lines.
+- ``Include`` directives are ignored.
+- Config options are not added/removed to/from the config file if their value is
+  the default value.
+
+
+AUTHORS
+-------
+Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Dennis Camera. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest
new file mode 100755
index 00000000..566bde90
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/manifest
@@ -0,0 +1,48 @@
+#!/bin/sh -e
+#
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$(cat "${__global:?}/explorer/os")
+
+state_should=$(cat "${__object:?}/parameter/state")
+
+case ${os}
+in
+	(alpine|centos|fedora|redhat|scientific|debian|devuan|ubuntu)
+		if test "${state_should}" != 'absent'
+		then
+			__package openssh-server --state present
+		fi
+		;;
+	(archlinux|gentoo|slackware|suse)
+		if test "${state_should}" != 'absent'
+		then
+			__package openssh --state present
+		fi
+		;;
+	(freebsd|netbsd|openbsd)
+		# whitelist
+		;;
+	(*)
+		printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \
+			"${os}" "${__type##*/}" >&2
+		printf 'Please contribute an implementation for it if you can.\n' >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__sshd_config/parameter/default/file b/cdist/conf/type/__sshd_config/parameter/default/file
new file mode 100644
index 00000000..d8ea5dfc
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/default/file
@@ -0,0 +1 @@
+/etc/ssh/sshd_config
diff --git a/cdist/conf/type/__sshd_config/parameter/default/state b/cdist/conf/type/__sshd_config/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__sshd_config/parameter/optional b/cdist/conf/type/__sshd_config/parameter/optional
new file mode 100644
index 00000000..922ab093
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/optional
@@ -0,0 +1,4 @@
+file
+option
+state
+value
diff --git a/cdist/conf/type/__sshd_config/parameter/optional_multiple b/cdist/conf/type/__sshd_config/parameter/optional_multiple
new file mode 100644
index 00000000..02b1d1a9
--- /dev/null
+++ b/cdist/conf/type/__sshd_config/parameter/optional_multiple
@@ -0,0 +1 @@
+match

From 82eadb69948e631041263938994be5aeae161ab0 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 19 Nov 2020 19:34:43 +0100
Subject: [PATCH 147/366] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index ff411a46..331acbeb 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,8 @@ next:
 	* Type __locale: Add state explorer (Matthias Stecher)
 	* Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera)
 	* New type: __hwclock (Dennis Camera)
+	* Type __hostname: Fix guessing SuSE OS version (Dennis Camera)
+	* New type: __sshd_config (Dennis Camera)
 
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)

From 803a9d62a7a6b7fd11844f6ea273f4b692c2334e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 20 Nov 2020 19:46:03 +0100
Subject: [PATCH 148/366] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 331acbeb..cd5dbb4f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,8 @@ next:
 	* New type: __hwclock (Dennis Camera)
 	* Type __hostname: Fix guessing SuSE OS version (Dennis Camera)
 	* New type: __sshd_config (Dennis Camera)
+	* New type: __localedef (Dennis Camera)
+	* Type __locale: Deprecate in favor of __localedef (Dennis Camera)
 
 6.9.1: 2020-11-08
 	* Type __file: Fix state pre-exists (Dennis Camera)

From 23e0da521c785715357b27d66007bd03fe173bf3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 20 Nov 2020 19:46:55 +0100
Subject: [PATCH 149/366] Release 6.9.2

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index cd5dbb4f..9a91bf57 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.2: 2020-11-20
 	* Documentation: Fix examples in best practice (Dennis Camera)
 	* Type __locale: Add state explorer (Matthias Stecher)
 	* Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera)

From 84172550df9e1800bc795dc51da01c9d3a0be9e0 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 30 Nov 2020 20:29:51 +0100
Subject: [PATCH 150/366] __iptables*: add IPv6 support

Because it currently only support IPv4. To implement this, it falls back
to IPv4 for backward compatibilty, but now supports rules for IPv6 and
both protocols at the same time.
---
 .../type/__iptables_apply/files/init-script   | 84 +++++++++++++++----
 cdist/conf/type/__iptables_rule/man.rst       | 38 ++++++++-
 cdist/conf/type/__iptables_rule/manifest      | 40 ++++++++-
 .../type/__iptables_rule/parameter/boolean    |  3 +
 4 files changed, 140 insertions(+), 25 deletions(-)
 create mode 100644 cdist/conf/type/__iptables_rule/parameter/boolean

diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script
index d9c79ef7..196f019b 100644
--- a/cdist/conf/type/__iptables_apply/files/init-script
+++ b/cdist/conf/type/__iptables_apply/files/init-script
@@ -1,6 +1,27 @@
 #!/bin/sh
-# Nico Schottelius
-# Zürisee, Mon Sep  2 18:38:27 CEST 2013
+#
+# 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is distributed with cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Originally written by:
+#   Nico Schottelius
+#   Zürisee, Mon Sep  2 18:38:27 CEST 2013
 #
 ### BEGIN INIT INFO
 # Provides:          iptables
@@ -15,33 +36,60 @@
 ### END INIT INFO
 
 
+# Read files and execute the content with the given commands
+#
+# Arguments:
+#  1:    Directory
+#  2..n: Commands which should be used to execute the file content
+gothrough() {
+    cd "$1" || return
+    shift
+
+    # iterate through all rules and continue if it's not a file
+    for rule in *; do
+        [ -f "$rule" ] || continue
+        echo "Appling iptables rule $rule ..."
+
+        # execute it with all commands specificed
+        ruleparam="$(cat "$rule")"
+        for cmd in "$@"; do
+            # Command and Rule should be split.
+            # shellcheck disable=SC2046
+            command $cmd $ruleparam
+        done
+    done
+}
+
+# Shortcut for iptables command to do IPv4 and v6
+iptables() {
+    command iptables "$@"
+    command ip6tables "$@"
+}
+
 basedir=/etc/iptables.d
-status="${basedir}/.pre-start"
+status4="${basedir}/.pre-start"
+status6="${basedir}/.pre-start6"
 
 case $1 in
     start)
         # Save status
-        iptables-save > "$status"
+        iptables-save > "$status4"
+        ip6tables-save > "$status6"
 
         # Apply our ruleset
-        cd "$basedir" || exit
-        count="$(find . ! -name . -prune | wc -l)"
-
-        # Only do something if there are rules
-        if [ "$count" -ge 1 ]; then
-            for rule in *; do
-                echo "Applying iptables rule $rule ..."
-                # Rule should be split.
-                # shellcheck disable=SC2046
-                iptables $(cat "$rule")
-            done
-        fi
+        gothrough "$basedir"     iptables
+        #gothrough "$basedir/v4"  iptables  # conflicts with $basedir
+        gothrough "$basedir/v6"  ip6tables
+        gothrough "$basedir/all" iptables  ip6tables
     ;;
 
     stop)
         # Restore from status before, if there is something to restore
-        if [ -f "$status" ]; then
-            iptables-restore < "$status"
+        if [ -f "$status4" ]; then
+            iptables-restore < "$status4"
+        fi
+        if [ -f "$status6" ]; then
+            ip6tables-restore < "$status6"
         fi
     ;;
     restart)
diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst
index 92d8859f..75d0740b 100644
--- a/cdist/conf/type/__iptables_rule/man.rst
+++ b/cdist/conf/type/__iptables_rule/man.rst
@@ -25,6 +25,24 @@ state
    'present' or 'absent', defaults to 'present'
 
 
+BOOLEAN PARAMETERS
+------------------
+All rules without any of this parameter will be threaten like ``--v4`` because
+of backward compatibility.
+
+v4
+    Explicitly set it as rule for IPv4. If IPv6 is set, too, it will be
+    threaten like ``--all``. Will be the default if nothing else is set.
+
+v6
+    Explicitly set it as rule for IPv6. If IPv4 is set, too, it will be
+    threaten like ``--all``.
+
+all
+    Set the rule for both IPv4 and IPv6. It will be saved separately from the
+    other rules.
+
+
 EXAMPLES
 --------
 
@@ -48,6 +66,16 @@ EXAMPLES
         --state absent
 
 
+    # IPv4-only rule for ICMPv4
+    __iptables_rule icmp-v4 --v4 --rule "-A INPUT -p icmp -j ACCEPT"
+    # IPv6-only rule for ICMPv6
+    __iptables_rule icmp-v6 --v6 --rule "-A INPUT -p icmpv6 -j ACCEPT"
+
+    # doing something for the dual stack
+    __iptables_rule fwd-eth0-eth1 --v4 --v6 --rule "-A INPUT -i eth0 -o eth1 -j ACCEPT"
+    __iptables_rule fwd-eth1-eth0 --all --rule "-A -o eth1 -i eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT"
+
+
 SEE ALSO
 --------
 :strong:`cdist-type__iptables_apply`\ (7), :strong:`iptables`\ (8)
@@ -56,11 +84,13 @@ SEE ALSO
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2013 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2013 Nico Schottelius.
+Copyright \(C) 2020 Matthias Stecher.
+You can redistribute it and/or modify it under the terms of the GNU
+General Public License as published by the Free Software Foundation,
+either version 3 of the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest
index ed78787f..27d00024 100755
--- a/cdist/conf/type/__iptables_rule/manifest
+++ b/cdist/conf/type/__iptables_rule/manifest
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
 #
 # This file is part of cdist.
 #
@@ -24,12 +25,36 @@ base_dir=/etc/iptables.d
 name="$__object_id"
 state="$(cat "$__object/parameter/state")"
 
+if [ -f "$__object/parameter/v4" ]; then
+    only_v4="yes"
+    # $specific_dir is $base_dir
+fi
+if [ -f "$__object/parameter/v6" ]; then
+    only_v6="yes"
+    specific_dir="$base_dir/v6"
+fi
+# If rules should be set for both protocols
+if ([ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]) \
+    || [ -f "$__object/parameter/all" ]; then
+
+    # all to a specific directory
+    specific_dir="$base_dir/all"
+fi
+
+# set rule directory based on if it's the base or subdirectory
+rule_dir="${specific_dir:-$base_dir}"
+
 ################################################################################
 # Basic setup
 #
 
 __directory "$base_dir" --state present
 
+# sub-directory if required
+if [ "$specific_dir" ]; then
+    require="__directory/$base_dir" __directory "$specific_dir" --state present
+fi
+
 # Have apply do the real job
 require="$__object_name" __iptables_apply
 
@@ -37,6 +62,15 @@ require="$__object_name" __iptables_apply
 # The rule
 #
 
-require="__directory/$base_dir" __file "$base_dir/${name}" \
-    --source "$__object/parameter/rule" \
-    --state "$state"
+for dir in "$base_dir" "$base_dir/v6" "$base_dir/all"; do
+    # defaults to absent except the directory that should contain the file
+    if [ "$rule_dir" = "$dir" ]; then
+        curr_state="$state"
+    else
+        curr_state="absent"
+    fi
+
+    require="__directory/$rule_dir" __file "$dir/$name" \
+        --source "$__object/parameter/rule" \
+        --state "$curr_state"
+done
diff --git a/cdist/conf/type/__iptables_rule/parameter/boolean b/cdist/conf/type/__iptables_rule/parameter/boolean
new file mode 100644
index 00000000..76882272
--- /dev/null
+++ b/cdist/conf/type/__iptables_rule/parameter/boolean
@@ -0,0 +1,3 @@
+all
+v4
+v6

From f568462e4981c8b6437c29f96963c3d8e7bed742 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 2 Dec 2020 17:48:41 +0100
Subject: [PATCH 151/366] __iptables_rule: fix shellcheck SC2235

---
 cdist/conf/type/__iptables_rule/manifest | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest
index 27d00024..d4394c25 100755
--- a/cdist/conf/type/__iptables_rule/manifest
+++ b/cdist/conf/type/__iptables_rule/manifest
@@ -34,8 +34,8 @@ if [ -f "$__object/parameter/v6" ]; then
     specific_dir="$base_dir/v6"
 fi
 # If rules should be set for both protocols
-if ([ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]) \
-    || [ -f "$__object/parameter/all" ]; then
+if { [ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]; } ||
+    [ -f "$__object/parameter/all" ]; then
 
     # all to a specific directory
     specific_dir="$base_dir/all"

From bee255c1ae0039e3d532317007f06fcd2e64e3bc Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 2 Dec 2020 18:04:50 +0100
Subject: [PATCH 152/366] __iptables_apply: man updates

---
 .../type/__iptables_apply/files/init-script   |  1 +
 cdist/conf/type/__iptables_apply/man.rst      | 21 +++++++++++++------
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script
index 196f019b..7faa2f92 100644
--- a/cdist/conf/type/__iptables_apply/files/init-script
+++ b/cdist/conf/type/__iptables_apply/files/init-script
@@ -61,6 +61,7 @@ gothrough() {
 }
 
 # Shortcut for iptables command to do IPv4 and v6
+# only applies to the "reset" target
 iptables() {
     command iptables "$@"
     command ip6tables "$@"
diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst
index 76e1f6bf..db0e7869 100644
--- a/cdist/conf/type/__iptables_apply/man.rst
+++ b/cdist/conf/type/__iptables_apply/man.rst
@@ -10,7 +10,14 @@ DESCRIPTION
 -----------
 This cdist type deploys an init script that triggers
 the configured rules and also re-applies them on
-configuration.
+configuration. Rules are written from __iptables_rule
+into the folder ``/etc/iptables.d/``.
+
+It reads all rules from the base folder as rules for IPv4.
+Rules in the subfolder ``v6/`` are IPv6 rules. Rules in
+the subfolder ``all/`` are applied to both rule tables. All
+files contain the arguments for a single ``iptables`` and/or
+``ip6tables`` command.
 
 
 REQUIRED PARAMETERS
@@ -24,7 +31,7 @@ None
 EXAMPLES
 --------
 
-None (__iptables_apply is used by __iptables_rule)
+None (__iptables_apply is used by __iptables_rule automaticly)
 
 
 SEE ALSO
@@ -35,11 +42,13 @@ SEE ALSO
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2013 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2013 Nico Schottelius.
+Copyright \(C) 2020 Matthias Stecher.
+You can redistribute it and/or modify it under the terms of the GNU
+General Public License as published by the Free Software Foundation,
+either version 3 of the License, or (at your option) any later version.

From a1db5c3d0e7b5899f1a877c5bab28ac4d4796f8c Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 2 Dec 2020 18:22:31 +0100
Subject: [PATCH 153/366] __iptables*: Update manpages for execution order

To make some thinks clear if someone needs it ..
---
 cdist/conf/type/__iptables_apply/man.rst | 10 ++++++++++
 cdist/conf/type/__iptables_rule/man.rst  |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst
index db0e7869..4109e789 100644
--- a/cdist/conf/type/__iptables_apply/man.rst
+++ b/cdist/conf/type/__iptables_apply/man.rst
@@ -19,6 +19,16 @@ the subfolder ``all/`` are applied to both rule tables. All
 files contain the arguments for a single ``iptables`` and/or
 ``ip6tables`` command.
 
+Rules are applied in the following order:
+1. All IPv4 rules
+2. All IPv6 rules
+2. All rules that should be applied to both tables
+
+The order of the rules that will be applied are definite
+from the result the shell glob returns, which should be
+alphabetical. If rules must be applied in a special order,
+prefix them with a number like ``02-some-rule``.
+
 
 REQUIRED PARAMETERS
 -------------------
diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst
index 75d0740b..86d38a34 100644
--- a/cdist/conf/type/__iptables_rule/man.rst
+++ b/cdist/conf/type/__iptables_rule/man.rst
@@ -11,6 +11,10 @@ DESCRIPTION
 This cdist type allows you to manage iptable rules
 in a distribution independent manner.
 
+See :strong:`cdist-type__iptables_apply`\ (7) for the
+execution order of these rules. It will be executed
+automaticly to apply all rules non-volaite.
+
 
 REQUIRED PARAMETERS
 -------------------

From 1055e92545e74cf7845ede8094e783b4103efbcc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 2 Dec 2020 19:54:41 +0100
Subject: [PATCH 154/366] [setup.py] Add cdist.scan to packages

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 858c2c17..bfc8b495 100644
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@ os.chdir(cur)
 
 setup(
     name="cdist",
-    packages=["cdist", "cdist.core", "cdist.exec", "cdist.util", ],
+    packages=["cdist", "cdist.core", "cdist.exec", "cdist.scan", "cdist.util"],
     package_data={'cdist': package_data},
     scripts=["bin/cdist", "bin/cdist-dump", "bin/cdist-new-type"],
     version=cdist.version.VERSION,

From c7fa2efe6b780d63a779eed310bfa4ebf926d27e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 4 Dec 2020 15:30:08 +0100
Subject: [PATCH 155/366] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 9a91bf57..ad0aa75d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)
+
 6.9.2: 2020-11-20
 	* Documentation: Fix examples in best practice (Dennis Camera)
 	* Type __locale: Add state explorer (Matthias Stecher)

From d44b5cfdc990ce020ffe3af5085bbb93b8ce64a8 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 4 Dec 2020 15:31:35 +0100
Subject: [PATCH 156/366] Release 6.9.3

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index ad0aa75d..6c272fd7 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)
 
 6.9.2: 2020-11-20

From ba7d16a155cef46230d1dc650119f3d542a7f7f4 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 4 Dec 2020 17:57:55 +0100
Subject: [PATCH 157/366] __iptables_*: correct manpage spelling

---
 cdist/conf/type/__iptables_apply/man.rst | 2 +-
 cdist/conf/type/__iptables_rule/man.rst  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst
index 4109e789..3bef92cc 100644
--- a/cdist/conf/type/__iptables_apply/man.rst
+++ b/cdist/conf/type/__iptables_apply/man.rst
@@ -41,7 +41,7 @@ None
 EXAMPLES
 --------
 
-None (__iptables_apply is used by __iptables_rule automaticly)
+None (__iptables_apply is used by __iptables_rule automatically)
 
 
 SEE ALSO
diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst
index 86d38a34..afb71e01 100644
--- a/cdist/conf/type/__iptables_rule/man.rst
+++ b/cdist/conf/type/__iptables_rule/man.rst
@@ -31,7 +31,7 @@ state
 
 BOOLEAN PARAMETERS
 ------------------
-All rules without any of this parameter will be threaten like ``--v4`` because
+All rules without any of these parameters will be treated like ``--v4`` because
 of backward compatibility.
 
 v4

From 2d19856840400af57bd6667cc868ab314a3e07c2 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 4 Dec 2020 18:26:03 +0100
Subject: [PATCH 158/366] [type/__package_pkgng_freebsd] Set ASSUME_ALWAYS_YES
 instead of -y

---
 cdist/conf/type/__package_pkgng_freebsd/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
index b5944177..05ba4cb2 100755
--- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
+++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote
@@ -75,7 +75,7 @@ execcmd(){
    esac
 
    if [ -z "${pkg_bootstrapped}" ]; then
-       echo "pkg bootstrap -y >/dev/null 2>&1"
+       echo "ASSUME_ALWAYS_YES=yes pkg bootstrap >/dev/null 2>&1"
    fi
 
    echo "$_cmd >/dev/null 2>&1"   # Silence the output of the command

From 087be130fa67d3fe195387ae0ab079f39c5066e1 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 4 Dec 2020 19:23:49 +0100
Subject: [PATCH 159/366] __iptables_apply: shorten copyright header

Do we need all the copyright header or is this sufficient? The licence
is given for cdist, but not on the target host. But it should be clear
anyway.
---
 .../type/__iptables_apply/files/init-script   | 34 ++++++-------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script
index 7faa2f92..e42017ae 100644
--- a/cdist/conf/type/__iptables_apply/files/init-script
+++ b/cdist/conf/type/__iptables_apply/files/init-script
@@ -1,28 +1,4 @@
 #!/bin/sh
-#
-# 2013 Nico Schottelius (nico-cdist at schottelius.org)
-# 2020 Matthias Stecher (matthiasstecher at gmx.de)
-#
-# This file is distributed with cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Originally written by:
-#   Nico Schottelius
-#   Zürisee, Mon Sep  2 18:38:27 CEST 2013
-#
 ### BEGIN INIT INFO
 # Provides:          iptables
 # Required-Start:    $local_fs $remote_fs
@@ -35,6 +11,16 @@
 #                    and saves/restores previous status
 ### END INIT INFO
 
+# Originally written by:
+#   Nico Schottelius
+#   Zürisee, Mon Sep  2 18:38:27 CEST 2013
+#
+# 2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is distributed with cdist and licenced under the
+# GNU GPLv3+ WITHOUT ANY WARRANTY.
+
 
 # Read files and execute the content with the given commands
 #

From 3930f69456fd5a0d108a107e4ee61d87c9a73a56 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sun, 6 Dec 2020 16:45:58 +0100
Subject: [PATCH 160/366] __block: fix escaping in here-doc

This changes the here-document to do not interpret any shell-things. It
also single-quotes some more strings that are printed to code-remote.

Fixes #838
---
 cdist/conf/type/__block/gencode-remote | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__block/gencode-remote b/cdist/conf/type/__block/gencode-remote
index 1f5cc033..acdb3286 100755
--- a/cdist/conf/type/__block/gencode-remote
+++ b/cdist/conf/type/__block/gencode-remote
@@ -46,10 +46,10 @@ fi
 
 remove_block() {
    cat << DONE
-tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
+tmpfile=\$(mktemp '${file}.cdist.XXXXXXXXXX')
 # preserve ownership and permissions of existing file
-if [ -f "$file" ]; then
-   cp -p "$file" "\$tmpfile"
+if [ -f '$file' ]; then
+   cp -p '$file' "\$tmpfile"
 fi
 awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
 {
@@ -63,8 +63,8 @@ awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
    } else {
       print
    }
-}' "$file" > "\$tmpfile"
-mv -f "\$tmpfile" "$file"
+}' '$file' > "\$tmpfile"
+mv -f "\$tmpfile" '$file'
 DONE
 }
 
@@ -77,7 +77,7 @@ case "$state_should" in
          echo add >> "$__messages_out"
       fi
       cat << DONE
-cat >> "$file" << ${__type##*/}_DONE
+cat >> '$file' << '${__type##*/}_DONE'
 $(cat "$block")
 ${__type##*/}_DONE
 DONE

From bed08c2c5c71ec4252743e0b64b75e7d8fedbd92 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sun, 6 Dec 2020 20:24:00 +0100
Subject: [PATCH 161/366] Deal with deprecation of imp module.

importlib has been a thing since Python 3.1, and imp has been deprecated since
3.4.

Insert random complaint here about not being able to use f-strings because they
were introduced in Python 3.6 and apparently we support Python 3.5 >,<.

Output diff before to after for ./bin/cdist-build-helper test (on heavy load):
```
1,2d0
< /usr/home/evilham/s/cdist/cdist/cdist/test/__main__.py:23: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
<   import imp
72c70
< ERROR: cdisttesthost: __file/tmp/foobar requires object __file without object id. Defined at /tmp/tmp.cdist.test.g87lx7c8/tmp.cdist.test.6ramsakx
---
> ERROR: cdisttesthost: __file/tmp/foobar requires object __file without object id. Defined at /tmp/tmp.cdist.test.aqdf6vjz/tmp.cdist.test.jgv3udel
76c74
< test_nonexistent_type_requirement (cdist.test.emulator.EmulatorTestCase) ... ERROR: cdisttesthost: __file/tmp/foobar requires object __does-not-exist/some-id, but type __does-not-exist does not exist. Defined at /tmp/tmp.cdist.test.mma5j8ln/tmp.cdist.test.3zg4by4d
---
> test_nonexistent_type_requirement (cdist.test.emulator.EmulatorTestCase) ... ERROR: cdisttesthost: __file/tmp/foobar requires object __does-not-exist/some-id, but type __does-not-exist does not exist. Defined at /tmp/tmp.cdist.test.t8d6ockr/tmp.cdist.test.uimxurg9
86c84
< test_initial_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running initial manifest /tmp/tmp.cdist.test.uvid60ij/759547ff4356de6e3d9e08522b0d0807/data/conf/manifest/dump_environment
---
> test_initial_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running initial manifest /tmp/tmp.cdist.test._cttcnrj/759547ff4356de6e3d9e08522b0d0807/data/conf/manifest/dump_environment
89c87
< test_type_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running type manifest /tmp/tmp.cdist.test.k1i2onpb/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__dump_environment/manifest for object __dump_environment/whatever
---
> test_type_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running type manifest /tmp/tmp.cdist.test.ukr7lrzd/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__dump_environment/manifest for object __dump_environment/whatever
272c270
< Ran 225 tests in 44.457s
---
> Ran 225 tests in 43.750s
```
---
 cdist/test/__main__.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/cdist/test/__main__.py b/cdist/test/__main__.py
index c8c7df3b..8049c752 100644
--- a/cdist/test/__main__.py
+++ b/cdist/test/__main__.py
@@ -20,7 +20,7 @@
 #
 #
 
-import imp
+import importlib
 import os
 import sys
 import unittest
@@ -37,8 +37,9 @@ for possible_test in os.listdir(base_dir):
 
 suites = []
 for test_module in test_modules:
-    module_parameters = imp.find_module(test_module, [base_dir])
-    module = imp.load_module("cdist.test." + test_module, *module_parameters)
+    module_spec = importlib.util.find_spec("cdist.test.{}".format(test_module))
+    module = importlib.util.module_from_spec(module_spec)
+    module_spec.loader.exec_module(module)
 
     suite = unittest.defaultTestLoader.loadTestsFromModule(module)
     # print("Got suite: " + suite.__str__())

From 29662961731ef07fbc20213d0d87da1469f0d27c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 7 Dec 2020 19:47:52 +0100
Subject: [PATCH 162/366] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 6c272fd7..8b35901b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
+	* Core: Deal with deprecated imp in unit tests (Evil Ham)
+
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)
 

From c5ca4cd2e13516dfb55371c1600e32297c3343e9 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 7 Dec 2020 19:59:05 +0100
Subject: [PATCH 163/366] __block: securly quote via the quote function

Because the function already exists, it will be used for the file to be
changed, too. Therefor, no quotes are required for that value.

The prefix and suffix match was also improved: There is no regex check
any more (the regex did checked the whole line); instead it will do it
simple.
---
 cdist/conf/type/__block/gencode-remote | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/cdist/conf/type/__block/gencode-remote b/cdist/conf/type/__block/gencode-remote
index acdb3286..7a1f4064 100755
--- a/cdist/conf/type/__block/gencode-remote
+++ b/cdist/conf/type/__block/gencode-remote
@@ -46,28 +46,29 @@ fi
 
 remove_block() {
    cat << DONE
-tmpfile=\$(mktemp '${file}.cdist.XXXXXXXXXX')
+tmpfile=\$(mktemp ${quoted_file}.cdist.XXXXXXXXXX)
 # preserve ownership and permissions of existing file
-if [ -f '$file' ]; then
-   cp -p '$file' "\$tmpfile"
+if [ -f $quoted_file ]; then
+   cp -p $quoted_file "\$tmpfile"
 fi
-awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ '
+awk -v prefix=$(quote "$prefix") -v suffix=$(quote "$suffix") '
 {
-   if (match(\$0,prefix)) {
+   if (\$0 == prefix) {
       triggered=1
    }
    if (triggered) {
-      if (match(\$0,suffix)) {
+      if (\$0 == suffix) {
          triggered=0
       }
    } else {
       print
    }
-}' '$file' > "\$tmpfile"
-mv -f "\$tmpfile" '$file'
+}' $quoted_file > "\$tmpfile"
+mv -f "\$tmpfile" $quoted_file
 DONE
 }
 
+quoted_file="$(quote "$file")"
 case "$state_should" in
    present)
       if [ "$state_is" = "changed" ]; then
@@ -77,7 +78,7 @@ case "$state_should" in
          echo add >> "$__messages_out"
       fi
       cat << DONE
-cat >> '$file' << '${__type##*/}_DONE'
+cat >> $quoted_file << '${__type##*/}_DONE'
 $(cat "$block")
 ${__type##*/}_DONE
 DONE

From 14c81d6c7e20a2ed04c7318f6e8ecb51098390e8 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 8 Dec 2020 07:16:26 +0100
Subject: [PATCH 164/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 8b35901b..4be08ae9 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
+	* Type __iptables: Add IPv6 support (Matthias Stecher)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From a58f5ffa7f59a7e28ba0a414097f154e4a497e88 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 8 Dec 2020 19:36:44 +0100
Subject: [PATCH 165/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 4be08ae9..3d6084f1 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
 	* Type __iptables: Add IPv6 support (Matthias Stecher)
+	* Type __block: Fix escaping in here-doc (Matthias Stecher)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From a5169ad858a4d6e9c378184db5736f55a86306e0 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 10 Dec 2020 21:24:26 +0100
Subject: [PATCH 166/366] new type __debian_backports

This new type will setup the backports distribution for the current
Debian release.
---
 cdist/conf/type/__debian_backports/man.rst    | 90 +++++++++++++++++++
 cdist/conf/type/__debian_backports/manifest   | 59 ++++++++++++
 .../parameter/default/mirror                  |  1 +
 .../parameter/default/state                   |  1 +
 .../__debian_backports/parameter/optional     |  2 +
 cdist/conf/type/__debian_backports/singleton  |  0
 6 files changed, 153 insertions(+)
 create mode 100644 cdist/conf/type/__debian_backports/man.rst
 create mode 100755 cdist/conf/type/__debian_backports/manifest
 create mode 100644 cdist/conf/type/__debian_backports/parameter/default/mirror
 create mode 100644 cdist/conf/type/__debian_backports/parameter/default/state
 create mode 100644 cdist/conf/type/__debian_backports/parameter/optional
 create mode 100644 cdist/conf/type/__debian_backports/singleton

diff --git a/cdist/conf/type/__debian_backports/man.rst b/cdist/conf/type/__debian_backports/man.rst
new file mode 100644
index 00000000..ba353f4e
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/man.rst
@@ -0,0 +1,90 @@
+cdist-type__debian_backports(7)
+===============================
+
+NAME
+----
+cdist-type__debian_backports - Install backports for Debain systems
+
+
+DESCRIPTION
+-----------
+This singleton type installs backports for the current Debian version.
+It aborts if backports are not supported for the specified os or no
+version codename could be fetched (like Debian unstable).
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    Represents the state of the backports repository. ``present`` or
+    ``absent``, defaults to ``present``.
+
+    Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
+
+mirror
+    The mirror to fetch the backports from. Will defaults to the Debian default
+    `<http://deb.debian.org/debian/>`_.
+
+    Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+MESSAGES
+--------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+   # setup the backports
+   __debian_backports
+   __debian_backports --state absent
+   __debian_backports --state present --mirror "http://ftp.de.debian.org/debian/"
+
+   # update
+   require="__debian_backports" __apt_update_index
+
+   # install a backports package
+   # currently for the buster release backports
+   require="__apt_update_index" __package_apt wireguard \
+        --target-release buster-backports
+
+
+ABORTS
+------
+Aborts if the detected os is not Debian.
+
+Aborts if no distribuition codename could be detected. This is common for the
+unstable distribution, but there is no backports repository for it already.
+
+
+SEE ALSO
+--------
+`Official Debian Backports site <https://backports.debian.org/>`_
+
+:strong:`cdist-type__apt_source`\ (7)
+
+
+AUTHORS
+-------
+Matthias Stecher <matthiasstecher at gmx.de>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Matthias Stecher. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__debian_backports/manifest
new file mode 100755
index 00000000..29bf9a43
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/manifest
@@ -0,0 +1,59 @@
+#!/bin/sh -e
+# __debian_backports/manifest
+#
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Enables/disables backports repository. Utilies __apt_source for it.
+#
+
+
+# detect backport distribution
+os="$(cat "$__global/explorer/os")"
+case "$os" in
+    debian)
+        # distribution codename from /etc/os-release
+        # lsb_release may not be given in all debian installations
+        dist="$(
+            . "$__global/explorer/os-release"
+            printf "%s" "$VERSION_CODENAME"
+        )"
+        ;;
+    *)
+        printf "Backports for %s are not supported!\n" "$os" >&2
+        exit 1
+        ;;
+esac
+
+# error if no codename given (e.g. on Debian unstable)
+if [ -z "$dist" ]; then
+    printf "No backports for unkown version of distribution %s!\n" "$os" >&2
+    exit 1
+fi
+
+
+# parameters
+state="$(cat "$__object/parameter/state")"
+mirror="$(cat "$__object/parameter/mirror")"
+
+# install the given backports repository
+__apt_source "${dist}-backports" \
+    --state "$state" \
+    --distribution "${dist}-backports" \
+    --component main \
+    --uri "$mirror"
diff --git a/cdist/conf/type/__debian_backports/parameter/default/mirror b/cdist/conf/type/__debian_backports/parameter/default/mirror
new file mode 100644
index 00000000..0965ef04
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/parameter/default/mirror
@@ -0,0 +1 @@
+http://deb.debian.org/debian/
diff --git a/cdist/conf/type/__debian_backports/parameter/default/state b/cdist/conf/type/__debian_backports/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__debian_backports/parameter/optional b/cdist/conf/type/__debian_backports/parameter/optional
new file mode 100644
index 00000000..4b05c235
--- /dev/null
+++ b/cdist/conf/type/__debian_backports/parameter/optional
@@ -0,0 +1,2 @@
+state
+mirror
diff --git a/cdist/conf/type/__debian_backports/singleton b/cdist/conf/type/__debian_backports/singleton
new file mode 100644
index 00000000..e69de29b

From 0d96b31b5696f95b559450e17c71914d0746c0ce Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 11 Dec 2020 18:13:44 +0100
Subject: [PATCH 167/366] __debian_backports: pass shellcheck for sourced file

Because the sourced explorer can't be detected by shellcheck, it will be
completely disabled. Changing the path to /etc/os-release isn't
deterministic either.

The shellcheck wiki page suggests to use `source=/dev/null` instead of
`disable=SC1090`, but it was choosen to completely avoid that check ..
---
 cdist/conf/type/__debian_backports/manifest | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__debian_backports/manifest
index 29bf9a43..661e5281 100755
--- a/cdist/conf/type/__debian_backports/manifest
+++ b/cdist/conf/type/__debian_backports/manifest
@@ -30,6 +30,7 @@ case "$os" in
         # distribution codename from /etc/os-release
         # lsb_release may not be given in all debian installations
         dist="$(
+            # shellcheck disable=SC1090
             . "$__global/explorer/os-release"
             printf "%s" "$VERSION_CODENAME"
         )"

From bc2948a8a552e3bda3ab9ae8e8e199e831d81af2 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 11 Dec 2020 19:37:53 +0100
Subject: [PATCH 168/366] ++scan stuff

---
 cdist/scan/scan.py           | 15 +++++++++++++++
 docs/dev/logs/2020-10-29.org | 10 +++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index fcbf1899..00858581 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -56,6 +56,8 @@ from scapy.all import *
 # Datetime overwrites scapy.all.datetime - needs to be imported AFTER
 import datetime
 
+import cdist.config
+
 log = logging.getLogger("scan")
 
 class Trigger(object):
@@ -120,6 +122,19 @@ class Scanner(object):
             with open(fname, "w") as fd:
                 fd.write(f"{now}\n")
 
+    def config(self):
+        """
+        Configure a host
+
+        - Assume we are only called if necessary
+        - However we need to ensure to not run in parallel
+        - Maybe keep dict storing per host processes
+        - Save the result
+        - Save the output -> probably aligned to config mode
+
+        """
+
+
     def start(self):
         self.process = Process(target=self.scan)
         self.process.start()
diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org
index 4461be8c..03d6b3f4 100644
--- a/docs/dev/logs/2020-10-29.org
+++ b/docs/dev/logs/2020-10-29.org
@@ -54,4 +54,12 @@ VERBOSE: scan: Host fe80::f29f:c2ff:fe7c:275e is alive
 VERBOSE: scan: Host fe80::ba69:f4ff:fec5:8db7 is alive
 VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f863 is alive
 VERBOSE: scan: Host fe80::21b:fcff:feee:f4bc is alive
-...
+** Better usage -> saving the env
+    sudo -E cdist scan -b -I wlan0 -vv
+** TODO Implement actual configuration step
+   - Also serves as a nice PoC
+   - Might need to escape literal IPv6 addresses for scp
+** TODO Define how to map link local address to something useful
+   - via reverse DNS?
+   - via link local in manifest?
+** TODO define ignorehosts?

From c4d19a23193ec13eda96a3d7536ba338d0801825 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 09:36:17 +0100
Subject: [PATCH 169/366] __debian_backports -> __apt_backports; add wider os
 support

As discussed in the chat, this type now supports a broader list of OSes
which it supports backports for. Because of this, it was renamed to
something more generic. "apt" should fit in.
---
 .../man.rst                                   | 27 ++++++++----
 .../manifest                                  | 41 ++++++++++++++-----
 .../parameter/default/state                   |  0
 .../parameter/optional                        |  0
 .../singleton                                 |  0
 .../parameter/default/mirror                  |  1 -
 6 files changed, 49 insertions(+), 20 deletions(-)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/man.rst (66%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/manifest (59%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/parameter/default/state (100%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/parameter/optional (100%)
 rename cdist/conf/type/{__debian_backports => __apt_backports}/singleton (100%)
 delete mode 100644 cdist/conf/type/__debian_backports/parameter/default/mirror

diff --git a/cdist/conf/type/__debian_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
similarity index 66%
rename from cdist/conf/type/__debian_backports/man.rst
rename to cdist/conf/type/__apt_backports/man.rst
index ba353f4e..7d269fbb 100644
--- a/cdist/conf/type/__debian_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -3,13 +3,13 @@ cdist-type__debian_backports(7)
 
 NAME
 ----
-cdist-type__debian_backports - Install backports for Debain systems
+cdist-type__apt_backports - Install backports
 
 
 DESCRIPTION
 -----------
-This singleton type installs backports for the current Debian version.
-It aborts if backports are not supported for the specified os or no
+This singleton type installs backports for the current OS release.
+It aborts if backports are not supported for the specified OS or no
 version codename could be fetched (like Debian unstable).
 
 
@@ -27,8 +27,8 @@ state
     Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
 
 mirror
-    The mirror to fetch the backports from. Will defaults to the Debian default
-    `<http://deb.debian.org/debian/>`_.
+    The mirror to fetch the backports from. Will defaults to the generic
+    mirror of the current OS.
 
     Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
 
@@ -49,12 +49,12 @@ EXAMPLES
 .. code-block:: sh
 
    # setup the backports
-   __debian_backports
-   __debian_backports --state absent
-   __debian_backports --state present --mirror "http://ftp.de.debian.org/debian/"
+   __apt_backports
+   __apt_backports --state absent
+   __apt_backports --state present --mirror "http://ftp.de.debian.org/debian/"
 
    # update
-   require="__debian_backports" __apt_update_index
+   require="__apt_backports" __apt_update_index
 
    # install a backports package
    # currently for the buster release backports
@@ -70,6 +70,15 @@ Aborts if no distribuition codename could be detected. This is common for the
 unstable distribution, but there is no backports repository for it already.
 
 
+CAVEATS
+-------
+For Ubuntu, it setup all componenents for the backports repository: ``main``,
+``restricted``, ``universe`` and ``multiverse``. The user may not want to
+install proprietary packages, which will only be installed if the user
+explicitly uses the backports target-release. The user may change this behavior
+to install backports packages without the need of explicitly select it.
+
+
 SEE ALSO
 --------
 `Official Debian Backports site <https://backports.debian.org/>`_
diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__apt_backports/manifest
similarity index 59%
rename from cdist/conf/type/__debian_backports/manifest
rename to cdist/conf/type/__apt_backports/manifest
index 661e5281..e5358dea 100755
--- a/cdist/conf/type/__debian_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -1,5 +1,5 @@
 #!/bin/sh -e
-# __debian_backports/manifest
+# __apt_backports/manifest
 #
 # 2020 Matthias Stecher (matthiasstecher at gmx.de)
 #
@@ -23,18 +23,34 @@
 #
 
 
+# Get the distribution codename by /etc/os-release.
+#  is already executed in a subshell by string substitution
+#  lsb_release may not be given in all installations
+codename_os_release() {
+    # shellcheck disable=SC1090
+    . "$__global/explorer/os-release"
+    printf "%s" "$VERSION_CODENAME"
+}
+
 # detect backport distribution
 os="$(cat "$__global/explorer/os")"
 case "$os" in
     debian)
-        # distribution codename from /etc/os-release
-        # lsb_release may not be given in all debian installations
-        dist="$(
-            # shellcheck disable=SC1090
-            . "$__global/explorer/os-release"
-            printf "%s" "$VERSION_CODENAME"
-        )"
+        dist="$( codename_os_release )"
+        components="main"
+        mirror="http://deb.debian.org/debian/"
         ;;
+    devuan)
+        dist="$( codename_os_release )"
+        components="main"
+        mirror="http://deb.devuan.org/merged"
+        ;;
+    ubuntu)
+        dist="$( codename_os_release )"
+        components="main restricted universe multiverse"
+        mirror="http://archive.ubuntu.com/ubuntu"
+        ;;
+
     *)
         printf "Backports for %s are not supported!\n" "$os" >&2
         exit 1
@@ -50,11 +66,16 @@ fi
 
 # parameters
 state="$(cat "$__object/parameter/state")"
-mirror="$(cat "$__object/parameter/mirror")"
+
+# mirror already set for the os, only override user-values
+if [ -f "$__object/parameter/mirror" ]; then
+    mirror="$(cat "$__object/parameter/mirror")"
+fi
+
 
 # install the given backports repository
 __apt_source "${dist}-backports" \
     --state "$state" \
     --distribution "${dist}-backports" \
-    --component main \
+    --component "$components" \
     --uri "$mirror"
diff --git a/cdist/conf/type/__debian_backports/parameter/default/state b/cdist/conf/type/__apt_backports/parameter/default/state
similarity index 100%
rename from cdist/conf/type/__debian_backports/parameter/default/state
rename to cdist/conf/type/__apt_backports/parameter/default/state
diff --git a/cdist/conf/type/__debian_backports/parameter/optional b/cdist/conf/type/__apt_backports/parameter/optional
similarity index 100%
rename from cdist/conf/type/__debian_backports/parameter/optional
rename to cdist/conf/type/__apt_backports/parameter/optional
diff --git a/cdist/conf/type/__debian_backports/singleton b/cdist/conf/type/__apt_backports/singleton
similarity index 100%
rename from cdist/conf/type/__debian_backports/singleton
rename to cdist/conf/type/__apt_backports/singleton
diff --git a/cdist/conf/type/__debian_backports/parameter/default/mirror b/cdist/conf/type/__debian_backports/parameter/default/mirror
deleted file mode 100644
index 0965ef04..00000000
--- a/cdist/conf/type/__debian_backports/parameter/default/mirror
+++ /dev/null
@@ -1 +0,0 @@
-http://deb.debian.org/debian/

From 49aec0b5e40ba05e7703565731aae6b0a9e5936a Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 09:40:47 +0100
Subject: [PATCH 170/366] __apt_backports: list supported OSes

The manpage now lists all OSes where this type supports backports.
---
 cdist/conf/type/__apt_backports/man.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
index 7d269fbb..fd311361 100644
--- a/cdist/conf/type/__apt_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -12,6 +12,12 @@ This singleton type installs backports for the current OS release.
 It aborts if backports are not supported for the specified OS or no
 version codename could be fetched (like Debian unstable).
 
+It supports backports from following OSes:
+
+- Debian
+- Devuan
+- Ubuntu
+
 
 REQUIRED PARAMETERS
 -------------------

From fafa3d9ea55f38ae0a350a463d64b7ef1d70c7bb Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 10:00:23 +0100
Subject: [PATCH 171/366] __apt_backports: update index if required

This type now automatically calls the type __apt_update_index to update
the package index if something changed.
---
 cdist/conf/type/__apt_backports/man.rst  | 13 +++++++------
 cdist/conf/type/__apt_backports/manifest |  5 ++++-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
index fd311361..c578ce6b 100644
--- a/cdist/conf/type/__apt_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -9,8 +9,11 @@ cdist-type__apt_backports - Install backports
 DESCRIPTION
 -----------
 This singleton type installs backports for the current OS release.
-It aborts if backports are not supported for the specified OS or no
-version codename could be fetched (like Debian unstable).
+It aborts if backports are not supported for the specified OS or
+no version codename could be fetched (like Debian unstable).
+
+The package index will be automatically updated by the type
+:strong:`cdist-type__apt_update_index`\ (7) if required.
 
 It supports backports from following OSes:
 
@@ -59,12 +62,9 @@ EXAMPLES
    __apt_backports --state absent
    __apt_backports --state present --mirror "http://ftp.de.debian.org/debian/"
 
-   # update
-   require="__apt_backports" __apt_update_index
-
    # install a backports package
    # currently for the buster release backports
-   require="__apt_update_index" __package_apt wireguard \
+   require="__apt_backports" __package_apt wireguard \
         --target-release buster-backports
 
 
@@ -90,6 +90,7 @@ SEE ALSO
 `Official Debian Backports site <https://backports.debian.org/>`_
 
 :strong:`cdist-type__apt_source`\ (7)
+:strong:`cdist-type__apt_update_index`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index e5358dea..c490a103 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -19,7 +19,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Enables/disables backports repository. Utilies __apt_source for it.
+# Enables/disables backports repository. Utilises __apt_source for it.
 #
 
 
@@ -79,3 +79,6 @@ __apt_source "${dist}-backports" \
     --distribution "${dist}-backports" \
     --component "$components" \
     --uri "$mirror"
+
+# update the index if the source changed
+require="__apt_source/${dist}-backports" __apt_update_index

From 645734c62959c694597826598be3b34c0edc79d2 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 12 Dec 2020 12:15:17 +0100
Subject: [PATCH 172/366] [explorer/os_version] Improve FreeBSD support.

It looks like uname -r is not the most reliable way to get the target patch
level for the target system.

For more information see:
https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
---
 cdist/conf/explorer/os_version | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index a7b1d3bc..3b02dedd 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -70,6 +70,11 @@ case "$("$__explorer/os")" in
    macosx)
       sw_vers -productVersion
    ;;
+   freebsd)
+      # Apparently uname -r is not a reliable way to get the patch level.
+      # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
+      freebsd-version
+   ;;
    *bsd|solaris)
       uname -r
    ;;

From fca35fc858d09fc649dc5e7f0964cef4af9e09f0 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 17:29:58 +0100
Subject: [PATCH 173/366] __apt_backports: fix explorer call

s/-/_/ because the explorers are following an other convention :-)
---
 cdist/conf/type/__apt_backports/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index c490a103..c2943725 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -28,7 +28,7 @@
 #  lsb_release may not be given in all installations
 codename_os_release() {
     # shellcheck disable=SC1090
-    . "$__global/explorer/os-release"
+    . "$__global/explorer/os_release"
     printf "%s" "$VERSION_CODENAME"
 }
 

From 27aca06fb893c84601e14bc30890ea6be9300dcd Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Sat, 12 Dec 2020 17:34:51 +0100
Subject: [PATCH 174/366] __apt_backports: undo __apt_update_index call

Becuase it is already done by __apt_source.
---
 cdist/conf/type/__apt_backports/man.rst  | 4 +---
 cdist/conf/type/__apt_backports/manifest | 3 ---
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst
index c578ce6b..7036fb84 100644
--- a/cdist/conf/type/__apt_backports/man.rst
+++ b/cdist/conf/type/__apt_backports/man.rst
@@ -12,8 +12,7 @@ This singleton type installs backports for the current OS release.
 It aborts if backports are not supported for the specified OS or
 no version codename could be fetched (like Debian unstable).
 
-The package index will be automatically updated by the type
-:strong:`cdist-type__apt_update_index`\ (7) if required.
+The package index will be automatically updated if required.
 
 It supports backports from following OSes:
 
@@ -90,7 +89,6 @@ SEE ALSO
 `Official Debian Backports site <https://backports.debian.org/>`_
 
 :strong:`cdist-type__apt_source`\ (7)
-:strong:`cdist-type__apt_update_index`\ (7)
 
 
 AUTHORS
diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index c2943725..bc47d8de 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -79,6 +79,3 @@ __apt_source "${dist}-backports" \
     --distribution "${dist}-backports" \
     --component "$components" \
     --uri "$mirror"
-
-# update the index if the source changed
-require="__apt_source/${dist}-backports" __apt_update_index

From 71f22831175ec41e9348a759fcdaf0eceb1a2a52 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 13 Dec 2020 16:03:39 +0100
Subject: [PATCH 175/366] ++changelog

---
 docs/changelog | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 3d6084f1..290f94d0 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -2,10 +2,12 @@ Changelog
 ---------
 
 next:
-	* __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
+	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
 	* Type __iptables: Add IPv6 support (Matthias Stecher)
 	* Type __block: Fix escaping in here-doc (Matthias Stecher)
+	* Explorer os_version: Improve FreeBSD support (Evil Ham)
+	* New type: __apt_backports (Matthias Stecher)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From 932e2496ed87b830a468a263cf1b1f5e6a88e2ef Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 15 Dec 2020 18:40:39 +0100
Subject: [PATCH 176/366] [type/__postgres_role] Lint

---
 .../conf/type/__postgres_role/explorer/state  | 32 ++++----
 .../conf/type/__postgres_role/gencode-remote  | 78 ++++++++++---------
 2 files changed, 58 insertions(+), 52 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index c8e1fa9d..110d29d5 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -11,32 +11,32 @@
 #
 # cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$("${__explorer}/os")"
+case $("${__explorer:?}/os")
 in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
+	(netbsd)
+		postgres_user='pgsql'
+		;;
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(*)
+		postgres_user='postgres'
+		;;
 esac
 
 
-name="$__object_id"
+rolename=${__object_id:?}
 
-if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='$name'\"")"
+if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='${rolename}'\"")"
 then
-    echo 'present'
+	echo 'present'
 else
-    echo 'absent'
+	echo 'absent'
 fi
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 282294c9..4fb761ed 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -11,55 +11,61 @@
 #
 # cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
+case $(cat "${__global:?}/explorer/os")
 in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
+	(netbsd)
+		postgres_user='pgsql'
+		;;
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(*)
+		postgres_user='postgres'
+		;;
 esac
 
 
-name="$__object_id"
-state_is="$(cat "$__object/explorer/state")"
-state_should="$(cat "$__object/parameter/state")"
+rolename=${__object_id:?}
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
 
-[ "$state_is" = "$state_should" ] && exit 0
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
 
-case "$state_should" in
-    present)
-        if [ -f "$__object/parameter/password" ]; then
-            password="$(cat "$__object/parameter/password")"
-        fi
-        booleans=""
-        for boolean in login createdb createrole superuser; do
-            if [ ! -f "$__object/parameter/$boolean" ]; then
-                boolean="no${boolean}"
-            fi
-            upper=$(echo $boolean | tr '[:lower:]' '[:upper:]')
-            booleans="$booleans $upper"
-        done
+case ${state_should}
+in
+	(present)
+		if test -f "${__object:?}/parameter/password"
+		then
+			password=$(cat "${__object:?}/parameter/password")
+		fi
+		booleans=
+		for boolean in login createdb createrole superuser
+		do
+			if test ! -f "${__object:?}/parameter/${boolean}"
+			then
+				boolean="no${boolean}"
+			fi
+			booleans="${booleans} $(echo ${boolean} | tr '[:lower:]' '[:upper:]')"
+		done
 
-        [ -n "$password" ] && password="PASSWORD '$password'"
-        cat << EOF
-su - '$postgres_user' -c "psql postgres -wc \"CREATE ROLE \\\\\"$name\\\\\" WITH $password $booleans;\""
+		[ -n "${password}" ] && password="PASSWORD '${password}'"
+		cat << EOF
+su - '${postgres_user}' -c "psql postgres -wc 'CREATE ROLE \\"${rolename}\\" WITH ${password} ${booleans};'"
 EOF
-    ;;
-    absent)
-        cat << EOF
-su - '$postgres_user' -c "dropuser \"$name\""
+		;;
+	(absent)
+		cat << EOF
+su - '${postgres_user}' -c "dropuser '${rolename}'"
 EOF
-    ;;
+		;;
 esac

From c36df82882fedb6ce3630e4718c46992a7c153a0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 15 Dec 2020 21:05:55 +0100
Subject: [PATCH 177/366] [type/__postgres_role] ALTER ROLE when parameters
 change

---
 .../conf/type/__postgres_role/explorer/state  | 42 +++++++++++-
 .../conf/type/__postgres_role/gencode-remote  | 65 +++++++++++++++----
 2 files changed, 91 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 110d29d5..033afcb2 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -31,11 +32,48 @@ in
 		;;
 esac
 
-
 rolename=${__object_id:?}
 
-if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='${rolename}'\"")"
+role_properties=$(
+	cmd=$(printf "psql -F '\034' -R '\036' -wAc \"SELECT * FROM pg_roles WHERE rolname='%s'\"" "${rolename}")
+	su -l "${postgres_user}" -c "${cmd}" \
+	| awk '
+	  BEGIN { RS = "\036"; FS = "\034" }
+	  /^\([0-9]+ rows?\)/ { exit }
+	  NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next }
+	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }
+	  '
+)
+
+if test -n "${role_properties}"
 then
+	# Check if the user's properties match the parameters
+	for prop in login createdb createrole superuser
+	do
+		bool_should=$(test -f "${__object:?}/parameter/${prop}" && echo 't' || echo 'f')
+		bool_is=$(
+			printf '%s\n' "${role_properties}" |
+			awk -F '=' -v key="${prop}" '
+			BEGIN {
+				if (key == "login")
+					key = "canlogin"
+				else if (key == "superuser")
+					key = "super"
+				key = "rol" key
+			}
+			$1 == key {
+				sub(/^[^=]*=/, "")
+				print
+			}
+			'
+		)
+
+		test "${bool_is}" = "${bool_should}" || {
+			echo 'different'
+			exit 0
+		}
+	done
+
 	echo 'present'
 else
 	echo 'absent'
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 4fb761ed..7837976c 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,6 +19,15 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+quote() {
+	if test $# -gt 0
+	then
+		printf '%s' "$*"
+	else
+		cat -
+	fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
+}
+
 case $(cat "${__global:?}/explorer/os")
 in
 	(netbsd)
@@ -44,28 +54,55 @@ fi
 case ${state_should}
 in
 	(present)
-		if test -f "${__object:?}/parameter/password"
+		if test -s "${__object:?}/parameter/password"
 		then
-			password=$(cat "${__object:?}/parameter/password")
+			quoted_password=$(
+				delim='$$'
+				while grep -q -F "${delim}" "${__object:?}/parameter/password"
+				do
+					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
+				done
+
+				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
+				# shellcheck disable=SC2016
+				printf '%s%s%s' "${delim}" "${raw_passwd%?.}" "${delim}"
+			)
 		fi
+
 		booleans=
 		for boolean in login createdb createrole superuser
 		do
-			if test ! -f "${__object:?}/parameter/${boolean}"
-			then
-				boolean="no${boolean}"
-			fi
-			booleans="${booleans} $(echo ${boolean} | tr '[:lower:]' '[:upper:]')"
+			booleans="${booleans}${booleans:+ }$(
+				if test -f "${__object:?}/parameter/${boolean}"
+				then
+					echo "${boolean}"
+				else
+					echo "no${boolean}"
+				fi \
+				| tr '[:lower:]' '[:upper:]')"
 		done
 
-		[ -n "${password}" ] && password="PASSWORD '${password}'"
-		cat << EOF
-su - '${postgres_user}' -c "psql postgres -wc 'CREATE ROLE \\"${rolename}\\" WITH ${password} ${booleans};'"
-EOF
+		case ${state_is}
+		in
+			(absent)
+				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				;;
+			(different)
+				query=$(printf 'ALTER ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				;;
+			(*)
+				exit 1	# TODO: error msg
+				;;
+		esac
+
+		psql_cmd=$(printf 'psql postgres -wc %s' "$(quote "${query}")" | quote)
+		printf "su -l '%s' -c %s\\n" "${postgres_user}" "${psql_cmd}"
 		;;
 	(absent)
-		cat << EOF
-su - '${postgres_user}' -c "dropuser '${rolename}'"
-EOF
+		printf "su -l '%s' -c 'dropuser '\\\\'%s\\\\'\\n" \
+			"${postgres_user}" \
+			"$(quote "${rolename}")"
 		;;
 esac

From 7b7ca4d385ccd025cbefced58419468651652da8 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Wed, 16 Dec 2020 19:06:50 +0100
Subject: [PATCH 178/366] [type/__postgres_role] Handle password changes

---
 .../conf/type/__postgres_role/explorer/state  | 76 +++++++++++++++++--
 .../conf/type/__postgres_role/gencode-remote  | 23 ++++--
 2 files changed, 88 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 033afcb2..4aadbdff 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -34,9 +34,23 @@ esac
 
 rolename=${__object_id:?}
 
+
+psql_query() {
+	su -l "${postgres_user}" -c "$(
+		printf "psql -F '\034' -R '\036' -wAc '%s'" \
+		"$(printf %s "$*" | sed "s/'/'\\\\''/g")"
+	)"
+}
+
+password_check_login() (
+	PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .)
+	PGPASSWORD=${PGPASSWORD%?.}
+	export PGPASSWORD
+	psql -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
+)
+
 role_properties=$(
-	cmd=$(printf "psql -F '\034' -R '\036' -wAc \"SELECT * FROM pg_roles WHERE rolname='%s'\"" "${rolename}")
-	su -l "${postgres_user}" -c "${cmd}" \
+	psql_query "SELECT * FROM pg_roles WHERE rolname = '${rolename}'" \
 	| awk '
 	  BEGIN { RS = "\036"; FS = "\034" }
 	  /^\([0-9]+ rows?\)/ { exit }
@@ -69,12 +83,62 @@ then
 		)
 
 		test "${bool_is}" = "${bool_should}" || {
-			echo 'different'
-			exit 0
+			state='different properties'
 		}
 	done
 
-	echo 'present'
+	# Check password
+	passwd_stored=$(
+		psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \
+		| awk 'BEGIN { RS = "\036" } NR == 2'
+		printf .
+	)
+	passwd_stored=${passwd_stored%?.}
+
+	passwd_should=$(cat "${__object}/parameter/password"; printf .)
+	passwd_should=${passwd_should%?.}
+
+	if expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
+	then
+		# SCRAM-SHA-256 "encrypted" password
+		# NOTE: There is currently no easy way to check SCRAM passwords
+		password_check_login || state="${state:-different} password"
+	elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null
+	then
+		# MD5 "encrypted" password
+		if command -v md5sum >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| md5sum - | sed -e 's/[^0-9a-f]*$//')
+		elif command -v gmd5sum >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| gmd5sum - | sed -e 's/[^0-9a-f]*$//')
+		elif command -v openssl >/dev/null 2>&1
+		then
+			should_md5=$(
+				printf '%s%s' "${passwd_should}" "${rolename}" \
+				| openssl dgst -md5 | sed 's/^.* //')
+		fi
+
+		if test -n "${should_md5}"
+		then
+			test "${passwd_stored}" = "md5${should_md5}" \
+			|| state="${state:-different} password"
+		else
+			password_check_login || state="${state:-different} password"
+		fi
+	else
+		# unencrypted password (unsupported since PostgreSQL 10)
+		test "${passwd_stored}" = "${passwd_should}" \
+		|| state="${state:-different} password"
+	fi
+
+	test -n "${state}" || state='present'
 else
-	echo 'absent'
+	state='absent'
 fi
+
+echo "${state}"
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 7837976c..fd590a4b 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -58,7 +58,9 @@ in
 		then
 			quoted_password=$(
 				delim='$$'
-				while grep -q -F "${delim}" "${__object:?}/parameter/password"
+				# NOTE: Strip away trailing $ because with it the check breaks
+				#       if the password ends with $ + random value.
+				while grep -q -F "${delim%$}" "${__object:?}/parameter/password"
 				do
 					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
 				done
@@ -88,12 +90,23 @@ in
 				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
 					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
 				;;
-			(different)
-				query=$(printf 'ALTER ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+			(different*)
+				query="ALTER ROLE \"${rolename}\" WITH"
+
+				if expr "${state_is}" : 'different.*properties' >/dev/null
+				then
+					query="${query} ${booleans}"
+				fi
+				if expr "${state_is}" : 'different.*password' >/dev/null
+				then
+					query="${query} PASSWORD ${quoted_password:-NULL}"
+				fi
+
+				query="${query};"
 				;;
 			(*)
-				exit 1	# TODO: error msg
+				printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2
+				exit 1
 				;;
 		esac
 

From 4859c27900d125841c703e0270dd8c0eae05d504 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 16:57:03 +0100
Subject: [PATCH 179/366] [type/__postgres_role] Refactor gencode-remote

---
 .../conf/type/__postgres_role/gencode-remote  | 65 ++++++++++---------
 1 file changed, 34 insertions(+), 31 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index fd590a4b..540eb606 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -51,9 +51,6 @@ then
 	exit 0
 fi
 
-case ${state_should}
-in
-	(present)
 		if test -s "${__object:?}/parameter/password"
 		then
 			quoted_password=$(
@@ -64,6 +61,11 @@ in
 				do
 					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
 				done
+psql_query() {
+	printf 'su -l %s -c %s\n' \
+		"$(quote "${postgres_user}")" \
+		"$(quote "psql postgres -wc $(quote "$1")")"
+}
 
 				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
 				# shellcheck disable=SC2016
@@ -71,51 +73,52 @@ in
 			)
 		fi
 
-		booleans=
-		for boolean in login createdb createrole superuser
-		do
-			booleans="${booleans}${booleans:+ }$(
-				if test -f "${__object:?}/parameter/${boolean}"
-				then
-					echo "${boolean}"
-				else
-					echo "no${boolean}"
-				fi \
-				| tr '[:lower:]' '[:upper:]')"
-		done
+role_properties_should() {
+	_props=
+	for _prop in login createdb createrole superuser
+	do
+		_props="${_props}${_props:+ }$(
+			if test -f "${__object:?}/parameter/${_prop}"
+			then
+				echo "${_prop}"
+			else
+				echo "no${_prop}"
+			fi \
+			| tr '[:lower:]' '[:upper:]')"
+	done
+	printf '%s\n' "${_props}"
+	unset _prop _props
+}
 
+case ${state_should}
+in
+	(present)
 		case ${state_is}
 		in
 			(absent)
-				query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "${booleans}" "${quoted_password:-NULL}")
+				psql_query "$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
+					"${rolename}" "$(role_properties_should)" "${quoted_password:-NULL}")"
 				;;
 			(different*)
-				query="ALTER ROLE \"${rolename}\" WITH"
-
 				if expr "${state_is}" : 'different.*properties' >/dev/null
 				then
-					query="${query} ${booleans}"
-				fi
-				if expr "${state_is}" : 'different.*password' >/dev/null
-				then
-					query="${query} PASSWORD ${quoted_password:-NULL}"
+					psql_query "ALTER ROLE \"${rolename}\" WITH $(role_properties_should);"
 				fi
 
-				query="${query};"
+				if expr "${state_is}" : 'different.*password' >/dev/null
+				then
+					psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD ${quoted_password:-NULL};"
+				fi
 				;;
 			(*)
 				printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2
 				exit 1
 				;;
 		esac
-
-		psql_cmd=$(printf 'psql postgres -wc %s' "$(quote "${query}")" | quote)
-		printf "su -l '%s' -c %s\\n" "${postgres_user}" "${psql_cmd}"
 		;;
 	(absent)
-		printf "su -l '%s' -c 'dropuser '\\\\'%s\\\\'\\n" \
-			"${postgres_user}" \
-			"$(quote "${rolename}")"
+		printf 'su -l %s -c %s\n' \
+			"$(quote "${postgres_user}")" \
+			"$(quote "dropuser $(quote "${rolename}")")"
 		;;
 esac

From 1180f13ed6c12e0277bebdf32bdd9840320d0fb9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 16:58:32 +0100
Subject: [PATCH 180/366] [type/__postgres_role] Fix setting password

We need to make sure that the password does not end up in ~/.psql_history.
---
 .../conf/type/__postgres_role/gencode-remote  | 38 ++++++++++---------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 540eb606..15c3e692 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -51,27 +51,29 @@ then
 	exit 0
 fi
 
-		if test -s "${__object:?}/parameter/password"
-		then
-			quoted_password=$(
-				delim='$$'
-				# NOTE: Strip away trailing $ because with it the check breaks
-				#       if the password ends with $ + random value.
-				while grep -q -F "${delim%$}" "${__object:?}/parameter/password"
-				do
-					delim="\$$(LC_ALL=C tr -cd '[:alpha:]' </dev/urandom | dd bs=1 count=4 2>/dev/null)$"
-				done
 psql_query() {
 	printf 'su -l %s -c %s\n' \
 		"$(quote "${postgres_user}")" \
 		"$(quote "psql postgres -wc $(quote "$1")")"
 }
 
-				raw_passwd=$(cat "${__object:?}/parameter/password"; printf .)
-				# shellcheck disable=SC2016
-				printf '%s%s%s' "${delim}" "${raw_passwd%?.}" "${delim}"
-			)
-		fi
+psql_set_password() {
+	# NOTE: Always make sure that the password does not end up in psql_history!
+	if test -s "${__object:?}/parameter/password"
+	then
+		cat <<-EOF
+		exec 3< "\${__object:?}/parameter/password"
+		su -l '${postgres_user}' -c 'psql -q postgres -w' <<'SQL'
+		\set HISTFILE /dev/null
+		\set pw \`cat <&3\`
+		ALTER ROLE "${rolename}" WITH PASSWORD :'pw';
+		SQL
+		exec 3<&-
+		EOF
+	else
+		psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD NULL;"
+	fi
+}
 
 role_properties_should() {
 	_props=
@@ -96,8 +98,8 @@ in
 		case ${state_is}
 		in
 			(absent)
-				psql_query "$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \
-					"${rolename}" "$(role_properties_should)" "${quoted_password:-NULL}")"
+				psql_query "CREATE ROLE \"${rolename}\" WITH $(role_properties_should);"
+				psql_set_password
 				;;
 			(different*)
 				if expr "${state_is}" : 'different.*properties' >/dev/null
@@ -107,7 +109,7 @@ in
 
 				if expr "${state_is}" : 'different.*password' >/dev/null
 				then
-					psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD ${quoted_password:-NULL};"
+					psql_set_password
 				fi
 				;;
 			(*)

From 99d82fd0d5ab6d6ccd95fe8a7d3b24f5d82a4cce Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 17 Dec 2020 17:05:58 +0100
Subject: [PATCH 181/366] [type/__postgres_role] Always set psql -q

---
 cdist/conf/type/__postgres_role/explorer/state | 4 ++--
 cdist/conf/type/__postgres_role/gencode-remote | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 4aadbdff..77779662 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -37,7 +37,7 @@ rolename=${__object_id:?}
 
 psql_query() {
 	su -l "${postgres_user}" -c "$(
-		printf "psql -F '\034' -R '\036' -wAc '%s'" \
+		printf "psql -q -F '\034' -R '\036' -wAc '%s'" \
 		"$(printf %s "$*" | sed "s/'/'\\\\''/g")"
 	)"
 }
@@ -46,7 +46,7 @@ password_check_login() (
 	PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .)
 	PGPASSWORD=${PGPASSWORD%?.}
 	export PGPASSWORD
-	psql -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
+	psql -q -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1
 )
 
 role_properties=$(
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 15c3e692..877c8135 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -54,7 +54,7 @@ fi
 psql_query() {
 	printf 'su -l %s -c %s\n' \
 		"$(quote "${postgres_user}")" \
-		"$(quote "psql postgres -wc $(quote "$1")")"
+		"$(quote "psql postgres -q -w -c $(quote "$1")")"
 }
 
 psql_set_password() {
@@ -63,7 +63,7 @@ psql_set_password() {
 	then
 		cat <<-EOF
 		exec 3< "\${__object:?}/parameter/password"
-		su -l '${postgres_user}' -c 'psql -q postgres -w' <<'SQL'
+		su -l '${postgres_user}' -c 'psql -q -w postgres' <<'SQL'
 		\set HISTFILE /dev/null
 		\set pw \`cat <&3\`
 		ALTER ROLE "${rolename}" WITH PASSWORD :'pw';

From 8dc2c4207cdd826b9e64cc91cfc7c918cddd5f35 Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Fri, 18 Dec 2020 11:16:28 +0100
Subject: [PATCH 182/366] Added optional dirmode parameter to set the mode of
 (optional) the directory.

---
 cdist/conf/type/__dot_file/man.rst                   | 3 +++
 cdist/conf/type/__dot_file/manifest                  | 2 ++
 cdist/conf/type/__dot_file/parameter/default/dirmode | 1 +
 cdist/conf/type/__dot_file/parameter/optional        | 1 +
 4 files changed, 7 insertions(+)
 create mode 100644 cdist/conf/type/__dot_file/parameter/default/dirmode

diff --git a/cdist/conf/type/__dot_file/man.rst b/cdist/conf/type/__dot_file/man.rst
index ae65eb95..ba7621a1 100644
--- a/cdist/conf/type/__dot_file/man.rst
+++ b/cdist/conf/type/__dot_file/man.rst
@@ -25,6 +25,9 @@ user
 OPTIONAL PARAMETERS
 -------------------
 
+dirmode
+    forwarded to :strong:`__directory` type as mode
+
 mode
     forwarded to :strong:`__file` type
 
diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest
index 5e4957e5..02dadf05 100755
--- a/cdist/conf/type/__dot_file/manifest
+++ b/cdist/conf/type/__dot_file/manifest
@@ -19,6 +19,7 @@ set -eu
 user="$(cat "${__object}/parameter/user")"
 home="$(cat "${__object}/explorer/home")"
 primary_group="$(cat "${__object}/explorer/primary_group")"
+dirmode="$(cat "${__object}/parameter/dirmode")"
 
 # Create parent directory. Type __directory has flag 'parents', but it
 # will leave us with root-owned directory in user home, which is not
@@ -36,6 +37,7 @@ export CDIST_ORDER_DEPENDENCY
 for dir ; do
 	__directory "${home}/${dir}"   \
 	    --group "${primary_group}" \
+            --mode "${dirmode}" \
 	    --owner "${user}"
 done
 
diff --git a/cdist/conf/type/__dot_file/parameter/default/dirmode b/cdist/conf/type/__dot_file/parameter/default/dirmode
new file mode 100644
index 00000000..e9745d1f
--- /dev/null
+++ b/cdist/conf/type/__dot_file/parameter/default/dirmode
@@ -0,0 +1 @@
+0700
diff --git a/cdist/conf/type/__dot_file/parameter/optional b/cdist/conf/type/__dot_file/parameter/optional
index ccab9fa6..9f7f83fb 100644
--- a/cdist/conf/type/__dot_file/parameter/optional
+++ b/cdist/conf/type/__dot_file/parameter/optional
@@ -1,3 +1,4 @@
 state
 mode
 source
+dirmode

From 4bae2863dbc1f6b60b8e797c2f121e1616a94c94 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 18 Dec 2020 12:54:33 +0100
Subject: [PATCH 183/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 290f94d0..3a623236 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __block: Fix escaping in here-doc (Matthias Stecher)
 	* Explorer os_version: Improve FreeBSD support (Evil Ham)
 	* New type: __apt_backports (Matthias Stecher)
+	* Type __dot_file: Add dirmode parameter (Mark Verboom)
 
 6.9.3: 2020-12-04
 	* pip install: Add cdist.scan to packages in setup.py (Dennis Camera)

From 7cf85c465980c74049f8e0258758c0fd8ea178b2 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 21 Dec 2020 19:21:51 +0100
Subject: [PATCH 184/366] Release 6.9.4

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 3a623236..35953a88 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)
 	* Type __iptables: Add IPv6 support (Matthias Stecher)

From 766198912d558e4630b258bba62e78587b3f0efe Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 5 Jan 2021 15:50:21 +0100
Subject: [PATCH 185/366] [type/__sshd_config] Produce error if invalid config
 file is generated

Previously, cdist would silently swallow the error (no invalid config file was
generated).

Reason: `set -e` does not exit if a command in a sub-command group fails,
it merely returns with a non-zero exit status.

e.g. the following snippet does not abort the script if sshd -t returns with a
non-zero exit status:

    set -e
    cmp -s old new || {
        # check config file and update it
        sshd -t -f new \
        && cat new >old
    }

or compressed:

    set -e
    false || { false && true; }
    echo $?
    # prints 1
---
 cdist/conf/type/__sshd_config/gencode-remote | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sshd_config/gencode-remote b/cdist/conf/type/__sshd_config/gencode-remote
index 0b44dfa7..275db4aa 100755
--- a/cdist/conf/type/__sshd_config/gencode-remote
+++ b/cdist/conf/type/__sshd_config/gencode-remote
@@ -91,7 +91,8 @@ awk $(drop_awk_comments "${__type:?}/files/update_sshd_config.awk") \\
 
 cmp -s $(quote "${sshd_config_file}") $(quote "${sshd_config_file}.tmp") || {
 	sshd -t -f $(quote "${sshd_config_file}.tmp") \\
-	&& cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}")
+	&& cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}") \\
+	|| exit  # stop if sshd_config file check fails
 }
 rm -f $(quote "${sshd_config_file}.tmp")
 EOF

From 8753b7eedf022a052ddc0b27fbc58ebf8fd638e1 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 5 Jan 2021 15:57:39 +0100
Subject: [PATCH 186/366] [type/__sshd_config] Make AuthenticationMethods and
 AuthorizedKeysFile singleton options

They were incorrectly treated as non-singleton options before.

cf. https://github.com/openssh/openssh-portable/blob/V_8_4/servconf.c#L2273
and https://github.com/openssh/openssh-portable/blob/V_8_4/servconf.c#L1899 resp.
---
 cdist/conf/type/__sshd_config/files/update_sshd_config.awk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
index d0bc2b4b..f7f30e87 100644
--- a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
+++ b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk
@@ -89,7 +89,7 @@ function strdelim(s) { return strdelim_internal(s, 1) }
 function strdelimw(s) { return strdelim_internal(s, 0) }
 
 function singleton_option(opt) {
-	return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|authenticationmethods|authorizedkeysfile|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/
+	return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/
 }
 
 function print_update() {

From bd8ab8f26fdc242f2eb77e58049d640f96d51a69 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 5 Jan 2021 17:00:55 +0100
Subject: [PATCH 187/366] [type/__sshd_config] Document "bug" in state explorer

---
 cdist/conf/type/__sshd_config/man.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cdist/conf/type/__sshd_config/man.rst b/cdist/conf/type/__sshd_config/man.rst
index 8b0069ac..c8e6b8ad 100644
--- a/cdist/conf/type/__sshd_config/man.rst
+++ b/cdist/conf/type/__sshd_config/man.rst
@@ -79,6 +79,10 @@ BUGS
 - ``Include`` directives are ignored.
 - Config options are not added/removed to/from the config file if their value is
   the default value.
+- | The explorer will incorrectly report ``absent`` if OpenSSH internally
+    transforms one value to another (e.g. ``permitrootlogin prohibit-password``
+    is transformed to ``permitrootlogin without-password``).
+  | Workaround: Use the value that OpenSSH uses internally.
 
 
 AUTHORS

From c819548343ad7bb24d023f94a32575e1b2520328 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 11 Jan 2021 09:50:12 +0100
Subject: [PATCH 188/366] Fix debug parameter

-d was removed from cdist in favor of mulitple -v and -l parameters, but
-d was not removed from preos.

Resolve #849.
---
 cdist/preos/debootstrap/files/code | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/preos/debootstrap/files/code b/cdist/preos/debootstrap/files/code
index 9e37003b..d836848c 100755
--- a/cdist/preos/debootstrap/files/code
+++ b/cdist/preos/debootstrap/files/code
@@ -22,7 +22,7 @@ set -e
 if [ "${debug}" ]
 then
     set -x
-    cdist_params="${cdist_params} -d"
+    cdist_params="${cdist_params} -l 3"
 fi
 
 bootstrap_dir="${target_dir}"

From 2954347771e00610a084c7b033016765f7a3d6d7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 14 Jan 2021 13:46:40 +0100
Subject: [PATCH 189/366] [type/__postgres_role] Add note regarding empty
 passwords

---
 cdist/conf/type/__postgres_role/gencode-remote | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 877c8135..d7631fbd 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -59,6 +59,8 @@ psql_query() {
 
 psql_set_password() {
 	# NOTE: Always make sure that the password does not end up in psql_history!
+	# NOTE: Never set an empty string as the password, because they can be
+	#       interpreted differently by different tooling.
 	if test -s "${__object:?}/parameter/password"
 	then
 		cat <<-EOF

From 6e9b13d94962e1bfb365efa849ebefc32854364b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 18 Jan 2021 06:22:32 +0100
Subject: [PATCH 190/366] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 35953a88..b2b35616 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,10 @@
 Changelog
 ---------
 
+next:
+	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
+	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
+
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)
 	* Core: Deal with deprecated imp in unit tests (Evil Ham)

From 92a50da4873ff4bfa85dde3b85e272704cda02c7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 18 Jan 2021 06:28:09 +0100
Subject: [PATCH 191/366] Fix pycodestyle issues

---
 cdist/scan/scan.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 5ca8fae2..b1d0e9e1 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -139,7 +139,6 @@ class Scanner(object):
 
         """
 
-
     def start(self):
         self.process = Process(target=self.scan)
         self.process.start()

From 35cde3e666c229ceb94a1c5074083f7ba905768a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 13:09:29 +0100
Subject: [PATCH 192/366] [type/__postgres_role] Fix state explorer when stored
 password is empty

---
 cdist/conf/type/__postgres_role/explorer/state | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 77779662..34069de9 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -95,13 +95,20 @@ then
 	)
 	passwd_stored=${passwd_stored%?.}
 
-	passwd_should=$(cat "${__object}/parameter/password"; printf .)
+	if test -f "${__object:?}/parameter/password"
+	then
+		passwd_should=$(cat "${__object:?}/parameter/password"; printf .)
+	fi
 	passwd_should=${passwd_should%?.}
 
-	if expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
+	if test -z "${passwd_stored}"
+	then
+		test -z "${passwd_should}" || state="${state:-different} password"
+	elif expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null
 	then
 		# SCRAM-SHA-256 "encrypted" password
-		# NOTE: There is currently no easy way to check SCRAM passwords
+		# NOTE: There is currently no easy way to check SCRAM passwords without
+		#       logging in
 		password_check_login || state="${state:-different} password"
 	elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null
 	then

From c51d68a7375915154a90c22e716e832f635ca878 Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Thu, 3 Dec 2020 18:48:04 +0100
Subject: [PATCH 193/366] [type/__postgres_conf] New type based on ALTER SYSTEM
 command

---
 .../conf/type/__postgres_conf/gencode-remote  | 92 +++++++++++++++++++
 cdist/conf/type/__postgres_conf/man.rst       | 55 +++++++++++
 .../__postgres_conf/parameter/default/state   |  1 +
 .../type/__postgres_conf/parameter/optional   |  2 +
 4 files changed, 150 insertions(+)
 create mode 100755 cdist/conf/type/__postgres_conf/gencode-remote
 create mode 100644 cdist/conf/type/__postgres_conf/man.rst
 create mode 100644 cdist/conf/type/__postgres_conf/parameter/default/state
 create mode 100644 cdist/conf/type/__postgres_conf/parameter/optional

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
new file mode 100755
index 00000000..a09e1873
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -0,0 +1,92 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+# 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$(cat "${__global}/explorer/os")
+state_should=$(cat "${__object}/parameter/state")
+
+if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ]
+then
+	echo "Invalid state '${state_should}'." \
+	     'Only "present" and "absent" are acceptable' >&2
+	exit 1
+fi
+
+if [ "${state_should}" = 'present' ] && [ ! -f "${__object}/parameter/value" ]
+then
+	echo 'Missing required parameter "value"' >&2
+	exit 1
+fi
+
+# Parameters
+conf_name="${__object_id}"
+if [ -f "${__object}/parameter/value" ]
+then
+	conf_value=$(cat "${__object}/parameter/value")
+fi
+
+if [ "${state_should}" = 'present' ]
+then
+	set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'"
+	check_command="SHOW ${conf_name}"
+else
+	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
+fi
+
+case $os
+in
+	openbsd|devuan)
+		case $os
+		in
+			openbsd)
+				postgres_user='_postgresql'
+				restart_command='/etc/rc.d/postgresql restart'
+				;;
+			devuan)
+				postgres_user='postgres'
+				restart_command='/etc/init.d/postgresql restart'
+				;;
+		esac
+		# needs two separate psql commands because ALTER SYSTEM
+		# cannot run inside a transaction block
+		cat <<-EOF
+		su - ${postgres_user} -c "psql postgres -twAc \
+			\"${set_command}\""
+		su - ${postgres_user} -c "psql postgres -twAc \
+			\"SELECT pg_reload_conf()\""
+		EOF
+		# check success (makes only sense if setting to a non-default value)
+		# and restart server if needed
+		if [ "${state_should}" = 'present' ]
+		then
+			cat <<-EOF
+			if [ \$(su - ${postgres_user} -c "psql postgres -twAc \"SHOW ${conf_name}\"") != '${conf_value}' ]
+			then
+				${restart_command}
+			fi
+			EOF
+		fi
+		;;
+	*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst
new file mode 100644
index 00000000..76016c6c
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/man.rst
@@ -0,0 +1,55 @@
+cdist-type__postgres_conf(7)
+============================
+
+Configure a PostgreSQL server.
+
+NOTE: This type might need to be run multiple times to apply all bits of the
+configuration due to ordering requirements.
+
+SSRQ <cdist--@--ssrq-sds-fds.ch>
+
+
+DESCRIPTION
+-----------
+Configure a PostgreSQL server using ALTER SYSTEM.
+
+
+REQUIRED PARAMETERS
+-------------------
+value
+    The value to setup (can be omitted when state is set to "absent").
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    "present" or "absent". Defaults to "present".
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+	# set timezone
+    __postgres_conf timezone --value Europe/Zurich
+
+    # reset maximum number of concurrent connections to default (normally 100)
+    __postgres_conf max_connections --state absent
+
+
+SEE ALSO
+--------
+- `cdist-type(7) <cdist-type.html>`_
+
+
+COPYING
+-------
+Copyright \(C) 2020 SSRQ (www.ssrq-sds-fds.ch).
+Free use of this software is granted under the terms
+of the GNU General Public License version 3 (GPLv3).
diff --git a/cdist/conf/type/__postgres_conf/parameter/default/state b/cdist/conf/type/__postgres_conf/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__postgres_conf/parameter/optional b/cdist/conf/type/__postgres_conf/parameter/optional
new file mode 100644
index 00000000..d0460d86
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/parameter/optional
@@ -0,0 +1,2 @@
+state
+value

From 534d5f6bb5dd3d8e9cb2256e75ed182d6530a892 Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Fri, 4 Dec 2020 14:31:04 +0100
Subject: [PATCH 194/366] [type/__postgres_conf] Fix errors found by ShellCheck

---
 cdist/conf/type/__postgres_conf/gencode-remote | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index a09e1873..e25514d0 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -20,8 +20,8 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$(cat "${__global}/explorer/os")
-state_should=$(cat "${__object}/parameter/state")
+os=$(cat "${__global:?}/explorer/os")
+state_should=$(cat "${__object:?}/parameter/state")
 
 if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ]
 then
@@ -37,7 +37,7 @@ then
 fi
 
 # Parameters
-conf_name="${__object_id}"
+conf_name="${__object_id:?}"
 if [ -f "${__object}/parameter/value" ]
 then
 	conf_value=$(cat "${__object}/parameter/value")
@@ -46,7 +46,6 @@ fi
 if [ "${state_should}" = 'present' ]
 then
 	set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'"
-	check_command="SHOW ${conf_name}"
 else
 	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
 fi

From 50bcd951055d7599ec340b4dea80a3a5d11dd9b0 Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Thu, 10 Dec 2020 11:45:32 +0100
Subject: [PATCH 195/366] [type/__postgres_conf] Remove faulty quotes

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index e25514d0..8ccb3b42 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -45,7 +45,7 @@ fi
 
 if [ "${state_should}" = 'present' ]
 then
-	set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'"
+	set_command="ALTER SYSTEM SET ${conf_name} = ${conf_value}"
 else
 	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
 fi

From b4060720dc3c7cc58aa992285877e4c009caa6be Mon Sep 17 00:00:00 2001
From: Beni Ruef <bernhard.ruef@ssrq-sds-fds.ch>
Date: Fri, 11 Dec 2020 11:12:16 +0100
Subject: [PATCH 196/366] [type/__postgres_conf] Fix psql options for ALTER
 command

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 8ccb3b42..7d86028e 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -67,7 +67,7 @@ in
 		# needs two separate psql commands because ALTER SYSTEM
 		# cannot run inside a transaction block
 		cat <<-EOF
-		su - ${postgres_user} -c "psql postgres -twAc \
+		su - ${postgres_user} -c "psql postgres -qwc \
 			\"${set_command}\""
 		su - ${postgres_user} -c "psql postgres -twAc \
 			\"SELECT pg_reload_conf()\""

From 1b49fec972ce5f486f2794571c8e892d1fa6cb51 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 19:15:49 +0100
Subject: [PATCH 197/366] [type/__postgres_conf] Refactor

---
 .../type/__postgres_conf/explorer/context     |  42 ++++++
 .../conf/type/__postgres_conf/explorer/state  |  60 ++++++++
 .../conf/type/__postgres_conf/gencode-remote  | 135 ++++++++++--------
 3 files changed, 180 insertions(+), 57 deletions(-)
 create mode 100644 cdist/conf/type/__postgres_conf/explorer/context
 create mode 100644 cdist/conf/type/__postgres_conf/explorer/state

diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context
new file mode 100644
index 00000000..3a2fa504
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/explorer/context
@@ -0,0 +1,42 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Returns the "context" of the configuration setting.
+# cf. also https://www.postgresql.org/docs/10/view-pg-settings.html
+
+os=$("${__explorer:?}/os")
+
+case ${os}
+in
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(devuan)
+		postgres_user='postgres'
+		;;
+	(*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
+
+conf_name=${__object_id:?}
+
+su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE name = '${conf_name}'\""
diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
new file mode 100644
index 00000000..da904b56
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -0,0 +1,60 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$("${__explorer:?}/os")
+
+case ${os}
+in
+	(openbsd)
+		postgres_user='_postgresql'
+		;;
+	(devuan)
+		postgres_user='postgres'
+		;;
+	(*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
+
+conf_name=${__object_id:?}
+
+if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
+	| cmp -s "${__object:?}/parameter/value" -
+then
+	echo present
+else
+	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT source FROM pg_settings WHERE name = '${conf_name}'\"")
+	in
+		('')
+			# invalid configuration parameter
+			# (error message was already printed by SHOW command above.
+			#  Yes, it's a hack)
+			exit 1
+			;;
+		(default)
+			echo absent
+			;;
+		(*)
+			echo different
+			;;
+	esac
+fi
diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 7d86028e..998b8582 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -1,7 +1,7 @@
 #!/bin/sh -e
 # -*- mode: sh; indent-tabs-mode: t -*-
 #
-# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+# 2019-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 # 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
@@ -21,71 +21,92 @@
 #
 
 os=$(cat "${__global:?}/explorer/os")
+state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
 
-if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ]
+conf_name=${__object_id:?}
+
+if test "${state_is}" = "${state_should}"
 then
-	echo "Invalid state '${state_should}'." \
-	     'Only "present" and "absent" are acceptable' >&2
-	exit 1
+	exit 0
 fi
 
-if [ "${state_should}" = 'present' ] && [ ! -f "${__object}/parameter/value" ]
-then
-	echo 'Missing required parameter "value"' >&2
-	exit 1
-fi
-
-# Parameters
-conf_name="${__object_id:?}"
-if [ -f "${__object}/parameter/value" ]
-then
-	conf_value=$(cat "${__object}/parameter/value")
-fi
-
-if [ "${state_should}" = 'present' ]
-then
-	set_command="ALTER SYSTEM SET ${conf_name} = ${conf_value}"
-else
-	set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT"
-fi
-
-case $os
-in
-	openbsd|devuan)
-		case $os
-		in
-			openbsd)
-				postgres_user='_postgresql'
-				restart_command='/etc/rc.d/postgresql restart'
-				;;
-			devuan)
-				postgres_user='postgres'
-				restart_command='/etc/init.d/postgresql restart'
-				;;
-		esac
-		# needs two separate psql commands because ALTER SYSTEM
-		# cannot run inside a transaction block
-		cat <<-EOF
-		su - ${postgres_user} -c "psql postgres -qwc \
-			\"${set_command}\""
-		su - ${postgres_user} -c "psql postgres -twAc \
-			\"SELECT pg_reload_conf()\""
-		EOF
-		# check success (makes only sense if setting to a non-default value)
-		# and restart server if needed
-		if [ "${state_should}" = 'present' ]
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
 		then
-			cat <<-EOF
-			if [ \$(su - ${postgres_user} -c "psql postgres -twAc \"SHOW ${conf_name}\"") != '${conf_value}' ]
-			then
-				${restart_command}
-			fi
-			EOF
+			# needs quoting
+			set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'"
+		else
+			set -- "$@" "${_arg}"
 		fi
+	done
+	unset _arg
+
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
+
+case ${os}
+in
+	(openbsd)
+		postgres_user='_postgresql'
+		restart_command='/etc/rc.d/postgresql restart'
 		;;
-	*)
+	(devuan)
+		postgres_user='postgres'
+		restart_command='/etc/init.d/postgresql restart'
+		;;
+	(*)
 		echo "Unsupported OS: ${os}" >&2
 		exit 1
 		;;
 esac
+
+
+psql_cmd() {
+	printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")"
+}
+
+case ${state_should}
+in
+	(present)
+		test -s "${__object:?}/parameter/value" || {
+			echo 'Missing required parameter --value' >&2
+			exit 1
+		}
+
+		cat <<-EOF
+		exec 3< "\${__object:?}/parameter/value"
+		$(psql_cmd postgres -tAw) <<'SQL'
+		\\set conf_value \`cat <&3\`
+		ALTER SYSTEM SET ${conf_name} = :'conf_value';
+		SELECT pg_reload_conf();
+		SQL
+		exec 3<&-
+		EOF
+		;;
+	(absent)
+		psql_cmd postgres -qwc "ALTER SYSTEM SET ${conf_name} TO DEFAULT"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		printf 'Only "present" and "absent" are acceptable.\n' >&2
+		exit 1
+		;;
+esac
+
+# check success (makes only sense if setting to a non-default value)
+# and restart server if needed
+if test "${state_should}" = 'present'
+then
+	cat <<-EOF
+
+	$(psql_cmd postgres -twAc "SHOW ${conf_name}") \\
+	| cmp -s "\${__object:?}/parameter/value" - || {
+		${restart_command}
+	}
+	EOF
+fi

From 803367b316f5c72c51153172a41b7a745f2d2d83 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 19:29:42 +0100
Subject: [PATCH 198/366] [type/__postgres_conf] Fix default detection when
 default is also set in config file

e.g. port is usually also set to the default value in postgresql.conf
---
 cdist/conf/type/__postgres_conf/explorer/state | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index da904b56..400b27e8 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -42,7 +42,7 @@ if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
 then
 	echo present
 else
-	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT source FROM pg_settings WHERE name = '${conf_name}'\"")
+	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE name = '${conf_name}'\"")
 	in
 		('')
 			# invalid configuration parameter

From 891c98567e1d94577dc4f20d3ec9f73f4927fd24 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 18 Jan 2021 19:39:56 +0100
Subject: [PATCH 199/366] [type/__postgres_conf] Compare configuration
 parameter names case insensitively

---
 cdist/conf/type/__postgres_conf/explorer/context | 3 ++-
 cdist/conf/type/__postgres_conf/explorer/state   | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context
index 3a2fa504..def5f676 100644
--- a/cdist/conf/type/__postgres_conf/explorer/context
+++ b/cdist/conf/type/__postgres_conf/explorer/context
@@ -39,4 +39,5 @@ esac
 
 conf_name=${__object_id:?}
 
-su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE name = '${conf_name}'\""
+# NOTE: SHOW/SET are case-insentitive, so this command should also be.
+su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE lower(name) = lower('${conf_name}')\""
diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 400b27e8..a4930296 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -42,7 +42,8 @@ if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
 then
 	echo present
 else
-	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE name = '${conf_name}'\"")
+	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
+	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"")
 	in
 		('')
 			# invalid configuration parameter

From 5051d4f40b7097f58431ce334f9ddf5b171dbc7f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 16:36:44 +0100
Subject: [PATCH 200/366] [type/__postgres_conf] Catch invalid values

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 998b8582..cc9b4e99 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -80,7 +80,7 @@ in
 
 		cat <<-EOF
 		exec 3< "\${__object:?}/parameter/value"
-		$(psql_cmd postgres -tAw) <<'SQL'
+		$(psql_cmd postgres -tAw -v ON_ERROR_STOP=on) <<'SQL'
 		\\set conf_value \`cat <&3\`
 		ALTER SYSTEM SET ${conf_name} = :'conf_value';
 		SELECT pg_reload_conf();

From 0f2ff477381d67de27149a92744f873322e44fd7 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 16:37:43 +0100
Subject: [PATCH 201/366] [type/__postgres_conf] Restart PostgreSQL server
 based on pending_restart column of pg_settings

---
 .../type/__postgres_conf/explorer/context     | 43 -------------------
 .../conf/type/__postgres_conf/gencode-remote  | 16 +++----
 2 files changed, 6 insertions(+), 53 deletions(-)
 delete mode 100644 cdist/conf/type/__postgres_conf/explorer/context

diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context
deleted file mode 100644
index def5f676..00000000
--- a/cdist/conf/type/__postgres_conf/explorer/context
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/sh -e
-# -*- mode: sh; indent-tabs-mode: t -*-
-#
-# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-# Returns the "context" of the configuration setting.
-# cf. also https://www.postgresql.org/docs/10/view-pg-settings.html
-
-os=$("${__explorer:?}/os")
-
-case ${os}
-in
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(devuan)
-		postgres_user='postgres'
-		;;
-	(*)
-		echo "Unsupported OS: ${os}" >&2
-		exit 1
-		;;
-esac
-
-conf_name=${__object_id:?}
-
-# NOTE: SHOW/SET are case-insentitive, so this command should also be.
-su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE lower(name) = lower('${conf_name}')\""
diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index cc9b4e99..fa930cc4 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -98,15 +98,11 @@ in
 		;;
 esac
 
-# check success (makes only sense if setting to a non-default value)
-# and restart server if needed
-if test "${state_should}" = 'present'
-then
-	cat <<-EOF
+# Restart PostgreSQL server if required to apply new configuration value
+cat <<EOF
 
-	$(psql_cmd postgres -twAc "SHOW ${conf_name}") \\
-	| cmp -s "\${__object:?}/parameter/value" - || {
-		${restart_command}
-	}
-	EOF
+if test 't' = "\$($(psql_cmd postgres -twAc "SELECT pending_restart FROM pg_settings WHERE lower(name) = lower('${conf_name}')"))"
+then
+	${restart_command}
 fi
+EOF

From 3f605c31ac7538d188a11b5d9d84cf2ed485bf23 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 17:18:27 +0100
Subject: [PATCH 202/366] [type/__postgres_conf] Add support for more init
 systems to restart service

---
 .../conf/type/__postgres_conf/gencode-remote  | 35 +++++++++++++++++--
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index fa930cc4..15d5bb3e 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -53,11 +53,9 @@ case ${os}
 in
 	(openbsd)
 		postgres_user='_postgresql'
-		restart_command='/etc/rc.d/postgresql restart'
 		;;
 	(devuan)
 		postgres_user='postgres'
-		restart_command='/etc/init.d/postgresql restart'
 		;;
 	(*)
 		echo "Unsupported OS: ${os}" >&2
@@ -103,6 +101,37 @@ cat <<EOF
 
 if test 't' = "\$($(psql_cmd postgres -twAc "SELECT pending_restart FROM pg_settings WHERE lower(name) = lower('${conf_name}')"))"
 then
-	${restart_command}
+	$(
+		init=$(cat "${__global:?}/explorer/init")
+		case ${init}
+		in
+			(systemd)
+				echo 'systemctl restart postgresql.service'
+				;;
+			(*openrc*)
+				echo 'rc-service postgresql restart'
+				;;
+			(sysvinit)
+				echo '/etc/init.d/postgresql restart'
+				;;
+			(init)
+				case $(cat "${__global:?}/explorer/kernel_name")
+				in
+					(FreeBSD)
+						echo 'service postgresql restart'
+						;;
+					(OpenBSD|NetBSD)
+						echo '/etc/rc.d/postgresql restart'
+						;;
+					(*)
+						echo "Unsupported operating system. Don't know how to restart services." >&2
+						exit 1
+				esac
+				;;
+			(*)
+				printf "Don't know how to restart services with your init (%s)\n" "${init}" >&2
+				exit 1
+		esac
+	)
 fi
 EOF

From 4967c7ebbb7e98af6aa2c7f8bc91511bcbbf9820 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 17:20:27 +0100
Subject: [PATCH 203/366] [type/__postgres_conf] Silence psql output

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 15d5bb3e..9b8227ee 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -78,7 +78,7 @@ in
 
 		cat <<-EOF
 		exec 3< "\${__object:?}/parameter/value"
-		$(psql_cmd postgres -tAw -v ON_ERROR_STOP=on) <<'SQL'
+		$(psql_cmd postgres -tAwq -o /dev/null -v ON_ERROR_STOP=on) <<'SQL'
 		\\set conf_value \`cat <&3\`
 		ALTER SYSTEM SET ${conf_name} = :'conf_value';
 		SELECT pg_reload_conf();

From f9ebb4333c85e41439635cb00a40c3e07d8a3683 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 19 Jan 2021 17:21:50 +0100
Subject: [PATCH 204/366] [type/__postgres_conf] Add NetBSD PostgreSQL UNIX
 user

---
 cdist/conf/type/__postgres_conf/gencode-remote | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 9b8227ee..8f8a2bfe 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -51,15 +51,14 @@ quote() {
 
 case ${os}
 in
+	(netbsd)
+		postgres_user='pgsql'
+		;;
 	(openbsd)
 		postgres_user='_postgresql'
 		;;
-	(devuan)
-		postgres_user='postgres'
-		;;
 	(*)
-		echo "Unsupported OS: ${os}" >&2
-		exit 1
+		postgres_user='postgres'
 		;;
 esac
 

From 6b18cace759d1345cec6c2168c3bd8f238f23bca Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 21 Jan 2021 19:29:07 +0100
Subject: [PATCH 205/366] [type/__postgres_conf] Catch connection errors early

---
 cdist/conf/type/__postgres_conf/explorer/state | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index a4930296..589925de 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -37,6 +37,11 @@ esac
 
 conf_name=${__object_id:?}
 
+su - "${postgres_user}" -c 'psql postgres -c "SELECT 1"' >/dev/null || {
+	echo 'Connection to PostgreSQL server failed' >&2
+	exit 1
+}
+
 if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
 	| cmp -s "${__object:?}/parameter/value" -
 then

From 8eccacec59b0badd7a6e2a40a27f02f0ff355ccf Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 4 Feb 2021 19:09:26 +0100
Subject: [PATCH 206/366] __package_pip: add optional dependencies

This is a poor implementation of optional dependencies for pip packages.
It ensures to install them if the package will be installed, but does
not take into account if they must be added/removed after the package is
already installed. Also, it will not be autoremoved, as all dependencies
will not be removed.
---
 cdist/conf/type/__package_pip/gencode-remote        | 13 +++++++++++++
 cdist/conf/type/__package_pip/man.rst               | 13 +++++++++++++
 .../type/__package_pip/parameter/optional_multiple  |  1 +
 3 files changed, 27 insertions(+)
 create mode 100644 cdist/conf/type/__package_pip/parameter/optional_multiple

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index a1375c2d..58c89040 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -56,6 +56,19 @@ fi
 
 case "$state_should" in
     present)
+        if [ -f "$__object/parameter/extra" ]
+        then
+            while read extra
+            do
+                if [ "$extras" ]; then
+                    extras="$extras,$extra"
+                else
+                    extras="$extra"
+                fi
+            done < "$__object/parameter/extra"
+            name="${name}[${extras}]"
+        fi
+
         if [ "$runas" ]
         then
             echo "su -c '$pip install -q $name' $runas"
diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index 234ceee2..64ad0358 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -22,6 +22,11 @@ OPTIONAL PARAMETERS
 name
     If supplied, use the name and not the object id as the package name.
 
+extra
+    Extra optional dependencies which should be installed along the selected
+    package. Can be specified multiple times. Will only be applied if the
+    package actually will be installed, but will not explicitly checked.
+
 pip
     Instead of using pip from PATH, use the specific pip path.
 
@@ -46,6 +51,14 @@ EXAMPLES
     # Use pip in a virtualenv located at /foo/shinken_virtualenv as user foo
     __package_pip pyro --state present --pip /foo/shinken_virtualenv/bin/pip --runas foo
 
+    # Install package with optional dependencies
+    __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails
+    # or do a little cheating
+    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails
+
+    # or take all extras
+    __package_pip mautrix-telegram --extra all
+
 
 SEE ALSO
 --------
diff --git a/cdist/conf/type/__package_pip/parameter/optional_multiple b/cdist/conf/type/__package_pip/parameter/optional_multiple
new file mode 100644
index 00000000..0f228715
--- /dev/null
+++ b/cdist/conf/type/__package_pip/parameter/optional_multiple
@@ -0,0 +1 @@
+extra

From 73a03d75d7d23fea292b5fd60aeb059d8c72c4ba Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 4 Feb 2021 19:18:02 +0100
Subject: [PATCH 207/366] __package_pip: fix shellcheck

---
 cdist/conf/type/__package_pip/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 58c89040..925eaf24 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -58,7 +58,7 @@ case "$state_should" in
     present)
         if [ -f "$__object/parameter/extra" ]
         then
-            while read extra
+            while read -r extra
             do
                 if [ "$extras" ]; then
                     extras="$extras,$extra"

From cda17be38a1030742ed224b2536fca517bf8c09b Mon Sep 17 00:00:00 2001
From: ssrq <ungleich@ssrq-sds-fds.ch>
Date: Mon, 8 Feb 2021 08:27:03 +0100
Subject: [PATCH 208/366] [explorer/memory] Clean up, return kiB for all
 systems, add SunOS

BSDs were MiB before.
---
 cdist/conf/explorer/memory | 85 ++++++++++++++++++++++++++++++--------
 1 file changed, 68 insertions(+), 17 deletions(-)

diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory
index 5ea15ada..63aba9c6 100755
--- a/cdist/conf/explorer/memory
+++ b/cdist/conf/explorer/memory
@@ -1,8 +1,9 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2014 Daniel Heule  (hda at sfs.biz)
 # 2014 Thomas Oettli (otho at sfs.biz)
 # Copyright 2017, Philippe Gregoire <pg@pgregoire.xyz>
+# 2020 Dennis Camera <dennis.camera at ssrq-sds-fds.ch>
 #
 # This file is part of cdist.
 #
@@ -19,24 +20,74 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
+# Returns the amount of memory physically installed in the system, or if that
+# cannot be determined the amount available to the operating system kernel,
+# in kibibytes (kiB).
 
-# FIXME: other system types (not linux ...)
+str2bytes() {
+	awk -F' ' '
+	$2 ==   "B" || !$2 { print $1 }
+	$2 ==  "kB" { print $1 * 1000 }
+	$2 ==  "MB" { print $1 * 1000 * 1000 }
+	$2 ==  "GB" { print $1 * 1000 * 1000 * 1000 }
+	$2 ==  "TB" { print $1 * 1000 * 1000 * 1000 * 1000 }
+	$2 == "kiB" { print $1 * 1024 }
+	$2 == "MiB" { print $1 * 1024 * 1024 }
+	$2 == "GiB" { print $1 * 1024 * 1024 * 1024 }
+	$2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }'
+}
 
-os=$("$__explorer/os")
-case "$os" in
-    "macosx")
-        echo "$(sysctl -n hw.memsize)/1024" | bc
-    ;;
+bytes2kib() {
+	set -- "$(cat)"
+	test "$1" -gt 0 && echo $(($1 / 1024))
+}
 
-    *"bsd")
-        PATH=$(getconf PATH)
-        echo "$(sysctl -n hw.physmem) / 1048576" | bc
-    ;;
 
-    *)
-    if [ -r /proc/meminfo ]; then
-        grep "MemTotal:" /proc/meminfo | awk '{print $2}'
-    fi
-    ;;
+case $(uname -s)
+in
+	(Darwin)
+		sysctl -n hw.memsize | bytes2kib
+		;;
+	(FreeBSD)
+		sysctl -n hw.realmem | bytes2kib
+		;;
+	(NetBSD|OpenBSD)
+		# NOTE: This reports "usable" memory, not physically installed memory.
+		command -p sysctl -n hw.physmem | bytes2kib
+		;;
+	(SunOS)
+		# Make sure that awk from xpg4 is used for the scripts to work
+		export PATH="/usr/xpg4/bin:${PATH}"
+		prtconf \
+		| awk -F ': ' '
+		  $1 == "Memory size" { sub(/Megabytes/, "MiB", $2); print $2 }
+		  /^$/ { exit }' \
+		| str2bytes \
+		| bytes2kib
+		;;
+	(Linux)
+		if test -d /sys/devices/system/memory
+		then
+			# Use memory blocks if the architecture (e.g. x86, PPC64, s390)
+			# supports them (they denote physical memory)
+			num_mem_blocks=$(cat /sys/devices/system/memory/memory[0-9]*/state | grep -cxF online)
+			mem_block_size=$(cat /sys/devices/system/memory/block_size_bytes)
+
+			echo $((num_mem_blocks * 0x$mem_block_size)) | bytes2kib && exit
+		fi
+		if test -r /proc/meminfo
+		then
+			# Fall back to meminfo file on other architectures (e.g. ARM, MIPS,
+			# PowerPC)
+			# NOTE: This is "usable" memory, not physically installed memory.
+			awk -F ': +' '$1 == "MemTotal" { sub(/B$/, "iB", $2); print $2 }' /proc/meminfo \
+			| str2bytes \
+			| bytes2kib
+		fi
+		;;
+	(*)
+		printf "Your kernel (%s) is currently not supported by the memory explorer\n" "$(uname -s)" >&2
+		printf "Please contribute an implementation for it if you can.\n" >&2
+		exit 1
+		;;
 esac

From 65a6a2ed52c56f553d65ad0aa0aaba3ee4579e3c Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 8 Feb 2021 08:28:31 +0100
Subject: [PATCH 209/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index b2b35616..52686617 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
+	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)

From bc145bbc27a45e382adb744e6774fb803f4fda0a Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 9 Feb 2021 19:58:47 +0100
Subject: [PATCH 210/366] [__letsencrypt_cert] Fix various issues with hooks.

Closes #853, see issue for full description / discussion.

Short summary:
- There was about 6.53% chances of `--renewal-hook` not being applied
- Using --automatic-renewal in one cert and not in another was an error.
- It was not possible to use different hooks for different certificates.
- FreeBSD support was utterly broken.
---
 cdist/conf/type/__letsencrypt_cert/man.rst    |  84 +++++++--
 cdist/conf/type/__letsencrypt_cert/manifest   | 171 ++++++++++++++----
 .../parameter/deprecated/automatic-renewal    |   2 +
 .../parameter/deprecated/renew-hook           |   2 +
 .../parameter/optional_multiple               |   3 +
 5 files changed, 220 insertions(+), 42 deletions(-)
 mode change 100755 => 100644 cdist/conf/type/__letsencrypt_cert/manifest
 create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal
 create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook

diff --git a/cdist/conf/type/__letsencrypt_cert/man.rst b/cdist/conf/type/__letsencrypt_cert/man.rst
index 85eb88ea..43be8424 100644
--- a/cdist/conf/type/__letsencrypt_cert/man.rst
+++ b/cdist/conf/type/__letsencrypt_cert/man.rst
@@ -1,16 +1,33 @@
 cdist-type__letsencrypt_cert(7)
 ===============================
 
+
 NAME
 ----
 
 cdist-type__letsencrypt_cert - Get an SSL certificate from Let's Encrypt
 
+
 DESCRIPTION
 -----------
 
 Automatically obtain a Let's Encrypt SSL certificate using Certbot.
 
+This type attempts to setup automatic renewals always. In many Linux
+distributions, that is the case out of the box, see:
+https://certbot.eff.org/docs/using.html#automated-renewals
+
+For Alpine Linux and Arch Linux, we setup a system-wide cronjob that
+attempts to renew certificates daily.
+
+If you are using FreeBSD, we configure periodic(8) as recommended by
+the port mantainer, so there will be a weekly attempt at renewal.
+
+If your OS is not mentioned here or on Certbot's docs as having
+support for automated renewals, please make sure you check your OS
+and possibly patch this type so the system-wide cronjob is installed.
+
+
 REQUIRED PARAMETERS
 -------------------
 
@@ -21,6 +38,7 @@ object id
 admin-email
     Where to send Let's Encrypt emails like "certificate needs renewal".
 
+
 OPTIONAL PARAMETERS
 -------------------
 
@@ -36,25 +54,68 @@ webroot
     The path to your webroot, as set up in your webserver config. If this
     parameter is not present, Certbot will be run in standalone mode.
 
+
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
 
-renew-hook
-    Renew hook command directly passed to Certbot in cron job.
-
 domain
     Domains to be included in the certificate. When specified then object id
     is not used as a domain.
 
+deploy-hook
+    Command to be executed only when the certificate associated with this
+    ``$__object_id`` is issued or renewed.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    For this command, the
+    shell variable ``$RENEWED_LINEAGE`` will point to the
+    config live subdirectory (for example,
+    ``/etc/letsencrypt/live/${__object_id}``) containing the
+    new certificates and keys; the shell variable
+    ``$RENEWED_DOMAINS`` will contain a space-delimited list
+    of renewed certificate domains (for example,
+    ``example.com www.example.com``)
+
+pre-hook
+    Command to be run in a shell before obtaining any
+    certificates.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    Note these run regardless of which certificate is attempted, you may want to
+    manage these system-wide hooks with ``__file`` in
+    ``/etc/letsencrypt/renewal-hooks/pre/``.
+
+    Intended primarily for renewal, where it
+    can be used to temporarily shut down a webserver that
+    might conflict with the standalone plugin. This will
+    only be called if a certificate is actually to be
+    obtained/renewed.
+
+post-hook
+    Command to be run in a shell after attempting to
+    obtain/renew certificates.
+    You can specify it multiple times, but any failure will prevent further
+    commands from being executed.
+
+    Note these run regardless of which certificate was attempted, you may want to
+    manage these system-wide hooks with ``__file`` in
+    ``/etc/letsencrypt/renewal-hooks/post/``.
+
+    Can be used to deploy
+    renewed certificates, or to restart any servers that
+    were stopped by --pre-hook. This is only run if an
+    attempt was made to obtain/renew a certificate.
+
+
 BOOLEAN PARAMETERS
 ------------------
 
-automatic-renewal
-    Install a cron job, which attempts to renew certificates daily.
-
 staging
     Obtain a test certificate from a staging server.
 
+
 MESSAGES
 --------
 
@@ -67,6 +128,7 @@ create
 remove
     Certificate was removed.
 
+
 EXAMPLES
 --------
 
@@ -75,8 +137,7 @@ EXAMPLES
     # use object id as domain
     __letsencrypt_cert example.com \
         --admin-email root@example.com \
-        --automatic-renewal \
-        --renew-hook "service nginx reload" \
+        --deploy-hook "service nginx reload" \
         --webroot /data/letsencrypt/root
 
 .. code-block:: sh
@@ -85,11 +146,10 @@ EXAMPLES
     # and example.com needs to be included again with domain parameter
     __letsencrypt_cert example.com \
         --admin-email root@example.com \
-        --automatic-renewal \
         --domain example.com \
         --domain foo.example.com \
         --domain bar.example.com \
-        --renew-hook "service nginx reload" \
+        --deploy-hook "service nginx reload" \
         --webroot /data/letsencrypt/root
 
 AUTHORS
@@ -99,11 +159,13 @@ AUTHORS
 | Kamila Součková <kamila--@--ksp.sk>
 | Darko Poljak <darko.poljak--@--gmail.com>
 | Ľubomír Kučera <lubomir.kucera.jr at gmail.com>
+| Evilham <contact@evilham.com>
+
 
 COPYING
 -------
 
-Copyright \(C) 2017-2018 Nico Schottelius, Kamila Součková, Darko Poljak and
+Copyright \(C) 2017-2021 Nico Schottelius, Kamila Součková, Darko Poljak and
 Ľubomír Kučera. You can redistribute it and/or modify it under the terms of
 the GNU General Public License as published by the Free Software Foundation,
 either version 3 of the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
old mode 100755
new mode 100644
index b4464366..9c8cc043
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -1,18 +1,20 @@
 #!/bin/sh
 
 certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")"
+state=$(cat "${__object}/parameter/state")
+os="$(cat "${__global:?}/explorer/os")"
 
 if [ -z "${certbot_fullpath}" ]; then
-	os="$(cat "${__global:?}/explorer/os")"
 	os_version="$(cat "${__global}/explorer/os_version")"
-
+	# Use this, very common value, as a default. It is OS-dependent
+	certbot_fullpath="/usr/bin/certbot"
 	case "$os" in
-        archlinux)
-            __package certbot
-            ;;
-        alpine)
-            __package certbot
-            ;;
+		archlinux)
+			__package certbot
+		;;
+		alpine)
+			__package certbot
+		;;
 		debian)
 			case "$os_version" in
 				8*)
@@ -48,9 +50,7 @@ if [ -z "${certbot_fullpath}" ]; then
 					exit 1
 				;;
 			esac
-
-			certbot_fullpath=/usr/bin/certbot
-			;;
+		;;
 		devuan)
 			case "$os_version" in
 				jessie)
@@ -83,17 +83,14 @@ if [ -z "${certbot_fullpath}" ]; then
 					exit 1
 				;;
 			esac
-
-			certbot_fullpath=/usr/bin/certbot
 		;;
 		freebsd)
-			__package py27-certbot
-
-			certbot_fullpath=/usr/local/bin/certbot
+			__package py37-certbot
+			certbot_fullpath="/usr/local/bin/certbot"
 		;;
 		ubuntu)
-            __package certbot
-            ;;
+			__package certbot
+		;;
 		*)
 			echo "Unsupported os: $os" >&2
 			exit 1
@@ -101,18 +98,130 @@ if [ -z "${certbot_fullpath}" ]; then
 	esac
 fi
 
-if [ -f "${__object}/parameter/automatic-renewal" ]; then
-	renew_hook_param="${__object}/parameter/renew-hook"
-	renew_hook=""
-	if [ -f "${renew_hook_param}" ]; then
-		while read -r hook; do
-			renew_hook="${renew_hook} --renew-hook \"${hook}\""
-		done < "${renew_hook_param}"
-	fi
+# Other OS-dependent values that we want to set every time
+LE_DIR="/etc/letsencrypt"
+certbot_cronjob_state="absent"
+case "$os" in
+	archlinux|alpine)
+		certbot_cronjob_state="present"
+	;;
+	freebsd)
+		LE_DIR="/usr/local/etc/letsencrypt"
+		# FreeBSD uses periodic(8) instead of crontabs for this
+		__line "periodic.conf_weekly_certbot" \
+			--file "/etc/periodic.conf" \
+			--regex "^(#[[:space:]]*)?weekly_certbot_enable=.*" \
+			--state "replace" \
+			--line 'weekly_certbot_enable="YES"'
+	;;
+	*)
+	;;
+esac
 
-	__cron letsencrypt-certbot  \
-		--user root \
-		--command "${certbot_fullpath} renew -q ${renew_hook}" \
-		--hour 0 \
-		--minute 47
+# This is only necessary in certain OS
+__cron letsencrypt-certbot \
+	--user root \
+	--command "${certbot_fullpath} renew -q" \
+	--hour 0 \
+	--minute 47 \
+	--state "${certbot_cronjob_state}"
+
+# Ensure hook directories
+HOOKS_DIR="${LE_DIR}/renewal-hooks"
+__directory "${LE_DIR}" --mode 0755
+require="__directory/${LE_DIR}" __directory "${HOOKS_DIR}" --mode 0755
+
+if [ -f "${__object}/parameter/domain" ]; then
+	domains="$(sort "${__object}/parameter/domain")"
+else
+	domains="${__object_id}"
 fi
+
+# Install hooks as needed
+for hook in deploy pre post; do
+	# Using something unique and specific to this object
+	hook_file="${HOOKS_DIR}/${hook}/${__object_id}.cdist.sh"
+	# Reasonable defaults
+	hook_source="${__object}/parameter/${hook}-hook"
+	hook_state="absent"
+	hook_contents_head="#!/bin/sh -e"
+	hook_contents_logic=""
+	hook_contents_tail=""
+
+	# Backwards compatibility
+	# Remove this when renew-hook is removed
+	# Falling back to renew-hook if deploy-hook is not passed
+	if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then
+		hook_source="${__object}/parameter/renew-hook"
+	fi
+	if [ "${state}" = "present" ] && \
+		[ -f "${hook_source}" ]; then
+		# This hook is to be installed, let's generate it with some
+		# safety boilerplate
+		# Since certbot runs all hooks for all renewal processes
+		# (at each state for deploy, pre, post), it is up to us to
+		# differentiate whether or not the hook must run
+		hook_state="present"
+		hook_contents_head="$(cat <<EOF
+#!/bin/sh -e
+#
+# Managed remotely with https://cdi.st
+#
+# Domains for which this hook is supposed to apply
+lineage="${LE_DIR}/live/${__object_id}"
+domains="\$(cat <<eof
+${domains}
+eof
+)"
+EOF
+)"
+		case "${hook}" in
+			pre|post)
+				# Certbot is kind of terrible, we have
+				# no way of knowing what domain/lineage the
+				# hook is running for
+				hook_contents_logic="$(cat <<EOF
+# pre/post-hooks apply always due to a certbot limitation
+APPLY_HOOK="YES"
+EOF
+)"
+			;;
+			deploy)
+				hook_contents_logic="$(cat <<EOF
+# certbot defines these:
+# RENEWED_DOMAINS: DOMAIN1,DOMAIN2
+# RENEWED_LINEAGE: /etc/letsencrypt/live/__object_id
+# It feels more stable to use RENEWED_LINEAGE
+if [ "\${lineage}" = "\${RENEWED_LINEAGE}" ]; then
+	APPLY_HOOK="YES"
+fi
+EOF
+)"
+			;;
+			*)
+				echo "Unknown hook '${hook}'" >> /dev/stderr
+				exit 1
+			;;
+		esac
+
+		hook_contents_tail="$(cat <<EOF
+if [ -n "\${APPLY_HOOK}" ]; then
+$(sed -e 's/^/	//' "${hook_source}")
+fi
+EOF
+)"
+	fi
+	# Ensure hook directory exists
+	require="__directory/${HOOKS_DIR}" __directory "${HOOKS_DIR}/${hook}" \
+		--mode 0755
+	require="__directory/${HOOKS_DIR}/${hook}" __file "${hook_file}" \
+		--mode 0555 \
+		--source '-' \
+		--state "${hook_state}" <<EOF
+${hook_contents_head}
+
+${hook_contents_logic}
+
+${hook_contents_tail}
+EOF
+done
diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal
new file mode 100644
index 00000000..dbfc5fa7
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal
@@ -0,0 +1,2 @@
+Deprecated in favour of consistent behaviour. It has no effect, see:
+https://code.ungleich.ch/ungleich-public/cdist/-/issues/853
diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook
new file mode 100644
index 00000000..be4a3460
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook
@@ -0,0 +1,2 @@
+This parameter has been deprecated in favour of --deploy-hook.
+See: https://code.ungleich.ch/ungleich-public/cdist/-/issues/853
diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple b/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple
index 0e866d45..bebb19bd 100644
--- a/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple
+++ b/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple
@@ -1,2 +1,5 @@
+deploy-hook
 domain
+post-hook
+pre-hook
 renew-hook

From e49da474c4e92775477c132833fb8c517db6921a Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 9 Feb 2021 20:29:17 +0100
Subject: [PATCH 211/366] [__letsencrypt_cert] Remove problematic trailing
 slash in sed.

Happy fingers are happy and like adding slashes places.
---
 cdist/conf/type/__letsencrypt_cert/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 9c8cc043..04edea1e 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -206,7 +206,7 @@ EOF
 
 		hook_contents_tail="$(cat <<EOF
 if [ -n "\${APPLY_HOOK}" ]; then
-$(sed -e 's/^/	//' "${hook_source}")
+$(sed -e 's/^/	/' "${hook_source}")
 fi
 EOF
 )"

From b832af5e3bce7d631dc1aafd6c4d9c2786391684 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Tue, 9 Feb 2021 20:53:58 +0100
Subject: [PATCH 212/366] [__letsencrypt_cert] Don't mess with user script
 indentation

This could break in odd ways if they passed sth like:
cat <<eof
bla bla
eof
---
 cdist/conf/type/__letsencrypt_cert/manifest | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 04edea1e..84221a80 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -206,7 +206,8 @@ EOF
 
 		hook_contents_tail="$(cat <<EOF
 if [ -n "\${APPLY_HOOK}" ]; then
-$(sed -e 's/^/	/' "${hook_source}")
+# Messing with indentation can eff up the users' scripts, let's not
+$(cat "${hook_source}")
 fi
 EOF
 )"

From aa80c09c803e132c3a05a642e9ada82ccb0e7dd8 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Wed, 10 Feb 2021 10:10:21 +0100
Subject: [PATCH 213/366] [__letsencrypt_cert] Move hook contents generation
 out of manifest

While there address some minor issues in the comments in the hook contents.
---
 .../type/__letsencrypt_cert/files/gen_hook.sh | 84 +++++++++++++++++++
 cdist/conf/type/__letsencrypt_cert/manifest   | 78 +----------------
 2 files changed, 88 insertions(+), 74 deletions(-)
 create mode 100644 cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh

diff --git a/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh b/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh
new file mode 100644
index 00000000..81ea4856
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh
@@ -0,0 +1,84 @@
+#!/bin/sh -e
+
+# It is expected that this defines hook_contents
+
+# Reasonable defaults
+hook_source="${__object}/parameter/${hook}-hook"
+hook_state="absent"
+hook_contents_head="#!/bin/sh -e"
+hook_contents_logic=""
+hook_contents_tail=""
+
+# Backwards compatibility
+# Remove this when renew-hook is removed
+# Falling back to renew-hook if deploy-hook is not passed
+if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then
+	hook_source="${__object}/parameter/renew-hook"
+fi
+if [ "${state}" = "present" ] && \
+	[ -f "${hook_source}" ]; then
+	# This hook is to be installed, let's generate it with some
+	# safety boilerplate
+	# Since certbot runs all hooks for all renewal processes
+	# (at each state for deploy, pre, post), it is up to us to
+	# differentiate whether or not the hook must run
+	hook_state="present"
+	hook_contents_head="$(cat <<EOF
+#!/bin/sh -e
+#
+# Managed remotely with https://cdi.st
+#
+# Domains for which this hook is supposed to apply
+lineage="${LE_DIR}/live/${__object_id}"
+domains="\$(cat <<eof
+${domains}
+eof
+)"
+EOF
+)"
+	case "${hook}" in
+		pre|post)
+			# Certbot is kind of terrible, we have
+			# no way of knowing what domain/lineage the
+			# hook is running for
+			hook_contents_logic="$(cat <<EOF
+# pre/post-hooks apply always due to a certbot limitation
+APPLY_HOOK="YES"
+EOF
+)"
+		;;
+		deploy)
+			hook_contents_logic="$(cat <<EOF
+# certbot defines these environment variables:
+# RENEWED_DOMAINS="DOMAIN1 DOMAIN2"
+# RENEWED_LINEAGE="/etc/letsencrypt/live/__object_id"
+# It feels more stable to use RENEWED_LINEAGE
+if [ "\${lineage}" = "\${RENEWED_LINEAGE}" ]; then
+	APPLY_HOOK="YES"
+fi
+EOF
+)"
+		;;
+		*)
+			echo "Unknown hook '${hook}'" >> /dev/stderr
+			exit 1
+		;;
+	esac
+
+	hook_contents_tail="$(cat <<EOF
+if [ -n "\${APPLY_HOOK}" ]; then
+	# Messing with indentation can eff up the users' scripts, let's not
+$(cat "${hook_source}")
+fi
+EOF
+)"
+fi
+
+hook_contents="$(cat <<EOF
+${hook_contents_head}
+
+${hook_contents_logic}
+
+${hook_contents_tail}
+EOF
+)"
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 84221a80..1df3574a 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -141,77 +141,11 @@ fi
 for hook in deploy pre post; do
 	# Using something unique and specific to this object
 	hook_file="${HOOKS_DIR}/${hook}/${__object_id}.cdist.sh"
-	# Reasonable defaults
-	hook_source="${__object}/parameter/${hook}-hook"
-	hook_state="absent"
-	hook_contents_head="#!/bin/sh -e"
-	hook_contents_logic=""
-	hook_contents_tail=""
 
-	# Backwards compatibility
-	# Remove this when renew-hook is removed
-	# Falling back to renew-hook if deploy-hook is not passed
-	if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then
-		hook_source="${__object}/parameter/renew-hook"
-	fi
-	if [ "${state}" = "present" ] && \
-		[ -f "${hook_source}" ]; then
-		# This hook is to be installed, let's generate it with some
-		# safety boilerplate
-		# Since certbot runs all hooks for all renewal processes
-		# (at each state for deploy, pre, post), it is up to us to
-		# differentiate whether or not the hook must run
-		hook_state="present"
-		hook_contents_head="$(cat <<EOF
-#!/bin/sh -e
-#
-# Managed remotely with https://cdi.st
-#
-# Domains for which this hook is supposed to apply
-lineage="${LE_DIR}/live/${__object_id}"
-domains="\$(cat <<eof
-${domains}
-eof
-)"
-EOF
-)"
-		case "${hook}" in
-			pre|post)
-				# Certbot is kind of terrible, we have
-				# no way of knowing what domain/lineage the
-				# hook is running for
-				hook_contents_logic="$(cat <<EOF
-# pre/post-hooks apply always due to a certbot limitation
-APPLY_HOOK="YES"
-EOF
-)"
-			;;
-			deploy)
-				hook_contents_logic="$(cat <<EOF
-# certbot defines these:
-# RENEWED_DOMAINS: DOMAIN1,DOMAIN2
-# RENEWED_LINEAGE: /etc/letsencrypt/live/__object_id
-# It feels more stable to use RENEWED_LINEAGE
-if [ "\${lineage}" = "\${RENEWED_LINEAGE}" ]; then
-	APPLY_HOOK="YES"
-fi
-EOF
-)"
-			;;
-			*)
-				echo "Unknown hook '${hook}'" >> /dev/stderr
-				exit 1
-			;;
-		esac
+	# This defines hook_contents
+	# shellcheck source=cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh
+	. "${__type}/files/gen_hook.sh"
 
-		hook_contents_tail="$(cat <<EOF
-if [ -n "\${APPLY_HOOK}" ]; then
-# Messing with indentation can eff up the users' scripts, let's not
-$(cat "${hook_source}")
-fi
-EOF
-)"
-	fi
 	# Ensure hook directory exists
 	require="__directory/${HOOKS_DIR}" __directory "${HOOKS_DIR}/${hook}" \
 		--mode 0755
@@ -219,10 +153,6 @@ EOF
 		--mode 0555 \
 		--source '-' \
 		--state "${hook_state}" <<EOF
-${hook_contents_head}
-
-${hook_contents_logic}
-
-${hook_contents_tail}
+${hook_contents}
 EOF
 done

From 4717e5ceff078111748824e4fb365df7979e5fe7 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 10:31:07 +0100
Subject: [PATCH 214/366] __package_pip: add extras explorer

The two new explorers detect all installed extras for this package.
---
 .../type/__package_pip/explorer/distinfo-dir  | 45 +++++++++++++++++++
 cdist/conf/type/__package_pip/explorer/extras | 26 +++++++++++
 cdist/conf/type/__package_pip/explorer/state  |  0
 3 files changed, 71 insertions(+)
 create mode 100755 cdist/conf/type/__package_pip/explorer/distinfo-dir
 create mode 100755 cdist/conf/type/__package_pip/explorer/extras
 mode change 100644 => 100755 cdist/conf/type/__package_pip/explorer/state

diff --git a/cdist/conf/type/__package_pip/explorer/distinfo-dir b/cdist/conf/type/__package_pip/explorer/distinfo-dir
new file mode 100755
index 00000000..18e169ae
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/distinfo-dir
@@ -0,0 +1,45 @@
+#!/bin/sh
+#
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+nameparam="$__object/parameter/name"
+if [ -f "$nameparam" ]; then
+    name=$(cat "$nameparam")
+else
+    name="$__object_id"
+fi
+
+pipparam="$__object/parameter/pip"
+if [ -f "$pipparam" ]; then
+    pip=$(cat "$pipparam")
+else
+    pip="$( "$__type_explorer/pip" )"
+fi
+
+
+if command -v "$pip" >/dev/null 2>&1; then
+    # assemble the path where pip stores all pip package info
+    "$pip" show "$name" \
+        | awk -F': ' '
+            $1 == "Name" {name=$2; gsub(/-/,"_",name); next}
+            $1 == "Version" {version=$2; next}
+            $1 == "Location" {location=$2; next}
+            END {if (version != "") printf "%s/%s-%s.dist-info", location, name, version}'
+fi
diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
new file mode 100755
index 00000000..92fc4732
--- /dev/null
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -0,0 +1,26 @@
+#!/bin/sh
+#
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+distinfo_dir="$("$__type_explorer/distinfo-dir")"
+if [ "$distinfo_dir" ]; then
+    # output all extras that are installed
+    awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA"
+fi
diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state
old mode 100644
new mode 100755

From 8dc6ab97384f2240e9218fbd4463ba12b004f44c Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 13:49:53 +0100
Subject: [PATCH 215/366] __package_pip: install not found extras

Compares the explorer against the parameters and install those extras
that are not already installed.
---
 cdist/conf/type/__package_pip/gencode-remote | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index 925eaf24..ad327e37 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -2,6 +2,7 @@
 #
 # 2012 Nico Schottelius (nico-cdist at schottelius.org)
 # 2016 Darko Poljak (darko.poljak at gmail.com)
+# 2021 Matthias Stecher (matthiasstecher at gmx.de)
 #
 # This file is part of cdist.
 #
@@ -58,14 +59,17 @@ case "$state_should" in
     present)
         if [ -f "$__object/parameter/extra" ]
         then
-            while read -r extra
-            do
-                if [ "$extras" ]; then
-                    extras="$extras,$extra"
-                else
-                    extras="$extra"
-                fi
-            done < "$__object/parameter/extra"
+            mkdir "$__object/files"
+            # sort and generalize all extras
+            sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params"
+            sort "$__object/explorer/extras" > "$__object/files/extras-explorer"
+
+            # assemble extras
+            # all extras are passed to pip in a comma-separated list
+            extras="$(comm -23 \
+                "$__object/files/extras-params" "$__object/files/extras-explorer" \
+                | sed ':a; $!N; s/\n/,/; ta'  # loops through all input lines and add commas between them
+            )"
             name="${name}[${extras}]"
         fi
 

From 2db0ef7c98226661e5e4b55c89180b31ba65b8bc Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 22:53:26 +0100
Subject: [PATCH 216/366] __package_pip: updating real detection of extras

As the previous detection took the wrong values, this explorer now
checks if packages for an extra are installed or not. If not, the extra
is not installed.

Based on the information of the explorer, it will install the package
again with the absent extras.
---
 cdist/conf/type/__package_pip/explorer/extras | 37 +++++++++++++++++--
 cdist/conf/type/__package_pip/gencode-remote  | 21 ++++-------
 2 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 92fc4732..2f5fcca6 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -17,10 +17,41 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+#
+# Checks if the given extras are really installed or not. It will be
+# done by querring all dependencies for that extra and return it as
+# "to be installed" if no dependency was found.
+#
 
 
 distinfo_dir="$("$__type_explorer/distinfo-dir")"
-if [ "$distinfo_dir" ]; then
-    # output all extras that are installed
-    awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA"
+
+# check if we have something to check
+if [ "$distinfo_dir" ] && [ -s "$__object/parameter/extra" ]
+then
+    # save cause freezing is slow
+    mkdir "$__object/files"
+    pip_freeze="$__object/files/pip-freeze.tmp"
+    pip3 freeze > "$pip_freeze"
+
+    for extra in $(cat "$__object/parameter/extra" | tr ',' '\n')
+    do
+        # create a grep BRE pattern to search all packages
+        grep_pattern="$(
+            awk -F'(: | ; )' -v check="$extra" '
+                $1 == "Requires-Dist" {
+                    split($2, r, " ");
+                    sub("extra == ", "", $3); gsub("'"'"'", "", $3);
+                    if($3 == check) print r[1]
+                }' "$distinfo_dir/METADATA" \
+                | sed ':a; $!N; s/\n/\\|/; ta'
+        )"
+
+        # echo the extra if no packages where found for it
+        # if there is no pattern, we don't need to search ;-)
+        if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze"
+        then
+            echo "$extra"
+        fi
+    done
 fi
diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote
index ad327e37..9abe28bf 100755
--- a/cdist/conf/type/__package_pip/gencode-remote
+++ b/cdist/conf/type/__package_pip/gencode-remote
@@ -26,7 +26,10 @@
 state_is=$(cat "$__object/explorer/state")
 state_should="$(cat "$__object/parameter/state")"
 
-[ "$state_is" = "$state_should" ] && exit 0
+# short circuit if state is the same and no extras to install
+[ "$state_is" = "$state_should" ] && ! [ -s "$__object/explorer/extras" ] \
+    && exit 0
+
 
 nameparam="$__object/parameter/name"
 if [ -f "$nameparam" ]; then
@@ -57,19 +60,11 @@ fi
 
 case "$state_should" in
     present)
-        if [ -f "$__object/parameter/extra" ]
+        if [ -s "$__object/explorer/extras" ]
         then
-            mkdir "$__object/files"
-            # sort and generalize all extras
-            sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params"
-            sort "$__object/explorer/extras" > "$__object/files/extras-explorer"
-
-            # assemble extras
-            # all extras are passed to pip in a comma-separated list
-            extras="$(comm -23 \
-                "$__object/files/extras-params" "$__object/files/extras-explorer" \
-                | sed ':a; $!N; s/\n/,/; ta'  # loops through all input lines and add commas between them
-            )"
+            # all extras are passed to pip in a comma-separated list in the name
+            # sed loops through all input lines and add commas between them
+            extras="$(sed ':a; $!N; s/\n/,/; ta' "$__object/explorer/extras")"
             name="${name}[${extras}]"
         fi
 

From 7398382890a4368e588e700183e6c777e90fb069 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Thu, 11 Feb 2021 23:12:10 +0100
Subject: [PATCH 217/366] __package_pip: fix shellcheck

Useless `cat $file`, use `< $file` instead.
---
 cdist/conf/type/__package_pip/explorer/extras | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 2f5fcca6..d57c69db 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -34,7 +34,7 @@ then
     pip_freeze="$__object/files/pip-freeze.tmp"
     pip3 freeze > "$pip_freeze"
 
-    for extra in $(cat "$__object/parameter/extra" | tr ',' '\n')
+    for extra in $(tr ',' '\n' < "$__object/parameter/extra")
     do
         # create a grep BRE pattern to search all packages
         grep_pattern="$(

From a9d7dfb2ed40d0a61978cc706ff469c354a6c8f5 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 12 Feb 2021 09:17:02 +0100
Subject: [PATCH 218/366] __package_pip: split extra 'all' to a list of all
 extras

This will fix if a package will be upgraded from some extras to all
extras. Previously, it will not work because some dependencies of 'all'
are already installed, so the feature 'all' is already installed.

Now, it will use a list of all extras to iterate over them separatly. This
will result it will never install all extras via `[all]`, but rather
`[foo,bar]`.
---
 cdist/conf/type/__package_pip/explorer/extras | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index d57c69db..be945e36 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -34,7 +34,14 @@ then
     pip_freeze="$__object/files/pip-freeze.tmp"
     pip3 freeze > "$pip_freeze"
 
-    for extra in $(tr ',' '\n' < "$__object/parameter/extra")
+    # If all is set, it searches all available extras to separatly check them.
+    # It would work with just 'all' (cause dependencies are given), but will
+    # not update if one extra is already present. Side effect is that it will
+    # not use [all] but instead name all extras seperatly.
+    for extra in $(if grep -qFx all "$__object/parameter/extra";
+        then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA";
+        else tr ',' '\n' < "$__object/parameter/extra";
+        fi)
     do
         # create a grep BRE pattern to search all packages
         grep_pattern="$(

From 951712740f95f8f1277f1295bcd97c0b9bafdc94 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 12 Feb 2021 13:42:51 +0100
Subject: [PATCH 219/366] __package_pip: update man.rst

Adjusted comments for `explorer/extras` and updated the man page for the
new behaviour of updating the extras.
---
 cdist/conf/type/__package_pip/explorer/extras |  7 +++---
 cdist/conf/type/__package_pip/man.rst         | 25 ++++++++++++-------
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index be945e36..8104ed47 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -35,15 +35,16 @@ then
     pip3 freeze > "$pip_freeze"
 
     # If all is set, it searches all available extras to separatly check them.
-    # It would work with just 'all' (cause dependencies are given), but will
-    # not update if one extra is already present. Side effect is that it will
-    # not use [all] but instead name all extras seperatly.
+    # It would work with just 'all' (cause dependencies are specified for
+    # 'all'), but will not update if one extra is already present. Side effect
+    # is that it will not use [all] but instead name all extras seperatly.
     for extra in $(if grep -qFx all "$__object/parameter/extra";
         then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA";
         else tr ',' '\n' < "$__object/parameter/extra";
         fi)
     do
         # create a grep BRE pattern to search all packages
+        # maybe a file full of patterns for -F could be written
         grep_pattern="$(
             awk -F'(: | ; )' -v check="$extra" '
                 $1 == "Requires-Dist" {
diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index 64ad0358..c013695c 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -24,8 +24,14 @@ name
 
 extra
     Extra optional dependencies which should be installed along the selected
-    package. Can be specified multiple times. Will only be applied if the
-    package actually will be installed, but will not explicitly checked.
+    package. Can be specified multiple times. Multiple extra optional
+    dependencies can also be specified in a comma-separated list to provide
+    a more pip-natvie style.
+
+    Extra optional dependencies will be installed even when the base package
+    is already installed. Notice that the type will not remove installed extras
+    that are not explicitly named for the type because pip does not offer a
+    management for orphaned packages and they may be used by other packages.
 
 pip
     Instead of using pip from PATH, use the specific pip path.
@@ -53,8 +59,8 @@ EXAMPLES
 
     # Install package with optional dependencies
     __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails
-    # or do a little cheating
-    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails
+    # the extras can also be specified comma-separated
+    __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails --extra postgres
 
     # or take all extras
     __package_pip mautrix-telegram --extra all
@@ -67,12 +73,13 @@ SEE ALSO
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Matthias Stecher <matthiasstecher--@--gmx.de>
 
 
 COPYING
 -------
-Copyright \(C) 2012 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2012 Nico Schottelius, 2021 Matthias Stecher. You can
+redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either
+version 3 of the License, or (at your option) any later version.

From 2ce1fce7677b64a8d17a86f29f676b1ba8475db2 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Mon, 15 Feb 2021 16:17:46 +0100
Subject: [PATCH 220/366] __package_pip: match package names case insensitive

Pip matches them insensitive, so we need to do the same to avoid
problems by saying extras are not installed but already is there in
place.
---
 cdist/conf/type/__package_pip/explorer/extras | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras
index 8104ed47..bbdc17ab 100755
--- a/cdist/conf/type/__package_pip/explorer/extras
+++ b/cdist/conf/type/__package_pip/explorer/extras
@@ -57,7 +57,8 @@ then
 
         # echo the extra if no packages where found for it
         # if there is no pattern, we don't need to search ;-)
-        if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze"
+        # pip matches packages case-insensetive, we need to do that, too
+        if [ "$grep_pattern" ] && ! grep -qi "$grep_pattern" "$pip_freeze"
         then
             echo "$extra"
         fi

From 0835f414a5b2ce41309ee9b265cb80abb1362563 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 16 Feb 2021 16:03:23 +0100
Subject: [PATCH 221/366] [type/__postgres_conf] Extract PostgreSQL service
 user detection to separate explorer

---
 .../__postgres_conf/explorer/postgres_user    | 64 +++++++++++++++++++
 .../conf/type/__postgres_conf/explorer/state  | 16 +----
 .../conf/type/__postgres_conf/gencode-remote  | 17 +----
 3 files changed, 67 insertions(+), 30 deletions(-)
 create mode 100644 cdist/conf/type/__postgres_conf/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_conf/explorer/postgres_user b/cdist/conf/type/__postgres_conf/explorer/postgres_user
new file mode 100644
index 00000000..c6582dc4
--- /dev/null
+++ b/cdist/conf/type/__postgres_conf/explorer/postgres_user
@@ -0,0 +1,64 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+os=$("${__explorer:?}/os")
+
+case ${os}
+in
+	(alpine)
+		echo 'postgres'
+		;;
+	(centos|rhel|scientific)
+		echo 'postgres'
+		;;
+	(debian|devuan|ubuntu)
+		echo 'postgres'
+		;;
+	(freebsd)
+		test -x /usr/local/etc/rc.d/postgresql || {
+			printf 'could not find postgresql rc script./n' >&2
+			exit 1
+		}
+		pg_status=$(/usr/local/etc/rc.d/postgresql onestatus) || {
+			printf 'postgresql daemon is not running.\n' >&2
+			exit 1
+		}
+		pg_pid=$(printf '%s\n' "${pg_status}" \
+			| sed -n 's/^pg_ctl:.*(PID: *\([0-9]*\))$/\1/p')
+
+		# PostgreSQL < 9.6: pgsql
+		# PostgreSQL >= 9.6: postgres
+		ps -o user -p "${pg_pid}" | sed -n '2p'
+		;;
+	(netbsd)
+		echo 'pgsql'
+		;;
+	(openbsd)
+		echo '_postgresql'
+		;;
+	(suse)
+		echo 'postgres'
+		;;
+	(*)
+		echo "Unsupported OS: ${os}" >&2
+		exit 1
+		;;
+esac
diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 589925de..de6e8aa0 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -19,21 +19,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$("${__explorer:?}/os")
-
-case ${os}
-in
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(devuan)
-		postgres_user='postgres'
-		;;
-	(*)
-		echo "Unsupported OS: ${os}" >&2
-		exit 1
-		;;
-esac
+postgres_user=$("${__type_explorer:?}/postgres_user")
 
 conf_name=${__object_id:?}
 
diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index 8f8a2bfe..d0d247b4 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -20,9 +20,9 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-os=$(cat "${__global:?}/explorer/os")
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
 conf_name=${__object_id:?}
 
@@ -49,19 +49,6 @@ quote() {
 	printf '%s' "$*"
 }
 
-case ${os}
-in
-	(netbsd)
-		postgres_user='pgsql'
-		;;
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(*)
-		postgres_user='postgres'
-		;;
-esac
-
 
 psql_cmd() {
 	printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")"
@@ -117,7 +104,7 @@ then
 				case $(cat "${__global:?}/explorer/kernel_name")
 				in
 					(FreeBSD)
-						echo 'service postgresql restart'
+						echo '/usr/local/etc/rc.d/postgresql restart'
 						;;
 					(OpenBSD|NetBSD)
 						echo '/etc/rc.d/postgresql restart'

From d1f45d3524f8a025488d0d2f020433b6efb7edb6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Fri, 19 Feb 2021 09:03:56 +0100
Subject: [PATCH 222/366] __package_pip: corrected typo in man

.. by fully replacing it with a smaller sentence.
---
 cdist/conf/type/__package_pip/man.rst | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst
index c013695c..5a2bc673 100644
--- a/cdist/conf/type/__package_pip/man.rst
+++ b/cdist/conf/type/__package_pip/man.rst
@@ -24,9 +24,8 @@ name
 
 extra
     Extra optional dependencies which should be installed along the selected
-    package. Can be specified multiple times. Multiple extra optional
-    dependencies can also be specified in a comma-separated list to provide
-    a more pip-natvie style.
+    package. Can be specified multiple times. Multiple extras can be passed
+    in one `--extra` as a comma-separated list.
 
     Extra optional dependencies will be installed even when the base package
     is already installed. Notice that the type will not remove installed extras

From 5e0572189fb930764318227fa9fd9f4966b0da99 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 22 Feb 2021 09:11:22 +0100
Subject: [PATCH 223/366] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 52686617..59c58e65 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,8 @@ next:
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
+	* Type __postgres_role: Implement modification of roles (Dennis Camera)
+	* Type __letsencrypt_cert: Fix issues with hooks (Evil Ham)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)

From 22f637c15b6616899c848e2a08c4e1c1b13f0f0b Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 23 Feb 2021 06:29:24 +0100
Subject: [PATCH 224/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 59c58e65..88dda0aa 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)
 	* Type __postgres_role: Implement modification of roles (Dennis Camera)
 	* Type __letsencrypt_cert: Fix issues with hooks (Evil Ham)
+	* Type __package_pip: Add optional extra dependencies param (Matthias Stecher)
 
 6.9.4: 2020-12-21
 	* Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera)

From 0734288483700e8e10cebd87c797b625aa83d55e Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Sun, 21 Feb 2021 19:59:57 +0000
Subject: [PATCH 225/366] First draft of __apt_pin

---
 cdist/conf/type/__apt_pin/man.rst             | 53 ++++++++++++++++
 cdist/conf/type/__apt_pin/manifest            | 63 +++++++++++++++++++
 .../type/__apt_pin/parameter/default/package  |  1 +
 .../type/__apt_pin/parameter/default/state    |  1 +
 cdist/conf/type/__apt_pin/parameter/optional  |  2 +
 cdist/conf/type/__apt_pin/parameter/required  |  2 +
 6 files changed, 122 insertions(+)
 create mode 100644 cdist/conf/type/__apt_pin/man.rst
 create mode 100755 cdist/conf/type/__apt_pin/manifest
 create mode 100644 cdist/conf/type/__apt_pin/parameter/default/package
 create mode 100644 cdist/conf/type/__apt_pin/parameter/default/state
 create mode 100644 cdist/conf/type/__apt_pin/parameter/optional
 create mode 100644 cdist/conf/type/__apt_pin/parameter/required

diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst
new file mode 100644
index 00000000..7fcae6f8
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/man.rst
@@ -0,0 +1,53 @@
+cdist-type__apt_pin(7)
+======================
+
+NAME
+----
+cdist-type__apt_pin - TODO
+
+
+DESCRIPTION
+-----------
+This space intentionally left blank.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # TODO
+    __apt_pin
+
+
+SEE ALSO
+--------
+:strong:`TODO`\ (7)
+
+
+AUTHORS
+-------
+Daniel Fancsali <fancsali@gmail.com>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Daniel Fancsali. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
new file mode 100755
index 00000000..8dd9770d
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -0,0 +1,63 @@
+#!/bin/sh -e
+#
+# 2021 Daniel Fancsali (fancsali@gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+os=$(cat "$__global/explorer/os")
+state="$(cat "$__object/parameter/state")"
+package="$(cat "$__object/parameter/package")"
+distribution="$(cat "$__object/parameter/distribution")"
+priority="$(cat "$__object/parameter/priority")"
+
+
+case "$os" in
+   debian|ubuntu|devuan)
+   ;;
+   *)
+      printf "This type is specific to Debian and it's derivatives" >&2
+      printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2
+      exit 1
+   ;;
+esac
+
+if [ "$package" = "*" ]; then
+    name="default"
+
+else
+    name="$__object_id"
+fi
+
+case $distribution in
+    stabletesting|unsatbel|experimental)
+        pin="release a=$distribution"
+        ;;
+    *)
+        pin="release n=$distribution"
+        ;;
+esac
+
+
+__file /etc/apt/preferences.d/$name \
+    --owner root --group root --mode 0644 \
+    --state "$state" \
+    --source - << EOF
+Package: $package
+Pin: $pin
+Pin-Priority: $priority
+EOF
diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package
new file mode 100644
index 00000000..72e8ffc0
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/default/package
@@ -0,0 +1 @@
+*
diff --git a/cdist/conf/type/__apt_pin/parameter/default/state b/cdist/conf/type/__apt_pin/parameter/default/state
new file mode 100644
index 00000000..e7f6134f
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/default/state
@@ -0,0 +1 @@
+present
diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional
new file mode 100644
index 00000000..52f01fd2
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/optional
@@ -0,0 +1,2 @@
+state
+package
diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required
new file mode 100644
index 00000000..4b4e9741
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/required
@@ -0,0 +1,2 @@
+distribution
+priority

From 1a74470c4d9b30e41a72fbfc084dc54ea44e643b Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Tue, 23 Feb 2021 09:37:36 +0000
Subject: [PATCH 226/366] __apt_pin: Always use $__object_id as preferences.d
 filename

---
 cdist/conf/type/__apt_pin/manifest | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index 8dd9770d..162b523f 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -36,12 +36,7 @@ case "$os" in
    ;;
 esac
 
-if [ "$package" = "*" ]; then
-    name="default"
-
-else
-    name="$__object_id"
-fi
+name="$__object_id"
 
 case $distribution in
     stabletesting|unsatbel|experimental)

From dc66efa690e15ff32d6836e23baa1cbec5eee1ed Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Tue, 23 Feb 2021 11:59:09 +0000
Subject: [PATCH 227/366] Fix shellcheck issues

---
 cdist/conf/type/__apt_pin/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index 162b523f..b1372ad0 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -48,7 +48,7 @@ case $distribution in
 esac
 
 
-__file /etc/apt/preferences.d/$name \
+__file "/etc/apt/preferences.d/$name" \
     --owner root --group root --mode 0644 \
     --state "$state" \
     --source - << EOF

From 60fd7ba1f38382761c366cbc0425ddaec62f1164 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 28 Feb 2021 13:37:23 +0100
Subject: [PATCH 228/366] Release 6.9.5

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 88dda0aa..95e36b88 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)
 	* Explorer memory: Fix result units; support Solaris (Dennis Camera)

From 8ef19d47f6b5fb6cdcba7a9a3b8890388dbd7023 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 1 Mar 2021 17:59:25 +0100
Subject: [PATCH 229/366] [type/__pyvenv] Fix example (--user -> --owner)

---
 cdist/conf/type/__pyvenv/man.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__pyvenv/man.rst b/cdist/conf/type/__pyvenv/man.rst
index 8085ff12..e2e4a1e6 100644
--- a/cdist/conf/type/__pyvenv/man.rst
+++ b/cdist/conf/type/__pyvenv/man.rst
@@ -61,7 +61,7 @@ EXAMPLES
     __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4
 
     # Create python virtualenv for user foo.
-    __pyvenv /home/foo/fooenv --group foo --user foo
+    __pyvenv /home/foo/fooenv --group foo --owner foo
 
     # Create python virtualenv with specific parameters.
     __pyvenv /home/services/djangoenv --venvparams "--copies --system-site-packages"

From e7d33891df945831d54d164eb13f2a9ae4f9dd8e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 2 Mar 2021 09:29:33 +0100
Subject: [PATCH 230/366] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 95e36b88..072fcb6b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __pyvenv: Fix user example in man page (Dennis Camera)
+
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)
 	* Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera)

From ea0126dd8117c051612e6b6abfe895f5f722c584 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Fri, 5 Mar 2021 16:11:49 +0100
Subject: [PATCH 231/366] Make local state dir available to custom remote
 scripts

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 cdist/config.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cdist/config.py b/cdist/config.py
index e84f6f84..19d5bd70 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -420,6 +420,9 @@ class Config:
                 exec_path=sys.argv[0],
                 save_output_streams=args.save_output_streams)
 
+            # Make __global state dir available to custom remote scripts.
+            os.environ['__global'] = local.base_path
+
             remote = cdist.exec.remote.Remote(
                 target_host=target_host,
                 remote_exec=remote_exec,

From ecba284fc8ce68486c5f1f87a863409fcad0f106 Mon Sep 17 00:00:00 2001
From: Steven Armstrong <steven@icarus.ethz.ch>
Date: Fri, 5 Mar 2021 16:13:02 +0100
Subject: [PATCH 232/366] changelog++

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 072fcb6b..ad93a595 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
+	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From fb19f342669d18e6decb21ee4cfb2b22e46748d9 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 9 Mar 2021 21:15:26 +0100
Subject: [PATCH 233/366] [type/__ssh_authorized_key] Only grep if file exists

---
 cdist/conf/type/__ssh_authorized_key/explorer/entry | 1 +
 cdist/conf/type/__ssh_authorized_key/gencode-remote | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__ssh_authorized_key/explorer/entry b/cdist/conf/type/__ssh_authorized_key/explorer/entry
index ccab0afc..aca0f2b9 100755
--- a/cdist/conf/type/__ssh_authorized_key/explorer/entry
+++ b/cdist/conf/type/__ssh_authorized_key/explorer/entry
@@ -25,6 +25,7 @@ type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ]
 if [ -n "${type_and_key}" ]
 then
     file="$(cat "$__object/parameter/file")"
+    test -e "$file" || exit 0
 
     # get any entries that match the type and key
 
diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote
index f37aa565..61c77fb9 100755
--- a/cdist/conf/type/__ssh_authorized_key/gencode-remote
+++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote
@@ -37,9 +37,9 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
 # preserve ownership and permissions of existing file
 if [ -f "$file" ]; then
    cp -p "$file" "\$tmpfile"
+   grep -v -F -x '$line' '$file' >\$tmpfile
 fi
-grep -v -F -x '$line' '$file' > \$tmpfile || true
-mv -f "\$tmpfile" "$file"
+cat "\$tmpfile" >"$file"
 DONE
 }
 

From 31cc592aa10902c3e83357ee4a0b9300b5e34efa Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 10 Mar 2021 19:25:04 +0100
Subject: [PATCH 234/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index ad93a595..c63b901a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
+	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From e47c4dd8a4cc441b94fe73d362d79525d01a4bb1 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 9 Mar 2021 19:59:05 +0100
Subject: [PATCH 235/366] [type/__sshd_config] Whitelist OpenBMC in manifest

---
 cdist/conf/type/__sshd_config/manifest | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest
index 566bde90..e37afebb 100755
--- a/cdist/conf/type/__sshd_config/manifest
+++ b/cdist/conf/type/__sshd_config/manifest
@@ -39,7 +39,14 @@ in
 	(freebsd|netbsd|openbsd)
 		# whitelist
 		;;
+	(openbmc-phosphor)
+		# whitelist
+		# OpenBMC can be configured with dropbear and OpenSSH.
+		# If dropbear is used, the state explorer will already fail because it
+		# cannot find the sshd binary.
+		;;
 	(*)
+		: "${__type:?}"  # make shellcheck happy
 		printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \
 			"${os}" "${__type##*/}" >&2
 		printf 'Please contribute an implementation for it if you can.\n' >&2

From 10ca1c12fd25a9f2b92cebbc5d375a1723ed5e2f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 12 Mar 2021 08:21:03 +0100
Subject: [PATCH 236/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index c63b901a..42a74d04 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
+	* Type __sshd_config: Whitelist OpenBMC (Dennis Camera)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From 7a0b697f4c394f1ae8a8941a59937007cfe474aa Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 21 Mar 2021 18:10:06 +0100
Subject: [PATCH 237/366] Implement maintaining object relationship graph

For each object maintain parent-child relationship graph, i.e. list of
parent objects ('parents' property) and list of children objects ('children'
property).

Objects without parent(s) are objects specified in init manifest.
Objects without children are object of types that do not reuse other types.
---
 cdist/core/cdist_object.py |  7 +++++++
 cdist/emulator.py          | 19 +++++++++++++++++++
 docs/src/cdist-cache.rst   | 17 +++++++++++++++++
 3 files changed, 43 insertions(+)

diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index 51d61e04..eea2c255 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -247,6 +247,13 @@ class CdistObject:
             lambda obj: os.path.join(obj.absolute_path, 'typeorder'))
     typeorder_dep = fsproperty.FileListProperty(
             lambda obj: os.path.join(obj.absolute_path, 'typeorder_dep'))
+    # objects without parents are objects specified in init manifest
+    parents = fsproperty.FileListProperty(
+            lambda obj: os.path.join(obj.absolute_path, 'parents'))
+    # objects without children are object of types that do not reuse other
+    # types
+    children = fsproperty.FileListProperty(
+            lambda obj: os.path.join(obj.absolute_path, 'children'))
 
     def cleanup(self):
         try:
diff --git a/cdist/emulator.py b/cdist/emulator.py
index a2bdc3d4..f1db862e 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -106,6 +106,7 @@ class Emulator:
             self.save_stdin()
             self.record_requirements()
             self.record_auto_requirements()
+            self.record_parent_child_relationships()
             self.log.trace("Finished %s %s" % (
                 self.cdist_object.path, self.parameters))
 
@@ -420,3 +421,21 @@ class Emulator:
                 self.log.debug("Recording autorequirement %s for %s",
                                current_object.name, parent.name)
                 parent.autorequire.append(current_object.name)
+
+    def record_parent_child_relationships(self):
+        # __object_name is the name of the object whose type manifest is
+        # currently executed
+        __object_name = self.env.get('__object_name', None)
+        if __object_name:
+            # The object whose type manifest is currently run
+            parent = self.cdist_object.object_from_name(__object_name)
+            # The object currently being defined
+            current_object = self.cdist_object
+            if current_object.name not in parent.children:
+                self.log.debug("Recording child %s for %s",
+                               current_object.name, parent.name)
+                parent.children.append(current_object.name)
+            if parent.name not in current_object.parents:
+                self.log.debug("Recording parent %s for %s",
+                               parent.name, current_object.name)
+                current_object.parents.append(parent.name)
diff --git a/docs/src/cdist-cache.rst b/docs/src/cdist-cache.rst
index d2d2d56c..d4159e77 100644
--- a/docs/src/cdist-cache.rst
+++ b/docs/src/cdist-cache.rst
@@ -61,6 +61,14 @@ Object cache overview
 ~~~~~~~~~~~~~~~~~~~~~
 Each object under :strong:`object` directory has its own structure.
 
+autorequire
+    file containing a list of object auto requirements
+
+children
+    file containing a list of object children, i.e. objects of types that this
+    type reuses (along with 'parents' it is used for maintaining parent-child
+    relationship graph)
+
 code-local
     code generated from gencode-local, present only if something is
     generated
@@ -80,6 +88,15 @@ parameter
     directory containing type parameter named files containing parameter
     values   
 
+parents
+    file containing a list of object parents, i.e. objects of types that reuse
+    this type (along with 'children' it is used for maintaining parent-child
+    relationship graph); objects without parents are objects specified in init
+    manifest
+
+require
+    file containing a list of object requirements
+
 source
     this type's source (init manifest)
 

From 167c2ad7ea2b895cdc7d89c72968d88c7d1ab34f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 30 Mar 2021 13:24:56 +0200
Subject: [PATCH 238/366] [type/__git] Fix if --owner / --group is numeric

Before, if --owner and/or --group was numeric, gencode-remote would generate
`chown` code every time.
---
 cdist/conf/type/__git/explorer/group | 20 +++++++++++++++++---
 cdist/conf/type/__git/explorer/owner | 20 +++++++++++++++++---
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group
index 3ddf9656..1365c60d 100644
--- a/cdist/conf/type/__git/explorer/group
+++ b/cdist/conf/type/__git/explorer/group
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id/.git"
+destination="/${__object_id:?}/.git"
 
-stat --print "%G" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((group_gid + 1)) -ge 0
+then
+	group_should=$(cat "${__object:?}/parameter/group")
+
+	if expr "${group_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${group_gid}"
+	else
+		printf '%s\n' "$(id -u -n "${group_gid}")"
+	fi
+fi
diff --git a/cdist/conf/type/__git/explorer/owner b/cdist/conf/type/__git/explorer/owner
index 4c3cd431..4a4d0d13 100644
--- a/cdist/conf/type/__git/explorer/owner
+++ b/cdist/conf/type/__git/explorer/owner
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id/.git"
+destination="/${__object_id:?}/.git"
 
-stat --print "%U" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((owner_uid + 1)) -ge 0
+then
+	owner_should=$(cat "${__object:?}/parameter/owner")
+
+	if expr "${owner_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${owner_uid}"
+	else
+		printf '%s\n' "$(id -u -n "${owner_uid}")"
+	fi
+fi

From 985252585ce83605787d2612fdc2a84b9788d943 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 30 Mar 2021 13:26:21 +0200
Subject: [PATCH 239/366] [type/__pyvenv] Fix if --owner / --group is numeric

Before, if --owner and/or --group was numeric, gencode-remote would generate
`chown` code every time.
---
 cdist/conf/type/__pyvenv/explorer/group | 20 +++++++++++++++++---
 cdist/conf/type/__pyvenv/explorer/owner | 20 +++++++++++++++++---
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group
index a655bda7..f31a1cb7 100755
--- a/cdist/conf/type/__pyvenv/explorer/group
+++ b/cdist/conf/type/__pyvenv/explorer/group
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id"
+destination="/${__object_id:?}"
 
-stat --print "%G" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((group_gid + 1)) -ge 0
+then
+	group_should=$(cat "${__object:?}/parameter/group")
+
+	if expr "${group_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${group_gid}"
+	else
+		printf '%s\n' "$(id -u -n "${group_gid}")"
+	fi
+fi
diff --git a/cdist/conf/type/__pyvenv/explorer/owner b/cdist/conf/type/__pyvenv/explorer/owner
index 8b3c7f8e..ebec751f 100755
--- a/cdist/conf/type/__pyvenv/explorer/owner
+++ b/cdist/conf/type/__pyvenv/explorer/owner
@@ -1,5 +1,19 @@
-#!/bin/sh
+#!/bin/sh -e
 
-destination="/$__object_id"
+destination="/${__object_id:?}"
 
-stat --print "%U" "${destination}" 2>/dev/null || exit 0
+# shellcheck disable=SC2012
+owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }')
+
+# NOTE: +1 because $((notanum)) prints 0.
+if test $((owner_uid + 1)) -ge 0
+then
+	owner_should=$(cat "${__object:?}/parameter/owner")
+
+	if expr "${owner_should}" : '[0-9]*$' >/dev/null
+	then
+		printf '%u\n' "${owner_uid}"
+	else
+		printf '%s\n' "$(id -u -n "${owner_uid}")"
+	fi
+fi

From 1e765fcab7bae02d6b380ca02e7cb04ec69ebdc5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 31 Mar 2021 07:54:00 +0200
Subject: [PATCH 240/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 42a74d04..03ae7566 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
 	* Type __sshd_config: Whitelist OpenBMC (Dennis Camera)
+	* Core: Maintain object relationship graph in cdist cache (Darko Poljak)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From f984a918b959a46e49fe8456a327eca8d44313de Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 30 Mar 2021 07:56:38 +0200
Subject: [PATCH 241/366] Fix log message string formatting

Use logging message format with args, instead of direct `%` or `str.format`.

Resolve #855.
---
 cdist/argparse.py                      |  8 +--
 cdist/config.py                        | 87 ++++++++++++--------------
 cdist/core/cdist_type.py               |  6 +-
 cdist/core/explorer.py                 | 26 ++++----
 cdist/core/manifest.py                 |  2 +-
 cdist/emulator.py                      | 37 +++++------
 cdist/exec/local.py                    | 16 +++--
 cdist/exec/remote.py                   | 14 ++---
 cdist/install.py                       |  2 +-
 cdist/inventory.py                     | 36 +++++------
 cdist/preos.py                         |  6 +-
 cdist/preos/debootstrap/debootstrap.py | 41 ++++++------
 cdist/scan/scan.py                     |  4 +-
 cdist/util/ipaddr.py                   |  6 +-
 14 files changed, 140 insertions(+), 151 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 88759d7b..cadac39a 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -533,10 +533,10 @@ def parse_and_configure(argv, singleton=True):
 
     log = logging.getLogger("cdist")
 
-    log.verbose("version %s" % cdist.VERSION)
-    log.trace('command line args: {}'.format(cfg.command_line_args))
-    log.trace('configuration: {}'.format(cfg.get_config()))
-    log.trace('configured args: {}'.format(args))
+    log.verbose("version %s", cdist.VERSION)
+    log.trace('command line args: %s', cfg.command_line_args)
+    log.trace('configuration: %s', cfg.get_config())
+    log.trace('configured args: %s', args)
 
     check_beta(vars(args))
 
diff --git a/cdist/config.py b/cdist/config.py
index 19d5bd70..d6fec55f 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -273,15 +273,15 @@ class Config:
                     host_tags = None
             host_base_path, hostdir = cls.create_host_base_dirs(
                 host, base_root_path)
-            log.debug("Base root path for target host \"{}\" is \"{}\"".format(
-                host, host_base_path))
+            log.debug("Base root path for target host \"%s\" is \"%s\"",
+                      host, host_base_path)
 
             hostcnt += 1
             if args.parallel:
                 pargs = (host, host_tags, host_base_path, hostdir, args, True,
                          configuration)
-                log.trace(("Args for multiprocessing operation "
-                           "for host {}: {}".format(host, pargs)))
+                log.trace("Args for multiprocessing operation for host %s: %s",
+                          host, pargs)
                 process_args.append(pargs)
             else:
                 try:
@@ -298,10 +298,10 @@ class Config:
             except cdist.Error:
                 failed_hosts.append(host)
         elif args.parallel:
-            log.trace("Multiprocessing start method is {}".format(
-                multiprocessing.get_start_method()))
-            log.trace(("Starting multiprocessing Pool for {} "
-                       "parallel host operation".format(args.parallel)))
+            log.trace("Multiprocessing start method is %s",
+                      multiprocessing.get_start_method())
+            log.trace("Starting multiprocessing Pool for %d parallel host"
+                      " operation", args.parallel)
 
             results = mp_pool_run(cls.onehost,
                                   process_args,
@@ -396,16 +396,13 @@ class Config:
 
             remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds(
                 args)
-            log.debug("remote_exec for host \"{}\": {}".format(
-                host, remote_exec))
-            log.debug("remote_copy for host \"{}\": {}".format(
-                host, remote_copy))
+            log.debug("remote_exec for host \"%s\": %s", host, remote_exec)
+            log.debug("remote_copy for host \"%s\": %s", host, remote_copy)
 
             family = cls._address_family(args)
-            log.debug("address family: {}".format(family))
+            log.debug("address family: %s", family)
             target_host = cls.resolve_target_addresses(host, family)
-            log.debug("target_host for host \"{}\": {}".format(
-                host, target_host))
+            log.debug("target_host for host \"%s\": %s", host, target_host)
 
             local = cdist.exec.local.Local(
                 target_host=target_host,
@@ -474,8 +471,8 @@ class Config:
         """Do what is most often done: deploy & cleanup"""
         start_time = time.time()
 
-        self.log.info("Starting {} run".format(
-            'dry' if self.dry_run else 'configuration'))
+        self.log.info("Starting %s run",
+                      'dry' if self.dry_run else 'configuration')
 
         self._init_files_dirs()
 
@@ -493,9 +490,9 @@ class Config:
         self._remove_files_dirs()
 
         self.local.save_cache(start_time)
-        self.log.info("Finished {} run in {:.2f} seconds".format(
+        self.log.info("Finished %s run in %.2f seconds",
                       'dry' if self.dry_run else 'successful',
-                      time.time() - start_time))
+                      time.time() - start_time)
 
     def cleanup(self):
         self.log.debug("Running cleanup commands")
@@ -519,8 +516,8 @@ class Config:
                 self.local.object_path, self.local.type_path,
                 self.local.object_marker_name):
             if cdist_object.cdist_type.is_install:
-                self.log.debug(("Running in config mode, ignoring install "
-                                "object: {0}").format(cdist_object))
+                self.log.debug("Running in config mode, ignoring install "
+                               "object: %s", cdist_object)
             else:
                 yield cdist_object
 
@@ -565,8 +562,7 @@ class Config:
         return objects_changed
 
     def _iterate_once_parallel(self):
-        self.log.debug("Iteration in parallel mode in {} jobs".format(
-            self.jobs))
+        self.log.debug("Iteration in parallel mode in %d jobs", self.jobs)
         objects_changed = False
 
         cargo = []
@@ -588,8 +584,8 @@ class Config:
             self.object_prepare(cargo[0])
             objects_changed = True
         elif cargo:
-            self.log.trace("Multiprocessing start method is {}".format(
-                multiprocessing.get_start_method()))
+            self.log.trace("Multiprocessing start method is %s",
+                           multiprocessing.get_start_method())
 
             self.log.trace("Multiprocessing cargo: %s", cargo)
 
@@ -603,9 +599,8 @@ class Config:
                                 "sequentially"))
                 self.explorer.transfer_type_explorers(cargo_types.pop())
             else:
-                self.log.trace(("Starting multiprocessing Pool for {} "
-                                "parallel types explorers transferring".format(
-                                    nt)))
+                self.log.trace("Starting multiprocessing Pool for %d "
+                               "parallel types explorers transferring", nt)
                 args = [
                     (ct, ) for ct in cargo_types
                 ]
@@ -614,8 +609,8 @@ class Config:
                 self.log.trace(("Multiprocessing for parallel transferring "
                                 "types' explorers finished"))
 
-            self.log.trace(("Starting multiprocessing Pool for {} parallel "
-                            "objects preparation".format(n)))
+            self.log.trace("Starting multiprocessing Pool for %d parallel "
+                           "objects preparation", n)
             args = [
                 (c, False, ) for c in cargo
             ]
@@ -667,10 +662,10 @@ class Config:
                 self.object_run(chunk[0])
                 objects_changed = True
             elif chunk:
-                self.log.trace("Multiprocessing start method is {}".format(
-                    multiprocessing.get_start_method()))
-                self.log.trace(("Starting multiprocessing Pool for {} "
-                               "parallel object run".format(n)))
+                self.log.trace("Multiprocessing start method is %s",
+                               multiprocessing.get_start_method())
+                self.log.trace("Starting multiprocessing Pool for %d "
+                               "parallel object run", n)
                 args = [
                     (c, ) for c in chunk
                 ]
@@ -794,9 +789,9 @@ class Config:
     def object_prepare(self, cdist_object, transfer_type_explorers=True):
         """Prepare object: Run type explorer + manifest"""
         self._handle_deprecation(cdist_object)
-        self.log.verbose("Preparing object {}".format(cdist_object.name))
-        self.log.verbose(
-            "Running manifest and explorers for " + cdist_object.name)
+        self.log.verbose("Preparing object %s", cdist_object.name)
+        self.log.verbose("Running manifest and explorers for %s",
+                         cdist_object.name)
         self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
         try:
             self.manifest.run_type_manifest(cdist_object)
@@ -810,13 +805,13 @@ class Config:
     def object_run(self, cdist_object):
         """Run gencode and code for an object"""
         try:
-            self.log.verbose("Running object " + cdist_object.name)
+            self.log.verbose("Running object %s", cdist_object.name)
             if cdist_object.state == core.CdistObject.STATE_DONE:
                 raise cdist.Error(("Attempting to run an already finished "
-                                   "object: %s"), cdist_object)
+                                   "object: {}").format(cdist_object))
 
             # Generate
-            self.log.debug("Generating code for %s" % (cdist_object.name))
+            self.log.debug("Generating code for %s", cdist_object.name)
             cdist_object.code_local = self.code.run_gencode_local(cdist_object)
             cdist_object.code_remote = self.code.run_gencode_remote(
                 cdist_object)
@@ -825,20 +820,20 @@ class Config:
 
             # Execute
             if cdist_object.code_local or cdist_object.code_remote:
-                self.log.info("Processing %s" % (cdist_object.name))
+                self.log.info("Processing %s", cdist_object.name)
             if not self.dry_run:
                 if cdist_object.code_local:
-                    self.log.trace("Executing local code for %s"
-                                   % (cdist_object.name))
+                    self.log.trace("Executing local code for %s",
+                                   cdist_object.name)
                     self.code.run_code_local(cdist_object)
                 if cdist_object.code_remote:
-                    self.log.trace("Executing remote code for %s"
-                                   % (cdist_object.name))
+                    self.log.trace("Executing remote code for %s",
+                                   cdist_object.name)
                     self.code.transfer_code_remote(cdist_object)
                     self.code.run_code_remote(cdist_object)
 
             # Mark this object as done
-            self.log.trace("Finishing run of " + cdist_object.name)
+            self.log.trace("Finishing run of %s", cdist_object.name)
             cdist_object.state = core.CdistObject.STATE_DONE
         except cdist.Error as e:
             raise cdist.CdistObjectError(cdist_object, e)
diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py
index c0329c8a..274de989 100644
--- a/cdist/core/cdist_type.py
+++ b/cdist/core/cdist_type.py
@@ -82,9 +82,9 @@ class CdistType:
                 yield cls(base_path, name)
             except InvalidTypeError as e:
                 # ignore invalid type, log warning and continue
-                msg = "Ignoring invalid type '%s' at '%s' defined at '%s'" % (
-                    e.type_path, e.type_absolute_path, e.source_path)
-                cls.log.warning(msg)
+                cls.log.warning("Ignoring invalid type '%s' at '%s' defined"
+                                " at '%s'", e.type_path, e.type_absolute_path,
+                                e.source_path)
                 # remove invalid from runtime conf dir
                 os.remove(e.type_absolute_path)
 
diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py
index a3baa959..caa12a7d 100644
--- a/cdist/core/explorer.py
+++ b/cdist/core/explorer.py
@@ -131,18 +131,17 @@ class Explorer:
             self._run_global_explorer(explorer, out_path)
 
     def _run_global_explorers_parallel(self, out_path):
-        self.log.debug("Running global explorers in {} parallel jobs".format(
-            self.jobs))
-        self.log.trace("Multiprocessing start method is {}".format(
-            multiprocessing.get_start_method()))
-        self.log.trace(("Starting multiprocessing Pool for global "
-                       "explorers run"))
+        self.log.debug("Running global explorers in %s parallel jobs",
+                       self.jobs)
+        self.log.trace("Multiprocessing start method is %s",
+                       multiprocessing.get_start_method())
+        self.log.trace("Starting multiprocessing Pool for global explorers"
+                       " run")
         args = [
             (e, out_path, ) for e in self.list_global_explorer_names()
         ]
         mp_pool_run(self._run_global_explorer, args, jobs=self.jobs)
-        self.log.trace(("Multiprocessing run for global explorers "
-                        "finished"))
+        self.log.trace("Multiprocessing run for global explorers finished")
 
     # logger is not pickable, so remove it when we pickle
     def __getstate__(self):
@@ -184,15 +183,14 @@ class Explorer:
         in the object.
 
         """
-        self.log.verbose("Running type explorers for {}".format(
-            cdist_object.cdist_type))
+        self.log.verbose("Running type explorers for %s",
+                         cdist_object.cdist_type)
         if transfer_type_explorers:
             self.log.trace("Transferring type explorers for type: %s",
                            cdist_object.cdist_type)
             self.transfer_type_explorers(cdist_object.cdist_type)
         else:
-            self.log.trace(("No need for transferring type explorers for "
-                            "type: %s"),
+            self.log.trace("No need for transferring type explorers for %s",
                            cdist_object.cdist_type)
         self.log.trace("Transferring object parameters for object: %s",
                        cdist_object.name)
@@ -236,8 +234,8 @@ class Explorer:
            remote side."""
         if cdist_type.explorers:
             if cdist_type.name in self._type_explorers_transferred:
-                self.log.trace(("Skipping retransfer of type explorers "
-                                "for: %s"), cdist_type)
+                self.log.trace("Skipping retransfer of type explorers for: %s",
+                               cdist_type)
             else:
                 source = os.path.join(self.local.type_path,
                                       cdist_type.explorer_path)
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 390340d4..3148d66c 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -161,7 +161,7 @@ class Manifest:
             raise NoInitialManifestError(initial_manifest, user_supplied)
 
         message_prefix = "initialmanifest"
-        self.log.verbose("Running initial manifest " + initial_manifest)
+        self.log.verbose("Running initial manifest %s", initial_manifest)
         which = "init"
         if self.local.save_output_streams:
             stderr_path = os.path.join(self.local.stderr_base_path, which)
diff --git a/cdist/emulator.py b/cdist/emulator.py
index f1db862e..f09c282d 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -107,8 +107,8 @@ class Emulator:
             self.record_requirements()
             self.record_auto_requirements()
             self.record_parent_child_relationships()
-            self.log.trace("Finished %s %s" % (
-                self.cdist_object.path, self.parameters))
+            self.log.trace("Finished %s %s", self.cdist_object.path,
+                           self.parameters)
 
     def __init_log(self):
         """Setup logging facility"""
@@ -170,7 +170,7 @@ class Emulator:
 
         # And finally parse/verify parameter
         self.args = parser.parse_args(self.argv[1:])
-        self.log.trace('Args: %s' % self.args)
+        self.log.trace('Args: %s', self.args)
 
     def init_object(self):
         # Initialize object - and ensure it is not in args
@@ -241,8 +241,8 @@ class Emulator:
                 raise cdist.Error(errmsg)
         else:
             if self.cdist_object.exists:
-                self.log.debug(('Object %s override forced with '
-                                'CDIST_OVERRIDE'), self.cdist_object.name)
+                self.log.debug('Object %s override forced with CDIST_OVERRIDE',
+                               self.cdist_object.name)
                 self.cdist_object.create(True)
             else:
                 self.cdist_object.create()
@@ -260,8 +260,8 @@ class Emulator:
             parent = self.cdist_object.object_from_name(__object_name)
             parent.typeorder.append(self.cdist_object.name)
             if self._order_dep_on():
-                self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
-                                ' for %s'), depname, parent.name)
+                self.log.trace('[ORDER_DEP] Adding %s to typeorder dep for %s',
+                               depname, parent.name)
                 parent.typeorder_dep.append(depname)
         elif self._order_dep_on():
             self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
@@ -301,16 +301,14 @@ class Emulator:
         try:
             cdist_object = self.cdist_object.object_from_name(requirement)
         except core.cdist_type.InvalidTypeError as e:
-            self.log.error(("%s requires object %s, but type %s does not"
-                            " exist. Defined at %s" % (
-                                self.cdist_object.name,
-                                requirement, e.name, self.object_source)))
+            self.log.error("%s requires object %s, but type %s does not"
+                           " exist. Defined at %s", self.cdist_object.name,
+                           requirement, e.name, self.object_source)
             raise
         except core.cdist_object.MissingObjectIdError:
-            self.log.error(("%s requires object %s without object id."
-                            " Defined at %s" % (self.cdist_object.name,
-                                                requirement,
-                                                self.object_source)))
+            self.log.error("%s requires object %s without object id."
+                           " Defined at %s", self.cdist_object.name,
+                           requirement, self.object_source)
             raise
 
         self.log.debug("Recording requirement %s for %s",
@@ -380,10 +378,9 @@ class Emulator:
                         self.env['require'] += " " + lastcreatedtype
                 else:
                     self.env['require'] = lastcreatedtype
-                self.log.debug(("Injecting require for "
-                                "CDIST_ORDER_DEPENDENCY: %s for %s"),
-                               lastcreatedtype,
-                               self.cdist_object.name)
+                self.log.debug("Injecting require for"
+                               " CDIST_ORDER_DEPENDENCY: %s for %s",
+                               lastcreatedtype, self.cdist_object.name)
             except IndexError:
                 # if no second last line, we are on the first type,
                 # so do not set a requirement
@@ -391,7 +388,7 @@ class Emulator:
 
         if "require" in self.env:
             requirements = self.env['require']
-            self.log.debug("reqs = " + requirements)
+            self.log.debug("reqs = %s", requirements)
             for requirement in self._parse_require(requirements):
                 # Ignore empty fields - probably the only field anyway
                 if len(requirement) == 0:
diff --git a/cdist/exec/local.py b/cdist/exec/local.py
index e0aab190..6713cd13 100644
--- a/cdist/exec/local.py
+++ b/cdist/exec/local.py
@@ -154,8 +154,8 @@ class Local:
         with open(self.object_marker_file, 'w') as fd:
             fd.write("%s\n" % self.object_marker_name)
 
-        self.log.trace("Object marker %s saved in %s" % (
-            self.object_marker_name, self.object_marker_file))
+        self.log.trace("Object marker %s saved in %s",
+                       self.object_marker_name, self.object_marker_file)
 
     def _init_cache_dir(self, cache_dir):
         home_dir = cdist.home_dir()
@@ -289,14 +289,12 @@ class Local:
         return cache_subpath
 
     def save_cache(self, start_time=time.time()):
-        self.log.trace("cache subpath pattern: {}".format(
-            self.cache_path_pattern))
+        self.log.trace("cache subpath pattern: %s", self.cache_path_pattern)
         cache_subpath = self._cache_subpath(start_time,
                                             self.cache_path_pattern)
-        self.log.debug("cache subpath: {}".format(cache_subpath))
+        self.log.debug("cache subpath: %s", cache_subpath)
         destination = os.path.join(self.cache_path, cache_subpath)
-        self.log.trace(("Saving cache: " + self.base_path + " to " +
-                        destination))
+        self.log.trace("Saving cache %s to %s", self.base_path, destination)
 
         if not os.path.exists(destination):
             shutil.move(self.base_path, destination)
@@ -332,7 +330,7 @@ class Local:
 
         # Iterate over all directories and link the to the output dir
         for conf_dir in self.conf_dirs:
-            self.log.debug("Checking conf_dir %s ..." % (conf_dir))
+            self.log.debug("Checking conf_dir %s ...", conf_dir)
             for sub_dir in CONF_SUBDIRS_LINKED:
                 current_dir = os.path.join(conf_dir, sub_dir)
 
@@ -350,7 +348,7 @@ class Local:
                     if os.path.exists(dst):
                         os.unlink(dst)
 
-                    self.log.trace("Linking %s to %s ..." % (src, dst))
+                    self.log.trace("Linking %s to %s ...", src, dst)
                     try:
                         os.symlink(src, dst)
                     except OSError as e:
diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py
index e5af2f34..ea85da4c 100644
--- a/cdist/exec/remote.py
+++ b/cdist/exec/remote.py
@@ -176,19 +176,19 @@ class Remote:
                 # create archive
                 tarpath, fcnt = autil.tar(source, self.archiving_mode)
                 if tarpath is None:
-                    self.log.trace(("Files count {} is lower than {} limit, "
-                                    "skipping archiving").format(
-                                        fcnt, autil.FILES_LIMIT))
+                    self.log.trace("Files count %d is lower than %d limit, "
+                                   "skipping archiving",
+                                   fcnt, autil.FILES_LIMIT)
                 else:
-                    self.log.trace(("Archiving mode, tarpath: %s, file count: "
-                                    "%s"), tarpath, fcnt)
+                    self.log.trace("Archiving mode, tarpath: %s, file count: "
+                                   "%s", tarpath, fcnt)
                     # get archive name
                     tarname = os.path.basename(tarpath)
                     self.log.trace("Archiving mode tarname: %s", tarname)
                     # archive path at the remote
                     desttarpath = os.path.join(destination, tarname)
-                    self.log.trace(
-                        "Archiving mode desttarpath: %s", desttarpath)
+                    self.log.trace("Archiving mode desttarpath: %s",
+                                   desttarpath)
                     # transfer archive to the remote side
                     self.log.trace("Archiving mode: transferring")
                     self._transfer_file(tarpath, desttarpath)
diff --git a/cdist/install.py b/cdist/install.py
index 7c894fe5..d077baef 100644
--- a/cdist/install.py
+++ b/cdist/install.py
@@ -47,4 +47,4 @@ class Install(cdist.config.Config):
                 yield cdist_object
             else:
                 self.log.debug("Running in install mode, ignoring non install"
-                               "object: {0}".format(cdist_object))
+                               "object: %s", cdist_object)
diff --git a/cdist/inventory.py b/cdist/inventory.py
index 0387f326..106052a2 100644
--- a/cdist/inventory.py
+++ b/cdist/inventory.py
@@ -92,7 +92,7 @@ class Inventory:
         self.init_db()
 
     def init_db(self):
-        self.log.trace("Init db: {}".format(self.db_basedir))
+        self.log.trace("Init db: %s", self.db_basedir)
         if not os.path.exists(self.db_basedir):
             os.makedirs(self.db_basedir, exist_ok=True)
         elif not os.path.isdir(self.db_basedir):
@@ -182,9 +182,9 @@ class Inventory:
         configuration = cfg.get_config(section='GLOBAL')
         determine_default_inventory_dir(args, configuration)
 
-        log.debug("Using inventory: {}".format(args.inventory_dir))
-        log.trace("Inventory args: {}".format(vars(args)))
-        log.trace("Inventory command: {}".format(args.subcommand))
+        log.debug("Using inventory: %s", args.inventory_dir)
+        log.trace("Inventory args: %s", vars(args))
+        log.trace("Inventory command: %s", args.subcommand)
 
         if args.subcommand == "list":
             c = InventoryList(hosts=args.host, istag=args.tag,
@@ -237,16 +237,16 @@ class InventoryList(Inventory):
     def _do_list(self, it_tags, it_hosts, check_func):
         if (it_tags is not None):
             param_tags = set(it_tags)
-            self.log.trace("param_tags: {}".format(param_tags))
+            self.log.trace("param_tags: %s", param_tags)
         else:
             param_tags = set()
         for host in it_hosts:
-            self.log.trace("host: {}".format(host))
+            self.log.trace("host: %s", host)
             tags = self._get_host_tags(host)
             if tags is None:
-                self.log.debug("Host \'{}\' not found, skipped".format(host))
+                self.log.debug("Host \'%s\' not found, skipped", host)
                 continue
-            self.log.trace("tags: {}".format(tags))
+            self.log.trace("tags: %s", tags)
             if check_func(tags, param_tags):
                 yield host, tags
 
@@ -308,11 +308,11 @@ class InventoryHost(Inventory):
 
     def _action(self, host):
         if self.action == "add":
-            self.log.debug("Adding host \'{}\'".format(host))
+            self.log.debug("Adding host \'%s\'", host)
         elif self.action == "del":
-            self.log.debug("Deleting host \'{}\'".format(host))
+            self.log.debug("Deleting host \'%s\'", host)
         hostpath = self._host_path(host)
-        self.log.trace("hostpath: {}".format(hostpath))
+        self.log.trace("hostpath: %s", hostpath)
         if self.action == "add" and not os.path.exists(hostpath):
             self._new_hostpath(hostpath)
         else:
@@ -372,23 +372,23 @@ class InventoryTag(Inventory):
             print("Host \'{}\' does not exist, skipping".format(host),
                   file=sys.stderr)
             return
-        self.log.trace("existing host_tags: {}".format(host_tags))
+        self.log.trace("existing host_tags: %s", host_tags)
         if self.action == "del" and self.all:
             host_tags = set()
         else:
             for tag in self.input_tags:
                 if self.action == "add":
-                    self.log.debug("Adding tag \'{}\' for host \'{}\'".format(
-                        tag, host))
+                    self.log.debug("Adding tag \'%s\' for host \'%s\'",
+                                   tag, host)
                     host_tags.add(tag)
                 elif self.action == "del":
-                    self.log.debug("Deleting tag \'{}\' for host "
-                                   "\'{}\'".format(tag, host))
+                    self.log.debug("Deleting tag \'%s\' for host \'%s\'",
+                                   tag, host)
                     if tag in host_tags:
                         host_tags.remove(tag)
-        self.log.trace("new host tags: {}".format(host_tags))
+        self.log.trace("new host tags: %s", host_tags)
         if not self._write_host_tags(host, host_tags):
-            self.log.trace("{} does not exist, skipped".format(host))
+            self.log.trace("%s does not exist, skipped", host)
 
     def run(self):
         if self.allhosts:
diff --git a/cdist/preos.py b/cdist/preos.py
index f8a5dd67..45711a41 100644
--- a/cdist/preos.py
+++ b/cdist/preos.py
@@ -49,7 +49,7 @@ def scan_preos_dir_plugins(dir):
                 c = cm[1]
                 yield from preos_plugin(c)
         except ImportError as e:
-            log.warning("Cannot import '{}': {}".format(module_name, e))
+            log.warning("Cannot import '%s': %s", module_name, e)
 
 
 def find_preos_plugins():
@@ -102,7 +102,7 @@ class PreOS:
         parser.add_argument('remainder_args', nargs=argparse.REMAINDER)
         args = parser.parse_args(argv[1:])
         cdist.argparse.handle_loglevel(args)
-        log.debug("preos args : {}".format(args))
+        log.debug("preos args : %s", args)
 
         conf_dirs = util.resolve_conf_dirs_from_config_and_args(args)
 
@@ -122,7 +122,7 @@ class PreOS:
                 func_args = [preos, args.remainder_args, ]
             else:
                 func_args = [args.remainder_args, ]
-            log.info("Running preos : {}".format(preos_name))
+            log.info("Running preos : %s", preos_name)
             func(*func_args)
         else:
             raise cdist.Error(
diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py
index a20cdb9c..f1f750ee 100644
--- a/cdist/preos/debootstrap/debootstrap.py
+++ b/cdist/preos/debootstrap/debootstrap.py
@@ -166,7 +166,7 @@ class Debian:
             args.pxe_boot_dir = os.path.realpath(args.pxe_boot_dir)
 
         cdist.argparse.handle_loglevel(args)
-        log.debug("preos: {}, args: {}".format(cls._preos_name, args))
+        log.debug("preos: %s, args: %s", cls._preos_name, args)
         try:
             env = vars(args)
             new_env = {}
@@ -190,27 +190,30 @@ class Debian:
             env = new_env
             env.update(os.environ)
             cls.update_env(env)
-            log.debug("preos: {} env: {}".format(cls._preos_name, env))
+            log.debug("preos: %s env: %s", cls._preos_name, env)
+
+            if log.getEffectiveLevel() <= logging.INFO:
+                info_msg = ["Running preos: {}, suite: {}, arch: {}".format(
+                    cls._preos_name, args.suite, args.arch), ]
+                if args.mirror:
+                    info_msg.append("mirror: {}".format(args.mirror))
+                if args.script:
+                    info_msg.append("script: {}".format(args.script))
+                if args.bootstrap:
+                    info_msg.append("bootstrapping")
+                if args.configure:
+                    info_msg.append("configuring")
+                if args.pxe_boot_dir:
+                    info_msg.append("creating PXE")
+                if args.drive:
+                    info_msg.append("creating bootable drive")
+                log.info(info_msg)
+
             cmd = os.path.join(cls._files_dir, "code")
-            info_msg = ["Running preos: {}, suite: {}, arch: {}".format(
-                cls._preos_name, args.suite, args.arch), ]
-            if args.mirror:
-                info_msg.append("mirror: {}".format(args.mirror))
-            if args.script:
-                info_msg.append("script: {}".format(args.script))
-            if args.bootstrap:
-                info_msg.append("bootstrapping")
-            if args.configure:
-                info_msg.append("configuring")
-            if args.pxe_boot_dir:
-                info_msg.append("creating PXE")
-            if args.drive:
-                info_msg.append("creating bootable drive")
-            log.info(info_msg)
-            log.debug("cmd={}".format(cmd))
+            log.debug("cmd=%s", cmd)
             subprocess.check_call(cmd, env=env, shell=True)
         except subprocess.CalledProcessError as e:
-            log.error("preos {} failed: {}".format(cls._preos_name, e))
+            log.error("preos %s failed: %s", cls._preos_name, e)
 
 
 class Ubuntu(Debian):
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index b1d0e9e1..9a0bcc52 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -94,7 +94,7 @@ class Trigger(object):
 
     def trigger(self, interface):
         packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
-        log.debug(f"Sending request on {interface}")
+        log.debug("Sending request on %s", interface)
         send(packet, verbose=self.verbose)
 
 
@@ -114,7 +114,7 @@ class Scanner(object):
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
             host = pkg['IPv6'].src
-            log.verbose(f"Host {host} is alive")
+            log.verbose("Host %s is alive", host)
 
             dir = os.path.join(self.outdir, host)
             fname = os.path.join(dir, "last_seen")
diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py
index 95ca74ee..d9e5f498 100644
--- a/cdist/util/ipaddr.py
+++ b/cdist/util/ipaddr.py
@@ -42,8 +42,7 @@ def resolve_target_host_name(host, family=0):
         # gethostbyaddr returns triple
         # (hostname, aliaslist, ipaddrlist)
         host_name = socket.gethostbyaddr(ip_addr)[0]
-        log.debug("derived host_name for host \"{}\": {}".format(
-            host, host_name))
+        log.debug("derived host_name for host \"%s\": %s", host, host_name)
     except (socket.gaierror, socket.herror) as e:
         # in case of error provide empty value
         host_name = ''
@@ -54,8 +53,7 @@ def resolve_target_fqdn(host):
     log = logging.getLogger(host)
     try:
         host_fqdn = socket.getfqdn(host)
-        log.debug("derived host_fqdn for host \"{}\": {}".format(
-            host, host_fqdn))
+        log.debug("derived host_fqdn for host \"%s\": %s", host, host_fqdn)
     except socket.herror as e:
         # in case of error provide empty value
         host_fqdn = ''

From 4c2d273f07e6cedb3dcb539d132cb0c3f8176d42 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 30 Mar 2021 07:56:38 +0200
Subject: [PATCH 242/366] Unify string formatting

Use one way of string formatting: replace old `%` style with new `str.format`.

Resolve #855.
---
 cdist/config.py                     |  4 ++--
 cdist/core/cdist_object.py          | 14 +++++++-------
 cdist/core/cdist_type.py            |  4 ++--
 cdist/core/code.py                  |  8 ++++----
 cdist/core/explorer.py              |  6 +++---
 cdist/core/manifest.py              | 11 +++++------
 cdist/emulator.py                   | 12 ++++++------
 cdist/exec/local.py                 | 11 ++++++-----
 cdist/exec/remote.py                | 11 +++++------
 cdist/message.py                    |  2 +-
 cdist/scan/scan.py                  |  2 +-
 cdist/shell.py                      |  2 +-
 cdist/test/cdist_object/__init__.py | 27 +++++++++++++++------------
 cdist/test/emulator/__init__.py     |  4 ++--
 cdist/test/exec/remote.py           |  8 ++++----
 cdist/test/message/__init__.py      |  2 +-
 cdist/util/fsproperty.py            |  4 ++--
 17 files changed, 67 insertions(+), 65 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index d6fec55f..adc460de 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -190,7 +190,7 @@ class Config:
                     fd.write(sys.stdin.read())
             except (IOError, OSError) as e:
                 raise cdist.Error(("Creating tempfile for stdin data "
-                                   "failed: %s" % e))
+                                   "failed: {}").format(e))
 
             args.manifest = initial_manifest_temp_path
             atexit.register(lambda: os.remove(initial_manifest_temp_path))
@@ -764,7 +764,7 @@ class Config:
 
             raise cdist.UnresolvableRequirementsError(
                     ("The requirements of the following objects could not be "
-                     "resolved:\n%s") % ("\n".join(info_string)))
+                     "resolved:\n{}").format("\n".join(info_string)))
 
     def _handle_deprecation(self, cdist_object):
         cdist_type = cdist_object.cdist_type
diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index eea2c255..ccf7392d 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -34,17 +34,17 @@ class IllegalObjectIdError(cdist.Error):
         self.message = message or 'Illegal object id'
 
     def __str__(self):
-        return '%s: %s' % (self.message, self.object_id)
+        return '{}: {}'.format(self.message, self.object_id)
 
 
 class MissingObjectIdError(cdist.Error):
     def __init__(self, type_name):
         self.type_name = type_name
-        self.message = ("Type %s requires object id (is not a "
-                        "singleton type)") % self.type_name
+        self.message = ("Type {} requires object id (is not a "
+                        "singleton type)").format(self.type_name)
 
     def __str__(self):
-        return '%s' % (self.message)
+        return '{}'.format(self.message)
 
 
 class CdistObject:
@@ -142,7 +142,7 @@ class CdistObject:
             if self.object_marker in self.object_id.split(os.sep):
                 raise IllegalObjectIdError(
                         self.object_id, ('object_id may not contain '
-                                         '\'%s\'') % self.object_marker)
+                                         '\'{}\'').format(self.object_marker))
             if '//' in self.object_id:
                 raise IllegalObjectIdError(
                         self.object_id, 'object_id may not contain //')
@@ -189,7 +189,7 @@ class CdistObject:
                               object_id=object_id)
 
     def __repr__(self):
-        return '<CdistObject %s>' % self.name
+        return '<CdistObject {}>'.format(self.name)
 
     def __eq__(self, other):
         """define equality as 'name is the same'"""
@@ -277,7 +277,7 @@ class CdistObject:
                 os.makedirs(path, exist_ok=allow_overwrite)
         except EnvironmentError as error:
             raise cdist.Error(('Error creating directories for cdist object: '
-                               '%s: %s') % (self, error))
+                               '{}: {}').format(self, error))
 
     def requirements_unfinished(self, requirements):
         """Return state whether requirements are satisfied"""
diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py
index 274de989..b68448bc 100644
--- a/cdist/core/cdist_type.py
+++ b/cdist/core/cdist_type.py
@@ -34,7 +34,7 @@ class InvalidTypeError(cdist.Error):
         self.source_path = os.path.realpath(self.type_absolute_path)
 
     def __str__(self):
-        return "Invalid type '%s' at '%s' defined at '%s'" % (
+        return "Invalid type '{}' at '{}' defined at '{}'".format(
                 self.type_path, self.type_absolute_path, self.source_path)
 
 
@@ -109,7 +109,7 @@ class CdistType:
         return cls._instances[name]
 
     def __repr__(self):
-        return '<CdistType %s>' % self.name
+        return '<CdistType {}>'.format(self.name)
 
     def __eq__(self, other):
         return isinstance(other, self.__class__) and self.name == other.name
diff --git a/cdist/core/code.py b/cdist/core/code.py
index 1e9b4f80..12888ba4 100644
--- a/cdist/core/code.py
+++ b/cdist/core/code.py
@@ -122,8 +122,8 @@ class Code:
 
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.cdist_type
-        script = os.path.join(self.local.type_path,
-                              getattr(cdist_type, 'gencode_%s_path' % which))
+        gencode_attr = getattr(cdist_type, 'gencode_{}_path'.format(which))
+        script = os.path.join(self.local.type_path, gencode_attr)
         if os.path.isfile(script):
             env = os.environ.copy()
             env.update(self.env)
@@ -167,8 +167,8 @@ class Code:
 
     def _run_code(self, cdist_object, which, env=None):
         which_exec = getattr(self, which)
-        script = os.path.join(which_exec.object_path,
-                              getattr(cdist_object, 'code_%s_path' % which))
+        code_attr = getattr(cdist_object, 'code_{}_path'.format(which))
+        script = os.path.join(which_exec.object_path, code_attr)
         if which_exec.save_output_streams:
             stderr_path = os.path.join(cdist_object.stderr_path,
                                        'code-' + which)
diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py
index caa12a7d..48414ade 100644
--- a/cdist/core/explorer.py
+++ b/cdist/core/explorer.py
@@ -160,8 +160,8 @@ class Explorer:
         self.remote.transfer(self.local.global_explorer_path,
                              self.remote.global_explorer_path,
                              self.jobs)
-        self.remote.run(["chmod", "0700",
-                         "%s/*" % (self.remote.global_explorer_path)])
+        self.remote.run(["chmod", "0700", "{}/*".format(
+            self.remote.global_explorer_path)])
 
     def run_global_explorer(self, explorer):
         """Run the given global explorer and return it's output."""
@@ -242,7 +242,7 @@ class Explorer:
                 destination = os.path.join(self.remote.type_path,
                                            cdist_type.explorer_path)
                 self.remote.transfer(source, destination)
-                self.remote.run(["chmod", "0700", "%s/*" % (destination)])
+                self.remote.run(["chmod", "0700", "{}/*".format(destination)])
                 self._type_explorers_transferred.append(cdist_type.name)
 
     def transfer_object_parameters(self, cdist_object):
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index 3148d66c..09e74dac 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -80,13 +80,12 @@ class NoInitialManifestError(cdist.Error):
 
         if user_supplied:
             if os.path.islink(manifest_path):
-                self.message = "%s: %s -> %s" % (
-                        msg_header, manifest_path,
-                        os.path.realpath(manifest_path))
+                self.message = "{}: {} -> {}".format(
+                    msg_header, manifest_path, os.path.realpath(manifest_path))
             else:
-                self.message = "%s: %s" % (msg_header, manifest_path)
+                self.message = "{}: {}".format(msg_header, manifest_path)
         else:
-            self.message = "%s" % (msg_header)
+            self.message = "{}".format(msg_header)
 
     def __str__(self):
         return repr(self.message)
@@ -107,7 +106,7 @@ class Manifest:
         self._open_logger()
 
         self.env = {
-            'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
+            'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']),
             # for use in type emulator
             '__cdist_type_base_path': self.local.type_path,
             '__global': self.local.base_path,
diff --git a/cdist/emulator.py b/cdist/emulator.py
index f09c282d..c55b47d2 100644
--- a/cdist/emulator.py
+++ b/cdist/emulator.py
@@ -36,8 +36,8 @@ from cdist.core.manifest import Manifest
 class MissingRequiredEnvironmentVariableError(cdist.Error):
     def __init__(self, name):
         self.name = name
-        self.message = ("Emulator requires the environment variable %s to be "
-                        "setup" % self.name)
+        self.message = ("Emulator requires the environment variable {} to be "
+                        "setup").format(self.name)
 
     def __str__(self):
         return self.message
@@ -231,13 +231,13 @@ class Emulator:
         if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env:
             obj_params = self._object_params_in_context()
             if obj_params != self.parameters:
-                errmsg = ("Object %s already exists with conflicting "
-                          "parameters:\n%s: %s\n%s: %s" % (
+                errmsg = ("Object {} already exists with conflicting "
+                          "parameters:\n{}: {}\n{}: {}").format(
                               self.cdist_object.name,
                               " ".join(self.cdist_object.source),
                               obj_params,
                               self.object_source,
-                              self.parameters))
+                              self.parameters)
                 raise cdist.Error(errmsg)
         else:
             if self.cdist_object.exists:
@@ -292,7 +292,7 @@ class Emulator:
                         fd.write(chunk)
                         chunk = self._read_stdin()
             except EnvironmentError as e:
-                raise cdist.Error('Failed to read from stdin: %s' % e)
+                raise cdist.Error('Failed to read from stdin: {}'.format(e))
 
     def record_requirement(self, requirement):
         """record requirement and return recorded requirement"""
diff --git a/cdist/exec/local.py b/cdist/exec/local.py
index 6713cd13..370336ad 100644
--- a/cdist/exec/local.py
+++ b/cdist/exec/local.py
@@ -152,7 +152,7 @@ class Local:
 
     def _setup_object_marker_file(self):
         with open(self.object_marker_file, 'w') as fd:
-            fd.write("%s\n" % self.object_marker_name)
+            fd.write("{}\n".format(self.object_marker_name))
 
         self.log.trace("Object marker %s saved in %s",
                        self.object_marker_name, self.object_marker_file)
@@ -184,7 +184,7 @@ class Local:
 
         """
         assert isinstance(command, (list, tuple)), (
-                "list or tuple argument expected, got: %s" % command)
+                "list or tuple argument expected, got: {}".format(command))
 
         quiet = self.quiet_mode or quiet_mode
         do_save_output = save_output and not quiet and self.save_output_streams
@@ -352,8 +352,9 @@ class Local:
                     try:
                         os.symlink(src, dst)
                     except OSError as e:
-                        raise cdist.Error("Linking %s %s to %s failed: %s" % (
-                            sub_dir, src, dst, e.__str__()))
+                        raise cdist.Error(
+                            "Linking {} {} to {} failed: {}".format(
+                                sub_dir, src, dst, e.__str__()))
 
     def _link_types_for_emulator(self):
         """Link emulator to types"""
@@ -366,5 +367,5 @@ class Local:
                 os.symlink(src, dst)
             except OSError as e:
                 raise cdist.Error(
-                        "Linking emulator from %s to %s failed: %s" % (
+                        "Linking emulator from {} to {} failed: {}".format(
                             src, dst, e.__str__()))
diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py
index ea85da4c..af9e70cb 100644
--- a/cdist/exec/remote.py
+++ b/cdist/exec/remote.py
@@ -24,12 +24,10 @@ import os
 import glob
 import subprocess
 import logging
-import multiprocessing
 
 import cdist
 import cdist.exec.util as util
 import cdist.util.ipaddr as ipaddr
-from cdist.mputil import mp_pool_run
 
 
 def _wrap_addr(addr):
@@ -262,9 +260,10 @@ class Remote:
         # remotely in e.g. csh and setting up CDIST_REMOTE_SHELL to e.g.
         # /bin/csh will execute this script in the right way.
         if env:
-            remote_env = [" export %s=%s;" % item for item in env.items()]
-            string_cmd = ("/bin/sh -c '" + " ".join(remote_env) +
-                          " ".join(command) + "'")
+            remote_env = [" export {env[0]}={env[1]};".format(env=item)
+                          for item in env.items()]
+            string_cmd = ("/bin/sh -c '{}{}'").format(" ".join(remote_env),
+                                                      " ".join(command))
             cmd.append(string_cmd)
         else:
             cmd.extend(command)
@@ -278,7 +277,7 @@ class Remote:
 
         """
         assert isinstance(command, (list, tuple)), (
-                "list or tuple argument expected, got: %s" % command)
+                "list or tuple argument expected, got: {}".format(command))
 
         close_stdout = False
         close_stderr = False
diff --git a/cdist/message.py b/cdist/message.py
index ffa8c2bb..0c4e21a6 100644
--- a/cdist/message.py
+++ b/cdist/message.py
@@ -70,7 +70,7 @@ class Message:
 
         with open(self.global_messages, 'a') as fd:
             for line in content:
-                fd.write("%s:%s" % (self.prefix, line))
+                fd.write("{}:{}".format(self.prefix, line))
 
     def merge_messages(self):
         self._merge_messages()
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 9a0bcc52..faee8a56 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -93,7 +93,7 @@ class Trigger(object):
             time.sleep(self.sleeptime)
 
     def trigger(self, interface):
-        packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest()
+        packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest()
         log.debug("Sending request on %s", interface)
         send(packet, verbose=self.verbose)
 
diff --git a/cdist/shell.py b/cdist/shell.py
index 04a68937..05803556 100644
--- a/cdist/shell.py
+++ b/cdist/shell.py
@@ -65,7 +65,7 @@ class Shell:
     def _init_environment(self):
         self.env = os.environ.copy()
         additional_env = {
-            'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
+            'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']),
             # for use in type emulator
             '__cdist_type_base_path': self.local.type_path,
             '__cdist_manifest': "cdist shell",
diff --git a/cdist/test/cdist_object/__init__.py b/cdist/test/cdist_object/__init__.py
index a9c20cd3..11619e96 100644
--- a/cdist/test/cdist_object/__init__.py
+++ b/cdist/test/cdist_object/__init__.py
@@ -86,8 +86,7 @@ class ObjectClassTestCase(test.CdistTestCase):
 
     def test_create_singleton(self):
         """Check whether creating an object without id (singleton) works"""
-        singleton = self.expected_objects[0].object_from_name(
-                "__test_singleton")
+        self.expected_objects[0].object_from_name("__test_singleton")
         # came here - everything fine
 
     def test_create_singleton_not_singleton_type(self):
@@ -126,16 +125,16 @@ class ObjectIdTestCase(test.CdistTestCase):
 
     def test_object_id_contains_object_marker(self):
         cdist_type = core.CdistType(type_base_path, '__third')
-        illegal_object_id = (
-            'object_id/may/not/contain/%s/anywhere' % OBJECT_MARKER_NAME)
+        illegal_object_id = 'object_id/may/not/contain/{}/anywhere'.format(
+            OBJECT_MARKER_NAME)
         with self.assertRaises(core.IllegalObjectIdError):
             core.CdistObject(cdist_type, self.object_base_path,
                              OBJECT_MARKER_NAME, illegal_object_id)
 
     def test_object_id_contains_object_marker_string(self):
         cdist_type = core.CdistType(type_base_path, '__third')
-        illegal_object_id = (
-            'object_id/may/contain_%s_in_filename' % OBJECT_MARKER_NAME)
+        illegal_object_id = 'object_id/may/contain_{}_in_filename'.format(
+            OBJECT_MARKER_NAME)
         core.CdistObject(cdist_type, self.object_base_path,
                          OBJECT_MARKER_NAME, illegal_object_id)
         # if we get here, the test passed
@@ -195,28 +194,32 @@ class ObjectTestCase(test.CdistTestCase):
 
     def test_path(self):
         self.assertEqual(self.cdist_object.path,
-                         "__third/moon/%s" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}".format(OBJECT_MARKER_NAME))
 
     def test_absolute_path(self):
         self.assertEqual(self.cdist_object.absolute_path,
                          os.path.join(self.object_base_path,
-                                      "__third/moon/%s" % OBJECT_MARKER_NAME))
+                                      "__third/moon/{}".format(
+                                          OBJECT_MARKER_NAME)))
 
     def test_code_local_path(self):
         self.assertEqual(self.cdist_object.code_local_path,
-                         "__third/moon/%s/code-local" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/code-local".format(
+                             OBJECT_MARKER_NAME))
 
     def test_code_remote_path(self):
         self.assertEqual(self.cdist_object.code_remote_path,
-                         "__third/moon/%s/code-remote" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/code-remote".format(
+                             OBJECT_MARKER_NAME))
 
     def test_parameter_path(self):
         self.assertEqual(self.cdist_object.parameter_path,
-                         "__third/moon/%s/parameter" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/parameter".format(
+                             OBJECT_MARKER_NAME))
 
     def test_explorer_path(self):
         self.assertEqual(self.cdist_object.explorer_path,
-                         "__third/moon/%s/explorer" % OBJECT_MARKER_NAME)
+                         "__third/moon/{}/explorer".format(OBJECT_MARKER_NAME))
 
     def test_parameters(self):
         expected_parameters = {'planet': 'Saturn', 'name': 'Prometheus'}
diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py
index befd7b57..4b2bc3ba 100644
--- a/cdist/test/emulator/__init__.py
+++ b/cdist/test/emulator/__init__.py
@@ -84,8 +84,8 @@ class EmulatorTestCase(test.CdistTestCase):
 
     def test_illegal_object_id_requirement(self):
         argv = ['__file', '/tmp/foobar']
-        self.env['require'] = (
-                "__file/bad/id/with/%s/inside") % self.local.object_marker_name
+        self.env['require'] = "__file/bad/id/with/{}/inside".format(
+            self.local.object_marker_name)
         emu = emulator.Emulator(argv, env=self.env)
         self.assertRaises(core.IllegalObjectIdError, emu.run)
 
diff --git a/cdist/test/exec/remote.py b/cdist/test/exec/remote.py
index a7fe384d..b23f8447 100644
--- a/cdist/test/exec/remote.py
+++ b/cdist/test/exec/remote.py
@@ -47,8 +47,8 @@ class RemoteTestCase(test.CdistTestCase):
             args = (self.target_host,)
         kwargs.setdefault('base_path', self.base_path)
         user = getpass.getuser()
-        kwargs.setdefault('remote_exec', 'ssh -o User=%s -q' % user)
-        kwargs.setdefault('remote_copy', 'scp -o User=%s -q' % user)
+        kwargs.setdefault('remote_exec', 'ssh -o User={} -q'.format(user))
+        kwargs.setdefault('remote_copy', 'scp -o User={} -q'.format(user))
         if 'stdout_base_path' not in kwargs:
             stdout_path = os.path.join(self.temp_dir, 'stdout')
             os.makedirs(stdout_path, exist_ok=True)
@@ -170,7 +170,7 @@ class RemoteTestCase(test.CdistTestCase):
         r = self.create_remote(remote_exec=remote_exec,
                                remote_copy=remote_copy)
         self.assertEqual(r.run('true', return_output=True),
-                         "%s\n" % self.target_host[0])
+                         "{}\n".format(self.target_host[0]))
 
     def test_run_script_target_host_in_env(self):
         handle, remote_exec_path = self.mkstemp(dir=self.temp_dir)
@@ -185,7 +185,7 @@ class RemoteTestCase(test.CdistTestCase):
         with os.fdopen(handle, "w") as fd:
             fd.writelines(["#!/bin/sh\n", "true"])
         self.assertEqual(r.run_script(script, return_output=True),
-                         "%s\n" % self.target_host[0])
+                         "{}\n".format(self.target_host[0]))
 
     def test_run_script_with_env_target_host_in_env(self):
         handle, script = self.mkstemp(dir=self.temp_dir)
diff --git a/cdist/test/message/__init__.py b/cdist/test/message/__init__.py
index 61cd5d97..55040fc9 100644
--- a/cdist/test/message/__init__.py
+++ b/cdist/test/message/__init__.py
@@ -67,7 +67,7 @@ class MessageTestCase(test.CdistTestCase):
     def test_message_merge_prefix(self):
         """Ensure messages are merged and are prefixed"""
 
-        expectedcontent = "%s:%s" % (self.prefix, self.content)
+        expectedcontent = "{}:{}".format(self.prefix, self.content)
 
         out = self.message.env['__messages_out']
 
diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py
index 1d76fd76..09e9cc19 100644
--- a/cdist/util/fsproperty.py
+++ b/cdist/util/fsproperty.py
@@ -30,7 +30,7 @@ class AbsolutePathRequiredError(cdist.Error):
         self.path = path
 
     def __str__(self):
-        return 'Absolute path required, got: %s' % self.path
+        return 'Absolute path required, got: {}'.format(self.path)
 
 
 class FileList(collections.MutableSequence):
@@ -218,7 +218,7 @@ class FileBasedProperty:
 
     def _get_attribute(self, instance, owner):
         name = self._get_property_name(owner)
-        attribute_name = '__%s' % name
+        attribute_name = '__{}'.format(name)
         if not hasattr(instance, attribute_name):
             path = self._get_path(instance)
             attribute_instance = self.attribute_class(path)

From ab811ad282882558a4639986e75c800e1b63c2e0 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 1 Apr 2021 15:37:11 +0200
Subject: [PATCH 243/366] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 03ae7566..db60c0c4 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,8 @@ next:
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)
 	* Type __sshd_config: Whitelist OpenBMC (Dennis Camera)
 	* Core: Maintain object relationship graph in cdist cache (Darko Poljak)
+	* Type __git: Fix numeric owner and group handling (Dennis Camera)
+	* Type __pyvenv: Fix numeric owner and group handling (Dennis Camera)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From 199effb7ef23980bc69f0233ff6d7ee465623f40 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 6 Apr 2021 19:35:14 +0200
Subject: [PATCH 244/366] Improve unfinished object requirements bool check

When we need only boolean value for unfinished object requirements then
we don't need to determine the whole list of unfinished objects.
---
 cdist/config.py            | 12 +++++++-----
 cdist/core/cdist_object.py | 13 ++++++++++++-
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index adc460de..f7da9b1d 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -537,7 +537,7 @@ class Config:
         objects_changed = False
 
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(
+            if cdist_object.has_requirements_unfinished(
                     cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
@@ -548,7 +548,7 @@ class Config:
                 self.object_prepare(cdist_object)
                 objects_changed = True
 
-            if cdist_object.requirements_unfinished(
+            if cdist_object.has_requirements_unfinished(
                     cdist_object.autorequire):
                 """The previous step created objects we depend on -
                     wait for them
@@ -567,7 +567,8 @@ class Config:
 
         cargo = []
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(cdist_object.requirements):
+            if cdist_object.has_requirements_unfinished(
+                    cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
 
@@ -621,12 +622,13 @@ class Config:
 
         del cargo[:]
         for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(cdist_object.requirements):
+            if cdist_object.has_requirements_unfinished(
+                    cdist_object.requirements):
                 """We cannot do anything for this poor object"""
                 continue
 
             if cdist_object.state == core.CdistObject.STATE_PREPARED:
-                if cdist_object.requirements_unfinished(
+                if cdist_object.has_requirements_unfinished(
                         cdist_object.autorequire):
                     """The previous step created objects we depend on -
                     wait for them
diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py
index ccf7392d..bb3a65bd 100644
--- a/cdist/core/cdist_object.py
+++ b/cdist/core/cdist_object.py
@@ -280,7 +280,7 @@ class CdistObject:
                                '{}: {}').format(self, error))
 
     def requirements_unfinished(self, requirements):
-        """Return state whether requirements are satisfied"""
+        """Return unsatisfied requirements"""
 
         object_list = []
 
@@ -291,3 +291,14 @@ class CdistObject:
                 object_list.append(cdist_object)
 
         return object_list
+
+    def has_requirements_unfinished(self, requirements):
+        """Return whether requirements are satisfied"""
+
+        for requirement in requirements:
+            cdist_object = self.object_from_name(requirement)
+
+            if cdist_object.state != self.STATE_DONE:
+                return True
+
+        return False

From 750c71fb5a9fcf4a30af0700b7a09e3501988a27 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Wed, 7 Apr 2021 09:07:29 +0200
Subject: [PATCH 245/366] Minor refactoring and remove code duplication

---
 cdist/config.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/cdist/config.py b/cdist/config.py
index f7da9b1d..638fdf0e 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -699,20 +699,22 @@ class Config:
             check for cycles.
         '''
         graph = {}
-        for cdist_object in self.object_list():
+
+        def _add_requirements(cdist_object, requirements):
             obj_name = cdist_object.name
             if obj_name not in graph:
                 graph[obj_name] = []
+
+            for requirement in cdist_object.requirements_unfinished(
+                    requirements):
+                graph[obj_name].append(requirement.name)
+
+        for cdist_object in self.object_list():
             if cdist_object.state == cdist_object.STATE_DONE:
                 continue
 
-            for requirement in cdist_object.requirements_unfinished(
-                    cdist_object.requirements):
-                graph[obj_name].append(requirement.name)
-
-            for requirement in cdist_object.requirements_unfinished(
-                    cdist_object.autorequire):
-                graph[obj_name].append(requirement.name)
+            _add_requirements(cdist_object, cdist_object.requirements)
+            _add_requirements(cdist_object, cdist_object.autorequire)
         return graph_check_cycle(graph)
 
     def iterate_until_finished(self):

From d2eec6066876eeb4a295b81a670c70b0ff8ad2a7 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 11 Apr 2021 23:05:48 +0300
Subject: [PATCH 246/366] [__download] make --sum optional

---
 cdist/conf/type/__download/explorer/state     |  6 ++++++
 cdist/conf/type/__download/man.rst            | 15 ++++++---------
 cdist/conf/type/__download/parameter/optional |  1 +
 cdist/conf/type/__download/parameter/required |  1 -
 4 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 00362545..68b517c5 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -8,6 +8,12 @@ then
     exit 0
 fi
 
+if [ ! -f "$__object/parameter/sum" ]
+then
+    echo 'present'
+    exit 0
+fi
+
 sum_should="$( cat "$__object/parameter/sum" )"
 
 if [ -f "$__object/parameter/cmd-sum" ]
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index 54503470..a1278cfb 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -8,9 +8,6 @@ cdist-type__download - Download a file
 
 DESCRIPTION
 -----------
-Destination (``$__object_id``) in target host must be persistent storage
-in order to calculate checksum and decide if file must be (re-)downloaded.
-
 By default type will try to use ``wget``, ``curl`` or ``fetch``.
 If download happens in target (see ``--download``) then type will
 fallback to (and install) ``wget``.
@@ -25,14 +22,14 @@ REQUIRED PARAMETERS
 url
    File's URL.
 
-sum
-   Checksum of file going to be downloaded.
-   By default output of ``cksum`` without filename is expected.
-   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
-
 
 OPTIONAL PARAMETERS
 -------------------
+sum
+   Checksum is used to decide if existing destination file must be redownloaded.
+   By default output of ``cksum`` without filename is expected.
+   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
+
 download
    If ``local`` (default), then download file to local storage and copy
    it to target host. If ``remote``, then download happens in target.
@@ -81,7 +78,7 @@ Ander Punnar <ander-at-kvlt-dot-ee>
 
 COPYING
 -------
-Copyright \(C) 2020 Ander Punnar. You can redistribute it
+Copyright \(C) 2021 Ander Punnar. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index 838e2fbf..d69e083e 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,3 +1,4 @@
+sum
 cmd-get
 cmd-sum
 download
diff --git a/cdist/conf/type/__download/parameter/required b/cdist/conf/type/__download/parameter/required
index 6ea4c38f..96cdd3b9 100644
--- a/cdist/conf/type/__download/parameter/required
+++ b/cdist/conf/type/__download/parameter/required
@@ -1,2 +1 @@
 url
-sum

From 9ec01d9f971339ee67c866c4a248047966f73ce1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 13 Apr 2021 12:22:45 +0200
Subject: [PATCH 247/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index db60c0c4..c8cb44da 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,7 @@ next:
 	* Core: Maintain object relationship graph in cdist cache (Darko Poljak)
 	* Type __git: Fix numeric owner and group handling (Dennis Camera)
 	* Type __pyvenv: Fix numeric owner and group handling (Dennis Camera)
+	* Type __download: Make sum parameter optional (Ander Punnar)
 
 6.9.5: 2021-02-28
 	* Core: preos: Fix passing cdist debug parameter (Darko Poljak)

From 92b8942a8c37bb985ef42991f3b7aaef0cd8073f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 09:00:32 +0200
Subject: [PATCH 248/366] [type/__postgres_conf] Add psql_exec function to
 state explorer

---
 cdist/conf/type/__postgres_conf/explorer/state | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index de6e8aa0..1a58751a 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -20,21 +20,24 @@
 #
 
 postgres_user=$("${__type_explorer:?}/postgres_user")
-
 conf_name=${__object_id:?}
 
-su - "${postgres_user}" -c 'psql postgres -c "SELECT 1"' >/dev/null || {
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_exec() {
+	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
+}
+
+psql_exec 'SELECT 1' >/dev/null || {
 	echo 'Connection to PostgreSQL server failed' >&2
 	exit 1
 }
 
-if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \
-	| cmp -s "${__object:?}/parameter/value" -
+if psql_exec "SHOW ${conf_name}" | cmp -s "${__object:?}/parameter/value" -
 then
 	echo present
 else
 	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
-	case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"")
+	case $(psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')")
 	in
 		('')
 			# invalid configuration parameter

From 2ccc03fef1e28c759a7918fa00ffcb45c76dc57a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 09:02:40 +0200
Subject: [PATCH 249/366] [type/__postgres_conf] Add psql_conf_cmp function to
 state explorer

---
 cdist/conf/type/__postgres_conf/explorer/state | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 1a58751a..be881be6 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -27,12 +27,16 @@ psql_exec() {
 	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
 }
 
+psql_conf_cmp() {
+	test "$(psql_exec "SHOW $1")" = "$2"
+}
+
 psql_exec 'SELECT 1' >/dev/null || {
 	echo 'Connection to PostgreSQL server failed' >&2
 	exit 1
 }
 
-if psql_exec "SHOW ${conf_name}" | cmp -s "${__object:?}/parameter/value" -
+if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
 then
 	echo present
 else

From e0416403c4862cb8bc0f41a7c6f8a33d1a8656a5 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 09:04:07 +0200
Subject: [PATCH 250/366] [type/__postgres_conf] Add psql_conf_source function
 to state explorer

---
 cdist/conf/type/__postgres_conf/explorer/state | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index be881be6..9f54724a 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -27,6 +27,10 @@ psql_exec() {
 	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
 }
 
+psql_conf_source() {
+	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
+	psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')"
+}
 psql_conf_cmp() {
 	test "$(psql_exec "SHOW $1")" = "$2"
 }
@@ -40,8 +44,7 @@ if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
 then
 	echo present
 else
-	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
-	case $(psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')")
+	case $(psql_conf_source "${conf_name}")
 	in
 		('')
 			# invalid configuration parameter

From 12c2995494ce582cb3adf1d942b36f6df035370a Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 14:46:36 +0200
Subject: [PATCH 251/366] [type/__postgres_conf] Implement complex state
 compare logic

---
 .../conf/type/__postgres_conf/explorer/state  | 166 +++++++++++++++++-
 1 file changed, 163 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index 9f54724a..e76540c5 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -22,6 +22,52 @@
 postgres_user=$("${__type_explorer:?}/postgres_user")
 conf_name=${__object_id:?}
 
+tolower() { printf '%s' "$*" | tr '[:upper:]' '[:lower:]'; }
+
+tobytes() {
+	# NOTE: This function treats everything as base 2.
+	#       It is not compatible with SI units.
+	awk 'BEGIN { FS = "\n" }
+	/TB$/ { $0 = ($0 * 1024) "GB" }
+	/GB$/ { $0 = ($0 * 1024) "MB" }
+	/MB$/ { $0 = ($0 * 1024) "kB" }
+	/kB$/ { $0 = ($0 * 1024) "B" }
+	 /B?$/ { sub(/ *B?$/, "") }
+	($0*1) == $0  # is number
+	' <<-EOF
+	$1
+	EOF
+}
+
+tomillisecs() {
+	awk 'BEGIN { FS = "\n" }
+	    /d$/ { $0 = ($0 * 24) "h" }
+	    /h$/ { $0 = ($0 * 60) "min" }
+	  /min$/ { $0 = ($0 * 60) "s" }
+	/[^m]s$/ { $0 = ($0 * 1000) "ms" }
+	   /ms$/ { $0 *= 1 }
+	($0*1) == $0  # is number
+	' <<-EOF
+	$1
+	EOF
+}
+
+tobool() {
+	# prints either 'on' or 'off'
+	case $(tolower "$1")
+	in
+		(t|true|y|yes|on|1)
+			echo 'on' ;;
+		(f|false|n|no|off|0)
+			echo 'off' ;;
+		(*)
+			printf 'Inavlid bool value: %s\n' "$2" >&2
+			return 1
+			;;
+	esac
+	return 0
+}
+
 quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
 psql_exec() {
 	su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")"
@@ -31,9 +77,123 @@ psql_conf_source() {
 	# NOTE: SHOW/SET are case-insentitive, so this command should also be.
 	psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')"
 }
-psql_conf_cmp() {
-	test "$(psql_exec "SHOW $1")" = "$2"
-}
+psql_conf_cmp() (
+	IFS='|' read -r lower_name vartype setting unit <<-EOF
+	$(psql_exec "SELECT lower(name), vartype, setting, unit FROM pg_settings WHERE lower(name) = lower('$1')")
+	EOF
+
+	should_value=$2
+	is_value=${setting}
+
+	# The following case contains special cases for special settings.
+	case ${lower_name}
+	in
+		(archive_command)
+			if test "${setting}" = '(disabled)'
+			then
+				# DAFUQ PostgreSQL?!
+				# PostgreSQL returns (disabled) if the feature is inactive.
+				# We cannot compare the values unless it is enabled, first.
+				return 0
+			fi
+			;;
+		(archive_mode|backslash_quote|constraint_exclusion|force_parallel_mode|huge_pages|synchronous_commit)
+			# Although only 'on', 'off' are documented, PostgreSQL accepts all
+			# the "likely" variants of "on" and "off".
+			case $(tolower "${should_value}")
+			in
+				(on|off|true|false|yes|no|1|0)
+					should_value=$(tobool "${should_value}")
+					;;
+			esac
+			;;
+	esac
+
+	case ${vartype}
+	in
+		(bool)
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported for vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			should_value=$(tobool "${should_value}")
+
+			test "${is_value}" = "${should_value}"
+			;;
+		(enum)
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported with vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			# NOTE: All enums that are currently defined are lower case, but
+			#       PostgreSQL also accepts upper case spelling.
+			should_value=$(tolower "$2")
+
+			test "${is_value}" = "${should_value}"
+			;;
+		(integer)
+			# split multiples from unit, first (e.g. 8kB -> 8, kB)
+			case ${unit}
+			in
+				([0-9]*)
+					multiple=${unit%%[!0-9]*}
+					unit=${unit##*[0-9 ]}
+					;;
+				(*) multiple=1 ;;
+			esac
+
+			is_value=$((setting * multiple))${unit}
+
+			if expr "${should_value}" : '-\{0,1\}[0-9]*$' >/dev/null
+			then
+				# default unit
+				should_value=$((should_value * multiple))${unit}
+			fi
+
+			# then, do conversion
+			# NOTE: these conversions work for integers only!
+			case ${unit}
+			in
+				(B|[kMGT]B)
+					# bytes
+					is_bytes=$(tobytes "${is_value}")
+					should_bytes=$(tobytes "${should_value}")
+
+					test $((is_bytes)) -eq $((should_bytes))
+					;;
+				(ms|s|min|h|d)
+					# seconds
+					is_ms=$(tomillisecs "${is_value}")
+					should_ms=$(tomillisecs "${should_value}")
+
+					test $((is_ms)) -eq $((should_ms))
+					;;
+				('')
+					# no unit
+					is_int=${is_value}
+					should_int=${should_value}
+
+					test $((is_int)) -eq $((should_int))
+					;;
+			esac
+			;;
+		(real|string)
+			# NOTE: reals could possibly have units, but currently there none.
+
+			test -z "${unit}" || {
+				# please fix the explorer if this error occurs.
+				printf 'units are not supported with vartype: %s\n' "${vartype}" >&2
+				exit 1
+			}
+
+			test "${is_value}" = "${should_value}"
+			;;
+	esac
+)
 
 psql_exec 'SELECT 1' >/dev/null || {
 	echo 'Connection to PostgreSQL server failed' >&2

From bef1433ba3adb022f8b50eb9df0fbbbf90757d76 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 14:46:50 +0200
Subject: [PATCH 252/366] [type/__postgres_conf] Accept empty values

---
 cdist/conf/type/__postgres_conf/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote
index d0d247b4..27651600 100755
--- a/cdist/conf/type/__postgres_conf/gencode-remote
+++ b/cdist/conf/type/__postgres_conf/gencode-remote
@@ -57,7 +57,7 @@ psql_cmd() {
 case ${state_should}
 in
 	(present)
-		test -s "${__object:?}/parameter/value" || {
+		test -n "${__object:?}/parameter/value" || {
 			echo 'Missing required parameter --value' >&2
 			exit 1
 		}

From 686e4f0f2d498ee39a69e43f7535171f16f5fcbc Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 15:01:38 +0200
Subject: [PATCH 253/366] [type/__postgres_conf] Reverse state logic (decide
 based on source first)

---
 .../conf/type/__postgres_conf/explorer/state  | 39 ++++++++++---------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state
index e76540c5..4b7b0a43 100644
--- a/cdist/conf/type/__postgres_conf/explorer/state
+++ b/cdist/conf/type/__postgres_conf/explorer/state
@@ -200,23 +200,24 @@ psql_exec 'SELECT 1' >/dev/null || {
 	exit 1
 }
 
-if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
-then
-	echo present
-else
-	case $(psql_conf_source "${conf_name}")
-	in
-		('')
-			# invalid configuration parameter
-			# (error message was already printed by SHOW command above.
-			#  Yes, it's a hack)
-			exit 1
-			;;
-		(default)
-			echo absent
-			;;
-		(*)
+case $(psql_conf_source "${conf_name}")
+in
+	('')
+		printf 'Invalid configuration parameter: %s\n' "${conf_name}" >&2
+		exit 1
+		;;
+	(default)
+		echo absent
+		;;
+	(*)
+		if ! test -f "${__object:?}/parameter/value"
+		then
+			echo present
+		elif psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")"
+		then
+			echo present
+		else
 			echo different
-			;;
-	esac
-fi
+		fi
+		;;
+esac

From 19bf37be1a8b0b099a7199d1565b74598010f614 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 15:55:52 +0200
Subject: [PATCH 254/366] [type/__postgres_conf] Update man.rst

---
 cdist/conf/type/__postgres_conf/man.rst | 39 ++++++++++++++-----------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst
index 76016c6c..e035f080 100644
--- a/cdist/conf/type/__postgres_conf/man.rst
+++ b/cdist/conf/type/__postgres_conf/man.rst
@@ -1,29 +1,27 @@
 cdist-type__postgres_conf(7)
 ============================
 
-Configure a PostgreSQL server.
-
-NOTE: This type might need to be run multiple times to apply all bits of the
-configuration due to ordering requirements.
-
-SSRQ <cdist--@--ssrq-sds-fds.ch>
+NAME
+----
+cdist-type__postgres_conf - Alter PostgreSQL configuration
 
 
 DESCRIPTION
 -----------
-Configure a PostgreSQL server using ALTER SYSTEM.
+Configure a running PostgreSQL server using ``ALTER SYSTEM``.
 
 
 REQUIRED PARAMETERS
 -------------------
 value
-    The value to setup (can be omitted when state is set to "absent").
+   The value to set (can be omitted if ``--state`` is set to ``absent``).
 
 
 OPTIONAL PARAMETERS
 -------------------
 state
-    "present" or "absent". Defaults to "present".
+   ``present`` or ``absent``.
+   Defaults to ``present``.
 
 
 BOOLEAN PARAMETERS
@@ -36,20 +34,27 @@ EXAMPLES
 
 .. code-block:: sh
 
-	# set timezone
-    __postgres_conf timezone --value Europe/Zurich
+   # set timezone
+   __postgres_conf timezone --value Europe/Zurich
 
-    # reset maximum number of concurrent connections to default (normally 100)
-    __postgres_conf max_connections --state absent
+   # reset maximum number of concurrent connections to default (normally 100)
+   __postgres_conf max_connections --state absent
 
 
 SEE ALSO
 --------
-- `cdist-type(7) <cdist-type.html>`_
+None.
+
+
+AUTHORS
+-------
+Beni Ruef (bernhard.ruef--@--ssrq-sds-fds.ch)
+Dennis Camera (dennis.camera--@--ssrq-sds-fds.ch)
 
 
 COPYING
 -------
-Copyright \(C) 2020 SSRQ (www.ssrq-sds-fds.ch).
-Free use of this software is granted under the terms
-of the GNU General Public License version 3 (GPLv3).
+Copyright \(C) 2019-2021 SSRQ (www.ssrq-sds-fds.ch).
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.

From 1c047353a95e7ac079ccf89af8fc232451b8f891 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Sat, 17 Apr 2021 09:57:10 +0200
Subject: [PATCH 255/366] [bin/cdist] Fix Python version check

---
 bin/cdist         | 8 +++++---
 cdist/__init__.py | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/bin/cdist b/bin/cdist
index ddaffa7f..adb06a8d 100755
--- a/bin/cdist
+++ b/bin/cdist
@@ -72,9 +72,11 @@ def commandline():
 
 
 if __name__ == "__main__":
-    if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION:
-        print('Python >= {} is required on the source host.'.format(
-                cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr)
+    if sys.version_info[:3] < cdist.MIN_SUPPORTED_PYTHON_VERSION:
+        print(
+            'Python >= {} is required on the source host.'.format(
+                ".".join(map(str, cdist.MIN_SUPPORTED_PYTHON_VERSION))),
+            file=sys.stderr)
         sys.exit(1)
 
     exit_code = 0
diff --git a/cdist/__init__.py b/cdist/__init__.py
index 44366cd0..31d49889 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -64,7 +64,7 @@ REMOTE_EXEC = "ssh -o User=root"
 REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
 
 
-MIN_SUPPORTED_PYTHON_VERSION = '3.5'
+MIN_SUPPORTED_PYTHON_VERSION = (3, 5)
 
 
 class Error(Exception):

From 1bb696a4100f161afada9b1e8d8ad2b8ab190f54 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 20 Apr 2021 07:33:07 +0200
Subject: [PATCH 256/366] Release 6.9.6

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index c8cb44da..70d0b960 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong
 	* Type __ssh_authorized_key: grep only if file exists (Dennis Camera)

From acf9bf91f17fb4ad0e5813c6e4b0b40fc7e027b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Thu, 22 Apr 2021 08:55:14 +0200
Subject: [PATCH 257/366] [scanner] error to stderr and exit when scapy is not
 available

---
 cdist/scan/commandline.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index eca4cf13..0d7fb0ca 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -20,6 +20,7 @@
 #
 
 import logging
+import sys
 
 log = logging.getLogger("scan")
 
@@ -31,7 +32,9 @@ def commandline(args):
     try:
         import cdist.scan.scan as scan
     except ModuleNotFoundError:
-        print('cdist scan requires scapy to be installed')
+        print('cdist scan requires scapy to be installed! Exiting.',
+                file=sys.stderr)
+        sys.exit(1)
 
     processes = []
 

From a4464209b6741f2c1764aa432c56cfa8128852a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Thu, 22 Apr 2021 09:29:53 +0200
Subject: [PATCH 258/366] [scanner] add minimal error handling, consolidate CLI
 args processing

---
 cdist/argparse.py         | 10 ++++--
 cdist/scan/commandline.py | 19 ++++++-----
 cdist/scan/scan.py        | 71 +++++++++++----------------------------
 3 files changed, 37 insertions(+), 63 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index cadac39a..153e7864 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -492,12 +492,16 @@ def get_parsers():
         help='Try to configure detected hosts')
     parser['scan'].add_argument(
         '-I', '--interfaces',
-        action='append',  default=[],
+        action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
     parser['scan'].add_argument(
         '-d', '--delay',
-        action='store',  default=3600,
-        help='How long to wait before reconfiguring after last try')
+        action='store',  default=3600, type=int,
+        help='How long (seconds) to wait before reconfiguring after last try')
+    parser['scan'].add_argument(
+        '-t', '--trigger-delay',
+        action='store',  default=5, type=int,
+        help='How long (seconds) to wait between ICMPv6 echo requests')
     parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
 
     for p in parser:
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 0d7fb0ca..dead5292 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -24,26 +24,29 @@ import sys
 
 log = logging.getLogger("scan")
 
-
-# define this outside of the class to not handle scapy import errors by default
+# CLI processing is defined outside of the main scan class to handle
+# non-available optional scapy dependency (instead of crashing mid-flight).
 def commandline(args):
     log.debug(args)
 
+    # Check if we have the optional scapy dependency available.
     try:
         import cdist.scan.scan as scan
     except ModuleNotFoundError:
-        print('cdist scan requires scapy to be installed! Exiting.',
-                file=sys.stderr)
+        log.error('cdist scan requires scapy to be installed. Exiting.')
         sys.exit(1)
 
-    processes = []
-
+    # Default operation mode.
     if not args.mode:
-        # By default scan and trigger, but do not call any action
+        # By default scan and trigger, but do not call any action.
         args.mode = ['scan', 'trigger', ]
 
+    # We run each component in a separate process since they
+    # must not block on each other.
+    processes = []
+
     if 'trigger' in args.mode:
-        t = scan.Trigger(interfaces=args.interfaces)
+        t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay)
         t.start()
         processes.append(t)
         log.debug("Trigger started")
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index faee8a56..633a5c06 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -61,20 +61,22 @@ import datetime
 
 import cdist.config
 
+logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("scan")
 
-
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
     """
 
-    def __init__(self, interfaces=None, verbose=False):
+    def __init__(self, interfaces, sleeptime, verbose=False):
         self.interfaces = interfaces
+
+        # Used by scapy / send in trigger/2.
         self.verbose = verbose
 
-        # Wait 5 seconds before triggering again - FIXME: add parameter
-        self.sleeptime = 5
+        # Delay in seconds between sent ICMPv6EchoRequests.
+        self.sleeptime = sleeptime
 
     def start(self):
         self.processes = []
@@ -93,9 +95,12 @@ class Trigger(object):
             time.sleep(self.sleeptime)
 
     def trigger(self, interface):
-        packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest()
-        log.debug("Sending request on %s", interface)
-        send(packet, verbose=self.verbose)
+        try:
+            log.debug("Sending ICMPv6EchoRequest on %s", interface)
+            packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest()
+            send(packet, verbose=self.verbose)
+        except Exception as e:
+            log.error( "Could not send ICMPv6EchoRequest: %s", e)
 
 
 class Scanner(object):
@@ -103,7 +108,7 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces=None, args=None, outdir=None):
+    def __init__(self, interfaces, args=None, outdir=None):
         self.interfaces = interfaces
 
         if outdir:
@@ -148,47 +153,9 @@ class Scanner(object):
 
     def scan(self):
         log.debug("Scanning - zzzzz")
-        sniff(iface=self.interfaces,
-              filter="icmp6",
-              prn=self.handle_pkg)
-
-
-if __name__ == '__main__':
-    t = Trigger(interfaces=["wlan0"])
-    t.start()
-
-    # Scanner can listen on many interfaces at the same time
-    s = Scanner(interfaces=["wlan0"])
-    s.scan()
-
-    # Join back the trigger processes
-    t.join()
-
-    # Test in my lan shows:
-    # [18:48] bridge:cdist% ls -1d fe80::*
-    # fe80::142d:f0a5:725b:1103
-    # fe80::20d:b9ff:fe49:ac11
-    # fe80::20d:b9ff:fe4c:547d
-    # fe80::219:d2ff:feb2:2e12
-    # fe80::21b:fcff:feee:f446
-    # fe80::21b:fcff:feee:f45c
-    # fe80::21b:fcff:feee:f4b1
-    # fe80::21b:fcff:feee:f4ba
-    # fe80::21b:fcff:feee:f4bc
-    # fe80::21b:fcff:feee:f4c1
-    # fe80::21d:72ff:fe86:46b
-    # fe80::42b0:34ff:fe6f:f6f0
-    # fe80::42b0:34ff:fe6f:f863
-    # fe80::42b0:34ff:fe6f:f9b2
-    # fe80::4a5d:60ff:fea1:e55f
-    # fe80::77a3:5e3f:82cc:f2e5
-    # fe80::9e93:4eff:fe6c:c1f4
-    # fe80::ba69:f4ff:fec5:6041
-    # fe80::ba69:f4ff:fec5:8db7
-    # fe80::bad8:12ff:fe65:313d
-    # fe80::bad8:12ff:fe65:d9b1
-    # fe80::ce2d:e0ff:fed4:2611
-    # fe80::ce32:e5ff:fe79:7ea7
-    # fe80::d66d:6dff:fe33:e00
-    # fe80::e2ff:f7ff:fe00:20e6
-    # fe80::f29f:c2ff:fe7c:275e
+        try:
+            sniff(iface=self.interfaces,
+                  filter="icmp6",
+                  prn=self.handle_pkg)
+        except Exception as e:
+            log.error( "Could not start listener: %s", e)

From bb24d632d62da8390dd1a724fc325a57526da40d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Thu, 22 Apr 2021 10:20:49 +0200
Subject: [PATCH 259/366] [scanner] implement the --list flag

---
 cdist/argparse.py         |  4 +++
 cdist/scan/commandline.py | 63 ++++++++++++++++++++++++++++-----------
 cdist/scan/scan.py        | 17 +++++++++++
 3 files changed, 67 insertions(+), 17 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index 153e7864..f390a974 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -486,6 +486,10 @@ def get_parsers():
         '-m', '--mode', help='Which modes should run',
         action='append', default=[],
         choices=['scan', 'trigger'])
+    parser['scan'].add_argument(
+        '--list',
+        action='store_true',
+        help='List the known hosts and exit')
     parser['scan'].add_argument(
         '--config',
         action='store_true',
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index dead5292..331694e4 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -21,26 +21,11 @@
 
 import logging
 import sys
+from datetime import datetime
 
 log = logging.getLogger("scan")
 
-# CLI processing is defined outside of the main scan class to handle
-# non-available optional scapy dependency (instead of crashing mid-flight).
-def commandline(args):
-    log.debug(args)
-
-    # Check if we have the optional scapy dependency available.
-    try:
-        import cdist.scan.scan as scan
-    except ModuleNotFoundError:
-        log.error('cdist scan requires scapy to be installed. Exiting.')
-        sys.exit(1)
-
-    # Default operation mode.
-    if not args.mode:
-        # By default scan and trigger, but do not call any action.
-        args.mode = ['scan', 'trigger', ]
-
+def run(scan, args):
     # We run each component in a separate process since they
     # must not block on each other.
     processes = []
@@ -59,3 +44,47 @@ def commandline(args):
 
     for process in processes:
         process.join()
+
+def list(scan, args):
+    s = scan.Scanner(interfaces=args.interfaces, args=args)
+    hosts = s.list()
+
+    # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
+    # 6 colons.
+    ipv6_max_size = 8 * 4 + 10
+    # We format dates as follow: YYYY-MM-DD HH:MM:SS
+    date_max_size =              8 + 2    + 6 + 2
+
+    print("{} | {}".format(
+        'link-local address'.ljust(ipv6_max_size),
+        'last seen'.ljust(date_max_size)))
+    print('=' * (ipv6_max_size + 3 + date_max_size))
+    for addr in hosts:
+        last_seen = datetime.strftime(
+                datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'),
+                '%Y-%m-%d %H:%M:%S')
+        print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size)))
+
+# CLI processing is defined outside of the main scan class to handle
+# non-available optional scapy dependency (instead of crashing mid-flight).
+def commandline(args):
+    log.debug(args)
+
+    # Check if we have the optional scapy dependency available.
+    try:
+        import cdist.scan.scan as scan
+    except ModuleNotFoundError:
+        log.error('cdist scan requires scapy to be installed. Exiting.')
+        sys.exit(1)
+
+    # Set default operation mode.
+    if not args.mode:
+        # By default scan and trigger, but do not call any action.
+        args.mode = ['scan', 'trigger', ]
+
+    # Print known hosts and exit is --list is specified - do not start
+    # the scanner.
+    if args.list:
+        list(scan, args)
+    else:
+        run(scan, args)
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 633a5c06..f3976370 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -132,6 +132,23 @@ class Scanner(object):
             with open(fname, "w") as fd:
                 fd.write(f"{now}\n")
 
+    def list(self):
+        hosts = dict()
+        for linklocal_addr in os.listdir(self.outdir):
+            workdir = os.path.join(self.outdir, linklocal_addr)
+            # We ignore any (unexpected) file in this directory.
+            if os.path.isdir(workdir):
+                last_seen='-'
+                last_seen_file = os.path.join(workdir, 'last_seen')
+                if os.path.isfile(last_seen_file):
+                    with open(last_seen_file, "r") as fd:
+                        last_seen = fd.readline()
+
+                hosts[linklocal_addr] = {'last_seen': last_seen}
+
+        return hosts
+
+
     def config(self):
         """
         Configure a host

From 13e2ad175f01b6cfbdb21ee50e85b6f3ba6fe750 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Sun, 25 Apr 2021 12:45:34 +0200
Subject: [PATCH 260/366] [scanner] add host class, name mapper and pre-config
 logic

---
 cdist/argparse.py         |   6 +-
 cdist/scan/commandline.py |  37 ++++++++----
 cdist/scan/scan.py        | 122 +++++++++++++++++++++++++++++---------
 3 files changed, 124 insertions(+), 41 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index f390a974..bedb23ac 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -485,7 +485,7 @@ def get_parsers():
     parser['scan'].add_argument(
         '-m', '--mode', help='Which modes should run',
         action='append', default=[],
-        choices=['scan', 'trigger'])
+        choices=['scan', 'trigger', 'config'])
     parser['scan'].add_argument(
         '--list',
         action='store_true',
@@ -498,6 +498,10 @@ def get_parsers():
         '-I', '--interfaces',
         action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
+    parser['scan'].add_argument(
+        '--name-mapper',
+        action='store',  default=None,
+        help='Map addresses to names, required for config mode')
     parser['scan'].add_argument(
         '-d', '--delay',
         action='store',  default=3600, type=int,
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 331694e4..1a0ab0a7 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -37,7 +37,10 @@ def run(scan, args):
         log.debug("Trigger started")
 
     if 'scan' in args.mode:
-        s = scan.Scanner(interfaces=args.interfaces, args=args)
+        s = scan.Scanner(
+                autoconfigure='config' in args.mode,
+                interfaces=args.interfaces,
+                name_mapper=args.name_mapper)
         s.start()
         processes.append(s)
         log.debug("Scanner started")
@@ -46,24 +49,27 @@ def run(scan, args):
         process.join()
 
 def list(scan, args):
-    s = scan.Scanner(interfaces=args.interfaces, args=args)
+    s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper)
     hosts = s.list()
 
     # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
     # 6 colons.
     ipv6_max_size = 8 * 4 + 10
-    # We format dates as follow: YYYY-MM-DD HH:MM:SS
-    date_max_size =              8 + 2    + 6 + 2
+    date_max_size = len(datetime.now().strftime(scan.datetime_format))
+    name_max_size = 25
 
-    print("{} | {}".format(
-        'link-local address'.ljust(ipv6_max_size),
-        'last seen'.ljust(date_max_size)))
-    print('=' * (ipv6_max_size + 3 + date_max_size))
-    for addr in hosts:
-        last_seen = datetime.strftime(
-                datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'),
-                '%Y-%m-%d %H:%M:%S')
-        print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size)))
+    print("{} | {} | {} | {}".format(
+        'name'.ljust(name_max_size),
+        'address'.ljust(ipv6_max_size),
+        'last seen'.ljust(date_max_size),
+        'last configured'.ljust(date_max_size)))
+    print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
+    for host in hosts:
+        print("{} | {} | {} | {}".format(
+            host.name(default='-').ljust(name_max_size),
+            host.address().ljust(ipv6_max_size),
+            host.last_seen().ljust(date_max_size),
+            host.last_configured().ljust(date_max_size)))
 
 # CLI processing is defined outside of the main scan class to handle
 # non-available optional scapy dependency (instead of crashing mid-flight).
@@ -82,6 +88,11 @@ def commandline(args):
         # By default scan and trigger, but do not call any action.
         args.mode = ['scan', 'trigger', ]
 
+    if 'config' in args.mode and args.name_mapper == None:
+        print('--name-mapper must be specified for scanner config mode.',
+                file=sys.stderr)
+        sys.exit(1)
+
     # Print known hosts and exit is --list is specified - do not start
     # the scanner.
     if args.list:
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index f3976370..459138e2 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -63,6 +63,69 @@ import cdist.config
 
 logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("scan")
+datetime_format = '%Y-%m-%d %H:%M:%S'
+
+class Host(object):
+    def __init__(self, addr, outdir, name_mapper=None):
+        self.addr = addr
+        self.workdir = os.path.join(outdir, addr)
+        self.name_mapper = name_mapper
+
+        os.makedirs(self.workdir, exist_ok=True)
+
+    def __get(self, key, default=None):
+        fname = os.path.join(self.workdir, key)
+        value=default
+        if os.path.isfile(fname):
+            with open(fname, "r") as fd:
+                value = fd.readline()
+        return value
+
+    def __set(self, key, value):
+        fname = os.path.join(self.workdir, key)
+        with open(fname, "w") as fd:
+            fd.write(f"{value}")
+
+    def name(self, default=None):
+        if self.name_mapper == None:
+            return default
+
+        fpath = os.path.join(os.getcwd(), self.name_mapper)
+        if os.path.isfile(fpath) and os.access(fpath, os.X_OK):
+             out = subprocess.run([fpath, self.addr], capture_output=True)
+             if out.returncode != 0:
+                 return default
+             else:
+                value = out.stdout.decode()
+                return (None if len(value) == 0 else value)
+        else:
+            return default
+
+    def address(self):
+        return self.addr
+
+    def last_seen(self, default=None):
+        raw = self.__get('last_seen')
+        if raw:
+            return datetime.datetime.strptime(raw, datetime_format)
+        else:
+            return default
+
+    def last_configured(self, default=None):
+        raw = self.__get('last_configured')
+        if raw:
+            return datetime.datetime.strptime(raw, datetime_format)
+        else:
+            return default
+
+    def seen(self):
+        now = datetime.datetime.now().strftime(datetime_format)
+        self.__set('last_seen', now)
+
+    def configure(self):
+        # TODO: configure.
+        now = datetime.datetime.now().strftime(datetime_format)
+        self.__set('last_configured', now)
 
 class Trigger(object):
     """
@@ -108,48 +171,43 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces, args=None, outdir=None):
+    def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None):
         self.interfaces = interfaces
+        self.autoconfigure=autoconfigure
+        self.name_mapper = name_mapper
+        self.config_delay = datetime.timedelta(seconds=3600)
 
         if outdir:
             self.outdir = outdir
         else:
             self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan')
+        os.makedirs(self.outdir, exist_ok=True)
+
+        self.running_configs = {}
 
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
-            host = pkg['IPv6'].src
-            log.verbose("Host %s is alive", host)
+            host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper)
+            if host.name():
+                log.verbose("Host %s (%s) is alive", host.name(), host.address())
+            else:
+                log.verbose("Host %s is alive", host.address())
+            host.seen()
 
-            dir = os.path.join(self.outdir, host)
-            fname = os.path.join(dir, "last_seen")
-
-            now = datetime.datetime.now()
-
-            os.makedirs(dir, exist_ok=True)
-
-            # FIXME: maybe adjust the format so we can easily parse again
-            with open(fname, "w") as fd:
-                fd.write(f"{now}\n")
+            # TODO check last config.
+            if self.autoconfigure and \
+                    host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now():
+                self.config(host)
 
     def list(self):
-        hosts = dict()
-        for linklocal_addr in os.listdir(self.outdir):
-            workdir = os.path.join(self.outdir, linklocal_addr)
-            # We ignore any (unexpected) file in this directory.
-            if os.path.isdir(workdir):
-                last_seen='-'
-                last_seen_file = os.path.join(workdir, 'last_seen')
-                if os.path.isfile(last_seen_file):
-                    with open(last_seen_file, "r") as fd:
-                        last_seen = fd.readline()
-
-                hosts[linklocal_addr] = {'last_seen': last_seen}
+        hosts = []
+        for addr in os.listdir(self.outdir):
+            hosts.append(Host(addr, self.outdir, self.name_mapper))
 
         return hosts
 
 
-    def config(self):
+    def config(self, host):
         """
         Configure a host
 
@@ -158,9 +216,19 @@ class Scanner(object):
         - Maybe keep dict storing per host processes
         - Save the result
         - Save the output -> probably aligned to config mode
-
         """
 
+        if host.name() == None:
+            log.debug("config - could not resolve name for %s, aborting.", host.address())
+            return
+
+        if self.running_configs.get(host.name()) != None:
+            log.debug("config - is already running for %s, aborting.", host.name())
+
+        log.info("config - running against host %s.", host.name())
+        p = host.configure()
+        self.running_configs[host.name()] = p
+
     def start(self):
         self.process = Process(target=self.scan)
         self.process.start()

From 512e9b23c0035b09ac1e7e76f706fee2e2ec481f Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 25 Apr 2021 15:53:40 +0200
Subject: [PATCH 261/366] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 70d0b960..c41743cd 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
+
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)
 	* Core: config: Make local state directory available to custom remotes (Steven Armstrong

From 6ac8cbf98f5d5018373db6eb9c038ef1c599ff2f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:33:56 +0200
Subject: [PATCH 262/366] [type/__postgres_database] Include postgres_user
 explorer from __postgres_conf

---
 .../explorer/postgres_user                    |  1 +
 .../type/__postgres_database/explorer/state   | 23 +++++--------------
 .../type/__postgres_database/gencode-remote   | 13 +----------
 3 files changed, 8 insertions(+), 29 deletions(-)
 create mode 120000 cdist/conf/type/__postgres_database/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_database/explorer/postgres_user b/cdist/conf/type/__postgres_database/explorer/postgres_user
new file mode 120000
index 00000000..714e7237
--- /dev/null
+++ b/cdist/conf/type/__postgres_database/explorer/postgres_user
@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user
\ No newline at end of file
diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state
index d68d4120..bb044e0e 100755
--- a/cdist/conf/type/__postgres_database/explorer/state
+++ b/cdist/conf/type/__postgres_database/explorer/state
@@ -1,6 +1,7 @@
 #!/bin/sh
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,25 +19,13 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$("${__explorer}/os")"
-in
-    netbsd)
-        postgres_user='pgsql'
-        ;;
-    openbsd)
-        postgres_user='_postgresql'
-        ;;
-    *)
-        postgres_user='postgres'
-        ;;
-esac
+postgres_user=$("${__type_explorer:?}/postgres_user")
 
+name=${__object_id:?}
 
-name="$__object_id"
-
-if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='$name'\"")"
+if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='${name}'\"")"
 then
-    echo 'present'
+	echo 'present'
 else
-    echo 'absent'
+	echo 'absent'
 fi
diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote
index 0f11cff4..d70fc436 100755
--- a/cdist/conf/type/__postgres_database/gencode-remote
+++ b/cdist/conf/type/__postgres_database/gencode-remote
@@ -18,23 +18,12 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
-in
-   netbsd)
-      postgres_user='pgsql'
-      ;;
-   openbsd)
-      postgres_user='_postgresql'
-      ;;
-   *)
-      postgres_user='postgres'
-      ;;
-esac
 
 
 name="$__object_id"
 state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
 if [ "$state_should" != "$state_is" ]; then
    case "$state_should" in

From 58b279a8d0121dcbef05a4999d1d88208656aeed Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:34:17 +0200
Subject: [PATCH 263/366] [type/__postgres_database] Improve quoting

---
 .../type/__postgres_database/gencode-remote   | 95 +++++++++++--------
 1 file changed, 55 insertions(+), 40 deletions(-)

diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote
index d70fc436..7d7d6fa2 100755
--- a/cdist/conf/type/__postgres_database/gencode-remote
+++ b/cdist/conf/type/__postgres_database/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,49 +19,63 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+quote() {
+	for _arg
+	do
+		shift
+		if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')"
+		then
+			# needs quoting
+			set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'"
+		else
+			set -- "$@" "${_arg}"
+		fi
+	done
+	unset _arg
 
+	# NOTE: Use printf because POSIX echo interprets escape sequences
+	printf '%s' "$*"
+}
 
-name="$__object_id"
-state_should="$(cat "$__object/parameter/state")"
-state_is="$(cat "$__object/explorer/state")"
 postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
-if [ "$state_should" != "$state_is" ]; then
-   case "$state_should" in
-      present)
-         owner=""
-         if [ -f "$__object/parameter/owner" ]; then
-            owner="-O \"$(cat "$__object/parameter/owner")\""
-         fi
+dbname=${__object_id:?}
+state_should=$(cat "${__object:?}/parameter/state")
+state_is=$(cat "${__object:?}/explorer/state")
 
-         template=""
-         if [ -f "$__object/parameter/template" ]; then
-            template="--template \"$(cat "$__object/parameter/template")\""
-         fi
-
-         encoding=""
-         if [ -f "$__object/parameter/encoding" ]; then
-            encoding="--encoding \"$(cat "$__object/parameter/encoding")\""
-         fi
-
-         lc_collate=""
-         if [ -f "$__object/parameter/lc-collate" ]; then
-            lc_collate="--lc-collate \"$(cat "$__object/parameter/lc-collate")\""
-         fi
-
-         lc_ctype=""
-         if [ -f "$__object/parameter/lc-ctype" ]; then
-            lc_ctype="--lc-ctype \"$(cat "$__object/parameter/lc-ctype")\""
-         fi
-
-         cat << EOF
-su - '$postgres_user' -c "createdb $owner \"$name\" $template $encoding $lc_collate $lc_ctype"
-EOF
-      ;;
-      absent)
-         cat << EOF
-su - '$postgres_user' -c "dropdb \"$name\""
-EOF
-      ;;
-   esac
+if test "${state_should}" = "$state_is"
+then
+	exit 0
 fi
+
+case ${state_should}
+in
+	(present)
+		set --
+
+		while read -r param_name opt
+		do
+			if test -f "${__object:?}/parameter/${param_name}"
+			then
+				set -- "$@" "${opt}" "$(cat "${__object:?}/parameter/${param_name}")"
+			fi
+		done <<-'EOF'
+		owner -O
+		template --template
+		encoding --encoding
+		lc_collate --lc-collate
+		lc_ctype --lc-ctype
+		EOF
+
+		set -- "$@" "${dbname}"
+
+		cat <<-EOF
+		su - $(quote "${postgres_user}") -c $(quote "$(quote createdb "$@")")
+		EOF
+		;;
+	(absent)
+		cat <<-EOF
+		su - $(quote "${postgres_user}") -c $(quote "$(quote dropdb "${dbname}")")
+		EOF
+		;;
+esac

From beb8da6d5fd141ac63eacfb526cfcc43cbf22d8f Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:48:51 +0200
Subject: [PATCH 264/366] [type/__postgres_role] Include postgres_user explorer
 from __postgres_conf

---
 .../type/__postgres_role/explorer/postgres_user   |  1 +
 cdist/conf/type/__postgres_role/explorer/state    | 14 +-------------
 cdist/conf/type/__postgres_role/gencode-remote    | 15 +--------------
 3 files changed, 3 insertions(+), 27 deletions(-)
 create mode 120000 cdist/conf/type/__postgres_role/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_role/explorer/postgres_user b/cdist/conf/type/__postgres_role/explorer/postgres_user
new file mode 120000
index 00000000..714e7237
--- /dev/null
+++ b/cdist/conf/type/__postgres_role/explorer/postgres_user
@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user
\ No newline at end of file
diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 34069de9..0b264e6f 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -19,19 +19,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case $("${__explorer:?}/os")
-in
-	(netbsd)
-		postgres_user='pgsql'
-		;;
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(*)
-		postgres_user='postgres'
-		;;
-esac
-
+postgres_user=$("${__type_explorer:?}/postgres_user")
 rolename=${__object_id:?}
 
 
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index d7631fbd..7324b80c 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -28,20 +28,7 @@ quote() {
 	fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
 }
 
-case $(cat "${__global:?}/explorer/os")
-in
-	(netbsd)
-		postgres_user='pgsql'
-		;;
-	(openbsd)
-		postgres_user='_postgresql'
-		;;
-	(*)
-		postgres_user='postgres'
-		;;
-esac
-
-
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 rolename=${__object_id:?}
 state_is=$(cat "${__object:?}/explorer/state")
 state_should=$(cat "${__object:?}/parameter/state")

From 3cf93249c38a6f27c097877add619b5d6499a049 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 15 Apr 2021 16:55:11 +0200
Subject: [PATCH 265/366] [type/__postgres_extension] Include postgres_user
 explorer from __postgres_conf

---
 .../__postgres_extension/explorer/postgres_user     |  1 +
 cdist/conf/type/__postgres_extension/gencode-remote | 13 +------------
 2 files changed, 2 insertions(+), 12 deletions(-)
 create mode 120000 cdist/conf/type/__postgres_extension/explorer/postgres_user

diff --git a/cdist/conf/type/__postgres_extension/explorer/postgres_user b/cdist/conf/type/__postgres_extension/explorer/postgres_user
new file mode 120000
index 00000000..714e7237
--- /dev/null
+++ b/cdist/conf/type/__postgres_extension/explorer/postgres_user
@@ -0,0 +1 @@
+../../__postgres_conf/explorer/postgres_user
\ No newline at end of file
diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote
index af9c97f1..a1c84828 100755
--- a/cdist/conf/type/__postgres_extension/gencode-remote
+++ b/cdist/conf/type/__postgres_extension/gencode-remote
@@ -22,18 +22,7 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-case "$(cat "${__global}/explorer/os")"
-in
-   netbsd)
-      postgres_user='pgsql'
-      ;;
-   openbsd)
-      postgres_user='_postgresql'
-      ;;
-   *)
-      postgres_user='postgres'
-      ;;
-esac
+postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
 
 dbname=$(    echo "$__object_id" | cut -d":" -f1 )

From 8296051653a8187374ad51a3a487315767b73a19 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 16 Apr 2021 19:22:58 +0200
Subject: [PATCH 266/366] [type/__postgres_extension] Add state explorer

---
 .../type/__postgres_extension/explorer/state  | 41 ++++++++++++++
 .../type/__postgres_extension/gencode-remote  | 44 ++++++++++-----
 cdist/conf/type/__postgres_extension/man.rst  | 55 +++++++++++--------
 3 files changed, 104 insertions(+), 36 deletions(-)
 create mode 100644 cdist/conf/type/__postgres_extension/explorer/state

diff --git a/cdist/conf/type/__postgres_extension/explorer/state b/cdist/conf/type/__postgres_extension/explorer/state
new file mode 100644
index 00000000..9d156be7
--- /dev/null
+++ b/cdist/conf/type/__postgres_extension/explorer/state
@@ -0,0 +1,41 @@
+#!/bin/sh -e
+# -*- mode: sh; indent-tabs-mode: t -*-
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Prints "present" if the extension is currently installed.
+# "absent" otherwise.
+
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+
+postgres_user=$("${__type_explorer:?}/postgres_user")
+
+IFS=: read -r dbname extname <<EOF
+${__object_id:?}
+EOF
+
+psql_exec() {
+	su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")"
+}
+
+if psql_exec "${dbname}" 'SELECT extname FROM pg_extension' | grep -qFx "${extname}"
+then
+	echo present
+else
+	echo absent
+fi
diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote
index a1c84828..4ce72622 100755
--- a/cdist/conf/type/__postgres_extension/gencode-remote
+++ b/cdist/conf/type/__postgres_extension/gencode-remote
@@ -2,9 +2,10 @@
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2013 Tomas Pospisek (tpo_deb at sourcepole.ch)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This type was created by Tomas Pospisek based on the
-#__postgres_role type by Steven Armstrong
+# __postgres_role type by Steven Armstrong.
 #
 # This file is part of cdist.
 #
@@ -24,19 +25,36 @@
 
 postgres_user=$(cat "${__object:?}/explorer/postgres_user")
 
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_cmd() {
+	printf 'su - %s -c %s\n' \
+		"$(quote "${postgres_user}")" \
+		"$(quote psql "$(quote "$1")" -c "$(quote "$2")")"
+}
 
-dbname=$(    echo "$__object_id" | cut -d":" -f1 )
-extension=$( echo "$__object_id" | cut -d":" -f2 )
 
-state_should=$( cat "$__object/parameter/state" )
+IFS=: read -r dbname extname <<EOF
+${__object_id:?}
+EOF
 
-case "$state_should" in
-    present)
-         cmd="CREATE EXTENSION IF NOT EXISTS $extension"
-         echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'"
-    ;;
-    absent)
-         cmd="DROP EXTENSION IF EXISTS $extension"
-         echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'"
-    ;;
+state_is=$(cat "${__object:?}/explorer/state")
+state_should=$(cat "${__object:?}/parameter/state")
+
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
+
+case ${state_should}
+in
+	(present)
+		psql_cmd "${dbname}" "CREATE EXTENSION ${extname}"
+		;;
+	(absent)
+		psql_cmd "${dbname}" "DROP EXTENSION ${extname}"
+		;;
+	(*)
+		printf 'Invalid --state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
 esac
diff --git a/cdist/conf/type/__postgres_extension/man.rst b/cdist/conf/type/__postgres_extension/man.rst
index 79645b2b..442239f6 100644
--- a/cdist/conf/type/__postgres_extension/man.rst
+++ b/cdist/conf/type/__postgres_extension/man.rst
@@ -3,32 +3,36 @@ cdist-type__postgres_extension(7)
 
 NAME
 ----
-cdist-type__postgres_extension - manage postgres extensions
+cdist-type__postgres_extension - Manage PostgreSQL extensions
 
 
 DESCRIPTION
 -----------
-This cdist type allows you to create or drop postgres extensions.
+This cdist type allows you to manage PostgreSQL extensions.
 
-The object you need to pass to __postgres_extension consists of
-the database name and the extension name joined by a colon in the
-following form:
-
-.. code-block:: sh
-
-    dbname:extension
-
-f.ex.
+The ``__object_id`` to pass to ``__postgres_extension`` is of the form
+``dbname:extension``, e.g.:
 
 .. code-block:: sh
 
     rails_test:unaccent
 
 
+**CAUTION!** Be careful when installing extensions from (untrusted) third-party
+sources:
+
+   | Installing an extension as superuser requires trusting that the extension's
+     author wrote the extension installation script in a secure fashion. It is
+     not terribly difficult for a malicious user to create trojan-horse objects
+     that will compromise later execution of a carelessly-written extension
+     script, allowing that user to acquire superuser privileges.
+   | – `<https://www.postgresql.org/docs/13/sql-createextension.html#id-1.9.3.64.7>`_
+
+
 OPTIONAL PARAMETERS
 -------------------
 state
-    either "present" or "absent", defaults to "present"
+    either ``present`` or ``absent``, defaults to ``present``.
 
 
 EXAMPLES
@@ -36,24 +40,29 @@ EXAMPLES
 
 .. code-block:: sh
 
-    __postgres_extension           rails_test:unaccent
-    __postgres_extension --present rails_test:unaccent
-    __postgres_extension --absent  rails_test:unaccent
+   # Install extension unaccent into database rails_test
+   __postgres_extension rails_test:unaccent
+
+   # Drop extension unaccent from database fails_test
+   __postgres_extension rails_test:unaccent --state absent
 
 
 SEE ALSO
 --------
-:strong:`cdist-type__postgre_database`\ (7)
+- :strong:`cdist-type__postgres_database`\ (7)
+- PostgreSQL "CREATE EXTENSION" documentation at:
+  `<http://www.postgresql.org/docs/current/static/sql-createextension.html>`_.
 
-Postgres "Create Extension" documentation at: <http://www.postgresql.org/docs/current/static/sql-createextension.html>.
 
-AUTHOR
+AUTHORS
 -------
-Tomas Pospisek <tpo_deb--@--sourcepole.ch>
+| Tomas Pospisek <tpo_deb--@--sourcepole.ch>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+
 
 COPYING
 -------
-Copyright \(C) 2014 Tomas Pospisek. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2014 Tomas Pospisek, 2021 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.

From 0d33407b182964672b1cfc3ffc49c7fa7cf462a0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Fri, 16 Apr 2021 19:35:26 +0200
Subject: [PATCH 267/366] [type/__postgres_database] Proper quoting in state
 explorer

---
 cdist/conf/type/__postgres_database/explorer/state | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state
index bb044e0e..6a25df86 100755
--- a/cdist/conf/type/__postgres_database/explorer/state
+++ b/cdist/conf/type/__postgres_database/explorer/state
@@ -21,9 +21,14 @@
 
 postgres_user=$("${__type_explorer:?}/postgres_user")
 
-name=${__object_id:?}
+dbname=${__object_id:?}
 
-if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='${name}'\"")"
+quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; }
+psql_exec() {
+	su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")"
+}
+
+if psql_exec postgres "SELECT datname FROM pg_database" | grep -qFx "${dbname}"
 then
 	echo 'present'
 else

From 0f05f38384c94b79232ca0917d1323eeefd6e1ed Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 19 Apr 2021 19:09:46 +0200
Subject: [PATCH 268/366] [type/__postgres_role] Treat --password '' like no
 --password

---
 cdist/conf/type/__postgres_role/explorer/state | 11 ++++-------
 cdist/conf/type/__postgres_role/gencode-remote |  2 +-
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state
index 0b264e6f..822816c1 100755
--- a/cdist/conf/type/__postgres_role/explorer/state
+++ b/cdist/conf/type/__postgres_role/explorer/state
@@ -43,8 +43,7 @@ role_properties=$(
 	  BEGIN { RS = "\036"; FS = "\034" }
 	  /^\([0-9]+ rows?\)/ { exit }
 	  NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next }
-	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }
-	  '
+	  NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }'
 )
 
 if test -n "${role_properties}"
@@ -78,12 +77,10 @@ then
 	# Check password
 	passwd_stored=$(
 		psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \
-		| awk 'BEGIN { RS = "\036" } NR == 2'
-		printf .
-	)
-	passwd_stored=${passwd_stored%?.}
+		| awk 'BEGIN { RS = "\036" } NR == 2 { printf "%s.", $0 }')
+	passwd_stored=${passwd_stored%.}
 
-	if test -f "${__object:?}/parameter/password"
+	if test -s "${__object:?}/parameter/password"
 	then
 		passwd_should=$(cat "${__object:?}/parameter/password"; printf .)
 	fi
diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote
index 7324b80c..4cb78330 100755
--- a/cdist/conf/type/__postgres_role/gencode-remote
+++ b/cdist/conf/type/__postgres_role/gencode-remote
@@ -46,7 +46,7 @@ psql_query() {
 
 psql_set_password() {
 	# NOTE: Always make sure that the password does not end up in psql_history!
-	# NOTE: Never set an empty string as the password, because they can be
+	# NOTE: Never set an empty string as the password, because it can be
 	#       interpreted differently by different tooling.
 	if test -s "${__object:?}/parameter/password"
 	then

From 92fff7cb77271009f25d999b6451dc1e43e9bb6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 26 Apr 2021 12:09:20 +0200
Subject: [PATCH 269/366] [scanner] fix crash on --list with name mapper
 provided

---
 cdist/scan/commandline.py | 10 ++++++++--
 cdist/scan/scan.py        |  2 +-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 1a0ab0a7..3eb7eec4 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -65,11 +65,17 @@ def list(scan, args):
         'last configured'.ljust(date_max_size)))
     print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
     for host in hosts:
+        last_seen = host.last_seen()
+        last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-'
+
+        last_configured = host.last_configured()
+        last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-'
+
         print("{} | {} | {} | {}".format(
             host.name(default='-').ljust(name_max_size),
             host.address().ljust(ipv6_max_size),
-            host.last_seen().ljust(date_max_size),
-            host.last_configured().ljust(date_max_size)))
+            last_seen.ljust(date_max_size),
+            last_configured.ljust(date_max_size)))
 
 # CLI processing is defined outside of the main scan class to handle
 # non-available optional scapy dependency (instead of crashing mid-flight).
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 459138e2..69a4121d 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -97,7 +97,7 @@ class Host(object):
                  return default
              else:
                 value = out.stdout.decode()
-                return (None if len(value) == 0 else value)
+                return (default if len(value) == 0 else value)
         else:
             return default
 

From 3a9dd5b1669fa29e1713cdadec1596d859d377f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 26 Apr 2021 12:09:55 +0200
Subject: [PATCH 270/366] [scanner] add minimal (non-configurable) config mode

---
 cdist/scan/scan.py | 35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 69a4121d..152abb4e 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -122,8 +122,20 @@ class Host(object):
         now = datetime.datetime.now().strftime(datetime_format)
         self.__set('last_seen', now)
 
+    # XXX: There's no easy way to use the config module without feeding it with
+    # CLI args. Might as well call everything from scratch!
     def configure(self):
-        # TODO: configure.
+        target = self.name() or self.address()
+        cmd = ['cdist', 'config', '-v', target ]
+
+        fname = os.path.join(self.workdir, 'last_configuration_log')
+        with open(fname, "w") as fd:
+            log.debug("Executing: %s", cmd)
+            completed_process = subprocess.run(cmd, stdout=fd, stderr=fd)
+            if completed_process.returncode != 0:
+                log.error("%s return with non-zero code %i - see %s for details.",
+                        cmd, completed_process.returncode, fname)
+
         now = datetime.datetime.now().strftime(datetime_format)
         self.__set('last_configured', now)
 
@@ -194,7 +206,7 @@ class Scanner(object):
                 log.verbose("Host %s is alive", host.address())
             host.seen()
 
-            # TODO check last config.
+            # Configure if needed.
             if self.autoconfigure and \
                     host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now():
                 self.config(host)
@@ -206,27 +218,18 @@ class Scanner(object):
 
         return hosts
 
-
     def config(self, host):
-        """
-        Configure a host
-
-        - Assume we are only called if necessary
-        - However we need to ensure to not run in parallel
-        - Maybe keep dict storing per host processes
-        - Save the result
-        - Save the output -> probably aligned to config mode
-        """
-
         if host.name() == None:
             log.debug("config - could not resolve name for %s, aborting.", host.address())
             return
 
-        if self.running_configs.get(host.name()) != None:
+        previous_config_process = self.running_configs.get(host.name())
+        if previous_config_process != None and previous_config_process.is_alive():
             log.debug("config - is already running for %s, aborting.", host.name())
 
-        log.info("config - running against host %s.", host.name())
-        p = host.configure()
+        log.info("config - running against host %s (%s).", host.name(), host.address())
+        p = Process(target=host.configure())
+        p.start()
         self.running_configs[host.name()] = p
 
     def start(self):

From 2232435c2206b2b3a9a5f2b976df16540b3f1eeb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 26 Apr 2021 14:39:26 +0200
Subject: [PATCH 271/366] [scanner] initial documentation

Note: still needs to patch main cdist(1) manpage
---
 cdist/scan/scan.py      | 32 -------------
 docs/src/cdist-scan.rst | 99 +++++++++++++++++++++++++++++++++++++++++
 docs/src/index.rst      |  1 +
 3 files changed, 100 insertions(+), 32 deletions(-)
 create mode 100644 docs/src/cdist-scan.rst

diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 152abb4e..2912dab3 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -19,38 +19,6 @@
 #
 #
 
-#
-# Interface to be implemented:
-# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated
-#   scan: scan / listen for icmp6 replies
-#   trigger: send trigger to multicast
-#   config: configure newly detected hosts
-#   install: install newly detected hosts
-#
-# Scanner logic
-#  - save results to configdir:
-#     basedir = ~/.cdist/scan/<ipv6-address>
-#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time
-#           or similar
-#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record
-#           unix time or similar
-#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record
-#           unix time or similar
-#
-#
-#
-#
-# cdist scan --list
-#       Show all known hosts including last seen flag
-#
-# Logic for reconfiguration:
-#
-#  - record when configured last time
-#  - introduce a parameter --reconfigure-after that takes time argument
-#  - reconfigure if a) host alive and b) reconfigure-after time passed
-#
-
-
 from multiprocessing import Process
 import os
 import logging
diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst
new file mode 100644
index 00000000..02193456
--- /dev/null
+++ b/docs/src/cdist-scan.rst
@@ -0,0 +1,99 @@
+Scan
+=====
+
+Description
+-----------
+Runs cdist as a daemon that discover/watch on hosts and reconfigure them
+periodically. It is especially useful in netboot-based environment where hosts
+boot unconfigured, and to ensure your infrastructure stays in sync with your
+configuration.
+
+This feature is still consider to be in **beta** stage.
+
+Usage (Examples)
+----------------
+
+Discover hosts on local network and configure those whose name is resolved by
+the name mapper script.
+
+.. code-block:: sh
+
+    $ cdist scan --beta --interface eth0 \
+      --mode scan --name-mapper path/to/script \
+      --mode trigger --mode config
+
+List known hosts and exit.
+
+.. code-block:: sh
+
+    $ cdist scan --beta --list --name-mapper path/to/script
+
+Please refer to `cdist(1)` for a detailed list of parameters.
+
+Modes
+-----
+
+The scanner has 3 modes that can be independently toggled. If the `--mode`
+parameter is not specified, only `tigger` and `scan` are enabled (= hosts are
+not configured).
+
+trigger
+  Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to
+  trigger detection by the `scan` module.
+
+scan
+  Watch for incoming ICMPv6 replies and optionally configure detected hosts.
+
+config
+  Enable configuration of hosts detected by `scan`.
+
+Name Mapper Script
+------------------
+
+The name mapper script takes an IPv6 address as first argument and writes the
+resolved name to stdout - if any. The script must be executable.
+
+Simplest script:
+
+.. code-block:: sh
+  #!/bin/sh
+
+  case "$1" in
+  	"fe80::20d:b9ff:fe57:3524")
+  		printf "my-host-01"
+  		;;
+  	"fe80::7603:bdff:fe05:89bb")
+  		printf "my-host-02"
+  		;;
+  esac
+
+Resolving name from `PTR` DNS record:
+
+.. code-block:: sh
+  #!/bin/sh
+
+  for cmd in dig sed; do
+  	if ! command -v $cmd > /dev/null; then
+  		exit 1
+  	fi
+  done
+
+  dig +short -x "$1" | sed -e 's/.$//'
+
+
+Trigger Source Script
+---------------------
+
+This script returns a list of addresses (separated by a newline) to be used by
+`trigger` mode. It is not used to map names. The script must be executable.
+
+Simplest script:
+
+.. code-block:: sh
+  #!/bin/sh
+
+  cat << EOF
+  server1.domain.tld
+  server2.domain.tld
+  server3.domain.tld
+  EOF
diff --git a/docs/src/index.rst b/docs/src/index.rst
index 31c044dc..369d5309 100644
--- a/docs/src/index.rst
+++ b/docs/src/index.rst
@@ -34,6 +34,7 @@ It natively supports IPv6 since the first release.
    cdist-parallelization
    cdist-inventory
    cdist-preos
+   cdist-scan
    cdist-integration
    cdist-reference
    cdist-best-practice

From a4122882f2a14034d1a219bd0314892176c5cb64 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Apr 2021 16:39:51 +0200
Subject: [PATCH 272/366] [type/__debconf_set_selections] Add state explorer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…and to make it work, replace --file with --line.

--file is deprecated because it does not work with the state explorer as the
contents of the file are not available on the target.
---
 .../__debconf_set_selections/explorer/state   | 124 ++++++++++++++++++
 .../__debconf_set_selections/gencode-remote   |  37 ++++--
 .../type/__debconf_set_selections/man.rst     |  49 ++++---
 .../type/__debconf_set_selections/manifest    |  21 +++
 .../parameter/{required => deprecated}        |   0
 .../parameter/optional_multiple               |   1 +
 6 files changed, 206 insertions(+), 26 deletions(-)
 create mode 100644 cdist/conf/type/__debconf_set_selections/explorer/state
 create mode 100755 cdist/conf/type/__debconf_set_selections/manifest
 rename cdist/conf/type/__debconf_set_selections/parameter/{required => deprecated} (100%)
 create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/optional_multiple

diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state
new file mode 100644
index 00000000..68d28941
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/explorer/state
@@ -0,0 +1,124 @@
+#!/bin/sh -e
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Determine current debconf selections' state.
+# Prints one of:
+#   present: all selections are already set as they should.
+#   different: one or more of the selections have a different value.
+#   absent: one or more of the selections are not (currently) defined.
+#
+
+test -x /usr/bin/perl || {
+	# cannot find perl (no perl ~ no debconf)
+	echo 'absent'
+	exit 0
+}
+
+linesfile="${__object:?}/parameter/line"
+test -s "${linesfile}" || {
+	if test -s "${__object:?}/parameter/file"
+	then
+		echo absent
+	else
+		echo present
+	fi
+	exit 0
+}
+
+/usr/bin/perl -- - "${linesfile}" <<'EOF'
+use strict;
+use warnings "all";
+
+use Debconf::Db;
+use Debconf::Question;
+
+# Extract @known... arrays from debconf-set-selections
+# These values are required to distinguish flags and values in the given lines.
+# DC: I couldn't think of a more ugly solution to the problem…
+my @knownflags;
+my @knowntypes;
+my $debconf_set_selections = '/usr/bin/debconf-set-selections';
+if (-e $debconf_set_selections) {
+	my $sed_known = 's/^my \(@known\(flags\|types\) = qw([a-z ]*);\).*$/\1/p';
+	eval `sed -n '$sed_known' '$debconf_set_selections'`;
+}
+
+sub mungeline ($) {
+	my $line = shift;
+	chomp $line;
+	$line =~ s/\r$//;
+	return $line;
+}
+
+sub fatal { printf STDERR @_; exit 1; }
+
+my $state = 'present';
+
+sub state {
+	my $new = shift;
+	if ($state eq 'present'
+	    or ($state eq 'different' and $new eq 'absent')) {
+		$state = $new;
+	}
+}
+
+Debconf::Db->load(readonly => 'true');
+
+while (<>) {
+	# Read and process lines (taken from debconf-set-selections)
+	$_ = mungeline($_);
+	while (/\\$/ && ! eof) {
+		s/\\$//;
+		$_ .= mungeline(<>);
+	}
+	next if /^\s*$/ || /^\s*\#/;
+
+	my ($owner, $label, $type, $content) = /^\s*(\S+)\s+(\S+)\s+(\S+)(?:\s(.*))?/
+		or fatal "invalid line: %s\n", $_;
+	$content = '' unless defined $content;
+
+
+	# Compare is and should state
+	my $q = Debconf::Question->get($label);
+
+	unless (defined $q) {
+		# probably a preseed
+		state 'absent';
+		next;
+	}
+
+	if (grep { $_ eq $q->type } @knownflags) {
+		# This line wants to set a flag, presumably.
+		if ($q->flag($q->type) ne $content) {
+			state 'different';
+		}
+	} else {
+		# Otherwise, it's probably a value…
+		if ($q->value ne $content) {
+			state 'different';
+		}
+
+		unless (grep { $_ eq $owner } (split /, /, $q->owners)) {
+			state 'different';
+		}
+	}
+}
+
+printf "%s\n", $state;
+EOF
diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote
index e99aef40..50d898c6 100755
--- a/cdist/conf/type/__debconf_set_selections/gencode-remote
+++ b/cdist/conf/type/__debconf_set_selections/gencode-remote
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org)
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,16 +18,32 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# Setup selections
-#
 
-filename="$(cat "$__object/parameter/file")"
-
-if [ "$filename" = "-" ]; then
-    filename="$__object/stdin"
+if test -f "${__object:?}/parameter/line"
+then
+	filename="${__object:?}/parameter/line"
+elif test -s "${__object:?}/parameter/file"
+then
+	filename=$(cat "${__object:?}/parameter/file")
+	if test "${filename}" = '-'
+	then
+		filename="${__object:?}/stdin"
+	fi
+else
+	printf 'Neither --line nor --file set.\n' >&2
+	exit 1
 fi
 
-echo "debconf-set-selections << __file-eof"
-cat "$filename"
-echo "__file-eof"
+# setting no lines makes no sense
+test -s "${filename}" || exit 0
+
+state_is=$(cat "${__object:?}/explorer/state")
+
+if test "${state_is}" != 'present'
+then
+	cat <<-CODE
+	debconf-set-selections <<'EOF'
+	$(cat "${filename}")
+	EOF
+	CODE
+fi
diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst
index 58c25b81..690e3e49 100644
--- a/cdist/conf/type/__debconf_set_selections/man.rst
+++ b/cdist/conf/type/__debconf_set_selections/man.rst
@@ -8,15 +8,33 @@ cdist-type__debconf_set_selections - Setup debconf selections
 
 DESCRIPTION
 -----------
-On Debian and alike systems debconf-set-selections(1) can be used
+On Debian and alike systems :strong:`debconf-set-selections`\ (1) can be used
 to setup configuration parameters.
 
 
 REQUIRED PARAMETERS
 -------------------
+cf. ``--line``.
+
+
+OPTIONAL PARAMETERS
+-------------------
 file
-   Use the given filename as input for debconf-set-selections(1)
-   If filename is "-", read from stdin.
+   Use the given filename as input for :strong:`debconf-set-selections`\ (1)
+   If filename is ``-``, read from stdin.
+
+   **This parameter is deprecated, because it doesn't work with state detection.**
+line
+   A line in :strong:`debconf-set-selections`\ (1) compatible format.
+   This parameter can be used multiple times to set multiple options.
+
+   (This parameter is actually required, but marked optional because the
+   deprecated ``--file`` is still accepted.)
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
 
 
 EXAMPLES
@@ -24,30 +42,29 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # Setup configuration for nslcd
-    __debconf_set_selections nslcd --file /path/to/file
+    # Setup gitolite's gituser
+    __debconf_set_selections nslcd --line 'gitolite gitolite/gituser string git'
 
-    # Setup configuration for nslcd from another type
-    __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd"
-
-    __debconf_set_selections nslcd --file - << eof
-    gitolite gitolite/gituser string git
-    eof
+    # Setup configuration for nslcd from a file.
+    # NB: Multiple lines can be passed to --line, although this can be considered a hack.
+    __debconf_set_selections nslcd --line "$(cat "${__files:?}/preseed/nslcd.debconf")"
 
 
 SEE ALSO
 --------
-:strong:`debconf-set-selections`\ (1), :strong:`cdist-type__update_alternatives`\ (7)
+- :strong:`cdist-type__update_alternatives`\ (7)
+- :strong:`debconf-set-selections`\ (1)
 
 
 AUTHORS
 -------
 Nico Schottelius <nico-cdist--@--schottelius.org>
+Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2014 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2011-2014 Nico Schottelius, 2021 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.
diff --git a/cdist/conf/type/__debconf_set_selections/manifest b/cdist/conf/type/__debconf_set_selections/manifest
new file mode 100755
index 00000000..0f4fb2e2
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/manifest
@@ -0,0 +1,21 @@
+#!/bin/sh -e
+#
+# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+__package_apt debconf
diff --git a/cdist/conf/type/__debconf_set_selections/parameter/required b/cdist/conf/type/__debconf_set_selections/parameter/deprecated
similarity index 100%
rename from cdist/conf/type/__debconf_set_selections/parameter/required
rename to cdist/conf/type/__debconf_set_selections/parameter/deprecated
diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple
new file mode 100644
index 00000000..a999a0c2
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple
@@ -0,0 +1 @@
+line

From 9cf19388abe95a5f8df9f5c94d2e54fa1060b2ef Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 26 Apr 2021 16:47:44 +0200
Subject: [PATCH 273/366] [type/__debconf_set_selections] Send message about
 each debconf setting that is changed

---
 cdist/conf/type/__debconf_set_selections/gencode-remote | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote
index 50d898c6..9ba28f09 100755
--- a/cdist/conf/type/__debconf_set_selections/gencode-remote
+++ b/cdist/conf/type/__debconf_set_selections/gencode-remote
@@ -46,4 +46,9 @@ then
 	$(cat "${filename}")
 	EOF
 	CODE
+
+	awk '
+		{
+			printf "set %s %s %s %s\n", $1, $2, $3, $4
+		}' "${filename}" >>"${__messages_out:?}"
 fi

From 3a25b804664963e092d1f75c1fdded25d7016ef3 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 26 Apr 2021 21:27:15 +0200
Subject: [PATCH 274/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index c41743cd..497ae91a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
+	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From a42ebc7a78e09e138954b1376376f7cb643420e0 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 27 Apr 2021 19:41:10 +0200
Subject: [PATCH 275/366] [type/__debconf_set_selections] Synchronise objects

Works around locking error:

	debconf: DbDriver "config": /var/cache/debconf/config.dat is locked by another process: Resource temporarily unavailable
---
 .../__debconf_set_selections/explorer/state   | 20 ++++++++++++++++++-
 .../type/__debconf_set_selections/nonparallel |  0
 2 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 cdist/conf/type/__debconf_set_selections/nonparallel

diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state
index 68d28941..f8a3f6c8 100644
--- a/cdist/conf/type/__debconf_set_selections/explorer/state
+++ b/cdist/conf/type/__debconf_set_selections/explorer/state
@@ -41,10 +41,15 @@ test -s "${linesfile}" || {
 	exit 0
 }
 
+# assert __type_explorer is set (because it is used by the Perl script)
+: "${__type_explorer:?}"
+
 /usr/bin/perl -- - "${linesfile}" <<'EOF'
 use strict;
 use warnings "all";
 
+use Fcntl qw(:DEFAULT :flock);
+
 use Debconf::Db;
 use Debconf::Question;
 
@@ -78,7 +83,20 @@ sub state {
 	}
 }
 
-Debconf::Db->load(readonly => 'true');
+
+# Load Debconf DB but manually lock on the state explorer script,
+# because Debconf aborts immediately if executed concurrently.
+# This is not really an ideal solution because the Debconf DB could be locked by
+# another process (e.g. apt-get), but no way to achieve this could be found.
+# If you know how to, please provide a patch.
+my $lockfile = "%ENV{'__type_explorer'}/state";
+if (open my $lock_fh, '+<', $lockfile) {
+   flock $lock_fh, LOCK_EX or die "Cannot lock $lockfile";
+}
+{
+	Debconf::Db->load(readonly => 'true');
+}
+
 
 while (<>) {
 	# Read and process lines (taken from debconf-set-selections)
diff --git a/cdist/conf/type/__debconf_set_selections/nonparallel b/cdist/conf/type/__debconf_set_selections/nonparallel
new file mode 100644
index 00000000..e69de29b

From c00c8c20127af9881b4e295b7732524e7e1c5031 Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Mon, 10 May 2021 12:08:22 +0200
Subject: [PATCH 276/366] [__apt_key*] Deprecate __apt_key_uri and improve
 __apt_key

Previously this type was falling back to using the deprecated apt-key(8) by
checking for existence of files/directories on the controller host in
gencode-remote.

Adding `--use-deprecated-apt-key` as an explicit boolean serves two purposes:
1. It prevents fallbacks that might end up doing the wrong thing
   (as was the case)
2. It allows for a simple way to remove keys from the keyring that were
   previously added with apt-key(8) to /etc/apt/trusted.gpg

This parameter is added marked as deprecated as is only intended use is to
migrate to directory-based keyrings as recommended by Debian for a few releases.
It will be removed when Debian 11 stops being supported.

During the review process of this merge request, it was noted that the state of
PGP Key Servers is somewhat suboptimal, that the examples encouraged bad
practise (it is trivial to produce collisions for short key IDs), and that
this use does not require the Web of Trust, but instead only the public key
that is signing the repository.

That is why this also adds `--source` as an argument allowing for in-type or
in-manifest provision of such public keys by the type/manifest maintainer and
the use of Key Servers is still supported, but discouraged.
---
 cdist/conf/type/__apt_key/explorer/state      |  27 +++--
 cdist/conf/type/__apt_key/gencode-remote      |  53 ++++-----
 cdist/conf/type/__apt_key/man.rst             |  84 ++++++++++----
 cdist/conf/type/__apt_key/manifest            | 104 +++++++++++++++++-
 cdist/conf/type/__apt_key/parameter/boolean   |   1 +
 .../deprecated/use-deprecated-apt-key         |   3 +
 cdist/conf/type/__apt_key/parameter/optional  |   5 +-
 cdist/conf/type/__apt_key_uri/deprecated      |   1 +
 8 files changed, 209 insertions(+), 69 deletions(-)
 create mode 100644 cdist/conf/type/__apt_key/parameter/boolean
 create mode 100644 cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key
 create mode 100644 cdist/conf/type/__apt_key_uri/deprecated

diff --git a/cdist/conf/type/__apt_key/explorer/state b/cdist/conf/type/__apt_key/explorer/state
index 38f1bd3c..8ab268c1 100755
--- a/cdist/conf/type/__apt_key/explorer/state
+++ b/cdist/conf/type/__apt_key/explorer/state
@@ -27,18 +27,25 @@ else
    keyid="$__object_id"
 fi
 
+# From apt-key(8):
+#   Use of apt-key is deprecated, except for the use of apt-key del in
+#   maintainer scripts to remove existing keys from the main keyring.
+#   If such usage of apt-key is desired the additional installation of
+#   the GNU Privacy Guard suite (packaged in gnupg) is required.
+if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then
+   if apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK"
+      then echo present
+      else echo absent
+   fi
+   exit
+fi
+
 keydir="$(cat "$__object/parameter/keydir")"
 keyfile="$keydir/$__object_id.gpg"
 
-if [ -d "$keydir" ]
+if [ -f "$keyfile" ]
 then
-   if [ -f "$keyfile" ]
-   then echo present
-   else echo absent
-   fi
-else
-   # fallback to deprecated apt-key
-   apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \
-      && echo present \
-      || echo absent
+   echo present
+   exit
 fi
+echo absent
diff --git a/cdist/conf/type/__apt_key/gencode-remote b/cdist/conf/type/__apt_key/gencode-remote
index 0c96ff67..17dc9bfc 100755
--- a/cdist/conf/type/__apt_key/gencode-remote
+++ b/cdist/conf/type/__apt_key/gencode-remote
@@ -25,11 +25,7 @@ else
 fi
 state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
-
-if [ "$state_should" = "$state_is" ]; then
-   # nothing to do
-   exit 0
-fi
+method="$(cat "$__object/key_method")"
 
 keydir="$(cat "$__object/parameter/keydir")"
 keyfile="$keydir/$__object_id.gpg"
@@ -37,30 +33,18 @@ keyfile="$keydir/$__object_id.gpg"
 case "$state_should" in
    present)
       keyserver="$(cat "$__object/parameter/keyserver")"
-
-      if [ -f "$__object/parameter/uri" ]; then
-         uri="$(cat "$__object/parameter/uri")"
-
-         if [ -d "$keydir" ]; then
-            cat << EOF
-
-curl -s -L \\
-    -o "$keyfile" \\
-    "$uri"
-
-key="\$( cat "$keyfile" )"
-
-if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK'
-then
-    echo "\$key" | gpg --dearmor > "$keyfile"
-fi
-
-EOF
-         else
-            # fallback to deprecated apt-key
-            echo "curl -s -L '$uri' | apt-key add -"
+      # Using __download or __file as key source
+      # Propagate messages if needed
+      if [ "${method}" = "uri" ] || [ "${method}" = "source" ]; then
+         if grep -Eq "^__(file|download)$keyfile" "$__messages_in"; then
+            echo "added '$keyid'" >> "$__messages_out"
          fi
-      elif [ -d "$keydir" ]; then
+         exit 0
+      elif [ "${state_is}" = "present" ]; then
+         exit 0
+      fi
+      # Using key servers to fetch the key
+      if [ ! -f "$__object/parameter/use-deprecated-apt-key" ]; then
          # we need to kill gpg after 30 seconds, because gpg
          # can get stuck if keyserver is not responding.
          # exporting env var and not exit 1,
@@ -100,13 +84,16 @@ EOF
       echo "added '$keyid'" >> "$__messages_out"
    ;;
    absent)
-      if [ -f "$keyfile" ]; then
-         echo "rm '$keyfile'"
-      else
+      # Removal for keys added from a keyserver without this flag
+      # is done in the manifest
+      if [ "$state_is" != "absent" ] && \
+            [ -f "$__object/parameter/use-deprecated-apt-key" ]; then
          # fallback to deprecated apt-key
          echo "apt-key del \"$keyid\""
+         echo "removed '$keyid'" >> "$__messages_out"
+      # Propagate messages if needed
+      elif grep -Eq "^__file$keyfile" "$__messages_in"; then
+         echo "removed '$keyid'" >> "$__messages_out"
       fi
-
-      echo "removed '$keyid'" >> "$__messages_out"
    ;;
 esac
diff --git a/cdist/conf/type/__apt_key/man.rst b/cdist/conf/type/__apt_key/man.rst
index 234bc715..e35eaa0f 100644
--- a/cdist/conf/type/__apt_key/man.rst
+++ b/cdist/conf/type/__apt_key/man.rst
@@ -10,6 +10,14 @@ DESCRIPTION
 -----------
 Manages the list of keys used by apt to authenticate packages.
 
+This is done by placing the requested key in a file named
+``$__object_id.gpg`` in the ``keydir`` directory.
+
+This is supported by modern releases of Debian-based distributions.
+
+In order of preference, exactly one of: ``source``, ``uri`` or ``keyid``
+must be specified.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -18,21 +26,49 @@ None.
 
 OPTIONAL PARAMETERS
 -------------------
+keydir
+   keyring directory, defaults to ``/etc/apt/trusted.pgp.d``, which is
+   enabled system-wide by default.
+
+source
+   path to a file containing the GPG key of the repository.
+   Using this is recommended as it ensures that the manifest/type manintainer
+   has validated the key.
+   If ``-``, the GPG key is read from the type's stdin.
+
 state
    'present' or 'absent'. Defaults to 'present'
 
+uri
+   the URI from which to download the key.
+   It is highly recommended that you only use protocols with TLS like HTTPS.
+   This uses ``__download`` but does not use checksums, if you want to ensure
+   that the key doesn't change, you are better off downloading it and using
+   ``--source``.
+
+
+DEPRECATED OPTIONAL PARAMETERS
+------------------------------
 keyid
-   the id of the key to add. Defaults to __object_id
+   the id of the key to download from the ``keyserver``.
+   This is to be used in absence of ``--source`` and ``--uri`` or together
+   with ``--use-deprecated-apt-key`` for key removal.
+   Defaults to ``$__object_id``.
 
 keyserver
-   the keyserver from which to fetch the key. If omitted the default set
-   in ./parameter/default/keyserver is used.
+   the keyserver from which to fetch the key.
+   Defaults to ``pool.sks-keyservers.net``.
 
-keydir
-   key save location, defaults to ``/etc/apt/trusted.pgp.d``
 
-uri
-   the URI from which to download the key
+DEPRECATED BOOLEAN PARAMETERS
+-----------------------------
+use-deprecated-apt-key
+   ``apt-key(8)`` will last be available in Debian 11 and Ubuntu 22.04.
+   You can use this parameter to force usage of ``apt-key(8)``.
+   Please only use this parameter to *remove* keys from the keyring,
+   in order to prepare for removal of ``apt-key``.
+   Adding keys should be done without this parameter.
+   This parameter will be removed when Debian 11 stops being supported.
 
 
 EXAMPLES
@@ -40,33 +76,39 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # Add Ubuntu Archive Automatic Signing Key
-    __apt_key 437D05B5
-    # Same thing
-    __apt_key 437D05B5 --state present
-    # Get rid of it
-    __apt_key 437D05B5 --state absent
+    # add a key that has been verified by a type maintainer
+    __apt_key jitsi_meet_2021 \
+       --source cdist-contrib/type/__jitsi_meet/files/apt_2021.gpg
 
-    # same thing with human readable name and explicit keyid
-    __apt_key UbuntuArchiveKey --keyid 437D05B5
+    # remove an old, deprecated or expired key
+    __apt_key jitsi_meet_2016 --state absent
 
-    # same thing with other keyserver
-    __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com
+    # Get rid of a key that might have been added to
+    # /etc/apt/trusted.gpg with apt-key
+    __apt_key 0x40976EAF437D05B5 --use-deprecated-apt-key --state absent
 
-    # download key from the internet
-    __apt_key rabbitmq \
-       --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc
+    # add a key that we define in-line
+    __apt_key jitsi_meet_2021 --source '-' <<EOF
+    -----BEGIN PGP PUBLIC KEY BLOCK-----
+    [...]
+    -----END PGP PUBLIC KEY BLOCK-----
+    EOF
+
+    # download or update key from the internet
+    __apt_key rabbitmq_2007 \
+       --uri https://www.rabbitmq.com/rabbitmq-signing-key-public.asc
 
 
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
 Ander Punnar <ander-at-kvlt-dot-ee>
+Evilham <contact~~@~~evilham.com>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can
+Copyright \(C) 2011-2021 Steven Armstrong, Ander Punnar and Evilham. You can
 redistribute it and/or modify it under the terms of the GNU General Public
 License as published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.
diff --git a/cdist/conf/type/__apt_key/manifest b/cdist/conf/type/__apt_key/manifest
index 010357cd..889a764a 100755
--- a/cdist/conf/type/__apt_key/manifest
+++ b/cdist/conf/type/__apt_key/manifest
@@ -2,7 +2,105 @@
 
 __package gnupg
 
-if [ -f "$__object/parameter/uri" ]
-then __package curl
-else __package dirmngr
+state_should="$(cat "${__object}/parameter/state")"
+
+incompatible_args()
+{
+	cat >> /dev/stderr <<-EOF
+	This type does not support --${1} and --${method} simultaneously.
+	EOF
+	exit 1
+}
+
+if [ -f "${__object}/parameter/source" ]; then
+	method="source"
+	src="$(cat "${__object}/parameter/source")"
+	if [ "${src}" = "-" ]; then
+		src="${__object}/stdin"
+	fi
+fi
+if [ -f "${__object}/parameter/uri" ]; then
+	if [ -n "${method}" ]; then
+		incompatible_args uri
+	fi
+	method="uri"
+	src="$(cat "${__object}/parameter/uri")"
+fi
+if [ -f "${__object}/parameter/keyid" ]; then
+	if [ -n "${method}" ]; then
+		incompatible_args keyid
+	fi
+	method="keyid"
+fi
+# Keep old default
+if [ -z "${method}" ]; then
+	method="keyid"
+fi
+# Save this for later in gencode-remote
+echo "${method}" > "${__object}/key_method"
+
+# Required remotely (most likely already installed)
+__package dirmngr
+# We need this in case a key has to be dearmor'd
+__package gnupg
+export require="__package/gnupg"
+
+if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then
+	# This is required if apt-key(8) is to be used
+	if [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then
+		incompatible_args use-deprecated-apt-key
+	fi
+else
+	if [ "${state_should}" = "absent" ] && \
+			[ -f "${__object}/parameter/keyid" ]; then
+		cat >> /dev/stderr <<EOF
+You can't reliably remove by keyid without --use-deprecated-apt-key.
+This would very likely do something you do not intend.
+EOF
+		exit 1
+	fi
+fi
+
+keydir="$(cat "${__object}/parameter/keydir")"
+keyfile="${keydir}/${__object_id}.gpg"
+keyfilecdist="${keyfile}.cdist"
+if [ "${state_should}" != "absent" ]; then
+	# Ensure keydir exists
+	__directory "${keydir}" --state exists --mode 0755
+fi
+
+if [ "${state_should}" = "absent" ]; then
+	__file "${keyfile}" --state "absent"
+	__file "${keyfilecdist}" --state "absent"
+elif [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then
+	dearmor="$(cat <<-EOF
+	if [ '${state_should}' = 'present' ]; then
+		# Dearmor if necessary
+		if grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK' '${keyfilecdist}'; then
+			gpg --dearmor < '${keyfilecdist}' > '${keyfile}'
+		else
+			cp '${keyfilecdist}' '${keyfile}'
+		fi
+		# Ensure permissions
+		chown root '${keyfile}'
+		chmod 0444 '${keyfile}'
+	fi
+	EOF
+	)"
+
+	if [ "${method}" = "uri" ]; then
+		__download "${keyfilecdist}" \
+			--url "${src}" \
+			--onchange "${dearmor}"
+		require="__download${keyfilecdist}" \
+			__file "${keyfile}" \
+				--owner root \
+				--mode 0444 \
+				--state pre-exists
+	else
+		__file "${keyfilecdist}" --state "${state_should}" \
+			--mode 0444 \
+			--source "${src}" \
+			--onchange "${dearmor}"
+	fi
 fi
diff --git a/cdist/conf/type/__apt_key/parameter/boolean b/cdist/conf/type/__apt_key/parameter/boolean
new file mode 100644
index 00000000..7263e1b2
--- /dev/null
+++ b/cdist/conf/type/__apt_key/parameter/boolean
@@ -0,0 +1 @@
+use-deprecated-apt-key
diff --git a/cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key b/cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key
new file mode 100644
index 00000000..d18ebe57
--- /dev/null
+++ b/cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key
@@ -0,0 +1,3 @@
+apt-key(8) will last be available in Debian 11 and Ubuntu 22.04.
+Use this flag *only* to migrate to placing a keyring directly in the
+/etc/apt/trusted.gpg.d/ directory with a descriptive name.
diff --git a/cdist/conf/type/__apt_key/parameter/optional b/cdist/conf/type/__apt_key/parameter/optional
index de647375..975d1fd8 100644
--- a/cdist/conf/type/__apt_key/parameter/optional
+++ b/cdist/conf/type/__apt_key/parameter/optional
@@ -1,5 +1,6 @@
-state
+keydir
 keyid
 keyserver
-keydir
+source
+state
 uri
diff --git a/cdist/conf/type/__apt_key_uri/deprecated b/cdist/conf/type/__apt_key_uri/deprecated
new file mode 100644
index 00000000..05550587
--- /dev/null
+++ b/cdist/conf/type/__apt_key_uri/deprecated
@@ -0,0 +1 @@
+Please migrate to using __apt_key key_id --uri URI.

From a696f3cf0026ec96e4cf7235ee8634391ccfecdb Mon Sep 17 00:00:00 2001
From: Evil Ham <ungleich@evilham.com>
Date: Mon, 10 May 2021 12:10:00 +0200
Subject: [PATCH 277/366] [__letsencrypt_cert] Revamp explorers, add locking.

This would fix #839

Certbot uses locking [1] even for read-only operations and does not properly
use exit codes, which means that sometimes it would print:
"Another instance of Certbot is already running" and exit with success.

However, the previous explorers would take that as the certificate being absent
and would trigger code generation.

The issue was made worse by having many explorers running certbot, so for N
certificates, we'd run certbot N*4 times, potentially "in parallel".

[1]: https://certbot.eff.org/docs/using.html#id5

This patch joins all explorers in one to avoid starting multiple remote python
processes and uses a cdist-specific lock in /tmp/certbot.cdist.lock with a
60 seconds timeout.

It has been tested with certbot 0.31.0 and 0.17 that the:

    from certbot.main import main

trick works. It is somewhat well documented so it can be somewhat relied upon.
---
 .../__letsencrypt_cert/explorer/certbot-path  |  3 -
 .../explorer/certificate-data                 | 78 +++++++++++++++++++
 .../explorer/certificate-domains              |  8 --
 .../explorer/certificate-exists               | 13 ----
 .../explorer/certificate-is-test              | 14 ----
 .../type/__letsencrypt_cert/gencode-remote    | 11 ++-
 cdist/conf/type/__letsencrypt_cert/manifest   |  2 +-
 7 files changed, 87 insertions(+), 42 deletions(-)
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certbot-path
 create mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-data
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists
 delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test

diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path
deleted file mode 100755
index 3c6076df..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh -e
-
-command -v certbot 2>/dev/null || true
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data
new file mode 100755
index 00000000..ff62e742
--- /dev/null
+++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data
@@ -0,0 +1,78 @@
+#!/bin/sh -e
+certbot_path="$(command -v certbot 2>/dev/null || true)"
+# Defaults
+certificate_exists="no"
+certificate_is_test="no"
+
+if [ -n "${certbot_path}" ]; then
+	# Find python executable that has access to certbot's module
+	python_path=$(sed -n '1s/^#! *//p' "${certbot_path}")
+
+	# Use a lock for cdist due to certbot not exiting with failure
+	# or having any flags for concurrent use.
+	_certbot() {
+		${python_path} - 2>/dev/null <<EOF
+from certbot.main import main
+import fcntl
+lock_file = "/tmp/certbot.cdist.lock"
+timeout=60
+with open(lock_file, 'w') as fd:
+    for i in range(timeout):
+        try:
+            # Get exclusive lock
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            break
+        except:
+            # Wait if that fails
+            import time
+            time.sleep(1)
+    else:
+        # Timed out, exit with failure
+        import sys
+        sys.exit(1)
+    # Do list certificates
+    main(["certificates", "--cert-name", "${__object_id:?}"])
+EOF
+	}
+
+
+	_certificate_exists() {
+		if grep -q "  Certificate Name: ${__object_id:?}$"; then
+			echo yes
+		else
+			echo no
+		fi
+	}
+
+	_certificate_is_test() {
+		if grep -q 'INVALID: TEST_CERT'; then
+			echo yes
+		else
+			echo no
+		fi
+	}
+
+	_certificate_domains() {
+		grep '    Domains: ' | cut -d ' ' -f 6- | tr ' ' '\n'
+	}
+
+	# Get data about all available certificates
+	certificates="$(_certbot)"
+
+	# Check whether or not the certificate exists
+	certificate_exists="$(echo "${certificates}" | _certificate_exists)"
+
+	# Check whether or not the certificate is for testing
+	certificate_is_test="$(echo "${certificates}" | _certificate_is_test)"
+
+	# Get domains for certificate
+	certificate_domains="$(echo "${certificates}" | _certificate_domains)"
+fi
+
+# Return received data
+cat <<EOF
+certbot_path:${certbot_path}
+certificate_exists:${certificate_exists}
+certificate_is_test:${certificate_is_test}
+${certificate_domains}
+EOF
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains
deleted file mode 100755
index db605b63..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	certbot certificates --cert-name "${__object_id:?}" | grep '    Domains: ' | \
-		cut -d ' ' -f 6- | tr ' ' '\n'
-fi
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists
deleted file mode 100755
index 4e6f44db..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	if certbot certificates | grep -q "  Certificate Name: ${__object_id:?}$"; then
-		echo yes
-	else
-		echo no
-	fi
-else
-	echo no
-fi
diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test
deleted file mode 100755
index 9b445059..00000000
--- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh -e
-
-certbot_path=$("${__type_explorer}/certbot-path")
-if [ -n "${certbot_path}" ]
-then
-	if certbot certificates --cert-name "${__object_id:?}" | \
-		grep -q 'INVALID: TEST_CERT'; then
-		echo yes
-	else
-		echo no
-	fi
-else
-	echo no
-fi
diff --git a/cdist/conf/type/__letsencrypt_cert/gencode-remote b/cdist/conf/type/__letsencrypt_cert/gencode-remote
index 375570a4..60a77ec3 100755
--- a/cdist/conf/type/__letsencrypt_cert/gencode-remote
+++ b/cdist/conf/type/__letsencrypt_cert/gencode-remote
@@ -1,6 +1,10 @@
 #!/bin/sh -e
 
-certificate_exists=$(cat "${__object:?}/explorer/certificate-exists")
+_explorer_var() {
+	grep "^$1:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-
+}
+
+certificate_exists="$(_explorer_var certificate_exists)"
 name="${__object_id:?}"
 state=$(cat "${__object}/parameter/state")
 
@@ -29,8 +33,9 @@ case "${state}" in
 		fi
 
 		if [ "${certificate_exists}" = "yes" ]; then
-			existing_domains="${__object}/explorer/certificate-domains"
-			certificate_is_test=$(cat "${__object}/explorer/certificate-is-test")
+			existing_domains=$(mktemp "${TMPDIR:-/tmp}/existing_domains.cdist.XXXXXXXXXX")
+			tail -n +4 "${__object:?}/explorer/certificate-data" | grep -v '^$' > "${existing_domains}"
+			certificate_is_test="$(_explorer_var certificate_is_test)"
 
 			sort -uo "${requested_domains}" "${requested_domains}"
 			sort -uo "${existing_domains}" "${existing_domains}"
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 1df3574a..6394f629 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")"
+certbot_fullpath="$(grep "^certbot_path:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-)"
 state=$(cat "${__object}/parameter/state")
 os="$(cat "${__global:?}/explorer/os")"
 

From f14623e45f56aec4bdce53bc2b367e11ff157865 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 10 May 2021 12:17:08 +0200
Subject: [PATCH 278/366] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 497ae91a..5c3b547a 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,8 @@ Changelog
 next:
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
+	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)
+	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 6210cccb28ace8ec2807ffe6ca9f9e4d9007990b Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Mon, 10 May 2021 12:34:04 +0200
Subject: [PATCH 279/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 5c3b547a..a7310788 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)
 	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
+	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking.
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 503a06ed28f743aad47797a0989be735c67b07a6 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 28 Apr 2021 13:32:10 +0300
Subject: [PATCH 280/366] [__git] fix group explorer

group name from numberic id wasn't resolved correctly.

try to use getent and fallback to reading /etc/group directly.
---
 cdist/conf/type/__git/explorer/group | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group
index 1365c60d..ab4396b1 100644
--- a/cdist/conf/type/__git/explorer/group
+++ b/cdist/conf/type/__git/explorer/group
@@ -14,6 +14,11 @@ then
 	then
 		printf '%u\n' "${group_gid}"
 	else
-		printf '%s\n' "$(id -u -n "${group_gid}")"
+		if command -v getent > /dev/null
+		then
+			getent group "${group_gid}" | cut -d : -f 1
+		else
+			awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group
+		fi
 	fi
 fi

From 75c71f69c1fed4371c5891ddcca0eaf28ad928e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 10:17:48 +0200
Subject: [PATCH 281/366] [scanner] pycodestyle compliance

---
 cdist/scan/commandline.py | 20 +++++++++++----
 cdist/scan/scan.py        | 54 ++++++++++++++++++++++++---------------
 2 files changed, 48 insertions(+), 26 deletions(-)

diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index 3eb7eec4..b42bc7b2 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -25,13 +25,15 @@ from datetime import datetime
 
 log = logging.getLogger("scan")
 
+
 def run(scan, args):
     # We run each component in a separate process since they
     # must not block on each other.
     processes = []
 
     if 'trigger' in args.mode:
-        t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay)
+        t = scan.Trigger(interfaces=args.interfaces,
+                         sleeptime=args.trigger_delay)
         t.start()
         processes.append(t)
         log.debug("Trigger started")
@@ -48,6 +50,7 @@ def run(scan, args):
     for process in processes:
         process.join()
 
+
 def list(scan, args):
     s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper)
     hosts = s.list()
@@ -66,10 +69,16 @@ def list(scan, args):
     print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
     for host in hosts:
         last_seen = host.last_seen()
-        last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-'
+        if last_seen:
+            last_seen = last_seen.strftime(scan.datetime_format)
+        else:
+            last_seen = '-'
 
         last_configured = host.last_configured()
-        last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-'
+        if last_configured:
+            last_configured = last_configured.strftime(scan.datetime_format)
+        else:
+            '-'
 
         print("{} | {} | {} | {}".format(
             host.name(default='-').ljust(name_max_size),
@@ -77,6 +86,7 @@ def list(scan, args):
             last_seen.ljust(date_max_size),
             last_configured.ljust(date_max_size)))
 
+
 # CLI processing is defined outside of the main scan class to handle
 # non-available optional scapy dependency (instead of crashing mid-flight).
 def commandline(args):
@@ -94,9 +104,9 @@ def commandline(args):
         # By default scan and trigger, but do not call any action.
         args.mode = ['scan', 'trigger', ]
 
-    if 'config' in args.mode and args.name_mapper == None:
+    if 'config' in args.mode and args.name_mapper is None:
         print('--name-mapper must be specified for scanner config mode.',
-                file=sys.stderr)
+              file=sys.stderr)
         sys.exit(1)
 
     # Print known hosts and exit is --list is specified - do not start
diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py
index 2912dab3..4a20f511 100644
--- a/cdist/scan/scan.py
+++ b/cdist/scan/scan.py
@@ -33,6 +33,7 @@ logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("scan")
 datetime_format = '%Y-%m-%d %H:%M:%S'
 
+
 class Host(object):
     def __init__(self, addr, outdir, name_mapper=None):
         self.addr = addr
@@ -43,7 +44,7 @@ class Host(object):
 
     def __get(self, key, default=None):
         fname = os.path.join(self.workdir, key)
-        value=default
+        value = default
         if os.path.isfile(fname):
             with open(fname, "r") as fd:
                 value = fd.readline()
@@ -55,15 +56,15 @@ class Host(object):
             fd.write(f"{value}")
 
     def name(self, default=None):
-        if self.name_mapper == None:
+        if self.name_mapper is None:
             return default
 
         fpath = os.path.join(os.getcwd(), self.name_mapper)
         if os.path.isfile(fpath) and os.access(fpath, os.X_OK):
-             out = subprocess.run([fpath, self.addr], capture_output=True)
-             if out.returncode != 0:
-                 return default
-             else:
+            out = subprocess.run([fpath, self.addr], capture_output=True)
+            if out.returncode != 0:
+                return default
+            else:
                 value = out.stdout.decode()
                 return (default if len(value) == 0 else value)
         else:
@@ -94,19 +95,20 @@ class Host(object):
     # CLI args. Might as well call everything from scratch!
     def configure(self):
         target = self.name() or self.address()
-        cmd = ['cdist', 'config', '-v', target ]
+        cmd = ['cdist', 'config', '-v', target]
 
         fname = os.path.join(self.workdir, 'last_configuration_log')
         with open(fname, "w") as fd:
             log.debug("Executing: %s", cmd)
             completed_process = subprocess.run(cmd, stdout=fd, stderr=fd)
             if completed_process.returncode != 0:
-                log.error("%s return with non-zero code %i - see %s for details.",
-                        cmd, completed_process.returncode, fname)
+                log.error("%s return with non-zero code %i - see %s for \
+                        details.", cmd, completed_process.returncode, fname)
 
         now = datetime.datetime.now().strftime(datetime_format)
         self.__set('last_configured', now)
 
+
 class Trigger(object):
     """
     Trigger an ICMPv6EchoReply from all hosts that are alive
@@ -140,10 +142,12 @@ class Trigger(object):
     def trigger(self, interface):
         try:
             log.debug("Sending ICMPv6EchoRequest on %s", interface)
-            packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest()
+            packet = IPv6(
+                    dst="ff02::1%{}".format(interface)
+                    ) / ICMPv6EchoRequest()
             send(packet, verbose=self.verbose)
         except Exception as e:
-            log.error( "Could not send ICMPv6EchoRequest: %s", e)
+            log.error("Could not send ICMPv6EchoRequest: %s", e)
 
 
 class Scanner(object):
@@ -151,9 +155,10 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None):
+    def __init__(self, interfaces, autoconfigure=False, outdir=None,
+                 name_mapper=None):
         self.interfaces = interfaces
-        self.autoconfigure=autoconfigure
+        self.autoconfigure = autoconfigure
         self.name_mapper = name_mapper
         self.config_delay = datetime.timedelta(seconds=3600)
 
@@ -169,14 +174,17 @@ class Scanner(object):
         if ICMPv6EchoReply in pkg:
             host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper)
             if host.name():
-                log.verbose("Host %s (%s) is alive", host.name(), host.address())
+                log.verbose("Host %s (%s) is alive", host.name(),
+                            host.address())
             else:
                 log.verbose("Host %s is alive", host.address())
+
             host.seen()
 
             # Configure if needed.
             if self.autoconfigure and \
-                    host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now():
+                    host.last_configured(default=datetime.datetime.min) + \
+                    self.config_delay < datetime.datetime.now():
                 self.config(host)
 
     def list(self):
@@ -187,15 +195,19 @@ class Scanner(object):
         return hosts
 
     def config(self, host):
-        if host.name() == None:
-            log.debug("config - could not resolve name for %s, aborting.", host.address())
+        if host.name() is None:
+            log.debug("config - could not resolve name for %s, aborting.",
+                      host.address())
             return
 
         previous_config_process = self.running_configs.get(host.name())
-        if previous_config_process != None and previous_config_process.is_alive():
-            log.debug("config - is already running for %s, aborting.", host.name())
+        if previous_config_process is not None and \
+                previous_config_process.is_alive():
+            log.debug("config - is already running for %s, aborting.",
+                      host.name())
 
-        log.info("config - running against host %s (%s).", host.name(), host.address())
+        log.info("config - running against host %s (%s).", host.name(),
+                 host.address())
         p = Process(target=host.configure())
         p.start()
         self.running_configs[host.name()] = p
@@ -214,4 +226,4 @@ class Scanner(object):
                   filter="icmp6",
                   prn=self.handle_pkg)
         except Exception as e:
-            log.error( "Could not start listener: %s", e)
+            log.error("Could not start listener: %s", e)

From ab10b453f275d4289f3d37988b3caec204227194 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 11:15:41 +0200
Subject: [PATCH 282/366] [scanner] populate cdist(1)

---
 docs/src/man1/cdist.rst | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 0ecb4a61..599ec3b7 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -88,6 +88,9 @@ SYNOPSIS
     cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t]
                [pattern]
 
+    cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list]
+               [-d CONFIG_DELAY] [-t TRIGGER_DELAY]
+
 
 DESCRIPTION
 -----------
@@ -641,6 +644,31 @@ Display information for cdist (global explorers, types).
 **-t, --types**
     Display info for types.
 
+SCAN
+----
+
+Runs cdist as a daemon that discover/watch on hosts and reconfigure them
+periodically.
+
+**-I INTERFACE, --interfaces INTERFACE**
+    Interface to listen on. Can be specified multiple times.
+
+**-m MODE, --mode MODE**
+    Scanner components to enable. Can be specified multiple time to enable more
+    than one component. Supported modes are: scan, trigger and config. Defaults
+    to tiggger and scan.
+
+**--name-mapper PATH_TO_SCRIPT**
+    Path to script used to resolve a remote host name from an IPv6 address.
+
+**--list**
+    List known hosts and exit.
+
+**-d CONFIG_DELAY, --config-delay CONFIG_DELAY**
+    How long (seconds) to wait before reconfiguring after last try (config mode only).
+
+**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY**
+    How long (seconds) to wait between ICMPv6 echo requests (trigger mode only).
 
 CONFIGURATION
 -------------

From b8733c65f52776facce7a8de0265538a856ead64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 11:26:35 +0200
Subject: [PATCH 283/366] [scanner] fix minor CLI handling and --list bugs /
 typo

---
 cdist/argparse.py         |  4 ++--
 cdist/scan/commandline.py | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/cdist/argparse.py b/cdist/argparse.py
index bedb23ac..f17315e7 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -495,7 +495,7 @@ def get_parsers():
         action='store_true',
         help='Try to configure detected hosts')
     parser['scan'].add_argument(
-        '-I', '--interfaces',
+        '-I', '--interface',
         action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
     parser['scan'].add_argument(
@@ -503,7 +503,7 @@ def get_parsers():
         action='store',  default=None,
         help='Map addresses to names, required for config mode')
     parser['scan'].add_argument(
-        '-d', '--delay',
+        '-d', '--config-delay',
         action='store',  default=3600, type=int,
         help='How long (seconds) to wait before reconfiguring after last try')
     parser['scan'].add_argument(
diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py
index b42bc7b2..ddbe4933 100644
--- a/cdist/scan/commandline.py
+++ b/cdist/scan/commandline.py
@@ -32,7 +32,7 @@ def run(scan, args):
     processes = []
 
     if 'trigger' in args.mode:
-        t = scan.Trigger(interfaces=args.interfaces,
+        t = scan.Trigger(interfaces=args.interface,
                          sleeptime=args.trigger_delay)
         t.start()
         processes.append(t)
@@ -41,7 +41,7 @@ def run(scan, args):
     if 'scan' in args.mode:
         s = scan.Scanner(
                 autoconfigure='config' in args.mode,
-                interfaces=args.interfaces,
+                interfaces=args.interface,
                 name_mapper=args.name_mapper)
         s.start()
         processes.append(s)
@@ -52,7 +52,7 @@ def run(scan, args):
 
 
 def list(scan, args):
-    s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper)
+    s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper)
     hosts = s.list()
 
     # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
@@ -75,10 +75,10 @@ def list(scan, args):
             last_seen = '-'
 
         last_configured = host.last_configured()
-        if last_configured:
+        if last_configured is not None:
             last_configured = last_configured.strftime(scan.datetime_format)
         else:
-            '-'
+            last_configured = '-'
 
         print("{} | {} | {} | {}".format(
             host.name(default='-').ljust(name_max_size),

From e0c52d0e1dfbaa3814a2ff26482177c2909a15ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Wed, 26 May 2021 11:27:11 +0200
Subject: [PATCH 284/366] [scanner] remove mention of non-implemented trigger
 soruce script

---
 docs/src/cdist-scan.rst | 21 ++-------------------
 1 file changed, 2 insertions(+), 19 deletions(-)

diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst
index 02193456..064e65ff 100644
--- a/docs/src/cdist-scan.rst
+++ b/docs/src/cdist-scan.rst
@@ -8,7 +8,8 @@ periodically. It is especially useful in netboot-based environment where hosts
 boot unconfigured, and to ensure your infrastructure stays in sync with your
 configuration.
 
-This feature is still consider to be in **beta** stage.
+This feature is still consider to be in **beta** stage, and only operate on
+IPv6 (including link-local).
 
 Usage (Examples)
 ----------------
@@ -79,21 +80,3 @@ Resolving name from `PTR` DNS record:
   done
 
   dig +short -x "$1" | sed -e 's/.$//'
-
-
-Trigger Source Script
----------------------
-
-This script returns a list of addresses (separated by a newline) to be used by
-`trigger` mode. It is not used to map names. The script must be executable.
-
-Simplest script:
-
-.. code-block:: sh
-  #!/bin/sh
-
-  cat << EOF
-  server1.domain.tld
-  server2.domain.tld
-  server3.domain.tld
-  EOF

From defa3c22eaef88652a2bec6135d038a84710c41e Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 29 May 2021 11:21:34 +0200
Subject: [PATCH 285/366] ++changelog

---
 docs/changelog | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index a7310788..403e7653 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,7 +6,8 @@ next:
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)
 	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
-	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking.
+	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham)
+	* Type __git: Fix group explorer (Ander Punnar)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From d596986af894749877fb3dcadf90628c9cfa9d13 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 31 May 2021 09:06:52 +0200
Subject: [PATCH 286/366] [type/__pyvenv] Fix group explorer

---
 cdist/conf/type/__pyvenv/explorer/group | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group
index f31a1cb7..922ce3df 100755
--- a/cdist/conf/type/__pyvenv/explorer/group
+++ b/cdist/conf/type/__pyvenv/explorer/group
@@ -14,6 +14,11 @@ then
 	then
 		printf '%u\n' "${group_gid}"
 	else
-		printf '%s\n' "$(id -u -n "${group_gid}")"
+		if command -v getent >/dev/null 2>&1
+		then
+			getent group "${group_gid}" | cut -d : -f 1
+		else
+			awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group
+		fi
 	fi
 fi

From 6ede76b08b96b7504c65ca2dc6737e31be7e7f24 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 8 Jun 2021 16:20:55 +0200
Subject: [PATCH 287/366] [type/__debconf_set_selections] man.rst: Fix line
 break in AUTHORS

---
 cdist/conf/type/__debconf_set_selections/man.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst
index 690e3e49..fd0040ae 100644
--- a/cdist/conf/type/__debconf_set_selections/man.rst
+++ b/cdist/conf/type/__debconf_set_selections/man.rst
@@ -58,8 +58,8 @@ SEE ALSO
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
-Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
+| Nico Schottelius <nico-cdist--@--schottelius.org>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING

From c308a2896971ac4808be0152630ef41b7f96a2de Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 10 Jun 2021 06:39:55 +0200
Subject: [PATCH 288/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 403e7653..97ea204b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham)
 	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham)
 	* Type __git: Fix group explorer (Ander Punnar)
+	* Type __pyvenv: Fix group explorer (Dennis Camera)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From a3102022e18a23ce9b54eeaf7415b55361f80bd1 Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Fri, 11 Jun 2021 15:05:17 +0100
Subject: [PATCH 289/366] More sensible defaults; reword debian-only error
 message

---
 cdist/conf/type/__apt_pin/manifest                  | 13 +++++++++----
 cdist/conf/type/__apt_pin/nonparallel               |  0
 cdist/conf/type/__apt_pin/parameter/default/package |  1 -
 3 files changed, 9 insertions(+), 5 deletions(-)
 create mode 100644 cdist/conf/type/__apt_pin/nonparallel
 delete mode 100644 cdist/conf/type/__apt_pin/parameter/default/package

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index b1372ad0..909bc80d 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -19,9 +19,17 @@
 #
 
 
+name="$__object_id"
+
 os=$(cat "$__global/explorer/os")
 state="$(cat "$__object/parameter/state")"
-package="$(cat "$__object/parameter/package")"
+
+if [ -f "$__object/parameter/package" ]; then
+    package="$(cat "$__object/parameter/package")"
+else
+    package=$name
+fi
+
 distribution="$(cat "$__object/parameter/distribution")"
 priority="$(cat "$__object/parameter/priority")"
 
@@ -31,13 +39,10 @@ case "$os" in
    ;;
    *)
       printf "This type is specific to Debian and it's derivatives" >&2
-      printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2
       exit 1
    ;;
 esac
 
-name="$__object_id"
-
 case $distribution in
     stabletesting|unsatbel|experimental)
         pin="release a=$distribution"
diff --git a/cdist/conf/type/__apt_pin/nonparallel b/cdist/conf/type/__apt_pin/nonparallel
new file mode 100644
index 00000000..e69de29b
diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package
deleted file mode 100644
index 72e8ffc0..00000000
--- a/cdist/conf/type/__apt_pin/parameter/default/package
+++ /dev/null
@@ -1 +0,0 @@
-*

From b726697e070e5266eae38c7951ced14a2305acb2 Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Fri, 11 Jun 2021 15:05:33 +0100
Subject: [PATCH 290/366] Add documentation

---
 cdist/conf/type/__apt_pin/man.rst | 37 ++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst
index 7fcae6f8..0c91cdec 100644
--- a/cdist/conf/type/__apt_pin/man.rst
+++ b/cdist/conf/type/__apt_pin/man.rst
@@ -13,11 +13,21 @@ This space intentionally left blank.
 
 REQUIRED PARAMETERS
 -------------------
-None.
+distribution
+   Specifies what distribution the package should be pinned to. Accepts both codenames (buster/bullseye/sid) and suite names (stable/testing/...).
 
 
 OPTIONAL PARAMETERS
 -------------------
+package
+   Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used.
+
+priority
+   The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority)
+
+state
+   Will be passed to underlying `__file` type; see there for valid values and defaults.
+
 None.
 
 
@@ -31,14 +41,31 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # TODO
-    __apt_pin
+   # Add the bullseye repo to buster, but do not install any pacakges by default
+   # only if explicitely asked for
+    __apt_pin bullseye-default \
+       --package "*" \
+       --distribution bullseye \
+       --priority -1
+
+    require="__apt_pin/bullseye-default" __apt_source bullseye \
+       --uri http://deb.debian.org/debian/ \
+       --distribution bullseye \
+       --component main
+       # TODO
+       __apt_pin
+
+    __apt_pin foo --package "foo foo-*" --distribution bullseye
+
+    __foo # Installs the `foo` package internally
+
+    __package foo-plugin-extras
 
 
 SEE ALSO
 --------
-:strong:`TODO`\ (7)
-
+:strong:`apt_preferences`\ (7)
+:strong:`cdist-type__file`\ (7)
 
 AUTHORS
 -------

From 7b3f268df25922c515174862da95569aea547367 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 22 Jun 2021 16:36:30 +0300
Subject: [PATCH 291/366] [__download] improvements

1. post download checksum verification
2. detect hashes without prefix
3. add optional --destination
4. updated man
---
 .../conf/type/__download/explorer/remote_cmd  |  19 ---
 .../type/__download/explorer/remote_cmd_get   |  16 +++
 .../type/__download/explorer/remote_cmd_sum   |  82 +++++++++++
 cdist/conf/type/__download/explorer/state     |  60 ++------
 cdist/conf/type/__download/gencode-local      | 131 +++++++++++++++---
 cdist/conf/type/__download/gencode-remote     |  46 +++++-
 cdist/conf/type/__download/man.rst            |  31 ++++-
 cdist/conf/type/__download/manifest           |   2 +-
 cdist/conf/type/__download/parameter/optional |   3 +-
 9 files changed, 292 insertions(+), 98 deletions(-)
 delete mode 100755 cdist/conf/type/__download/explorer/remote_cmd
 create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_get
 create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_sum

diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd
deleted file mode 100755
index e3e35b45..00000000
--- a/cdist/conf/type/__download/explorer/remote_cmd
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh -e
-
-if [ -f "$__object/parameter/cmd-get" ]
-then
-    cmd="$( cat "$__object/parameter/cmd-get" )"
-
-elif command -v curl > /dev/null
-then
-    cmd="curl -L -o - '%s'"
-
-elif command -v fetch > /dev/null
-then
-    cmd="fetch -o - '%s'"
-
-else
-    cmd="wget -O - '%s'"
-fi
-
-echo "$cmd"
diff --git a/cdist/conf/type/__download/explorer/remote_cmd_get b/cdist/conf/type/__download/explorer/remote_cmd_get
new file mode 100755
index 00000000..9f1cd59c
--- /dev/null
+++ b/cdist/conf/type/__download/explorer/remote_cmd_get
@@ -0,0 +1,16 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cat "$__object/parameter/cmd-get"
+elif
+    command -v curl > /dev/null
+then
+    echo "curl -sSL -o - '%s'"
+elif
+    command -v fetch > /dev/null
+then
+    echo "fetch -o - '%s'"
+else
+    echo "wget -O - '%s'"
+fi
diff --git a/cdist/conf/type/__download/explorer/remote_cmd_sum b/cdist/conf/type/__download/explorer/remote_cmd_sum
new file mode 100755
index 00000000..84df663c
--- /dev/null
+++ b/cdist/conf/type/__download/explorer/remote_cmd_sum
@@ -0,0 +1,82 @@
+#!/bin/sh -e
+
+if [ ! -f "$__object/parameter/sum" ]
+then
+    exit 0
+fi
+
+if [ -f "$__object/parameter/cmd-sum" ]
+then
+    cat "$__object/parameter/cmd-sum"
+    exit 0
+fi
+
+sum_should="$( cat "$__object/parameter/sum" )"
+
+if echo "$sum_should" | grep -Fq ':'
+then
+    sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
+else
+    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+    then
+        sum_hash='cksum'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
+    then
+        sum_hash='md5'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
+    then
+        sum_hash='sha1'
+    elif
+        echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
+    then
+        sum_hash='sha256'
+    else
+        echo 'hash format detection failed' >&2
+        exit 1
+    fi
+fi
+
+os="$( "$__explorer/os" )"
+
+case "$sum_hash" in
+    cksum)
+        echo "cksum %s | awk '{print \$1\" \"\$2}'"
+    ;;
+    md5)
+        case "$os" in
+            freebsd)
+                echo "md5 -q %s"
+            ;;
+            *)
+                echo "md5sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    sha1)
+        case "$os" in
+            freebsd)
+                echo "sha1 -q %s"
+            ;;
+            *)
+                echo "sha1sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    sha256)
+        case "$os" in
+            freebsd)
+                echo "sha256 -q %s"
+            ;;
+            *)
+                echo "sha256sum %s | awk '{print \$1}'"
+            ;;
+        esac
+    ;;
+    *)
+        # we arrive here only if --sum is given with unknown format prefix
+        echo "unknown hash format: $sum_hash" >&2
+        exit 1
+    ;;
+esac
diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 68b517c5..881a1c09 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -1,6 +1,11 @@
 #!/bin/sh -e
 
-dst="/$__object_id"
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
+else
+    dst="/$__object_id"
+fi
 
 if [ ! -f "$dst" ]
 then
@@ -16,57 +21,18 @@ fi
 
 sum_should="$( cat "$__object/parameter/sum" )"
 
-if [ -f "$__object/parameter/cmd-sum" ]
+if echo "$sum_should" | grep -Fq ':'
 then
-    # shellcheck disable=SC2059
-    sum_is="$( eval "$( printf \
-        "$( cat "$__object/parameter/cmd-sum" )" \
-        "$dst" )" )"
-else
-    os="$( "$__explorer/os" )"
-
-    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
-    then
-        sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
-
-    elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="md5:$( md5 -q "$dst" )"
-            ;;
-            *)
-                sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-
-    elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="sha1:$( sha1 -q "$dst" )"
-            ;;
-            *)
-                sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-
-    elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
-    then
-        case "$os" in
-            freebsd)
-                sum_is="sha256:$( sha256 -q "$dst" )"
-            ;;
-            *)
-                sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
-            ;;
-        esac
-    fi
+    sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
 fi
 
+sum_cmd="$( "$__type_explorer/remote_cmd_sum" )"
+
+sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )"
+
 if [ -z "$sum_is" ]
 then
-    echo 'no checksum from target' >&2
+    echo 'existing destination checksum failed' >&2
     exit 1
 fi
 
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 571d2c3c..d1b0d0d5 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -11,34 +11,133 @@ fi
 
 url="$( cat "$__object/parameter/url" )"
 
-tmp="$( mktemp )"
-
-dst="/$__object_id"
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
+else
+    dst="/$__object_id"
+fi
 
 if [ -f "$__object/parameter/cmd-get" ]
 then
     cmd="$( cat "$__object/parameter/cmd-get" )"
 
-elif command -v wget > /dev/null
-then
-    cmd="wget -O - '%s'"
-
 elif command -v curl > /dev/null
 then
-    cmd="curl -L -o - '%s'"
+    cmd="curl -sSL -o - '%s'"
 
 elif command -v fetch > /dev/null
 then
     cmd="fetch -o - '%s'"
 
+elif command -v wget > /dev/null
+then
+    cmd="wget -O - '%s'"
+
 else
-    echo 'no usable locally installed utility for downloading' >&2
+    echo 'local download failed, no usable utility' >&2
     exit 1
 fi
 
-printf "$cmd > %s\n" \
-    "$url" \
-    "$tmp"
+echo "download_tmp=\"\$( mktemp )\""
+
+# shellcheck disable=SC2059
+printf "$cmd > \"\$download_tmp\"\n" "$url"
+
+if [ -f "$__object/parameter/sum" ]
+then
+    sum_should="$( cat "$__object/parameter/sum" )"
+
+    if [ -f "$__object/parameter/cmd-sum" ]
+    then
+        local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
+    else
+        if echo "$sum_should" | grep -Fq ':'
+        then
+            sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
+
+            sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
+        else
+            if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+            then
+                sum_hash='cksum'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
+            then
+                sum_hash='md5'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
+            then
+                sum_hash='sha1'
+            elif
+                echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
+            then
+                sum_hash='sha256'
+            else
+                echo 'hash format detection failed' >&2
+                exit 1
+            fi
+        fi
+
+        case "$sum_hash" in
+            cksum)
+                local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'"
+            ;;
+            md5)
+                if command -v md5 > /dev/null
+                then
+                    local_cmd_sum="md5 -q %s"
+                elif
+                    command -v md5sum > /dev/null
+                then
+                    local_cmd_sum="md5sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            sha1)
+                if command -v sha1 > /dev/null
+                then
+                    local_cmd_sum="sha1 -q %s"
+                elif
+                    command -v sha1sum > /dev/null
+                then
+                    local_cmd_sum="sha1sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            sha256)
+                if command -v sha256 > /dev/null
+                then
+                    local_cmd_sum="sha256 -q %s"
+                elif
+                    command -v sha256sum > /dev/null
+                then
+                    local_cmd_sum="sha256sum %s | awk '{print \$1}'"
+                fi
+            ;;
+            *)
+                # we arrive here only if --sum is given with unknown format prefix
+                echo "unknown hash format: $sum_hash" >&2
+                exit 1
+            ;;
+        esac
+
+        if [ -z "$local_cmd_sum" ]
+        then
+            echo 'local checksum verification failed, no usable utility' >&2
+            exit 1
+        fi
+    fi
+
+    # shellcheck disable=SC2059
+    echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\""
+
+    echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
+
+    echo "echo 'local download checksum mismatch' >&2"
+
+    echo "rm -f \"\$download_tmp\""
+
+    echo 'exit 1; fi'
+fi
 
 if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$'
 then
@@ -47,12 +146,10 @@ else
     target_host="$__target_host"
 fi
 
-printf '%s %s %s:%s\n' \
+# shellcheck disable=SC2016
+printf '%s "$download_tmp" %s:%s\n' \
     "$__remote_copy" \
-    "$tmp" \
     "$target_host" \
     "$dst"
 
-echo "rm -f '$tmp'"
-
-echo 'downloaded' > "$__messages_out"
+echo "rm -f \"\$download_tmp\""
diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
index 029a0801..e49bcec3 100755
--- a/cdist/conf/type/__download/gencode-remote
+++ b/cdist/conf/type/__download/gencode-remote
@@ -6,17 +6,51 @@ state_is="$( cat "$__object/explorer/state" )"
 
 if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
 then
-    cmd="$( cat "$__object/explorer/remote_cmd" )"
+    cmd_get="$( cat "$__object/explorer/remote_cmd_get" )"
 
     url="$( cat "$__object/parameter/url" )"
 
-    dst="/$__object_id"
+    if [ -f "$__object/parameter/destination" ]
+    then
+        dst="$( cat "$__object/parameter/destination" )"
+    else
+        dst="/$__object_id"
+    fi
 
-    printf "$cmd > %s\n" \
-        "$url" \
-        "$dst"
+    echo "download_tmp=\"\$( mktemp )\""
 
-    echo 'downloaded' > "$__messages_out"
+    # shellcheck disable=SC2059
+    printf "$cmd_get > \"\$download_tmp\"\n" "$url"
+
+    if [ -f "$__object/parameter/sum" ]
+    then
+        sum_should="$( cat "$__object/parameter/sum" )"
+
+        if [ -f "$__object/parameter/cmd-sum" ]
+        then
+            remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
+        else
+            remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )"
+
+            if echo "$sum_should" | grep -Fq ':'
+            then
+                sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
+            fi
+        fi
+
+        # shellcheck disable=SC2059
+        echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\""
+
+        echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
+
+        echo "echo 'remote download checksum mismatch' >&2"
+
+        echo "rm -f \"\$download_tmp\""
+
+        echo 'exit 1; fi'
+    fi
+
+    echo "mv \"\$download_tmp\" '$dst'"
 fi
 
 if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ]
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index a1278cfb..c16510a9 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -8,7 +8,7 @@ cdist-type__download - Download a file
 
 DESCRIPTION
 -----------
-By default type will try to use ``wget``, ``curl`` or ``fetch``.
+By default type will try to use ``curl``, ``fetch`` or ``wget``.
 If download happens in target (see ``--download``) then type will
 fallback to (and install) ``wget``.
 
@@ -16,6 +16,8 @@ If download happens in local machine, then environment variables like
 ``{http,https,ftp}_proxy`` etc can be used on cdist execution
 (``http_proxy=foo cdist config ...``).
 
+To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -25,14 +27,29 @@ url
 
 OPTIONAL PARAMETERS
 -------------------
+destination
+   Downloaded file's destination in target. If unset, ``$__object_id`` is used.
+
 sum
-   Checksum is used to decide if existing destination file must be redownloaded.
-   By default output of ``cksum`` without filename is expected.
-   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
+   Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256.
+
+   Type tries to detect hash format with regexes, but prefixes
+   ``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported.
+
+   Checksum have two purposes - state check and post-download verification.
+   In state check, if destination checksum mismatches, then content of URL
+   will be downloaded to temporary file. If downloaded temporary file's
+   checksum matches, then it will be moved to destination (overwritten).
+
+   For local downloads it is expected that usable utilities for checksum
+   calculation exist in the system.
 
 download
-   If ``local`` (default), then download file to local storage and copy
-   it to target host. If ``remote``, then download happens in target.
+   If ``local`` (default), then file is downloaded to local storage and copied
+   to target host. If ``remote``, then download happens in target.
+
+   For local downloads it is expected that usable utilities for downloading
+   exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``.
 
 cmd-get
    Command used for downloading.
@@ -62,7 +79,7 @@ EXAMPLES
     require='__directory/opt/cpma' \
         __download /opt/cpma/cnq3.zip \
             --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
-            --sum md5:46da3021ca9eace277115ec9106c5b46
+            --sum 46da3021ca9eace277115ec9106c5b46
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest
index 7ec8d86d..3d4c498b 100755
--- a/cdist/conf/type/__download/manifest
+++ b/cdist/conf/type/__download/manifest
@@ -1,6 +1,6 @@
 #!/bin/sh -e
 
-if grep -Eq '^wget' "$__object/explorer/remote_cmd"
+if grep -Eq '^wget' "$__object/explorer/remote_cmd_get"
 then
     __package wget
 fi
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index d69e083e..e809ef78 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,5 +1,6 @@
-sum
 cmd-get
 cmd-sum
+destination
 download
 onchange
+sum

From 2db40d8d704b427768307fbea29384bd3dc8dbd7 Mon Sep 17 00:00:00 2001
From: fancsali <fancsali@gmail.com>
Date: Mon, 28 Jun 2021 12:54:20 +0200
Subject: [PATCH 292/366] Use $__remote_exec and thus the ssh multiplexing

---
 cdist/conf/type/__rsync/gencode-local | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index e36ded2f..36addc36 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -36,4 +36,5 @@ fi
 
 echo rsync -a \
     --no-owner --no-group \
+    -e "$__remote_exec" \
     -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"

From d937d53f3dfd10830a07aee0450596eea62f2a1a Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Mon, 28 Jun 2021 18:09:35 +0100
Subject: [PATCH 293/366] Add quotes to rsync command

---
 cdist/conf/type/__rsync/gencode-local | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index 36addc36..f1bddc16 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -36,5 +36,5 @@ fi
 
 echo rsync -a \
     --no-owner --no-group \
-    -e "$__remote_exec" \
+    -e \"$__remote_exec\" \
     -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"

From 60753ddfcc3ac49303d572da7f6a68f398c02227 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 1 Jul 2021 14:42:10 +0300
Subject: [PATCH 294/366] fix shellcheck

---
 cdist/conf/type/__download/explorer/state | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 881a1c09..8c5d5ce1 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -28,6 +28,7 @@ fi
 
 sum_cmd="$( "$__type_explorer/remote_cmd_sum" )"
 
+# shellcheck disable=SC2059
 sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )"
 
 if [ -z "$sum_is" ]

From a90e642c1354cf01be7c9a1ba8a468ad624c4202 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 1 Jul 2021 14:50:40 +0300
Subject: [PATCH 295/366] update README

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index de6901c7..a468dd86 100644
--- a/README.md
+++ b/README.md
@@ -24,8 +24,8 @@ For community-maintained types there is
 
 ## Participating
 
-IRC: ``#cdist`` @ freenode
+IRC: ``#cdist`` @ [libera](https://libera.chat)
 
 Matrix: ``#cdist:ungleich.ch``
 
-Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist
+Matrix and IRC are bridged.

From 243a4b904a2de638d00bac57f6e762a076f9ae54 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 2 Jul 2021 06:50:02 +0200
Subject: [PATCH 296/366] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 97ea204b..d48c657e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,8 @@ next:
 	* Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham)
 	* Type __git: Fix group explorer (Ander Punnar)
 	* Type __pyvenv: Fix group explorer (Dennis Camera)
+	* Type __download: Improve checksum verification, add optional --destination (Ander Punnar)
+	* Type __debconf_set_selections: Add state explorer (Dennis Camera)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 30ba796d060fd3d0c4affd4167f14784156b5354 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 1 Jul 2021 11:49:07 +0300
Subject: [PATCH 297/366] new type: __snakeoil_cert

---
 .../__snakeoil_cert/explorer/ssl-cert-group   |  8 ++
 .../conf/type/__snakeoil_cert/explorer/state  | 24 ++++++
 .../conf/type/__snakeoil_cert/gencode-remote  | 73 +++++++++++++++++++
 cdist/conf/type/__snakeoil_cert/man.rst       | 60 +++++++++++++++
 .../parameter/default/cert-path               |  1 +
 .../parameter/default/key-path                |  1 +
 .../parameter/default/key-type                |  1 +
 .../type/__snakeoil_cert/parameter/optional   |  4 +
 8 files changed, 172 insertions(+)
 create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group
 create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/state
 create mode 100755 cdist/conf/type/__snakeoil_cert/gencode-remote
 create mode 100644 cdist/conf/type/__snakeoil_cert/man.rst
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/cert-path
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-path
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-type
 create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/optional

diff --git a/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group
new file mode 100755
index 00000000..a6cb3dfd
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+if grep -Eq '^ssl-cert:' /etc/group
+then
+    echo 'present'
+else
+    echo 'absent'
+fi
diff --git a/cdist/conf/type/__snakeoil_cert/explorer/state b/cdist/conf/type/__snakeoil_cert/explorer/state
new file mode 100755
index 00000000..cc5aae0b
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/explorer/state
@@ -0,0 +1,24 @@
+#!/bin/sh -e
+
+key_path="$( cat "$__object/parameter/key-path" )"
+
+if echo "$key_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    key_path="$( printf "$key_path" "$__object_id" )"
+fi
+
+cert_path="$( cat "$__object/parameter/cert-path" )"
+
+if echo "$cert_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    cert_path="$( printf "$cert_path" "$__object_id" )"
+fi
+
+if [ ! -f "$key_path" ] || [ ! -f "$cert_path" ]
+then
+    echo 'absent'
+else
+    echo 'present'
+fi
diff --git a/cdist/conf/type/__snakeoil_cert/gencode-remote b/cdist/conf/type/__snakeoil_cert/gencode-remote
new file mode 100755
index 00000000..8ffbfad1
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/gencode-remote
@@ -0,0 +1,73 @@
+#!/bin/sh -e
+
+state="$( cat "$__object/explorer/state" )"
+
+if [ "$state" = 'present' ]
+then
+    exit 0
+fi
+
+if [ -f "$__object/parameter/common-name" ]
+then
+    common_name="$( cat "$__object/parameter/common-name" )"
+else
+    common_name="$__object_id"
+fi
+
+key_path="$( cat "$__object/parameter/key-path" )"
+
+if echo "$key_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    key_path="$( printf "$key_path" "$__object_id" )"
+fi
+
+cert_path="$( cat "$__object/parameter/cert-path" )"
+
+if echo "$cert_path" | grep -Fq '%s'
+then
+    # shellcheck disable=SC2059
+    cert_path="$( printf "$cert_path" "$__object_id" )"
+fi
+
+key_type="$( cat "$__object/parameter/key-type" )"
+
+key_type_arg="$( echo "$key_type" | cut -d : -f 2 )"
+
+case "$key_type" in
+    rsa:*)
+        echo "openssl genrsa -out '$key_path' $key_type_arg"
+    ;;
+    ec:*)
+        echo "openssl ecparam -name $key_type_arg -genkey -noout -out '$key_path'"
+    ;;
+esac
+
+# shellcheck disable=SC2016
+echo 'csr_path="$( mktemp )"'
+
+echo "openssl req -new -subj '/CN=$common_name' -key '$key_path' -out \"\$csr_path\""
+
+echo "openssl x509 -req -sha256 -days 3650 -in \"\$csr_path\" -signkey '$key_path' -out '$cert_path'"
+
+# shellcheck disable=SC2016
+echo 'rm -f "$csr_path"'
+
+if [ "$( cat "$__object/explorer/ssl-cert-group" )" = 'present' ]
+then
+    key_group='ssl-cert'
+else
+    key_group='root'
+fi
+
+echo "chmod 640 '$key_path'"
+
+echo "chown root '$key_path'"
+
+echo "chgrp $key_group '$key_path'"
+
+echo "chmod 644 '$cert_path'"
+
+echo "chown root '$cert_path'"
+
+echo "chgrp root '$cert_path'"
diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst
new file mode 100644
index 00000000..0b547804
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/man.rst
@@ -0,0 +1,60 @@
+cdist-type__snakeoil_cert(7)
+============================
+
+NAME
+----
+cdist-type__snakeoil_cert - Generate self-signed certificate
+
+
+DESCRIPTION
+-----------
+The purpose of this type is to generate **self-signed** certificate and private key
+for **testing purposes**. Certificate will expire in 3650 days.
+
+Certificate's and key's access bits will be ``644`` and ``640`` respectively.
+If target system has ``ssl-cert`` group, then it will be used as key's group.
+Use ``require='__snakeoil_cert/...' __file ...`` to override.
+
+
+OPTIONAL PARAMETERS
+-------------------
+common-name
+   Defaults to ``$__object_id``.
+
+key-path
+   ``%s`` in path will be replaced with ``$__object_id``.
+   Defaults to ``/etc/ssl/private/%s.pem``.
+
+key-type
+   Possible values are ``rsa:$bits`` and ``ec:$name``.
+   For possible EC names see ``openssl ecparam -list_curves``.
+   Defaults to ``rsa:2048``.
+
+cert-path
+   ``%s`` in path will be replaced with ``$__object_id``.
+   Defaults to ``/etc/ssl/certs/%s.pem``.
+
+
+EXAMPLES
+--------
+.. code-block:: sh
+	__snakeoil_cert localhost-rsa \
+	    --common-name localhost \
+	    --key-type rsa:4096
+
+	__snakeoil_cert localhost-ec \
+	    --common-name localhost \
+	    --key-type ec:prime256v1
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path
new file mode 100644
index 00000000..4bbae089
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path
@@ -0,0 +1 @@
+/etc/ssl/certs/%s.pem
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-path b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path
new file mode 100644
index 00000000..86eb9359
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path
@@ -0,0 +1 @@
+/etc/ssl/private/%s.pem
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-type b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type
new file mode 100644
index 00000000..f13f8ada
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type
@@ -0,0 +1 @@
+rsa:2048
diff --git a/cdist/conf/type/__snakeoil_cert/parameter/optional b/cdist/conf/type/__snakeoil_cert/parameter/optional
new file mode 100644
index 00000000..76d08c0a
--- /dev/null
+++ b/cdist/conf/type/__snakeoil_cert/parameter/optional
@@ -0,0 +1,4 @@
+common-name
+key-path
+key-type
+cert-path

From 853e5cf7b4f615e1470c7ec2cfa8fe68cfa85ce7 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Mon, 5 Jul 2021 09:07:06 +0200
Subject: [PATCH 298/366] ++changelog

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index d48c657e..0c3c64e1 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -11,6 +11,8 @@ next:
 	* Type __pyvenv: Fix group explorer (Dennis Camera)
 	* Type __download: Improve checksum verification, add optional --destination (Ander Punnar)
 	* Type __debconf_set_selections: Add state explorer (Dennis Camera)
+	* Core: Implement usable cdist scan (Timothée Floure)
+	* New type: __snakeoil_cert (Ander Punnar)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From be92731c5c8a8543448f0d87fafae67e22ac76a1 Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Mon, 5 Jul 2021 12:38:26 +0100
Subject: [PATCH 299/366] Shell check quoting

We're actually echo-ing the command, hence the escape in front of the
quotes - the issue Shellcheck alludes too would actually occur, had the
escaping bakcslashes been omitted.
---
 cdist/conf/type/__rsync/gencode-local | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index f1bddc16..be4feabb 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -34,7 +34,8 @@ if [ -f "$__object/parameter/rsync-opts" ]; then
     done < "$__object/parameter/rsync-opts"
 fi
 
+# shellcheck disable=SC2086
 echo rsync -a \
     --no-owner --no-group \
-    -e \"$__remote_exec\" \
+    -e \"${__remote_exec}\" \
     -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"

From 521241d74102e37fae5f55552e1ef565d26ea9d2 Mon Sep 17 00:00:00 2001
From: fancsali <fancsali@gmail.com>
Date: Mon, 5 Jul 2021 15:28:05 +0200
Subject: [PATCH 300/366] Refine docs even more

---
 cdist/conf/type/__apt_pin/man.rst | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst
index 0c91cdec..4229c0cd 100644
--- a/cdist/conf/type/__apt_pin/man.rst
+++ b/cdist/conf/type/__apt_pin/man.rst
@@ -3,12 +3,12 @@ cdist-type__apt_pin(7)
 
 NAME
 ----
-cdist-type__apt_pin - TODO
+cdist-type__apt_pin - Manage apt pinning rules
 
 
 DESCRIPTION
 -----------
-This space intentionally left blank.
+Adds/removes/edits rules to pin some packages to a specific distribution. Useful if using multiple debian repositories at the same time. (Useful, if one wants to use a few specific packages from backports or perhaps Debain testing... or even sid.)
 
 
 REQUIRED PARAMETERS
@@ -20,7 +20,7 @@ distribution
 OPTIONAL PARAMETERS
 -------------------
 package
-   Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used.
+   Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used.
 
 priority
    The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority)
@@ -28,7 +28,6 @@ priority
 state
    Will be passed to underlying `__file` type; see there for valid values and defaults.
 
-None.
 
 
 BOOLEAN PARAMETERS
@@ -41,8 +40,8 @@ EXAMPLES
 
 .. code-block:: sh
 
-   # Add the bullseye repo to buster, but do not install any pacakges by default
-   # only if explicitely asked for
+   # Add the bullseye repo to buster, but do not install any packages by default,
+   # only if explicitely asked for (-1 means "never" for apt)
     __apt_pin bullseye-default \
        --package "*" \
        --distribution bullseye \
@@ -52,19 +51,19 @@ EXAMPLES
        --uri http://deb.debian.org/debian/ \
        --distribution bullseye \
        --component main
-       # TODO
-       __apt_pin
 
     __apt_pin foo --package "foo foo-*" --distribution bullseye
 
-    __foo # Installs the `foo` package internally
+    __foo # Assuming, this installs the `foo` package internally
 
-    __package foo-plugin-extras
+    __package foo-plugin-extras # Assuming we also need some extra stuff
 
 
 SEE ALSO
 --------
-:strong:`apt_preferences`\ (7)
+:strong:`apt_preferences`\ (5)
+:strong:`cdist-type__apt_source`\ (7)
+:strong:`cdist-type__apt_backports`\ (7)
 :strong:`cdist-type__file`\ (7)
 
 AUTHORS

From 166b58aeea09da41086525849291c70a3c3a571c Mon Sep 17 00:00:00 2001
From: fancsali <fancsali@gmail.com>
Date: Mon, 5 Jul 2021 15:32:27 +0200
Subject: [PATCH 301/366] Fix typo in distro names...

---
 cdist/conf/type/__apt_pin/manifest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index 909bc80d..e72a8fdd 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -44,7 +44,7 @@ case "$os" in
 esac
 
 case $distribution in
-    stabletesting|unsatbel|experimental)
+    stable|testing|unstable|experimental)
         pin="release a=$distribution"
         ;;
     *)

From 485283f2e531f5fa23a876fa52411e0030f32589 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 7 Jul 2021 20:47:22 +0300
Subject: [PATCH 302/366] new type: __sed

---
 cdist/conf/type/__sed/explorer/file           | 22 +++++++++
 cdist/conf/type/__sed/gencode-remote          | 46 +++++++++++++++++
 cdist/conf/type/__sed/man.rst                 | 49 +++++++++++++++++++
 cdist/conf/type/__sed/parameter/boolean       |  1 +
 cdist/conf/type/__sed/parameter/optional      |  1 +
 .../type/__sed/parameter/required_multiple    |  1 +
 6 files changed, 120 insertions(+)
 create mode 100755 cdist/conf/type/__sed/explorer/file
 create mode 100755 cdist/conf/type/__sed/gencode-remote
 create mode 100644 cdist/conf/type/__sed/man.rst
 create mode 100644 cdist/conf/type/__sed/parameter/boolean
 create mode 100644 cdist/conf/type/__sed/parameter/optional
 create mode 100644 cdist/conf/type/__sed/parameter/required_multiple

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
new file mode 100755
index 00000000..a40f71b3
--- /dev/null
+++ b/cdist/conf/type/__sed/explorer/file
@@ -0,0 +1,22 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/file" ]
+then
+    file="$( cat "$__object/parameter/file" )"
+else
+    file="/$__object_id"
+fi
+
+if [ -f "$file" ]
+then
+    if [ -s "$file" ]
+    then
+        cat "$file"
+    else
+        echo "$file is empty" >&2
+        exit 1
+    fi
+else
+    echo "$file do not exist" >&2
+    exit 1
+fi
diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
new file mode 100755
index 00000000..391e3bc1
--- /dev/null
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -0,0 +1,46 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/file" ]
+then
+    file="$( cat "$__object/parameter/file" )"
+else
+    file="/$__object_id"
+fi
+
+script="$( cat "$__object/parameter/script" )"
+
+if [ "$script" = '-' ]
+then
+    script="$( cat "$__object/stdin" )"
+elif
+    [ -f "$script" ]
+then
+    script="$( cat "$script" )"
+fi
+
+file_from_target="$__object/explorer/file"
+
+sed_cmd='sed'
+
+if [ -f "$__object/parameter/regexp-extended" ]
+then
+    sed_cmd="$sed_cmd --regexp-extended"
+fi
+
+if ! echo "$script" \
+    | "$sed_cmd" -f - "$file_from_target" \
+    | diff "$file_from_target" - \
+    > /dev/null
+then
+    echo 'tmp="$( mktemp )"'
+
+    echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
+
+    echo "$script"
+
+    echo 'EOF'
+
+    echo "cp \"\$tmp\" '$file'"
+    
+    echo 'rm -f "$tmp"'
+fi
diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst
new file mode 100644
index 00000000..fbac212d
--- /dev/null
+++ b/cdist/conf/type/__sed/man.rst
@@ -0,0 +1,49 @@
+cdist-type__sed(7)
+==================
+
+NAME
+----
+cdist-type__sed - Transform text files with ``sed``
+
+
+DESCRIPTION
+-----------
+TODO
+
+
+REQUIRED MULTIPLE PARAMETERS
+----------------------------
+script
+   TODO
+
+
+OPTIONAL PARAMETERS
+-------------------
+file
+   TODO
+
+
+BOOLEAN PARAMETERS
+------------------
+regexp-extended
+   TODO
+
+
+EXAMPLES
+--------
+.. code-block:: sh
+
+   true
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
diff --git a/cdist/conf/type/__sed/parameter/boolean b/cdist/conf/type/__sed/parameter/boolean
new file mode 100644
index 00000000..1ad75c5d
--- /dev/null
+++ b/cdist/conf/type/__sed/parameter/boolean
@@ -0,0 +1 @@
+regexp-extended
diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional
new file mode 100644
index 00000000..f73f3093
--- /dev/null
+++ b/cdist/conf/type/__sed/parameter/optional
@@ -0,0 +1 @@
+file
diff --git a/cdist/conf/type/__sed/parameter/required_multiple b/cdist/conf/type/__sed/parameter/required_multiple
new file mode 100644
index 00000000..84f7e31d
--- /dev/null
+++ b/cdist/conf/type/__sed/parameter/required_multiple
@@ -0,0 +1 @@
+script

From 7a5896acfad0f992d6e7fe9cbe0bd08035ce19db Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 7 Jul 2021 21:23:25 +0300
Subject: [PATCH 303/366] add --onchange, fix shellcheck

---
 cdist/conf/type/__sed/gencode-remote     | 9 ++++++++-
 cdist/conf/type/__sed/man.rst            | 3 +++
 cdist/conf/type/__sed/parameter/optional | 1 +
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 391e3bc1..6b1bd216 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -32,6 +32,7 @@ if ! echo "$script" \
     | diff "$file_from_target" - \
     > /dev/null
 then
+    # shellcheck disable=SC2016
     echo 'tmp="$( mktemp )"'
 
     echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
@@ -41,6 +42,12 @@ then
     echo 'EOF'
 
     echo "cp \"\$tmp\" '$file'"
-    
+
+    # shellcheck disable=SC2016
     echo 'rm -f "$tmp"'
+
+    if [ -f "$__object/parameter/onchange" ]
+    then
+        cat "$__object/parameter/onchange"
+    fi
 fi
diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst
index fbac212d..4a728184 100644
--- a/cdist/conf/type/__sed/man.rst
+++ b/cdist/conf/type/__sed/man.rst
@@ -22,6 +22,9 @@ OPTIONAL PARAMETERS
 file
    TODO
 
+onchange
+   TODO
+
 
 BOOLEAN PARAMETERS
 ------------------
diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional
index f73f3093..fa86f917 100644
--- a/cdist/conf/type/__sed/parameter/optional
+++ b/cdist/conf/type/__sed/parameter/optional
@@ -1 +1,2 @@
 file
+onchange

From cf0032d667eb4bd8b7fe1bebacec341bbf71ed42 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 7 Jul 2021 21:28:00 +0300
Subject: [PATCH 304/366] add messaging and exit earlier

---
 cdist/conf/type/__sed/gencode-remote | 42 +++++++++++++++-------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 6b1bd216..836506f0 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -27,27 +27,31 @@ then
     sed_cmd="$sed_cmd --regexp-extended"
 fi
 
-if ! echo "$script" \
+if echo "$script" \
     | "$sed_cmd" -f - "$file_from_target" \
     | diff "$file_from_target" - \
     > /dev/null
 then
-    # shellcheck disable=SC2016
-    echo 'tmp="$( mktemp )"'
-
-    echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
-
-    echo "$script"
-
-    echo 'EOF'
-
-    echo "cp \"\$tmp\" '$file'"
-
-    # shellcheck disable=SC2016
-    echo 'rm -f "$tmp"'
-
-    if [ -f "$__object/parameter/onchange" ]
-    then
-        cat "$__object/parameter/onchange"
-    fi
+    exit 0
+fi
+
+# shellcheck disable=SC2016
+echo 'tmp="$( mktemp )"'
+
+echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
+
+echo "$script"
+
+echo 'EOF'
+
+echo "cp \"\$tmp\" '$file'"
+
+# shellcheck disable=SC2016
+echo 'rm -f "$tmp"'
+
+echo 'change' >> "$__messages_out"
+
+if [ -f "$__object/parameter/onchange" ]
+then
+    cat "$__object/parameter/onchange"
 fi

From 3e76d1cd3fbe53fa77c460cb2ce8416698b101bd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 8 Jul 2021 08:09:05 +0200
Subject: [PATCH 305/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 0c3c64e1..04f826f0 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -13,6 +13,7 @@ next:
 	* Type __debconf_set_selections: Add state explorer (Dennis Camera)
 	* Core: Implement usable cdist scan (Timothée Floure)
 	* New type: __snakeoil_cert (Ander Punnar)
+	* Type __rsync: Honour $__remote_exec env var (Daniel Fancsali)
 
 6.9.6: 2021-04-20
 	* Type __pyvenv: Fix user example in man page (Dennis Camera)

From 77dab4c5c63070aef962875af7fe8b1565f5ba78 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 10 Jul 2021 20:37:02 +0200
Subject: [PATCH 306/366] Release 6.9.7

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 04f826f0..284293c1 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)
 	* Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)

From 65c43d3c1db938ac0063f54b1cb6b5090bb0a665 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 10 Jul 2021 21:02:27 +0200
Subject: [PATCH 307/366] Fix docs code block errors

---
 cdist/conf/type/__snakeoil_cert/man.rst | 1 +
 docs/src/cdist-scan.rst                 | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst
index 0b547804..b0b0a2e9 100644
--- a/cdist/conf/type/__snakeoil_cert/man.rst
+++ b/cdist/conf/type/__snakeoil_cert/man.rst
@@ -38,6 +38,7 @@ cert-path
 EXAMPLES
 --------
 .. code-block:: sh
+
 	__snakeoil_cert localhost-rsa \
 	    --common-name localhost \
 	    --key-type rsa:4096
diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst
index 064e65ff..86b7fab6 100644
--- a/docs/src/cdist-scan.rst
+++ b/docs/src/cdist-scan.rst
@@ -57,6 +57,7 @@ resolved name to stdout - if any. The script must be executable.
 Simplest script:
 
 .. code-block:: sh
+
   #!/bin/sh
 
   case "$1" in
@@ -71,6 +72,7 @@ Simplest script:
 Resolving name from `PTR` DNS record:
 
 .. code-block:: sh
+
   #!/bin/sh
 
   for cmd in dig sed; do

From 0e611af2a6388572eef5112c2ffaed082803965c Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 13 Jul 2021 00:13:22 +0300
Subject: [PATCH 308/366] [__rsync] rewrite

---
 cdist/conf/type/__rsync/gencode-local         | 123 +++++++++++++-----
 cdist/conf/type/__rsync/gencode-remote        |  37 ------
 cdist/conf/type/__rsync/man.rst               | 107 +++++----------
 cdist/conf/type/__rsync/manifest              |  18 ---
 .../type/__rsync/parameter/default/options    |   1 +
 cdist/conf/type/__rsync/parameter/optional    |   4 +-
 .../type/__rsync/parameter/optional_multiple  |   2 +-
 7 files changed, 130 insertions(+), 162 deletions(-)
 delete mode 100755 cdist/conf/type/__rsync/gencode-remote
 create mode 100644 cdist/conf/type/__rsync/parameter/default/options

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index be4feabb..612d237e 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -1,41 +1,100 @@
 #!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
-source=$(cat "$__object/parameter/source")
-remote_user=$(cat "$__object/parameter/remote-user")
+if ! command -v rsync > /dev/null
+then
+    echo 'rsync is missing in local machine' >&2
+    exit 1
+fi
 
-if [ -f "$__object/parameter/destination" ]; then
-    destination=$(cat "$__object/parameter/destination")
+src="$( cat "$__object/parameter/source" )"
+
+if [ ! -e "$src" ]
+then
+    echo "$src not found" >&2
+    exit 1
+fi
+
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
 else
-    destination="/$__object_id"
+    dst="/$__object_id"
 fi
 
-set --
-if [ -f "$__object/parameter/rsync-opts" ]; then
-    while read -r opts; do
-        set -- "$@" "--$opts"
-    done < "$__object/parameter/rsync-opts"
+# if source is directory, then make sure that
+# source and destination are ending with slash,
+# because this is what you almost always want when
+# rsyncing two directories.
+
+if [ -d "$src" ]
+then
+    if ! echo "$src" | grep -Eq '/$'
+    then
+        src="$src/"
+    fi
+
+    if ! echo "$dst" | grep -Eq '/$'
+    then
+        dst="$dst/"
+    fi
 fi
 
+remote_user="$( cat "$__object/parameter/remote-user" )"
+
+options="$( cat "$__object/parameter/options" )"
+
+if [ -f "$__object/parameter/option" ]
+then
+    while read -r l
+    do
+        # there's a limitation in argparse: value can't begin with '-'.
+        # to workaround this, let's prefix opts with '\' in manifest and remove here.
+        # read more about argparse issue: https://bugs.python.org/issue9334
+
+        options="$options $( echo "$l" | sed 's/\\//g' )"
+    done \
+        < "$__object/parameter/option"
+fi
+
+if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ]
+then
+    options="$options --chown="
+
+    if [ -f "$__object/parameter/owner" ]
+    then
+        owner="$( cat "$__object/parameter/owner" )"
+        options="$options$owner"
+    fi
+
+    if [ -f "$__object/parameter/group" ]
+    then
+        group="$( cat "$__object/parameter/group" )"
+        options="$options:$group"
+    fi
+fi
+
+if [ -f "$__object/parameter/mode" ]
+then
+    mode="$( cat "$__object/parameter/mode" )"
+    options="$options --chmod=$mode"
+fi
+
+# IMPORTANT
+#
+# 1. we first dry-run rsync with change summary to find out
+#    if there are any changes and code generation is needed.
+# 2. normally, to get current state or target host, we run
+#    such operations in type explorers, but that's not
+#    possible due to how rsync works.
+# 3. redirecting output of dry-run to stderr to ease debugging.
+# 4. to understand how that cryptic regex works, please
+#    open rsync manpage and read about --itemize-changes.
+
 # shellcheck disable=SC2086
-echo rsync -a \
-    --no-owner --no-group \
-    -e \"${__remote_exec}\" \
-    -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"
+if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \
+    | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2
+then
+    exit 0
+fi
+
+echo "rsync $options $src $remote_user@$__target_host:$dst"
diff --git a/cdist/conf/type/__rsync/gencode-remote b/cdist/conf/type/__rsync/gencode-remote
deleted file mode 100755
index 074246af..00000000
--- a/cdist/conf/type/__rsync/gencode-remote
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-if [ -f "$__object/parameter/destination" ]; then
-    destination=$(cat "$__object/parameter/destination")
-else
-    destination="/$__object_id"
-fi
-
-ownergroup=""
-if [ -f "$__object/parameter/owner" ]; then
-    ownergroup=$(cat "$__object/parameter/owner")
-fi
-if [ -f "$__object/parameter/group" ]; then
-    ownergroup="${ownergroup}:$(cat "$__object/parameter/group")"
-fi
-
-if [ "$ownergroup" ]; then
-    echo chown -R "$ownergroup" "$destination"
-fi
diff --git a/cdist/conf/type/__rsync/man.rst b/cdist/conf/type/__rsync/man.rst
index 94b06d63..88019c92 100644
--- a/cdist/conf/type/__rsync/man.rst
+++ b/cdist/conf/type/__rsync/man.rst
@@ -3,112 +3,73 @@ cdist-type__rsync(7)
 
 NAME
 ----
-cdist-type__rsync - Mirror directories using rsync
+cdist-type__rsync - Mirror directories using ``rsync``
 
 
 DESCRIPTION
 -----------
-WARNING: This type is of BETA quality:
-
-- it has not been tested widely
-- interfaces *may* change
-- if there is a better approach to solve the problem -> the type may even vanish
-
-If you are fine with these constraints, please read on.
-
-
-This cdist type allows you to mirror local directories to the
-target host using rsync. Rsync will be installed in the manifest of the type.
-If group or owner are giveng, a recursive chown will be executed on the 
-target host.
-
-A slash will be appended to the source directory so that only the contents
-of the directory are taken and not the directory name itself.
+The purpose of this type is to bring power of ``rsync`` into ``cdist``.
 
 
 REQUIRED PARAMETERS
 -------------------
 source
-    Where to take files from
+   Source directory in local machine.
+   If source is directory, slash (``/``) will be added to source and destination paths.
 
 
 OPTIONAL PARAMETERS
 -------------------
-group
-   Group to chgrp to.
+destination
+   Destination directory. Defaults to ``$__object_id``.
 
 owner
-   User to chown to.
+   Will be passed to ``rsync`` as ``--chown=OWNER``.
+   Read ``rsync(1)`` for more details.
 
-destination
-    Use this as the base destination instead of the object id
+group
+   Will be passed to ``rsync`` as ``--chown=:GROUP``.
+   Read ``rsync(1)`` for more details.
+
+mode
+   Will be passed to ``rsync`` as ``--chmod=MODE``.
+   Read ``rsync(1)`` for more details.
+
+options
+   Defaults to ``--recursive --links --perms --times``.
+   Due to `bug in Python's argparse<https://bugs.python.org/issue9334>`_, value must be prefixed with ``\``.
 
 remote-user
-    Use this user instead of the default "root" for rsync operations.
+   Defaults to ``root``.
 
 
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
-rsync-opts
-    Use this option to give rsync options with.
-    See rsync(1) for available options.
-    Only "--" options are supported.
-    Write the options without the beginning "--"
-    Can be specified multiple times.
-
-
-MESSAGES
---------
-NONE
+option
+   Pass additional options to ``rsync``.
+   See ``rsync(1)`` for all possible options.
+   Due to `bug in Python's argparse<https://bugs.python.org/issue9334>`_, value must be prefixed with ``\``.
 
 
 EXAMPLES
 --------
-
 .. code-block:: sh
 
-    # You can use any source directory
-    __rsync /tmp/testdir \
-        --source /etc
-
-    # Use source from type
-    __rsync /etc \
-        --source "$__type/files/package"
-
-    # Allow multiple __rsync objects to write to the same dir
-    __rsync mystuff \
-        --destination /usr/local/bin \
-        --source "$__type/files/package"
-
-    __rsync otherstuff \
-        --destination /usr/local/bin \
-        --source "$__type/files/package2"
-
-    # Use rsync option --exclude
-    __rsync /tmp/testdir \
-        --source /etc \
-        --rsync-opts exclude=sshd_conf
-
-    # Use rsync with multiple options --exclude --dry-run
-    __rsync /tmp/testing \
-        --source /home/tester \
-        --rsync-opts exclude=id_rsa \
-        --rsync-opts dry-run
-
-
-SEE ALSO
---------
-:strong:`rsync`\ (1)
+    __rsync /var/www/example.com \
+        --owner root \
+        --group www-data \
+        --mode 'D750,F640' \
+        --source "$__files/example.com/www"
 
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+Ander Punnar <ander-at-kvlt-dot-ee>
 
 
 COPYING
 -------
-Copyright \(C) 2015 Nico Schottelius. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
diff --git a/cdist/conf/type/__rsync/manifest b/cdist/conf/type/__rsync/manifest
index 9bd44c6d..64fa804e 100755
--- a/cdist/conf/type/__rsync/manifest
+++ b/cdist/conf/type/__rsync/manifest
@@ -1,21 +1,3 @@
 #!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
 __package rsync
diff --git a/cdist/conf/type/__rsync/parameter/default/options b/cdist/conf/type/__rsync/parameter/default/options
new file mode 100644
index 00000000..d967b110
--- /dev/null
+++ b/cdist/conf/type/__rsync/parameter/default/options
@@ -0,0 +1 @@
+--recursive --links --perms --times
diff --git a/cdist/conf/type/__rsync/parameter/optional b/cdist/conf/type/__rsync/parameter/optional
index ac2b2390..833e9bbe 100644
--- a/cdist/conf/type/__rsync/parameter/optional
+++ b/cdist/conf/type/__rsync/parameter/optional
@@ -1,4 +1,6 @@
 destination
-owner
 group
+mode
+options
+owner
 remote-user
diff --git a/cdist/conf/type/__rsync/parameter/optional_multiple b/cdist/conf/type/__rsync/parameter/optional_multiple
index fdb7cd88..01925a15 100644
--- a/cdist/conf/type/__rsync/parameter/optional_multiple
+++ b/cdist/conf/type/__rsync/parameter/optional_multiple
@@ -1 +1 @@
-rsync-opts
+option

From 46b5c24cd240cd9006d93cfcc12a4d81b46a5238 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 18 Jul 2021 16:25:00 +0300
Subject: [PATCH 309/366] use $__remote_exec for RSYNC_RSH

---
 cdist/conf/type/__rsync/gencode-local | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local
index 612d237e..e9f3c131 100755
--- a/cdist/conf/type/__rsync/gencode-local
+++ b/cdist/conf/type/__rsync/gencode-local
@@ -90,6 +90,8 @@ fi
 # 4. to understand how that cryptic regex works, please
 #    open rsync manpage and read about --itemize-changes.
 
+export RSYNC_RSH="$__remote_exec"
+
 # shellcheck disable=SC2086
 if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \
     | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2
@@ -97,4 +99,6 @@ then
     exit 0
 fi
 
+echo "export RSYNC_RSH='$__remote_exec'"
+
 echo "rsync $options $src $remote_user@$__target_host:$dst"

From 5229337611b7e7afb2597729a786c637a8bec1f6 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 18 Jul 2021 17:41:29 +0200
Subject: [PATCH 310/366] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 284293c1..f0746218 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Type __rsync: Rewrite (Ander Punnar)
+
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
 	* Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera)

From de116661613002e91b54194ac8f0d5c3dd3eebbd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sun, 18 Jul 2021 17:45:19 +0200
Subject: [PATCH 311/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index f0746218..ea51ac1b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,7 @@ Changelog
 
 next:
 	* Type __rsync: Rewrite (Ander Punnar)
+	* New type: __apt_pin (Daniel Fancsali)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 24c9406ea0f2c6c8edc87e5bbc1be25b5e8e1572 Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Mon, 19 Jul 2021 12:13:23 +0200
Subject: [PATCH 312/366] [explorer/os_version] Convert Devuan ceres to version
 number

Conversion of Devuan ceres to version numbers is done based on Devuan codenames.
The version number is the version number of the final release - 0.01.

Analogous to Debian.
---
 cdist/conf/explorer/os_version | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 3b02dedd..6c94915c 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -1,6 +1,7 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,12 +18,11 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
 # All os variables are lower case
 #
-#
 
-case "$("$__explorer/os")" in
+case $("${__explorer:?}/os")
+in
    amazon)
       cat /etc/system-release
    ;;
@@ -59,7 +59,23 @@ case "$("$__explorer/os")" in
       esac
    ;;
    devuan)
-      cat /etc/devuan_version
+      devuan_version=$(cat /etc/devuan_version)
+      case ${devuan_version}
+      in
+         (*/ceres)
+            # ceres versions don't have a number, so we decode by codename:
+            case ${devuan_version}
+            in
+               (chimaera/ceres) echo 3.99 ;;
+               (beowulf/ceres) echo 2.99 ;;
+               (ascii/ceres) echo 1.99 ;;
+               (*) exit 1
+            esac
+            ;;
+         (*)
+            echo "${devuan_version}"
+            ;;
+      esac
    ;;
    fedora)
       cat /etc/fedora-release

From fbc9594729ece74b2f8612fe8046ddea966e8c05 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 20 Jul 2021 06:38:46 +0200
Subject: [PATCH 313/366] ++changelog

---
 docs/changelog | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/docs/changelog b/docs/changelog
index ea51ac1b..55d7cb73 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)
+	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)
@@ -146,7 +147,7 @@ next:
 	* Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham)
 	* Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham)
 	* Configuration: Add notes to cdist.cfg.skeleton (Evil Ham)
-	* Explorers cpu_cores, memory: Improve *BSD support (Evil Ham)
+	* Explorers cpu_cores, memory: Improve BSD support (Evil Ham)
 	* Core: Remove debug logging noise (Evil Ham)
 
 6.5.4: 2020-04-11
@@ -211,7 +212,7 @@ next:
 	* Documentation: PreOS english nitpicking (Evil Ham)
 	* Documentation: Add installing from source with signature verification (Darko Poljak)
 	* Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak)
-	* Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham)
+	* Type __start_on_boot: Docs: remove unsupported BSD claim (Evil Ham)
 	* New type: __openldap_server (Evil Ham)
 
 6.2.0: 2019-11-30
@@ -1070,9 +1071,9 @@ next:
 	* Removed type __removeline (replaced by __line) (Nico Schottelius)
 	* Type __directory: Parameter --parents and --recursive are now boolean (Nico Schottelius)
 	* Type __package_apt, __package_luarocks, __package_opkg,
-		__package_pacman, __package_pkg_freebsd, __package_pkg_openbsd,
-		__package_rubygem, __package_yum, __process:
-			Parameter state accepts only "present" and "absent" (Nico Schottelius)
+	  __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd,
+	  __package_rubygem, __package_yum, __process:
+	  Parameter state accepts only "present" and "absent" (Nico Schottelius)
 	* Dist: Initial support for pypi packaging (Nico Schottelius)
 
 2.0.15: 2012-11-02

From c7daaabc6c28cbfaebaf6c620cc3cac130684d2c Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Tue, 20 Jul 2021 09:03:16 +0200
Subject: [PATCH 314/366] [docs] Bump copyright year to 2021

---
 docs/src/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/conf.py b/docs/src/conf.py
index 47765413..a3dfafca 100644
--- a/docs/src/conf.py
+++ b/docs/src/conf.py
@@ -56,7 +56,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'cdist'
-copyright = 'ungleich GmbH 2020'
+copyright = 'ungleich GmbH 2021'
 # author = 'Darko Poljak'
 
 # The version info for the project you're documenting, acts as replacement for

From fed01ded83b217a3ad147f2403ca863be373eeaa Mon Sep 17 00:00:00 2001
From: Dennis Camera <dennis.camera@ssrq-sds-fds.ch>
Date: Thu, 22 Jul 2021 11:17:41 +0200
Subject: [PATCH 315/366] [cdist.log] Define custom log functions on
 logging.Logger

Define out custom logger functions on logging.Logger so that they are passed on
to all other loggers.

Also, the logger functions need to take a self argument so that they can log on
the corrent Logger.
---
 cdist/log.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/cdist/log.py b/cdist/log.py
index 113f3b4c..62e457fe 100644
--- a/cdist/log.py
+++ b/cdist/log.py
@@ -36,25 +36,27 @@ import threading
 logging.OFF = logging.CRITICAL + 10  # disable logging
 logging.addLevelName(logging.OFF, 'OFF')
 
+
 logging.VERBOSE = logging.INFO - 5
 logging.addLevelName(logging.VERBOSE, 'VERBOSE')
 
 
-def _verbose(msg, *args, **kwargs):
-    logging.log(logging.VERBOSE, msg, *args, **kwargs)
+def _verbose(self, msg, *args, **kwargs):
+    self.log(logging.VERBOSE, msg, args, **kwargs)
 
 
-logging.verbose = _verbose
+logging.Logger.verbose = _verbose
+
 
 logging.TRACE = logging.DEBUG - 5
 logging.addLevelName(logging.TRACE, 'TRACE')
 
 
-def _trace(msg, *args, **kwargs):
-    logging.log(logging.TRACE, msg, *args, **kwargs)
+def _trace(self, msg, *args, **kwargs):
+    self.log(logging.TRACE, msg, *args, **kwargs)
 
 
-logging.trace = _trace
+logging.Logger.trace = _trace
 
 
 class CdistFormatter(logging.Formatter):

From 71fee1fd6b4874abffee5173a96c31842a9583bd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 23 Jul 2021 08:06:45 +0200
Subject: [PATCH 316/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 55d7cb73..1273e432 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -5,6 +5,7 @@ next:
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
+	* Core: Fix logging bug (Dennis Camera)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 67bcc6cae38e2d480c3336b6a4d59d01196190e3 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sat, 24 Jul 2021 02:37:58 +0200
Subject: [PATCH 317/366] Improve Makefile compatibility and build docs

We now use `$(MAKE)` for subsequent calls to `make`.
This means that systems that do not default to GNU make can run `gmake man` and
produce the man pages.

While there also document a dependency on the rtd theme for sphinx.
---
 Makefile                   | 6 +++---
 docs/src/cdist-install.rst | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 3712511c..89286310 100644
--- a/Makefile
+++ b/Makefile
@@ -35,9 +35,9 @@ DOCS_SRC_DIR=./docs/src
 SPEECHDIR=./docs/speeches
 TYPEDIR=./cdist/conf/type
 
-SPHINXM=make -C $(DOCS_SRC_DIR) man
-SPHINXH=make -C $(DOCS_SRC_DIR) html
-SPHINXC=make -C $(DOCS_SRC_DIR) clean
+SPHINXM=$(MAKE) -C $(DOCS_SRC_DIR) man
+SPHINXH=$(MAKE) -C $(DOCS_SRC_DIR) html
+SPHINXC=$(MAKE) -C $(DOCS_SRC_DIR) clean
 
 ################################################################################
 # Manpages
diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst
index 18863145..390ab9ec 100644
--- a/docs/src/cdist-install.rst
+++ b/docs/src/cdist-install.rst
@@ -12,7 +12,7 @@ This is the machine from which you will configure target hosts.
  * /bin/sh: A POSIX like shell (for instance bash, dash, zsh)
  * Python >= 3.5
  * SSH client
- * sphinx (for building html docs and/or the man pages)
+ * sphinx with the rtd theme (for building html docs and/or the man pages)
 
 Target Hosts
 ~~~~~~~~~~~~

From cb8695cc88478640d7e76992438d90d3d6a68a90 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Sat, 24 Jul 2021 12:53:39 +0200
Subject: [PATCH 318/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 1273e432..181c4139 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -6,6 +6,7 @@ next:
 	* New type: __apt_pin (Daniel Fancsali)
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
 	* Core: Fix logging bug (Dennis Camera)
+	* Build: Improve Makefile compatibility (Evilham)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 4156fea9001aa267c8b173247cada2f919511c1b Mon Sep 17 00:00:00 2001
From: Joachim Desroches <joachim.desroches@epfl.ch>
Date: Wed, 28 Jul 2021 12:56:39 +0200
Subject: [PATCH 319/366] [filesystem] Add ubuntu as supported distribution.

---
 cdist/conf/type/__filesystem/explorer/lsblk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk
index 9be3c575..d376c09f 100644
--- a/cdist/conf/type/__filesystem/explorer/lsblk
+++ b/cdist/conf/type/__filesystem/explorer/lsblk
@@ -27,7 +27,7 @@ else
 fi
 
 case "$os" in
-    alpine|centos|fedora|redhat|suse|gentoo)
+    alpine|centos|fedora|gentoo|redhat|suse|ubuntu)
         if [ ! -x "$(command -v lsblk)" ]; then
             echo "lsblk is required for __filesystem type" >&2
             exit 1

From 542674dae81ab03a4de2c4ad0b2eaf264ee2c442 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Fri, 30 Jul 2021 10:30:33 +0200
Subject: [PATCH 320/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 181c4139..9fc10b20 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)
 	* Core: Fix logging bug (Dennis Camera)
 	* Build: Improve Makefile compatibility (Evilham)
+	* Type __filesystem: Support ubuntu (Joachim Desroches)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 53334fb4eb311550af7f5b73f279a2e86fa1c504 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 4 Aug 2021 19:50:10 +0200
Subject: [PATCH 321/366] [explorer/os_version] Fix for FreeBSD < 10.0 (again)

---
 cdist/conf/explorer/os_version | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 6c94915c..cc976608 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -89,7 +89,14 @@ in
    freebsd)
       # Apparently uname -r is not a reliable way to get the patch level.
       # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
-      freebsd-version
+      if command -v freebsd-version >/dev/null 2>&1
+      then
+         # get userland version
+         freebsd-version -u
+      else
+         # fallback to kernel release for FreeBSD < 10.0
+         uname -r
+      fi
    ;;
    *bsd|solaris)
       uname -r

From e108cbc205cb5e7ac0d2e07b82cef4c83eaa285f Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Tue, 3 Aug 2021 13:20:43 +0200
Subject: [PATCH 322/366] [explorer/os_version] Ubuntu: fall back to
 os-release/lsb-release files

---
 cdist/conf/explorer/os_version | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 6c94915c..e70fe7f4 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -21,6 +21,17 @@
 # All os variables are lower case
 #
 
+rc_getvar() {
+   awk -F= -v varname="$2" '
+      function unquote(s) {
+         if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/)
+            return substr(s, 2, length(s) - 2)
+         else
+            return s
+      }
+      $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1"
+}
+
 case $("${__explorer:?}/os")
 in
    amazon)
@@ -114,7 +125,20 @@ in
       fi
    ;;
    ubuntu)
-      lsb_release -sr
+      if command -v lsb_release >/dev/null 2>&1
+      then
+         lsb_release -sr
+      elif test -r /usr/lib/os-release
+      then
+         # fallback to /usr/lib/os-release if lsb_release is not present (like
+         # on minimized Ubuntu installations)
+         rc_getvar /usr/lib/os-release VERSION_ID
+      elif test -r /etc/lsb-release
+      then
+         # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old
+         # versions without /usr/lib/os-release.
+         rc_getvar /etc/lsb-release DISTRIB_RELEASE
+      fi
    ;;
    alpine)
        cat /etc/alpine-release

From 83fe6e9f5b2537db73d5c6a142b6d24eef75ac58 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Tue, 3 Aug 2021 19:26:55 +0200
Subject: [PATCH 323/366] [explorer/memory] Fix conversion of large numbers (>=
 2GiB)

At least mawk uses scientific notation when using print for
numbers >=2^31 (INT_MAX of a signed 32-bit int).

`printf "%.f\n"` works around this.
---
 cdist/conf/explorer/memory | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory
index 63aba9c6..c6d113cf 100755
--- a/cdist/conf/explorer/memory
+++ b/cdist/conf/explorer/memory
@@ -27,19 +27,18 @@
 str2bytes() {
 	awk -F' ' '
 	$2 ==   "B" || !$2 { print $1 }
-	$2 ==  "kB" { print $1 * 1000 }
-	$2 ==  "MB" { print $1 * 1000 * 1000 }
-	$2 ==  "GB" { print $1 * 1000 * 1000 * 1000 }
-	$2 ==  "TB" { print $1 * 1000 * 1000 * 1000 * 1000 }
-	$2 == "kiB" { print $1 * 1024 }
-	$2 == "MiB" { print $1 * 1024 * 1024 }
-	$2 == "GiB" { print $1 * 1024 * 1024 * 1024 }
-	$2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }'
+	$2 ==  "kB" { printf "%.f\n", ($1 * 1000) }
+	$2 ==  "MB" { printf "%.f\n", ($1 * 1000 * 1000) }
+	$2 ==  "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) }
+	$2 ==  "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) }
+	$2 == "kiB" { printf "%.f\n", ($1 * 1024) }
+	$2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) }
+	$2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) }
+	$2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }'
 }
 
 bytes2kib() {
-	set -- "$(cat)"
-	test "$1" -gt 0 && echo $(($1 / 1024))
+	awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }'
 }
 
 

From a7d6481a7ddc7cb72b1a55bfca7fdfed20514a62 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Mon, 2 Aug 2021 21:23:50 +0200
Subject: [PATCH 324/366] [type/__update_alternatives] Secure cdist-defined
 environment variables with :?

---
 .../__update_alternatives/explorer/alternatives     |  2 +-
 cdist/conf/type/__update_alternatives/explorer/link |  6 +++---
 .../type/__update_alternatives/explorer/path_is     |  4 ++--
 .../explorer/path_should_state                      |  2 +-
 .../conf/type/__update_alternatives/gencode-remote  | 13 ++++++-------
 5 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives
index 34aaca56..ecc62f4b 100755
--- a/cdist/conf/type/__update_alternatives/explorer/alternatives
+++ b/cdist/conf/type/__update_alternatives/explorer/alternatives
@@ -1,4 +1,4 @@
 #!/bin/sh -e
 
-update-alternatives --display "$__object_id" 2>/dev/null \
+update-alternatives --display "${__object_id:?}" 2>/dev/null \
     | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link
index 6519e7c2..c6fd1c98 100755
--- a/cdist/conf/type/__update_alternatives/explorer/link
+++ b/cdist/conf/type/__update_alternatives/explorer/link
@@ -18,12 +18,12 @@ for altdir in \
     /var/lib/dpkg/alternatives \
     /var/lib/alternatives
 do
-    if [ ! -f "$altdir/$__object_id" ]
+    if [ ! -f "$altdir/${__object_id:?}" ]
     then
         continue
     fi
 
-    link="$( awk 'NR==2' "$altdir/$__object_id" )"
+    link="$( awk 'NR==2' "$altdir/${__object_id:?}" )"
 
     if [ -n "$link" ]
     then
@@ -33,7 +33,7 @@ done
 
 if [ -z "$link" ]
 then
-    echo "unable to get link for $__object_id" >&2
+    echo "unable to get link for ${__object_id:?}" >&2
     exit 1
 fi
 
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
index fc304d5d..a24bd40e 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_is
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -1,11 +1,11 @@
 #!/bin/sh -e
 
-path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \
+path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \
     | awk '/link currently points to/ {print $5}' )"
 
 if [ -z "$path_is" ]
 then
-    echo "unable to get current path for $__object_id" >&2
+    echo "unable to get current path for ${__object_id:?}" >&2
     exit 1
 fi
 
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state
index 59e015c5..b74a7ee8 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_should_state
+++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state
@@ -1,6 +1,6 @@
 #!/bin/sh -e
 
-if [ -f "$( cat "$__object/parameter/path" )" ]
+if [ -f "$( cat "${__object:?}/parameter/path" )" ]
 then
     echo 'present'
 else
diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index e393cdef..13666805 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -18,26 +18,25 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 
-path_is="$( cat "$__object/explorer/path_is" )"
+path_is="$( cat "${__object:?}/explorer/path_is" )"
 
-path_should="$( cat "$__object/parameter/path" )"
+path_should="$( cat "${__object:?}/parameter/path" )"
 
 if [ "$path_is" = "$path_should" ]
 then
     exit 0
 fi
 
-if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ]
+if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \
+    && [ -z "${__cdist_dry_run+dry run}" ]
 then
     echo "$path_should does not exist in target" >&2
     exit 1
 fi
 
-name="$__object_id"
+name=${__object_id:?}
 
-alternatives="$( cat "$__object/explorer/alternatives" )"
-
-if ! echo "$alternatives" | grep -Fxq "$path_should"
+if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives"
 then
     if [ ! -f "$__object/parameter/install" ]
     then

From 0b3b47396f2aafa377e3d5d9a13f51ace2303d41 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Mon, 2 Aug 2021 21:25:08 +0200
Subject: [PATCH 325/366] [type/__update_alternatives] dry-run fixes

---
 cdist/conf/type/__update_alternatives/explorer/link |  5 ++++-
 .../type/__update_alternatives/explorer/path_is     |  5 ++++-
 .../conf/type/__update_alternatives/gencode-remote  | 13 ++++++++-----
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link
index c6fd1c98..d1087c75 100755
--- a/cdist/conf/type/__update_alternatives/explorer/link
+++ b/cdist/conf/type/__update_alternatives/explorer/link
@@ -31,8 +31,11 @@ do
     fi
 done
 
-if [ -z "$link" ]
+if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then
+    # NOTE: ignore error for dry-runs because a package providing the link
+    #       might be managed by another cdist object (which wasn't executed,
+    #       because dry run…).
     echo "unable to get link for ${__object_id:?}" >&2
     exit 1
 fi
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
index a24bd40e..9208df7b 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_is
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -3,8 +3,11 @@
 path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \
     | awk '/link currently points to/ {print $5}' )"
 
-if [ -z "$path_is" ]
+if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then
+    # NOTE: ignore error for dry-runs because a package providing the
+    #       alternative might be managed by another cdist object (which
+    #       wasn't executed, because dry run…).
     echo "unable to get current path for ${__object_id:?}" >&2
     exit 1
 fi
diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote
index 13666805..e91ea78f 100755
--- a/cdist/conf/type/__update_alternatives/gencode-remote
+++ b/cdist/conf/type/__update_alternatives/gencode-remote
@@ -38,16 +38,19 @@ name=${__object_id:?}
 
 if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives"
 then
-    if [ ! -f "$__object/parameter/install" ]
+    if [ -f "${__object:?}/parameter/install" ]
     then
+        link="$( cat "${__object:?}/explorer/link" )"
+        echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
+    elif [ -z "${__cdist_dry_run+dry run}" ]
+    then
+        # NOTE: ignore error for dry-runs because a package providing the link
+        #       to be installed might be managed by another cdist object (which
+        #       wasn't executed, because dry run…).
         echo "$path_should is not in $name alternatives." >&2
         echo 'Please install missing packages or use --install to add path to alternatives.' >&2
         exit 1
     fi
-
-    link="$( cat "$__object/explorer/link" )"
-
-    echo "update-alternatives --install '$link' '$name' '$path_should' 1000"
 fi
 
 echo "update-alternatives --set '$name' '$path_should'"

From bbcc81a9841f2619e1b9e13b25a941337489a681 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 4 Aug 2021 21:44:04 +0200
Subject: [PATCH 326/366] [type/__update_alternatives] Fix for non-English
 locales

Since update-alternatives(1) is localized, screen scraping its output breaks
if the locale is set to non-English.
---
 cdist/conf/type/__update_alternatives/explorer/alternatives | 4 ++--
 cdist/conf/type/__update_alternatives/explorer/path_is      | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives
index ecc62f4b..bb1619a9 100755
--- a/cdist/conf/type/__update_alternatives/explorer/alternatives
+++ b/cdist/conf/type/__update_alternatives/explorer/alternatives
@@ -1,4 +1,4 @@
 #!/bin/sh -e
 
-update-alternatives --display "${__object_id:?}" 2>/dev/null \
-    | awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
+LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \
+| awk -F ' - ' '/priority [0-9]+$/ { print $1 }'
diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is
index 9208df7b..5cf4fa4b 100755
--- a/cdist/conf/type/__update_alternatives/explorer/path_is
+++ b/cdist/conf/type/__update_alternatives/explorer/path_is
@@ -1,7 +1,8 @@
 #!/bin/sh -e
 
-path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \
-    | awk '/link currently points to/ {print $5}' )"
+path_is=$(
+    LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \
+    | awk '/link currently points to/ { print $5 }')
 
 if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ]
 then

From 2a0c073d4021206c7459015cefaba218004235ce Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 4 Aug 2021 21:54:17 +0200
Subject: [PATCH 327/366] [explorer/os_version] Fix for legacy Mac OS X
 versions

---
 cdist/conf/explorer/os_version | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 3b02dedd..96eca1ee 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -68,7 +68,8 @@ case "$("$__explorer/os")" in
       cat /etc/gentoo-release
    ;;
    macosx)
-      sw_vers -productVersion
+      # NOTE: Legacy versions (< 10.3) do not support options
+      sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }'
    ;;
    freebsd)
       # Apparently uname -r is not a reliable way to get the patch level.

From 3ae5a606ca7182ec7fe13134670400506e198ec1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Thu, 5 Aug 2021 10:27:51 +0200
Subject: [PATCH 328/366] ++changelog

---
 docs/changelog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 9fc10b20..f9409d7e 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,10 @@ next:
 	* Core: Fix logging bug (Dennis Camera)
 	* Build: Improve Makefile compatibility (Evilham)
 	* Type __filesystem: Support ubuntu (Joachim Desroches)
+	* Explorer os_version: Fall back to os-release/lsb-release file on Ubuntu (Dennis Camera)
+	* Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera)
+	* Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera)
+	* Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From edcac70b2a99ce532dd5bca9a0b8fc5bf6dc5148 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Wed, 21 Jul 2021 13:22:34 +0200
Subject: [PATCH 329/366] [explorer/machine_type] Reimplement

---
 cdist/conf/explorer/machine_type | 986 ++++++++++++++++++++++++++++---
 1 file changed, 904 insertions(+), 82 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 1c84f4d7..29f98849 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -1,8 +1,6 @@
-#!/bin/sh
+#!/bin/sh -e
 #
-# 2014 Daniel Heule  (hda at sfs.biz)
-# 2014 Thomas Oettli (otho at sfs.biz)
-# 2020 Evilham (contact at evilham.com)
+# 2021 Dennis Camera (cdist at dtnr.ch)
 #
 # This file is part of cdist.
 #
@@ -19,91 +17,915 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
+# This explorer tries to determine what type of machine the target to be
+# configured is (container, virtual machine, bare-metal).
+#
+# It will print one line for each layer it can detect.
+# The format of all lines is: TYPE[ VERB VENDOR]
+#
+# VERB does not have a special meaning, it is just for better readability.
+#
+# e.g.
+# container on lxc
+# virtual by kvm-spapr
+#
+# The third word of each line can be composed of different parts concatenated with a `-'
+# (minus) character, with each component being a specification of the previous,
+# e.g.:
+#  - lxc-libvirt             (LXC container, managed by libvirt)
+#  - lpar-s390 / lpar-power  (LPAR running on IBM S/390 or POWER, respectively)
+#  - xen-hvm / xen-pv        (Xen HVM vs para-virtualization)
+#
+# If this explorer cannot determine any information it will print nothing.
+#
 
-os=$("$__explorer/os")
+# Add /sbin and /usr/sbin to the path so we can find system
+# binaries like dmidecode.
+PATH=$(getconf PATH 2>/dev/null) || PATH='/usr/bin:/bin'
+PATH="/sbin:/usr/sbin:${PATH}"
+export PATH
 
-vendor_string_to_machine_type() {
-    for vendor in vmware bochs kvm qemu virtualbox bhyve; do
-        if echo "${1}" | grep -q -i "${vendor}"; then
-            if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then
-                vendor="kvm"
-            fi
-            echo "virtual_by_${vendor}"
-            exit
-        fi
-    done
+arch=$(uname -m | sed -e 's/i.86/i386/' -e 's/arm.*/arm/')
+uname_s=$(uname -s)
+
+
+is_command() { command -v "$1" >/dev/null 2>&1; }
+
+is_oneof() (
+	x=$1; shift
+	for y
+	do
+		test "${x}" = "${y}" || continue
+		return 0
+	done
+	return 1
+)
+
+tolower() { LC_ALL=C tr '[:upper:]' '[:lower:]'; }
+
+# shellcheck disable=SC2086
+glob_exists() { set -- $1; test -e "$1"; }
+
+get_dmi_field() {
+	if is_oneof "${uname_s}" NetBSD
+	then
+		case $1
+		in
+			(system-manufacturer) _mib=machdep.dmi.system-vendor ;;
+			(system-product-name) _mib=machdep.dmi.system-product ;;
+			(system-version|system-uuid) _mib=machdep.dmi.$1 ;;
+			(bios-vendor|bios-version) _mib=machdep.dmi.$1 ;;
+			(biod-release-date) _mib=machdep.dmi.bios-date ;;
+			(*) _mib= ;;
+		esac
+
+		test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return
+	fi
+
+	if is_command dmidecode
+	then
+		dmidecode -s "$1"
+	elif test -d "${dmi_sysfs-}"
+	then
+		case $1
+		in
+			(system-manufacturer) _filename=sys_vendor ;;
+			(system-product-name) _filename=product_name ;;
+			(*) _filename=$(echo "$1" | tr - _) ;;
+		esac
+		if test -r "${dmi_sysfs-}/${_filename}"
+		then
+			cat "${dmi_sysfs}/${_filename}"
+		fi
+		unset _filename
+	elif test "${uname_s}" = OpenBSD
+	then
+		# NOTE: something similar to system-manufacutrer and system-product-name
+		#       is available on OpenBSD in sysctl
+		case $1
+		in
+			(system-manufacturer) _mib=hw.vendor ;;
+			(system-product-name) _mib=hw.product ;;
+			(*) _mib= ;;
+		esac
+
+		test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return
+	fi
+
+	return 1
 }
 
-case "$os" in
-    "freebsd")
-        # FreeBSD does not have /proc/cpuinfo even when procfs is used.
-        # Instead there is a sysctl kern.vm_guest.
-        # Which is 'none' if physical, else the virtualisation.
-        vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)"
-        if [ -n "${vm_guest}" ]; then
-            if [ "${vm_guest}" = "none" ]; then
-                echo "physical"
-                exit
-            fi
-            echo "virtual_by_${vm_guest}"
-            exit
-        fi
-    ;;
+has_cpuinfo() { test -e /proc/cpuinfo; }
 
-    "openbsd")
-        # OpenBSD can also use the sysctl's: hw.vendor or hw.product.
-        # Note we can be reasonably sure about a machine being virtualised
-        # as long as we can identify the virtualisation technology.
-        # But not so much about it being physical...
-        # Patches are welcome / reach out if you have better ideas.
-        for sysctl in hw.vendor hw.product; do
-            # This exits if we can make a reasonable judgement
-            vendor_string_to_machine_type "$(sysctl -n "${sysctl}")"
-        done
-    ;;
+get_sysctl() {
+	is_command sysctl && sysctl -n "$1" 2>/dev/null
+}
 
-    *)
-        # Defaulting to linux for compatibility with previous cdist behaviour
 
-        if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then
-            echo openvz
-            exit
-        fi
+# Check for container
 
-        if [ -e "/proc/1/environ" ] &&
-            tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then
-            echo lxc
-            exit
-        fi
+has_ct_pid_1() {
+	test -r /run/systemd/container -o -r /proc/1/environ
+}
 
-        if [ -r /proc/cpuinfo ]; then
-            # this should only exist on virtual guest machines,
-            # tested on vmware, xen, kvm, bhyve
-            if grep -q "hypervisor" /proc/cpuinfo; then
-                # this file is aviable in xen guest systems
-                if [ -r /sys/hypervisor/type ]; then
-                    if grep -q -i "xen" /sys/hypervisor/type; then
-                        echo virtual_by_xen
-                        exit
-                    fi
-                else
-                    for vendor_file in /sys/class/dmi/id/product_name \
-                                       /sys/class/dmi/id/sys_vendor \
-                                       /sys/class/dmi/id/chasis_vendor; do
-                        if [ -r ${vendor_file} ]; then
-                            # This exits if we can make a reasonable judgement
-                            vendor_string_to_machine_type "$(cat "${vendor_file}")"
-                        fi
-                    done
-                fi
-                echo "virtual_by_unknown"
-                exit
-            else
-                echo "physical"
-                exit
-            fi
-        fi
-    ;;
-esac
+translate_container_name() {
+	case $1
+	in
+		('lxc')
+			echo lxc ;;
+		('lxc-libvirt')
+			echo lxc-libvirt ;;
+		('podman')
+			echo podman ;;
+		('systemd-nspawn')
+			echo systemd_nspawn ;;
+		(*)
+			return 1 ;;
+	esac
+	return 0
+}
 
-echo "unknown"
+check_ct_pid_1() {
+	if test -r /run/systemd/container
+	then
+		translate_container_name "$(head -n1 /run/systemd/container)" \
+		&& return 0
+	fi
+
+	if test -r /proc/1/environ
+	then
+		translate_container_name "$(
+			LC_ALL=C tr '\000' '\n' </proc/1/environ \
+			| sed -n -e 's/^container=//p')" \
+		&& return 0
+	fi
+
+	return 1
+}
+
+has_ct_cgroup() {
+	test -r /proc/self/cgroup
+}
+
+check_ct_cgroup() {
+	if grep -q -e ':/docker/' /proc/self/cgroup
+	then
+		echo docker
+	elif grep -q -e ':/lxc/.*\.libvirt-lxc$' /proc/self/cgroup
+	then
+		echo libvirt-lxc
+	elif grep -q -F '/libpod-' /proc/self/cgroup
+	then
+		echo podman
+	else
+		return 1
+	fi
+}
+
+check_ct_files() {
+	if test -e /.dockerenv -o -e /run/.dockerenv -o -e /.dockerinit
+	then
+		echo docker
+	elif test -f /run/.containerenv
+	then
+		# https://github.com/containers/podman/issues/3586#issuecomment-661918679
+		echo podman
+	elif test -d /proc/vz -a ! -d /proc/bc -a -f /proc/user_beancounters \
+		|| test -d /.cpt_hardlink_dir_*/
+	then
+		# Virtuozzo / OpenVZ
+		#
+		# /proc/vz: always exists if OpenVZ kernel is running
+		# /proc/bc: exists only on node (not inside container)
+		# /proc/user_beancounters - exists inside container
+		#
+		# /.cpt_hardlink_dir_*/ may be (?) created by vzctl:
+		# https://github.com/kolyshkin/vzctl/commit/09e974f
+
+		echo openvz
+	elif grep -qie 'Microsoft\|WSL' /proc/sys/kernel/osrelease /proc/version 2>/dev/null
+	then
+		# https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364
+		echo wsl
+	elif test -d /var/.cagefs
+	then
+		# https://docs.cloudlinux.com/cloudlinux_os_components/#cagefs
+		# CageFS is not "really" a container, but it isn't a chroot either.
+		echo cagefs
+	elif test -e /proc/self/status && grep -q -e '^VxID: [0-9]\{1,\}' /proc/self/status
+	then
+		# Linux-VServer
+		if grep -q -x -F 'VxID: 0' /proc/self/status
+		then
+			# host
+			return 1
+		else
+			# guest
+			echo linux_vserver
+		fi
+	else
+		return 1
+	fi
+}
+
+check_ct_os_specific() (
+	if jailed=$(get_sysctl security.jail.jailed) && test "${jailed}" = 1
+	then
+		# FreeBSD jail
+		echo jail
+		return 0
+	fi
+
+	if is_command zonename && test "$(zonename)" != global
+	then
+		# Solaris zone
+		echo zone
+		return 0
+	fi
+
+	return 1
+)
+
+
+# Check for hypervisor
+
+guess_hypervisor_from_cpu_model() {
+	case $1
+	in
+		(*\ KVM\ *)
+			echo kvm ;;
+		(*\ QEMU\ *|QEMU\ *)
+			echo qemu ;;
+		(*)
+			return 1 ;;
+	esac
+}
+
+has_vm_cpuinfo() { has_cpuinfo; }
+
+check_vm_cpuinfo() {
+	if grep -q -F 'User Mode Linux' /proc/cpuinfo \
+		|| grep -q -F 'UML' /proc/cpuinfo
+	then
+		# User Mode Linux
+		echo uml
+	elif grep -q -e '^vendor_id.*: PowerVM Lx86' /proc/cpuinfo
+	then
+		# IBM PowerVM Lx86 (Linux/x86 emulator)
+		echo powervm_lx86
+	elif grep -q -e '^vendor_id.*: IBM/S390' /proc/cpuinfo
+	then
+		# IBM SystemZ (S/390)
+		if test -f /proc/sysinfo
+		then
+			if grep -q -e '^VM[0-9]* Control Program: KVM/Linux' /proc/sysinfo
+			then
+				echo kvm-s390
+				return 0
+			elif grep -q -e '^VM[0-9]* Control Program: z/VM' /proc/sysinfo
+			then
+				echo zvm
+				return 0
+			elif grep -q -e '^LPAR ' /proc/sysinfo
+			then
+				echo zvm-lpar
+				return 0
+			fi
+		fi
+		return 1
+	else
+		if grep -q -e '^model name.*:' /proc/cpuinfo
+		then
+			sed -n -e 's/^model name[^:]*: *//p' /proc/cpuinfo \
+			| while read -r _cpu_model
+			  do
+				  guess_hypervisor_from_cpu_model "${_cpu_model}"
+			  done \
+			| sort \
+			| uniq -c \
+			| awk '
+			  { if ($1 > most_c) { most_c = $1; most_s = $2 } }
+			  END {
+				if (most_s) print most_s
+				exit !most_s
+			  }' \
+			&& return 0
+		fi
+		return 1
+	fi
+}
+
+check_vm_arch_specific() {
+	case ${arch}
+	in
+		(ppc64|ppc64le)
+			# Check PPC64 LPAR, KVM
+
+			# example /proc/cpuinfo line indicating 'not baremetal'
+			# platform	: pSeries
+			#
+			# example /proc/ppc64/lparcfg systemtype line
+			# system_type=IBM pSeries (emulated by qemu)
+
+			if has_cpuinfo && grep -q -e 'platform.**pSeries' /proc/cpuinfo
+			then
+				if test -e /proc/ppc64/lparcfg
+				then
+					# Assume LPAR, now detect shared or dedicated
+					if grep -q -x -F 'shared_processor_mode=1' /proc/ppc64/lparcfg
+					then
+						echo powervm-shared
+						return 0
+					else
+						echo powervm-dedicated
+						return 0
+					fi
+				fi
+			fi
+			;;
+		(sparc*)
+			# Check for SPARC LDoms
+
+			if test -e /dev/mdesc
+			then
+				if test -d /sys/class/vlds/ctrl -a -d /sys/class/vlds/sp
+				then
+					# control LDom
+					return 1
+				else
+					# guest LDom
+					echo ldom-sparc
+				fi
+
+				# MDPROP=/usr/lib/ldoms/mdprop.py
+				# if test -x "${MDPROP}"
+				# then
+				# 	if test -n "$("${MDPROP}" -v iodevice device-type=pciex)"
+				# 	then
+				# 		echo ldoms-root
+				# 		echo ldoms-io
+				# 	elif test -n "$("${MDPROP}" -v iov-device vf-id=0)"
+				# 	then
+				# 		echo ldoms-io
+				# 	fi
+				# fi
+				return 0
+			fi
+			;;
+		(i?86|x86*|amd64|i86pc)
+			# Check CPUID
+			#
+			# Many fullvirt hypervisors give an indication through CPUID.  Use
+			# the virt-what helper program to get this information if available.
+
+			for CPUID_HELPER in \
+				$(command -v virt-what-cpuid-helper 2>/dev/null) \
+				/usr/lib/x86_64-*/virt-what-cpuid-helper \
+				/usr/lib/i?86-*/virt-what-cpuid-helper \
+				/usr/lib/virt-what/virt-what-cpuid-helper
+			do
+				if test -x "${CPUID_HELPER:?}"; then break; fi
+			done
+
+			if test -x "${CPUID_HELPER-}"
+			then
+				case $(command "${CPUID_HELPER}")
+				in
+					('bhyve bhyve ')
+						echo bhyve
+						;;
+					('LKVMLKVMLKVM')
+						echo lkvm
+						;;
+					('KVMKVMKVM')
+						echo kvm
+						;;
+					('TCGTCGTCGTCG')
+						echo qemu-tcg
+						;;
+					('Microsoft Hv')
+						# http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx
+						echo hyperv
+						;;
+					('OpenBSDVMM58')
+						# OpenBSD/VMM
+						echo openbsd_vmm
+						;;
+					('VMwareVMware')
+						# check added by Chetan Loke.
+						echo vmware
+						;;
+					('XenVMMXenVMM')
+						if has dmi
+						then
+							# https://access.redhat.com/solutions/222903
+							echo xen-hvm
+						else
+							echo xen-paravirt
+						fi
+						;;
+					(*)
+						return 1 ;;
+				esac
+				return 0
+			fi
+
+			unset CPUID_HELPER
+
+			# VMM CPUID flag denotes that this system is running under a VMM
+			if is_oneof "${uname_s}" Darwin
+			then
+				get_sysctl machdep.cpu.features | tr ' ' '\n' | grep -qixF VMM \
+				&& return 0
+			fi
+			if has_cpuinfo \
+				&& grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo
+			then
+				return 0
+			fi
+			;;
+		(ia64)
+			if test -d /sys/bus/xen -a ! -d /sys/bus/xen-backend
+			then
+				# PV-on-HVM drivers installed in a Xen guest
+				echo xen-hvm
+				return 0
+			fi
+			;;
+	esac
+	return 1
+}
+
+has_vm_dmi() {
+	# Check for various products in SMBIOS/DMI.
+	# Note that DMI doesn't exist on all architectures (only x86 and some ARM).
+	# On other architectures the $dmi variable will be empty.
+
+	if test -d /sys/class/dmi/id/
+	then
+		dmi_sysfs=/sys/class/dmi/id
+	elif test -d /sys/devices/virtual/dmi/id/
+	then
+		dmi_sysfs=/sys/devices/virtual/dmi/id
+	fi
+
+	# shellcheck disable=SC2015
+	{
+		is_command dmidecode \
+		&& (
+			# dmidecode needs to exit 0 and not print the No SMBIOS/DMI line
+			dmi_out=$(dmidecode 2>&1) \
+			&& ! printf '%s\n' "${dmi_out}" \
+				 | grep -qF 'No SMBIOS nor DMI entry point found, sorry.'
+			) \
+		|| test -d "${dmi_sysfs}"
+	}
+}
+
+check_vm_dmi() {
+	case $(get_dmi_field system-product-name)
+	in
+		(*.metal)
+			if test "$(get_dmi_field system-manufacturer)" = 'Amazon EC2'
+			then
+				# AWS EC2 bare metal -> no virtualisation
+				return 1
+			fi
+			;;
+		('BHYVE')
+			echo bhyve
+			return 0
+			;;
+		('Google Compute Engine')
+			echo gce
+			return 0
+			;;
+		('RHEV Hypervisor')
+			# Red Hat Enterprise Virtualization
+			echo rhev
+			return 0
+			;;
+		('KVM'|'Bochs'|'KVM Virtual Machine')
+			echo kvm
+			return 0
+			;;
+		('Parallels Virtual Platform')
+			echo parallels
+			return 0
+			;;
+		('VirtualBox')
+			echo virtualbox
+			return 0
+			;;
+		('VMware Virtual Platform')
+			echo vmware
+			return 0
+			;;
+	esac
+
+	case $(get_dmi_field system-manufacturer)
+	in
+		('Alibaba'*)
+			case $(get_dmi_field system-product-name)
+			in
+				('Alibaba Cloud ECS')
+					echo alibaba-ecs
+					;;
+				(*)
+					echo alibaba
+					;;
+			esac
+			return 0
+			;;
+		('Amazon EC2')
+			# AWS on bare-metal or KVM
+			echo aws-ec2
+			return 0
+			;;
+		('innotek GmbH'|'Oracle Corporation')
+			echo virtualbox
+			return 0
+			;;
+		('Joyent')
+			if test "$(get_dmi_field system-product-name)" = 'SmartDC HVM'
+			then
+				# SmartOS KVM
+				echo kvm-smartdc_hvm
+				return 0
+			fi
+			;;
+		('Microsoft Corporation'*)
+			if test "$(get_dmi_field system-product-name)" = 'Virtual Machine'
+			then
+				if test -e /proc/irq/7/hyperv \
+				|| expr "$(get_dmi_field bios-version)" : 'VRTUAL.*' >/dev/null
+				then
+					echo hyperv
+					return 0
+				fi
+
+				case $(get_dmi_field system-version)
+				in
+					(VPC[0-9]*|VS2005*|*[Vv]irtual*[Pp][Cc]*)
+						echo virtualpc
+						return 0
+						;;
+					(*)
+						echo hyperv
+						return 0
+						;;
+				esac
+			fi
+			;;
+		('Nutanix')
+			# Nutanix AHV. Similar to KVM.
+			if test "$(get_dmi_field system-product-name)" = 'AHV'
+			then
+				echo nutanix_ahv
+				return 0
+			fi
+			;;
+		('oVirt')
+			echo ovirt
+			return 0
+			;;
+		('Parallels Software International Inc.')
+			echo parallels
+			return 0
+			;;
+		('QEMU')
+			echo qemu
+			return 0
+			;;
+		('VMware, Inc.')
+			echo vmware
+			return 0
+			;;
+	esac
+
+	case $(get_dmi_field bios-vendor)
+	in
+		('Amazon EC2')
+			# AWS on bare-metal or KVM
+			echo aws-ec2
+			return 0
+			;;
+		('BHYVE')
+			echo bhyve
+			return 0
+			;;
+		('innotek GmbH')
+			echo virtualbox
+			return 0
+			;;
+		('Parallels Software International Inc.')
+			echo parallels
+			return 0
+			;;
+		('Xen')
+			if get_dmi_field bios-version | grep -q -e '\([0-9]\{1,\}\.\)\{2\}amazon'
+			then
+				# AWS on Xen
+				echo aws-xen
+				return 0
+			fi
+			;;
+	esac
+
+	return 1
+}
+
+check_vm_hyp_specific() {
+	if is_command vmware-checkvm && vmware-checkvm >/dev/null
+	then
+		# vmware-checkvm is provided by VMware's open-vm-tools
+		echo vmware
+		return 0
+	elif test -d /proc/xen
+	then
+		test -r /proc/xen/capabilities &&
+		if grep -q -F 'control_d' /proc/xen/capabilities 2>/dev/null
+		then
+			# Xen dom0
+			return 1
+		else
+			# Xen domU
+			echo xen
+			return 0
+		fi
+	fi
+	return 1
+}
+
+has_vm_dt() {
+	# OpenFirmware/Das U-Boot device-tree
+	test -d /proc/device-tree
+}
+
+check_vm_dt() {
+	case ${arch}
+	in
+		(arm|aarch64)
+			if test -r /proc/device-tree/hypervisor/compatible
+			then
+				if grep -q -F 'xen' /proc/device-tree/hypervisor/compatible
+				then
+					echo xen
+					return 0
+				elif grep -q -F 'vmware' /proc/device-tree/hypervisor/compatible
+				then
+					# e.g. VMware ESXi on ARM
+					echo vmware
+					return 0
+				fi
+			fi
+			if glob_exists /proc/device-tree/fw-cfg@*/compatible
+			then
+				# qemu,fw-cfg-mmio
+				sed -e 's/,.*$//' /proc/device-tree/fw-cfg@*/compatible | head -n1
+				return 0
+			fi
+			if grep -q -F 'dummy-virt' /proc/device-tree/compatible
+			then
+				echo lkvm
+				return 0
+			fi
+			;;
+		(ppc64*)
+			if test -d /proc/device-tree/hypervisor \
+				&& grep -qF 'linux,kvm' /proc/device-tree/hypervisor/compatible
+			then
+				# We are running as a spapr KVM guest on ppc64
+				echo kvm-spapr
+				return 0
+			fi
+			if test -r /proc/device-tree/ibm,partition-name \
+				&& test -r /proc/device-tree/hmc-managed\? \
+				&& test -r /proc/device-tree/chosen/qemu,graphic-width
+			then
+				echo powervm
+			fi
+			;;
+	esac
+	return 1
+}
+
+has_vm_sys_hypervisor() {
+	test -d /sys/hypervisor/
+}
+
+check_vm_sys_hypervisor() {
+	test -r /sys/hypervisor/type &&
+	case $(head -n1 /sys/hypervisor/type)
+	in
+		(xen)
+			# Ordinary kernel with pv_ops.	There does not seem to be
+			# enough information at present to tell whether this is dom0
+			# or domU.
+			echo xen
+			return 0
+			;;
+	esac
+	return 1
+}
+
+check_vm_os_specific() {
+	_hyp_generic=false
+
+	case ${uname_s}
+	in
+		(Darwin)
+			if hv_vmm_present=$(get_sysctl kern.hv_vmm_present) \
+				&& test "${hv_vmm_present}" -ne 0
+			then
+				_hyp_generic=true
+			fi
+			;;
+		(FreeBSD)
+			# FreeBSD does not have /proc/cpuinfo even when procfs is used.
+			# Instead there is a sysctl kern.vm_guest.
+			# Which is 'none' if physical, else the virtualisation.
+			vm_guest=$(get_sysctl kern.vm_guest | tolower) &&
+			case ${vm_guest}
+			in
+				(none) ;;
+				(generic) _hyp_generic=true ;;
+				(*)
+					# kernel could detect hypervisor
+					case ${vm_guest}
+					in
+						(hv) echo hyperv ;;
+						(vbox) echo virtualbox ;;
+						(*) echo "${vm_guest}" ;;
+					esac
+					return 0
+					;;
+			esac
+			;;
+		(NetBSD)
+			machdep_hv=$(get_sysctl machdep.hypervisor | tolower) &&
+			case ${machdep_hv}
+			in
+				(none) ;;
+				(generic) _hyp_generic=true ;;
+				(*)
+					# kernel could detect hypervisor
+					case ${machdep_hv}
+					in
+						(hyper-v) echo hyperv ;;
+						(xenhvm*) echo xen-hvm ;;
+						(xenpv*) echo xen-pv ;;
+						(xen*) echo xen ;;
+						(*) echo "${machdep_hv}" ;;
+					esac
+					return 0
+					;;
+			esac
+			;;
+		(OpenBSD)
+			if is_command hostctl && glob_exists /dev/pvbus[0-9]*
+			then
+				for _pvbus in /dev/pvbus[0-9]*
+				do
+					_h_out=$(hostctl -f "${_pvbus}" -t 2>/dev/null) || continue
+					case $(expr "${_h_out}" : '[^:]*: *\(.*\)$')
+					in
+						(KVM) echo kvm ;;
+						(Hyper-V) echo hyperv ;;
+						(VMware) echo vmware ;;
+						(Xen) echo xen ;;
+						(bhyve) echo bhyve ;;
+						(OpenBSD) echo openbsd_vmm ;;
+					esac
+					return 0
+				done
+			fi
+			;;
+		(SunOS)
+			diag_conf=$(prtdiag | sed -n -e 's/.*Configuration: *//p' -e '/^$/q')
+			# NOTE: Don't use -e or -F in Solaris grep
+			if printf '%s\n' "${diag_conf}" | grep -q -i QEMU
+			then
+				echo qemu
+				return 0
+			elif printf '%s\n' "${diag_conf}" | grep -q -i VMware
+			then
+				echo vmware
+				return 0
+			fi
+			;;
+		(Linux)
+			if is_command dmesg
+			then
+				while read -r line
+				do
+					case ${line}
+					in
+						('Booting paravirtualized kernel on ')
+							case $(expr "${line}" : '.* kernel on \(.*\)')
+							in
+								('Xen')
+									echo xen-pv; return 0 ;;
+								('bare hardware')
+									return 1 ;;
+							esac
+							;;
+						('Hypervisor detected')
+							case $(expr "${line}" : '.*: *\(.*\)')
+							in
+								('ACRN')
+									echo acrn ;;
+								('Jailhouse')
+									echo jailhouse ;;
+								('KVM')
+									echo kvm ;;
+								('Microsoft Hyper-V')
+									echo hyperv ;;
+								('VMware')
+									echo vmware ;;
+								('Xen HVM')
+									echo xen-hvm ;;
+								('Xen PV')
+									echo xen-pv ;;
+							esac
+							return 0
+							;;
+						(lpar:*' under hypervisor')
+							return 0 ;;
+					esac
+				done <<-EOF
+				$(dmesg 2>/dev/null | awk '
+				/Booting paravirtualized kernel on /
+				/Hypervisor detected: /
+				/lpar: .* under hypervisor/
+				')
+				EOF
+			fi
+	esac
+
+	# Try to guess hypervisor based on CPU model (sysctl hw.model if available)
+	if cpu_model=$(get_sysctl hw.model)
+	then
+		guess_hypervisor_from_cpu_model "${cpu_model}" && return 0
+	fi
+
+	if ${_hyp_generic}
+	then
+		# cannot say which hypervisor, but one was detected
+		return 0
+	else
+		return 1
+	fi
+}
+
+run_stage() {
+	if type "has_$1_$2" >/dev/null 2>&1
+	then
+		"has_$1_$2"
+	else
+		true
+	fi \
+	&& "check_$1_$2"
+}
+
+
+# Execute container stages
+
+for stage in \
+	pid_1 cgroup files os_specific
+do
+	ctengine=$(run_stage ct ${stage}) || continue
+	is_contained=true
+	if test -n "${ctengine}"
+	then
+		echo container on "${ctengine}"
+		break
+	fi
+done
+if ${is_contained:-false} && test -z "${ctengine}"
+then
+	# none of the stages could determine the specific container engine, but
+	# we are running in some container.
+	echo container
+fi
+
+
+# Execute virtual machine / hypervisor stages
+
+for stage in \
+	os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
+do
+	hypervisor=$(run_stage vm ${stage}) || continue
+	is_virtual=true
+	if test -n "${hypervisor}"
+	then
+		echo virtual by "${hypervisor}"
+		break
+	fi
+done
+if ${is_virtual:-false} && test -z "${hypervisor}"
+then
+	# none of the stages could determine the specific hypervisor, but
+	# we are virtual.
+	echo virtual
+fi

From abc6d009b21b0d1ce3fc5107201e30740f127200 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 19:29:41 +0200
Subject: [PATCH 330/366] [explorer/machine_type] Print top most machine layer
 as first line (fallback to physical)

---
 cdist/conf/explorer/machine_type | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 29f98849..90c441da 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -26,17 +26,19 @@
 # VERB does not have a special meaning, it is just for better readability.
 #
 # e.g.
+# container
 # container on lxc
 # virtual by kvm-spapr
 #
-# The third word of each line can be composed of different parts concatenated with a `-'
-# (minus) character, with each component being a specification of the previous,
-# e.g.:
+# The third word of each line (except the first) can be composed of different
+# parts concatenated with a `-' (minus) character, with each component being
+# a specification of the previous, e.g.:
 #  - lxc-libvirt             (LXC container, managed by libvirt)
 #  - lpar-s390 / lpar-power  (LPAR running on IBM S/390 or POWER, respectively)
 #  - xen-hvm / xen-pv        (Xen HVM vs para-virtualization)
 #
-# If this explorer cannot determine any information it will print nothing.
+# If this explorer cannot collect enough information about virtualization it
+# will fall back to 'physical'.
 #
 
 # Add /sbin and /usr/sbin to the path so we can find system
@@ -121,6 +123,10 @@ get_sysctl() {
 	is_command sysctl && sysctl -n "$1" 2>/dev/null
 }
 
+detected_layer() {
+	test -n "${_toplayer:-}" || echo "${_toplayer:=${1:?}}"
+}
+
 
 # Check for container
 
@@ -895,6 +901,7 @@ for stage in \
 	pid_1 cgroup files os_specific
 do
 	ctengine=$(run_stage ct ${stage}) || continue
+	detected_layer 'container'
 	is_contained=true
 	if test -n "${ctengine}"
 	then
@@ -916,6 +923,7 @@ for stage in \
 	os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
 do
 	hypervisor=$(run_stage vm ${stage}) || continue
+	detected_layer 'virtual machine'
 	is_virtual=true
 	if test -n "${hypervisor}"
 	then
@@ -929,3 +937,8 @@ then
 	# we are virtual.
 	echo virtual
 fi
+
+
+# Fallback
+
+detected_layer physical

From 2ffa895f578f12d615ed65b9a93f3e2ae07f1d07 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 19:57:24 +0200
Subject: [PATCH 331/366] [explorer/machine_type] Remove CPUID check

it's a lot of code and depends on a binary helper unlikely to be installed.
---
 cdist/conf/explorer/machine_type | 59 --------------------------------
 1 file changed, 59 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 90c441da..7ce035e3 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -379,65 +379,6 @@ check_vm_arch_specific() {
 			fi
 			;;
 		(i?86|x86*|amd64|i86pc)
-			# Check CPUID
-			#
-			# Many fullvirt hypervisors give an indication through CPUID.  Use
-			# the virt-what helper program to get this information if available.
-
-			for CPUID_HELPER in \
-				$(command -v virt-what-cpuid-helper 2>/dev/null) \
-				/usr/lib/x86_64-*/virt-what-cpuid-helper \
-				/usr/lib/i?86-*/virt-what-cpuid-helper \
-				/usr/lib/virt-what/virt-what-cpuid-helper
-			do
-				if test -x "${CPUID_HELPER:?}"; then break; fi
-			done
-
-			if test -x "${CPUID_HELPER-}"
-			then
-				case $(command "${CPUID_HELPER}")
-				in
-					('bhyve bhyve ')
-						echo bhyve
-						;;
-					('LKVMLKVMLKVM')
-						echo lkvm
-						;;
-					('KVMKVMKVM')
-						echo kvm
-						;;
-					('TCGTCGTCGTCG')
-						echo qemu-tcg
-						;;
-					('Microsoft Hv')
-						# http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx
-						echo hyperv
-						;;
-					('OpenBSDVMM58')
-						# OpenBSD/VMM
-						echo openbsd_vmm
-						;;
-					('VMwareVMware')
-						# check added by Chetan Loke.
-						echo vmware
-						;;
-					('XenVMMXenVMM')
-						if has dmi
-						then
-							# https://access.redhat.com/solutions/222903
-							echo xen-hvm
-						else
-							echo xen-paravirt
-						fi
-						;;
-					(*)
-						return 1 ;;
-				esac
-				return 0
-			fi
-
-			unset CPUID_HELPER
-
 			# VMM CPUID flag denotes that this system is running under a VMM
 			if is_oneof "${uname_s}" Darwin
 			then

From 23fbfaf0352eadaabd43504d3ee074bb9f696fcd Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 21:29:24 +0200
Subject: [PATCH 332/366] [explorer/machine_type] Use systemd-detect-virt (if
 available) to detect containers and VMs

---
 cdist/conf/explorer/machine_type | 49 ++++++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 7ce035e3..1a38fda0 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -130,6 +130,25 @@ detected_layer() {
 
 # Check for container
 
+has_ct_systemd() {
+	is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^  -c'
+}
+
+check_ct_systemd() (
+	_ctengine=$(systemd-detect-virt -c 2>/dev/null) &&
+	case ${_ctengine}
+	in
+		(''|'none')
+			return 1 ;;
+		('container-other')
+			return 0 ;;
+		('systemd-nspawn')
+			echo systemd_nspawn ;;
+		(*)
+			echo "${_ctengine}" ;;
+	esac
+)
+
 has_ct_pid_1() {
 	test -r /run/systemd/container -o -r /proc/1/environ
 }
@@ -267,6 +286,32 @@ guess_hypervisor_from_cpu_model() {
 	esac
 }
 
+has_vm_systemd() {
+	is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^  -v'
+}
+
+check_vm_systemd() (
+	_hypervisor=$(systemd-detect-virt -v 2>/dev/null) &&
+	case ${_hypervisor}
+	in
+		(''|'none')
+			return 1 ;;
+		('amazon')
+			echo aws ;;
+		('bochs')
+			echo kvm ;;
+		('microsoft')
+			# assumption
+			echo hyperv ;;
+		('oracle')
+			echo virtualbox ;;
+		('vm-other')
+			return 0 ;;
+		(*)
+			echo "${_hypervisor}" ;;
+	esac
+)
+
 has_vm_cpuinfo() { has_cpuinfo; }
 
 check_vm_cpuinfo() {
@@ -839,7 +884,7 @@ run_stage() {
 # Execute container stages
 
 for stage in \
-	pid_1 cgroup files os_specific
+	systemd pid_1 cgroup files os_specific
 do
 	ctengine=$(run_stage ct ${stage}) || continue
 	detected_layer 'container'
@@ -861,7 +906,7 @@ fi
 # Execute virtual machine / hypervisor stages
 
 for stage in \
-	os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
+	systemd os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific
 do
 	hypervisor=$(run_stage vm ${stage}) || continue
 	detected_layer 'virtual machine'

From 4a05669765c2c00c19af0ef1b607b0f2efa10f42 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sat, 31 Jul 2021 22:01:28 +0200
Subject: [PATCH 333/366] [explorer/machine_type] Implement chroot detection

---
 cdist/conf/explorer/machine_type | 53 ++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 1a38fda0..10c914ba 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -53,6 +53,21 @@ uname_s=$(uname -s)
 
 is_command() { command -v "$1" >/dev/null 2>&1; }
 
+files_same() {
+	# shellcheck disable=SC2012
+	LC_ALL=C df -P "$1" "$2" 2>/dev/null | {
+		read -r _  # skip header line
+		read -r fs1 _ _ _ _ mp1
+		read -r fs2 _ _ _ _ mp2
+		test "${fs1}" = "${fs2}" -a "${mp1}" = "${mp2}" || return 1
+	} &&
+	ls -1Ldi "$1" "$2" 2>/dev/null | {
+		read -r ino1 _
+		read -r ino2 _
+		test "${ino1}" = "${ino2}" || return 1
+	}
+}
+
 is_oneof() (
 	x=$1; shift
 	for y
@@ -128,6 +143,32 @@ detected_layer() {
 }
 
 
+# Check for chroot
+
+has_chroot_systemd() {
+	is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^  -r'
+}
+
+check_chroot_systemd() {
+	systemd-detect-virt -r
+}
+
+has_chroot_debian_ischroot() {
+	is_command ischroot
+}
+
+check_chroot_debian_ischroot() {
+	ischroot --default-false
+}
+
+has_chroot_procfs() {
+	test -d /proc/
+}
+
+check_chroot_procfs() {
+	test -e /proc/1/root && ! files_same /proc/1/root /
+}
+
 # Check for container
 
 has_ct_systemd() {
@@ -881,6 +922,18 @@ run_stage() {
 }
 
 
+# Execute chroot stages
+
+for stage in \
+	procfs debian_ischroot systemd
+do
+	run_stage chroot ${stage} || continue
+	detected_layer 'chroot'
+	echo chroot
+	break
+done
+
+
 # Execute container stages
 
 for stage in \

From 5af1317c2969995871da1d17b951de60dbc3fd09 Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sun, 1 Aug 2021 16:40:20 +0200
Subject: [PATCH 334/366] [explorer/machine_type] Try to detect chroot path

---
 cdist/conf/explorer/machine_type | 38 ++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index 10c914ba..fa68ec4d 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -166,7 +166,28 @@ has_chroot_procfs() {
 }
 
 check_chroot_procfs() {
-	test -e /proc/1/root && ! files_same /proc/1/root /
+	if test -e /proc/1/root && ! files_same /proc/1/root /
+	then
+		# try to determine where the chroot has been mounted
+		(
+			rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}')
+
+			if test -e "${rootdev}"
+			then
+				# escape chroot to determine where the device containing the
+				# chroot's / is mounted
+				rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}')
+
+				# shellcheck disable=SC2012
+				root_ino=$(ls -1di / | awk '{print $1}')
+
+				# Get mount point
+				chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}"
+			fi
+		)
+		return 0
+	fi
+	return 1
 }
 
 # Check for container
@@ -927,11 +948,20 @@ run_stage() {
 for stage in \
 	procfs debian_ischroot systemd
 do
-	run_stage chroot ${stage} || continue
+	chrootpnt=$(run_stage chroot ${stage}) || continue
+	is_chrooted=true
 	detected_layer 'chroot'
-	echo chroot
-	break
+	if test -n "${chrootpnt}"
+	then
+		echo chroot at "${chrootpnt}"
+		break
+	fi
 done
+if ${is_chrooted:-false} && test -z "${chrootpnt}"
+then
+	# could determine chroot, but not its mount point
+	echo chroot
+fi
 
 
 # Execute container stages

From 05c2a62191f533ceff1e074df571278a103d75ab Mon Sep 17 00:00:00 2001
From: Dennis Camera <cdist@dtnr.ch>
Date: Sun, 1 Aug 2021 23:09:02 +0200
Subject: [PATCH 335/366] [explorer/machine_type] Implement chroot detection
 using /proc/.../mountinfo

---
 cdist/conf/explorer/machine_type | 64 +++++++++++++++++++++-----------
 1 file changed, 43 insertions(+), 21 deletions(-)

diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type
index fa68ec4d..00646c75 100755
--- a/cdist/conf/explorer/machine_type
+++ b/cdist/conf/explorer/machine_type
@@ -165,30 +165,52 @@ has_chroot_procfs() {
 	test -d /proc/
 }
 
-check_chroot_procfs() {
+check_chroot_procfs() (
+	is_chroot=false  # default
 	if test -e /proc/1/root && ! files_same /proc/1/root /
 	then
-		# try to determine where the chroot has been mounted
-		(
-			rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}')
-
-			if test -e "${rootdev}"
-			then
-				# escape chroot to determine where the device containing the
-				# chroot's / is mounted
-				rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}')
-
-				# shellcheck disable=SC2012
-				root_ino=$(ls -1di / | awk '{print $1}')
-
-				# Get mount point
-				chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}"
-			fi
-		)
-		return 0
+		is_chroot=true
 	fi
-	return 1
-}
+	if test -e /proc/1/mountinfo -a -e /proc/self/mountinfo
+	then
+		has_mountinfo=true
+		cmp -s /proc/1/mountinfo /proc/self/mountinfo || is_chroot=true
+	fi
+
+	if ${is_chroot}
+	then
+		# try to determine where the chroot has been mounted
+		rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}')
+
+		if test -e "${rootdev}"
+		then
+			# escape chroot to determine where the device containing the
+			# chroot's / is mounted
+			rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}')
+
+			# shellcheck disable=SC2012
+			root_ino=$(ls -1di / | awk '{print $1}')
+
+			# escape chroot and find mount point by inode
+			chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}"
+		elif ${has_mountinfo}
+		then
+			while read -r mntid _ _ _ cmntpnt _
+			do
+				read -r _ _ _ _ hmntpnt _ <<-EOF
+				$(grep -e "^$((mntid)) " /proc/1/mountinfo)
+				EOF
+				printf '%s\n' "${hmntpnt%${cmntpnt}}"
+			done </proc/self/mountinfo \
+			| sort -u \
+			| head -n 1  # just take the first...
+		fi
+
+		return 0
+	else
+		return 1
+	fi
+)
 
 # Check for container
 

From 67f85546ec3bf865d368f268fdbbc121ccfd5742 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 23 Aug 2021 09:57:20 +0300
Subject: [PATCH 336/366] [explorer/os_version] add new debian code names:
 bookworm and trixie

---
 cdist/conf/explorer/os_version | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index 7bc6dd6b..aea3c43f 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -54,6 +54,8 @@ in
               # sid versions don't have a number, so we decode by codename:
               case $(expr "$debian_version" : '\([a-z]\{1,\}\)/')
               in
+                  trixie) echo 12.99 ;;
+                  bookworm) echo 11.99 ;;
                   bullseye) echo 10.99 ;;
                   buster) echo 9.99 ;;
                   stretch) echo 8.99 ;;

From e1e134899811e51e4694e89e421790e41cb894bf Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Mon, 23 Aug 2021 10:44:48 +0300
Subject: [PATCH 337/366] [explorer/os_version] use 99.99 as fallback for
 unknown code names in */sid

---
 cdist/conf/explorer/os_version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version
index aea3c43f..bbc9e4f0 100755
--- a/cdist/conf/explorer/os_version
+++ b/cdist/conf/explorer/os_version
@@ -63,7 +63,7 @@ in
                   wheezy) echo 6.99 ;;
                   squeeze) echo 5.99 ;;
                   lenny) echo 4.99 ;;
-                  *) exit 1
+                  *) echo 99.99 ;;
               esac
               ;;
           *)

From 46ed48d546af6ff663b131c803a9027b19798ba1 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 08:09:47 +0200
Subject: [PATCH 338/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index f9409d7e..8507c663 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,7 @@ next:
 	* Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera)
 	* Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera)
 	* Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera)
+	* Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar)
 
 6.9.7: 2021-07-10
 	* New type: __postgres_conf (Beni Ruef, Dennis Camera)

From 0546283d0ebc0058dbff68cec9d18b9281c26edd Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 20:32:44 +0200
Subject: [PATCH 339/366] Update shellcheck disable

---
 cdist/conf/type/__apt_backports/manifest             | 1 +
 cdist/conf/type/__package_pkg_freebsd/gencode-remote | 1 +
 cdist/conf/type/__ssh_authorized_keys/explorer/keys  | 1 +
 3 files changed, 3 insertions(+)

diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest
index bc47d8de..6fcd9212 100755
--- a/cdist/conf/type/__apt_backports/manifest
+++ b/cdist/conf/type/__apt_backports/manifest
@@ -28,6 +28,7 @@
 #  lsb_release may not be given in all installations
 codename_os_release() {
     # shellcheck disable=SC1090
+    # shellcheck disable=SC1091
     . "$__global/explorer/os_release"
     printf "%s" "$VERSION_CODENAME"
 }
diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote
index 3f88f6bc..ca9aa45a 100755
--- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote
+++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote
@@ -37,6 +37,7 @@ assert ()                 #  If condition false,
 	then
 		echo "Assertion failed:  \"$1\""
 		# shellcheck disable=SC2039
+		# shellcheck disable=SC3044
 		echo "File \"$0\", line $lineno, called by $(caller 0)"
 		exit $E_ASSERT_FAILED
 	fi
diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
index cec25746..9694a64b 100755
--- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys
+++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys
@@ -1,6 +1,7 @@
 #!/bin/sh -e
 
 # shellcheck disable=SC1090
+# shellcheck disable=SC1091
 file="$( . "$__type_explorer/file" )"
 
 if [ -f "$file" ]

From 44741e714b16f7a00bf84bd54ae33eacc7593192 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 20:25:49 +0200
Subject: [PATCH 340/366] Release 6.9.8

---
 docs/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog b/docs/changelog
index 8507c663..dcdc4b3d 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,7 +1,7 @@
 Changelog
 ---------
 
-next:
+6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)
 	* Explorer os_version: Convert Devuan ceres to version number (Dennis Camera)

From b8eb6e984c1638e8e167394c6b3aa482cb8aad49 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 24 Aug 2021 20:47:50 +0200
Subject: [PATCH 341/366] ++changelog

---
 docs/changelog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index dcdc4b3d..693d028f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -1,6 +1,9 @@
 Changelog
 ---------
 
+next:
+	* Explorer machine_type: Rewrite (Dennis Camera)
+
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)
 	* New type: __apt_pin (Daniel Fancsali)

From d7fdc8006f747be86b64a2790266b50219bab507 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 21:54:45 +0300
Subject: [PATCH 342/366] allow empty file

---
 cdist/conf/type/__sed/explorer/file | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
index a40f71b3..5f0f4edd 100755
--- a/cdist/conf/type/__sed/explorer/file
+++ b/cdist/conf/type/__sed/explorer/file
@@ -9,13 +9,7 @@ fi
 
 if [ -f "$file" ]
 then
-    if [ -s "$file" ]
-    then
-        cat "$file"
-    else
-        echo "$file is empty" >&2
-        exit 1
-    fi
+    cat "$file"
 else
     echo "$file do not exist" >&2
     exit 1

From 0f6e48dbc657a9d9145b965eacf79f021a1419f5 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:24:26 +0300
Subject: [PATCH 343/366] use $__object/tempfile in target instead of mktemp,
 add comments

---
 cdist/conf/type/__sed/gencode-remote | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 836506f0..df929b42 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -35,8 +35,11 @@ then
     exit 0
 fi
 
+# we can't use -i, because it's not posix, so we fly with tempfile and cp
+# and we use cp because we want to preserve destination file's attributes
+
 # shellcheck disable=SC2016
-echo 'tmp="$( mktemp )"'
+echo 'tmp="$__object/tempfile"'
 
 echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF"
 

From 90488fcebcc00f3b4ea260867f8b31625df61e46 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:27:42 +0300
Subject: [PATCH 344/366] use -e

---
 cdist/conf/type/__sed/explorer/file | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
index 5f0f4edd..9262b613 100755
--- a/cdist/conf/type/__sed/explorer/file
+++ b/cdist/conf/type/__sed/explorer/file
@@ -7,10 +7,10 @@ else
     file="/$__object_id"
 fi
 
-if [ -f "$file" ]
+if [ ! -e "$file" ]
 then
-    cat "$file"
-else
     echo "$file do not exist" >&2
     exit 1
 fi
+
+cat "$file"

From b7f392fa371ff78591abb910f7e224e7587c81be Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:38:55 +0300
Subject: [PATCH 345/366] use -E for better compat (not really sure if it is
 posix at all)

---
 cdist/conf/type/__sed/gencode-remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index df929b42..625448e6 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -24,7 +24,7 @@ sed_cmd='sed'
 
 if [ -f "$__object/parameter/regexp-extended" ]
 then
-    sed_cmd="$sed_cmd --regexp-extended"
+    sed_cmd="$sed_cmd -E"
 fi
 
 if echo "$script" \

From aabef7f44a98ad144a7e1fe68bfb562079916ef8 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:40:06 +0300
Subject: [PATCH 346/366] remove reading script from file

---
 cdist/conf/type/__sed/gencode-remote | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index 625448e6..be7690e1 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -12,10 +12,6 @@ script="$( cat "$__object/parameter/script" )"
 if [ "$script" = '-' ]
 then
     script="$( cat "$__object/stdin" )"
-elif
-    [ -f "$script" ]
-then
-    script="$( cat "$script" )"
 fi
 
 file_from_target="$__object/explorer/file"

From 5bf0c71e7af49ff6e7c1cccec192f763a019503e Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Tue, 14 Sep 2021 22:45:36 +0300
Subject: [PATCH 347/366] update man

---
 cdist/conf/type/__sed/man.rst | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst
index 4a728184..86789363 100644
--- a/cdist/conf/type/__sed/man.rst
+++ b/cdist/conf/type/__sed/man.rst
@@ -8,35 +8,40 @@ cdist-type__sed - Transform text files with ``sed``
 
 DESCRIPTION
 -----------
-TODO
+Transform text files with ``sed``.
 
 
 REQUIRED MULTIPLE PARAMETERS
 ----------------------------
 script
-   TODO
+   ``sed`` script.
+   If ``-`` then the script is read from ``stdin``.
 
 
 OPTIONAL PARAMETERS
 -------------------
 file
-   TODO
+   Path to the file. Defaults to ``$__object_id``.
 
 onchange
-   TODO
+   Execute this command if ``sed`` changes file.
 
 
 BOOLEAN PARAMETERS
 ------------------
 regexp-extended
-   TODO
+   Use extended regular expressions in the script.
+   Might not be supported with every ``sed`` version.
 
 
 EXAMPLES
 --------
+
 .. code-block:: sh
 
-   true
+   __sed /tmp/foobar --script 's/foo/bar/'
+
+   echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script -
 
 
 AUTHORS

From cd4acde67ec9c3967e6e08450734cf65040e787f Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 15 Sep 2021 09:22:27 +0300
Subject: [PATCH 348/366] grammar

---
 cdist/conf/type/__sed/explorer/file | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file
index 9262b613..ec3d0fe8 100755
--- a/cdist/conf/type/__sed/explorer/file
+++ b/cdist/conf/type/__sed/explorer/file
@@ -9,7 +9,7 @@ fi
 
 if [ ! -e "$file" ]
 then
-    echo "$file do not exist" >&2
+    echo "$file does not exist" >&2
     exit 1
 fi
 

From 7b6789ddeb74450f70991efe957e3eb0f11043f6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 15:04:12 +0200
Subject: [PATCH 349/366] __package_update_index: fix complain about suite
 change

2 of 4th fix for ticket #861
---
 .../type/__package_update_index/gencode-remote     | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote
index 803468b5..a10c16d3 100755
--- a/cdist/conf/type/__package_update_index/gencode-remote
+++ b/cdist/conf/type/__package_update_index/gencode-remote
@@ -41,7 +41,19 @@ fi
 case "$type" in
     yum) ;;
     apt)
-        echo "apt-get --quiet update"
+        # There are special arguments to apt(8) to prevent aborts if apt woudn't been
+        # updated after the 19th April 2021 till the bullseye release. The additional
+        # arguments acknoledge the happend suite change (the apt(8) update does the
+        # same by itself).
+        #
+        # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+        # allows backward compatablility to pre-buster Debian versions.
+        #
+        # See more: ticket #861
+        # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+        apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
+        echo "apt-get --quiet $apt_opts update"
         echo "apt-cache updated (age was: $currage)" >> "$__messages_out"
         ;;
     pacman)

From 12787ffe2c48f5c6d534d10f8faadde30db477ac Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 15:12:20 +0200
Subject: [PATCH 350/366] __apt_source: fix complain about suite change

3 of 4th fix for ticket #861
---
 cdist/conf/type/__apt_source/gencode-remote | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_source/gencode-remote b/cdist/conf/type/__apt_source/gencode-remote
index 1e8592c6..973b0f6c 100755
--- a/cdist/conf/type/__apt_source/gencode-remote
+++ b/cdist/conf/type/__apt_source/gencode-remote
@@ -22,7 +22,21 @@
 name="$__object_id"
 destination="/etc/apt/sources.list.d/${name}.list"
 
+# There are special arguments to apt(8) to prevent aborts if apt woudn't been
+# updated after the 19th April 2021 till the bullseye release. The additional
+# arguments acknoledge the happend suite change (the apt(8) update does the
+# same by itself).
+#
+# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+# allows backward compatablility to pre-buster Debian versions.
+#
+# See more: ticket #861
+# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
+# run 'apt-get update' only if something changed with our sources.list file
+#   it will be run a second time on error as a redundancy messure to success
 if grep -q "^__file${destination}" "$__messages_in"; then
-   printf 'apt-get update || apt-get update\n'
+   printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts"
 fi
 

From d246e067109dc8b85ea89668f742affbb221d1a6 Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 14:49:23 +0200
Subject: [PATCH 351/366] __apt_update_index: fix complain about suite change

1 of 4th fix for ticket #861
---
 .../conf/type/__apt_update_index/gencode-remote  | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__apt_update_index/gencode-remote b/cdist/conf/type/__apt_update_index/gencode-remote
index 70b59710..2d7f9030 100755
--- a/cdist/conf/type/__apt_update_index/gencode-remote
+++ b/cdist/conf/type/__apt_update_index/gencode-remote
@@ -18,9 +18,23 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+
+# There are special arguments to apt(8) to prevent aborts if apt woudn't been
+# updated after the 19th April 2021 till the bullseye release. The additional
+# arguments acknoledge the happend suite change (the apt(8) update does the
+# same by itself).
+#
+# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+# allows backward compatablility to pre-buster Debian versions.
+#
+# See more: ticket #861
+# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
 # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists
+#   it will be run a second time on error as a redundancy messure to success
 cat << DONE
 if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then
-   apt-get update || apt-get update
+   apt-get $apt_opts update || apt-get $apt_opts update
 fi
 DONE

From 3d7b31cbb43067271c2c510281f90f19dc5ec5fb Mon Sep 17 00:00:00 2001
From: Matthias Stecher <matthiasstecher@gmx.de>
Date: Wed, 15 Sep 2021 15:22:16 +0200
Subject: [PATCH 352/366] __package_apt: fix complain about suite change

the last fix for ticket #861 :-)
---
 cdist/conf/type/__package_apt/gencode-remote | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote
index fbfca330..79c0d9d3 100755
--- a/cdist/conf/type/__package_apt/gencode-remote
+++ b/cdist/conf/type/__package_apt/gencode-remote
@@ -81,12 +81,24 @@ aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::=
 
 case "$state_should" in
     present)
+        # There are special arguments to apt(8) to prevent aborts if apt woudn't been
+        # updated after the 19th April 2021 till the bullseye release. The additional
+        # arguments acknoledge the happend suite change (the apt(8) update does the
+        # same by itself).
+        #
+        # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter
+        # allows backward compatablility to pre-buster Debian versions.
+        #
+        # See more: ticket #861
+        # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861
+        apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true"
+
         # following is bit ugly, but important hack.
         # due to how cdist config run works, there isn't
         # currently better way to do it :(
         cat << EOF
 if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ]
-then echo apt-get update > /dev/null 2>&1 || true
+then echo apt-get $apt_opts update > /dev/null 2>&1 || true
 fi
 EOF
         if [ -n "$version" ]; then

From 72ff48154c958ebdedc8b6d919b6f372efa63be2 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Thu, 16 Sep 2021 21:36:39 +0300
Subject: [PATCH 353/366] add comments, add -u to diff

---
 cdist/conf/type/__sed/gencode-remote | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote
index be7690e1..f99c5a88 100755
--- a/cdist/conf/type/__sed/gencode-remote
+++ b/cdist/conf/type/__sed/gencode-remote
@@ -14,6 +14,8 @@ then
     script="$( cat "$__object/stdin" )"
 fi
 
+# since stdin is not available in explorer, we pull file from target with explorer
+
 file_from_target="$__object/explorer/file"
 
 sed_cmd='sed'
@@ -23,10 +25,10 @@ then
     sed_cmd="$sed_cmd -E"
 fi
 
-if echo "$script" \
-    | "$sed_cmd" -f - "$file_from_target" \
-    | diff "$file_from_target" - \
-    > /dev/null
+# do sed dry run, diff result and if no change, then there's nothing to do
+# also redirect diff's output to stderr for debugging purposes
+
+if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2
 then
     exit 0
 fi

From bf222d05435000063930ff199bd0dec94d7d8cd5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.ch>
Date: Tue, 21 Sep 2021 08:55:54 +0200
Subject: [PATCH 354/366] ++changelog

---
 docs/changelog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 693d028f..ab637f5f 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -3,6 +3,11 @@ Changelog
 
 next:
 	* Explorer machine_type: Rewrite (Dennis Camera)
+	* New type: __sed (Ander Punnar)
+	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From 15c642a9b7af42c368eb5a8e22cab5f07b39b95e Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 12:06:45 +0200
Subject: [PATCH 355/366] [__debconf_set_selections] Fix --file not being
 supported

Even if deprecated, the parameter *must* be supported, which isn't the case
right now.

This was due to a misunderstanding of how deprecating parameters work, see:
https://www.cdi.st/manual/latest/cdist-type.html#deprecated-parameters
---
 .../conf/type/__debconf_set_selections/parameter/deprecated/file | 1 +
 .../__debconf_set_selections/parameter/{deprecated => optional}  | 0
 2 files changed, 1 insertion(+)
 create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/deprecated/file
 rename cdist/conf/type/__debconf_set_selections/parameter/{deprecated => optional} (100%)

diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file
new file mode 100644
index 00000000..09db545a
--- /dev/null
+++ b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file
@@ -0,0 +1 @@
+'file' has been deprecated in favour of 'line' in order to provide idempotency.
diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated b/cdist/conf/type/__debconf_set_selections/parameter/optional
similarity index 100%
rename from cdist/conf/type/__debconf_set_selections/parameter/deprecated
rename to cdist/conf/type/__debconf_set_selections/parameter/optional

From 5b7cca99f73e4ec3dd5cdb90e4dbfd40ec8fe349 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 12:09:42 +0200
Subject: [PATCH 356/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index ab637f5f..142602c8 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -8,6 +8,7 @@ next:
 	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
+	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From fc9bd40c9a34cf54e2d6cfd2fec089bd25e1b172 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 13:14:57 +0200
Subject: [PATCH 357/366] Improve bullseye support, perticularly
 __letsencrypt_cert

---
 cdist/conf/type/__grafana_dashboard/manifest | 2 +-
 cdist/conf/type/__letsencrypt_cert/manifest  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest
index d145c4c3..0d944482 100755
--- a/cdist/conf/type/__grafana_dashboard/manifest
+++ b/cdist/conf/type/__grafana_dashboard/manifest
@@ -15,7 +15,7 @@ case $os in
                 # Differntation not needed anymore
                 apt_source_distribution=stable
                 ;;
-            10*)
+            10*|11*)
                 # Differntation not needed anymore
                 apt_source_distribution=stable
                 ;;
diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest
index 6394f629..638a99e0 100644
--- a/cdist/conf/type/__letsencrypt_cert/manifest
+++ b/cdist/conf/type/__letsencrypt_cert/manifest
@@ -41,7 +41,7 @@ if [ -z "${certbot_fullpath}" ]; then
 					require="__apt_source/stretch-backports" __package_apt certbot \
 						--target-release stretch-backports
 				;;
-				10*)
+				10*|11*)
 					__package_apt certbot
 				;;
 

From 560374a6861281695b61c8c2298123be41d92326 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Fri, 1 Oct 2021 13:16:11 +0200
Subject: [PATCH 358/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 142602c8..6f717cf4 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -9,6 +9,7 @@ next:
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
+	* Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From c33d99ee120ad180daba144be80a25d77b473f56 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Sun, 31 Oct 2021 17:38:10 +0100
Subject: [PATCH 359/366] [__haproxy_dualstack] New type with PROXY protocol
 support

This is backwards compatible with what is already used internally @ungleich, but
adds on top of that the ability to customise ports and, most importantly, it
adds PROXY protocol support.
---
 .../conf/type/__haproxy_dualstack/files/http  |   8 +
 .../conf/type/__haproxy_dualstack/files/https |  10 ++
 .../conf/type/__haproxy_dualstack/files/imaps |  12 ++
 .../conf/type/__haproxy_dualstack/files/smtps |  12 ++
 cdist/conf/type/__haproxy_dualstack/man.rst   | 121 ++++++++++++++
 cdist/conf/type/__haproxy_dualstack/manifest  | 155 ++++++++++++++++++
 .../parameter/default/protocol                |   1 +
 .../parameter/optional_multiple               |   3 +
 cdist/conf/type/__haproxy_dualstack/singleton |   0
 9 files changed, 322 insertions(+)
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/http
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/https
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/imaps
 create mode 100644 cdist/conf/type/__haproxy_dualstack/files/smtps
 create mode 100644 cdist/conf/type/__haproxy_dualstack/man.rst
 create mode 100644 cdist/conf/type/__haproxy_dualstack/manifest
 create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/default/protocol
 create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple
 create mode 100644 cdist/conf/type/__haproxy_dualstack/singleton

diff --git a/cdist/conf/type/__haproxy_dualstack/files/http b/cdist/conf/type/__haproxy_dualstack/files/http
new file mode 100644
index 00000000..0508a465
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/http
@@ -0,0 +1,8 @@
+frontend http
+	bind	BIND@:80
+	mode	http
+	option	httplog
+	default_backend	http
+
+backend http
+	mode	http
diff --git a/cdist/conf/type/__haproxy_dualstack/files/https b/cdist/conf/type/__haproxy_dualstack/files/https
new file mode 100644
index 00000000..73deac46
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/https
@@ -0,0 +1,10 @@
+frontend https
+	bind	BIND@:443
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	https
+
+backend https
+	mode	tcp
diff --git a/cdist/conf/type/__haproxy_dualstack/files/imaps b/cdist/conf/type/__haproxy_dualstack/files/imaps
new file mode 100644
index 00000000..b1ec3793
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/imaps
@@ -0,0 +1,12 @@
+frontend imaps
+	bind	BIND@:143
+	bind	BIND@:993
+
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	imaps
+
+backend imaps
+	mode	tcp
diff --git a/cdist/conf/type/__haproxy_dualstack/files/smtps b/cdist/conf/type/__haproxy_dualstack/files/smtps
new file mode 100644
index 00000000..dce6ed4a
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/files/smtps
@@ -0,0 +1,12 @@
+frontend smtps
+	bind	BIND@:25
+	bind	BIND@:465
+
+	mode	tcp
+	option	tcplog
+	tcp-request	inspect-delay 5s
+	tcp-request	content accept if { req_ssl_hello_type 1 }
+	default_backend	smtps
+
+backend smtps
+	mode	tcp
diff --git a/cdist/conf/type/__haproxy_dualstack/man.rst b/cdist/conf/type/__haproxy_dualstack/man.rst
new file mode 100644
index 00000000..6c131cbe
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/man.rst
@@ -0,0 +1,121 @@
+cdist-type__haproxy_dualstack(7)
+================================
+
+
+NAME
+----
+cdist-type__haproxy_dualstack - Proxy services from a dual-stack server
+
+
+DESCRIPTION
+-----------
+This (singleton) type installs and configures haproxy to act as a dual-stack
+proxy for single-stack services.
+
+This can be useful to add IPv4 support to IPv6-only services while only using
+one IPv4 for many such services.
+
+By default this type uses the plain TCP proxy mode, which means that there is no
+need for TLS termination on this host when SNI is supported.
+This also means that proxied services will not receive the client's IP address,
+but will see the proxy's IP address instead (that of `$__target_host`).
+
+This can be solved by using the PROXY protocol, but do take into account that,
+e.g. nginx cannot serve both regular HTTP(S) and PROXY protocols on the same
+port, so you will need to use other ports for that.
+
+As a recommendation in this type: use TCP ports 8080 and 591 respectively to
+serve HTTP and HTTPS using the PROXY protocol.
+
+See the EXAMPLES for more details.
+
+
+OPTIONAL PARAMETERS
+-------------------
+v4proxy
+    Proxy incoming IPv4 connections to the equivalent IPv6 endpoint.
+    In its simplest use, it must be a NAME with an `AAAA` DNS entry, which is
+    the IP address actually providing the proxied services.
+    The full format of this argument is:
+    `[proxy:]NAME[[:PROTOCOL_1=PORT_1]...[:PROTOCOL_N=PORT_N]]`
+    Where starting with `proxy:` determines that the PROXY protocol must be
+    used and each `:PROTOCOL=PORT` (e.g. `:http=8080` or `:https=591`) is a PORT
+    override for the given PROTOCOL (see `--protocol`), if not present the
+    PROTOCOL's default port will be used.
+
+
+v6proxy
+    Proxy incoming IPv6 connections to the equivalent IPv4 endpoint.
+    In its simplest use, it must be a NAME with an `A` DNS entry, which is
+    the IP address actually providing the proxied services.
+    See `--v4proxy` for more options and details.
+
+protocol
+    Can be passed multiple times or as a space-separated list of protocols.
+    Currently supported protocols are: `http`, `https`, `imaps`, `smtps`.
+    This defaults to: `http https imaps smtps`.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Proxy the IPv6-only services so IPv4-only clients can access them
+    # This uses HAProxy's TCP mode for http, https, imaps and smtps
+    __haproxy_dualstack \
+        --v4proxy ipv6.chat \
+        --v4proxy matrix.ungleich.ch
+
+    # Proxy the IPv6-only HTTP(S) services so IPv4-only clients can access them
+    # Note this means that the backend IPv6-only server will only see
+    # the IPv6 address of the haproxy host managed by cdist, which can be
+    # troublesome if this information is relevant for analytics/security/...
+    # See the PROXY example below
+    __haproxy_dualstack \
+        --protocol http --protocol https \
+        --v4proxy ipv6.chat \
+        --v4proxy matrix.ungleich.ch
+
+    # Use the PROXY protocol to proxy the IPv6-only HTTP(S) services enabling
+    # IPv4-only clients to access them while maintaining the client's IP address
+    __haproxy_dualstack \
+        --protocol http --protocol https \
+        --v4proxy proxy:ipv6.chat:http=8080:https=591 \
+        --v4proxy proxy:matrix.ungleich.ch:http=8080:https=591
+    # Note however that the PROXY protocol is not compatible with regular
+    # HTTP(S) protocols, so your nginx will have to listen on different ports
+    # with the PROXY settings.
+    # Note that you will need to restrict access to the 8080 port to prevent
+    # Client IP spoofing.
+    # This can be something like:
+    # server {
+    #     # listen for regular HTTP connections
+    #     listen [::]:80 default_server;
+    #     listen 80 default_server;
+    #     # listen for PROXY HTTP connections
+    #     listen [::]:8080 proxy_protocol;
+    #     # Accept the Client's IP from the PROXY protocol
+    #     real_ip_header proxy_protocol;
+    # }
+
+
+SEE ALSO
+--------
+- https://www.haproxy.com/blog/enhanced-ssl-load-balancing-with-server-name-indication-sni-tls-extension/
+- https://www.haproxy.com/blog/haproxy/proxy-protocol/
+- https://docs.nginx.com/nginx/admin-guide/load-balancer/using-proxy-protocol/
+
+
+AUTHORS
+-------
+ungleich <foss--@--ungleich.ch>
+Evilham <cvs--@--evilham.com>
+
+
+COPYING
+-------
+Copyright \(C) 2021 ungleich glarus ag. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
diff --git a/cdist/conf/type/__haproxy_dualstack/manifest b/cdist/conf/type/__haproxy_dualstack/manifest
new file mode 100644
index 00000000..d110eea6
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/manifest
@@ -0,0 +1,155 @@
+#!/bin/sh -eu
+
+__package haproxy
+require="__package/haproxy" __start_on_boot haproxy
+
+tmpdir="$__object/files"
+mkdir "$tmpdir"
+configtmp="$__object/files/haproxy.cfg"
+
+os=$(cat "$__global/explorer/os")
+case $os in
+	freebsd)
+		CONFIG_FILE="/usr/local/etc/haproxy.conf"
+		cat <<EOF > "$configtmp"
+global
+	maxconn	4000
+	user	nobody
+	group	nogroup
+	daemon
+
+EOF
+
+		;;
+	*)
+		CONFIG_FILE="/etc/haproxy/haproxy.cfg"
+		cat <<EOF > "$configtmp"
+global
+	log	[::1] local2
+	chroot	/var/lib/haproxy
+	pidfile	/var/run/haproxy.pid
+	maxconn	4000
+	user	haproxy
+	group	haproxy
+	daemon
+
+	# turn on stats unix socket
+	stats socket	/var/lib/haproxy/stats
+
+EOF
+		;;
+esac
+
+cat <<EOF >> "$configtmp"
+defaults
+	retries	3
+	log	global
+	timeout http-request	10s
+	timeout queue		1m
+	timeout connect		10s
+	timeout client		1m
+	timeout server		1m
+	timeout http-keep-alive	10s
+	timeout check		10s
+EOF
+
+dig_cmd="$(command -v dig || true)"
+get_ip() {
+	# Usage: get_ip (ipv4|ipv6) NAME
+	# uses "dig" if available, else fallback to "host"
+	case $1 in
+		ipv4)
+			if [ -n "${dig_cmd}" ]; then
+				${dig_cmd} +short A "$2"
+			else
+				host -t A "$2" | cut -d ' ' -f 4 | grep -v 'found:'
+			fi
+			;;
+		ipv6)
+			if [ -n "${dig_cmd}" ]; then
+				${dig_cmd} +short AAAA "$2"
+			else
+				host -t AAAA "$2" | cut -d ' ' -f 5 | grep -v 'NXDOMAIN'
+			fi
+			;;
+	esac
+}
+
+PROTOCOLS="$(cat "$__object/parameter/protocol")"
+
+for proxy in v4proxy v6proxy; do
+	param=$__object/parameter/$proxy
+	# no backend? skip generating code
+	if [ ! -f "$param" ]; then
+		continue
+	fi
+
+	# turn backend name into bind parameter: v4backend -> ipv4@
+	bind=$(echo $proxy | sed -e 's/^/ip/' -e 's/proxy//')
+
+	case $bind in
+		ipv4)
+			backendproto=ipv6
+			;;
+		ipv6)
+			backendproto=ipv4
+			;;
+	esac
+
+	for proto in ${PROTOCOLS}; do
+		# Add protocol "header"
+		printf "\n# %s %s \n" "${bind}" "${proto}" >> "$configtmp"
+
+		sed -e "s/BIND/$bind/" \
+			-e "s/\(frontend[[:space:]].*\)/\1$bind/" \
+			-e "s/\(backend[[:space:]].*\)/\\1$bind/" \
+			"$__type/files/$proto" >> "$configtmp"
+
+		while read -r hostdefinition; do
+			if echo "$hostdefinition" | grep -qE '^proxy:'; then
+				# Proxy protocol was requested
+				host="$(echo "$hostdefinition" | sed -E 's/^proxy:([^:]+).*$/\1/')"
+				send_proxy=" send-proxy"
+			else
+				# Just use tcp proxy mode
+				host="$hostdefinition"
+				send_proxy=""
+			fi
+			if echo "$hostdefinition" | grep -qE ":${proto}="; then
+				# Use custom port definition if requested
+				port="$(echo "$hostdefinition" | sed -E "s/^(.*:)?${proto}=([0-9]+).*$/:\2/")"
+			else
+				# Else use the default
+				port=""
+			fi
+			servername=$host
+
+			res=$(get_ip "$bind" "$servername")
+
+			if [ -z "$res" ]; then
+				echo "$servername does not resolve - aborting config" >&2
+				exit 1
+			fi
+
+			# Treat protocols without TLS+SNI specially
+			if [ "$proto" = http ]; then
+				echo "	use-server $servername if { hdr(host) -i $host }" >> "$configtmp"
+			else
+				echo "	use-server $servername if { req_ssl_sni -i $host }" >> "$configtmp"
+			fi
+
+			# Create the "server" itself.
+			# Note that port and send_proxy will be empty unless
+			# they were requested by the type user
+			echo "	server $servername ${backendproto}@${host}${port}${send_proxy}" >> "$configtmp"
+
+		done < "$param"
+	done
+done
+
+# Create config file
+require="__package/haproxy" __file ${CONFIG_FILE} --source "$configtmp" --mode 0644
+
+require="__file${CONFIG_FILE}" __check_messages "haproxy_reload" \
+	--pattern "^__file${CONFIG_FILE}" \
+	--execute "service haproxy reload || service haproxy restart"
diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol
new file mode 100644
index 00000000..dc8bb7bf
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol
@@ -0,0 +1 @@
+http https imaps smtps
diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple
new file mode 100644
index 00000000..8c482bd4
--- /dev/null
+++ b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple
@@ -0,0 +1,3 @@
+protocol
+v4proxy
+v6proxy
diff --git a/cdist/conf/type/__haproxy_dualstack/singleton b/cdist/conf/type/__haproxy_dualstack/singleton
new file mode 100644
index 00000000..e69de29b

From e2500248f2ddc83129e77f2e6b8dffb64904dbae Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Wed, 3 Nov 2021 11:03:33 +0100
Subject: [PATCH 360/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 6f717cf4..99a8c08b 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -4,6 +4,7 @@ Changelog
 next:
 	* Explorer machine_type: Rewrite (Dennis Camera)
 	* New type: __sed (Ander Punnar)
+	* New type: __haproxy_dualstack (Evilham and ungleich)
 	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)

From 3a321469a8ba5aea55220bd70bd4900de732e917 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Tue, 16 Nov 2021 11:11:45 +0100
Subject: [PATCH 361/366] Python 3.10: collections.X -> collections.abc.X

---
 cdist/integration.py     | 2 +-
 cdist/util/fsproperty.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cdist/integration.py b/cdist/integration.py
index 17b65f09..04470ea7 100644
--- a/cdist/integration.py
+++ b/cdist/integration.py
@@ -84,7 +84,7 @@ def _process_hosts_simple(action, host, manifest, verbose,
     """
     if isinstance(host, str):
         hosts = [host, ]
-    elif isinstance(host, collections.Iterable):
+    elif isinstance(host, collections.abc.Iterable):
         hosts = host
     else:
         raise cdist.Error('Invalid host argument: {}'.format(host))
diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py
index 09e9cc19..6bf935e8 100644
--- a/cdist/util/fsproperty.py
+++ b/cdist/util/fsproperty.py
@@ -33,7 +33,7 @@ class AbsolutePathRequiredError(cdist.Error):
         return 'Absolute path required, got: {}'.format(self.path)
 
 
-class FileList(collections.MutableSequence):
+class FileList(collections.abc.MutableSequence):
     """A list that stores it's state in a file.
 
     """
@@ -102,7 +102,7 @@ class FileList(collections.MutableSequence):
         self.__write(lines)
 
 
-class DirectoryDict(collections.MutableMapping):
+class DirectoryDict(collections.abc.MutableMapping):
     """A dict that stores it's items as files in a directory.
 
     """

From 6e3ad11ea0177865e7e0288b1765267df8d5d020 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Thu, 23 Dec 2021 20:07:28 +0100
Subject: [PATCH 362/366] [__package_upgrade_all] Add new --apt-with-new-pkgs
 argument

---
 cdist/conf/type/__package_upgrade_all/gencode-remote    | 6 +++++-
 cdist/conf/type/__package_upgrade_all/man.rst           | 8 ++++++++
 cdist/conf/type/__package_upgrade_all/parameter/boolean | 1 +
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote
index 38aa001e..d332e851 100755
--- a/cdist/conf/type/__package_upgrade_all/gencode-remote
+++ b/cdist/conf/type/__package_upgrade_all/gencode-remote
@@ -28,6 +28,10 @@ apt_clean="$__object/parameter/apt-clean"
 
 apt_dist_upgrade="$__object/parameter/apt-dist-upgrade"
 
+if [ -f "$__object/parameter/apt-with-new-pkgs" ]; then
+	apt_with_new_pkgs="--with-new-pkgs"
+fi
+
 if [ -f "$type" ]; then
     type="$(cat "$type")"
 else
@@ -54,7 +58,7 @@ case "$type" in
     apt)
         if [ -f "$apt_dist_upgrade" ]
         then echo "$aptget dist-upgrade"
-        else echo "$aptget upgrade"
+        else echo "$aptget $apt_with_new_pkgs upgrade"
         fi
 
         if [ -f "$apt_clean" ]
diff --git a/cdist/conf/type/__package_upgrade_all/man.rst b/cdist/conf/type/__package_upgrade_all/man.rst
index e9e2b8ce..0c116bac 100644
--- a/cdist/conf/type/__package_upgrade_all/man.rst
+++ b/cdist/conf/type/__package_upgrade_all/man.rst
@@ -33,6 +33,14 @@ BOOLEAN PARAMETERS
 apt-dist-upgrade
     Do dist-upgrade instead of upgrade.
 
+apt-with-new-pkg
+    Allow installing new packages when used in conjunction with
+    upgrade. This is useful if the update of an installed package
+    requires new dependencies to be installed. Instead of holding the
+    package back upgrade will upgrade the package and install the new
+    dependencies. Note that upgrade with this option will never remove
+    packages, only allow adding new ones.
+
 apt-clean
     Clean out the local repository of retrieved package files.
 
diff --git a/cdist/conf/type/__package_upgrade_all/parameter/boolean b/cdist/conf/type/__package_upgrade_all/parameter/boolean
index 7a56a34b..cd22eb90 100644
--- a/cdist/conf/type/__package_upgrade_all/parameter/boolean
+++ b/cdist/conf/type/__package_upgrade_all/parameter/boolean
@@ -1,2 +1,3 @@
 apt-clean
 apt-dist-upgrade
+apt-with-new-pkgs

From c2c5668b704e1648ff6c8fb88219badddd028346 Mon Sep 17 00:00:00 2001
From: Evilham <cvs@evilham.com>
Date: Thu, 23 Dec 2021 20:08:49 +0100
Subject: [PATCH 363/366] ++changelog

---
 docs/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/changelog b/docs/changelog
index 99a8c08b..26d89057 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -7,6 +7,7 @@ next:
 	* New type: __haproxy_dualstack (Evilham and ungleich)
 	* Type __apt_update_index: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_update_index: Fix complaint about suite change (Matthias Stecher)
+	* Type __package_upgrade_all: Add new --apt-with-new-pkgs argument (Evilham)
 	* Type __apt_source: Fix complaint about suite change (Matthias Stecher)
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)

From 08ff41efded6e3112fc462ba13d9166e620b4082 Mon Sep 17 00:00:00 2001
From: Mark Verboom <mark@verboom.net>
Date: Tue, 8 Mar 2022 12:04:58 +0100
Subject: [PATCH 364/366] Added rm of tmpfile.

---
 cdist/conf/type/__ssh_authorized_key/gencode-remote | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote
index 61c77fb9..cbffde94 100755
--- a/cdist/conf/type/__ssh_authorized_key/gencode-remote
+++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote
@@ -40,6 +40,7 @@ if [ -f "$file" ]; then
    grep -v -F -x '$line' '$file' >\$tmpfile
 fi
 cat "\$tmpfile" >"$file"
+rm -f "\$tmpfile"
 DONE
 }
 

From e0150e779681e232f95bdbefd957c666f05daa89 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Wed, 9 Mar 2022 16:16:44 +0100
Subject: [PATCH 365/366] ++changes

---
 docs/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog b/docs/changelog
index 26d89057..81be51f6 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -12,6 +12,8 @@ next:
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
 	* Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham)
+	* Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom)
+	* Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

From bd44c023d33eb51a09afad12b18c082f6a3ae36d Mon Sep 17 00:00:00 2001
From: Daniel Fancsali <fancsali@gmail.com>
Date: Fri, 11 Jun 2021 11:22:31 +0100
Subject: [PATCH 366/366] Fix typos; add default priority; comments in
 generated files

---
 cdist/conf/type/__apt_pin/manifest                   | 5 +++++
 cdist/conf/type/__apt_pin/parameter/default/priority | 1 +
 cdist/conf/type/__apt_pin/parameter/optional         | 1 +
 cdist/conf/type/__apt_pin/parameter/required         | 1 -
 4 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 cdist/conf/type/__apt_pin/parameter/default/priority

diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest
index e72a8fdd..983b2b42 100755
--- a/cdist/conf/type/__apt_pin/manifest
+++ b/cdist/conf/type/__apt_pin/manifest
@@ -57,6 +57,11 @@ __file "/etc/apt/preferences.d/$name" \
     --owner root --group root --mode 0644 \
     --state "$state" \
     --source - << EOF
+# Created by cdist ${__type##*/}
+# Do not change. Changes will be overwritten.
+#
+
+# $name
 Package: $package
 Pin: $pin
 Pin-Priority: $priority
diff --git a/cdist/conf/type/__apt_pin/parameter/default/priority b/cdist/conf/type/__apt_pin/parameter/default/priority
new file mode 100644
index 00000000..1b79f38e
--- /dev/null
+++ b/cdist/conf/type/__apt_pin/parameter/default/priority
@@ -0,0 +1 @@
+500
diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional
index 52f01fd2..847e703d 100644
--- a/cdist/conf/type/__apt_pin/parameter/optional
+++ b/cdist/conf/type/__apt_pin/parameter/optional
@@ -1,2 +1,3 @@
 state
 package
+priority
diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required
index 4b4e9741..c8572d92 100644
--- a/cdist/conf/type/__apt_pin/parameter/required
+++ b/cdist/conf/type/__apt_pin/parameter/required
@@ -1,2 +1 @@
 distribution
-priority