Merge branch 'master' of code.ungleich.ch:ungleich-public/cdist

cdist/conf/type/__acl/gencode-remote

@@ -20,7 +20,13 @@
 
 file_is="$( cat "$__object/explorer/file_is" )"
 
-[ "$file_is" = 'missing' ] && [ -z "$__cdist_dry_run" ] && exit 0
+if [ "$file_is" = 'missing' ] \
+    && [ -z "$__cdist_dry_run" ] \
+    && \( [ ! -f "$__object/parameter/file" ] \
+        || [ ! -f "$__object/parameter/directory" ] \)
+then
+    exit 0
+fi
 
 os="$( cat "$__global/explorer/os" )"
 
@@ -28,7 +34,20 @@ acl_path="/$__object_id"
 
 acl_is="$( cat "$__object/explorer/acl_is" )"
 
-if [ -f "$__object/parameter/acl" ]
+if [ -f "$__object/parameter/source" ]
+then
+    acl_source="$( cat "$__object/parameter/source" )"
+
+    if [ "$acl_source" = '-' ]
+    then
+        acl_should="$( cat "$__object/stdin" )"
+    else
+        acl_should="$( grep -Ev '^#|^$' "$acl_source" )"
+    fi
+elif [ -f "$__object/parameter/entry" ]
+then
+    acl_should="$( cat "$__object/parameter/entry" )"
+elif [ -f "$__object/parameter/acl" ]
 then
     acl_should="$( cat "$__object/parameter/acl" )"
 elif

cdist/conf/type/__acl/man.rst

@@ -15,10 +15,24 @@ See ``setfacl`` and ``acl`` manpages for more details.
 
 REQUIRED MULTIPLE PARAMETERS
 ----------------------------
-acl
+entry
    Set ACL entry following ``getfacl`` output syntax.
 
 
+OPTIONAL PARAMETERS
+-------------------
+source
+   Read ACL entries from stdin or file.
+   Ordering of entries is not important.
+   When reading from file, comments and empty lines are ignored.
+
+file
+   Create/change file with ``__file`` using ``user:group:mode`` pattern.
+
+directory
+   Create/change directory with ``__directory`` using ``user:group:mode`` pattern.
+
+
 BOOLEAN PARAMETERS
 ------------------
 default
@@ -36,8 +50,8 @@ remove
 
 DEPRECATED PARAMETERS
 ---------------------
-Parameters ``user``, ``group``, ``mask`` and ``other`` are deprecated and they
+Parameters ``acl``, ``user``, ``group``, ``mask`` and ``other`` are deprecated and they
-will be removed in future versions. Please use ``acl`` parameter instead.
+will be removed in future versions. Please use ``entry`` parameter instead.
 
 
 EXAMPLES
@@ -49,27 +63,38 @@ EXAMPLES
         --default \
         --recursive \
         --remove \
-        --acl user:alice:rwx \
+        --entry user:alice:rwx \
-        --acl user:bob:r-x \
+        --entry user:bob:r-x \
-        --acl group:project-group:rwx \
+        --entry group:project-group:rwx \
-        --acl group:some-other-group:r-x \
+        --entry group:some-other-group:r-x \
-        --acl mask::r-x \
+        --entry mask::r-x \
-        --acl other::r-x
+        --entry other::r-x
 
     # give Alice read-only access to subdir,
     # but don't allow her to see parent content.
 
     __acl /srv/project2 \
         --remove \
-        --acl default:group:secret-project:rwx \
+        --entry default:group:secret-project:rwx \
-        --acl group:secret-project:rwx \
+        --entry group:secret-project:rwx \
-        --acl user:alice:--x
+        --entry user:alice:--x
 
     __acl /srv/project2/subdir \
         --default \
         --remove \
-        --acl group:secret-project:rwx \
+        --entry group:secret-project:rwx \
-        --acl user:alice:r-x
+        --entry user:alice:r-x
+
+    # read acl from stdin
+    echo 'user:alice:rwx' \
+        | __acl /path/to/directory --source -
+
+    # create/change directory too
+    __acl /path/to/directory \
+        --default \
+        --remove \
+        --directory root:root:770 \
+        --entry user:nobody:rwx
 
 
 AUTHORS

cdist/conf/type/__acl/manifest

@@ -0,0 +1,11 @@
+#!/bin/sh -e
+
+for p in file directory
+do
+    [ ! -f "$__object/parameter/$p" ] && continue
+
+    "__$p" "/$__object_id" \
+        --owner "$( awk -F: '{print $1}' "$__object/parameter/$p" )" \
+        --group "$( awk -F: '{print $2}' "$__object/parameter/$p" )" \
+        --mode  "$( awk -F: '{print $3}' "$__object/parameter/$p" )"
+done

cdist/conf/type/__acl/parameter/deprecated/acl

@@ -0,0 +1 @@
+see manual for details

cdist/conf/type/__acl/parameter/optional

@@ -1,2 +1,5 @@
 mask
 other
+source
+file
+directory

cdist/conf/type/__acl/parameter/optional_multiple

@@ -1,3 +1,4 @@
+entry
 acl
 user
 group

cdist/conf/type/__apt_unattended_upgrades/man.rst

@@ -0,0 +1,68 @@
+cdist-type__apt_unattended_upgrades(7)
+======================================
+
+NAME
+----
+cdist-type__apt_unattended_upgrades - automatic installation of updates
+
+
+DESCRIPTION
+-----------
+
+Install and configure unattended-upgrades package.
+
+For more information see https://wiki.debian.org/UnattendedUpgrades.
+
+
+OPTIONAL MULTIPLE PARAMETERS
+----------------------------
+option
+   Set options for unattended-upgrades. See examples.
+
+   Supported options with default values (as of 2020-01-17) are:
+
+   - AutoFixInterruptedDpkg, default is "true"
+   - MinimalSteps, default is "true"
+   - InstallOnShutdown, default is "false"
+   - Mail, default is "" (empty)
+   - MailOnlyOnError, default is "false"
+   - Remove-Unused-Kernel-Packages, default is "true"
+   - Remove-New-Unused-Dependencies, default is "true"
+   - Remove-Unused-Dependencies, default is "false"
+   - Automatic-Reboot, default is "false"
+   - Automatic-Reboot-WithUsers, default is "true"
+   - Automatic-Reboot-Time, default is "02:00"
+   - SyslogEnable, default is "false"
+   - SyslogFacility, default is "daemon"
+   - OnlyOnACPower, default is "true"
+   - Skip-Updates-On-Metered-Connections, default is "true"
+   - Verbose, default is "false"
+   - Debug, default is "false"
+
+blacklist
+   Python regular expressions, matching packages to exclude from upgrading.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __apt_unattended_upgrades \
+        --option Mail=root \
+        --option MailOnlyOnError=true \
+        --blacklist multipath-tools \
+        --blacklist open-iscsi
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option) any
+later version.

cdist/conf/type/__apt_unattended_upgrades/manifest

@@ -0,0 +1,80 @@
+#!/bin/sh -e
+#
+# 2020 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+__package unattended-upgrades
+
+export require='__package/unattended-upgrades'
+
+# in normal circumstances 20auto-upgrades is managed
+# by debconf and it can only contain these lines
+
+__file /etc/apt/apt.conf.d/20auto-upgrades \
+    --owner root \
+    --group root \
+    --mode 644 \
+    --source - << EOF
+APT::Periodic::Update-Package-Lists "1";
+APT::Periodic::Unattended-Upgrade "1";
+EOF
+
+# lets not write into upstream 50unattended-upgrades file,
+# but use our own config file to avoid clashes
+
+conf_file='/etc/apt/apt.conf.d/51unattended-upgrades-cdist'
+
+conf='# this file is managed by cdist'
+
+if [ -f "$__object/parameter/option" ]
+then
+    o=''
+
+    while read -r l
+    do
+        o="$( printf '%s\nUnattended-Upgrade::%s "%s";\n' "$o" "${l%%=*}" "${l#*=}" )"
+    done \
+        < "$__object/parameter/option"
+
+    conf="$( printf '%s\n%s\n' "$conf" "$o" )"
+fi
+
+if [ -f "$__object/parameter/blacklist" ]
+then
+    b='Unattended-Upgrade::Package-Blacklist {'
+
+    while read -r l
+    do
+        b="$( printf '%s\n"%s";\n' "$b" "$l" )"
+    done \
+        < "$__object/parameter/blacklist"
+
+    conf="$( printf '%s\n%s\n}\n' "$conf" "$b" )"
+fi
+
+if [ "$( echo "$conf" | wc -l )" -gt 1 ]
+then
+    echo "$conf" \
+        | __file "$conf_file" \
+            --owner root \
+            --group root \
+            --mode 644 \
+            --source -
+else
+    __file "$conf_file" --state absent
+fi

cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple

@@ -0,0 +1,2 @@
+option
+blacklist

cdist/conf/type/__line/explorer/state

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2018 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,6 +19,14 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+if [ -f "$__object/parameter/file" ]; then
+   file=$(cat "$__object/parameter/file")
+else
+   file="/$__object_id"
+fi
+
+[ -f "$file" ] || exit 0
+
 if [ -f "$__object/parameter/before" ]; then
    position="before"
 elif [ -f "$__object/parameter/after" ]; then
@@ -33,63 +42,56 @@ else
    needle="line"
 fi
 
-if [ -f "$__object/parameter/file" ]; then
-   file="$(cat "$__object/parameter/file")"
-else
-   file="/$__object_id"
-fi
-
-if [ ! -f "$file" ]; then
-  echo "file_missing"
-  exit 0
-fi
-
 awk -v position="$position" -v needle="$needle" '
 function _find(_text, _pattern) {
    if (needle == "regex") {
       return match(_text, _pattern)
    } else {
-      return index(_text, _pattern)
+      return index(_text, _pattern) == 1
    }
 }
 BEGIN {
    getline anchor < (ENVIRON["__object"] "/parameter/" position)
    getline pattern < (ENVIRON["__object"] "/parameter/" needle)
-   state = "absent"
+
+   found_line = 0
+   correct_pos = (position != "after" && position != "before")
 }
 {
    if (position == "after") {
       if (match($0, anchor)) {
          getline
          if (_find($0, pattern)) {
-            state = "present"
+            found_line++
-         }
+            correct_pos = 1
-         else {
-            state = "wrongposition"
-         }
             exit 0
          }
+      } else if (_find($0, pattern)) {
+         found_line++
       }
-   else if (position == "before") {
+   } else if (position == "before") {
       if (_find($0, pattern)) {
+         found_line++
          getline
          if (match($0, anchor)) {
-            state = "present"
+            correct_pos = 1
-         }
-         else {
-            state = "wrongposition"
-         }
             exit 0
          }
       }
-   else {
+   } else {
       if (_find($0, pattern)) {
-         state = "present"
+         found_line++
          exit 0
       }
    }
 }
 END {
-   print state
+   if (found_line && correct_pos) {
+      print "present"
+   } else if (found_line) {
+      print "wrongposition"
+   } else {
+      print "absent"
+   }
 }
 ' "$file"

cdist/conf/type/__line/gencode-remote

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2018 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -23,9 +24,20 @@ if [ -f "$__object/parameter/before" ] && [ -f "$__object/parameter/after" ]; th
    exit 1
 fi
 
+if [ -f "$__object/parameter/file" ]; then
+   file="$(cat "$__object/parameter/file")"
+else
+   file="/$__object_id"
+fi
+
 state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
 
+if [ -z "$state_is" ]; then
+   printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2
+   exit 1
+fi
+
 if [ "$state_should" = "$state_is" ]; then
    # nothing to do
    exit 0
@@ -46,12 +58,6 @@ else
    needle="line"
 fi
 
-if [ -f "$__object/parameter/file" ]; then
-   file="$(cat "$__object/parameter/file")"
-else
-   file="/$__object_id"
-fi
-
 add=0
 remove=0
 case "$state_should" in
@@ -104,10 +110,12 @@ BEGIN {
       if (anchor && match(\$0, anchor)) {
          if (position == "before") {
             print line
+            add = 0
             print
          } else if (position == "after") {
             print
             print line
+            add = 0
          }
          next
       }
@@ -115,7 +123,7 @@ BEGIN {
    print
 }
 END {
-   if (add && position == "end") {
+   if (add) {
       print line
    }
 }

cdist/conf/type/__postgres_database/gencode-remote

@@ -43,10 +43,14 @@ if [ "$state_should" != "$state_is" ]; then
          if [ -f "$__object/parameter/owner" ]; then
             owner="-O \"$(cat "$__object/parameter/owner")\""
          fi
-         echo "su - '$postgres_user' -c \"createdb $owner \"$name\"\""
+         cat << EOF
+su - '$postgres_user' -c "createdb $owner \"$name\""
+EOF
       ;;
       absent)
-         echo "su - '$postgres_user' -c \"dropdb \"$name\"\""
+         cat << EOF
+su - '$postgres_user' -c "dropdb \"$name\""
+EOF
       ;;
    esac
 fi

cdist/conf/type/__postgres_role/gencode-remote

@@ -53,11 +53,13 @@ case "$state_should" in
         done
 
         [ -n "$password" ] && password="PASSWORD '$password'"
-
+        cat << EOF
-        cmd="CREATE ROLE \"$name\" WITH $password $booleans"
+su - '$postgres_user' -c "psql postgres -wc 'CREATE ROLE \"$name\" WITH $password $booleans;'"
-        echo "su - '$postgres_user' -c \"psql postgres -wc \\\"$cmd\\\"\""
+EOF
     ;;
     absent)
-        echo "su - '$postgres_user' -c \"dropuser \\\"$name\\\"\""
+        cat << EOF
+su - '$postgres_user' -c "dropuser \"$name\""
+EOF
     ;;
 esac

cdist/exec/util.py

@@ -24,6 +24,7 @@ import os
 from tempfile import TemporaryFile
 
 import cdist
+import cdist.configuration
 
 
 # IMPORTANT:
@@ -200,3 +201,9 @@ def resolve_conf_dirs(configuration, add_conf_dirs):
         conf_dirs.extend(add_conf_dirs)
     conf_dirs = set(conf_dirs)
     return conf_dirs
+
+
+def resolve_conf_dirs_from_config_and_args(args):
+    cfg = cdist.configuration.Configuration(args)
+    configuration = cfg.get_config(section='GLOBAL')
+    return resolve_conf_dirs(configuration, args.conf_dir)

cdist/info.py

@@ -53,10 +53,7 @@ class Info(object):
 
     @classmethod
     def commandline(cls, args):
-        cfg = cdist.configuration.Configuration(args)
+        conf_dirs = util.resolve_conf_dirs_from_config_and_args(args)
-        configuration = cfg.get_config(section='GLOBAL')
-        conf_dirs = util.resolve_conf_dirs(configuration,
-                                           args.conf_dir)
         c = cls(conf_dirs, args)
         c.run()
 
@@ -170,7 +167,8 @@ class Info(object):
 
     def run(self):
         rv = []
-        for conf_path in self.conf_dirs:
+        for cp in self.conf_dirs:
+            conf_path = os.path.expanduser(cp)
             if self.all or self.display_global_explorers:
                 rv.extend((x, 'E', ) for x in self._get_global_explorers(
                     conf_path))

cdist/preos.py

@@ -5,8 +5,9 @@ import inspect
 import argparse
 import cdist
 import logging
-import re
 import cdist.argparse
+import cdist.configuration
+import cdist.exec.util as util
 
 
 _PREOS_CALL = "commandline"
@@ -24,16 +25,6 @@ def extend_plugins_path(dirs):
             _PLUGINS_PATH.append(preos_dir)
 
 
-cdist_home = cdist.home_dir()
-if cdist_home:
-    extend_plugins_path((cdist_home, ))
-x = 'CDIST_PATH'
-if x in os.environ:
-    vals = re.split(r'(?<!\\):', os.environ[x])
-    vals = [x for x in vals if x]
-    extend_plugins_path(vals)
-
-
 def preos_plugin(obj):
     """It is preos if _PREOS_MARKER is True and has _PREOS_CALL."""
     if hasattr(obj, _PREOS_MARKER):
@@ -102,6 +93,9 @@ class PreOS(object):
                             help=('Add configuration directory (one that '
                                   'contains "preos" subdirectory)'),
                             action='append')
+        parser.add_argument('-g', '--config-file',
+                            help='Use specified custom configuration file.',
+                            dest="config_file", required=False)
         parser.add_argument('-L', '--list-preoses',
                             help='List available PreOS-es',
                             action='store_true', default=False)
@@ -110,8 +104,9 @@ class PreOS(object):
         cdist.argparse.handle_loglevel(args)
         log.debug("preos args : {}".format(args))
 
-        if args.conf_dir:
+        conf_dirs = util.resolve_conf_dirs_from_config_and_args(args)
-            extend_plugins_path(args.conf_dir)
+
+        extend_plugins_path(conf_dirs)
         sys.path.extend(_PLUGINS_PATH)
         cls.preoses = find_preoses()
 

docs/changelog

@@ -1,6 +1,20 @@
 Changelog
 ---------
 
+6.5.0: 2020-01-23
+	* Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar)
+	* Core: preos: Fix missing configuration file usage, support -g, --config-file option (Darko Poljak)
+	* Core info command: Support tilde expansion of conf directories (Darko Poljak)
+	* Types __postgres_*: Fix edge cases in quoted identifiers (Timothée Floure)
+	* New type: __apt_unattended_upgrades (Ander Punnar)
+	* Type __line: Bugfixes: (Dennis Camera)
+		- ensure the line is only added once
+		- always add line to end if anchor is not found
+		- match line at the beginning when not regex
+		- fix incorrect 'wrongposition' in state explorer
+		- produce error when file does not exist
+	* Type __acl: Add --source, --file and --directory parameters (Ander Punnar)
+
 6.4.0: 2020-01-04
 	* Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius)
 	* Type __install_chroot_umount: Bugfix: type was not using __chroot_umount/manifest (Steven Armstrong)

docs/src/conf.py

@@ -56,7 +56,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'cdist'
-copyright = 'ungleich GmbH 2019'
+copyright = 'ungleich GmbH 2020'
 # author = 'Darko Poljak'
 
 # The version info for the project you're documenting, acts as replacement for

docs/src/man1/cdist.rst

@@ -59,7 +59,9 @@ SYNOPSIS
                          [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t]
                          [host [host ...]]
 
-    cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-L] [preos] ...
+    cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-g CONFIG_FILE]
+                [-L]
+                [preos] ...
 
     cdist preos [preos-options] debian [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
                                        [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
@@ -464,6 +466,9 @@ Create PreOS.
 **-c CONF_DIR, --conf-dir CONF_DIR**
     Add configuration directory (one that contains "preos" subdirectory).
 
+**-g CONFIG_FILE, --config-file CONFIG_FILE**
+    Use specified custom configuration file.
+
 **-L, --list-preoses**
     List available PreOS-es.