Merge branch 'master' into beta

.gitlab-ci.yml

@@ -1,6 +1,8 @@
 stages:
     - test
 
+image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
+
 unit_tests:
     stage: test
     script:

README

@@ -1,7 +0,0 @@
-cdist
------
-
-cdist is a usable configuration management system.
-
-For the web documentation have a look at https://www.cdi.st/
-or at docs/src for reStructuredText manual.

README.md

@@ -0,0 +1,31 @@
+# cdist
+
+**cdist** is a usable configuration management system.
+
+It adheres to the [**KISS principle**](https://en.wikipedia.org/wiki/KISS_principle)
+and is being used in small up to enterprise grade environments.
+
+For more information have a look at [**homepage**](https://cdi.st)
+or at **``docs/src``** for manual in **reStructuredText** format.
+
+## Contributing
+
+Merge/Pull requests can be made in both
+[upstream **GitLab**](https://code.ungleich.ch/ungleich-public/cdist/merge_requests)
+(managed by [**ungleich**](https://ungleich.ch))
+and [**GitHub** project](https://github.com/ungleich/cdist/pulls).
+
+Issues can be made and other project management activites happen
+[**only in GitLab**](https://code.ungleich.ch/ungleich-public/cdist)
+(needs [**ungleich** account](https://account.ungleich.ch)).
+
+For community-maintained types there is
+[**cdist-contrib** project](https://code.ungleich.ch/ungleich-public/cdist-contrib).
+
+## Participating
+
+IRC: ``#cdist`` @ freenode
+
+Matrix: ``#cdist:ungleich.ch``
+
+Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist

cdist/conf/explorer/cpu_cores

@@ -32,6 +32,10 @@ case "$os" in
         sysctl -n hw.ncpuonline
     ;;
 
+    "freebsd"|"netbsd")
+        sysctl -n hw.ncpu
+    ;;
+
     *)
         if [ -r /proc/cpuinfo ]; then
             cores="$(grep "core id" /proc/cpuinfo | sort | uniq | wc -l)"

cdist/conf/explorer/memory

@@ -29,7 +29,7 @@ case "$os" in
         echo "$(sysctl -n hw.memsize)/1024" | bc
     ;;
 
-    "openbsd")
+    *"bsd")
         echo "$(sysctl -n hw.physmem) / 1048576" | bc
     ;;
 

cdist/conf/type/__cdist/manifest

@@ -37,6 +37,7 @@ source="$(cat "$__object/parameter/source")"
 # out of it
 home=/home/$username
 
+# shellcheck disable=SC2086
 __user "$username" --home "$home" $shell
 
 require="__user/$username" __directory "$home" \

cdist/conf/type/__motd/gencode-remote

@@ -29,7 +29,18 @@ case "$os" in
         echo "uname -snrvm > /var/run/motd"
         echo "cat /etc/motd.tail >> /var/run/motd"
     ;;
+    freebsd)
+        # FreeBSD only updates /etc/motd on boot,
+	# as seen in /etc/rc.d/motd
+	echo "uname -sri > /etc/motd"
+	echo "cat /etc/motd.template >> /etc/motd"
+	# FreeBSD 13 starts treating motd slightly different from previous
+	# versions this ensures hosts have the expected config.
+	echo "rm /etc/motd.template || true"
+	echo "service motd start"
+    ;;
     *)
+        # Other OS tend to treat /etc/motd statically
         exit 0
     ;;
 esac

cdist/conf/type/__motd/man.rst

@@ -10,6 +10,13 @@ DESCRIPTION
 -----------
 This cdist type allows you to easily setup /etc/motd.
 
+.. note::
+      In some OS, motd is a bit special, check `motd(5)`.
+      Currently Debian, Devuan, Ubuntu and FreeBSD are taken into account.
+      If your OS of choice does something besides /etc/motd, check the source
+      and contribute support for it.
+      Otherwise it will likely just work.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -20,6 +27,7 @@ OPTIONAL PARAMETERS
 -------------------
 source
    If supplied, copy this file from the host running cdist to the target.
+   If source is '-' (dash), take what was written to stdin as the file content.
    If not supplied, a default message will be placed onto the target.
 
 
@@ -34,6 +42,15 @@ EXAMPLES
     # Supply source file from a different type
     __motd --source "$__type/files/my-motd"
 
+    # Supply source from stdin
+    __motd --source "-" <<EOF
+    Take this kiss upon the brow!
+    And, in parting from you now,
+    Thus much let me avow-
+    You are not wrong, who deem
+    That my days have been a dream
+    EOF
+
 
 AUTHORS
 -------
@@ -42,7 +59,7 @@ Nico Schottelius <nico-cdist--@--schottelius.org>
 
 COPYING
 -------
-Copyright \(C) 2011 Nico Schottelius. You can redistribute it
+Copyright \(C) 2020 Nico Schottelius. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.

cdist/conf/type/__motd/manifest

@@ -34,9 +34,17 @@ os=$(cat "$__global/explorer/os")
 
 case "$os" in
    debian|ubuntu|devuan)
+      # Debian-based systems use /etc/motd.tail as a template
       destination=/etc/motd.tail
    ;;
+   freebsd)
+      # FreeBSD uses motd.template to prepend system information on boot
+      # (this actually only applies starting with version 13,
+      #  but we fix that for whatever version in gencode-remote)
+      destination=/etc/motd.template
+   ;;
    *)
+      # Most UNIX systems, including other Linux and OpenBSD just use /etc/motd
       destination=/etc/motd
    ;;
 esac

cdist/conf/type/__openldap_server/man.rst

@@ -92,6 +92,9 @@ tls-ca
     Required if `tls-cert` is defined.
     Path in the remote hosts to the PEM-encoded CA certificate file.
 
+extra-config
+    Custom settings to be added in `slapd.conf(5)`.
+
 
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------

cdist/conf/type/__openldap_server/manifest

@@ -9,6 +9,7 @@ slapd_modules=$(cat "${__object}/parameter/module" 2>/dev/null || true)
 schemas=$(cat "${__object}/parameter/schema")
 slapd_urls=$(tr '\n' ' ' < "${__object}/parameter/slapd-url")
 tls_cipher_suite=$(cat "${__object}/parameter/tls-cipher-suite" 2>/dev/null || true)
+extra_config=$(cat "${__object}/parameter/extra-config" || true)
 
 
 os="$(cat "${__global}/explorer/os")"
@@ -168,6 +169,7 @@ if [ -z "${_skip_letsencrypt_cert}" ]; then
         staging=""
     fi
 
+    # shellcheck disable=SC2086
     __letsencrypt_cert "${name}" --admin-email "${admin_email}" \
         --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R openldap:openldap ${SLAPD_DIR}/sasl2 && service slapd restart" \
         --automatic-renewal ${staging}
@@ -230,6 +232,8 @@ index uid,memberUid eq,pres,sub
 index nisMapName,nisMapEntry eq,pres,sub
 index entryCSN,entryUUID eq
 
+${extra_config}
+
 serverid ${serverid}
 EOF
 

cdist/conf/type/__openldap_server/parameter/optional

@@ -6,3 +6,4 @@ tls-cipher-suite
 tls-cert
 tls-privkey
 tls-ca
+extra-config

cdist/conf/type/__pf_apply/deprecated

@@ -0,0 +1 @@
+Consider moving to __pf_apply_anchor. Get in touch if you need __pf_apply.

cdist/conf/type/__pf_apply_anchor/gencode-remote

@@ -1,6 +1,6 @@
-#!/bin/sh
+#!/bin/sh -e
 #
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
+# 2016 Kamila Součková (coding at kamila.is)
 #
 # This file is part of cdist.
 #
@@ -18,24 +18,16 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Get the 256 bit SHA2 checksum of the pf ruleset on the target host.
+# Apply pf(4) ruleset on *BSD
 #
 
-# Debug
+ANCHORS_DIR="/etc/pf.d"
-#exec >&2
-#set -x
 
-# Check /etc/rc.conf for pf's configuration file name. Default to /etc/pf.conf
+if [ -f "${__object}/parameter/anchor_name" ]; then
-# See if file exists and if so, get checksum
+    anchor_name="$(cat "${__object}/parameter/anchor_name")"
-
+else
-RC="/etc/rc.conf"
+    anchor_name="${__object_id}"
-TMP="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')"
-PFCONF="${TMP:-"/etc/pf.conf"}"
-
-if [ -f "${PFCONF}" ]; then	# The pf config file exists, find its cksum.
-	cksum -o 1 "${PFCONF}" | cut -d= -f2 | awk '{print $1}'
 fi
+anchor_file="${ANCHORS_DIR}/${anchor_name}"
 
-# Debug
+echo "pfctl -a \"${anchor_name}\" -f \"${anchor_file}\""
-#set +x
-

cdist/conf/type/__pf_apply_anchor/man.rst

@@ -0,0 +1,62 @@
+cdist-type__pf_apply_anchor(7)
+==============================
+
+NAME
+----
+cdist-type__pf_apply_anchor - Apply a pf(4) anchor on $__target_host
+
+
+DESCRIPTION
+-----------
+This type is used on \*BSD systems to manage anchors for the pf firewall.
+
+Notice this type does not take care of copying the ruleset, that must be
+done by the user with, e.g. `__file`.
+
+
+OPTIONAL PARAMETERS
+-------------------
+anchor_name
+   The name of the anchor to apply. If not set, `${__object_id}` is used.
+   This type requires `/etc/pf.d/${anchor_name}` to exist on
+   `$__target_host`.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Copy anchor file to ${__target_host}
+    __file "/etc/pf.d/80_dns" --source - <<EOF
+    # Managed remotely, changes will be lost
+
+    pass quick proto {tcp,udp} from any to any port domain
+    EOF
+
+    # Apply the anchor
+    require="__file/etc/pf.d/80_dns" __pf_apply_anchor 80_dns
+    # This is roughly equivalent to:
+    #   pfctl -a "${anchor_name}" -f "/etc/pf.d/${anchor_name}"
+
+
+SEE ALSO
+--------
+:strong:`pf`\ (4)
+
+
+AUTHORS
+-------
+Evilham <contact--@--evilham.com>
+Kamila Součková <coding--@--kamila.is>
+Jake Guffey <jake.guffey--@--eprotex.com>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Evilham.
+Copyright \(C) 2016 Kamila Součková.
+Copyright \(C) 2012 Jake Guffey. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__pf_apply_anchor/parameter/optional

@@ -0,0 +1 @@
+anchor_name

cdist/conf/type/__pf_ruleset/gencode-local

@@ -1,81 +0,0 @@
-#!/bin/sh -e
-#
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Manage pf(4) on *BSD
-#
-
-# Debug
-#exec >&2
-#set -x
-
-# Send files to $__target_host via $__remote_copy
-
-uname=$(uname)   # Need to know what the cdist host is running so we know how to compute the ruleset's checksum
-state=$(cat "$__object/parameter/state")
-
-if [ "$state" = "absent" ]; then   # There is nothing more for a *local* script to do
-   exit 0
-fi
-
-if [ -f "$__object/parameter/source" ]; then
-   source=$(cat "$__object/parameter/source")
-fi
-
-rcvar=$(cat "$__object/explorer/rcvar")
-cksum=$(cat "$__object/explorer/cksum")
-
-
-cat <<EOF
-case $uname in
-   Darwin)
-      currentSum=\$(cksum -o 1 ${source} | cut '-d ' -f1)
-      ;;
-   Linux)
-      currentSum=\$(cksum ${source} | cut '-d ' -f1)
-      ;;
-   FreeBSD)
-      currentSum=\$(cksum -o 1 ${source} | cut -d= -f2 | sed 's/ //g')
-      ;;
-   *)
-      echo "Sorry, I do not know how to find a cksum on ${uname}." >&2
-      exit 1
-      ;;
-esac
-
-# IPv6 fix
-if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$')
-then
-    my_target_host="[${__target_host}]"
-else
-    my_target_host="${__target_host}"
-fi
-
-if [ -n "${cksum}" ]; then
-   if [ ! "\${currentSum}" = "${cksum}" ]; then
-      $__remote_copy "${source}" "\${my_target_host}:${rcvar}.new"
-   fi
-else # File just doesn't exist yet
-   $__remote_copy "${source}" "\${my_target_host}:${rcvar}.new"
-fi
-EOF
-
-# Debug
-#exec +x
-

cdist/conf/type/__pf_ruleset/man.rst

@@ -10,6 +10,9 @@ DESCRIPTION
 -----------
 This type is used on \*BSD systems to manage the pf firewall's ruleset.
 
+It will also enable and disable the pf firewall as requested in the `state`
+parameter.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -20,9 +23,8 @@ state
 OPTIONAL PARAMETERS
 -------------------
 source
-   If supplied, use to define the ruleset to load onto the $__target_host for pf(4).
+   Required when state is "present".
-   Note that this type is almost useless without a ruleset defined, but it's technically not
+   Defines the ruleset to load onto the $__target_host for `pf(4)`.
-   needed, e.g. for the case of disabling the firewall temporarily.
 
 
 EXAMPLES
@@ -30,10 +32,10 @@ EXAMPLES
 
 .. code-block:: sh
 
-    # Remove the current ruleset in place
+    # Remove the current ruleset in place and disable pf
     __pf_ruleset --state absent
 
-    # Enable the firewall with the ruleset defined in $__manifest/files/pf.conf
+    # Enable pf with the ruleset defined in $__manifest/files/pf.conf
     __pf_ruleset --state present --source $__manifest/files/pf.conf
 
 
@@ -44,11 +46,13 @@ SEE ALSO
 
 AUTHORS
 -------
+Kamila Součková <coding--@--kamila.is>
 Jake Guffey <jake.guffey--@--eprotex.com>
 
 
 COPYING
 -------
+Copyright \(C) 2016 Kamila Součková.
 Copyright \(C) 2012 Jake Guffey. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the

cdist/conf/type/__pf_ruleset/manifest

@@ -1,6 +1,6 @@
 #!/bin/sh -e
 #
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
+# 2016 Kamila Součková (coding at kamila.is)
 #
 # This file is part of cdist.
 #
@@ -21,29 +21,26 @@
 # Manage pf(4) on *BSD
 #
 
-# Debug
+rcvar="$(cat "${__object}/explorer/rcvar")"
-#exec >&2
+state="$(cat "${__object}/parameter/state")"
-#set -x
+if [ -f "${__object}/parameter/source" ]; then
-
+    source="$(cat "${__object}/parameter/source")"
-# Remove ${rcvar} in the case of --state absent
-
-state=$(cat "$__object/parameter/state")
-rcvar=$(cat "$__object/explorer/rcvar")
-
-if [ "$state" = "present" ]; then   # There is nothing more for a *remote* script to do
-   exit 0
-elif [ "$state" = "absent" ]; then
-   # --state absent, so ensure that .new doesn't exist and that conf is renamed to .old
-   cat <<EOF
-   if [ -f "${rcvar}.new" ]; then
-      rm "${rcvar}.new"
-   fi
-   if [ -f "${rcvar}" ]; then
-      mv "${rcvar}" "${rcvar}.old"
-   fi
-EOF
-else
-   echo "Unknown state ${state}!" >&2
-   exit 1
 fi
 
+if [ "${state}" = "absent" ]; then
+    action="/etc/rc.d/pf stop"
+else
+    action="/etc/rc.d/pf reload || /etc/rc.d/pf start"
+fi
+
+__key_value __pf_ruleset/rcvar \
+    --state "${state}" \
+    --file /etc/rc.conf \
+    --delimiter "=" \
+    --key "pf_enable" \
+    --value "YES"
+
+require="__key_value/__pf_ruleset/rcvar" __config_file "${rcvar}" \
+    --source "${source}" \
+    --state "${state}" \
+    --onchange "${action}"

cdist/conf/type/__postfix/manifest

@@ -19,16 +19,4 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-
+__package postfix --state present
-os=$(cat "$__global/explorer/os")
-
-case "$os" in
-   alpine|ubuntu|debian|archlinux|suse|scientific|centos|devuan)
-      __package postfix --state present
-   ;;
-   *)
-      echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
-      echo "Please contribute an implementation for it if you can." >&2
-      exit 1
-   ;;
-esac

cdist/conf/type/__postgres_database/gencode-remote

@@ -43,8 +43,29 @@ if [ "$state_should" != "$state_is" ]; then
          if [ -f "$__object/parameter/owner" ]; then
             owner="-O \"$(cat "$__object/parameter/owner")\""
          fi
+
+         template=""
+         if [ -f "$__object/parameter/template" ]; then
+            template="--template \"$(cat "$__object/parameter/template")\""
+         fi
+
+         encoding=""
+         if [ -f "$__object/parameter/encoding" ]; then
+            encoding="--encoding \"$(cat "$__object/parameter/encoding")\""
+         fi
+
+         lc_collate=""
+         if [ -f "$__object/parameter/lc-collate" ]; then
+            lc_collate="--lc-collate \"$(cat "$__object/parameter/lc-collate")\""
+         fi
+
+         lc_ctype=""
+         if [ -f "$__object/parameter/lc-ctype" ]; then
+            lc_ctype="--lc-ctype \"$(cat "$__object/parameter/lc-ctype")\""
+         fi
+
          cat << EOF
-su - '$postgres_user' -c "createdb $owner \"$name\""
+su - '$postgres_user' -c "createdb $owner \"$name\" $template $encoding $lc_collate $lc_ctype"
 EOF
       ;;
       absent)

cdist/conf/type/__postgres_database/man.rst

@@ -14,10 +14,22 @@ This cdist type allows you to create or drop postgres databases.
 OPTIONAL PARAMETERS
 -------------------
 state
-   either 'present' or 'absent', defaults to 'present'.
+   Either 'present' or 'absent', defaults to 'present'.
 
 owner
-   the role owning this database
+   Specifies the database user who will own the new database.
+
+encoding
+   Specifies the character encoding scheme to be used in this database.
+
+lc-collate
+   Specifies the LC_COLLATE setting to be used in this database.
+
+lc-ctype
+   Specifies the LC_CTYPE setting to be used in this database.
+
+template
+   Specifies the template database from which to build this database.
 
 
 EXAMPLES

cdist/conf/type/__postgres_database/parameter/optional

@@ -1,2 +1,6 @@
 state
 owner
+encoding
+lc-collate
+lc-ctype
+template

cdist/configuration.py

@@ -274,7 +274,8 @@ class Configuration(metaclass=Singleton):
              os.path.isfile(_local_config_file))):
         _local_config_file = os.path.join(
             os.environ.get('XDG_CONFIG_HOME',
-                           os.path.expanduser('~/.config/cdist')),
+                           os.path.expanduser('~/.config/')),
+            'cdist',
             _config_basename)
     _dist_config_file = os.path.join(
         os.path.abspath(os.path.join(os.path.dirname(cdist.__file__), "conf")),

cdist/exec/remote.py

@@ -280,9 +280,6 @@ class Remote(object):
         assert isinstance(command, (list, tuple)), (
                 "list or tuple argument expected, got: %s" % command)
 
-        if return_output and stdout is not subprocess.PIPE:
-            self.log.debug("return_output is True, ignoring stdout")
-
         close_stdout = False
         close_stderr = False
         if self.save_output_streams:

configuration/cdist.cfg.skeleton

@@ -19,6 +19,9 @@
 #     such as ':' for POSIX or ';' for Windows.
 #     If also specified at command line then values from command line are
 #     appended to this value.
+#     Notice that this works in a "last one wins" fashion, so if a type is redefined
+#     in multiple conf_dirs, the last one in which it is defined will be used.
+#     Consider using a unique prefix for your own roles if this can be an issue.
 # conf_dir = <dir1>:<dir2>
 #
 # init_manifest

docs/changelog

@@ -5,6 +5,19 @@ next:
 	* Core: Add trigger functionality (Nico Schottelius, Darko Poljak)
 	* Core: Implement core support for python types (Darko Poljak)
 
+6.5.5: 2020-05-01
+	* Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches)
+	* Type __postgres_database: Add encoding, lc-collate, lc-ctype, template parameters (Timothée Floure)
+	* Type __motd: Improve documentation and support for FreeBSD (Evil Ham)
+	* Type __openldap_server: Support extra config parameter (Evil Ham)
+	* Type __postfix: Automagically support more OSs by not checking too much (Evil Ham)
+	* New type: __pf_apply_anchor (Kamila Součková, Evil Ham)
+	* Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham)
+	* Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham)
+	* Configuration: Add notes to cdist.cfg.skeleton (Evil Ham)
+	* Explorers cpu_cores, memory: Improve *BSD support (Evil Ham)
+	* Core: Remove debug logging noise (Evil Ham)
+
 6.5.4: 2020-04-11
 	* Explorer init: Do not grep on non-existent init (Steven Armstrong)
 	* Core: Bugfix to preserve conf dirs order (Steven Armstrong)

other/ci/Dockerfile

@@ -0,0 +1,8 @@
+FROM alpine:latest
+
+COPY ./repositories /etc/apk/
+
+RUN apk update
+RUN apk upgrade
+RUN apk add python3 py3-pycodestyle rsync make shellcheck git
+RUN apk fix

other/ci/README.md

@@ -0,0 +1,3 @@
+This container is used for cdist's CI pipeline, and deployed in ungleich's docker registry at:
+
+	code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest

other/ci/repositories

@@ -0,0 +1,3 @@
+https://mirror.ungleich.ch/mirror/packages/alpine/edge/main
+https://mirror.ungleich.ch/mirror/packages/alpine/edge/community
+https://mirror.ungleich.ch/mirror/packages/alpine/edge/testing

scripts/cdist-dump

@@ -224,6 +224,7 @@ hor_line()
 if [ "${do_global_explorer}" ]
 then
     print_verbose 2 "Dumping global explorers"
+    # shellcheck disable=SC2086
     set -- "$@" ${or} \( \
                 -path "*/explorer/*" -a \
                 ! -path "*/conf/*" -a \