Merge branch 'to-upstream' into local

.gitlab-ci.yml

@@ -1,6 +1,8 @@
 stages:
     - test
 
+image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
+
 unit_tests:
     stage: test
     script:

cdist/conf/explorer/init

@@ -221,6 +221,7 @@ check_systemstarter() {
 
 check_sysvinit() (
 	init_path=${1:-/sbin/init}
+	test -x "${init_path}" || return 1
 	grep -q 'INIT_VERSION=sysvinit-[0-9.]*' "${init_path}" || return 1
 
 	# It is quite common to use SysVinit to stack other init systemd

cdist/conf/type/__cdist/manifest

@@ -37,6 +37,7 @@ source="$(cat "$__object/parameter/source")"
 # out of it
 home=/home/$username
 
+# shellcheck disable=SC2086
 __user "$username" --home "$home" $shell
 
 require="__user/$username" __directory "$home" \

cdist/conf/type/__consul_agent/manifest

@@ -1,7 +1,7 @@
 #!/bin/sh -e
 #
 # 2015 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2015-2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2015-2020 Nico Schottelius (nico-cdist at schottelius.org)
 # 2019 Timothée Floure (timothee.floure at ungleich.ch)
 #
 # This file is part of cdist.
@@ -37,10 +37,22 @@ fi
 # Those are default that might be overriden by os-specific logic.
 
 data_dir="/var/lib/consul"
-conf_dir="/etc/consul/conf.d"
+
-conf_file="config.json"
+
+
 tls_dir="$conf_dir/tls"
 
+case "$os" in
+    alpine)
+        conf_dir="/etc/consul"
+        conf_file="server.json"
+        ;;
+    *)
+        conf_dir="/etc/consul/conf.d"
+        conf_file="config.json"
+        ;;
+esac
+
 ###
 # Sane deployment, based on distribution package when available.
 
@@ -220,7 +232,7 @@ if [ -f "$__object/parameter/ca-file-source" ] || \
    [ -f "$__object/parameter/cert-file-source" ] || \
    [ -f "$__object/parameter/key-file-source" ]; then
 
-   requires="$config_deployment_requires" __directory $tls_dir \
+   requires="$config_deployment_requires" __directory "$tls_dir" \
      --owner root --group "$group" --mode 750 --state "$state"
 
    # Append to service restart requirements.

cdist/conf/type/__letsencrypt_cert/man.rst

@@ -59,13 +59,13 @@ MESSAGES
 --------
 
 change
-    Certificte was changed.
+    Certificate was changed.
 
 create
-    Certificte was created.
+    Certificate was created.
 
 remove
-    Certificte was removed.
+    Certificate was removed.
 
 EXAMPLES
 --------

cdist/conf/type/__motd/gencode-remote

@@ -29,7 +29,18 @@ case "$os" in
         echo "uname -snrvm > /var/run/motd"
         echo "cat /etc/motd.tail >> /var/run/motd"
     ;;
+    freebsd)
+        # FreeBSD only updates /etc/motd on boot,
+	# as seen in /etc/rc.d/motd
+	echo "uname -sri > /etc/motd"
+	echo "cat /etc/motd.template >> /etc/motd"
+	# FreeBSD 13 starts treating motd slightly different from previous
+	# versions this ensures hosts have the expected config.
+	echo "rm /etc/motd.template || true"
+	echo "service motd start"
+    ;;
     *)
+        # Other OS tend to treat /etc/motd statically
         exit 0
     ;;
 esac

cdist/conf/type/__motd/man.rst

@@ -10,6 +10,13 @@ DESCRIPTION
 -----------
 This cdist type allows you to easily setup /etc/motd.
 
+.. note::
+      In some OS, motd is a bit special, check `motd(5)`.
+      Currently Debian, Devuan, Ubuntu and FreeBSD are taken into account.
+      If your OS of choice does something besides /etc/motd, check the source
+      and contribute support for it.
+      Otherwise it will likely just work.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -20,6 +27,7 @@ OPTIONAL PARAMETERS
 -------------------
 source
    If supplied, copy this file from the host running cdist to the target.
+   If source is '-' (dash), take what was written to stdin as the file content.
    If not supplied, a default message will be placed onto the target.
 
 
@@ -34,6 +42,15 @@ EXAMPLES
     # Supply source file from a different type
     __motd --source "$__type/files/my-motd"
 
+    # Supply source from stdin
+    __motd --source "-" <<EOF
+    Take this kiss upon the brow!
+    And, in parting from you now,
+    Thus much let me avow-
+    You are not wrong, who deem
+    That my days have been a dream
+    EOF
+
 
 AUTHORS
 -------
@@ -42,7 +59,7 @@ Nico Schottelius <nico-cdist--@--schottelius.org>
 
 COPYING
 -------
-Copyright \(C) 2011 Nico Schottelius. You can redistribute it
+Copyright \(C) 2020 Nico Schottelius. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.

cdist/conf/type/__motd/manifest

@@ -34,9 +34,17 @@ os=$(cat "$__global/explorer/os")
 
 case "$os" in
    debian|ubuntu|devuan)
+      # Debian-based systems use /etc/motd.tail as a template
       destination=/etc/motd.tail
    ;;
+   freebsd)
+      # FreeBSD uses motd.template to prepend system information on boot
+      # (this actually only applies starting with version 13,
+      #  but we fix that for whatever version in gencode-remote)
+      destination=/etc/motd.template
+   ;;
    *)
+      # Most UNIX systems, including other Linux and OpenBSD just use /etc/motd
       destination=/etc/motd
    ;;
 esac

cdist/conf/type/__openldap_server/manifest

@@ -169,6 +169,7 @@ if [ -z "${_skip_letsencrypt_cert}" ]; then
         staging=""
     fi
 
+    # shellcheck disable=SC2086
     __letsencrypt_cert "${name}" --admin-email "${admin_email}" \
         --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R openldap:openldap ${SLAPD_DIR}/sasl2 && service slapd restart" \
         --automatic-renewal ${staging}

cdist/conf/type/__package_update_index/explorer/currage

@@ -24,18 +24,18 @@ case "$type" in
         if [ -f "/var/cache/apt/pkgcache.bin" ]; then
             echo $(($(date +"%s")-$(stat --format '%Y' /var/cache/apt/pkgcache.bin)))
         else
-            echo 0
+            echo -- -1
         fi
         ;;
     pacman)
         if [ -d "/var/lib/pacman/sync" ]; then
             echo $(($(date +"%s")-$(stat --format '%Y' /var/lib/pacman/sync)))
         else
-            echo 0
+            echo -- -1
         fi
         ;;
     alpine)
-        echo 0
+        echo -- -1
         ;;
     *)  echo "Your specified type ($type) is currently not supported." >&2
         echo "Please contribute an implementation for it if you can." >&2

cdist/conf/type/__package_update_index/gencode-remote

@@ -31,7 +31,8 @@ if [ -n "$maxage" ]; then
     if [ "$type" != "apt" ] && [ "$type" != "pacman" ]; then
         echo "ERROR: \"--maxage\" only supported for \"apt\" or \"pacman\" pkg-manager." >&2
         exit 1
-    elif [ "$currage" -lt "$maxage" ]; then
+    # do not exit if no value found (represented as -1)
+    elif [ "$currage" -ne -1 ] && [ "$currage" -lt "$maxage" ]; then
         exit 0 # no need to update
     fi
 fi

cdist/conf/type/__pyvenv/gencode-remote

@@ -37,11 +37,21 @@ mode="$(cat "$__object/parameter/mode")"
 destination="/$__object_id"
 venvparams="$(cat "$__object/parameter/venvparams")"
 pyvenvparam="$__object/parameter/pyvenv"
+
+os=$(cat "$__global/explorer/os")
+
 if [ -f "$pyvenvparam" ]
 then
     pyvenv=$(cat "$pyvenvparam")
 else
-    pyvenv="pyvenv"
+    case "$os" in
+        alpine) # no pyvenv on alpine - I assume others will follow
+            pyvenv="python3 -m venv"
+            ;;
+        *)
+            pyvenv="pyvenv"
+            ;;
+    esac
 fi
 
 case $state_should in

cdist/exec/util.py

@@ -22,6 +22,7 @@
 import subprocess
 import os
 from tempfile import TemporaryFile
+from collections import OrderedDict
 
 import cdist
 import cdist.configuration
@@ -199,7 +200,9 @@ def resolve_conf_dirs(configuration, add_conf_dirs):
 
     if add_conf_dirs:
         conf_dirs.extend(add_conf_dirs)
-    conf_dirs = set(conf_dirs)
+
+    # Remove duplicates.
+    conf_dirs = list(OrderedDict.fromkeys(conf_dirs))
     return conf_dirs
 
 

docs/changelog

@@ -1,8 +1,17 @@
 Changelog
 ---------
 
-next:
+6.5.4: 2020-04-11
+	* Explorer init: Do not grep on non-existent init (Steven Armstrong)
+	* Core: Bugfix to preserve conf dirs order (Steven Armstrong)
+
+6.5.3: 2020-04-03
+	* Type __cron: Make non parallel due to race condition (Nico Schottelius)
+	* Type __pyvenv: Use python3 -m venv on Alpine (Nico Schottelius)
 	* Type __user: Fix missing shadow for alpine (llnu)
+	* Type __consule_agent: Make conf_dir dependent on OS - fixes Alpine (Nico Schottelius)
+	* Type __letsencrypt_cert: Fix typo (Andrew Schleifer)
+	* Type __package_update_index: Fix maxage false positives (Matthias Stecher)
 
 6.5.2: 2020-02-27
 	* Type __update_alternatives: Add state explorer (Ander Punnar)

other/ci/Dockerfile

@@ -0,0 +1,8 @@
+FROM alpine:latest
+
+COPY ./repositories /etc/apk/
+
+RUN apk update
+RUN apk upgrade
+RUN apk add python3 py3-pycodestyle rsync make shellcheck git
+RUN apk fix

other/ci/README.md

@@ -0,0 +1,3 @@
+This container is used for cdist's CI pipeline, and deployed in ungleich's docker registry at:
+
+	code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest

other/ci/repositories

@@ -0,0 +1,3 @@
+https://mirror.ungleich.ch/mirror/packages/alpine/edge/main
+https://mirror.ungleich.ch/mirror/packages/alpine/edge/community
+https://mirror.ungleich.ch/mirror/packages/alpine/edge/testing

scripts/cdist-dump

@@ -224,6 +224,7 @@ hor_line()
 if [ "${do_global_explorer}" ]
 then
     print_verbose 2 "Dumping global explorers"
+    # shellcheck disable=SC2086
     set -- "$@" ${or} \( \
                 -path "*/explorer/*" -a \
                 ! -path "*/conf/*" -a \