Merge branch 'master' into beta

cdist/conf/explorer/cpu_cores

@@ -33,6 +33,7 @@ case "$os" in
     ;;
 
     "freebsd"|"netbsd")
+        PATH=$(getconf PATH)
         sysctl -n hw.ncpu
     ;;
 

cdist/conf/explorer/disks

@@ -30,9 +30,8 @@ case $uname_s in
         sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+'
     ;;
     NetBSD)
-        PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin"
+        PATH=$(getconf PATH)
-        sysctl -n hw.disknames \
+        sysctl -n hw.disknames | awk -v RS=' ' '/^[lsw]d[0-9]+/'
-        | awk 'BEGIN { RS = " " } /^[lsw]d[0-9]+/'
     ;;
     Linux)
         # list of major device numbers toexclude:

cdist/conf/explorer/memory

@@ -30,6 +30,7 @@ case "$os" in
     ;;
 
     *"bsd")
+        PATH=$(getconf PATH)
         echo "$(sysctl -n hw.physmem) / 1048576" | bc
     ;;
 

cdist/conf/type/__directory/explorer/stat

@@ -30,10 +30,10 @@ fallback() {
    gid=$(echo "$ls_line" | awk '{ print $4 }')
 
    owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd)
-   group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
+   group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
-   mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }')
+   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[lst]/)*2^(9+i/3)}printf("%04o",k)}')
 
    printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\n' \
       "$("$__type_explorer/type")" \
@@ -45,56 +45,27 @@ fallback() {
 # nothing to work with, nothing we could do
 [ -e "$destination" ] || exit 0
 
-if ! command -v stat >/dev/null
+command -v stat >/dev/null 2>&1 || {
-then
    fallback
    exit
-fi
+}
 
-case $("$__explorer/os") in
+case $("$__explorer/os")
-   "freebsd"|"netbsd"|"openbsd"|"macosx")
+in
-      stat -f "type: %HT
+   freebsd|netbsd|openbsd|macosx)
+      stat -f 'type: %HT
 owner: %Du %Su
 group: %Dg %Sg
-mode: %Lp %Sp
+mode: %Mp%03Lp %Sp
-" "$destination" | awk '/^type/ { print tolower($0); next } { print }'
+' "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
-    solaris)
-        ls1="$( ls -ld "$destination" )"
-        ls2="$( ls -ldn "$destination" )"
-
-        if [ -f "$__object/parameter/mode" ]
-        then mode_should="$( cat "$__object/parameter/mode" )"
-        fi
-
-        # yes, it is ugly hack, but if you know better way...
-        if [ -z "$( find "$destination" -perm "$mode_should" )" ]
-        then octets=888
-        else octets="$( echo "$mode_should" | sed 's/^0//' )"
-        fi
-
-        case "$( echo "$ls1" | cut -c1-1 )" in
-            -) echo 'type: regular file' ;;
-            d) echo 'type: directory' ;;
-        esac
-
-        echo "owner: $( echo "$ls2" \
-            | awk '{print $3}' ) $( echo "$ls1" \
-                | awk '{print $3}' )"
-
-        echo "group: $( echo "$ls2" \
-            | awk '{print $4}' ) $( echo "$ls1" \
-                | awk '{print $4}' )"
-
-        echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )"
-    ;;
    *)
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
       #       we fall through to the shell fallback.
-       stat -c "type: %F
+      stat -c 'type: %F
 owner: %u %U
 group: %g %G
-mode: %a %A" "$destination" 2>/dev/null || fallback
+mode: %04a %A' "$destination" 2>/dev/null || fallback
-   ;;
+      ;;
 esac

cdist/conf/type/__directory/gencode-remote

@@ -97,9 +97,11 @@ case "$state_should" in
             value_should="$(cat "$__object/parameter/$attribute")"
             value_is="$(get_current_value "$attribute" "$value_should")"
 
-            # change 0xxx format to xxx format => same as stat returns
+            # format mode in four digits => same as stat returns
             if [ "$attribute" = mode ]; then
-                value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')"
+                # Convert to four-digit octal number (printf interprets
+                # strings with leading 0s as octal!)
+                value_should=$(printf '%04o' "0${value_should}")
             fi
 
             if [ "$set_attributes" = 1 ] || [ "$value_should" != "$value_is" ]; then

cdist/conf/type/__download/explorer/remote_cmd

@@ -0,0 +1,19 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cmd="$( cat "$__object/parameter/cmd-get" )"
+
+elif command -v curl > /dev/null
+then
+    cmd="curl -L -o - '%s'"
+
+elif command -v fetch > /dev/null
+then
+    cmd="fetch -o - '%s'"
+
+else
+    cmd="wget -O - '%s'"
+fi
+
+echo "$cmd"

cdist/conf/type/__download/explorer/state

@@ -0,0 +1,72 @@
+#!/bin/sh -e
+
+dst="/$__object_id"
+
+if [ ! -f "$dst" ]
+then
+    echo 'absent'
+    exit 0
+fi
+
+sum_should="$( cat "$__object/parameter/sum" )"
+
+if [ -f "$__object/parameter/cmd-sum" ]
+then
+    # shellcheck disable=SC2059
+    sum_is="$( eval "$( printf \
+        "$( cat "$__object/parameter/cmd-sum" )" \
+        "$dst" )" )"
+else
+    os="$( "$__explorer/os" )"
+
+    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+    then
+        sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
+
+    elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="md5:$( md5 -q "$dst" )"
+            ;;
+            *)
+                sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+
+    elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="sha1:$( sha1 -q "$dst" )"
+            ;;
+            *)
+                sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+
+    elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="sha256:$( sha256 -q "$dst" )"
+            ;;
+            *)
+                sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+    fi
+fi
+
+if [ -z "$sum_is" ]
+then
+    echo 'no checksum from target' >&2
+    exit 1
+fi
+
+if [ "$sum_is" = "$sum_should" ]
+then
+    echo 'present'
+else
+    echo 'mismatch'
+fi

cdist/conf/type/__download/gencode-local

@@ -0,0 +1,58 @@
+#!/bin/sh -e
+
+download="$( cat "$__object/parameter/download" )"
+
+state_is="$( cat "$__object/explorer/state" )"
+
+if [ "$download" != 'local' ] || [ "$state_is" = 'present' ]
+then
+    exit 0
+fi
+
+url="$( cat "$__object/parameter/url" )"
+
+tmp="$( mktemp )"
+
+dst="/$__object_id"
+
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cmd="$( cat "$__object/parameter/cmd-get" )"
+
+elif command -v wget > /dev/null
+then
+    cmd="wget -O - '%s'"
+
+elif command -v curl > /dev/null
+then
+    cmd="curl -L -o - '%s'"
+
+elif command -v fetch > /dev/null
+then
+    cmd="fetch -o - '%s'"
+
+else
+    echo 'no usable locally installed utility for downloading' >&2
+    exit 1
+fi
+
+printf "$cmd > %s\n" \
+    "$url" \
+    "$tmp"
+
+if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$'
+then
+    target_host="[$__target_host]"
+else
+    target_host="$__target_host"
+fi
+
+printf '%s %s %s:%s\n' \
+    "$__remote_copy" \
+    "$tmp" \
+    "$target_host" \
+    "$dst"
+
+echo "rm -f '$tmp'"
+
+echo 'downloaded' > "$__messages_out"

cdist/conf/type/__download/gencode-remote

@@ -0,0 +1,25 @@
+#!/bin/sh -e
+
+download="$( cat "$__object/parameter/download" )"
+
+state_is="$( cat "$__object/explorer/state" )"
+
+if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
+then
+    cmd="$( cat "$__object/explorer/remote_cmd" )"
+
+    url="$( cat "$__object/parameter/url" )"
+
+    dst="/$__object_id"
+
+    printf "$cmd > %s\n" \
+        "$url" \
+        "$dst"
+
+    echo 'downloaded' > "$__messages_out"
+fi
+
+if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ]
+then
+    cat "$__object/parameter/onchange"
+fi

cdist/conf/type/__download/man.rst

@@ -0,0 +1,86 @@
+cdist-type__download(7)
+=======================
+
+NAME
+----
+cdist-type__download - Download a file
+
+
+DESCRIPTION
+-----------
+Destination (``$__object_id``) in target host must be persistent storage
+in order to calculate checksum and decide if file must be (re-)downloaded.
+
+By default type will try to use ``wget``, ``curl`` or ``fetch``.
+If download happens in target (see ``--download``) then type will
+fallback to (and install) ``wget``.
+
+If download happens in local machine, then environment variables like
+``{http,https,ftp}_proxy`` etc can be used on cdist execution
+(``http_proxy=foo cdist config ...``).
+
+
+REQUIRED PARAMETERS
+-------------------
+url
+   File's URL.
+
+sum
+   Checksum of file going to be downloaded.
+   By default output of ``cksum`` without filename is expected.
+   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
+
+onchange
+   Execute this command after download.
+
+
+OPTIONAL PARAMETERS
+-------------------
+download
+   If ``local`` (default), then download file to local storage and copy
+   it to target host. If ``remote``, then download happens in target.
+
+cmd-get
+   Command used for downloading.
+   Command must output to ``stdout``.
+   Parameter will be used for ``printf`` and must include only one
+   format specification ``%s`` which will become URL.
+   For example: ``wget -O - '%s'``.
+
+cmd-sum
+   Command used for checksum calculation.
+   Command output and ``--sum`` parameter must match.
+   Parameter will be used for ``printf`` and must include only one
+   format specification ``%s`` which will become destination.
+   For example: ``md5sum '%s' | awk '{print $1}'``.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __directory /opt/cpma
+
+    require='__directory/opt/cpma' \
+        __download /opt/cpma/cnq3.zip \
+            --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
+            --sum md5:46da3021ca9eace277115ec9106c5b46
+
+    require='__download/opt/cpma/cnq3.zip' \
+        __unpack /opt/cpma/cnq3.zip \
+            --move-existing-destination \
+            --destination /opt/cpma/server
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__download/manifest

@@ -0,0 +1,6 @@
+#!/bin/sh -e
+
+if grep -Eq '^wget' "$__object/explorer/remote_cmd"
+then
+    __package wget
+fi

cdist/conf/type/__download/parameter/default/download

@@ -0,0 +1 @@
+local

cdist/conf/type/__download/parameter/optional

@@ -0,0 +1,4 @@
+cmd-get
+cmd-sum
+download
+onchange

cdist/conf/type/__download/parameter/required

@@ -0,0 +1,2 @@
+url
+sum

cdist/conf/type/__file/explorer/stat

@@ -31,10 +31,10 @@ fallback() {
    gid=$(echo "$ls_line" | awk '{ print $4 }')
 
    owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd)
-   group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
+   group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
 
    mode_text=$(echo "$ls_line" | awk '{ print $1 }')
-   mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }')
+   mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[lst]/)*2^(9+i/3)}printf("%04o",k)}')
 
    size=$(echo "$ls_line" | awk '{ print $5 }')
    links=$(echo "$ls_line" | awk '{ print $2 }')
@@ -53,64 +53,32 @@ fallback() {
 [ -e "$destination" ] || exit 0
 
 
-if ! command -v stat >/dev/null
+command -v stat >/dev/null 2>&1 || {
-then
    fallback
    exit
-fi
+}
 
 
 case $("$__explorer/os")
 in
    freebsd|netbsd|openbsd|macosx)
-      stat -f "type: %HT
+      stat -f 'type: %HT
 owner: %Du %Su
 group: %Dg %Sg
-mode: %Lp %Sp
+mode: %Mp%03Lp %Sp
 size: %Dz
 links: %Dl
-" "$destination" | awk '/^type/ { print tolower($0); next } { print }'
+' "$destination" | awk '/^type/ { print tolower($0); next } { print }'
       ;;
-    solaris)
-        ls1="$( ls -ld "$destination" )"
-        ls2="$( ls -ldn "$destination" )"
-
-        if [ -f "$__object/parameter/mode" ]
-        then mode_should="$( cat "$__object/parameter/mode" )"
-        fi
-
-        # yes, it is ugly hack, but if you know better way...
-        if [ -z "$( find "$destination" -perm "$mode_should" )" ]
-        then octets=888
-        else octets="$( echo "$mode_should" | sed 's/^0//' )"
-        fi
-
-        case "$( echo "$ls1" | cut -c1-1 )" in
-            -) echo 'type: regular file' ;;
-            d) echo 'type: directory' ;;
-        esac
-
-        echo "owner: $( echo "$ls2" \
-            | awk '{print $3}' ) $( echo "$ls1" \
-                | awk '{print $3}' )"
-
-        echo "group: $( echo "$ls2" \
-            | awk '{print $4}' ) $( echo "$ls1" \
-                | awk '{print $4}' )"
-
-        echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )"
-        echo "size: $( echo "$ls1" | awk '{print $5}' )"
-        echo "links: $( echo "$ls1" | awk '{print $2}' )"
-    ;;
    *)
       # NOTE: Do not use --printf here as it is not supported by BusyBox stat.
       # NOTE: BusyBox's stat might not support the "-c" option, in which case
       #       we fall through to the shell fallback.
-      stat -c "type: %F
+      stat -c 'type: %F
 owner: %u %U
 group: %g %G
-mode: %a %A
+mode: %04a %A
 size: %s
-links: %h" "$destination" 2>/dev/null || fallback
+links: %h' "$destination" 2>/dev/null || fallback
-         ;;
+      ;;
 esac

cdist/conf/type/__file_old/gencode-remote

@@ -68,9 +68,11 @@ case "$state_should" in
             if [ -f "$__object/parameter/$attribute" ]; then
                 value_should="$(cat "$__object/parameter/$attribute")"
 
-                # change 0xxx format to xxx format => same as stat returns
+                # format mode in four digits => same as stat returns
                 if [ "$attribute" = mode ]; then
-                    value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')"
+                    # Convert to four-digit octal number (printf interprets
+                    # strings with leading 0s as octal!)
+                    value_should=$(printf '%04o' "0${value_should}")
                 fi
 
                 value_is="$(get_current_value "$attribute" "$value_should")"

cdist/conf/type/__hosts/man.rst

@@ -25,6 +25,10 @@ ip
     state is ``present``, this parameter is mandatory, if state is
     ``absent``, this parameter is silently ignored.
 
+alias
+    An alias for the hostname.
+    This parameter can be specified multiple times (once per alias).
+
 EXAMPLES
 --------
 
@@ -36,6 +40,8 @@ EXAMPLES
     # previously configured via __hosts.
     __hosts happy --state absent
 
+    __hosts srv1.example.com --ip 192.168.0.42 --alias srv1
+
 SEE ALSO
 --------
 
@@ -43,13 +49,14 @@ SEE ALSO
 
 AUTHORS
 -------
-
+| Dmitry Bogatov <KAction@gnu.org>
-Dmitry Bogatov <KAction@gnu.org>
+| Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
 
-Copyright (C) 2015,2016 Dmitry Bogatov. Free use of this software is granted
+Copyright \(C) 2015-2016 Dmitry Bogatov, 2019 Dennis Camera.
-under the terms of the GNU General Public License version 3 or later
+You can redistribute it and/or modify it under the terms of the GNU General
-(GPLv3+).
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.

cdist/conf/type/__hosts/manifest

@@ -1,29 +1,42 @@
 #!/bin/sh -e
-# Copyright (C) 2015  Bogatov Dmitry <KAction@gnu.org>
 #
-# This program is free software: you can redistribute it and/or modify
+# Copyright (C) 2015  Bogatov Dmitry <KAction@gnu.org>
+# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
+# cdist is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-set -ue
+#
 
-hostname="$__object_id"
+set -e
-state="$(cat "$__object/parameter/state")"
-marker="# __hosts/$hostname"
 
-set -- "__hosts/$hostname" --file /etc/hosts --state "$state"
+hostname=$__object_id
+state=$(cat "${__object}/parameter/state")
+marker="# __hosts/${hostname}"
 
-if [ "$state" = absent ] ; then
+if test "${state}" != 'absent'
-	__line "$@" --regex "$marker"
+then
+	ip=$(cat "${__object}/parameter/ip")
+	if test -s "${__object}/parameter/alias"
+	then
+		aliases=$(while read -r a; do printf '\t%s' "$a"; done <"$__object/parameter/alias")
+	fi
+
+	set -- --line "$(printf '%s\t%s%s  %s' \
+		"${ip}" "${hostname}" "${aliases}" "${marker}")"
 else
-	ip="$(cat "$__object/parameter/ip")"
+	set -- --regex "$(echo "${marker}" | sed -e 's/\./\\./')$"
-	__line "$@" --line "$ip $hostname $marker"
 fi
+
+__line "/etc/hosts:${hostname}" --file /etc/hosts --state "${state}" "$@"

cdist/conf/type/__hosts/parameter/optional_multiple

@@ -0,0 +1 @@
+alias

cdist/conf/type/__key_value/explorer/state

@@ -40,7 +40,9 @@ else
 fi
 export key state delimiter value exact_delimiter
 
-awk -f - "$file" <<"AWK_EOF"
+awk_bin=$(PATH=$(getconf PATH 2>/dev/null) && command -v awk || echo awk)
+
+"${awk_bin}" -f - "$file" <<"AWK_EOF"
 BEGIN {
     state=ENVIRON["state"]
     key=ENVIRON["key"]

cdist/conf/type/__key_value/files/remote_script.sh

@@ -24,7 +24,10 @@ if [ -f "$file" ]; then
 else
     touch "$file"
 fi
-awk -f - "$file" >"$tmpfile" <<"AWK_EOF"
+
+awk_bin=$(PATH=$(getconf PATH 2>/dev/null) && command -v awk || echo awk)
+
+"${awk_bin}" -f - "$file" >"$tmpfile" <<"AWK_EOF"
 BEGIN {
     # import variables in a secure way ..
     state=ENVIRON["state"]

cdist/conf/type/__key_value/gencode-remote

@@ -25,7 +25,7 @@ state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
 fire_onchange=''
 
-if [  "$state_is" = "$state_should" ]; then
+if [ "$state_is" = "$state_should" ]; then
     exit 0
 fi
 

cdist/conf/type/__locale_system/manifest

@@ -3,6 +3,7 @@
 # 2012-2016 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2016 Carlos Ortigoza (carlos.ortigoza at ungleich.ch)
 # 2016 Nico Schottelius (nico.schottelius at ungleich.ch)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -23,17 +24,151 @@
 # Configure system-wide locale by modifying i18n file.
 #
 
+key=$__object_id
+onchange_cmd=  # none, by default
+quote_value=false
+
+catval() {
+    # shellcheck disable=SC2059
+    printf "$($quote_value && echo '"%s"' || echo '%s')" "$(cat "$1")"
+}
+
+state_should=$(cat "${__object}/parameter/state")
+
 os=$(cat "$__global/explorer/os")
 
-case "$os" in
+case $os
-    debian|ubuntu)
+in
+    debian)
+        os_version=$(cat "${__global}/explorer/os_version")
+        if expr "${os_version}" '>=' 4 >/dev/null
+        then
+            # Debian 4 (etch) and later
+            locale_conf="/etc/default/locale"
+        else
+            locale_conf="/etc/environment"
+        fi
+        ;;
+    devuan)
         locale_conf="/etc/default/locale"
         ;;
+    ubuntu)
+        os_version=$(cat "${__global}/explorer/os_version")
+        if expr "${os_version}" '>=' 6.10 >/dev/null
+        then
+            # Ubuntu 6.10 (edgy) and later
+            locale_conf="/etc/default/locale"
+        else
+            locale_conf="/etc/environment"
+        fi
+        ;;
     archlinux)
         locale_conf="/etc/locale.conf"
         ;;
-    redhat|centos)
+    centos|redhat|scientific)
-        locale_conf="/etc/sysconfig/i18n"
+        # shellcheck source=/dev/null
+        version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
+        if expr "${version_id}" '>=' 7 >/dev/null
+        then
+            locale_conf="/etc/locale.conf"
+        else
+            locale_conf="/etc/sysconfig/i18n"
+        fi
+        ;;
+    fedora)
+        # shellcheck source=/dev/null
+        version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}")
+        if expr "${version_id}" '>=' 18 >/dev/null
+        then
+            locale_conf="/etc/locale.conf"
+            quote_value=false
+        else
+            locale_conf="/etc/sysconfig/i18n"
+        fi
+        ;;
+    gentoo)
+        case $(cat "${__global}/explorer/init")
+        in
+            (*openrc*)
+                locale_conf="/etc/env.d/02locale"
+                onchange_cmd="env-update --no-ldconfig"
+                quote_value=true
+                ;;
+            (systemd)
+                locale_conf="/etc/locale.conf"
+                ;;
+        esac
+        ;;
+    freebsd|netbsd)
+        # NetBSD doesn't have a separate configuration file to set locales.
+        # In FreeBSD locales could be configured via /etc/login.conf but parsing
+        # that would be annoying, so the shell login file will have to do.
+        # "Non-POSIX" shells like csh will not be updated here.
+
+        locale_conf="/etc/profile"
+        quote_value=true
+        value="$(catval "${__object}/parameter/value"); export ${key}"
+        ;;
+    solaris)
+        locale_conf="/etc/default/init"
+        locale_conf_group="sys"
+
+        if expr "$(cat "${__global}/explorer/os_version")" '>=' 5.11 >/dev/null
+        then
+            # mode on Oracle Solaris 11 is actually 0444,
+            # but the write bit makes sense, IMO
+            locale_conf_mode=0644
+
+            # Oracle Solaris 11.2 and later uses SMF to store environment info.
+            # This is a hack, but I didn't feel like modifying the whole type
+            # just for some Oracle nonsense.
+            # 11.3 apparently added nlsadm(1m), but it is missing from 11.2.
+            # Illumos continues to use /etc/default/init
+            # NOTE: Remember not to use "cool" POSIX features like -q or -e with
+            # Solaris grep.
+            release_regex='Oracle Solaris 11.[2-9][0-9]*'
+            case $state_should
+            in
+                (present)
+                    svccfg_cmd="svccfg -s svc:/system/environment:init setprop environment/${key} = astring: '$(cat "${__object}/parameter/value")'"
+                    ;;
+                (absent)
+                    svccfg_cmd="svccfg -s svc:/system/environment:init delprop environment/${key}"
+                    ;;
+            esac
+            refresh_cmd='svcadm refresh svc:/system/environment'
+            onchange_cmd="grep '${release_regex}' /etc/release >&- || exit 0; ${svccfg_cmd:-:} && ${refresh_cmd}"
+        else
+            locale_conf_mode=0555
+        fi
+        ;;
+    slackware)
+        # NOTE: lang.csh (csh config) is ignored here.
+        locale_conf="/etc/profile.d/lang.sh"
+        locale_conf_mode=0755
+        key="export ${__object_id}"
+        ;;
+    suse)
+        os_version=$(cat "${__global}/explorer/os_version")
+        os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)')
+
+        # https://documentation.suse.com/sles/15-SP2/html/SLES-all/cha-suse.html#sec-suse-l10n
+        if expr "${os_major}" '>=' 15 \& "${os_major}" != 42
+        then
+            # It seems that starting with SuSE 15 the systemd /etc/locale.conf
+            # is the preferred way to set locales, although
+            # /etc/sysconfig/language is still available.
+            # Older documentation doesn't mention /etc/locale.conf, even though
+            # is it created when localectl is used.
+            locale_conf="/etc/locale.conf"
+        else
+            locale_conf="/etc/sysconfig/language"
+            quote_value=true
+            key="RC_${__object_id}"
+        fi
+        ;;
+    voidlinux)
+        locale_conf="/etc/locale.conf"
         ;;
     *)
         echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
@@ -42,14 +177,16 @@ case "$os" in
         ;;
 esac
 
-__file "$locale_conf" \
+__file "${locale_conf}" --state exists \
-       --owner root --group root --mode 644 \
+    --owner "${locale_conf_owner:-0}" \
-       --state exists
+    --group "${locale_conf_group:-0}" \
+    --mode "${locale_conf_mode:-0644}"
 
-require="__file/$locale_conf" \
+require="__file/${locale_conf}" \
-       __key_value "$locale_conf:$__object_id" \
+__key_value "${locale_conf}:${key#export }" \
-       --file "$locale_conf" \
+    --file "${locale_conf}" \
-       --key "$__object_id" \
+    --key "${key}" \
-       --delimiter = \
+    --delimiter '=' --exact_delimiter \
-       --state "$(cat "$__object/parameter/state")" \
+    --state "${state_should}" \
-       --value "$(cat "$__object/parameter/value")"
+    --value "${value:-$(catval "${__object}/parameter/value")}" \
+    --onchange "${onchange_cmd}"

cdist/conf/type/__openldap_server/man.rst

@@ -103,8 +103,8 @@ syncrepl-host
     Set once per host that will replicate the directory.
 
 module
-    LDAP module to load. See `slapd.conf(5)`.
+    LDAP module to load. See `slapd.conf(5)`. Some dependencies might have to
-    Default value is OS-dependent, see manifest.
+    be installed beforehand. Default value is OS-dependent, see manifest.
 
 schema
     Name of LDAP schema to load. Must be the name without extension of a

cdist/conf/type/__openldap_server/manifest

@@ -25,6 +25,7 @@ case "${os}" in
         SLAPD_DATA_DIR="/var/db/openldap-data"
         SLAPD_RUN_DIR="/var/run/openldap"
         SLAPD_MODULE_PATH="/usr/local/libexec/openldap"
+        SLAPD_MODULE_TYPE="la"
         if [ -z "${slapd_modules}" ]; then
             # It looks like ppolicy and syncprov must be compiled
             slapd_modules="back_mdb back_monitor"
@@ -43,13 +44,34 @@ case "${os}" in
         SLAPD_DATA_DIR="/var/lib/ldap"
         SLAPD_RUN_DIR="/var/run/slapd"
         SLAPD_MODULE_PATH="/usr/lib/ldap"
+        SLAPD_MODULE_TYPE="la"
         if [ -z "${slapd_modules}" ]; then
             slapd_modules="back_mdb ppolicy syncprov back_monitor"
         fi
+        CONF_OWNER="openldap"
+        CONF_GROUP="openldap"
         if [ -z "${tls_cipher_suite}" ]; then
             tls_cipher_suite="NORMAL"
         fi
         ;;
+    alpine)
+        PKGS="openldap openldap-clients"
+        ETC="/etc"
+        SLAPD_DIR="/etc/openldap"
+        SLAPD_DATA_DIR="/var/lib/openldap"
+        SLAPD_RUN_DIR="/var/run/openldap"
+        SLAPD_MODULE_PATH="/usr/lib/openldap"
+        SLAPD_MODULE_TYPE="so"
+        if [ -z "${slapd_modules}" ]; then
+            slapd_modules="back_mdb ppolicy syncprov back_monitor"
+            PKGS="$PKGS openldap-back-mdb openldap-back-monitor openldap-overlay-all"
+        fi
+        CONF_OWNER="ldap"
+        CONF_GROUP="$SLAPD_USER"
+        if [ -z "${tls_cipher_suite}" ]; then
+            tls_cipher_suite="DEFAULT"
+        fi
+        ;;
     *)
         echo "Don't know the openldap defaults for: $os" >&2
         exit 1
@@ -156,6 +178,12 @@ case "${os}" in
                --line "SLAPD_SERVICES=\"${slapd_urls}\"" \
                --state present
         ;;
+    alpine)
+        require="__package/${PKG_MAIN}" __line add_slapd_services \
+               --file ${ETC}/conf.d/slapd \
+               --line "command_args=\"-h '${slapd_urls}'\"" \
+               --state present
+        ;;
     *)
         # Nothing to do here, move on.
         ;;
@@ -170,20 +198,22 @@ if [ -z "${_skip_letsencrypt_cert}" ]; then
     fi
 
     # shellcheck disable=SC2086
-    __letsencrypt_cert "${name}" --admin-email "${admin_email}" \
+    __directory ${SLAPD_DIR}/sasl2
-        --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R openldap:openldap ${SLAPD_DIR}/sasl2 && service slapd restart" \
+    require="__directory/${SLAPD_DIR}/sasl2" __letsencrypt_cert "${name}" \
-        --automatic-renewal ${staging}
+        --admin-email "${admin_email}" \
+        --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R ${CONF_OWNER}:${CONF_GROUP} ${SLAPD_DIR}/sasl2 && service slapd restart" \
+        --automatic-renewal "${staging}"
 fi
 
 require="__package/${PKG_MAIN}" __directory ${SLAPD_DIR}/slapd.d --state absent
 
 if [ -z "${_skip_letsencrypt_cert}" ]; then
     require="__package/${PKG_MAIN} __letsencrypt_cert/${name}" \
-           __file ${SLAPD_DIR}/slapd.conf --owner ${CONF_OWNER} --group ${CONF_GROUP} --mode 644 \
+           __file "${SLAPD_DIR}/slapd.conf" --owner "${CONF_OWNER}" --group "${CONF_GROUP}" --mode 644 \
            --source "${ldapconf}"
 else
     require="__package/${PKG_MAIN}" \
-           __file ${SLAPD_DIR}/slapd.conf --owner ${CONF_OWNER} --group ${CONF_GROUP} --mode 644 \
+           __file "${SLAPD_DIR}/slapd.conf" --owner "${CONF_OWNER}" --group "${CONF_GROUP}" --mode 644 \
            --source "${ldapconf}"
 fi
 
@@ -210,7 +240,7 @@ done
 # Add specified modules
 echo "modulepath ${SLAPD_MODULE_PATH}" >> "${ldapconf}"
 for module in ${slapd_modules}; do
-    echo "moduleload ${module}.la" >> "${ldapconf}"
+    echo "moduleload ${module}.${SLAPD_MODULE_TYPE}" >> "${ldapconf}"
 done
 
 # Rest of the config

cdist/conf/type/__package_apt/gencode-remote

@@ -64,7 +64,7 @@ esac
 
 # Hint if we need to avoid questions at some point:
 # DEBIAN_PRIORITY=critical can reduce the number of questions
-aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes --no-install-recommends -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
+aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o APT::Install-Recommends=0 -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\""
 
 if [ "$state_is" = "$state_should" ]; then
     if [ -z "$version" ] || [ "$version" = "$version_is" ]; then

cdist/conf/type/__package_opkg/explorer/pkg_status

@@ -1,7 +1,8 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2011 Nico Schottelius (nico-cdist at schottelius.org)
 # 2012 Giel van Schijndel (giel plus cdist at mortis dot eu)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,21 +20,78 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Retrieve the status of a package - parsed opkg output
+# Retrieve the status of a package - parses opkg output
 #
 
-if [ -f "$__object/parameter/name" ]; then
+readonly __type_path=${__object%%${__object_id}*}
-   name="$(cat "$__object/parameter/name")"
+test -d "${__type_path}" || { echo 'Cannot determine __type_path' >&2; exit 1; }
+readonly LOCKFILE="${__type_path:?}/.cdist_opkg.lock"
+
+if command -v flock >/dev/null 2>&1
+then
+	# use flock (if available) on FD 9
+	_lock() {
+		exec 9<>"${LOCKFILE:?}"
+		flock -x 9
+		echo $$>&9
+	}
+	_unlock() {
+		:>"${LOCKFILE:?}"
+		flock -u 9
+		exec 9<&-
+	}
 else
-   name="$__object_id"
+	# fallback to mkdir if flock is missing
+	_lock() {
+		until mkdir "${LOCKFILE:?}.dir" 2>/dev/null
+		do
+			while test -d "${LOCKFILE}.dir"
+			do
+				# DEBUG:
+				# printf 'Locked by PID: %u\n' "$(cat "${LOCKFILE}.dir/pid")"
+				sleep 1
+			done
+		done
+		echo $$ >"${LOCKFILE:?}.dir/pid"
+	}
+	_unlock() {
+		test -d "${LOCKFILE}.dir" || return 0
+		if test -s "${LOCKFILE}.dir/pid"
+		then
+			test "$(cat "${LOCKFILE}.dir/pid")" = $$ || return 1
+			rm "${LOCKFILE:?}.dir/pid"
+		fi
+		rmdir "${LOCKFILE:?}.dir"
+	}
 fi
 
-# Except dpkg failing, if package is not known / installed
+
-if opkg status "$name" 2>/dev/null | grep -q "^Status: install user installed$"; then
+if test -f "${__object}/parameter/name"
-   echo "present"
+then
-   exit 0
+	pkg_name=$(cat "${__object}/parameter/name")
-elif [ "$(opkg info "$name" 2> /dev/null | wc -l)" -eq 0 ]; then
+else
-   echo "absent notpresent"
+	pkg_name=$__object_id
-   exit 0
+fi
+
+
+# NOTE: We need to lock parallel execution of type explorers and code-remote
+# because opkg will try to acquire the OPKG lock (usually /var/lock/opkg.lock)
+# using lockf(2) for every operation.
+# It will not wait for the lock but terminate with an error.
+# This leads to incorrect 'absent notpresent' statuses when parallel execution
+# is enabled.
+trap _unlock EXIT
+_lock
+
+
+# Except opkg failing, if package is not known / installed
+if opkg status "${pkg_name}" 2>/dev/null \
+	| grep -q -e '^Status: [^ ][^ ]* [^ ][^ ]* installed$'
+then
+	echo 'present'
+elif opkg info "${pkg_name}" 2>/dev/null | grep -q .
+then
+	echo 'absent notpresent'
+else
+	echo 'absent'
 fi
-echo "absent"

cdist/conf/type/__package_opkg/gencode-remote

@@ -2,6 +2,7 @@
 #
 # 2011,2013 Nico Schottelius (nico-cdist at schottelius.org)
 # 2012 Giel van Schijndel (giel plus cdist at mortis dot eu)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,41 +20,50 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Manage packages on OpenWRT and co.
+# Manage packages on OpenWrt, optware, and co.
 #
 
-if [ -f "$__object/parameter/name" ]; then
+if test -f "${__object}/parameter/name"
-   name="$(cat "$__object/parameter/name")"
+then
+	name=$(cat "${__object}/parameter/name")
 else
-   name="$__object_id"
+	name=$__object_id
 fi
 
-state_should="$(cat "$__object/parameter/state")"
+state_should=$(cat "${__object}/parameter/state")
+state_is=$(cat "${__object}/explorer/pkg_status")
 
-state_is="$(cat "$__object/explorer/pkg_status")"
+case $state_is
-case "$state_is" in
+in
-    absent*)
+	(absent*)
-       present="$(echo "$state_is" | cut -d ' ' -f 2)"
+		presence=$(echo "${state_is}" | cut -d ' ' -f 2)
-       state_is="absent"
+		state_is='absent'
-    ;;
+		;;
 esac
 
-[ "$state_is" = "$state_should" ] && exit 0
+if test "${state_is}" = "${state_should}"
+then
+	exit 0
+fi
 
-case "$state_should" in
+
-    present)
+case $state_should
-        if [ "$present" = "notpresent" ]; then
+in
-            echo "opkg --verbosity=0 update"
+	(present)
-        fi
+		if test "${presence}" = 'notpresent'
-        echo "opkg --verbosity=0 install '$name'"
+		then
-        echo "installed" >> "$__messages_out"
+			echo 'opkg --verbosity=0 update'
-    ;;
+		fi
-    absent)
+
-        echo "opkg --verbosity=0 remove '$name'"
+		printf "opkg --verbosity=0 install '%s'\n" "${name}"
-        echo "removed" >> "$__messages_out"
+		echo 'installed' >>"${__messages_out}"
-    ;;
+		;;
-    *)
+	(absent)
-        echo "Unknown state: ${state_should}" >&2
+		printf "opkg --verbosity=0 remove '%s'" "${name}"
-        exit 1
+		echo 'removed' >>"${__messages_out}"
-    ;;
+		;;
+	(*)
+		printf 'Unknown state: %s\n' "${state_should}" >&2
+		exit 1
+		;;
 esac

cdist/conf/type/__pf_apply/deprecated

@@ -1 +0,0 @@
-Consider moving to __pf_apply_anchor. Get in touch if you need __pf_apply.

cdist/conf/type/__pf_apply/explorer/rcvar

@@ -1,36 +0,0 @@
-#!/bin/sh
-#
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Get the location of the pf ruleset on the target host.
-#
-
-# Debug
-#exec >&2
-#set -x
-
-# Check /etc/rc.conf for pf's configuration file name. Default to /etc/pf.conf
-
-RC="/etc/rc.conf"
-PFCONF="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')"
-echo "${PFCONF:-"/etc/pf.conf"}"
-
-# Debug
-#set +x
-

cdist/conf/type/__pf_apply/gencode-remote

@@ -1,51 +0,0 @@
-#!/bin/sh -e
-#
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Apply pf(4) ruleset on *BSD
-#
-
-# Debug
-#exec >&2
-#set -x
-
-rcvar=$(cat "$__object/explorer/rcvar")
-
-cat <<EOF
-if [ -f "${rcvar}.old" ]; then # rcvar.old exists, we must need to disable pf
-   # Disable pf
-   # If it already is disabled, pfctl -d returns 1, go on with life
-   pfctl -d || true
-   # Cleanup
-   rm -f "${rcvar}.old"
-elif [ -f "${rcvar}.new" ]; then # rcvar.new exists, we must need to apply it
-   # Ensure that pf is enabled in the first place
-   # If it already is enabled, pfctl -e returns 1, go on with life
-   mv "${rcvar}.new" "${rcvar}"
-   pfctl -e || true
-   pfctl -f "${rcvar}"
-   if [ "\$?" -ne "0" ]; then # failed to configure new ruleset
-      echo "Failed to configure the new ruleset on ${__target_host}!" >&2
-   fi
-fi
-EOF
-
-# Debug
-#set +x
-

cdist/conf/type/__pf_apply/man.rst

@@ -1,55 +0,0 @@
-cdist-type__pf_apply(7)
-=======================
-
-NAME
-----
-cdist-type__pf_apply - Apply pf(4) ruleset on \*BSD
-
-
-DESCRIPTION
------------
-This type is used on \*BSD systems to manage the pf firewall's active ruleset.
-
-
-REQUIRED PARAMETERS
--------------------
-NONE
-
-
-OPTIONAL PARAMETERS
--------------------
-NONE
-
-
-EXAMPLES
---------
-
-.. code-block:: sh
-
-    # Modify the ruleset on $__target_host:
-    __pf_ruleset --state present --source /my/pf/ruleset.conf
-    require="__pf_ruleset" \
-       __pf_apply
-
-    # Remove the ruleset on $__target_host (implies disabling pf(4):
-    __pf_ruleset --state absent
-    require="__pf_ruleset" \
-       __pf_apply
-
-
-SEE ALSO
---------
-:strong:`pf`\ (4), :strong:`cdist-type__pf_ruleset`\ (7)
-
-
-AUTHORS
--------
-Jake Guffey <jake.guffey--@--eprotex.com>
-
-
-COPYING
--------
-Copyright \(C) 2012 Jake Guffey. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.

cdist/conf/type/__postfix_master/gencode-remote

@@ -67,7 +67,7 @@ case "$state_should" in
          remove_entry
       fi
       cat << DONE
-cat >> "$config" << ${__type##*/}_DONE
+cat >> "$config" << "${__type##*/}_DONE"
 $(cat "$entry")
 ${__type##*/}_DONE
 DONE

cdist/conf/type/__postfix_master/parameter/optional

@@ -4,6 +4,5 @@ unpriv
 chroot
 wakeup
 maxproc
-option
 comment
 state

cdist/conf/type/__postfix_master/parameter/optional_multiple

@@ -0,0 +1 @@
+option

cdist/conf/type/__sysctl/explorer/value

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
 #
@@ -18,5 +18,10 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+if test "$(uname -s)" = NetBSD
+then
+	PATH=$(getconf PATH)
+fi
+
 # get the current runtime value
-sysctl -n "$__object_id" || true
+sysctl -n "${__object_id}" || true

cdist/conf/type/__sysctl/gencode-remote

@@ -44,6 +44,8 @@ case "$os" in
       flag='-w'
       ;;
    netbsd)
+      # shellcheck disable=SC2016
+      echo 'PATH=$(getconf PATH)'
       flag='-w'
       ;;
    freebsd|openbsd)

cdist/conf/type/__sysctl/man.rst

@@ -26,6 +26,13 @@ EXAMPLES
 
     __sysctl net.ipv4.ip_forward --value 1
 
+    # On some operating systems, e.g. NetBSD, to prevent an error if the
+    # MIB style name does not exist (e.g. optional kernel components),
+    # name and value can be separated by `?=`. The same effect can be achieved
+    # in cdist by appending a `?` to the key:
+
+    __sysctl ddb.onpanic? --value -1
+
 
 AUTHORS
 -------

cdist/conf/type/__unpack/explorer/state

@@ -0,0 +1,37 @@
+#!/bin/sh -e
+
+src="/$__object_id"
+
+if [ -f "$__object/parameter/sum-file" ]
+then
+    src_sum_was_file="$( cat "$__object/parameter/sum-file" )"
+else
+    src_sum_was_file="$src.cdist__unpack_sum"
+fi
+
+if [ ! -f "$src" ]
+then
+    if [ -n "$__cdist_dry_run" ]
+    then
+        echo 'mismatch'
+    else
+        echo 'missing'
+    fi
+else
+    if [ ! -f "$src_sum_was_file" ]
+    then
+        echo 'mismatch'
+        exit 0
+    fi
+
+    src_sum_was="$( cat "$src_sum_was_file" )"
+
+    src_sum_is="$( cksum "$src" | awk '{ print $1$2 }' )"
+
+    if [ "$src_sum_was" = "$src_sum_is" ]
+    then
+        echo 'match'
+    else
+        echo 'mismatch'
+    fi
+fi

cdist/conf/type/__unpack/gencode-remote

@@ -0,0 +1,75 @@
+#!/bin/sh -e
+
+if grep -Eq '^(missing|match)$' "$__object/explorer/state"
+then
+    exit 0
+fi
+
+os="$( cat "$__global/explorer/os" )"
+
+src="/$__object_id"
+
+dst="$( sed 's/\/$//' "$__object/parameter/destination" )"
+
+cmd=''
+
+case "$src" in
+    *.tar|*.tgz|*.tar.*)
+        cmd="mkdir -p '$dst' && tar --directory='$dst' --extract --file='$src'"
+
+        if [ -f "$__object/parameter/tar-strip" ]
+        then
+            tar_strip="$( cat "$__object/parameter/tar-strip" )"
+
+            cmd="$cmd --strip-components=$tar_strip"
+        fi
+    ;;
+    *.7z)
+        case "$os" in
+            centos|fedora|redhat)
+                cmd='7za'
+            ;;
+            *)
+                cmd='7zr'
+            ;;
+        esac
+
+        cmd="$cmd e -aoa -o'$dst' '$src'"
+    ;;
+    *.bz2)
+        cmd="bunzip2 --stdout '$src' > '$dst'"
+    ;;
+    *.gz)
+        cmd="gunzip --stdout '$src' > '$dst'"
+    ;;
+    *.lzma|*.xz)
+        cmd="xz --uncompress --stdout '$src' > '$dst'"
+    ;;
+    *.rar)
+        cmd="unrar x -o+ '$src' '$dst/'"
+    ;;
+    *.zip)
+        cmd="unzip -o '$src' -d '$dst'"
+    ;;
+esac
+
+if [ -f "$__object/parameter/backup-destination" ]
+then
+    echo "if [ -e '$dst' ]; then mv '$dst' '$dst.cdist__unpack_backup_$( date +%s )'; fi"
+fi
+
+echo "$cmd"
+
+if [ -f "$__object/parameter/sum-file" ]
+then
+    sum_file="$( cat "$__object/parameter/sum-file" )"
+else
+    sum_file="$src.cdist__unpack_sum"
+fi
+
+echo "cksum '$src' | awk '{ print \$1\$2 }' > '$sum_file'"
+
+if [ ! -f "$__object/parameter/preserve-archive" ]
+then
+    echo "rm -f '$src'"
+fi

cdist/conf/type/__unpack/man.rst

@@ -0,0 +1,79 @@
+cdist-type__unpack(7)
+=====================
+
+NAME
+----
+cdist-type__unpack - Unpack archives
+
+
+DESCRIPTION
+-----------
+Unpack ``.tar``, ``.tgz``, ``.tar.*``, ``.7z``, ``.bz2``, ``.gz``,
+``.lzma``, ``.xz``, ``.rar`` and ``.zip`` archives. Archive type is
+detected by extension.
+
+To achieve idempotency, checksum file will be created in target. See
+``--sum-file`` parameter for details.
+
+
+REQUIRED PARAMETERS
+-------------------
+destination
+   Depending on archive format file or directory to where archive
+   contents will be written.
+
+
+OPTIONAL PARAMETERS
+-------------------
+sum-file
+    Override archive's checksum file in target. By default
+    ``XXX.cdist__unpack_sum`` will be used, where ``XXX`` is source
+    archive path. This file must be kept in target's persistent storage.
+
+tar-strip
+    Tarball specific. See ``man tar`` for ``--strip-components``.
+
+
+OPTIONAL BOOLEAN PARAMETERS
+---------------------------
+backup-destination
+    By default destination file will be overwritten. In case destination
+    is directory, files from archive will be added to or overwritten in
+    directory. This parameter moves existing destination to
+    ``XXX.cdist__unpack_backup_YYY``, where ``XXX`` is destination and
+    ``YYY`` current UNIX timestamp.
+
+preserve-archive
+    Don't delete archive after unpacking.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __directory /opt/cpma
+
+    require='__directory/opt/cpma' \
+        __download /opt/cpma/cnq3.zip \
+            --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
+            --sum md5:46da3021ca9eace277115ec9106c5b46
+
+    require='__download/opt/cpma/cnq3.zip' \
+        __unpack /opt/cpma/cnq3.zip \
+            --backup-destination \
+            --preserve-archive \
+            --destination /opt/cpma/server
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__unpack/manifest

@@ -0,0 +1,41 @@
+#!/bin/sh -e
+
+os="$( cat "$__global/explorer/os" )"
+
+src="/$__object_id"
+
+case "$src" in
+    *.7z)
+        __package p7zip
+    ;;
+    *.bz2)
+        case "$os" in
+            freebsd)
+                # bzip2 is part of freebsd base system
+            ;;
+            *)
+                __package bzip2
+            ;;
+        esac
+    ;;
+    *.lzma|*.xz|*.txz)
+        case "$os" in
+            debian|ubuntu|devuan)
+                __package xz-utils
+            ;;
+            alpine|centos)
+                __package xz
+            ;;
+        esac
+    ;;
+    *.rar)
+        case "$os" in
+            debian|ubuntu|devuan|alpine|freebsd)
+                __package unrar
+            ;;
+        esac
+    ;;
+    *.zip)
+        __package unzip
+    ;;
+esac

cdist/conf/type/__unpack/parameter/boolean

@@ -0,0 +1,2 @@
+backup-destination
+preserve-archive

cdist/conf/type/__unpack/parameter/optional

@@ -0,0 +1,2 @@
+sum-file
+tar-strip

cdist/conf/type/__unpack/parameter/required

@@ -0,0 +1 @@
+destination

cdist/conf/type/__unpack/test/README

@@ -0,0 +1,3 @@
+./make-test-files.sh
+./make-init-manifest.sh | cdist config -i - localhost
+sudo find /tmp/cdist__unpack_test/ -type f -exec cat {} \; | sort

cdist/conf/type/__unpack/test/make-init-manifest.sh

@@ -0,0 +1,22 @@
+#!/bin/sh -e
+
+p="$( pwd )"
+d=/tmp/cdist__unpack_test
+
+echo 'export CDIST_ORDER_DEPENDENCY=1'
+
+echo "__directory $d"
+
+find "$p" -name 'test.*' -and -not -name '*.cdist__unpack_sum' \
+    | sort \
+    | while read -r l
+do
+    n="$( basename "$l" )"
+
+    printf '__unpack %s --destination %s/%s\n' \
+        "$l" \
+        "$d" \
+        "$n"
+done
+
+echo "__clean_path $p --pattern '.+/test\..+'"

cdist/conf/type/__unpack/test/make-test-files.sh

@@ -0,0 +1,44 @@
+#!/bin/sh -ex
+
+echo test.7z > test
+7z a test.7z test > /dev/null
+
+echo test.bz2 > test
+bzip2 test
+
+echo test.gz > test
+gzip test
+
+echo test.lzma > test
+lzma test
+
+echo test.rar > test
+rar a test.rar test > /dev/null
+
+echo test.tar.bz2 > test
+tar cf test.tar test
+bzip2 test.tar
+
+echo test.tar.xz > test
+tar cf test.tar test
+xz test.tar
+
+echo test.tgz > test
+tar cf test.tar test
+gzip test.tar
+mv test.tar.gz test.tgz
+
+echo test.tar.gz > test
+tar cf test.tar test
+gzip test.tar
+
+echo test.tar > test
+tar cf test.tar test
+
+echo test.xz > test
+xz test
+
+echo test.zip > test
+zip test.zip test > /dev/null
+
+rm test

cdist/conf/type/__user/explorer/shadow

@@ -23,18 +23,25 @@
 
 name=$__object_id
 
-case $("$__explorer/os") in
+case $("${__explorer}/os") in
-  'freebsd'|'netbsd'|'openbsd'|'alpine')
+  freebsd|netbsd)
     database='passwd'
     ;;
-  # Default to using shadow passwords
+  openbsd)
+    database='master.passwd'
+    ;;
   *)
+    # Default to using shadow passwords
     database='shadow'
     ;;
 esac
 
-if command -v getent >/dev/null; then
+if command -v getent >/dev/null 2>&1
-  getent "$database" "$name" || true
+then
-elif [ -f /etc/shadow ]; then
+  # shellcheck disable=SC2015
-  grep "^${name}:" /etc/shadow || true
+  getent "${database}" "${name}" 2>/dev/null && exit || true  # fallback to file
+fi
+if test -n "${database}" -a -f "/etc/${database}"
+then
+  grep -e "^${name}:" "/etc/${database}" || true  # ignore failure
 fi

cdist/conf/type/__user/manifest

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -17,16 +18,37 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
 # Manage users.
+#
 
-os=$(cat "$__global/explorer/os")
+case $(cat "${__global}/explorer/os")
-
+in
-case "$os" in
+	(alpine)
-    alpine)
+		__package shadow
-        __package shadow
+		;;
-        ;;
+	(openwrt)
-    *)
+		case $(cat "${__object}/parameter/state")
-        :
+		in
-        ;;
+			(present)
+				if test -s "${__object}/explorer/passwd"
+				then
+					# NOTE: The package might not be required if no changes
+					# are required, but determining if changes are required is
+					# out of scope here, and 40k should be okay, I hope.
+					__package shadow-usermod
+				else
+					__package shadow-useradd
+				fi
+				;;
+			(absent)
+				if test -s "${__object}/explorer/passwd"
+				then
+					__package shadow-userdel
+				fi
+				;;
+		esac
+		;;
+	(*)
+		:
+		;;
 esac

cdist/config.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
@@ -73,7 +72,7 @@ def _graph_dfs_cycle(graph, node, path):
     return False
 
 
-class Config(object):
+class Config:
     """Cdist main class to hold arbitrary data"""
 
     # list of paths (files and/or directories) that will be removed on finish

cdist/core/cdist_object.py

@@ -47,7 +47,7 @@ class MissingObjectIdError(cdist.Error):
         return '%s' % (self.message)
 
 
-class CdistObject(object):
+class CdistObject:
     """Represents a cdist object.
 
     All interaction with objects in cdist should be done through this class.

cdist/core/cdist_type.py

@@ -38,7 +38,7 @@ class InvalidTypeError(cdist.Error):
                 self.type_path, self.type_absolute_path, self.source_path)
 
 
-class CdistType(object):
+class CdistType:
     """Represents a cdist type.
 
     All interaction with types in cdist should be done through this class.

cdist/core/code.py

@@ -94,7 +94,7 @@ code-remote
 '''
 
 
-class Code(object):
+class Code:
     """Generates and executes cdist code scripts.
 
     """

cdist/core/explorer.py

@@ -63,7 +63,7 @@ type explorer is:
 '''
 
 
-class Explorer(object):
+class Explorer:
     """Executes cdist explorers.
 
     """

cdist/core/manifest.py

@@ -95,7 +95,7 @@ class NoInitialManifestError(cdist.Error):
         return repr(self.message)
 
 
-class Manifest(object):
+class Manifest:
     """Executes cdist manifests.
 
     """

cdist/emulator.py

@@ -58,7 +58,7 @@ class DefaultList(list):
             return cls(initial.split('\n'))
 
 
-class Emulator(object):
+class Emulator:
     def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ):
         self.argv = argv
         self.stdin = stdin
@@ -93,10 +93,6 @@ class Emulator(object):
         self.type_name = os.path.basename(argv[0])
         self.cdist_type = core.CdistType(self.type_base_path, self.type_name)
 
-        # If set then object alreay exists and this var holds existing
-        # requirements.
-        self._existing_reqs = None
-
         self.__init_log()
 
     def run(self):
@@ -253,9 +249,6 @@ class Emulator(object):
                 self.parameters[key] = value
 
         if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env:
-            # Make existing requirements a set so that we can compare it
-            # later with new requirements.
-            self._existing_reqs = set(self.cdist_object.requirements)
             obj_params = self._object_params_in_context()
             if obj_params != self.parameters:
                 errmsg = ("Object %s already exists with conflicting "
@@ -274,23 +267,26 @@ class Emulator(object):
             else:
                 self.cdist_object.create()
             self.cdist_object.parameters = self.parameters
-            # record the created object in typeorder file
+        # Do the following recording even if object exists, but with
-            with open(self.typeorder_path, 'a') as typeorderfile:
+        # different requirements.
-                print(self.cdist_object.name, file=typeorderfile)
+
-            # record the created object in parent object typeorder file
+        # record the created object in typeorder file
-            __object_name = self.env.get('__object_name', None)
+        with open(self.typeorder_path, 'a') as typeorderfile:
-            depname = self.cdist_object.name
+            print(self.cdist_object.name, file=typeorderfile)
-            if __object_name:
+        # record the created object in parent object typeorder file
-                parent = self.cdist_object.object_from_name(__object_name)
+        __object_name = self.env.get('__object_name', None)
-                parent.typeorder.append(self.cdist_object.name)
+        depname = self.cdist_object.name
-                if self._order_dep_on():
+        if __object_name:
-                    self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
+            parent = self.cdist_object.object_from_name(__object_name)
-                                    ' for %s'), depname, parent.name)
+            parent.typeorder.append(self.cdist_object.name)
-                    parent.typeorder_dep.append(depname)
+            if self._order_dep_on():
-            elif self._order_dep_on():
+                self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
-                self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
+                                ' for %s'), depname, parent.name)
-                               depname)
+                parent.typeorder_dep.append(depname)
-                self._add_typeorder_dep(depname)
+        elif self._order_dep_on():
+            self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
+                           depname)
+            self._add_typeorder_dep(depname)
 
         # Record / Append source
         self.cdist_object.source.append(self.object_source)
@@ -345,8 +341,6 @@ class Emulator(object):
         # This ensures pattern matching is done against sanitised list
         self.cdist_object.requirements.append(cdist_object.name)
 
-        return cdist_object.name
-
     def _order_dep_on(self):
         return os.path.exists(self.order_dep_state_path)
 
@@ -415,7 +409,6 @@ class Emulator(object):
                 # so do not set a requirement
                 pass
 
-        reqs = set()
         if "require" in self.env:
             requirements = self.env['require']
             self.log.debug("reqs = " + requirements)
@@ -423,23 +416,7 @@ class Emulator(object):
                 # Ignore empty fields - probably the only field anyway
                 if len(requirement) == 0:
                     continue
-                object_name = self.record_requirement(requirement)
+                self.record_requirement(requirement)
-                reqs.add(object_name)
-        if self._existing_reqs is not None:
-            # If object exists then compare existing and new requirements.
-            if self._existing_reqs != reqs:
-                warnmsg = ("Object {} already exists with requirements:\n"
-                           "{}: {}\n"
-                           "{}: {}\n"
-                           "Dependency resolver could not handle dependencies "
-                           "as expected.".format(
-                               self.cdist_object.name,
-                               " ".join(self.cdist_object.source),
-                               self._existing_reqs,
-                               self.object_source,
-                               reqs
-                           ))
-                self.log.warning(warnmsg)
 
     def record_auto_requirements(self):
         """An object shall automatically depend on all objects that it

cdist/exec/local.py

@@ -42,7 +42,7 @@ import functools
 CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ]
 
 
-class Local(object):
+class Local:
     """Execute commands locally.
 
     All interaction with the local side should be done through this class.

cdist/exec/remote.py

@@ -49,7 +49,7 @@ class DecodeError(cdist.Error):
         return "Cannot decode output of " + " ".join(self.command)
 
 
-class Remote(object):
+class Remote:
     """Execute commands remotely.
 
     All interaction with the remote side should be done through this class.

cdist/hostsource.py

@@ -41,7 +41,7 @@ def hostfile_process_line(line, strip_func=str.strip):
         return None
 
 
-class HostSource(object):
+class HostSource:
     """
     Host source object.
     Source can be a sequence or filename (stdin if \'-\').

cdist/info.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2019-2020 Darko Poljak (darko.poljak at gmail.com)
@@ -29,8 +28,7 @@ import glob
 import fnmatch
 
 
-class Info(object):
+class Info:
-
     def __init__(self, conf_dirs, args):
         self.conf_dirs = conf_dirs
         self.all = args.all

cdist/install.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2013-2019 Steven Armstrong (steven-cdist at armstrong.cc)

cdist/integration.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2017 Darko Poljak (darko.poljak at gmail.com)

cdist/inventory.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2016 Darko Poljak (darko.poljak at gmail.com)
@@ -80,7 +79,7 @@ def rstrip_nl(s):
     return str.rstrip(s, "\n")
 
 
-class Inventory(object):
+class Inventory:
     """Inventory main class"""
 
     def __init__(self, db_basedir=dist_inventory_db, configuration=None):

cdist/log.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)

cdist/message.py

@@ -27,7 +27,7 @@ import tempfile
 log = logging.getLogger(__name__)
 
 
-class Message(object):
+class Message:
     """Support messaging between types
 
     """

cdist/preos.py

@@ -78,7 +78,7 @@ def get_available_preoses_string(cls):
     return "Available PreOS-es:\n{}".format("\n".join(preoses))
 
 
-class PreOS(object):
+class PreOS:
     preoses = None
 
     @classmethod

cdist/preos/debootstrap/debootstrap.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # 2016 Darko Poljak (darko.poljak at ungleich.ch)
@@ -31,7 +30,7 @@ import os
 import subprocess
 
 
-class Debian(object):
+class Debian:
     _preos_name = 'debian'
     _cdist_preos = True
 

cdist/shell.py

@@ -32,10 +32,8 @@ import cdist.config
 log = logging.getLogger(__name__)
 
 
-class Shell(object):
+class Shell:
-
     def __init__(self, shell=None):
-
         self.shell = shell
 
         self.target_host = (

cdist/test/config/__init__.py

@@ -44,7 +44,7 @@ expected_object_names = sorted([
     '__third/moon'])
 
 
-class CdistObjectErrorContext(object):
+class CdistObjectErrorContext:
     def __init__(self, original_error):
         self.original_error = original_error
 

cdist/util/fsproperty.py

@@ -177,7 +177,7 @@ class DirectoryDict(collections.MutableMapping):
             raise cdist.Error(str(e))
 
 
-class FileBasedProperty(object):
+class FileBasedProperty:
     attribute_class = None
 
     def __init__(self, path):
@@ -189,7 +189,7 @@ class FileBasedProperty(object):
 
         Usage with a sublcass:
 
-        class Foo(object):
+        class Foo:
             # note that the actual DirectoryDict is stored as __parameters
             # on the instance
             parameters = DirectoryDictProperty(

cdist/util/ipaddr.py

@@ -45,8 +45,6 @@ def resolve_target_host_name(host, family=0):
         log.debug("derived host_name for host \"{}\": {}".format(
             host, host_name))
     except (socket.gaierror, socket.herror) as e:
-        log.warning("Could not derive host_name for {}"
-                    ", $host_name will be empty. Error is: {}".format(host, e))
         # in case of error provide empty value
         host_name = ''
     return host_name
@@ -59,8 +57,6 @@ def resolve_target_fqdn(host):
         log.debug("derived host_fqdn for host \"{}\": {}".format(
             host, host_fqdn))
     except socket.herror as e:
-        log.warning("Could not derive host_fqdn for {}"
-                    ", $host_fqdn will be empty. Error is: {}".format(host, e))
         # in case of error provide empty value
         host_fqdn = ''
     return host_fqdn

docs/changelog

@@ -5,6 +5,25 @@ next:
 	* Core: Add trigger functionality (Nico Schottelius, Darko Poljak)
 	* Core: Implement core support for python types (Darko Poljak)
 
+6.7.0: 2020-07-28
+	* Delete deprecated type: __pf_apply (Darko Poljak)
+	* New type: __download (Ander Punnar)
+	* Type __locale_system: Add devuan support (Dennis Camera)
+	* Type __package_opkg: Add locking (Dennis Camera)
+	* Type __hosts: Add --alias parameter (Dennis Camera)
+	* Type __user: Fix shadow explorer for OpenBSD (Dennis Camera)
+	* Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak)
+	* Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera)
+	* Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali)
+	* Type __user: Install user packages on OpenWrt (Dennis Camera)
+	* Type __openldap_server: Add Alpine support (Timothée Floure)
+	* Type __package_apt: Fix for legacy APT versions that do not support --no-install-recommends (Dennis Camera)
+	* Type __key_value: Get awk from POSIX PATH (Dennis Camera)
+	* New type: __unpack (Ander Punnar)
+	* Type __locale_system: Support more OSes (Dennis Camera)
+	* Explorers cpu_cores, disks, memory: Fix for NetBSD (Dennis Camera)
+	* Type __sysctl: Fix for NetBSD (Dennis Camera)
+
 6.6.0: 2020-06-17
 	* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)
 	* Core: Support colored log output (Evil Ham)