From 3e48ef9e11795bf1c8a59ae202ed28f58cff1753 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 11 Nov 2020 14:09:26 +0100 Subject: [PATCH 01/86] [type/__hostname] Lint - Error if expected environment variables are unset - Always wrap variable expansions in {} --- cdist/conf/type/__hostname/gencode-remote | 82 ++++++------ cdist/conf/type/__hostname/manifest | 147 +++++++++++----------- 2 files changed, 114 insertions(+), 115 deletions(-) diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote index 02afcbfb..9447daa9 100755 --- a/cdist/conf/type/__hostname/gencode-remote +++ b/cdist/conf/type/__hostname/gencode-remote @@ -20,26 +20,27 @@ # along with cdist. If not, see . # -os=$(cat "$__global/explorer/os") -name_running=$(cat "$__global/explorer/hostname") -has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl") +os=$(cat "${__global:?}/explorer/os") +name_running=$(cat "${__global:?}/explorer/hostname") +has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl") -if test -s "$__object/parameter/name" +if test -s "${__object:?}/parameter/name" then - name_should=$(cat "$__object/parameter/name") + name_should=$(cat "${__object:?}/parameter/name") else - case $os + case ${os} in # RedHat-derivatives and BSDs - centos|fedora|redhat|scientific|freebsd|macosx|netbsd|openbsd) + (centos|fedora|redhat|scientific|freebsd|macosx|netbsd|openbsd) # Hostname is FQDN - name_should="${__target_host}" - ;; - *) + name_should=${__target_host:?} + ;; + (*) # Hostname is only first component of FQDN - name_should="${__target_host%%.*}" - ;; + name_should=${__target_host:?} + name_should=${name_should%%.*} + ;; esac fi @@ -47,46 +48,46 @@ fi ################################################################################ # Check if the (running) hostname is already correct # -test "$name_running" != "$name_should" || exit 0 +test "${name_running}" != "${name_should}" || exit 0 ################################################################################ # Setup hostname # -echo 'changed' >>"$__messages_out" +echo 'changed' >>"${__messages_out:?}" # Use the good old way to set the hostname. -case $os +case ${os} in - alpine|debian|devuan|ubuntu) + (alpine|debian|devuan|ubuntu) echo 'hostname -F /etc/hostname' - ;; - archlinux) + ;; + (archlinux) echo 'command -v hostnamectl >/dev/null 2>&1' \ - "&& hostnamectl set-hostname '$name_should'" \ - "|| hostname '$name_should'" - ;; - centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void) - echo "hostname '$name_should'" - ;; - openwrt) - echo "echo '$name_should' >/proc/sys/kernel/hostname" - ;; - macosx) - echo "scutil --set HostName '$name_should'" - ;; - solaris) - echo "uname -S '$name_should'" - ;; - slackware|suse|opensuse-leap) + "&& hostnamectl set-hostname '${name_should}'" \ + "|| hostname '${name_should}'" + ;; + (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void) + echo "hostname '${name_should}'" + ;; + (openwrt) + echo "echo '${name_should}' >/proc/sys/kernel/hostname" + ;; + (macosx) + echo "scutil --set HostName '${name_should}'" + ;; + (solaris) + echo "uname -S '${name_should}'" + ;; + (slackware|suse|opensuse-leap) # We do not read from /etc/HOSTNAME, because the running # hostname is the first component only while the file contains # the FQDN. - echo "hostname '$name_should'" - ;; - *) + echo "hostname '${name_should}'" + ;; + (*) # Fall back to set the hostname using hostnamectl, if available. - if test -n "$has_hostnamectl" + if test -n "${has_hostnamectl}" then # Don't use hostnamectl as the primary means to set the hostname for # systemd systems, because it cannot be trusted to work reliably and @@ -97,7 +98,8 @@ in echo "test \"\$(hostname)\" = \"\$(cat /etc/hostname)\"" \ " || hostname -F /etc/hostname" else - printf "echo 'Unsupported OS: %s' >&2\nexit 1\n" "$os" + printf "echo 'Unsupported OS: %s' >&2\n" "${os}" + printf 'exit 1\n' fi - ;; + ;; esac diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest index bf8a331c..125bb2fb 100755 --- a/cdist/conf/type/__hostname/manifest +++ b/cdist/conf/type/__hostname/manifest @@ -20,69 +20,65 @@ # along with cdist. If not, see . # -not_supported() { - echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 - echo "Please contribute an implementation for it if you can." >&2 - exit 1 -} - set_hostname_systemd() { echo "$1" | __file /etc/hostname --source - } -os=$(cat "$__global/explorer/os") -os_version=$(cat "$__global/explorer/os_version") -os_major=$(echo "$os_version" | grep -o '^[0-9][0-9]*' || true) +os=$(cat "${__global:?}/explorer/os") +os_version=$(cat "${__global:?}/explorer/os_version") +os_major=$(echo "${os_version}" | grep -o '^[0-9][0-9]*' || true) -max_len=$(cat "$__object/explorer/max_len") -has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl") +max_len=$(cat "${__object:?}/explorer/max_len") +has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl") -if test -s "$__object/parameter/name" +if test -s "${__object:?}/parameter/name" then - name_should=$(cat "$__object/parameter/name") + name_should=$(cat "${__object:?}/parameter/name") else - case $os + case ${os} in # RedHat-derivatives and BSDs - centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware) + (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware) # Hostname is FQDN - name_should="${__target_host}" - ;; - suse|opensuse-leap) + name_should=${__target_host:?} + ;; + (suse|opensuse-leap) + name_should=${__target_host:?} + # Classic SuSE stores the FQDN in /etc/HOSTNAME, while # systemd does not. The running hostname is the first # component in both cases. # In versions before 15.x, the FQDN is stored in /etc/hostname. - if test -n "$has_hostnamectl" && test "$os_major" -ge 15 \ - && test "$os_major" -ne 42 + if test -n "${has_hostnamectl}" \ + && test "${os_major}" -ge 15 \ + && test "${os_major}" -ne 42 then - name_should="${__target_host%%.*}" - else - name_should="${__target_host}" + name_should=${name_should%%.*} fi - ;; + ;; *) # Hostname is only first component of FQDN on all other systems. - name_should="${__target_host%%.*}" - ;; + name_should=${__target_host:?} + name_should=${name_should%%.*} + ;; esac fi -if test -n "$max_len" && test "$(printf '%s' "$name_should" | wc -c)" -gt "$max_len" +if test -n "${max_len}" && test "$(printf '%s' "${name_should}" | wc -c)" -gt "${max_len}" then printf "Host name too long. Up to %u characters allowed.\n" "${max_len}" >&2 exit 1 fi -case $os +case ${os} in - alpine|debian|devuan|ubuntu|void) - echo "$name_should" | __file /etc/hostname --source - - ;; - archlinux) - if test -n "$has_hostnamectl" + (alpine|debian|devuan|ubuntu|void) + echo "${name_should}" | __file /etc/hostname --source - + ;; + (archlinux) + if test -n "${has_hostnamectl}" then - set_hostname_systemd "$name_should" + set_hostname_systemd "${name_should}" else echo 'Ancient ArchLinux variants without hostnamectl are not supported.' >&2 exit 1 @@ -97,8 +93,8 @@ in # --value "\"$name_should\"" fi ;; - centos|fedora|redhat|scientific) - if test -z "$has_hostnamectl" + (centos|fedora|redhat|scientific) + if test -z "${has_hostnamectl}" then # Only write to /etc/sysconfig/network on non-systemd versions. # On systemd-based versions this entry is ignored. @@ -106,63 +102,62 @@ in --file /etc/sysconfig/network \ --delimiter '=' --exact_delimiter \ --key HOSTNAME \ - --value "\"$name_should\"" + --value "\"${name_should}\"" else - set_hostname_systemd "$name_should" + set_hostname_systemd "${name_should}" fi - ;; - gentoo) + ;; + (gentoo) # Only write to /etc/conf.d/hostname on OpenRC-based installations. # On systemd use hostnamectl(1) in gencode-remote. - if test -z "$has_hostnamectl" + if test -z "${has_hostnamectl}" then __key_value '/etc/conf.d/hostname:hostname' \ --file /etc/conf.d/hostname \ --delimiter '=' --exact_delimiter \ --key 'hostname' \ - --value "\"$name_should\"" + --value "\"${name_should}\"" else set_hostname_systemd "$name_should" fi - ;; - freebsd) + ;; + (freebsd) __key_value '/etc/rc.conf:hostname' \ --file /etc/rc.conf \ --delimiter '=' --exact_delimiter \ --key 'hostname' \ - --value "\"$name_should\"" - ;; - macosx) + --value "\"${name_should}\"" + ;; + (macosx) # handled in gencode-remote - : - ;; - netbsd) + ;; + (netbsd) __key_value '/etc/rc.conf:hostname' \ --file /etc/rc.conf \ --delimiter '=' --exact_delimiter \ --key 'hostname' \ - --value "\"$name_should\"" + --value "\"${name_should}\"" # To avoid confusion, ensure that the hostname is only stored once. __file /etc/myname --state absent - ;; - openbsd) - echo "$name_should" | __file /etc/myname --source - - ;; - openwrt) - __uci system.@system[0].hostname --value "$name_should" + ;; + (openbsd) + echo "${name_should}" | __file /etc/myname --source - + ;; + (openwrt) + __uci system.@system[0].hostname --value "${name_should}" # --transaction hostname - ;; - slackware) + ;; + (slackware) # We write the FQDN into /etc/HOSTNAME. But /etc/rc.d/rc.M will only # read the first component from this file and set it as the running # hostname on boot. - echo "$name_should" | __file /etc/HOSTNAME --source - - ;; - solaris) - echo "$name_should" | __file /etc/nodename --source - - ;; - suse|opensuse-leap) + echo "${name_should}" | __file /etc/HOSTNAME --source - + ;; + (solaris) + echo "${name_should}" | __file /etc/nodename --source - + ;; + (suse|opensuse-leap) # Modern SuSE provides /etc/HOSTNAME as a symlink for # backwards-compatibility. Unfortunately it cannot be used # here as __file does not follow the symlink. @@ -171,23 +166,25 @@ in # not work correctly on openSUSE 12.x which provides # hostnamectl but not /etc/hostname. - if test -n "$has_hostnamectl" -a "$os_major" -gt 12 + if test -n "${has_hostnamectl}" -a "${os_major}" -gt 12 then - hostname_file='/etc/hostname' + hostname_file=/etc/hostname else - hostname_file='/etc/HOSTNAME' + hostname_file=/etc/HOSTNAME fi - echo "$name_should" | __file "$hostname_file" --source - - ;; - *) + echo "${name_should}" | __file "${hostname_file}" --source - + ;; + (*) # On other operating systems we fall back to systemd's # hostnamectl if available… - if test -n "$has_hostnamectl" + if test -n "${has_hostnamectl}" then - set_hostname_systemd "$name_should" + set_hostname_systemd "${name_should}" else - not_supported + echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 fi - ;; + ;; esac From 702f3eba4f8f88b634a010b377d077d501a6f0e9 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 11 Nov 2020 14:15:02 +0100 Subject: [PATCH 02/86] [type/__hostname] Remove opensuse-leap OS string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit everything should be suse now… --- cdist/conf/type/__hostname/gencode-remote | 2 +- cdist/conf/type/__hostname/manifest | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote index 9447daa9..c1a97ac8 100755 --- a/cdist/conf/type/__hostname/gencode-remote +++ b/cdist/conf/type/__hostname/gencode-remote @@ -79,7 +79,7 @@ in (solaris) echo "uname -S '${name_should}'" ;; - (slackware|suse|opensuse-leap) + (slackware|suse) # We do not read from /etc/HOSTNAME, because the running # hostname is the first component only while the file contains # the FQDN. diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest index 125bb2fb..d044072b 100755 --- a/cdist/conf/type/__hostname/manifest +++ b/cdist/conf/type/__hostname/manifest @@ -42,7 +42,7 @@ else # Hostname is FQDN name_should=${__target_host:?} ;; - (suse|opensuse-leap) + (suse) name_should=${__target_host:?} # Classic SuSE stores the FQDN in /etc/HOSTNAME, while @@ -157,7 +157,7 @@ in (solaris) echo "${name_should}" | __file /etc/nodename --source - ;; - (suse|opensuse-leap) + (suse) # Modern SuSE provides /etc/HOSTNAME as a symlink for # backwards-compatibility. Unfortunately it cannot be used # here as __file does not follow the symlink. From 87a0d91587568594a616c7f3588bc74106312761 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 11 Nov 2020 14:20:18 +0100 Subject: [PATCH 03/86] [type/__hostname] Fix OS version detection for SuSE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit everything should be suse now… --- cdist/conf/type/__hostname/manifest | 39 ++++++++++++++++------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest index d044072b..b80aa2ef 100755 --- a/cdist/conf/type/__hostname/manifest +++ b/cdist/conf/type/__hostname/manifest @@ -25,8 +25,6 @@ set_hostname_systemd() { } os=$(cat "${__global:?}/explorer/os") -os_version=$(cat "${__global:?}/explorer/os_version") -os_major=$(echo "${os_version}" | grep -o '^[0-9][0-9]*' || true) max_len=$(cat "${__object:?}/explorer/max_len") has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl") @@ -38,24 +36,10 @@ else case ${os} in # RedHat-derivatives and BSDs - (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware) + (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware|suse) # Hostname is FQDN name_should=${__target_host:?} ;; - (suse) - name_should=${__target_host:?} - - # Classic SuSE stores the FQDN in /etc/HOSTNAME, while - # systemd does not. The running hostname is the first - # component in both cases. - # In versions before 15.x, the FQDN is stored in /etc/hostname. - if test -n "${has_hostnamectl}" \ - && test "${os_major}" -ge 15 \ - && test "${os_major}" -ne 42 - then - name_should=${name_should%%.*} - fi - ;; *) # Hostname is only first component of FQDN on all other systems. name_should=${__target_host:?} @@ -158,6 +142,27 @@ in echo "${name_should}" | __file /etc/nodename --source - ;; (suse) + if test -s "${__global:?}/explorer/os_release" + then + # shellcheck source=/dev/null + os_version=$(. "${__global:?}/explorer/os_release" && echo "${VERSION}") + else + os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global:?}/explorer/os_version") + fi + os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)') + + # Classic SuSE stores the FQDN in /etc/HOSTNAME, while + # systemd does not. The running hostname is the first + # component in both cases. + # In versions before 15.x, the FQDN is stored in /etc/hostname. + if test -n "${has_hostnamectl}" \ + && test "${os_major}" -ge 15 \ + && test "${os_major}" -ne 42 + then + # strip away everything but the first part from $name_should + name_should=${name_should%%.*} + fi + # Modern SuSE provides /etc/HOSTNAME as a symlink for # backwards-compatibility. Unfortunately it cannot be used # here as __file does not follow the symlink. From f75d4772090d3a9cfd798dd193cd324083a9b3a7 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 8 Nov 2020 13:45:12 +0100 Subject: [PATCH 04/86] Deprecate __locale and replace with __localedef --- cdist/conf/type/__locale/deprecated | 1 + cdist/conf/type/__localedef/gencode-remote | 60 +++++++++++++++++++ cdist/conf/type/__localedef/man.rst | 55 +++++++++++++++++ cdist/conf/type/__localedef/manifest | 41 +++++++++++++ .../type/__localedef/parameter/default/state | 1 + .../conf/type/__localedef/parameter/optional | 1 + 6 files changed, 159 insertions(+) create mode 100644 cdist/conf/type/__locale/deprecated create mode 100755 cdist/conf/type/__localedef/gencode-remote create mode 100644 cdist/conf/type/__localedef/man.rst create mode 100755 cdist/conf/type/__localedef/manifest create mode 100644 cdist/conf/type/__localedef/parameter/default/state create mode 100644 cdist/conf/type/__localedef/parameter/optional diff --git a/cdist/conf/type/__locale/deprecated b/cdist/conf/type/__locale/deprecated new file mode 100644 index 00000000..5a06b28e --- /dev/null +++ b/cdist/conf/type/__locale/deprecated @@ -0,0 +1 @@ +This type is deprecated. Please use __localedef instead. diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote new file mode 100755 index 00000000..1feb9884 --- /dev/null +++ b/cdist/conf/type/__localedef/gencode-remote @@ -0,0 +1,60 @@ +#!/bin/sh -e +# +# 2013-2019 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Let localedef do the magic +# + +locale="$__object_id" + +# Hardcoded, create a pull request with +# branching on $os in case it is at another location +alias=/usr/share/locale/locale.alias + +input=$(echo "$locale" | cut -d . -f 1) +charmap=$(echo "$locale" | cut -d . -f 2) + +# Adding locale? The name is de_CH.UTF-8 +# Removing locale? The name is de_CH.utf8. +# W-T-F! +locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/') + +state=$(cat "$__object/parameter/state") + +os=$(cat "$__global/explorer/os") + +# Nothing to be done on alpine +case "$os" in + alpine) + exit 0 + ;; +esac + +case "$state" in + present) + echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale" + ;; + absent) + echo localedef --delete-from-archive "$locale_remove" + ;; + *) + echo "Unsupported state: $state" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__localedef/man.rst b/cdist/conf/type/__localedef/man.rst new file mode 100644 index 00000000..0ccf484a --- /dev/null +++ b/cdist/conf/type/__localedef/man.rst @@ -0,0 +1,55 @@ +cdist-type__localedef(7) +======================== + +NAME +---- +cdist-type__localedef - Define and remove system locales + + +DESCRIPTION +----------- +This cdist type allows you to define locales on the system using +:strong:`localedef`\ (1) or remove them. +On systems that don't support definition of new locales, the type will raise an +error. + + +OPTIONAL PARAMETERS +------------------- +state + ``present`` or ``absent``. Defaults to ``present``. + + +EXAMPLES +-------- + +.. code-block:: sh + + # Add locale de_CH.UTF-8 + __localedef de_CH.UTF-8 + + # Same as above, but more explicit + __localedef de_CH.UTF-8 --state present + + # Remove colourful British English + __localedef en_GB.UTF-8 --state absent + + +SEE ALSO +-------- +:strong:`locale`\ (1), +:strong:`localedef`\ (1), +:strong:`cdist-type__locale_system`\ (7) + + +AUTHORS +------- +| Dennis Camera +| Nico Schottelius + + +COPYING +------- +Copyright \(C) 2013-2019 Nico Schottelius, 2020 Dennis Camera. Free use of this +software is granted under the terms of the GNU General Public License version 3 +or later (GPLv3+). diff --git a/cdist/conf/type/__localedef/manifest b/cdist/conf/type/__localedef/manifest new file mode 100755 index 00000000..9f1e17ac --- /dev/null +++ b/cdist/conf/type/__localedef/manifest @@ -0,0 +1,41 @@ +#!/bin/sh -e +# +# 2013-2019 Nico Schottelius (nico-cdist at schottelius.org) +# 2015 David Hürlimann (david at ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Install required packages +# + +os=$(cat "$__global/explorer/os") + + +case "$os" in + debian|devuan) + # Debian needs a seperate package + __package locales --state present + ;; + archlinux|suse|ubuntu|scientific|centos|alpine) + : + ;; + *) + echo "Sorry, do not know how to handle os: $os" >&2 + echo "Please edit the type ${__type##*/} to fix this." >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__localedef/parameter/default/state b/cdist/conf/type/__localedef/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__localedef/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__localedef/parameter/optional b/cdist/conf/type/__localedef/parameter/optional new file mode 100644 index 00000000..ff72b5c7 --- /dev/null +++ b/cdist/conf/type/__localedef/parameter/optional @@ -0,0 +1 @@ +state From 54e689f7c2b3bc25aedb2aaf9f5aece289ec5b66 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 8 Nov 2020 13:49:04 +0100 Subject: [PATCH 05/86] [type/__localedef] Add state explorer --- cdist/conf/type/__localedef/explorer/state | 71 ++++++++++++++++++++++ cdist/conf/type/__localedef/gencode-remote | 13 +++- 2 files changed, 82 insertions(+), 2 deletions(-) create mode 100755 cdist/conf/type/__localedef/explorer/state diff --git a/cdist/conf/type/__localedef/explorer/state b/cdist/conf/type/__localedef/explorer/state new file mode 100755 index 00000000..d8db29c5 --- /dev/null +++ b/cdist/conf/type/__localedef/explorer/state @@ -0,0 +1,71 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# This explorer determines if the locale is defined on the target system. +# Will print nothing on error. +# +# Possible output: +# present: +# the main locale (and possibly aliases) is present +# absent: +# neither the main locale nor any aliases are present +# + +command -v locale >/dev/null 2>&1 || exit 0 + +locales=$(locale -a) + +parse_locale() { + # This function will split locales into their parts. Locale strings are + # usually of the form: [language[_territory][.codeset][@modifier]] + # For simplicity, language and territory are not separated by this function. + # Old Linux systems were also using "english" or "german" as locale strings. + # Usage: parse_locale locale_str lang_var codeset_var modifier_var + eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')" + eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')" + eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')" +} + +format_locale() { + # Usage: format_locale language codeset modifier + printf '%s' "$1" + test -z "$2" || printf '.%s' "$2" + test -z "$3" || printf '@%s' "$3" + printf '\n' +} + +gnu_normalize_codeset() { + # reimplementation of glibc/locale/programs/localedef.c normalize_codeset() + echo "$*" | tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]' +} + +locale_available() ( + echo "${locales}" | grep -qxF "$1" || { + # glibc uses "normalized" locale names in archives. + # If a locale is stored in an archive, the normalized name will be + # printed by locale, so that needs to be checked, too. + localename=$( + parse_locale "$1" _lang _codeset _modifier \ + && format_locale "${_lang:?}" "$(gnu_normalize_codeset "${_codeset?}")" \ + "${_modifier?}") + echo "${locales}" | grep -qxF "${localename}" + } +) + +locale_available "${__object_id:?}" && echo present || echo absent diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote index 1feb9884..9ea3f6f1 100755 --- a/cdist/conf/type/__localedef/gencode-remote +++ b/cdist/conf/type/__localedef/gencode-remote @@ -35,7 +35,8 @@ charmap=$(echo "$locale" | cut -d . -f 2) # W-T-F! locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/') -state=$(cat "$__object/parameter/state") +state_is=$(cat "${__object:?}/explorer/state") +state_should=$(cat "${__object:?}/parameter/state") os=$(cat "$__global/explorer/os") @@ -46,7 +47,15 @@ case "$os" in ;; esac -case "$state" in +# NOTE: If state explorer fails (e.g. locale(1) missing), the following check +# will always fail and let definition/removal run. +if test "${state_is}" = "${state_should}" +then + exit 0 +fi + +case ${state_should} +in present) echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale" ;; From cc29e54b851212568c3f6eddf4bf0a64c1eabb0d Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 23 Jul 2020 23:20:39 +0200 Subject: [PATCH 06/86] [type/__localedef] Differentiate between OSes and better handling of normalized locale names --- cdist/conf/type/__localedef/gencode-remote | 117 +++++++++++++++------ 1 file changed, 82 insertions(+), 35 deletions(-) diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote index 9ea3f6f1..b9cc68e8 100755 --- a/cdist/conf/type/__localedef/gencode-remote +++ b/cdist/conf/type/__localedef/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2013-2019 Nico Schottelius (nico-cdist at schottelius.org) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,35 +18,16 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# Manage system locales using localedef(1). # -# Let localedef do the magic -# - -locale="$__object_id" - -# Hardcoded, create a pull request with -# branching on $os in case it is at another location -alias=/usr/share/locale/locale.alias - -input=$(echo "$locale" | cut -d . -f 1) -charmap=$(echo "$locale" | cut -d . -f 2) - -# Adding locale? The name is de_CH.UTF-8 -# Removing locale? The name is de_CH.utf8. -# W-T-F! -locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/') state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") -os=$(cat "$__global/explorer/os") - -# Nothing to be done on alpine -case "$os" in - alpine) - exit 0 - ;; -esac +test "${state_should}" = 'present' -o "${state_should}" = 'absent' || { + printf 'Invalid state: %s\n' "${state_should}" >&2 + exit 1 +} # NOTE: If state explorer fails (e.g. locale(1) missing), the following check # will always fail and let definition/removal run. @@ -54,16 +36,81 @@ then exit 0 fi -case ${state_should} +locale=${__object_id:?} +os=$(cat "${__global:?}/explorer/os") + +if expr "${locale}" : '.*/' >/dev/null +then + printf 'Paths as locales are not supported.\n' >&2 + printf '__object_id is: %s\n' "${locale}" >&2 + exit 1 +fi + +parse_locale() { + # This function will split locales into their parts. Locale strings are + # usually of the form: [language[_territory][.codeset][@modifier]] + # For simplicity, language and territory are not separated by this function. + # Old Linux systems were also using "english" or "german" as locale strings. + # Usage: parse_locale locale_str lang_var codeset_var modifier_var + eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')" + eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')" + eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')" +} + +format_locale() { + # Usage: format_locale language codeset modifier + printf '%s' "$1" + test -z "$2" || printf '.%s' "$2" + test -z "$3" || printf '@%s' "$3" + printf '\n' +} + +gnu_normalize_codeset() { + echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]' +} + + +: "${lang=}" "${codeset=}" "${modifier=}" # declare variables for shellcheck +parse_locale "${locale}" lang codeset modifier + + +case ${os} in - present) - echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale" - ;; - absent) - echo localedef --delete-from-archive "$locale_remove" - ;; - *) - echo "Unsupported state: $state" >&2 - exit 1 - ;; + (alpine|openwrt) + printf '%s does not support locales.\n' "${os}" >&2 + exit 1 + ;; + (archlinux|debian|devuan|ubuntu|suse|centos|fedora|redhat|scientific) + # FIXME: The code below only works for glibc-based installations. + + # NOTE: Hardcoded, create a pull request in case it is at another + # location for some opther distro. + # NOTE: locale.alias can be symlinked (e.g. Debian) + aliasfile='/usr/share/locale/locale.alias' + + case ${state_should} + in + (present) + input=$(format_locale "${lang}" '' "${modifier}") + cat <<-EOF + set -- + if test -e '${aliasfile}' + then + set -- -A '${aliasfile}' + fi + + localedef -i '${input}' -f '${codeset}' "\$@" '${locale}' + EOF + ;; + (absent) + localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}") + printf "localedef --delete-from-archive '%s'\n" "${localename}" + ;; + esac + ;; + (*) + echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 + ;; esac From f44888f192d78d8ce1e3583f70f94b0c2039c1e2 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 8 Nov 2020 14:02:08 +0100 Subject: [PATCH 07/86] [type/__localedef] Only install dependencies in manifest. OS checking moved to gencode-remote --- cdist/conf/type/__localedef/manifest | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/cdist/conf/type/__localedef/manifest b/cdist/conf/type/__localedef/manifest index 9f1e17ac..3ab3ad8c 100755 --- a/cdist/conf/type/__localedef/manifest +++ b/cdist/conf/type/__localedef/manifest @@ -2,6 +2,7 @@ # # 2013-2019 Nico Schottelius (nico-cdist at schottelius.org) # 2015 David Hürlimann (david at ungleich.ch) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,24 +19,12 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# -# Install required packages +# Install required packages. # -os=$(cat "$__global/explorer/os") - - -case "$os" in - debian|devuan) - # Debian needs a seperate package - __package locales --state present - ;; - archlinux|suse|ubuntu|scientific|centos|alpine) - : - ;; - *) - echo "Sorry, do not know how to handle os: $os" >&2 - echo "Please edit the type ${__type##*/} to fix this." >&2 - exit 1 - ;; +case $(cat "${__global:?}/explorer/os") +in + (debian|devuan) + __package_apt locales --state present + ;; esac From dcef2c19f5b87e3a9594f01fefab66cc7488dce7 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 8 Nov 2020 14:07:42 +0100 Subject: [PATCH 08/86] [type/__localedef] Add support for FreeBSD --- cdist/conf/type/__localedef/gencode-remote | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote index b9cc68e8..80e7eb1c 100755 --- a/cdist/conf/type/__localedef/gencode-remote +++ b/cdist/conf/type/__localedef/gencode-remote @@ -108,6 +108,25 @@ in ;; esac ;; + (freebsd) + case ${state_should} + in + (present) + if expr "$(grep -oe '^[0-9]*' "${__global:?}/explorer/os_version")" '>=' 11 >/dev/null + then + # localedef(1) is available with FreeBSD >= 11 + printf "localedef -i '%s' -f '%s' '%s'\n" "${input}" "${codeset}" "${locale}" + else + printf 'localedef(1) was added to FreeBSD starting with version 11.\n' >&2 + printf 'Please upgrade your FreeBSD installation to use %s.\n' "${__type##*/}" >&2 + exit 1 + fi + ;; + (absent) + printf "rm -R '/usr/share/locale/%s'\n" "${locale}" + ;; + esac + ;; (*) echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2 echo "Please contribute an implementation for it if you can." >&2 From c1c60e3374e0cb93ea182c4cbcb813150756c3e4 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 8 Nov 2020 15:24:46 +0100 Subject: [PATCH 09/86] [type/__localedef] Blacklist OpenBSD and NetBSD --- cdist/conf/type/__localedef/gencode-remote | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote index 80e7eb1c..af1a77f7 100755 --- a/cdist/conf/type/__localedef/gencode-remote +++ b/cdist/conf/type/__localedef/gencode-remote @@ -127,6 +127,12 @@ in ;; esac ;; + (netbsd|openbsd) + # NetBSD/OpenBSD are missing localedef(1). + # We also do not delete defined locales because they can't be recreated. + echo "${os} is lacking localedef(1). Locale management unavailable." >&2 + exit 1 + ;; (*) echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2 echo "Please contribute an implementation for it if you can." >&2 From 575bb62dc507fb8c294319f8db20ba45595e6869 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 13 Nov 2020 18:42:04 +0100 Subject: [PATCH 10/86] [type/__localedef] Externalise functions to separate files --- .../conf/type/__localedef/files/lib/glibc.sh | 5 ++++ .../conf/type/__localedef/files/lib/locale.sh | 20 +++++++++++++ cdist/conf/type/__localedef/gencode-remote | 29 ++++--------------- 3 files changed, 30 insertions(+), 24 deletions(-) create mode 100644 cdist/conf/type/__localedef/files/lib/glibc.sh create mode 100644 cdist/conf/type/__localedef/files/lib/locale.sh diff --git a/cdist/conf/type/__localedef/files/lib/glibc.sh b/cdist/conf/type/__localedef/files/lib/glibc.sh new file mode 100644 index 00000000..6ace80d4 --- /dev/null +++ b/cdist/conf/type/__localedef/files/lib/glibc.sh @@ -0,0 +1,5 @@ +# -*- mode: sh; indent-tabs-mode: t -*- + +gnu_normalize_codeset() { + echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]' +} diff --git a/cdist/conf/type/__localedef/files/lib/locale.sh b/cdist/conf/type/__localedef/files/lib/locale.sh new file mode 100644 index 00000000..b5e61374 --- /dev/null +++ b/cdist/conf/type/__localedef/files/lib/locale.sh @@ -0,0 +1,20 @@ +# -*- mode: sh; indent-tabs-mode:t -*- + +parse_locale() { + # This function will split locales into their parts. Locale strings are + # usually of the form: [language[_territory][.codeset][@modifier]] + # For simplicity, language and territory are not separated by this function. + # Old Linux systems were also using "english" or "german" as locale strings. + # Usage: parse_locale locale_str lang_var codeset_var modifier_var + eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')" + eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')" + eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')" +} + +format_locale() { + # Usage: format_locale language codeset modifier + printf '%s' "$1" + test -z "$2" || printf '.%s' "$2" + test -z "$3" || printf '@%s' "$3" + printf '\n' +} diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote index af1a77f7..17941c63 100755 --- a/cdist/conf/type/__localedef/gencode-remote +++ b/cdist/conf/type/__localedef/gencode-remote @@ -21,6 +21,11 @@ # Manage system locales using localedef(1). # +# shellcheck source=cdist/conf/type/__localedef/files/lib/locale.sh +. "${__type:?}/files/lib/locale.sh" +# shellcheck source=cdist/conf/type/__localedef/files/lib/glibc.sh +. "${__type:?}/files/lib/glibc.sh" + state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") @@ -46,30 +51,6 @@ then exit 1 fi -parse_locale() { - # This function will split locales into their parts. Locale strings are - # usually of the form: [language[_territory][.codeset][@modifier]] - # For simplicity, language and territory are not separated by this function. - # Old Linux systems were also using "english" or "german" as locale strings. - # Usage: parse_locale locale_str lang_var codeset_var modifier_var - eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')" - eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')" - eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')" -} - -format_locale() { - # Usage: format_locale language codeset modifier - printf '%s' "$1" - test -z "$2" || printf '.%s' "$2" - test -z "$3" || printf '@%s' "$3" - printf '\n' -} - -gnu_normalize_codeset() { - echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]' -} - - : "${lang=}" "${codeset=}" "${modifier=}" # declare variables for shellcheck parse_locale "${locale}" lang codeset modifier From eeb98719197a09691af60fa26c0b64b3aaf8960e Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 14 Nov 2020 09:54:01 +0100 Subject: [PATCH 11/86] [type/__localedef] glibc: Also delete aliases when removing a locale --- cdist/conf/type/__localedef/gencode-remote | 15 +++++++++++++-- cdist/conf/type/__localedef/man.rst | 5 +++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote index 17941c63..d7fd6942 100755 --- a/cdist/conf/type/__localedef/gencode-remote +++ b/cdist/conf/type/__localedef/gencode-remote @@ -84,8 +84,19 @@ in EOF ;; (absent) - localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}") - printf "localedef --delete-from-archive '%s'\n" "${localename}" + main_localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}") + + cat <<-EOF + while read -r _alias _localename + do + if test "\${_localename}" = '$(format_locale "${lang}" "${codeset}")' + then + localedef --delete-from-archive "\${_alias}" + fi + done <'${aliasfile}' + + localedef --delete-from-archive '${main_localename}' + EOF ;; esac ;; diff --git a/cdist/conf/type/__localedef/man.rst b/cdist/conf/type/__localedef/man.rst index 0ccf484a..454ce9d1 100644 --- a/cdist/conf/type/__localedef/man.rst +++ b/cdist/conf/type/__localedef/man.rst @@ -13,6 +13,11 @@ This cdist type allows you to define locales on the system using On systems that don't support definition of new locales, the type will raise an error. +**NB:** This type respects the glibc ``locale.alias`` file, +i.e. it defines alias locales or deletes aliases of a locale when it is removed. +It is not possible, however, to use alias names to define locales or only remove +certain aliases of a locale. + OPTIONAL PARAMETERS ------------------- From 87faffd8750cac94190707c2b37233229c3fbde2 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 14 Nov 2020 10:41:50 +0100 Subject: [PATCH 12/86] [type/__localdef] Also check for aliases in state explorer --- cdist/conf/type/__localedef/explorer/state | 31 +++++++++++++++++++++- cdist/conf/type/__localedef/gencode-remote | 7 +++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__localedef/explorer/state b/cdist/conf/type/__localedef/explorer/state index d8db29c5..3ba57661 100755 --- a/cdist/conf/type/__localedef/explorer/state +++ b/cdist/conf/type/__localedef/explorer/state @@ -25,8 +25,14 @@ # the main locale (and possibly aliases) is present # absent: # neither the main locale nor any aliases are present +# alias-present: +# the main locale is absent, but at least one of its aliases is present # +# Hardcoded, create a pull request in case it is at another location for +# some other distro. (cf. gencode-remote) +aliasfile='/usr/share/locale/locale.alias' + command -v locale >/dev/null 2>&1 || exit 0 locales=$(locale -a) @@ -68,4 +74,27 @@ locale_available() ( } ) -locale_available "${__object_id:?}" && echo present || echo absent +if locale_available "${__object_id:?}" +then + echo present +else + # NOTE: locale.alias can be symlinked. + if test -e "${aliasfile}" + then + # Check if one of the aliases of the locale is defined + baselocale=$( + parse_locale "${__object_id:?}" _lang _codeset _modifiers \ + && format_locale "${_lang}" "${_codeset}") + while read -r _alias _localename + do + if test "${_localename}" = "${baselocale}" \ + && echo "${locales}" | grep -qxF "${_alias}" + then + echo alias-present + exit 0 + fi + done <"${aliasfile}" + fi + + echo absent +fi diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote index d7fd6942..4538151f 100755 --- a/cdist/conf/type/__localedef/gencode-remote +++ b/cdist/conf/type/__localedef/gencode-remote @@ -94,9 +94,12 @@ in localedef --delete-from-archive "\${_alias}" fi done <'${aliasfile}' - - localedef --delete-from-archive '${main_localename}' EOF + + if test "${state_is}" = present + then + printf "localedef --delete-from-archive '%s'\n" "${main_localename}" + fi ;; esac ;; From 9d4f69250ea0e6bb733cde1e42183480f48b5d8f Mon Sep 17 00:00:00 2001 From: ssrq Date: Thu, 19 Nov 2020 19:33:47 +0100 Subject: [PATCH 13/86] __sshd config: New type --- cdist/conf/type/__sshd_config/explorer/state | 121 ++++++++ .../files/update_sshd_config.awk | 293 ++++++++++++++++++ cdist/conf/type/__sshd_config/gencode-remote | 97 ++++++ cdist/conf/type/__sshd_config/man.rst | 94 ++++++ cdist/conf/type/__sshd_config/manifest | 48 +++ .../type/__sshd_config/parameter/default/file | 1 + .../__sshd_config/parameter/default/state | 1 + .../type/__sshd_config/parameter/optional | 4 + .../__sshd_config/parameter/optional_multiple | 1 + 9 files changed, 660 insertions(+) create mode 100644 cdist/conf/type/__sshd_config/explorer/state create mode 100644 cdist/conf/type/__sshd_config/files/update_sshd_config.awk create mode 100755 cdist/conf/type/__sshd_config/gencode-remote create mode 100644 cdist/conf/type/__sshd_config/man.rst create mode 100755 cdist/conf/type/__sshd_config/manifest create mode 100644 cdist/conf/type/__sshd_config/parameter/default/file create mode 100644 cdist/conf/type/__sshd_config/parameter/default/state create mode 100644 cdist/conf/type/__sshd_config/parameter/optional create mode 100644 cdist/conf/type/__sshd_config/parameter/optional_multiple diff --git a/cdist/conf/type/__sshd_config/explorer/state b/cdist/conf/type/__sshd_config/explorer/state new file mode 100644 index 00000000..75c68b8a --- /dev/null +++ b/cdist/conf/type/__sshd_config/explorer/state @@ -0,0 +1,121 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Determines the current state of the config option. +# Possible output: +# - present: "should" option present in config file +# - default: the "should" option is the default -> don’t know if present +# - absent: no such option present in config file +# + +joinlines() { sed -n -e H -e "\${x;s/^\\n//;s/\\n/${1:?}/g;p;}"; } +trlower() { tr '[:upper:]' '[:lower:]'; } +tolower() { printf '%s' "$*" | trlower; } + +default_value() { + sshd -T -f /dev/null -C "$(make_conn_spec)" \ + | sed -n -e 's/^'"$(tolower "${1:?}")"'[[:blank:]]\{1,\}//p' +} + +make_conn_spec() { + if test -s "${__object:?}/parameter/match" + then + _match_file="${__object:?}/parameter/match" + else + _match_file='/dev/null' + fi + + for _kw in \ + addr=Address \ + user=User \ + host=Host \ + laddr=LocalAddress \ + lport=LocalPort \ + rdomain=RDomain + do + _specname=${_kw%%=*} + _confname=$(tolower "${_kw#*=}") + while read -r _k _v + do + if test "$(tolower "${_k}")" = "${_confname}" + then + printf '%s=%s\n' "${_specname}" "${_v}" + continue 2 + fi + done <"${_match_file}" + + # NOTE: Print test spec even for empty keys to suppress errors like: + # 'Match User' in configuration but 'user' not in connection test specification. + # except lport: + # Invalid port '' in test mode specification lport= + test "${_specname}" = 'lport' || printf '%s=\n' "${_specname}" + done \ + | joinlines ',' + unset _match_file +} + +sshd_config_file=$(cat "${__object:?}/parameter/file") +state_should=$(cat "${__object:?}/parameter/state") + +if test -s "${__object:?}/parameter/option" +then + option_name=$(cat "${__object:?}/parameter/option") +else + option_name=${__object_id:?} +fi + +value_should=$(cat "${__object:?}/parameter/value" 2>/dev/null) \ +|| test "${state_should}" = absent || exit 0 # param optional if --state absent + +command -v sshd >/dev/null 2>&1 || { + echo 'Cannot find sshd.' >&2 + exit 1 +} + +test -e "${sshd_config_file}" || { + echo 'absent' + exit 0 +} + +value_is=$( + sshd -T -f "${sshd_config_file}" -C "$(make_conn_spec)" \ + | sed -n -e 's/^'"$(tolower "${option_name}")"'[[:blank:]]\{1,\}//p') + +if printf '%s\n' "${value_is}" | { + if test -n "${value_should}" + then + grep -q -x -F "${value_should}" + else + # if no value provided, assume "any" value + grep -q -e . + fi + } +then + if default_value "${option_name}" | grep -q -x -F "${value_is}" + then + # Might produce false positives for default values. + # TODO: Manual checking should be done, but for simplicity, this case is + # currently ignored here. + echo default + else + echo present + fi +else + echo absent +fi diff --git a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk new file mode 100644 index 00000000..d0bc2b4b --- /dev/null +++ b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk @@ -0,0 +1,293 @@ +# -*- mode: awk; indent-tabs-mode: t -*- + +function usage() { + print_err("Usage: awk -f update_sshd_config.awk -- -o set|unset [-m 'User git'] -l 'X11Forwarding no' /etc/ssh/sshd_config") +} + +function print_err(s) { print s | "cat >&2" } + +function alength(a, i) { + for (i = 0; (i + 1) in a; ++i); + return i +} + +function join(sep, a, i, s) { + for (i = i ? i : 1; i in a; i++) + s = s sep a[i] + return substr(s, 2) +} + +function getopt(opts, argv, target, files, i, c, lv, idx, nf) { + # trivial getopt(3) implementation; only basic functionality + if (argv[1] == "--") i++ + for (i += 1; i in argv; i++) { + if (lv) { target[c] = argv[i]; lv = 0; continue } + if (argv[i] ~ /^-/) { + c = substr(argv[i], 2, 1) + idx = index(opts, c) + if (!idx) { + print_err(sprintf("invalid option -%c\n", c)) + continue + } + if (substr(opts, idx + 1, 1) == ":") { + # option takes argument + if (length(argv[i]) > 2) + target[c] = substr(argv[i], 3) + else + lv = 1 + } else { + target[c] = 1 + } + } else + files[++nf] = argv[i] + } +} + +# tokenise configuration line +# this function mimics the counterpart in OpenSSH (misc.c) +# but it returns two (next token SUBSEP rest) because I didn’t want to have to +# simulate any pointer magic. +function strdelim_internal(s, split_equals, old) { + if (!s) + return "" + + old = s + + if (!match(s, WHITESPACE "|" QUOTE "" (split_equals ? "|" EQUALS : ""))) + return s + + s = substr(s, RSTART) + old = substr(old, 1, RSTART - 1) + + if (s ~ "^" QUOTE) { + old = substr(old, 2) + + # Find matching quote + if (match(s, QUOTE)) { + old = substr(old, 1, RSTART) + # s = substr() + if (match(s, "^" WHITESPACE "*")) + s = substr(s, RLENGTH) + return old + } else { + # no matching quote + return "" + } + } + + if (match(s, "^" WHITESPACE "+")) { + sub("^" WHITESPACE "+", "", s) + if (split_equals) + sub(EQUALS WHITESPACE "*", "", s) + } else if (s ~ "^" EQUALS) { + s = substr(s, 2) + } + + return old SUBSEP s +} +function strdelim(s) { return strdelim_internal(s, 1) } +function strdelimw(s) { return strdelim_internal(s, 0) } + +function singleton_option(opt) { + return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|authenticationmethods|authorizedkeysfile|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/ +} + +function print_update() { + if (mode) { + if (match_only) printf "\t" + printf "%s\n", line_should + updated = 1 + } +} + +BEGIN { + FS = "\n" # disable field splitting + + WHITESPACE = "[ \t]" # servconf.c, misc.c:strdelim_internal (without line breaks, cf. bugs) + QUOTE = "[\"]" # misc.c:strdelim_internal + EQUALS = "[=]" + + split("", opts) + split("", files) + getopt("ho:l:m:", ARGV, opts, files) + + if (opts["h"]) { usage(); exit (e="0") } + + line_should = opts["l"] + match_only = opts["m"] + num_files = alength(files) + + if (num_files != 1 || !opts["o"] || !line_should) { + usage() + exit (e=126) + } + + if (opts["o"] == "set") { + mode = 1 + } else if (opts["o"] == "unset") { + mode = 0 + } else { + print_err(sprintf("invalid mode %s\n", mode)) + exit (e=1) + } + + if (mode) { + # loop over sshd_config twice! + ARGV[2] = ARGV[1] = files[1] + ARGC = 3 + } else { + # only loop once + ARGV[1] = files[1] + ARGC = 2 + } + + split(strdelim(line_should), should, SUBSEP) + option_should = tolower(should[1]) + value_should = should[2] +} + +{ + line = $0 + + # Strip trailing whitespace. Allow \f (form feed) at EOL only + sub("(" WHITESPACE "|\f)*$", "", line) + + # Strip leading whitespace + sub("^" WHITESPACE "*", "", line) + + if (match(line, "^#" WHITESPACE "*")) { + prefix = substr(line, RSTART, RLENGTH) + line = substr(line, RSTART + RLENGTH) + } else { + prefix = "" + } + + line_type = "invalid" + option_is = value_is = "" + + if (line) { + split(strdelim(line), toks, SUBSEP) + + if (tolower(toks[1]) == "match") { + MATCH = (prefix ~ /^#/ ? "#" : "") join(" ", toks, 2) + line_type = "match" + } else if (toks[1] ~ /^[A-Za-z][A-Za-z0-9]+$/) { + # This could be an option line + line_type = "option" + option_is = tolower(toks[1]) + value_is = toks[2] + } + } else { + line_type = "empty" + } +} + +# mode: unset + +!mode { + # delete matching config + if (prefix !~ /^#/) + if (MATCH == match_only && option_is == option_should) + if (!value_should || value_should == value_is) + next + + print + next +} + + +# mode: set + +mode && NR == FNR { + if (line_type == "option") { + if (MATCH !~ /^#/) { + if (prefix ~ /^#/) { + # comment line + last_occ[MATCH, "#" option_is] = FNR + } else { + # option line + last_occ[MATCH, option_is] = FNR + } + last_occ[MATCH] = FNR + } + } else if (line_type == "invalid" && !prefix) { + # INVALID LINE + print_err(sprintf("%s: syntax error on line %u\n", ARGV[0], FNR)) + } + + next +} + +# before second pass prepare hashes containing location information to be used +# in the second pass. +mode && NR > FNR && FNR == 1 { + # First we drop the locations of commented-out options if a non-commented + # option is available. If a non-commented option is available, we will + # append new config options there to have them all at one place. + for (k in last_occ) { + if (k ~ /^#/) { + # delete entries of commented out match blocks + delete last_occ[k] + continue + } + + split(k, parts, SUBSEP) + + if (parts[2] ~ /^#/ && ((parts[1], substr(parts[2], 2)) in last_occ)) + delete last_occ[k] + } + + # Reverse the option => line mapping. The line_map allows for easier lookups + # in the second pass. + # We only keep options, not top-level keywords, because we can only have + # one entry per line and there are conflicts with last lines of "sections". + for (k in last_occ) { + if (!index(k, SUBSEP)) continue + line_map[last_occ[k]] = k + } +} + +# Second pass +mode && line_map[FNR] == match_only SUBSEP option_should && !updated { + split(line_map[FNR], parts, SUBSEP) + + # If option allows multiple values, print current value + if (!singleton_option(parts[2])) { + if (value_should != value_is) + print + } + + print_update() + + next +} + +mode { print } + +# Is a comment option +mode && line_map[FNR] == match_only SUBSEP "#" option_should && !updated { + print_update() +} + +# Last line of the should match section +mode && last_occ[match_only] == FNR && !updated { + # NOTE: Inserting empty lines is only cosmetic. It is only done if + # different options are next to each other and not in a match block + # (match blocks are usually not in the default config and thus don’t + # contain commented blocks.) + if (line && option_is != option_should && !MATCH) + print "" + print_update() +} + +END { + if (e) exit e + + if (mode && !updated) { + if (match_only && MATCH != match_only) { + printf "\nMatch %s\n", match_only + } + + print_update() + } +} diff --git a/cdist/conf/type/__sshd_config/gencode-remote b/cdist/conf/type/__sshd_config/gencode-remote new file mode 100755 index 00000000..0b44dfa7 --- /dev/null +++ b/cdist/conf/type/__sshd_config/gencode-remote @@ -0,0 +1,97 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +joinlines() { sed -n -e H -e "\${x;s/^\\n//;s/\\n/${1:?}/g;p;}"; } + +state_is=$(cat "${__object:?}/explorer/state") +state_should=$(cat "${__object:?}/parameter/state") + +if test "${state_is}" = "${state_should}" -o "${state_is}" = 'default' +then + # nothing to do (if the value is the default, ignore its state) + exit 0 +fi + +case ${state_should} +in + (present) + mode='set' + ;; + (absent) + mode='unset' + ;; + (*) + printf 'Invalid --state: %s\n' "${state_should}" >&2 + exit 1 + ;; +esac + +sshd_config_file=$(cat "${__object:?}/parameter/file") + +quote() { printf "'%s'" "$(printf '%s' "$*" | sed -e "s/'/'\\\\''/g")"; } +drop_awk_comments() { quote "$(sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@")"; } + +# Ensure the sshd_config file is there +cat <$(quote "${sshd_config_file}") + chown 0:0 $(quote "${sshd_config_file}") + chmod 0644 $(quote "${sshd_config_file}") +} + +EOF + +match_only= +if test -s "${__object:?}/parameter/match" +then + match_only=$(joinlines ' ' <"${__object:?}/parameter/match") +fi + +if test -s "${__object:?}/parameter/option" +then + option_line=$(cat "${__object:?}/parameter/option") +else + option_line=${__object_id:?} +fi + +if test -s "${__object:?}/parameter/value" +then + option_line="${option_line} $(cat "${__object:?}/parameter/value")" +fi + +# Send message on config update +printf '%s%s %s\n' "${mode}" "${match_only:+ [${match_only}]}" \ + "${option_line}" >>"${__messages_out:?}" + +# Update sshd_config (remote code) +cat <$(quote "${sshd_config_file}.tmp") \\ +|| exit + +cmp -s $(quote "${sshd_config_file}") $(quote "${sshd_config_file}.tmp") || { + sshd -t -f $(quote "${sshd_config_file}.tmp") \\ + && cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}") +} +rm -f $(quote "${sshd_config_file}.tmp") +EOF diff --git a/cdist/conf/type/__sshd_config/man.rst b/cdist/conf/type/__sshd_config/man.rst new file mode 100644 index 00000000..8b0069ac --- /dev/null +++ b/cdist/conf/type/__sshd_config/man.rst @@ -0,0 +1,94 @@ +cdist-type__sshd_config(7) +========================== + +NAME +---- +cdist-type__sshd_config - Manage options in sshd_config + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +file + The path to the sshd_config file to edit. + Defaults to ``/etc/ssh/sshd_config``. +match + Restrict this option to apply only for certain connections. + Allowed values are what would be allowed to be written after a ``Match`` + keyword in ``sshd_config``, e.g. ``--match 'User anoncvs'``. + + Can be used multiple times. All of the values are ANDed together. +option + The name of the option to manipulate. Defaults to ``__object_id``. +state + Can be: + + - ``present``: ensure a matching config line is present (or the default + value). + - ``absent``: ensure no matching config line is present. +value + The option's value to be assigned to the option (if ``--state present``) or + removed (if ``--state absent``). + + This option is required if ``--state present``. If not specified and + ``--state absent``, all values for the given option are removed. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # Disallow root logins with password + __sshd_config PermitRootLogin --value without-password + + # Disallow password-based authentication + __sshd_config PasswordAuthentication --value no + + # Accept the EDITOR environment variable + __sshd_config AcceptEnv:EDITOR --option AcceptEnv --value EDITOR + + # Force command for connections as git user + __sshd_config git@ForceCommand --match 'User git' --option ForceCommand \ + --value 'cd ~git && exec git-shell ${SSH_ORIGINAL_COMMAND:+-c "${SSH_ORIGINAL_COMMAND}"}' + + +SEE ALSO +-------- +:strong:`sshd_config`\ (5) + + +BUGS +---- +- This type assumes a nicely formatted config file, + i.e. no config options spanning multiple lines. +- ``Include`` directives are ignored. +- Config options are not added/removed to/from the config file if their value is + the default value. + + +AUTHORS +------- +Dennis Camera + + +COPYING +------- +Copyright \(C) 2020 Dennis Camera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest new file mode 100755 index 00000000..566bde90 --- /dev/null +++ b/cdist/conf/type/__sshd_config/manifest @@ -0,0 +1,48 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$(cat "${__global:?}/explorer/os") + +state_should=$(cat "${__object:?}/parameter/state") + +case ${os} +in + (alpine|centos|fedora|redhat|scientific|debian|devuan|ubuntu) + if test "${state_should}" != 'absent' + then + __package openssh-server --state present + fi + ;; + (archlinux|gentoo|slackware|suse) + if test "${state_should}" != 'absent' + then + __package openssh --state present + fi + ;; + (freebsd|netbsd|openbsd) + # whitelist + ;; + (*) + printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \ + "${os}" "${__type##*/}" >&2 + printf 'Please contribute an implementation for it if you can.\n' >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__sshd_config/parameter/default/file b/cdist/conf/type/__sshd_config/parameter/default/file new file mode 100644 index 00000000..d8ea5dfc --- /dev/null +++ b/cdist/conf/type/__sshd_config/parameter/default/file @@ -0,0 +1 @@ +/etc/ssh/sshd_config diff --git a/cdist/conf/type/__sshd_config/parameter/default/state b/cdist/conf/type/__sshd_config/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__sshd_config/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__sshd_config/parameter/optional b/cdist/conf/type/__sshd_config/parameter/optional new file mode 100644 index 00000000..922ab093 --- /dev/null +++ b/cdist/conf/type/__sshd_config/parameter/optional @@ -0,0 +1,4 @@ +file +option +state +value diff --git a/cdist/conf/type/__sshd_config/parameter/optional_multiple b/cdist/conf/type/__sshd_config/parameter/optional_multiple new file mode 100644 index 00000000..02b1d1a9 --- /dev/null +++ b/cdist/conf/type/__sshd_config/parameter/optional_multiple @@ -0,0 +1 @@ +match From 82eadb69948e631041263938994be5aeae161ab0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 19 Nov 2020 19:34:43 +0100 Subject: [PATCH 14/86] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index ff411a46..331acbeb 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,8 @@ next: * Type __locale: Add state explorer (Matthias Stecher) * Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera) * New type: __hwclock (Dennis Camera) + * Type __hostname: Fix guessing SuSE OS version (Dennis Camera) + * New type: __sshd_config (Dennis Camera) 6.9.1: 2020-11-08 * Type __file: Fix state pre-exists (Dennis Camera) From 803a9d62a7a6b7fd11844f6ea273f4b692c2334e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 20 Nov 2020 19:46:03 +0100 Subject: [PATCH 15/86] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 331acbeb..cd5dbb4f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,8 @@ next: * New type: __hwclock (Dennis Camera) * Type __hostname: Fix guessing SuSE OS version (Dennis Camera) * New type: __sshd_config (Dennis Camera) + * New type: __localedef (Dennis Camera) + * Type __locale: Deprecate in favor of __localedef (Dennis Camera) 6.9.1: 2020-11-08 * Type __file: Fix state pre-exists (Dennis Camera) From 23e0da521c785715357b27d66007bd03fe173bf3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 20 Nov 2020 19:46:55 +0100 Subject: [PATCH 16/86] Release 6.9.2 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index cd5dbb4f..9a91bf57 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.2: 2020-11-20 * Documentation: Fix examples in best practice (Dennis Camera) * Type __locale: Add state explorer (Matthias Stecher) * Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera) From 84172550df9e1800bc795dc51da01c9d3a0be9e0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 30 Nov 2020 20:29:51 +0100 Subject: [PATCH 17/86] __iptables*: add IPv6 support Because it currently only support IPv4. To implement this, it falls back to IPv4 for backward compatibilty, but now supports rules for IPv6 and both protocols at the same time. --- .../type/__iptables_apply/files/init-script | 84 +++++++++++++++---- cdist/conf/type/__iptables_rule/man.rst | 38 ++++++++- cdist/conf/type/__iptables_rule/manifest | 40 ++++++++- .../type/__iptables_rule/parameter/boolean | 3 + 4 files changed, 140 insertions(+), 25 deletions(-) create mode 100644 cdist/conf/type/__iptables_rule/parameter/boolean diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script index d9c79ef7..196f019b 100644 --- a/cdist/conf/type/__iptables_apply/files/init-script +++ b/cdist/conf/type/__iptables_apply/files/init-script @@ -1,6 +1,27 @@ #!/bin/sh -# Nico Schottelius -# Zürisee, Mon Sep 2 18:38:27 CEST 2013 +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2020 Matthias Stecher (matthiasstecher at gmx.de) +# +# This file is distributed with cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Originally written by: +# Nico Schottelius +# Zürisee, Mon Sep 2 18:38:27 CEST 2013 # ### BEGIN INIT INFO # Provides: iptables @@ -15,33 +36,60 @@ ### END INIT INFO +# Read files and execute the content with the given commands +# +# Arguments: +# 1: Directory +# 2..n: Commands which should be used to execute the file content +gothrough() { + cd "$1" || return + shift + + # iterate through all rules and continue if it's not a file + for rule in *; do + [ -f "$rule" ] || continue + echo "Appling iptables rule $rule ..." + + # execute it with all commands specificed + ruleparam="$(cat "$rule")" + for cmd in "$@"; do + # Command and Rule should be split. + # shellcheck disable=SC2046 + command $cmd $ruleparam + done + done +} + +# Shortcut for iptables command to do IPv4 and v6 +iptables() { + command iptables "$@" + command ip6tables "$@" +} + basedir=/etc/iptables.d -status="${basedir}/.pre-start" +status4="${basedir}/.pre-start" +status6="${basedir}/.pre-start6" case $1 in start) # Save status - iptables-save > "$status" + iptables-save > "$status4" + ip6tables-save > "$status6" # Apply our ruleset - cd "$basedir" || exit - count="$(find . ! -name . -prune | wc -l)" - - # Only do something if there are rules - if [ "$count" -ge 1 ]; then - for rule in *; do - echo "Applying iptables rule $rule ..." - # Rule should be split. - # shellcheck disable=SC2046 - iptables $(cat "$rule") - done - fi + gothrough "$basedir" iptables + #gothrough "$basedir/v4" iptables # conflicts with $basedir + gothrough "$basedir/v6" ip6tables + gothrough "$basedir/all" iptables ip6tables ;; stop) # Restore from status before, if there is something to restore - if [ -f "$status" ]; then - iptables-restore < "$status" + if [ -f "$status4" ]; then + iptables-restore < "$status4" + fi + if [ -f "$status6" ]; then + ip6tables-restore < "$status6" fi ;; restart) diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst index 92d8859f..75d0740b 100644 --- a/cdist/conf/type/__iptables_rule/man.rst +++ b/cdist/conf/type/__iptables_rule/man.rst @@ -25,6 +25,24 @@ state 'present' or 'absent', defaults to 'present' +BOOLEAN PARAMETERS +------------------ +All rules without any of this parameter will be threaten like ``--v4`` because +of backward compatibility. + +v4 + Explicitly set it as rule for IPv4. If IPv6 is set, too, it will be + threaten like ``--all``. Will be the default if nothing else is set. + +v6 + Explicitly set it as rule for IPv6. If IPv4 is set, too, it will be + threaten like ``--all``. + +all + Set the rule for both IPv4 and IPv6. It will be saved separately from the + other rules. + + EXAMPLES -------- @@ -48,6 +66,16 @@ EXAMPLES --state absent + # IPv4-only rule for ICMPv4 + __iptables_rule icmp-v4 --v4 --rule "-A INPUT -p icmp -j ACCEPT" + # IPv6-only rule for ICMPv6 + __iptables_rule icmp-v6 --v6 --rule "-A INPUT -p icmpv6 -j ACCEPT" + + # doing something for the dual stack + __iptables_rule fwd-eth0-eth1 --v4 --v6 --rule "-A INPUT -i eth0 -o eth1 -j ACCEPT" + __iptables_rule fwd-eth1-eth0 --all --rule "-A -o eth1 -i eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT" + + SEE ALSO -------- :strong:`cdist-type__iptables_apply`\ (7), :strong:`iptables`\ (8) @@ -56,11 +84,13 @@ SEE ALSO AUTHORS ------- Nico Schottelius +Matthias Stecher COPYING ------- -Copyright \(C) 2013 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2013 Nico Schottelius. +Copyright \(C) 2020 Matthias Stecher. +You can redistribute it and/or modify it under the terms of the GNU +General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest index ed78787f..27d00024 100755 --- a/cdist/conf/type/__iptables_rule/manifest +++ b/cdist/conf/type/__iptables_rule/manifest @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2020 Matthias Stecher (matthiasstecher at gmx.de) # # This file is part of cdist. # @@ -24,12 +25,36 @@ base_dir=/etc/iptables.d name="$__object_id" state="$(cat "$__object/parameter/state")" +if [ -f "$__object/parameter/v4" ]; then + only_v4="yes" + # $specific_dir is $base_dir +fi +if [ -f "$__object/parameter/v6" ]; then + only_v6="yes" + specific_dir="$base_dir/v6" +fi +# If rules should be set for both protocols +if ([ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]) \ + || [ -f "$__object/parameter/all" ]; then + + # all to a specific directory + specific_dir="$base_dir/all" +fi + +# set rule directory based on if it's the base or subdirectory +rule_dir="${specific_dir:-$base_dir}" + ################################################################################ # Basic setup # __directory "$base_dir" --state present +# sub-directory if required +if [ "$specific_dir" ]; then + require="__directory/$base_dir" __directory "$specific_dir" --state present +fi + # Have apply do the real job require="$__object_name" __iptables_apply @@ -37,6 +62,15 @@ require="$__object_name" __iptables_apply # The rule # -require="__directory/$base_dir" __file "$base_dir/${name}" \ - --source "$__object/parameter/rule" \ - --state "$state" +for dir in "$base_dir" "$base_dir/v6" "$base_dir/all"; do + # defaults to absent except the directory that should contain the file + if [ "$rule_dir" = "$dir" ]; then + curr_state="$state" + else + curr_state="absent" + fi + + require="__directory/$rule_dir" __file "$dir/$name" \ + --source "$__object/parameter/rule" \ + --state "$curr_state" +done diff --git a/cdist/conf/type/__iptables_rule/parameter/boolean b/cdist/conf/type/__iptables_rule/parameter/boolean new file mode 100644 index 00000000..76882272 --- /dev/null +++ b/cdist/conf/type/__iptables_rule/parameter/boolean @@ -0,0 +1,3 @@ +all +v4 +v6 From f568462e4981c8b6437c29f96963c3d8e7bed742 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 2 Dec 2020 17:48:41 +0100 Subject: [PATCH 18/86] __iptables_rule: fix shellcheck SC2235 --- cdist/conf/type/__iptables_rule/manifest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest index 27d00024..d4394c25 100755 --- a/cdist/conf/type/__iptables_rule/manifest +++ b/cdist/conf/type/__iptables_rule/manifest @@ -34,8 +34,8 @@ if [ -f "$__object/parameter/v6" ]; then specific_dir="$base_dir/v6" fi # If rules should be set for both protocols -if ([ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]) \ - || [ -f "$__object/parameter/all" ]; then +if { [ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]; } || + [ -f "$__object/parameter/all" ]; then # all to a specific directory specific_dir="$base_dir/all" From bee255c1ae0039e3d532317007f06fcd2e64e3bc Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 2 Dec 2020 18:04:50 +0100 Subject: [PATCH 19/86] __iptables_apply: man updates --- .../type/__iptables_apply/files/init-script | 1 + cdist/conf/type/__iptables_apply/man.rst | 21 +++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script index 196f019b..7faa2f92 100644 --- a/cdist/conf/type/__iptables_apply/files/init-script +++ b/cdist/conf/type/__iptables_apply/files/init-script @@ -61,6 +61,7 @@ gothrough() { } # Shortcut for iptables command to do IPv4 and v6 +# only applies to the "reset" target iptables() { command iptables "$@" command ip6tables "$@" diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst index 76e1f6bf..db0e7869 100644 --- a/cdist/conf/type/__iptables_apply/man.rst +++ b/cdist/conf/type/__iptables_apply/man.rst @@ -10,7 +10,14 @@ DESCRIPTION ----------- This cdist type deploys an init script that triggers the configured rules and also re-applies them on -configuration. +configuration. Rules are written from __iptables_rule +into the folder ``/etc/iptables.d/``. + +It reads all rules from the base folder as rules for IPv4. +Rules in the subfolder ``v6/`` are IPv6 rules. Rules in +the subfolder ``all/`` are applied to both rule tables. All +files contain the arguments for a single ``iptables`` and/or +``ip6tables`` command. REQUIRED PARAMETERS @@ -24,7 +31,7 @@ None EXAMPLES -------- -None (__iptables_apply is used by __iptables_rule) +None (__iptables_apply is used by __iptables_rule automaticly) SEE ALSO @@ -35,11 +42,13 @@ SEE ALSO AUTHORS ------- Nico Schottelius +Matthias Stecher COPYING ------- -Copyright \(C) 2013 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2013 Nico Schottelius. +Copyright \(C) 2020 Matthias Stecher. +You can redistribute it and/or modify it under the terms of the GNU +General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later version. From a1db5c3d0e7b5899f1a877c5bab28ac4d4796f8c Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 2 Dec 2020 18:22:31 +0100 Subject: [PATCH 20/86] __iptables*: Update manpages for execution order To make some thinks clear if someone needs it .. --- cdist/conf/type/__iptables_apply/man.rst | 10 ++++++++++ cdist/conf/type/__iptables_rule/man.rst | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst index db0e7869..4109e789 100644 --- a/cdist/conf/type/__iptables_apply/man.rst +++ b/cdist/conf/type/__iptables_apply/man.rst @@ -19,6 +19,16 @@ the subfolder ``all/`` are applied to both rule tables. All files contain the arguments for a single ``iptables`` and/or ``ip6tables`` command. +Rules are applied in the following order: +1. All IPv4 rules +2. All IPv6 rules +2. All rules that should be applied to both tables + +The order of the rules that will be applied are definite +from the result the shell glob returns, which should be +alphabetical. If rules must be applied in a special order, +prefix them with a number like ``02-some-rule``. + REQUIRED PARAMETERS ------------------- diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst index 75d0740b..86d38a34 100644 --- a/cdist/conf/type/__iptables_rule/man.rst +++ b/cdist/conf/type/__iptables_rule/man.rst @@ -11,6 +11,10 @@ DESCRIPTION This cdist type allows you to manage iptable rules in a distribution independent manner. +See :strong:`cdist-type__iptables_apply`\ (7) for the +execution order of these rules. It will be executed +automaticly to apply all rules non-volaite. + REQUIRED PARAMETERS ------------------- From 1055e92545e74cf7845ede8094e783b4103efbcc Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 2 Dec 2020 19:54:41 +0100 Subject: [PATCH 21/86] [setup.py] Add cdist.scan to packages --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 858c2c17..bfc8b495 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ os.chdir(cur) setup( name="cdist", - packages=["cdist", "cdist.core", "cdist.exec", "cdist.util", ], + packages=["cdist", "cdist.core", "cdist.exec", "cdist.scan", "cdist.util"], package_data={'cdist': package_data}, scripts=["bin/cdist", "bin/cdist-dump", "bin/cdist-new-type"], version=cdist.version.VERSION, From c7fa2efe6b780d63a779eed310bfa4ebf926d27e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 4 Dec 2020 15:30:08 +0100 Subject: [PATCH 22/86] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 9a91bf57..ad0aa75d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * pip install: Add cdist.scan to packages in setup.py (Dennis Camera) + 6.9.2: 2020-11-20 * Documentation: Fix examples in best practice (Dennis Camera) * Type __locale: Add state explorer (Matthias Stecher) From d44b5cfdc990ce020ffe3af5085bbb93b8ce64a8 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 4 Dec 2020 15:31:35 +0100 Subject: [PATCH 23/86] Release 6.9.3 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index ad0aa75d..6c272fd7 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.3: 2020-12-04 * pip install: Add cdist.scan to packages in setup.py (Dennis Camera) 6.9.2: 2020-11-20 From ba7d16a155cef46230d1dc650119f3d542a7f7f4 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 4 Dec 2020 17:57:55 +0100 Subject: [PATCH 24/86] __iptables_*: correct manpage spelling --- cdist/conf/type/__iptables_apply/man.rst | 2 +- cdist/conf/type/__iptables_rule/man.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst index 4109e789..3bef92cc 100644 --- a/cdist/conf/type/__iptables_apply/man.rst +++ b/cdist/conf/type/__iptables_apply/man.rst @@ -41,7 +41,7 @@ None EXAMPLES -------- -None (__iptables_apply is used by __iptables_rule automaticly) +None (__iptables_apply is used by __iptables_rule automatically) SEE ALSO diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst index 86d38a34..afb71e01 100644 --- a/cdist/conf/type/__iptables_rule/man.rst +++ b/cdist/conf/type/__iptables_rule/man.rst @@ -31,7 +31,7 @@ state BOOLEAN PARAMETERS ------------------ -All rules without any of this parameter will be threaten like ``--v4`` because +All rules without any of these parameters will be treated like ``--v4`` because of backward compatibility. v4 From 2d19856840400af57bd6667cc868ab314a3e07c2 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 4 Dec 2020 18:26:03 +0100 Subject: [PATCH 25/86] [type/__package_pkgng_freebsd] Set ASSUME_ALWAYS_YES instead of -y --- cdist/conf/type/__package_pkgng_freebsd/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote index b5944177..05ba4cb2 100755 --- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote @@ -75,7 +75,7 @@ execcmd(){ esac if [ -z "${pkg_bootstrapped}" ]; then - echo "pkg bootstrap -y >/dev/null 2>&1" + echo "ASSUME_ALWAYS_YES=yes pkg bootstrap >/dev/null 2>&1" fi echo "$_cmd >/dev/null 2>&1" # Silence the output of the command From 087be130fa67d3fe195387ae0ab079f39c5066e1 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 4 Dec 2020 19:23:49 +0100 Subject: [PATCH 26/86] __iptables_apply: shorten copyright header Do we need all the copyright header or is this sufficient? The licence is given for cdist, but not on the target host. But it should be clear anyway. --- .../type/__iptables_apply/files/init-script | 34 ++++++------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script index 7faa2f92..e42017ae 100644 --- a/cdist/conf/type/__iptables_apply/files/init-script +++ b/cdist/conf/type/__iptables_apply/files/init-script @@ -1,28 +1,4 @@ #!/bin/sh -# -# 2013 Nico Schottelius (nico-cdist at schottelius.org) -# 2020 Matthias Stecher (matthiasstecher at gmx.de) -# -# This file is distributed with cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Originally written by: -# Nico Schottelius -# Zürisee, Mon Sep 2 18:38:27 CEST 2013 -# ### BEGIN INIT INFO # Provides: iptables # Required-Start: $local_fs $remote_fs @@ -35,6 +11,16 @@ # and saves/restores previous status ### END INIT INFO +# Originally written by: +# Nico Schottelius +# Zürisee, Mon Sep 2 18:38:27 CEST 2013 +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2020 Matthias Stecher (matthiasstecher at gmx.de) +# +# This file is distributed with cdist and licenced under the +# GNU GPLv3+ WITHOUT ANY WARRANTY. + # Read files and execute the content with the given commands # From 3930f69456fd5a0d108a107e4ee61d87c9a73a56 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 6 Dec 2020 16:45:58 +0100 Subject: [PATCH 27/86] __block: fix escaping in here-doc This changes the here-document to do not interpret any shell-things. It also single-quotes some more strings that are printed to code-remote. Fixes #838 --- cdist/conf/type/__block/gencode-remote | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__block/gencode-remote b/cdist/conf/type/__block/gencode-remote index 1f5cc033..acdb3286 100755 --- a/cdist/conf/type/__block/gencode-remote +++ b/cdist/conf/type/__block/gencode-remote @@ -46,10 +46,10 @@ fi remove_block() { cat << DONE -tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX) +tmpfile=\$(mktemp '${file}.cdist.XXXXXXXXXX') # preserve ownership and permissions of existing file -if [ -f "$file" ]; then - cp -p "$file" "\$tmpfile" +if [ -f '$file' ]; then + cp -p '$file' "\$tmpfile" fi awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ ' { @@ -63,8 +63,8 @@ awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ ' } else { print } -}' "$file" > "\$tmpfile" -mv -f "\$tmpfile" "$file" +}' '$file' > "\$tmpfile" +mv -f "\$tmpfile" '$file' DONE } @@ -77,7 +77,7 @@ case "$state_should" in echo add >> "$__messages_out" fi cat << DONE -cat >> "$file" << ${__type##*/}_DONE +cat >> '$file' << '${__type##*/}_DONE' $(cat "$block") ${__type##*/}_DONE DONE From bed08c2c5c71ec4252743e0b64b75e7d8fedbd92 Mon Sep 17 00:00:00 2001 From: Evilham Date: Sun, 6 Dec 2020 20:24:00 +0100 Subject: [PATCH 28/86] Deal with deprecation of imp module. importlib has been a thing since Python 3.1, and imp has been deprecated since 3.4. Insert random complaint here about not being able to use f-strings because they were introduced in Python 3.6 and apparently we support Python 3.5 >,<. Output diff before to after for ./bin/cdist-build-helper test (on heavy load): ``` 1,2d0 < /usr/home/evilham/s/cdist/cdist/cdist/test/__main__.py:23: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses < import imp 72c70 < ERROR: cdisttesthost: __file/tmp/foobar requires object __file without object id. Defined at /tmp/tmp.cdist.test.g87lx7c8/tmp.cdist.test.6ramsakx --- > ERROR: cdisttesthost: __file/tmp/foobar requires object __file without object id. Defined at /tmp/tmp.cdist.test.aqdf6vjz/tmp.cdist.test.jgv3udel 76c74 < test_nonexistent_type_requirement (cdist.test.emulator.EmulatorTestCase) ... ERROR: cdisttesthost: __file/tmp/foobar requires object __does-not-exist/some-id, but type __does-not-exist does not exist. Defined at /tmp/tmp.cdist.test.mma5j8ln/tmp.cdist.test.3zg4by4d --- > test_nonexistent_type_requirement (cdist.test.emulator.EmulatorTestCase) ... ERROR: cdisttesthost: __file/tmp/foobar requires object __does-not-exist/some-id, but type __does-not-exist does not exist. Defined at /tmp/tmp.cdist.test.t8d6ockr/tmp.cdist.test.uimxurg9 86c84 < test_initial_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running initial manifest /tmp/tmp.cdist.test.uvid60ij/759547ff4356de6e3d9e08522b0d0807/data/conf/manifest/dump_environment --- > test_initial_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running initial manifest /tmp/tmp.cdist.test._cttcnrj/759547ff4356de6e3d9e08522b0d0807/data/conf/manifest/dump_environment 89c87 < test_type_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running type manifest /tmp/tmp.cdist.test.k1i2onpb/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__dump_environment/manifest for object __dump_environment/whatever --- > test_type_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running type manifest /tmp/tmp.cdist.test.ukr7lrzd/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__dump_environment/manifest for object __dump_environment/whatever 272c270 < Ran 225 tests in 44.457s --- > Ran 225 tests in 43.750s ``` --- cdist/test/__main__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cdist/test/__main__.py b/cdist/test/__main__.py index c8c7df3b..8049c752 100644 --- a/cdist/test/__main__.py +++ b/cdist/test/__main__.py @@ -20,7 +20,7 @@ # # -import imp +import importlib import os import sys import unittest @@ -37,8 +37,9 @@ for possible_test in os.listdir(base_dir): suites = [] for test_module in test_modules: - module_parameters = imp.find_module(test_module, [base_dir]) - module = imp.load_module("cdist.test." + test_module, *module_parameters) + module_spec = importlib.util.find_spec("cdist.test.{}".format(test_module)) + module = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(module) suite = unittest.defaultTestLoader.loadTestsFromModule(module) # print("Got suite: " + suite.__str__()) From 29662961731ef07fbc20213d0d87da1469f0d27c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 7 Dec 2020 19:47:52 +0100 Subject: [PATCH 29/86] ++changelog --- docs/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog b/docs/changelog index 6c272fd7..8b35901b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,10 @@ Changelog --------- +next: + * __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) + * Core: Deal with deprecated imp in unit tests (Evil Ham) + 6.9.3: 2020-12-04 * pip install: Add cdist.scan to packages in setup.py (Dennis Camera) From c5ca4cd2e13516dfb55371c1600e32297c3343e9 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 7 Dec 2020 19:59:05 +0100 Subject: [PATCH 30/86] __block: securly quote via the quote function Because the function already exists, it will be used for the file to be changed, too. Therefor, no quotes are required for that value. The prefix and suffix match was also improved: There is no regex check any more (the regex did checked the whole line); instead it will do it simple. --- cdist/conf/type/__block/gencode-remote | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/cdist/conf/type/__block/gencode-remote b/cdist/conf/type/__block/gencode-remote index acdb3286..7a1f4064 100755 --- a/cdist/conf/type/__block/gencode-remote +++ b/cdist/conf/type/__block/gencode-remote @@ -46,28 +46,29 @@ fi remove_block() { cat << DONE -tmpfile=\$(mktemp '${file}.cdist.XXXXXXXXXX') +tmpfile=\$(mktemp ${quoted_file}.cdist.XXXXXXXXXX) # preserve ownership and permissions of existing file -if [ -f '$file' ]; then - cp -p '$file' "\$tmpfile" +if [ -f $quoted_file ]; then + cp -p $quoted_file "\$tmpfile" fi -awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ ' +awk -v prefix=$(quote "$prefix") -v suffix=$(quote "$suffix") ' { - if (match(\$0,prefix)) { + if (\$0 == prefix) { triggered=1 } if (triggered) { - if (match(\$0,suffix)) { + if (\$0 == suffix) { triggered=0 } } else { print } -}' '$file' > "\$tmpfile" -mv -f "\$tmpfile" '$file' +}' $quoted_file > "\$tmpfile" +mv -f "\$tmpfile" $quoted_file DONE } +quoted_file="$(quote "$file")" case "$state_should" in present) if [ "$state_is" = "changed" ]; then @@ -77,7 +78,7 @@ case "$state_should" in echo add >> "$__messages_out" fi cat << DONE -cat >> '$file' << '${__type##*/}_DONE' +cat >> $quoted_file << '${__type##*/}_DONE' $(cat "$block") ${__type##*/}_DONE DONE From 14c81d6c7e20a2ed04c7318f6e8ecb51098390e8 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 8 Dec 2020 07:16:26 +0100 Subject: [PATCH 31/86] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 8b35901b..4be08ae9 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) * Core: Deal with deprecated imp in unit tests (Evil Ham) + * Type __iptables: Add IPv6 support (Matthias Stecher) 6.9.3: 2020-12-04 * pip install: Add cdist.scan to packages in setup.py (Dennis Camera) From a58f5ffa7f59a7e28ba0a414097f154e4a497e88 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 8 Dec 2020 19:36:44 +0100 Subject: [PATCH 32/86] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 4be08ae9..3d6084f1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) * Core: Deal with deprecated imp in unit tests (Evil Ham) * Type __iptables: Add IPv6 support (Matthias Stecher) + * Type __block: Fix escaping in here-doc (Matthias Stecher) 6.9.3: 2020-12-04 * pip install: Add cdist.scan to packages in setup.py (Dennis Camera) From a5169ad858a4d6e9c378184db5736f55a86306e0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 10 Dec 2020 21:24:26 +0100 Subject: [PATCH 33/86] new type __debian_backports This new type will setup the backports distribution for the current Debian release. --- cdist/conf/type/__debian_backports/man.rst | 90 +++++++++++++++++++ cdist/conf/type/__debian_backports/manifest | 59 ++++++++++++ .../parameter/default/mirror | 1 + .../parameter/default/state | 1 + .../__debian_backports/parameter/optional | 2 + cdist/conf/type/__debian_backports/singleton | 0 6 files changed, 153 insertions(+) create mode 100644 cdist/conf/type/__debian_backports/man.rst create mode 100755 cdist/conf/type/__debian_backports/manifest create mode 100644 cdist/conf/type/__debian_backports/parameter/default/mirror create mode 100644 cdist/conf/type/__debian_backports/parameter/default/state create mode 100644 cdist/conf/type/__debian_backports/parameter/optional create mode 100644 cdist/conf/type/__debian_backports/singleton diff --git a/cdist/conf/type/__debian_backports/man.rst b/cdist/conf/type/__debian_backports/man.rst new file mode 100644 index 00000000..ba353f4e --- /dev/null +++ b/cdist/conf/type/__debian_backports/man.rst @@ -0,0 +1,90 @@ +cdist-type__debian_backports(7) +=============================== + +NAME +---- +cdist-type__debian_backports - Install backports for Debain systems + + +DESCRIPTION +----------- +This singleton type installs backports for the current Debian version. +It aborts if backports are not supported for the specified os or no +version codename could be fetched (like Debian unstable). + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +state + Represents the state of the backports repository. ``present`` or + ``absent``, defaults to ``present``. + + Will be directly passed to :strong:`cdist-type__apt_source`\ (7). + +mirror + The mirror to fetch the backports from. Will defaults to the Debian default + ``_. + + Will be directly passed to :strong:`cdist-type__apt_source`\ (7). + + +BOOLEAN PARAMETERS +------------------ +None. + + +MESSAGES +-------- +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # setup the backports + __debian_backports + __debian_backports --state absent + __debian_backports --state present --mirror "http://ftp.de.debian.org/debian/" + + # update + require="__debian_backports" __apt_update_index + + # install a backports package + # currently for the buster release backports + require="__apt_update_index" __package_apt wireguard \ + --target-release buster-backports + + +ABORTS +------ +Aborts if the detected os is not Debian. + +Aborts if no distribuition codename could be detected. This is common for the +unstable distribution, but there is no backports repository for it already. + + +SEE ALSO +-------- +`Official Debian Backports site `_ + +:strong:`cdist-type__apt_source`\ (7) + + +AUTHORS +------- +Matthias Stecher + + +COPYING +------- +Copyright \(C) 2020 Matthias Stecher. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__debian_backports/manifest new file mode 100755 index 00000000..29bf9a43 --- /dev/null +++ b/cdist/conf/type/__debian_backports/manifest @@ -0,0 +1,59 @@ +#!/bin/sh -e +# __debian_backports/manifest +# +# 2020 Matthias Stecher (matthiasstecher at gmx.de) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Enables/disables backports repository. Utilies __apt_source for it. +# + + +# detect backport distribution +os="$(cat "$__global/explorer/os")" +case "$os" in + debian) + # distribution codename from /etc/os-release + # lsb_release may not be given in all debian installations + dist="$( + . "$__global/explorer/os-release" + printf "%s" "$VERSION_CODENAME" + )" + ;; + *) + printf "Backports for %s are not supported!\n" "$os" >&2 + exit 1 + ;; +esac + +# error if no codename given (e.g. on Debian unstable) +if [ -z "$dist" ]; then + printf "No backports for unkown version of distribution %s!\n" "$os" >&2 + exit 1 +fi + + +# parameters +state="$(cat "$__object/parameter/state")" +mirror="$(cat "$__object/parameter/mirror")" + +# install the given backports repository +__apt_source "${dist}-backports" \ + --state "$state" \ + --distribution "${dist}-backports" \ + --component main \ + --uri "$mirror" diff --git a/cdist/conf/type/__debian_backports/parameter/default/mirror b/cdist/conf/type/__debian_backports/parameter/default/mirror new file mode 100644 index 00000000..0965ef04 --- /dev/null +++ b/cdist/conf/type/__debian_backports/parameter/default/mirror @@ -0,0 +1 @@ +http://deb.debian.org/debian/ diff --git a/cdist/conf/type/__debian_backports/parameter/default/state b/cdist/conf/type/__debian_backports/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__debian_backports/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__debian_backports/parameter/optional b/cdist/conf/type/__debian_backports/parameter/optional new file mode 100644 index 00000000..4b05c235 --- /dev/null +++ b/cdist/conf/type/__debian_backports/parameter/optional @@ -0,0 +1,2 @@ +state +mirror diff --git a/cdist/conf/type/__debian_backports/singleton b/cdist/conf/type/__debian_backports/singleton new file mode 100644 index 00000000..e69de29b From 0d96b31b5696f95b559450e17c71914d0746c0ce Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 11 Dec 2020 18:13:44 +0100 Subject: [PATCH 34/86] __debian_backports: pass shellcheck for sourced file Because the sourced explorer can't be detected by shellcheck, it will be completely disabled. Changing the path to /etc/os-release isn't deterministic either. The shellcheck wiki page suggests to use `source=/dev/null` instead of `disable=SC1090`, but it was choosen to completely avoid that check .. --- cdist/conf/type/__debian_backports/manifest | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__debian_backports/manifest index 29bf9a43..661e5281 100755 --- a/cdist/conf/type/__debian_backports/manifest +++ b/cdist/conf/type/__debian_backports/manifest @@ -30,6 +30,7 @@ case "$os" in # distribution codename from /etc/os-release # lsb_release may not be given in all debian installations dist="$( + # shellcheck disable=SC1090 . "$__global/explorer/os-release" printf "%s" "$VERSION_CODENAME" )" From bc2948a8a552e3bda3ab9ae8e8e199e831d81af2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 11 Dec 2020 19:37:53 +0100 Subject: [PATCH 35/86] ++scan stuff --- cdist/scan/scan.py | 15 +++++++++++++++ docs/dev/logs/2020-10-29.org | 10 +++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index fcbf1899..00858581 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -56,6 +56,8 @@ from scapy.all import * # Datetime overwrites scapy.all.datetime - needs to be imported AFTER import datetime +import cdist.config + log = logging.getLogger("scan") class Trigger(object): @@ -120,6 +122,19 @@ class Scanner(object): with open(fname, "w") as fd: fd.write(f"{now}\n") + def config(self): + """ + Configure a host + + - Assume we are only called if necessary + - However we need to ensure to not run in parallel + - Maybe keep dict storing per host processes + - Save the result + - Save the output -> probably aligned to config mode + + """ + + def start(self): self.process = Process(target=self.scan) self.process.start() diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org index 4461be8c..03d6b3f4 100644 --- a/docs/dev/logs/2020-10-29.org +++ b/docs/dev/logs/2020-10-29.org @@ -54,4 +54,12 @@ VERBOSE: scan: Host fe80::f29f:c2ff:fe7c:275e is alive VERBOSE: scan: Host fe80::ba69:f4ff:fec5:8db7 is alive VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f863 is alive VERBOSE: scan: Host fe80::21b:fcff:feee:f4bc is alive -... +** Better usage -> saving the env + sudo -E cdist scan -b -I wlan0 -vv +** TODO Implement actual configuration step + - Also serves as a nice PoC + - Might need to escape literal IPv6 addresses for scp +** TODO Define how to map link local address to something useful + - via reverse DNS? + - via link local in manifest? +** TODO define ignorehosts? From c4d19a23193ec13eda96a3d7536ba338d0801825 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 12 Dec 2020 09:36:17 +0100 Subject: [PATCH 36/86] __debian_backports -> __apt_backports; add wider os support As discussed in the chat, this type now supports a broader list of OSes which it supports backports for. Because of this, it was renamed to something more generic. "apt" should fit in. --- .../man.rst | 27 ++++++++---- .../manifest | 41 ++++++++++++++----- .../parameter/default/state | 0 .../parameter/optional | 0 .../singleton | 0 .../parameter/default/mirror | 1 - 6 files changed, 49 insertions(+), 20 deletions(-) rename cdist/conf/type/{__debian_backports => __apt_backports}/man.rst (66%) rename cdist/conf/type/{__debian_backports => __apt_backports}/manifest (59%) rename cdist/conf/type/{__debian_backports => __apt_backports}/parameter/default/state (100%) rename cdist/conf/type/{__debian_backports => __apt_backports}/parameter/optional (100%) rename cdist/conf/type/{__debian_backports => __apt_backports}/singleton (100%) delete mode 100644 cdist/conf/type/__debian_backports/parameter/default/mirror diff --git a/cdist/conf/type/__debian_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst similarity index 66% rename from cdist/conf/type/__debian_backports/man.rst rename to cdist/conf/type/__apt_backports/man.rst index ba353f4e..7d269fbb 100644 --- a/cdist/conf/type/__debian_backports/man.rst +++ b/cdist/conf/type/__apt_backports/man.rst @@ -3,13 +3,13 @@ cdist-type__debian_backports(7) NAME ---- -cdist-type__debian_backports - Install backports for Debain systems +cdist-type__apt_backports - Install backports DESCRIPTION ----------- -This singleton type installs backports for the current Debian version. -It aborts if backports are not supported for the specified os or no +This singleton type installs backports for the current OS release. +It aborts if backports are not supported for the specified OS or no version codename could be fetched (like Debian unstable). @@ -27,8 +27,8 @@ state Will be directly passed to :strong:`cdist-type__apt_source`\ (7). mirror - The mirror to fetch the backports from. Will defaults to the Debian default - ``_. + The mirror to fetch the backports from. Will defaults to the generic + mirror of the current OS. Will be directly passed to :strong:`cdist-type__apt_source`\ (7). @@ -49,12 +49,12 @@ EXAMPLES .. code-block:: sh # setup the backports - __debian_backports - __debian_backports --state absent - __debian_backports --state present --mirror "http://ftp.de.debian.org/debian/" + __apt_backports + __apt_backports --state absent + __apt_backports --state present --mirror "http://ftp.de.debian.org/debian/" # update - require="__debian_backports" __apt_update_index + require="__apt_backports" __apt_update_index # install a backports package # currently for the buster release backports @@ -70,6 +70,15 @@ Aborts if no distribuition codename could be detected. This is common for the unstable distribution, but there is no backports repository for it already. +CAVEATS +------- +For Ubuntu, it setup all componenents for the backports repository: ``main``, +``restricted``, ``universe`` and ``multiverse``. The user may not want to +install proprietary packages, which will only be installed if the user +explicitly uses the backports target-release. The user may change this behavior +to install backports packages without the need of explicitly select it. + + SEE ALSO -------- `Official Debian Backports site `_ diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__apt_backports/manifest similarity index 59% rename from cdist/conf/type/__debian_backports/manifest rename to cdist/conf/type/__apt_backports/manifest index 661e5281..e5358dea 100755 --- a/cdist/conf/type/__debian_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -1,5 +1,5 @@ #!/bin/sh -e -# __debian_backports/manifest +# __apt_backports/manifest # # 2020 Matthias Stecher (matthiasstecher at gmx.de) # @@ -23,18 +23,34 @@ # +# Get the distribution codename by /etc/os-release. +# is already executed in a subshell by string substitution +# lsb_release may not be given in all installations +codename_os_release() { + # shellcheck disable=SC1090 + . "$__global/explorer/os-release" + printf "%s" "$VERSION_CODENAME" +} + # detect backport distribution os="$(cat "$__global/explorer/os")" case "$os" in debian) - # distribution codename from /etc/os-release - # lsb_release may not be given in all debian installations - dist="$( - # shellcheck disable=SC1090 - . "$__global/explorer/os-release" - printf "%s" "$VERSION_CODENAME" - )" + dist="$( codename_os_release )" + components="main" + mirror="http://deb.debian.org/debian/" ;; + devuan) + dist="$( codename_os_release )" + components="main" + mirror="http://deb.devuan.org/merged" + ;; + ubuntu) + dist="$( codename_os_release )" + components="main restricted universe multiverse" + mirror="http://archive.ubuntu.com/ubuntu" + ;; + *) printf "Backports for %s are not supported!\n" "$os" >&2 exit 1 @@ -50,11 +66,16 @@ fi # parameters state="$(cat "$__object/parameter/state")" -mirror="$(cat "$__object/parameter/mirror")" + +# mirror already set for the os, only override user-values +if [ -f "$__object/parameter/mirror" ]; then + mirror="$(cat "$__object/parameter/mirror")" +fi + # install the given backports repository __apt_source "${dist}-backports" \ --state "$state" \ --distribution "${dist}-backports" \ - --component main \ + --component "$components" \ --uri "$mirror" diff --git a/cdist/conf/type/__debian_backports/parameter/default/state b/cdist/conf/type/__apt_backports/parameter/default/state similarity index 100% rename from cdist/conf/type/__debian_backports/parameter/default/state rename to cdist/conf/type/__apt_backports/parameter/default/state diff --git a/cdist/conf/type/__debian_backports/parameter/optional b/cdist/conf/type/__apt_backports/parameter/optional similarity index 100% rename from cdist/conf/type/__debian_backports/parameter/optional rename to cdist/conf/type/__apt_backports/parameter/optional diff --git a/cdist/conf/type/__debian_backports/singleton b/cdist/conf/type/__apt_backports/singleton similarity index 100% rename from cdist/conf/type/__debian_backports/singleton rename to cdist/conf/type/__apt_backports/singleton diff --git a/cdist/conf/type/__debian_backports/parameter/default/mirror b/cdist/conf/type/__debian_backports/parameter/default/mirror deleted file mode 100644 index 0965ef04..00000000 --- a/cdist/conf/type/__debian_backports/parameter/default/mirror +++ /dev/null @@ -1 +0,0 @@ -http://deb.debian.org/debian/ From 49aec0b5e40ba05e7703565731aae6b0a9e5936a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 12 Dec 2020 09:40:47 +0100 Subject: [PATCH 37/86] __apt_backports: list supported OSes The manpage now lists all OSes where this type supports backports. --- cdist/conf/type/__apt_backports/man.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst index 7d269fbb..fd311361 100644 --- a/cdist/conf/type/__apt_backports/man.rst +++ b/cdist/conf/type/__apt_backports/man.rst @@ -12,6 +12,12 @@ This singleton type installs backports for the current OS release. It aborts if backports are not supported for the specified OS or no version codename could be fetched (like Debian unstable). +It supports backports from following OSes: + +- Debian +- Devuan +- Ubuntu + REQUIRED PARAMETERS ------------------- From fafa3d9ea55f38ae0a350a463d64b7ef1d70c7bb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 12 Dec 2020 10:00:23 +0100 Subject: [PATCH 38/86] __apt_backports: update index if required This type now automatically calls the type __apt_update_index to update the package index if something changed. --- cdist/conf/type/__apt_backports/man.rst | 13 +++++++------ cdist/conf/type/__apt_backports/manifest | 5 ++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst index fd311361..c578ce6b 100644 --- a/cdist/conf/type/__apt_backports/man.rst +++ b/cdist/conf/type/__apt_backports/man.rst @@ -9,8 +9,11 @@ cdist-type__apt_backports - Install backports DESCRIPTION ----------- This singleton type installs backports for the current OS release. -It aborts if backports are not supported for the specified OS or no -version codename could be fetched (like Debian unstable). +It aborts if backports are not supported for the specified OS or +no version codename could be fetched (like Debian unstable). + +The package index will be automatically updated by the type +:strong:`cdist-type__apt_update_index`\ (7) if required. It supports backports from following OSes: @@ -59,12 +62,9 @@ EXAMPLES __apt_backports --state absent __apt_backports --state present --mirror "http://ftp.de.debian.org/debian/" - # update - require="__apt_backports" __apt_update_index - # install a backports package # currently for the buster release backports - require="__apt_update_index" __package_apt wireguard \ + require="__apt_backports" __package_apt wireguard \ --target-release buster-backports @@ -90,6 +90,7 @@ SEE ALSO `Official Debian Backports site `_ :strong:`cdist-type__apt_source`\ (7) +:strong:`cdist-type__apt_update_index`\ (7) AUTHORS diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index e5358dea..c490a103 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -19,7 +19,7 @@ # along with cdist. If not, see . # # -# Enables/disables backports repository. Utilies __apt_source for it. +# Enables/disables backports repository. Utilises __apt_source for it. # @@ -79,3 +79,6 @@ __apt_source "${dist}-backports" \ --distribution "${dist}-backports" \ --component "$components" \ --uri "$mirror" + +# update the index if the source changed +require="__apt_source/${dist}-backports" __apt_update_index From 645734c62959c694597826598be3b34c0edc79d2 Mon Sep 17 00:00:00 2001 From: Evilham Date: Sat, 12 Dec 2020 12:15:17 +0100 Subject: [PATCH 39/86] [explorer/os_version] Improve FreeBSD support. It looks like uname -r is not the most reliable way to get the target patch level for the target system. For more information see: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743 --- cdist/conf/explorer/os_version | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index a7b1d3bc..3b02dedd 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -70,6 +70,11 @@ case "$("$__explorer/os")" in macosx) sw_vers -productVersion ;; + freebsd) + # Apparently uname -r is not a reliable way to get the patch level. + # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743 + freebsd-version + ;; *bsd|solaris) uname -r ;; From fca35fc858d09fc649dc5e7f0964cef4af9e09f0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 12 Dec 2020 17:29:58 +0100 Subject: [PATCH 40/86] __apt_backports: fix explorer call s/-/_/ because the explorers are following an other convention :-) --- cdist/conf/type/__apt_backports/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index c490a103..c2943725 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -28,7 +28,7 @@ # lsb_release may not be given in all installations codename_os_release() { # shellcheck disable=SC1090 - . "$__global/explorer/os-release" + . "$__global/explorer/os_release" printf "%s" "$VERSION_CODENAME" } From 27aca06fb893c84601e14bc30890ea6be9300dcd Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 12 Dec 2020 17:34:51 +0100 Subject: [PATCH 41/86] __apt_backports: undo __apt_update_index call Becuase it is already done by __apt_source. --- cdist/conf/type/__apt_backports/man.rst | 4 +--- cdist/conf/type/__apt_backports/manifest | 3 --- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst index c578ce6b..7036fb84 100644 --- a/cdist/conf/type/__apt_backports/man.rst +++ b/cdist/conf/type/__apt_backports/man.rst @@ -12,8 +12,7 @@ This singleton type installs backports for the current OS release. It aborts if backports are not supported for the specified OS or no version codename could be fetched (like Debian unstable). -The package index will be automatically updated by the type -:strong:`cdist-type__apt_update_index`\ (7) if required. +The package index will be automatically updated if required. It supports backports from following OSes: @@ -90,7 +89,6 @@ SEE ALSO `Official Debian Backports site `_ :strong:`cdist-type__apt_source`\ (7) -:strong:`cdist-type__apt_update_index`\ (7) AUTHORS diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index c2943725..bc47d8de 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -79,6 +79,3 @@ __apt_source "${dist}-backports" \ --distribution "${dist}-backports" \ --component "$components" \ --uri "$mirror" - -# update the index if the source changed -require="__apt_source/${dist}-backports" __apt_update_index From 71f22831175ec41e9348a759fcdaf0eceb1a2a52 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 13 Dec 2020 16:03:39 +0100 Subject: [PATCH 42/86] ++changelog --- docs/changelog | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 3d6084f1..290f94d0 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,10 +2,12 @@ Changelog --------- next: - * __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) + * Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) * Core: Deal with deprecated imp in unit tests (Evil Ham) * Type __iptables: Add IPv6 support (Matthias Stecher) * Type __block: Fix escaping in here-doc (Matthias Stecher) + * Explorer os_version: Improve FreeBSD support (Evil Ham) + * New type: __apt_backports (Matthias Stecher) 6.9.3: 2020-12-04 * pip install: Add cdist.scan to packages in setup.py (Dennis Camera) From 932e2496ed87b830a468a263cf1b1f5e6a88e2ef Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 15 Dec 2020 18:40:39 +0100 Subject: [PATCH 43/86] [type/__postgres_role] Lint --- .../conf/type/__postgres_role/explorer/state | 32 ++++---- .../conf/type/__postgres_role/gencode-remote | 78 ++++++++++--------- 2 files changed, 58 insertions(+), 52 deletions(-) diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index c8e1fa9d..110d29d5 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -11,32 +11,32 @@ # # cdist is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -case "$("${__explorer}/os")" +case $("${__explorer:?}/os") in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; + (netbsd) + postgres_user='pgsql' + ;; + (openbsd) + postgres_user='_postgresql' + ;; + (*) + postgres_user='postgres' + ;; esac -name="$__object_id" +rolename=${__object_id:?} -if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='$name'\"")" +if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='${rolename}'\"")" then - echo 'present' + echo 'present' else - echo 'absent' + echo 'absent' fi diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 282294c9..4fb761ed 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -11,55 +11,61 @@ # # cdist is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -case "$(cat "${__global}/explorer/os")" +case $(cat "${__global:?}/explorer/os") in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; + (netbsd) + postgres_user='pgsql' + ;; + (openbsd) + postgres_user='_postgresql' + ;; + (*) + postgres_user='postgres' + ;; esac -name="$__object_id" -state_is="$(cat "$__object/explorer/state")" -state_should="$(cat "$__object/parameter/state")" +rolename=${__object_id:?} +state_is=$(cat "${__object:?}/explorer/state") +state_should=$(cat "${__object:?}/parameter/state") -[ "$state_is" = "$state_should" ] && exit 0 +if test "${state_is}" = "${state_should}" +then + exit 0 +fi -case "$state_should" in - present) - if [ -f "$__object/parameter/password" ]; then - password="$(cat "$__object/parameter/password")" - fi - booleans="" - for boolean in login createdb createrole superuser; do - if [ ! -f "$__object/parameter/$boolean" ]; then - boolean="no${boolean}" - fi - upper=$(echo $boolean | tr '[:lower:]' '[:upper:]') - booleans="$booleans $upper" - done +case ${state_should} +in + (present) + if test -f "${__object:?}/parameter/password" + then + password=$(cat "${__object:?}/parameter/password") + fi + booleans= + for boolean in login createdb createrole superuser + do + if test ! -f "${__object:?}/parameter/${boolean}" + then + boolean="no${boolean}" + fi + booleans="${booleans} $(echo ${boolean} | tr '[:lower:]' '[:upper:]')" + done - [ -n "$password" ] && password="PASSWORD '$password'" - cat << EOF -su - '$postgres_user' -c "psql postgres -wc \"CREATE ROLE \\\\\"$name\\\\\" WITH $password $booleans;\"" + [ -n "${password}" ] && password="PASSWORD '${password}'" + cat << EOF +su - '${postgres_user}' -c "psql postgres -wc 'CREATE ROLE \\"${rolename}\\" WITH ${password} ${booleans};'" EOF - ;; - absent) - cat << EOF -su - '$postgres_user' -c "dropuser \"$name\"" + ;; + (absent) + cat << EOF +su - '${postgres_user}' -c "dropuser '${rolename}'" EOF - ;; + ;; esac From c36df82882fedb6ce3630e4718c46992a7c153a0 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 15 Dec 2020 21:05:55 +0100 Subject: [PATCH 44/86] [type/__postgres_role] ALTER ROLE when parameters change --- .../conf/type/__postgres_role/explorer/state | 42 +++++++++++- .../conf/type/__postgres_role/gencode-remote | 65 +++++++++++++++---- 2 files changed, 91 insertions(+), 16 deletions(-) diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 110d29d5..033afcb2 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -31,11 +32,48 @@ in ;; esac - rolename=${__object_id:?} -if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='${rolename}'\"")" +role_properties=$( + cmd=$(printf "psql -F '\034' -R '\036' -wAc \"SELECT * FROM pg_roles WHERE rolname='%s'\"" "${rolename}") + su -l "${postgres_user}" -c "${cmd}" \ + | awk ' + BEGIN { RS = "\036"; FS = "\034" } + /^\([0-9]+ rows?\)/ { exit } + NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next } + NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i } + ' +) + +if test -n "${role_properties}" then + # Check if the user's properties match the parameters + for prop in login createdb createrole superuser + do + bool_should=$(test -f "${__object:?}/parameter/${prop}" && echo 't' || echo 'f') + bool_is=$( + printf '%s\n' "${role_properties}" | + awk -F '=' -v key="${prop}" ' + BEGIN { + if (key == "login") + key = "canlogin" + else if (key == "superuser") + key = "super" + key = "rol" key + } + $1 == key { + sub(/^[^=]*=/, "") + print + } + ' + ) + + test "${bool_is}" = "${bool_should}" || { + echo 'different' + exit 0 + } + done + echo 'present' else echo 'absent' diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 4fb761ed..7837976c 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,6 +19,15 @@ # along with cdist. If not, see . # +quote() { + if test $# -gt 0 + then + printf '%s' "$*" + else + cat - + fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" +} + case $(cat "${__global:?}/explorer/os") in (netbsd) @@ -44,28 +54,55 @@ fi case ${state_should} in (present) - if test -f "${__object:?}/parameter/password" + if test -s "${__object:?}/parameter/password" then - password=$(cat "${__object:?}/parameter/password") + quoted_password=$( + delim='$$' + while grep -q -F "${delim}" "${__object:?}/parameter/password" + do + delim="\$$(LC_ALL=C tr -cd '[:alpha:]' /dev/null)$" + done + + raw_passwd=$(cat "${__object:?}/parameter/password"; printf .) + # shellcheck disable=SC2016 + printf '%s%s%s' "${delim}" "${raw_passwd%?.}" "${delim}" + ) fi + booleans= for boolean in login createdb createrole superuser do - if test ! -f "${__object:?}/parameter/${boolean}" - then - boolean="no${boolean}" - fi - booleans="${booleans} $(echo ${boolean} | tr '[:lower:]' '[:upper:]')" + booleans="${booleans}${booleans:+ }$( + if test -f "${__object:?}/parameter/${boolean}" + then + echo "${boolean}" + else + echo "no${boolean}" + fi \ + | tr '[:lower:]' '[:upper:]')" done - [ -n "${password}" ] && password="PASSWORD '${password}'" - cat << EOF -su - '${postgres_user}' -c "psql postgres -wc 'CREATE ROLE \\"${rolename}\\" WITH ${password} ${booleans};'" -EOF + case ${state_is} + in + (absent) + query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \ + "${rolename}" "${booleans}" "${quoted_password:-NULL}") + ;; + (different) + query=$(printf 'ALTER ROLE "%s" WITH %s PASSWORD %s;' \ + "${rolename}" "${booleans}" "${quoted_password:-NULL}") + ;; + (*) + exit 1 # TODO: error msg + ;; + esac + + psql_cmd=$(printf 'psql postgres -wc %s' "$(quote "${query}")" | quote) + printf "su -l '%s' -c %s\\n" "${postgres_user}" "${psql_cmd}" ;; (absent) - cat << EOF -su - '${postgres_user}' -c "dropuser '${rolename}'" -EOF + printf "su -l '%s' -c 'dropuser '\\\\'%s\\\\'\\n" \ + "${postgres_user}" \ + "$(quote "${rolename}")" ;; esac From 7b7ca4d385ccd025cbefced58419468651652da8 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 16 Dec 2020 19:06:50 +0100 Subject: [PATCH 45/86] [type/__postgres_role] Handle password changes --- .../conf/type/__postgres_role/explorer/state | 76 +++++++++++++++++-- .../conf/type/__postgres_role/gencode-remote | 23 ++++-- 2 files changed, 88 insertions(+), 11 deletions(-) diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 033afcb2..4aadbdff 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -34,9 +34,23 @@ esac rolename=${__object_id:?} + +psql_query() { + su -l "${postgres_user}" -c "$( + printf "psql -F '\034' -R '\036' -wAc '%s'" \ + "$(printf %s "$*" | sed "s/'/'\\\\''/g")" + )" +} + +password_check_login() ( + PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .) + PGPASSWORD=${PGPASSWORD%?.} + export PGPASSWORD + psql -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1 +) + role_properties=$( - cmd=$(printf "psql -F '\034' -R '\036' -wAc \"SELECT * FROM pg_roles WHERE rolname='%s'\"" "${rolename}") - su -l "${postgres_user}" -c "${cmd}" \ + psql_query "SELECT * FROM pg_roles WHERE rolname = '${rolename}'" \ | awk ' BEGIN { RS = "\036"; FS = "\034" } /^\([0-9]+ rows?\)/ { exit } @@ -69,12 +83,62 @@ then ) test "${bool_is}" = "${bool_should}" || { - echo 'different' - exit 0 + state='different properties' } done - echo 'present' + # Check password + passwd_stored=$( + psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \ + | awk 'BEGIN { RS = "\036" } NR == 2' + printf . + ) + passwd_stored=${passwd_stored%?.} + + passwd_should=$(cat "${__object}/parameter/password"; printf .) + passwd_should=${passwd_should%?.} + + if expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null + then + # SCRAM-SHA-256 "encrypted" password + # NOTE: There is currently no easy way to check SCRAM passwords + password_check_login || state="${state:-different} password" + elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null + then + # MD5 "encrypted" password + if command -v md5sum >/dev/null 2>&1 + then + should_md5=$( + printf '%s%s' "${passwd_should}" "${rolename}" \ + | md5sum - | sed -e 's/[^0-9a-f]*$//') + elif command -v gmd5sum >/dev/null 2>&1 + then + should_md5=$( + printf '%s%s' "${passwd_should}" "${rolename}" \ + | gmd5sum - | sed -e 's/[^0-9a-f]*$//') + elif command -v openssl >/dev/null 2>&1 + then + should_md5=$( + printf '%s%s' "${passwd_should}" "${rolename}" \ + | openssl dgst -md5 | sed 's/^.* //') + fi + + if test -n "${should_md5}" + then + test "${passwd_stored}" = "md5${should_md5}" \ + || state="${state:-different} password" + else + password_check_login || state="${state:-different} password" + fi + else + # unencrypted password (unsupported since PostgreSQL 10) + test "${passwd_stored}" = "${passwd_should}" \ + || state="${state:-different} password" + fi + + test -n "${state}" || state='present' else - echo 'absent' + state='absent' fi + +echo "${state}" diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 7837976c..fd590a4b 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -58,7 +58,9 @@ in then quoted_password=$( delim='$$' - while grep -q -F "${delim}" "${__object:?}/parameter/password" + # NOTE: Strip away trailing $ because with it the check breaks + # if the password ends with $ + random value. + while grep -q -F "${delim%$}" "${__object:?}/parameter/password" do delim="\$$(LC_ALL=C tr -cd '[:alpha:]' /dev/null)$" done @@ -88,12 +90,23 @@ in query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \ "${rolename}" "${booleans}" "${quoted_password:-NULL}") ;; - (different) - query=$(printf 'ALTER ROLE "%s" WITH %s PASSWORD %s;' \ - "${rolename}" "${booleans}" "${quoted_password:-NULL}") + (different*) + query="ALTER ROLE \"${rolename}\" WITH" + + if expr "${state_is}" : 'different.*properties' >/dev/null + then + query="${query} ${booleans}" + fi + if expr "${state_is}" : 'different.*password' >/dev/null + then + query="${query} PASSWORD ${quoted_password:-NULL}" + fi + + query="${query};" ;; (*) - exit 1 # TODO: error msg + printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2 + exit 1 ;; esac From 4859c27900d125841c703e0270dd8c0eae05d504 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 17 Dec 2020 16:57:03 +0100 Subject: [PATCH 46/86] [type/__postgres_role] Refactor gencode-remote --- .../conf/type/__postgres_role/gencode-remote | 65 ++++++++++--------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index fd590a4b..540eb606 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -51,9 +51,6 @@ then exit 0 fi -case ${state_should} -in - (present) if test -s "${__object:?}/parameter/password" then quoted_password=$( @@ -64,6 +61,11 @@ in do delim="\$$(LC_ALL=C tr -cd '[:alpha:]' /dev/null)$" done +psql_query() { + printf 'su -l %s -c %s\n' \ + "$(quote "${postgres_user}")" \ + "$(quote "psql postgres -wc $(quote "$1")")" +} raw_passwd=$(cat "${__object:?}/parameter/password"; printf .) # shellcheck disable=SC2016 @@ -71,51 +73,52 @@ in ) fi - booleans= - for boolean in login createdb createrole superuser - do - booleans="${booleans}${booleans:+ }$( - if test -f "${__object:?}/parameter/${boolean}" - then - echo "${boolean}" - else - echo "no${boolean}" - fi \ - | tr '[:lower:]' '[:upper:]')" - done +role_properties_should() { + _props= + for _prop in login createdb createrole superuser + do + _props="${_props}${_props:+ }$( + if test -f "${__object:?}/parameter/${_prop}" + then + echo "${_prop}" + else + echo "no${_prop}" + fi \ + | tr '[:lower:]' '[:upper:]')" + done + printf '%s\n' "${_props}" + unset _prop _props +} +case ${state_should} +in + (present) case ${state_is} in (absent) - query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \ - "${rolename}" "${booleans}" "${quoted_password:-NULL}") + psql_query "$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \ + "${rolename}" "$(role_properties_should)" "${quoted_password:-NULL}")" ;; (different*) - query="ALTER ROLE \"${rolename}\" WITH" - if expr "${state_is}" : 'different.*properties' >/dev/null then - query="${query} ${booleans}" - fi - if expr "${state_is}" : 'different.*password' >/dev/null - then - query="${query} PASSWORD ${quoted_password:-NULL}" + psql_query "ALTER ROLE \"${rolename}\" WITH $(role_properties_should);" fi - query="${query};" + if expr "${state_is}" : 'different.*password' >/dev/null + then + psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD ${quoted_password:-NULL};" + fi ;; (*) printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2 exit 1 ;; esac - - psql_cmd=$(printf 'psql postgres -wc %s' "$(quote "${query}")" | quote) - printf "su -l '%s' -c %s\\n" "${postgres_user}" "${psql_cmd}" ;; (absent) - printf "su -l '%s' -c 'dropuser '\\\\'%s\\\\'\\n" \ - "${postgres_user}" \ - "$(quote "${rolename}")" + printf 'su -l %s -c %s\n' \ + "$(quote "${postgres_user}")" \ + "$(quote "dropuser $(quote "${rolename}")")" ;; esac From 1180f13ed6c12e0277bebdf32bdd9840320d0fb9 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 17 Dec 2020 16:58:32 +0100 Subject: [PATCH 47/86] [type/__postgres_role] Fix setting password We need to make sure that the password does not end up in ~/.psql_history. --- .../conf/type/__postgres_role/gencode-remote | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 540eb606..15c3e692 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -51,27 +51,29 @@ then exit 0 fi - if test -s "${__object:?}/parameter/password" - then - quoted_password=$( - delim='$$' - # NOTE: Strip away trailing $ because with it the check breaks - # if the password ends with $ + random value. - while grep -q -F "${delim%$}" "${__object:?}/parameter/password" - do - delim="\$$(LC_ALL=C tr -cd '[:alpha:]' /dev/null)$" - done psql_query() { printf 'su -l %s -c %s\n' \ "$(quote "${postgres_user}")" \ "$(quote "psql postgres -wc $(quote "$1")")" } - raw_passwd=$(cat "${__object:?}/parameter/password"; printf .) - # shellcheck disable=SC2016 - printf '%s%s%s' "${delim}" "${raw_passwd%?.}" "${delim}" - ) - fi +psql_set_password() { + # NOTE: Always make sure that the password does not end up in psql_history! + if test -s "${__object:?}/parameter/password" + then + cat <<-EOF + exec 3< "\${__object:?}/parameter/password" + su -l '${postgres_user}' -c 'psql -q postgres -w' <<'SQL' + \set HISTFILE /dev/null + \set pw \`cat <&3\` + ALTER ROLE "${rolename}" WITH PASSWORD :'pw'; + SQL + exec 3<&- + EOF + else + psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD NULL;" + fi +} role_properties_should() { _props= @@ -96,8 +98,8 @@ in case ${state_is} in (absent) - psql_query "$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \ - "${rolename}" "$(role_properties_should)" "${quoted_password:-NULL}")" + psql_query "CREATE ROLE \"${rolename}\" WITH $(role_properties_should);" + psql_set_password ;; (different*) if expr "${state_is}" : 'different.*properties' >/dev/null @@ -107,7 +109,7 @@ in if expr "${state_is}" : 'different.*password' >/dev/null then - psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD ${quoted_password:-NULL};" + psql_set_password fi ;; (*) From 99d82fd0d5ab6d6ccd95fe8a7d3b24f5d82a4cce Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 17 Dec 2020 17:05:58 +0100 Subject: [PATCH 48/86] [type/__postgres_role] Always set psql -q --- cdist/conf/type/__postgres_role/explorer/state | 4 ++-- cdist/conf/type/__postgres_role/gencode-remote | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 4aadbdff..77779662 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -37,7 +37,7 @@ rolename=${__object_id:?} psql_query() { su -l "${postgres_user}" -c "$( - printf "psql -F '\034' -R '\036' -wAc '%s'" \ + printf "psql -q -F '\034' -R '\036' -wAc '%s'" \ "$(printf %s "$*" | sed "s/'/'\\\\''/g")" )" } @@ -46,7 +46,7 @@ password_check_login() ( PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .) PGPASSWORD=${PGPASSWORD%?.} export PGPASSWORD - psql -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1 + psql -q -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1 ) role_properties=$( diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 15c3e692..877c8135 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -54,7 +54,7 @@ fi psql_query() { printf 'su -l %s -c %s\n' \ "$(quote "${postgres_user}")" \ - "$(quote "psql postgres -wc $(quote "$1")")" + "$(quote "psql postgres -q -w -c $(quote "$1")")" } psql_set_password() { @@ -63,7 +63,7 @@ psql_set_password() { then cat <<-EOF exec 3< "\${__object:?}/parameter/password" - su -l '${postgres_user}' -c 'psql -q postgres -w' <<'SQL' + su -l '${postgres_user}' -c 'psql -q -w postgres' <<'SQL' \set HISTFILE /dev/null \set pw \`cat <&3\` ALTER ROLE "${rolename}" WITH PASSWORD :'pw'; From 8dc2c4207cdd826b9e64cc91cfc7c918cddd5f35 Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Fri, 18 Dec 2020 11:16:28 +0100 Subject: [PATCH 49/86] Added optional dirmode parameter to set the mode of (optional) the directory. --- cdist/conf/type/__dot_file/man.rst | 3 +++ cdist/conf/type/__dot_file/manifest | 2 ++ cdist/conf/type/__dot_file/parameter/default/dirmode | 1 + cdist/conf/type/__dot_file/parameter/optional | 1 + 4 files changed, 7 insertions(+) create mode 100644 cdist/conf/type/__dot_file/parameter/default/dirmode diff --git a/cdist/conf/type/__dot_file/man.rst b/cdist/conf/type/__dot_file/man.rst index ae65eb95..ba7621a1 100644 --- a/cdist/conf/type/__dot_file/man.rst +++ b/cdist/conf/type/__dot_file/man.rst @@ -25,6 +25,9 @@ user OPTIONAL PARAMETERS ------------------- +dirmode + forwarded to :strong:`__directory` type as mode + mode forwarded to :strong:`__file` type diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest index 5e4957e5..02dadf05 100755 --- a/cdist/conf/type/__dot_file/manifest +++ b/cdist/conf/type/__dot_file/manifest @@ -19,6 +19,7 @@ set -eu user="$(cat "${__object}/parameter/user")" home="$(cat "${__object}/explorer/home")" primary_group="$(cat "${__object}/explorer/primary_group")" +dirmode="$(cat "${__object}/parameter/dirmode")" # Create parent directory. Type __directory has flag 'parents', but it # will leave us with root-owned directory in user home, which is not @@ -36,6 +37,7 @@ export CDIST_ORDER_DEPENDENCY for dir ; do __directory "${home}/${dir}" \ --group "${primary_group}" \ + --mode "${dirmode}" \ --owner "${user}" done diff --git a/cdist/conf/type/__dot_file/parameter/default/dirmode b/cdist/conf/type/__dot_file/parameter/default/dirmode new file mode 100644 index 00000000..e9745d1f --- /dev/null +++ b/cdist/conf/type/__dot_file/parameter/default/dirmode @@ -0,0 +1 @@ +0700 diff --git a/cdist/conf/type/__dot_file/parameter/optional b/cdist/conf/type/__dot_file/parameter/optional index ccab9fa6..9f7f83fb 100644 --- a/cdist/conf/type/__dot_file/parameter/optional +++ b/cdist/conf/type/__dot_file/parameter/optional @@ -1,3 +1,4 @@ state mode source +dirmode From 4bae2863dbc1f6b60b8e797c2f121e1616a94c94 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 18 Dec 2020 12:54:33 +0100 Subject: [PATCH 50/86] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 290f94d0..3a623236 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * Type __block: Fix escaping in here-doc (Matthias Stecher) * Explorer os_version: Improve FreeBSD support (Evil Ham) * New type: __apt_backports (Matthias Stecher) + * Type __dot_file: Add dirmode parameter (Mark Verboom) 6.9.3: 2020-12-04 * pip install: Add cdist.scan to packages in setup.py (Dennis Camera) From 7cf85c465980c74049f8e0258758c0fd8ea178b2 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 21 Dec 2020 19:21:51 +0100 Subject: [PATCH 51/86] Release 6.9.4 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 3a623236..35953a88 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.4: 2020-12-21 * Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) * Core: Deal with deprecated imp in unit tests (Evil Ham) * Type __iptables: Add IPv6 support (Matthias Stecher) From 766198912d558e4630b258bba62e78587b3f0efe Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 5 Jan 2021 15:50:21 +0100 Subject: [PATCH 52/86] [type/__sshd_config] Produce error if invalid config file is generated Previously, cdist would silently swallow the error (no invalid config file was generated). Reason: `set -e` does not exit if a command in a sub-command group fails, it merely returns with a non-zero exit status. e.g. the following snippet does not abort the script if sshd -t returns with a non-zero exit status: set -e cmp -s old new || { # check config file and update it sshd -t -f new \ && cat new >old } or compressed: set -e false || { false && true; } echo $? # prints 1 --- cdist/conf/type/__sshd_config/gencode-remote | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__sshd_config/gencode-remote b/cdist/conf/type/__sshd_config/gencode-remote index 0b44dfa7..275db4aa 100755 --- a/cdist/conf/type/__sshd_config/gencode-remote +++ b/cdist/conf/type/__sshd_config/gencode-remote @@ -91,7 +91,8 @@ awk $(drop_awk_comments "${__type:?}/files/update_sshd_config.awk") \\ cmp -s $(quote "${sshd_config_file}") $(quote "${sshd_config_file}.tmp") || { sshd -t -f $(quote "${sshd_config_file}.tmp") \\ - && cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}") + && cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}") \\ + || exit # stop if sshd_config file check fails } rm -f $(quote "${sshd_config_file}.tmp") EOF From 8753b7eedf022a052ddc0b27fbc58ebf8fd638e1 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 5 Jan 2021 15:57:39 +0100 Subject: [PATCH 53/86] [type/__sshd_config] Make AuthenticationMethods and AuthorizedKeysFile singleton options They were incorrectly treated as non-singleton options before. cf. https://github.com/openssh/openssh-portable/blob/V_8_4/servconf.c#L2273 and https://github.com/openssh/openssh-portable/blob/V_8_4/servconf.c#L1899 resp. --- cdist/conf/type/__sshd_config/files/update_sshd_config.awk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk index d0bc2b4b..f7f30e87 100644 --- a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk +++ b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk @@ -89,7 +89,7 @@ function strdelim(s) { return strdelim_internal(s, 1) } function strdelimw(s) { return strdelim_internal(s, 0) } function singleton_option(opt) { - return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|authenticationmethods|authorizedkeysfile|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/ + return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/ } function print_update() { From bd8ab8f26fdc242f2eb77e58049d640f96d51a69 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 5 Jan 2021 17:00:55 +0100 Subject: [PATCH 54/86] [type/__sshd_config] Document "bug" in state explorer --- cdist/conf/type/__sshd_config/man.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cdist/conf/type/__sshd_config/man.rst b/cdist/conf/type/__sshd_config/man.rst index 8b0069ac..c8e6b8ad 100644 --- a/cdist/conf/type/__sshd_config/man.rst +++ b/cdist/conf/type/__sshd_config/man.rst @@ -79,6 +79,10 @@ BUGS - ``Include`` directives are ignored. - Config options are not added/removed to/from the config file if their value is the default value. +- | The explorer will incorrectly report ``absent`` if OpenSSH internally + transforms one value to another (e.g. ``permitrootlogin prohibit-password`` + is transformed to ``permitrootlogin without-password``). + | Workaround: Use the value that OpenSSH uses internally. AUTHORS From c819548343ad7bb24d023f94a32575e1b2520328 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 11 Jan 2021 09:50:12 +0100 Subject: [PATCH 55/86] Fix debug parameter -d was removed from cdist in favor of mulitple -v and -l parameters, but -d was not removed from preos. Resolve #849. --- cdist/preos/debootstrap/files/code | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/preos/debootstrap/files/code b/cdist/preos/debootstrap/files/code index 9e37003b..d836848c 100755 --- a/cdist/preos/debootstrap/files/code +++ b/cdist/preos/debootstrap/files/code @@ -22,7 +22,7 @@ set -e if [ "${debug}" ] then set -x - cdist_params="${cdist_params} -d" + cdist_params="${cdist_params} -l 3" fi bootstrap_dir="${target_dir}" From 2954347771e00610a084c7b033016765f7a3d6d7 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 14 Jan 2021 13:46:40 +0100 Subject: [PATCH 56/86] [type/__postgres_role] Add note regarding empty passwords --- cdist/conf/type/__postgres_role/gencode-remote | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 877c8135..d7631fbd 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -59,6 +59,8 @@ psql_query() { psql_set_password() { # NOTE: Always make sure that the password does not end up in psql_history! + # NOTE: Never set an empty string as the password, because they can be + # interpreted differently by different tooling. if test -s "${__object:?}/parameter/password" then cat <<-EOF From 6e9b13d94962e1bfb365efa849ebefc32854364b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 18 Jan 2021 06:22:32 +0100 Subject: [PATCH 57/86] ++changelog --- docs/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog b/docs/changelog index 35953a88..b2b35616 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,10 @@ Changelog --------- +next: + * Core: preos: Fix passing cdist debug parameter (Darko Poljak) + * Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera) + 6.9.4: 2020-12-21 * Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) * Core: Deal with deprecated imp in unit tests (Evil Ham) From 92a50da4873ff4bfa85dde3b85e272704cda02c7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 18 Jan 2021 06:28:09 +0100 Subject: [PATCH 58/86] Fix pycodestyle issues --- cdist/scan/scan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 5ca8fae2..b1d0e9e1 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -139,7 +139,6 @@ class Scanner(object): """ - def start(self): self.process = Process(target=self.scan) self.process.start() From 35cde3e666c229ceb94a1c5074083f7ba905768a Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 18 Jan 2021 13:09:29 +0100 Subject: [PATCH 59/86] [type/__postgres_role] Fix state explorer when stored password is empty --- cdist/conf/type/__postgres_role/explorer/state | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 77779662..34069de9 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -95,13 +95,20 @@ then ) passwd_stored=${passwd_stored%?.} - passwd_should=$(cat "${__object}/parameter/password"; printf .) + if test -f "${__object:?}/parameter/password" + then + passwd_should=$(cat "${__object:?}/parameter/password"; printf .) + fi passwd_should=${passwd_should%?.} - if expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null + if test -z "${passwd_stored}" + then + test -z "${passwd_should}" || state="${state:-different} password" + elif expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null then # SCRAM-SHA-256 "encrypted" password - # NOTE: There is currently no easy way to check SCRAM passwords + # NOTE: There is currently no easy way to check SCRAM passwords without + # logging in password_check_login || state="${state:-different} password" elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null then From 8eccacec59b0badd7a6e2a40a27f02f0ff355ccf Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 4 Feb 2021 19:09:26 +0100 Subject: [PATCH 60/86] __package_pip: add optional dependencies This is a poor implementation of optional dependencies for pip packages. It ensures to install them if the package will be installed, but does not take into account if they must be added/removed after the package is already installed. Also, it will not be autoremoved, as all dependencies will not be removed. --- cdist/conf/type/__package_pip/gencode-remote | 13 +++++++++++++ cdist/conf/type/__package_pip/man.rst | 13 +++++++++++++ .../type/__package_pip/parameter/optional_multiple | 1 + 3 files changed, 27 insertions(+) create mode 100644 cdist/conf/type/__package_pip/parameter/optional_multiple diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index a1375c2d..58c89040 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -56,6 +56,19 @@ fi case "$state_should" in present) + if [ -f "$__object/parameter/extra" ] + then + while read extra + do + if [ "$extras" ]; then + extras="$extras,$extra" + else + extras="$extra" + fi + done < "$__object/parameter/extra" + name="${name}[${extras}]" + fi + if [ "$runas" ] then echo "su -c '$pip install -q $name' $runas" diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst index 234ceee2..64ad0358 100644 --- a/cdist/conf/type/__package_pip/man.rst +++ b/cdist/conf/type/__package_pip/man.rst @@ -22,6 +22,11 @@ OPTIONAL PARAMETERS name If supplied, use the name and not the object id as the package name. +extra + Extra optional dependencies which should be installed along the selected + package. Can be specified multiple times. Will only be applied if the + package actually will be installed, but will not explicitly checked. + pip Instead of using pip from PATH, use the specific pip path. @@ -46,6 +51,14 @@ EXAMPLES # Use pip in a virtualenv located at /foo/shinken_virtualenv as user foo __package_pip pyro --state present --pip /foo/shinken_virtualenv/bin/pip --runas foo + # Install package with optional dependencies + __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails + # or do a little cheating + __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails + + # or take all extras + __package_pip mautrix-telegram --extra all + SEE ALSO -------- diff --git a/cdist/conf/type/__package_pip/parameter/optional_multiple b/cdist/conf/type/__package_pip/parameter/optional_multiple new file mode 100644 index 00000000..0f228715 --- /dev/null +++ b/cdist/conf/type/__package_pip/parameter/optional_multiple @@ -0,0 +1 @@ +extra From 73a03d75d7d23fea292b5fd60aeb059d8c72c4ba Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 4 Feb 2021 19:18:02 +0100 Subject: [PATCH 61/86] __package_pip: fix shellcheck --- cdist/conf/type/__package_pip/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index 58c89040..925eaf24 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -58,7 +58,7 @@ case "$state_should" in present) if [ -f "$__object/parameter/extra" ] then - while read extra + while read -r extra do if [ "$extras" ]; then extras="$extras,$extra" From cda17be38a1030742ed224b2536fca517bf8c09b Mon Sep 17 00:00:00 2001 From: ssrq Date: Mon, 8 Feb 2021 08:27:03 +0100 Subject: [PATCH 62/86] [explorer/memory] Clean up, return kiB for all systems, add SunOS BSDs were MiB before. --- cdist/conf/explorer/memory | 85 ++++++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 17 deletions(-) diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory index 5ea15ada..63aba9c6 100755 --- a/cdist/conf/explorer/memory +++ b/cdist/conf/explorer/memory @@ -1,8 +1,9 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Daniel Heule (hda at sfs.biz) # 2014 Thomas Oettli (otho at sfs.biz) # Copyright 2017, Philippe Gregoire +# 2020 Dennis Camera # # This file is part of cdist. # @@ -19,24 +20,74 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# +# Returns the amount of memory physically installed in the system, or if that +# cannot be determined the amount available to the operating system kernel, +# in kibibytes (kiB). -# FIXME: other system types (not linux ...) +str2bytes() { + awk -F' ' ' + $2 == "B" || !$2 { print $1 } + $2 == "kB" { print $1 * 1000 } + $2 == "MB" { print $1 * 1000 * 1000 } + $2 == "GB" { print $1 * 1000 * 1000 * 1000 } + $2 == "TB" { print $1 * 1000 * 1000 * 1000 * 1000 } + $2 == "kiB" { print $1 * 1024 } + $2 == "MiB" { print $1 * 1024 * 1024 } + $2 == "GiB" { print $1 * 1024 * 1024 * 1024 } + $2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }' +} -os=$("$__explorer/os") -case "$os" in - "macosx") - echo "$(sysctl -n hw.memsize)/1024" | bc - ;; +bytes2kib() { + set -- "$(cat)" + test "$1" -gt 0 && echo $(($1 / 1024)) +} - *"bsd") - PATH=$(getconf PATH) - echo "$(sysctl -n hw.physmem) / 1048576" | bc - ;; - *) - if [ -r /proc/meminfo ]; then - grep "MemTotal:" /proc/meminfo | awk '{print $2}' - fi - ;; +case $(uname -s) +in + (Darwin) + sysctl -n hw.memsize | bytes2kib + ;; + (FreeBSD) + sysctl -n hw.realmem | bytes2kib + ;; + (NetBSD|OpenBSD) + # NOTE: This reports "usable" memory, not physically installed memory. + command -p sysctl -n hw.physmem | bytes2kib + ;; + (SunOS) + # Make sure that awk from xpg4 is used for the scripts to work + export PATH="/usr/xpg4/bin:${PATH}" + prtconf \ + | awk -F ': ' ' + $1 == "Memory size" { sub(/Megabytes/, "MiB", $2); print $2 } + /^$/ { exit }' \ + | str2bytes \ + | bytes2kib + ;; + (Linux) + if test -d /sys/devices/system/memory + then + # Use memory blocks if the architecture (e.g. x86, PPC64, s390) + # supports them (they denote physical memory) + num_mem_blocks=$(cat /sys/devices/system/memory/memory[0-9]*/state | grep -cxF online) + mem_block_size=$(cat /sys/devices/system/memory/block_size_bytes) + + echo $((num_mem_blocks * 0x$mem_block_size)) | bytes2kib && exit + fi + if test -r /proc/meminfo + then + # Fall back to meminfo file on other architectures (e.g. ARM, MIPS, + # PowerPC) + # NOTE: This is "usable" memory, not physically installed memory. + awk -F ': +' '$1 == "MemTotal" { sub(/B$/, "iB", $2); print $2 }' /proc/meminfo \ + | str2bytes \ + | bytes2kib + fi + ;; + (*) + printf "Your kernel (%s) is currently not supported by the memory explorer\n" "$(uname -s)" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; esac From 65a6a2ed52c56f553d65ad0aa0aaba3ee4579e3c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 8 Feb 2021 08:28:31 +0100 Subject: [PATCH 63/86] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index b2b35616..52686617 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Core: preos: Fix passing cdist debug parameter (Darko Poljak) * Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera) + * Explorer memory: Fix result units; support Solaris (Dennis Camera) 6.9.4: 2020-12-21 * Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) From bc145bbc27a45e382adb744e6774fb803f4fda0a Mon Sep 17 00:00:00 2001 From: Evilham Date: Tue, 9 Feb 2021 19:58:47 +0100 Subject: [PATCH 64/86] [__letsencrypt_cert] Fix various issues with hooks. Closes #853, see issue for full description / discussion. Short summary: - There was about 6.53% chances of `--renewal-hook` not being applied - Using --automatic-renewal in one cert and not in another was an error. - It was not possible to use different hooks for different certificates. - FreeBSD support was utterly broken. --- cdist/conf/type/__letsencrypt_cert/man.rst | 84 +++++++-- cdist/conf/type/__letsencrypt_cert/manifest | 171 ++++++++++++++---- .../parameter/deprecated/automatic-renewal | 2 + .../parameter/deprecated/renew-hook | 2 + .../parameter/optional_multiple | 3 + 5 files changed, 220 insertions(+), 42 deletions(-) mode change 100755 => 100644 cdist/conf/type/__letsencrypt_cert/manifest create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook diff --git a/cdist/conf/type/__letsencrypt_cert/man.rst b/cdist/conf/type/__letsencrypt_cert/man.rst index 85eb88ea..43be8424 100644 --- a/cdist/conf/type/__letsencrypt_cert/man.rst +++ b/cdist/conf/type/__letsencrypt_cert/man.rst @@ -1,16 +1,33 @@ cdist-type__letsencrypt_cert(7) =============================== + NAME ---- cdist-type__letsencrypt_cert - Get an SSL certificate from Let's Encrypt + DESCRIPTION ----------- Automatically obtain a Let's Encrypt SSL certificate using Certbot. +This type attempts to setup automatic renewals always. In many Linux +distributions, that is the case out of the box, see: +https://certbot.eff.org/docs/using.html#automated-renewals + +For Alpine Linux and Arch Linux, we setup a system-wide cronjob that +attempts to renew certificates daily. + +If you are using FreeBSD, we configure periodic(8) as recommended by +the port mantainer, so there will be a weekly attempt at renewal. + +If your OS is not mentioned here or on Certbot's docs as having +support for automated renewals, please make sure you check your OS +and possibly patch this type so the system-wide cronjob is installed. + + REQUIRED PARAMETERS ------------------- @@ -21,6 +38,7 @@ object id admin-email Where to send Let's Encrypt emails like "certificate needs renewal". + OPTIONAL PARAMETERS ------------------- @@ -36,25 +54,68 @@ webroot The path to your webroot, as set up in your webserver config. If this parameter is not present, Certbot will be run in standalone mode. + OPTIONAL MULTIPLE PARAMETERS ---------------------------- -renew-hook - Renew hook command directly passed to Certbot in cron job. - domain Domains to be included in the certificate. When specified then object id is not used as a domain. +deploy-hook + Command to be executed only when the certificate associated with this + ``$__object_id`` is issued or renewed. + You can specify it multiple times, but any failure will prevent further + commands from being executed. + + For this command, the + shell variable ``$RENEWED_LINEAGE`` will point to the + config live subdirectory (for example, + ``/etc/letsencrypt/live/${__object_id}``) containing the + new certificates and keys; the shell variable + ``$RENEWED_DOMAINS`` will contain a space-delimited list + of renewed certificate domains (for example, + ``example.com www.example.com``) + +pre-hook + Command to be run in a shell before obtaining any + certificates. + You can specify it multiple times, but any failure will prevent further + commands from being executed. + + Note these run regardless of which certificate is attempted, you may want to + manage these system-wide hooks with ``__file`` in + ``/etc/letsencrypt/renewal-hooks/pre/``. + + Intended primarily for renewal, where it + can be used to temporarily shut down a webserver that + might conflict with the standalone plugin. This will + only be called if a certificate is actually to be + obtained/renewed. + +post-hook + Command to be run in a shell after attempting to + obtain/renew certificates. + You can specify it multiple times, but any failure will prevent further + commands from being executed. + + Note these run regardless of which certificate was attempted, you may want to + manage these system-wide hooks with ``__file`` in + ``/etc/letsencrypt/renewal-hooks/post/``. + + Can be used to deploy + renewed certificates, or to restart any servers that + were stopped by --pre-hook. This is only run if an + attempt was made to obtain/renew a certificate. + + BOOLEAN PARAMETERS ------------------ -automatic-renewal - Install a cron job, which attempts to renew certificates daily. - staging Obtain a test certificate from a staging server. + MESSAGES -------- @@ -67,6 +128,7 @@ create remove Certificate was removed. + EXAMPLES -------- @@ -75,8 +137,7 @@ EXAMPLES # use object id as domain __letsencrypt_cert example.com \ --admin-email root@example.com \ - --automatic-renewal \ - --renew-hook "service nginx reload" \ + --deploy-hook "service nginx reload" \ --webroot /data/letsencrypt/root .. code-block:: sh @@ -85,11 +146,10 @@ EXAMPLES # and example.com needs to be included again with domain parameter __letsencrypt_cert example.com \ --admin-email root@example.com \ - --automatic-renewal \ --domain example.com \ --domain foo.example.com \ --domain bar.example.com \ - --renew-hook "service nginx reload" \ + --deploy-hook "service nginx reload" \ --webroot /data/letsencrypt/root AUTHORS @@ -99,11 +159,13 @@ AUTHORS | Kamila Součková | Darko Poljak | Ľubomír Kučera +| Evilham + COPYING ------- -Copyright \(C) 2017-2018 Nico Schottelius, Kamila Součková, Darko Poljak and +Copyright \(C) 2017-2021 Nico Schottelius, Kamila Součková, Darko Poljak and Ľubomír Kučera. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest old mode 100755 new mode 100644 index b4464366..9c8cc043 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -1,18 +1,20 @@ #!/bin/sh certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")" +state=$(cat "${__object}/parameter/state") +os="$(cat "${__global:?}/explorer/os")" if [ -z "${certbot_fullpath}" ]; then - os="$(cat "${__global:?}/explorer/os")" os_version="$(cat "${__global}/explorer/os_version")" - + # Use this, very common value, as a default. It is OS-dependent + certbot_fullpath="/usr/bin/certbot" case "$os" in - archlinux) - __package certbot - ;; - alpine) - __package certbot - ;; + archlinux) + __package certbot + ;; + alpine) + __package certbot + ;; debian) case "$os_version" in 8*) @@ -48,9 +50,7 @@ if [ -z "${certbot_fullpath}" ]; then exit 1 ;; esac - - certbot_fullpath=/usr/bin/certbot - ;; + ;; devuan) case "$os_version" in jessie) @@ -83,17 +83,14 @@ if [ -z "${certbot_fullpath}" ]; then exit 1 ;; esac - - certbot_fullpath=/usr/bin/certbot ;; freebsd) - __package py27-certbot - - certbot_fullpath=/usr/local/bin/certbot + __package py37-certbot + certbot_fullpath="/usr/local/bin/certbot" ;; ubuntu) - __package certbot - ;; + __package certbot + ;; *) echo "Unsupported os: $os" >&2 exit 1 @@ -101,18 +98,130 @@ if [ -z "${certbot_fullpath}" ]; then esac fi -if [ -f "${__object}/parameter/automatic-renewal" ]; then - renew_hook_param="${__object}/parameter/renew-hook" - renew_hook="" - if [ -f "${renew_hook_param}" ]; then - while read -r hook; do - renew_hook="${renew_hook} --renew-hook \"${hook}\"" - done < "${renew_hook_param}" - fi +# Other OS-dependent values that we want to set every time +LE_DIR="/etc/letsencrypt" +certbot_cronjob_state="absent" +case "$os" in + archlinux|alpine) + certbot_cronjob_state="present" + ;; + freebsd) + LE_DIR="/usr/local/etc/letsencrypt" + # FreeBSD uses periodic(8) instead of crontabs for this + __line "periodic.conf_weekly_certbot" \ + --file "/etc/periodic.conf" \ + --regex "^(#[[:space:]]*)?weekly_certbot_enable=.*" \ + --state "replace" \ + --line 'weekly_certbot_enable="YES"' + ;; + *) + ;; +esac - __cron letsencrypt-certbot \ - --user root \ - --command "${certbot_fullpath} renew -q ${renew_hook}" \ - --hour 0 \ - --minute 47 +# This is only necessary in certain OS +__cron letsencrypt-certbot \ + --user root \ + --command "${certbot_fullpath} renew -q" \ + --hour 0 \ + --minute 47 \ + --state "${certbot_cronjob_state}" + +# Ensure hook directories +HOOKS_DIR="${LE_DIR}/renewal-hooks" +__directory "${LE_DIR}" --mode 0755 +require="__directory/${LE_DIR}" __directory "${HOOKS_DIR}" --mode 0755 + +if [ -f "${__object}/parameter/domain" ]; then + domains="$(sort "${__object}/parameter/domain")" +else + domains="${__object_id}" fi + +# Install hooks as needed +for hook in deploy pre post; do + # Using something unique and specific to this object + hook_file="${HOOKS_DIR}/${hook}/${__object_id}.cdist.sh" + # Reasonable defaults + hook_source="${__object}/parameter/${hook}-hook" + hook_state="absent" + hook_contents_head="#!/bin/sh -e" + hook_contents_logic="" + hook_contents_tail="" + + # Backwards compatibility + # Remove this when renew-hook is removed + # Falling back to renew-hook if deploy-hook is not passed + if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then + hook_source="${__object}/parameter/renew-hook" + fi + if [ "${state}" = "present" ] && \ + [ -f "${hook_source}" ]; then + # This hook is to be installed, let's generate it with some + # safety boilerplate + # Since certbot runs all hooks for all renewal processes + # (at each state for deploy, pre, post), it is up to us to + # differentiate whether or not the hook must run + hook_state="present" + hook_contents_head="$(cat <> /dev/stderr + exit 1 + ;; + esac + + hook_contents_tail="$(cat < Date: Tue, 9 Feb 2021 20:29:17 +0100 Subject: [PATCH 65/86] [__letsencrypt_cert] Remove problematic trailing slash in sed. Happy fingers are happy and like adding slashes places. --- cdist/conf/type/__letsencrypt_cert/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 9c8cc043..04edea1e 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -206,7 +206,7 @@ EOF hook_contents_tail="$(cat < Date: Tue, 9 Feb 2021 20:53:58 +0100 Subject: [PATCH 66/86] [__letsencrypt_cert] Don't mess with user script indentation This could break in odd ways if they passed sth like: cat < Date: Wed, 10 Feb 2021 10:10:21 +0100 Subject: [PATCH 67/86] [__letsencrypt_cert] Move hook contents generation out of manifest While there address some minor issues in the comments in the hook contents. --- .../type/__letsencrypt_cert/files/gen_hook.sh | 84 +++++++++++++++++++ cdist/conf/type/__letsencrypt_cert/manifest | 78 +---------------- 2 files changed, 88 insertions(+), 74 deletions(-) create mode 100644 cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh diff --git a/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh b/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh new file mode 100644 index 00000000..81ea4856 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh @@ -0,0 +1,84 @@ +#!/bin/sh -e + +# It is expected that this defines hook_contents + +# Reasonable defaults +hook_source="${__object}/parameter/${hook}-hook" +hook_state="absent" +hook_contents_head="#!/bin/sh -e" +hook_contents_logic="" +hook_contents_tail="" + +# Backwards compatibility +# Remove this when renew-hook is removed +# Falling back to renew-hook if deploy-hook is not passed +if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then + hook_source="${__object}/parameter/renew-hook" +fi +if [ "${state}" = "present" ] && \ + [ -f "${hook_source}" ]; then + # This hook is to be installed, let's generate it with some + # safety boilerplate + # Since certbot runs all hooks for all renewal processes + # (at each state for deploy, pre, post), it is up to us to + # differentiate whether or not the hook must run + hook_state="present" + hook_contents_head="$(cat <> /dev/stderr + exit 1 + ;; + esac + + hook_contents_tail="$(cat <> /dev/stderr - exit 1 - ;; - esac + # This defines hook_contents + # shellcheck source=cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh + . "${__type}/files/gen_hook.sh" - hook_contents_tail="$(cat < Date: Thu, 11 Feb 2021 10:31:07 +0100 Subject: [PATCH 68/86] __package_pip: add extras explorer The two new explorers detect all installed extras for this package. --- .../type/__package_pip/explorer/distinfo-dir | 45 +++++++++++++++++++ cdist/conf/type/__package_pip/explorer/extras | 26 +++++++++++ cdist/conf/type/__package_pip/explorer/state | 0 3 files changed, 71 insertions(+) create mode 100755 cdist/conf/type/__package_pip/explorer/distinfo-dir create mode 100755 cdist/conf/type/__package_pip/explorer/extras mode change 100644 => 100755 cdist/conf/type/__package_pip/explorer/state diff --git a/cdist/conf/type/__package_pip/explorer/distinfo-dir b/cdist/conf/type/__package_pip/explorer/distinfo-dir new file mode 100755 index 00000000..18e169ae --- /dev/null +++ b/cdist/conf/type/__package_pip/explorer/distinfo-dir @@ -0,0 +1,45 @@ +#!/bin/sh +# +# 2021 Matthias Stecher (matthiasstecher at gmx.de) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +nameparam="$__object/parameter/name" +if [ -f "$nameparam" ]; then + name=$(cat "$nameparam") +else + name="$__object_id" +fi + +pipparam="$__object/parameter/pip" +if [ -f "$pipparam" ]; then + pip=$(cat "$pipparam") +else + pip="$( "$__type_explorer/pip" )" +fi + + +if command -v "$pip" >/dev/null 2>&1; then + # assemble the path where pip stores all pip package info + "$pip" show "$name" \ + | awk -F': ' ' + $1 == "Name" {name=$2; gsub(/-/,"_",name); next} + $1 == "Version" {version=$2; next} + $1 == "Location" {location=$2; next} + END {if (version != "") printf "%s/%s-%s.dist-info", location, name, version}' +fi diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras new file mode 100755 index 00000000..92fc4732 --- /dev/null +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -0,0 +1,26 @@ +#!/bin/sh +# +# 2021 Matthias Stecher (matthiasstecher at gmx.de) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +distinfo_dir="$("$__type_explorer/distinfo-dir")" +if [ "$distinfo_dir" ]; then + # output all extras that are installed + awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA" +fi diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state old mode 100644 new mode 100755 From 8dc6ab97384f2240e9218fbd4463ba12b004f44c Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 11 Feb 2021 13:49:53 +0100 Subject: [PATCH 69/86] __package_pip: install not found extras Compares the explorer against the parameters and install those extras that are not already installed. --- cdist/conf/type/__package_pip/gencode-remote | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index 925eaf24..ad327e37 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -2,6 +2,7 @@ # # 2012 Nico Schottelius (nico-cdist at schottelius.org) # 2016 Darko Poljak (darko.poljak at gmail.com) +# 2021 Matthias Stecher (matthiasstecher at gmx.de) # # This file is part of cdist. # @@ -58,14 +59,17 @@ case "$state_should" in present) if [ -f "$__object/parameter/extra" ] then - while read -r extra - do - if [ "$extras" ]; then - extras="$extras,$extra" - else - extras="$extra" - fi - done < "$__object/parameter/extra" + mkdir "$__object/files" + # sort and generalize all extras + sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params" + sort "$__object/explorer/extras" > "$__object/files/extras-explorer" + + # assemble extras + # all extras are passed to pip in a comma-separated list + extras="$(comm -23 \ + "$__object/files/extras-params" "$__object/files/extras-explorer" \ + | sed ':a; $!N; s/\n/,/; ta' # loops through all input lines and add commas between them + )" name="${name}[${extras}]" fi From 2db0ef7c98226661e5e4b55c89180b31ba65b8bc Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 11 Feb 2021 22:53:26 +0100 Subject: [PATCH 70/86] __package_pip: updating real detection of extras As the previous detection took the wrong values, this explorer now checks if packages for an extra are installed or not. If not, the extra is not installed. Based on the information of the explorer, it will install the package again with the absent extras. --- cdist/conf/type/__package_pip/explorer/extras | 37 +++++++++++++++++-- cdist/conf/type/__package_pip/gencode-remote | 21 ++++------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index 92fc4732..2f5fcca6 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -17,10 +17,41 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# +# Checks if the given extras are really installed or not. It will be +# done by querring all dependencies for that extra and return it as +# "to be installed" if no dependency was found. +# distinfo_dir="$("$__type_explorer/distinfo-dir")" -if [ "$distinfo_dir" ]; then - # output all extras that are installed - awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA" + +# check if we have something to check +if [ "$distinfo_dir" ] && [ -s "$__object/parameter/extra" ] +then + # save cause freezing is slow + mkdir "$__object/files" + pip_freeze="$__object/files/pip-freeze.tmp" + pip3 freeze > "$pip_freeze" + + for extra in $(cat "$__object/parameter/extra" | tr ',' '\n') + do + # create a grep BRE pattern to search all packages + grep_pattern="$( + awk -F'(: | ; )' -v check="$extra" ' + $1 == "Requires-Dist" { + split($2, r, " "); + sub("extra == ", "", $3); gsub("'"'"'", "", $3); + if($3 == check) print r[1] + }' "$distinfo_dir/METADATA" \ + | sed ':a; $!N; s/\n/\\|/; ta' + )" + + # echo the extra if no packages where found for it + # if there is no pattern, we don't need to search ;-) + if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze" + then + echo "$extra" + fi + done fi diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index ad327e37..9abe28bf 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -26,7 +26,10 @@ state_is=$(cat "$__object/explorer/state") state_should="$(cat "$__object/parameter/state")" -[ "$state_is" = "$state_should" ] && exit 0 +# short circuit if state is the same and no extras to install +[ "$state_is" = "$state_should" ] && ! [ -s "$__object/explorer/extras" ] \ + && exit 0 + nameparam="$__object/parameter/name" if [ -f "$nameparam" ]; then @@ -57,19 +60,11 @@ fi case "$state_should" in present) - if [ -f "$__object/parameter/extra" ] + if [ -s "$__object/explorer/extras" ] then - mkdir "$__object/files" - # sort and generalize all extras - sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params" - sort "$__object/explorer/extras" > "$__object/files/extras-explorer" - - # assemble extras - # all extras are passed to pip in a comma-separated list - extras="$(comm -23 \ - "$__object/files/extras-params" "$__object/files/extras-explorer" \ - | sed ':a; $!N; s/\n/,/; ta' # loops through all input lines and add commas between them - )" + # all extras are passed to pip in a comma-separated list in the name + # sed loops through all input lines and add commas between them + extras="$(sed ':a; $!N; s/\n/,/; ta' "$__object/explorer/extras")" name="${name}[${extras}]" fi From 7398382890a4368e588e700183e6c777e90fb069 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 11 Feb 2021 23:12:10 +0100 Subject: [PATCH 71/86] __package_pip: fix shellcheck Useless `cat $file`, use `< $file` instead. --- cdist/conf/type/__package_pip/explorer/extras | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index 2f5fcca6..d57c69db 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -34,7 +34,7 @@ then pip_freeze="$__object/files/pip-freeze.tmp" pip3 freeze > "$pip_freeze" - for extra in $(cat "$__object/parameter/extra" | tr ',' '\n') + for extra in $(tr ',' '\n' < "$__object/parameter/extra") do # create a grep BRE pattern to search all packages grep_pattern="$( From a9d7dfb2ed40d0a61978cc706ff469c354a6c8f5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 12 Feb 2021 09:17:02 +0100 Subject: [PATCH 72/86] __package_pip: split extra 'all' to a list of all extras This will fix if a package will be upgraded from some extras to all extras. Previously, it will not work because some dependencies of 'all' are already installed, so the feature 'all' is already installed. Now, it will use a list of all extras to iterate over them separatly. This will result it will never install all extras via `[all]`, but rather `[foo,bar]`. --- cdist/conf/type/__package_pip/explorer/extras | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index d57c69db..be945e36 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -34,7 +34,14 @@ then pip_freeze="$__object/files/pip-freeze.tmp" pip3 freeze > "$pip_freeze" - for extra in $(tr ',' '\n' < "$__object/parameter/extra") + # If all is set, it searches all available extras to separatly check them. + # It would work with just 'all' (cause dependencies are given), but will + # not update if one extra is already present. Side effect is that it will + # not use [all] but instead name all extras seperatly. + for extra in $(if grep -qFx all "$__object/parameter/extra"; + then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA"; + else tr ',' '\n' < "$__object/parameter/extra"; + fi) do # create a grep BRE pattern to search all packages grep_pattern="$( From 951712740f95f8f1277f1295bcd97c0b9bafdc94 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 12 Feb 2021 13:42:51 +0100 Subject: [PATCH 73/86] __package_pip: update man.rst Adjusted comments for `explorer/extras` and updated the man page for the new behaviour of updating the extras. --- cdist/conf/type/__package_pip/explorer/extras | 7 +++--- cdist/conf/type/__package_pip/man.rst | 25 ++++++++++++------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index be945e36..8104ed47 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -35,15 +35,16 @@ then pip3 freeze > "$pip_freeze" # If all is set, it searches all available extras to separatly check them. - # It would work with just 'all' (cause dependencies are given), but will - # not update if one extra is already present. Side effect is that it will - # not use [all] but instead name all extras seperatly. + # It would work with just 'all' (cause dependencies are specified for + # 'all'), but will not update if one extra is already present. Side effect + # is that it will not use [all] but instead name all extras seperatly. for extra in $(if grep -qFx all "$__object/parameter/extra"; then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA"; else tr ',' '\n' < "$__object/parameter/extra"; fi) do # create a grep BRE pattern to search all packages + # maybe a file full of patterns for -F could be written grep_pattern="$( awk -F'(: | ; )' -v check="$extra" ' $1 == "Requires-Dist" { diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst index 64ad0358..c013695c 100644 --- a/cdist/conf/type/__package_pip/man.rst +++ b/cdist/conf/type/__package_pip/man.rst @@ -24,8 +24,14 @@ name extra Extra optional dependencies which should be installed along the selected - package. Can be specified multiple times. Will only be applied if the - package actually will be installed, but will not explicitly checked. + package. Can be specified multiple times. Multiple extra optional + dependencies can also be specified in a comma-separated list to provide + a more pip-natvie style. + + Extra optional dependencies will be installed even when the base package + is already installed. Notice that the type will not remove installed extras + that are not explicitly named for the type because pip does not offer a + management for orphaned packages and they may be used by other packages. pip Instead of using pip from PATH, use the specific pip path. @@ -53,8 +59,8 @@ EXAMPLES # Install package with optional dependencies __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails - # or do a little cheating - __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails + # the extras can also be specified comma-separated + __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails --extra postgres # or take all extras __package_pip mautrix-telegram --extra all @@ -67,12 +73,13 @@ SEE ALSO AUTHORS ------- -Nico Schottelius +| Nico Schottelius +| Matthias Stecher COPYING ------- -Copyright \(C) 2012 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2012 Nico Schottelius, 2021 Matthias Stecher. You can +redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either +version 3 of the License, or (at your option) any later version. From 2ce1fce7677b64a8d17a86f29f676b1ba8475db2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 15 Feb 2021 16:17:46 +0100 Subject: [PATCH 74/86] __package_pip: match package names case insensitive Pip matches them insensitive, so we need to do the same to avoid problems by saying extras are not installed but already is there in place. --- cdist/conf/type/__package_pip/explorer/extras | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index 8104ed47..bbdc17ab 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -57,7 +57,8 @@ then # echo the extra if no packages where found for it # if there is no pattern, we don't need to search ;-) - if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze" + # pip matches packages case-insensetive, we need to do that, too + if [ "$grep_pattern" ] && ! grep -qi "$grep_pattern" "$pip_freeze" then echo "$extra" fi From d1f45d3524f8a025488d0d2f020433b6efb7edb6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 19 Feb 2021 09:03:56 +0100 Subject: [PATCH 75/86] __package_pip: corrected typo in man .. by fully replacing it with a smaller sentence. --- cdist/conf/type/__package_pip/man.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst index c013695c..5a2bc673 100644 --- a/cdist/conf/type/__package_pip/man.rst +++ b/cdist/conf/type/__package_pip/man.rst @@ -24,9 +24,8 @@ name extra Extra optional dependencies which should be installed along the selected - package. Can be specified multiple times. Multiple extra optional - dependencies can also be specified in a comma-separated list to provide - a more pip-natvie style. + package. Can be specified multiple times. Multiple extras can be passed + in one `--extra` as a comma-separated list. Extra optional dependencies will be installed even when the base package is already installed. Notice that the type will not remove installed extras From 5e0572189fb930764318227fa9fd9f4966b0da99 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 22 Feb 2021 09:11:22 +0100 Subject: [PATCH 76/86] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 52686617..59c58e65 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,8 @@ next: * Core: preos: Fix passing cdist debug parameter (Darko Poljak) * Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera) * Explorer memory: Fix result units; support Solaris (Dennis Camera) + * Type __postgres_role: Implement modification of roles (Dennis Camera) + * Type __letsencrypt_cert: Fix issues with hooks (Evil Ham) 6.9.4: 2020-12-21 * Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) From 22f637c15b6616899c848e2a08c4e1c1b13f0f0b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 23 Feb 2021 06:29:24 +0100 Subject: [PATCH 77/86] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 59c58e65..88dda0aa 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * Explorer memory: Fix result units; support Solaris (Dennis Camera) * Type __postgres_role: Implement modification of roles (Dennis Camera) * Type __letsencrypt_cert: Fix issues with hooks (Evil Ham) + * Type __package_pip: Add optional extra dependencies param (Matthias Stecher) 6.9.4: 2020-12-21 * Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) From 60fd7ba1f38382761c366cbc0425ddaec62f1164 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 28 Feb 2021 13:37:23 +0100 Subject: [PATCH 78/86] Release 6.9.5 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 88dda0aa..95e36b88 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) * Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera) * Explorer memory: Fix result units; support Solaris (Dennis Camera) From 8ef19d47f6b5fb6cdcba7a9a3b8890388dbd7023 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 1 Mar 2021 17:59:25 +0100 Subject: [PATCH 79/86] [type/__pyvenv] Fix example (--user -> --owner) --- cdist/conf/type/__pyvenv/man.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__pyvenv/man.rst b/cdist/conf/type/__pyvenv/man.rst index 8085ff12..e2e4a1e6 100644 --- a/cdist/conf/type/__pyvenv/man.rst +++ b/cdist/conf/type/__pyvenv/man.rst @@ -61,7 +61,7 @@ EXAMPLES __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4 # Create python virtualenv for user foo. - __pyvenv /home/foo/fooenv --group foo --user foo + __pyvenv /home/foo/fooenv --group foo --owner foo # Create python virtualenv with specific parameters. __pyvenv /home/services/djangoenv --venvparams "--copies --system-site-packages" From e7d33891df945831d54d164eb13f2a9ae4f9dd8e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 2 Mar 2021 09:29:33 +0100 Subject: [PATCH 80/86] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 95e36b88..072fcb6b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __pyvenv: Fix user example in man page (Dennis Camera) + 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) * Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera) From ea0126dd8117c051612e6b6abfe895f5f722c584 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 5 Mar 2021 16:11:49 +0100 Subject: [PATCH 81/86] Make local state dir available to custom remote scripts Signed-off-by: Steven Armstrong --- cdist/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cdist/config.py b/cdist/config.py index e84f6f84..19d5bd70 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -420,6 +420,9 @@ class Config: exec_path=sys.argv[0], save_output_streams=args.save_output_streams) + # Make __global state dir available to custom remote scripts. + os.environ['__global'] = local.base_path + remote = cdist.exec.remote.Remote( target_host=target_host, remote_exec=remote_exec, From ecba284fc8ce68486c5f1f87a863409fcad0f106 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 5 Mar 2021 16:13:02 +0100 Subject: [PATCH 82/86] changelog++ Signed-off-by: Steven Armstrong --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 072fcb6b..ad93a595 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Type __pyvenv: Fix user example in man page (Dennis Camera) + * Core: config: Make local state directory available to custom remotes (Steven Armstrong 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) From fb19f342669d18e6decb21ee4cfb2b22e46748d9 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 9 Mar 2021 21:15:26 +0100 Subject: [PATCH 83/86] [type/__ssh_authorized_key] Only grep if file exists --- cdist/conf/type/__ssh_authorized_key/explorer/entry | 1 + cdist/conf/type/__ssh_authorized_key/gencode-remote | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__ssh_authorized_key/explorer/entry b/cdist/conf/type/__ssh_authorized_key/explorer/entry index ccab0afc..aca0f2b9 100755 --- a/cdist/conf/type/__ssh_authorized_key/explorer/entry +++ b/cdist/conf/type/__ssh_authorized_key/explorer/entry @@ -25,6 +25,7 @@ type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ] if [ -n "${type_and_key}" ] then file="$(cat "$__object/parameter/file")" + test -e "$file" || exit 0 # get any entries that match the type and key diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index f37aa565..61c77fb9 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -37,9 +37,9 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX) # preserve ownership and permissions of existing file if [ -f "$file" ]; then cp -p "$file" "\$tmpfile" + grep -v -F -x '$line' '$file' >\$tmpfile fi -grep -v -F -x '$line' '$file' > \$tmpfile || true -mv -f "\$tmpfile" "$file" +cat "\$tmpfile" >"$file" DONE } From 31cc592aa10902c3e83357ee4a0b9300b5e34efa Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 10 Mar 2021 19:25:04 +0100 Subject: [PATCH 84/86] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index ad93a595..c63b901a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __pyvenv: Fix user example in man page (Dennis Camera) * Core: config: Make local state directory available to custom remotes (Steven Armstrong + * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) From e47c4dd8a4cc441b94fe73d362d79525d01a4bb1 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 9 Mar 2021 19:59:05 +0100 Subject: [PATCH 85/86] [type/__sshd_config] Whitelist OpenBMC in manifest --- cdist/conf/type/__sshd_config/manifest | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest index 566bde90..e37afebb 100755 --- a/cdist/conf/type/__sshd_config/manifest +++ b/cdist/conf/type/__sshd_config/manifest @@ -39,7 +39,14 @@ in (freebsd|netbsd|openbsd) # whitelist ;; + (openbmc-phosphor) + # whitelist + # OpenBMC can be configured with dropbear and OpenSSH. + # If dropbear is used, the state explorer will already fail because it + # cannot find the sshd binary. + ;; (*) + : "${__type:?}" # make shellcheck happy printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \ "${os}" "${__type##*/}" >&2 printf 'Please contribute an implementation for it if you can.\n' >&2 From 10ca1c12fd25a9f2b92cebbc5d375a1723ed5e2f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 12 Mar 2021 08:21:03 +0100 Subject: [PATCH 86/86] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index c63b901a..42a74d04 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * Type __pyvenv: Fix user example in man page (Dennis Camera) * Core: config: Make local state directory available to custom remotes (Steven Armstrong * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) + * Type __sshd_config: Whitelist OpenBMC (Dennis Camera) 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak)