From 3e48ef9e11795bf1c8a59ae202ed28f58cff1753 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 11 Nov 2020 14:09:26 +0100 Subject: [PATCH 01/52] [type/__hostname] Lint - Error if expected environment variables are unset - Always wrap variable expansions in {} --- cdist/conf/type/__hostname/gencode-remote | 82 ++++++------ cdist/conf/type/__hostname/manifest | 147 +++++++++++----------- 2 files changed, 114 insertions(+), 115 deletions(-) diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote index 02afcbfb..9447daa9 100755 --- a/cdist/conf/type/__hostname/gencode-remote +++ b/cdist/conf/type/__hostname/gencode-remote @@ -20,26 +20,27 @@ # along with cdist. If not, see . # -os=$(cat "$__global/explorer/os") -name_running=$(cat "$__global/explorer/hostname") -has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl") +os=$(cat "${__global:?}/explorer/os") +name_running=$(cat "${__global:?}/explorer/hostname") +has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl") -if test -s "$__object/parameter/name" +if test -s "${__object:?}/parameter/name" then - name_should=$(cat "$__object/parameter/name") + name_should=$(cat "${__object:?}/parameter/name") else - case $os + case ${os} in # RedHat-derivatives and BSDs - centos|fedora|redhat|scientific|freebsd|macosx|netbsd|openbsd) + (centos|fedora|redhat|scientific|freebsd|macosx|netbsd|openbsd) # Hostname is FQDN - name_should="${__target_host}" - ;; - *) + name_should=${__target_host:?} + ;; + (*) # Hostname is only first component of FQDN - name_should="${__target_host%%.*}" - ;; + name_should=${__target_host:?} + name_should=${name_should%%.*} + ;; esac fi @@ -47,46 +48,46 @@ fi ################################################################################ # Check if the (running) hostname is already correct # -test "$name_running" != "$name_should" || exit 0 +test "${name_running}" != "${name_should}" || exit 0 ################################################################################ # Setup hostname # -echo 'changed' >>"$__messages_out" +echo 'changed' >>"${__messages_out:?}" # Use the good old way to set the hostname. -case $os +case ${os} in - alpine|debian|devuan|ubuntu) + (alpine|debian|devuan|ubuntu) echo 'hostname -F /etc/hostname' - ;; - archlinux) + ;; + (archlinux) echo 'command -v hostnamectl >/dev/null 2>&1' \ - "&& hostnamectl set-hostname '$name_should'" \ - "|| hostname '$name_should'" - ;; - centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void) - echo "hostname '$name_should'" - ;; - openwrt) - echo "echo '$name_should' >/proc/sys/kernel/hostname" - ;; - macosx) - echo "scutil --set HostName '$name_should'" - ;; - solaris) - echo "uname -S '$name_should'" - ;; - slackware|suse|opensuse-leap) + "&& hostnamectl set-hostname '${name_should}'" \ + "|| hostname '${name_should}'" + ;; + (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|gentoo|void) + echo "hostname '${name_should}'" + ;; + (openwrt) + echo "echo '${name_should}' >/proc/sys/kernel/hostname" + ;; + (macosx) + echo "scutil --set HostName '${name_should}'" + ;; + (solaris) + echo "uname -S '${name_should}'" + ;; + (slackware|suse|opensuse-leap) # We do not read from /etc/HOSTNAME, because the running # hostname is the first component only while the file contains # the FQDN. - echo "hostname '$name_should'" - ;; - *) + echo "hostname '${name_should}'" + ;; + (*) # Fall back to set the hostname using hostnamectl, if available. - if test -n "$has_hostnamectl" + if test -n "${has_hostnamectl}" then # Don't use hostnamectl as the primary means to set the hostname for # systemd systems, because it cannot be trusted to work reliably and @@ -97,7 +98,8 @@ in echo "test \"\$(hostname)\" = \"\$(cat /etc/hostname)\"" \ " || hostname -F /etc/hostname" else - printf "echo 'Unsupported OS: %s' >&2\nexit 1\n" "$os" + printf "echo 'Unsupported OS: %s' >&2\n" "${os}" + printf 'exit 1\n' fi - ;; + ;; esac diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest index bf8a331c..125bb2fb 100755 --- a/cdist/conf/type/__hostname/manifest +++ b/cdist/conf/type/__hostname/manifest @@ -20,69 +20,65 @@ # along with cdist. If not, see . # -not_supported() { - echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 - echo "Please contribute an implementation for it if you can." >&2 - exit 1 -} - set_hostname_systemd() { echo "$1" | __file /etc/hostname --source - } -os=$(cat "$__global/explorer/os") -os_version=$(cat "$__global/explorer/os_version") -os_major=$(echo "$os_version" | grep -o '^[0-9][0-9]*' || true) +os=$(cat "${__global:?}/explorer/os") +os_version=$(cat "${__global:?}/explorer/os_version") +os_major=$(echo "${os_version}" | grep -o '^[0-9][0-9]*' || true) -max_len=$(cat "$__object/explorer/max_len") -has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl") +max_len=$(cat "${__object:?}/explorer/max_len") +has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl") -if test -s "$__object/parameter/name" +if test -s "${__object:?}/parameter/name" then - name_should=$(cat "$__object/parameter/name") + name_should=$(cat "${__object:?}/parameter/name") else - case $os + case ${os} in # RedHat-derivatives and BSDs - centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware) + (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware) # Hostname is FQDN - name_should="${__target_host}" - ;; - suse|opensuse-leap) + name_should=${__target_host:?} + ;; + (suse|opensuse-leap) + name_should=${__target_host:?} + # Classic SuSE stores the FQDN in /etc/HOSTNAME, while # systemd does not. The running hostname is the first # component in both cases. # In versions before 15.x, the FQDN is stored in /etc/hostname. - if test -n "$has_hostnamectl" && test "$os_major" -ge 15 \ - && test "$os_major" -ne 42 + if test -n "${has_hostnamectl}" \ + && test "${os_major}" -ge 15 \ + && test "${os_major}" -ne 42 then - name_should="${__target_host%%.*}" - else - name_should="${__target_host}" + name_should=${name_should%%.*} fi - ;; + ;; *) # Hostname is only first component of FQDN on all other systems. - name_should="${__target_host%%.*}" - ;; + name_should=${__target_host:?} + name_should=${name_should%%.*} + ;; esac fi -if test -n "$max_len" && test "$(printf '%s' "$name_should" | wc -c)" -gt "$max_len" +if test -n "${max_len}" && test "$(printf '%s' "${name_should}" | wc -c)" -gt "${max_len}" then printf "Host name too long. Up to %u characters allowed.\n" "${max_len}" >&2 exit 1 fi -case $os +case ${os} in - alpine|debian|devuan|ubuntu|void) - echo "$name_should" | __file /etc/hostname --source - - ;; - archlinux) - if test -n "$has_hostnamectl" + (alpine|debian|devuan|ubuntu|void) + echo "${name_should}" | __file /etc/hostname --source - + ;; + (archlinux) + if test -n "${has_hostnamectl}" then - set_hostname_systemd "$name_should" + set_hostname_systemd "${name_should}" else echo 'Ancient ArchLinux variants without hostnamectl are not supported.' >&2 exit 1 @@ -97,8 +93,8 @@ in # --value "\"$name_should\"" fi ;; - centos|fedora|redhat|scientific) - if test -z "$has_hostnamectl" + (centos|fedora|redhat|scientific) + if test -z "${has_hostnamectl}" then # Only write to /etc/sysconfig/network on non-systemd versions. # On systemd-based versions this entry is ignored. @@ -106,63 +102,62 @@ in --file /etc/sysconfig/network \ --delimiter '=' --exact_delimiter \ --key HOSTNAME \ - --value "\"$name_should\"" + --value "\"${name_should}\"" else - set_hostname_systemd "$name_should" + set_hostname_systemd "${name_should}" fi - ;; - gentoo) + ;; + (gentoo) # Only write to /etc/conf.d/hostname on OpenRC-based installations. # On systemd use hostnamectl(1) in gencode-remote. - if test -z "$has_hostnamectl" + if test -z "${has_hostnamectl}" then __key_value '/etc/conf.d/hostname:hostname' \ --file /etc/conf.d/hostname \ --delimiter '=' --exact_delimiter \ --key 'hostname' \ - --value "\"$name_should\"" + --value "\"${name_should}\"" else set_hostname_systemd "$name_should" fi - ;; - freebsd) + ;; + (freebsd) __key_value '/etc/rc.conf:hostname' \ --file /etc/rc.conf \ --delimiter '=' --exact_delimiter \ --key 'hostname' \ - --value "\"$name_should\"" - ;; - macosx) + --value "\"${name_should}\"" + ;; + (macosx) # handled in gencode-remote - : - ;; - netbsd) + ;; + (netbsd) __key_value '/etc/rc.conf:hostname' \ --file /etc/rc.conf \ --delimiter '=' --exact_delimiter \ --key 'hostname' \ - --value "\"$name_should\"" + --value "\"${name_should}\"" # To avoid confusion, ensure that the hostname is only stored once. __file /etc/myname --state absent - ;; - openbsd) - echo "$name_should" | __file /etc/myname --source - - ;; - openwrt) - __uci system.@system[0].hostname --value "$name_should" + ;; + (openbsd) + echo "${name_should}" | __file /etc/myname --source - + ;; + (openwrt) + __uci system.@system[0].hostname --value "${name_should}" # --transaction hostname - ;; - slackware) + ;; + (slackware) # We write the FQDN into /etc/HOSTNAME. But /etc/rc.d/rc.M will only # read the first component from this file and set it as the running # hostname on boot. - echo "$name_should" | __file /etc/HOSTNAME --source - - ;; - solaris) - echo "$name_should" | __file /etc/nodename --source - - ;; - suse|opensuse-leap) + echo "${name_should}" | __file /etc/HOSTNAME --source - + ;; + (solaris) + echo "${name_should}" | __file /etc/nodename --source - + ;; + (suse|opensuse-leap) # Modern SuSE provides /etc/HOSTNAME as a symlink for # backwards-compatibility. Unfortunately it cannot be used # here as __file does not follow the symlink. @@ -171,23 +166,25 @@ in # not work correctly on openSUSE 12.x which provides # hostnamectl but not /etc/hostname. - if test -n "$has_hostnamectl" -a "$os_major" -gt 12 + if test -n "${has_hostnamectl}" -a "${os_major}" -gt 12 then - hostname_file='/etc/hostname' + hostname_file=/etc/hostname else - hostname_file='/etc/HOSTNAME' + hostname_file=/etc/HOSTNAME fi - echo "$name_should" | __file "$hostname_file" --source - - ;; - *) + echo "${name_should}" | __file "${hostname_file}" --source - + ;; + (*) # On other operating systems we fall back to systemd's # hostnamectl if available… - if test -n "$has_hostnamectl" + if test -n "${has_hostnamectl}" then - set_hostname_systemd "$name_should" + set_hostname_systemd "${name_should}" else - not_supported + echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 fi - ;; + ;; esac From 702f3eba4f8f88b634a010b377d077d501a6f0e9 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 11 Nov 2020 14:15:02 +0100 Subject: [PATCH 02/52] [type/__hostname] Remove opensuse-leap OS string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit everything should be suse now… --- cdist/conf/type/__hostname/gencode-remote | 2 +- cdist/conf/type/__hostname/manifest | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote index 9447daa9..c1a97ac8 100755 --- a/cdist/conf/type/__hostname/gencode-remote +++ b/cdist/conf/type/__hostname/gencode-remote @@ -79,7 +79,7 @@ in (solaris) echo "uname -S '${name_should}'" ;; - (slackware|suse|opensuse-leap) + (slackware|suse) # We do not read from /etc/HOSTNAME, because the running # hostname is the first component only while the file contains # the FQDN. diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest index 125bb2fb..d044072b 100755 --- a/cdist/conf/type/__hostname/manifest +++ b/cdist/conf/type/__hostname/manifest @@ -42,7 +42,7 @@ else # Hostname is FQDN name_should=${__target_host:?} ;; - (suse|opensuse-leap) + (suse) name_should=${__target_host:?} # Classic SuSE stores the FQDN in /etc/HOSTNAME, while @@ -157,7 +157,7 @@ in (solaris) echo "${name_should}" | __file /etc/nodename --source - ;; - (suse|opensuse-leap) + (suse) # Modern SuSE provides /etc/HOSTNAME as a symlink for # backwards-compatibility. Unfortunately it cannot be used # here as __file does not follow the symlink. From 87a0d91587568594a616c7f3588bc74106312761 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 11 Nov 2020 14:20:18 +0100 Subject: [PATCH 03/52] [type/__hostname] Fix OS version detection for SuSE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit everything should be suse now… --- cdist/conf/type/__hostname/manifest | 39 ++++++++++++++++------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest index d044072b..b80aa2ef 100755 --- a/cdist/conf/type/__hostname/manifest +++ b/cdist/conf/type/__hostname/manifest @@ -25,8 +25,6 @@ set_hostname_systemd() { } os=$(cat "${__global:?}/explorer/os") -os_version=$(cat "${__global:?}/explorer/os_version") -os_major=$(echo "${os_version}" | grep -o '^[0-9][0-9]*' || true) max_len=$(cat "${__object:?}/explorer/max_len") has_hostnamectl=$(cat "${__object:?}/explorer/has_hostnamectl") @@ -38,24 +36,10 @@ else case ${os} in # RedHat-derivatives and BSDs - (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware) + (centos|fedora|redhat|scientific|freebsd|netbsd|openbsd|slackware|suse) # Hostname is FQDN name_should=${__target_host:?} ;; - (suse) - name_should=${__target_host:?} - - # Classic SuSE stores the FQDN in /etc/HOSTNAME, while - # systemd does not. The running hostname is the first - # component in both cases. - # In versions before 15.x, the FQDN is stored in /etc/hostname. - if test -n "${has_hostnamectl}" \ - && test "${os_major}" -ge 15 \ - && test "${os_major}" -ne 42 - then - name_should=${name_should%%.*} - fi - ;; *) # Hostname is only first component of FQDN on all other systems. name_should=${__target_host:?} @@ -158,6 +142,27 @@ in echo "${name_should}" | __file /etc/nodename --source - ;; (suse) + if test -s "${__global:?}/explorer/os_release" + then + # shellcheck source=/dev/null + os_version=$(. "${__global:?}/explorer/os_release" && echo "${VERSION}") + else + os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global:?}/explorer/os_version") + fi + os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)') + + # Classic SuSE stores the FQDN in /etc/HOSTNAME, while + # systemd does not. The running hostname is the first + # component in both cases. + # In versions before 15.x, the FQDN is stored in /etc/hostname. + if test -n "${has_hostnamectl}" \ + && test "${os_major}" -ge 15 \ + && test "${os_major}" -ne 42 + then + # strip away everything but the first part from $name_should + name_should=${name_should%%.*} + fi + # Modern SuSE provides /etc/HOSTNAME as a symlink for # backwards-compatibility. Unfortunately it cannot be used # here as __file does not follow the symlink. From f75d4772090d3a9cfd798dd193cd324083a9b3a7 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 8 Nov 2020 13:45:12 +0100 Subject: [PATCH 04/52] Deprecate __locale and replace with __localedef --- cdist/conf/type/__locale/deprecated | 1 + cdist/conf/type/__localedef/gencode-remote | 60 +++++++++++++++++++ cdist/conf/type/__localedef/man.rst | 55 +++++++++++++++++ cdist/conf/type/__localedef/manifest | 41 +++++++++++++ .../type/__localedef/parameter/default/state | 1 + .../conf/type/__localedef/parameter/optional | 1 + 6 files changed, 159 insertions(+) create mode 100644 cdist/conf/type/__locale/deprecated create mode 100755 cdist/conf/type/__localedef/gencode-remote create mode 100644 cdist/conf/type/__localedef/man.rst create mode 100755 cdist/conf/type/__localedef/manifest create mode 100644 cdist/conf/type/__localedef/parameter/default/state create mode 100644 cdist/conf/type/__localedef/parameter/optional diff --git a/cdist/conf/type/__locale/deprecated b/cdist/conf/type/__locale/deprecated new file mode 100644 index 00000000..5a06b28e --- /dev/null +++ b/cdist/conf/type/__locale/deprecated @@ -0,0 +1 @@ +This type is deprecated. Please use __localedef instead. diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote new file mode 100755 index 00000000..1feb9884 --- /dev/null +++ b/cdist/conf/type/__localedef/gencode-remote @@ -0,0 +1,60 @@ +#!/bin/sh -e +# +# 2013-2019 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Let localedef do the magic +# + +locale="$__object_id" + +# Hardcoded, create a pull request with +# branching on $os in case it is at another location +alias=/usr/share/locale/locale.alias + +input=$(echo "$locale" | cut -d . -f 1) +charmap=$(echo "$locale" | cut -d . -f 2) + +# Adding locale? The name is de_CH.UTF-8 +# Removing locale? The name is de_CH.utf8. +# W-T-F! +locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/') + +state=$(cat "$__object/parameter/state") + +os=$(cat "$__global/explorer/os") + +# Nothing to be done on alpine +case "$os" in + alpine) + exit 0 + ;; +esac + +case "$state" in + present) + echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale" + ;; + absent) + echo localedef --delete-from-archive "$locale_remove" + ;; + *) + echo "Unsupported state: $state" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__localedef/man.rst b/cdist/conf/type/__localedef/man.rst new file mode 100644 index 00000000..0ccf484a --- /dev/null +++ b/cdist/conf/type/__localedef/man.rst @@ -0,0 +1,55 @@ +cdist-type__localedef(7) +======================== + +NAME +---- +cdist-type__localedef - Define and remove system locales + + +DESCRIPTION +----------- +This cdist type allows you to define locales on the system using +:strong:`localedef`\ (1) or remove them. +On systems that don't support definition of new locales, the type will raise an +error. + + +OPTIONAL PARAMETERS +------------------- +state + ``present`` or ``absent``. Defaults to ``present``. + + +EXAMPLES +-------- + +.. code-block:: sh + + # Add locale de_CH.UTF-8 + __localedef de_CH.UTF-8 + + # Same as above, but more explicit + __localedef de_CH.UTF-8 --state present + + # Remove colourful British English + __localedef en_GB.UTF-8 --state absent + + +SEE ALSO +-------- +:strong:`locale`\ (1), +:strong:`localedef`\ (1), +:strong:`cdist-type__locale_system`\ (7) + + +AUTHORS +------- +| Dennis Camera +| Nico Schottelius + + +COPYING +------- +Copyright \(C) 2013-2019 Nico Schottelius, 2020 Dennis Camera. Free use of this +software is granted under the terms of the GNU General Public License version 3 +or later (GPLv3+). diff --git a/cdist/conf/type/__localedef/manifest b/cdist/conf/type/__localedef/manifest new file mode 100755 index 00000000..9f1e17ac --- /dev/null +++ b/cdist/conf/type/__localedef/manifest @@ -0,0 +1,41 @@ +#!/bin/sh -e +# +# 2013-2019 Nico Schottelius (nico-cdist at schottelius.org) +# 2015 David Hürlimann (david at ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Install required packages +# + +os=$(cat "$__global/explorer/os") + + +case "$os" in + debian|devuan) + # Debian needs a seperate package + __package locales --state present + ;; + archlinux|suse|ubuntu|scientific|centos|alpine) + : + ;; + *) + echo "Sorry, do not know how to handle os: $os" >&2 + echo "Please edit the type ${__type##*/} to fix this." >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__localedef/parameter/default/state b/cdist/conf/type/__localedef/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__localedef/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__localedef/parameter/optional b/cdist/conf/type/__localedef/parameter/optional new file mode 100644 index 00000000..ff72b5c7 --- /dev/null +++ b/cdist/conf/type/__localedef/parameter/optional @@ -0,0 +1 @@ +state From 54e689f7c2b3bc25aedb2aaf9f5aece289ec5b66 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 8 Nov 2020 13:49:04 +0100 Subject: [PATCH 05/52] [type/__localedef] Add state explorer --- cdist/conf/type/__localedef/explorer/state | 71 ++++++++++++++++++++++ cdist/conf/type/__localedef/gencode-remote | 13 +++- 2 files changed, 82 insertions(+), 2 deletions(-) create mode 100755 cdist/conf/type/__localedef/explorer/state diff --git a/cdist/conf/type/__localedef/explorer/state b/cdist/conf/type/__localedef/explorer/state new file mode 100755 index 00000000..d8db29c5 --- /dev/null +++ b/cdist/conf/type/__localedef/explorer/state @@ -0,0 +1,71 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# This explorer determines if the locale is defined on the target system. +# Will print nothing on error. +# +# Possible output: +# present: +# the main locale (and possibly aliases) is present +# absent: +# neither the main locale nor any aliases are present +# + +command -v locale >/dev/null 2>&1 || exit 0 + +locales=$(locale -a) + +parse_locale() { + # This function will split locales into their parts. Locale strings are + # usually of the form: [language[_territory][.codeset][@modifier]] + # For simplicity, language and territory are not separated by this function. + # Old Linux systems were also using "english" or "german" as locale strings. + # Usage: parse_locale locale_str lang_var codeset_var modifier_var + eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')" + eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')" + eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')" +} + +format_locale() { + # Usage: format_locale language codeset modifier + printf '%s' "$1" + test -z "$2" || printf '.%s' "$2" + test -z "$3" || printf '@%s' "$3" + printf '\n' +} + +gnu_normalize_codeset() { + # reimplementation of glibc/locale/programs/localedef.c normalize_codeset() + echo "$*" | tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]' +} + +locale_available() ( + echo "${locales}" | grep -qxF "$1" || { + # glibc uses "normalized" locale names in archives. + # If a locale is stored in an archive, the normalized name will be + # printed by locale, so that needs to be checked, too. + localename=$( + parse_locale "$1" _lang _codeset _modifier \ + && format_locale "${_lang:?}" "$(gnu_normalize_codeset "${_codeset?}")" \ + "${_modifier?}") + echo "${locales}" | grep -qxF "${localename}" + } +) + +locale_available "${__object_id:?}" && echo present || echo absent diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote index 1feb9884..9ea3f6f1 100755 --- a/cdist/conf/type/__localedef/gencode-remote +++ b/cdist/conf/type/__localedef/gencode-remote @@ -35,7 +35,8 @@ charmap=$(echo "$locale" | cut -d . -f 2) # W-T-F! locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/') -state=$(cat "$__object/parameter/state") +state_is=$(cat "${__object:?}/explorer/state") +state_should=$(cat "${__object:?}/parameter/state") os=$(cat "$__global/explorer/os") @@ -46,7 +47,15 @@ case "$os" in ;; esac -case "$state" in +# NOTE: If state explorer fails (e.g. locale(1) missing), the following check +# will always fail and let definition/removal run. +if test "${state_is}" = "${state_should}" +then + exit 0 +fi + +case ${state_should} +in present) echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale" ;; From cc29e54b851212568c3f6eddf4bf0a64c1eabb0d Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 23 Jul 2020 23:20:39 +0200 Subject: [PATCH 06/52] [type/__localedef] Differentiate between OSes and better handling of normalized locale names --- cdist/conf/type/__localedef/gencode-remote | 117 +++++++++++++++------ 1 file changed, 82 insertions(+), 35 deletions(-) diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote index 9ea3f6f1..b9cc68e8 100755 --- a/cdist/conf/type/__localedef/gencode-remote +++ b/cdist/conf/type/__localedef/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2013-2019 Nico Schottelius (nico-cdist at schottelius.org) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,35 +18,16 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# Manage system locales using localedef(1). # -# Let localedef do the magic -# - -locale="$__object_id" - -# Hardcoded, create a pull request with -# branching on $os in case it is at another location -alias=/usr/share/locale/locale.alias - -input=$(echo "$locale" | cut -d . -f 1) -charmap=$(echo "$locale" | cut -d . -f 2) - -# Adding locale? The name is de_CH.UTF-8 -# Removing locale? The name is de_CH.utf8. -# W-T-F! -locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/') state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") -os=$(cat "$__global/explorer/os") - -# Nothing to be done on alpine -case "$os" in - alpine) - exit 0 - ;; -esac +test "${state_should}" = 'present' -o "${state_should}" = 'absent' || { + printf 'Invalid state: %s\n' "${state_should}" >&2 + exit 1 +} # NOTE: If state explorer fails (e.g. locale(1) missing), the following check # will always fail and let definition/removal run. @@ -54,16 +36,81 @@ then exit 0 fi -case ${state_should} +locale=${__object_id:?} +os=$(cat "${__global:?}/explorer/os") + +if expr "${locale}" : '.*/' >/dev/null +then + printf 'Paths as locales are not supported.\n' >&2 + printf '__object_id is: %s\n' "${locale}" >&2 + exit 1 +fi + +parse_locale() { + # This function will split locales into their parts. Locale strings are + # usually of the form: [language[_territory][.codeset][@modifier]] + # For simplicity, language and territory are not separated by this function. + # Old Linux systems were also using "english" or "german" as locale strings. + # Usage: parse_locale locale_str lang_var codeset_var modifier_var + eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')" + eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')" + eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')" +} + +format_locale() { + # Usage: format_locale language codeset modifier + printf '%s' "$1" + test -z "$2" || printf '.%s' "$2" + test -z "$3" || printf '@%s' "$3" + printf '\n' +} + +gnu_normalize_codeset() { + echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]' +} + + +: "${lang=}" "${codeset=}" "${modifier=}" # declare variables for shellcheck +parse_locale "${locale}" lang codeset modifier + + +case ${os} in - present) - echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale" - ;; - absent) - echo localedef --delete-from-archive "$locale_remove" - ;; - *) - echo "Unsupported state: $state" >&2 - exit 1 - ;; + (alpine|openwrt) + printf '%s does not support locales.\n' "${os}" >&2 + exit 1 + ;; + (archlinux|debian|devuan|ubuntu|suse|centos|fedora|redhat|scientific) + # FIXME: The code below only works for glibc-based installations. + + # NOTE: Hardcoded, create a pull request in case it is at another + # location for some opther distro. + # NOTE: locale.alias can be symlinked (e.g. Debian) + aliasfile='/usr/share/locale/locale.alias' + + case ${state_should} + in + (present) + input=$(format_locale "${lang}" '' "${modifier}") + cat <<-EOF + set -- + if test -e '${aliasfile}' + then + set -- -A '${aliasfile}' + fi + + localedef -i '${input}' -f '${codeset}' "\$@" '${locale}' + EOF + ;; + (absent) + localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}") + printf "localedef --delete-from-archive '%s'\n" "${localename}" + ;; + esac + ;; + (*) + echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 + ;; esac From f44888f192d78d8ce1e3583f70f94b0c2039c1e2 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 8 Nov 2020 14:02:08 +0100 Subject: [PATCH 07/52] [type/__localedef] Only install dependencies in manifest. OS checking moved to gencode-remote --- cdist/conf/type/__localedef/manifest | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/cdist/conf/type/__localedef/manifest b/cdist/conf/type/__localedef/manifest index 9f1e17ac..3ab3ad8c 100755 --- a/cdist/conf/type/__localedef/manifest +++ b/cdist/conf/type/__localedef/manifest @@ -2,6 +2,7 @@ # # 2013-2019 Nico Schottelius (nico-cdist at schottelius.org) # 2015 David Hürlimann (david at ungleich.ch) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,24 +19,12 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# -# Install required packages +# Install required packages. # -os=$(cat "$__global/explorer/os") - - -case "$os" in - debian|devuan) - # Debian needs a seperate package - __package locales --state present - ;; - archlinux|suse|ubuntu|scientific|centos|alpine) - : - ;; - *) - echo "Sorry, do not know how to handle os: $os" >&2 - echo "Please edit the type ${__type##*/} to fix this." >&2 - exit 1 - ;; +case $(cat "${__global:?}/explorer/os") +in + (debian|devuan) + __package_apt locales --state present + ;; esac From dcef2c19f5b87e3a9594f01fefab66cc7488dce7 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 8 Nov 2020 14:07:42 +0100 Subject: [PATCH 08/52] [type/__localedef] Add support for FreeBSD --- cdist/conf/type/__localedef/gencode-remote | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote index b9cc68e8..80e7eb1c 100755 --- a/cdist/conf/type/__localedef/gencode-remote +++ b/cdist/conf/type/__localedef/gencode-remote @@ -108,6 +108,25 @@ in ;; esac ;; + (freebsd) + case ${state_should} + in + (present) + if expr "$(grep -oe '^[0-9]*' "${__global:?}/explorer/os_version")" '>=' 11 >/dev/null + then + # localedef(1) is available with FreeBSD >= 11 + printf "localedef -i '%s' -f '%s' '%s'\n" "${input}" "${codeset}" "${locale}" + else + printf 'localedef(1) was added to FreeBSD starting with version 11.\n' >&2 + printf 'Please upgrade your FreeBSD installation to use %s.\n' "${__type##*/}" >&2 + exit 1 + fi + ;; + (absent) + printf "rm -R '/usr/share/locale/%s'\n" "${locale}" + ;; + esac + ;; (*) echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2 echo "Please contribute an implementation for it if you can." >&2 From c1c60e3374e0cb93ea182c4cbcb813150756c3e4 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 8 Nov 2020 15:24:46 +0100 Subject: [PATCH 09/52] [type/__localedef] Blacklist OpenBSD and NetBSD --- cdist/conf/type/__localedef/gencode-remote | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote index 80e7eb1c..af1a77f7 100755 --- a/cdist/conf/type/__localedef/gencode-remote +++ b/cdist/conf/type/__localedef/gencode-remote @@ -127,6 +127,12 @@ in ;; esac ;; + (netbsd|openbsd) + # NetBSD/OpenBSD are missing localedef(1). + # We also do not delete defined locales because they can't be recreated. + echo "${os} is lacking localedef(1). Locale management unavailable." >&2 + exit 1 + ;; (*) echo "Your operating system (${os}) is currently not supported by this type (${__type##*/})." >&2 echo "Please contribute an implementation for it if you can." >&2 From 575bb62dc507fb8c294319f8db20ba45595e6869 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 13 Nov 2020 18:42:04 +0100 Subject: [PATCH 10/52] [type/__localedef] Externalise functions to separate files --- .../conf/type/__localedef/files/lib/glibc.sh | 5 ++++ .../conf/type/__localedef/files/lib/locale.sh | 20 +++++++++++++ cdist/conf/type/__localedef/gencode-remote | 29 ++++--------------- 3 files changed, 30 insertions(+), 24 deletions(-) create mode 100644 cdist/conf/type/__localedef/files/lib/glibc.sh create mode 100644 cdist/conf/type/__localedef/files/lib/locale.sh diff --git a/cdist/conf/type/__localedef/files/lib/glibc.sh b/cdist/conf/type/__localedef/files/lib/glibc.sh new file mode 100644 index 00000000..6ace80d4 --- /dev/null +++ b/cdist/conf/type/__localedef/files/lib/glibc.sh @@ -0,0 +1,5 @@ +# -*- mode: sh; indent-tabs-mode: t -*- + +gnu_normalize_codeset() { + echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]' +} diff --git a/cdist/conf/type/__localedef/files/lib/locale.sh b/cdist/conf/type/__localedef/files/lib/locale.sh new file mode 100644 index 00000000..b5e61374 --- /dev/null +++ b/cdist/conf/type/__localedef/files/lib/locale.sh @@ -0,0 +1,20 @@ +# -*- mode: sh; indent-tabs-mode:t -*- + +parse_locale() { + # This function will split locales into their parts. Locale strings are + # usually of the form: [language[_territory][.codeset][@modifier]] + # For simplicity, language and territory are not separated by this function. + # Old Linux systems were also using "english" or "german" as locale strings. + # Usage: parse_locale locale_str lang_var codeset_var modifier_var + eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')" + eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')" + eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')" +} + +format_locale() { + # Usage: format_locale language codeset modifier + printf '%s' "$1" + test -z "$2" || printf '.%s' "$2" + test -z "$3" || printf '@%s' "$3" + printf '\n' +} diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote index af1a77f7..17941c63 100755 --- a/cdist/conf/type/__localedef/gencode-remote +++ b/cdist/conf/type/__localedef/gencode-remote @@ -21,6 +21,11 @@ # Manage system locales using localedef(1). # +# shellcheck source=cdist/conf/type/__localedef/files/lib/locale.sh +. "${__type:?}/files/lib/locale.sh" +# shellcheck source=cdist/conf/type/__localedef/files/lib/glibc.sh +. "${__type:?}/files/lib/glibc.sh" + state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") @@ -46,30 +51,6 @@ then exit 1 fi -parse_locale() { - # This function will split locales into their parts. Locale strings are - # usually of the form: [language[_territory][.codeset][@modifier]] - # For simplicity, language and territory are not separated by this function. - # Old Linux systems were also using "english" or "german" as locale strings. - # Usage: parse_locale locale_str lang_var codeset_var modifier_var - eval "${2:?}"="$(expr "$1" : '\([^.@]*\)')" - eval "${3:?}"="$(expr "$1" : '[^.]*\.\([^@]*\)')" - eval "${4:?}"="$(expr "$1" : '.*@\(.*\)$')" -} - -format_locale() { - # Usage: format_locale language codeset modifier - printf '%s' "$1" - test -z "$2" || printf '.%s' "$2" - test -z "$3" || printf '@%s' "$3" - printf '\n' -} - -gnu_normalize_codeset() { - echo "$*" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]' -} - - : "${lang=}" "${codeset=}" "${modifier=}" # declare variables for shellcheck parse_locale "${locale}" lang codeset modifier From eeb98719197a09691af60fa26c0b64b3aaf8960e Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 14 Nov 2020 09:54:01 +0100 Subject: [PATCH 11/52] [type/__localedef] glibc: Also delete aliases when removing a locale --- cdist/conf/type/__localedef/gencode-remote | 15 +++++++++++++-- cdist/conf/type/__localedef/man.rst | 5 +++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote index 17941c63..d7fd6942 100755 --- a/cdist/conf/type/__localedef/gencode-remote +++ b/cdist/conf/type/__localedef/gencode-remote @@ -84,8 +84,19 @@ in EOF ;; (absent) - localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}") - printf "localedef --delete-from-archive '%s'\n" "${localename}" + main_localename=$(format_locale "${lang}" "$(gnu_normalize_codeset "${codeset}")" "${modifier}") + + cat <<-EOF + while read -r _alias _localename + do + if test "\${_localename}" = '$(format_locale "${lang}" "${codeset}")' + then + localedef --delete-from-archive "\${_alias}" + fi + done <'${aliasfile}' + + localedef --delete-from-archive '${main_localename}' + EOF ;; esac ;; diff --git a/cdist/conf/type/__localedef/man.rst b/cdist/conf/type/__localedef/man.rst index 0ccf484a..454ce9d1 100644 --- a/cdist/conf/type/__localedef/man.rst +++ b/cdist/conf/type/__localedef/man.rst @@ -13,6 +13,11 @@ This cdist type allows you to define locales on the system using On systems that don't support definition of new locales, the type will raise an error. +**NB:** This type respects the glibc ``locale.alias`` file, +i.e. it defines alias locales or deletes aliases of a locale when it is removed. +It is not possible, however, to use alias names to define locales or only remove +certain aliases of a locale. + OPTIONAL PARAMETERS ------------------- From 87faffd8750cac94190707c2b37233229c3fbde2 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 14 Nov 2020 10:41:50 +0100 Subject: [PATCH 12/52] [type/__localdef] Also check for aliases in state explorer --- cdist/conf/type/__localedef/explorer/state | 31 +++++++++++++++++++++- cdist/conf/type/__localedef/gencode-remote | 7 +++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__localedef/explorer/state b/cdist/conf/type/__localedef/explorer/state index d8db29c5..3ba57661 100755 --- a/cdist/conf/type/__localedef/explorer/state +++ b/cdist/conf/type/__localedef/explorer/state @@ -25,8 +25,14 @@ # the main locale (and possibly aliases) is present # absent: # neither the main locale nor any aliases are present +# alias-present: +# the main locale is absent, but at least one of its aliases is present # +# Hardcoded, create a pull request in case it is at another location for +# some other distro. (cf. gencode-remote) +aliasfile='/usr/share/locale/locale.alias' + command -v locale >/dev/null 2>&1 || exit 0 locales=$(locale -a) @@ -68,4 +74,27 @@ locale_available() ( } ) -locale_available "${__object_id:?}" && echo present || echo absent +if locale_available "${__object_id:?}" +then + echo present +else + # NOTE: locale.alias can be symlinked. + if test -e "${aliasfile}" + then + # Check if one of the aliases of the locale is defined + baselocale=$( + parse_locale "${__object_id:?}" _lang _codeset _modifiers \ + && format_locale "${_lang}" "${_codeset}") + while read -r _alias _localename + do + if test "${_localename}" = "${baselocale}" \ + && echo "${locales}" | grep -qxF "${_alias}" + then + echo alias-present + exit 0 + fi + done <"${aliasfile}" + fi + + echo absent +fi diff --git a/cdist/conf/type/__localedef/gencode-remote b/cdist/conf/type/__localedef/gencode-remote index d7fd6942..4538151f 100755 --- a/cdist/conf/type/__localedef/gencode-remote +++ b/cdist/conf/type/__localedef/gencode-remote @@ -94,9 +94,12 @@ in localedef --delete-from-archive "\${_alias}" fi done <'${aliasfile}' - - localedef --delete-from-archive '${main_localename}' EOF + + if test "${state_is}" = present + then + printf "localedef --delete-from-archive '%s'\n" "${main_localename}" + fi ;; esac ;; From 9d4f69250ea0e6bb733cde1e42183480f48b5d8f Mon Sep 17 00:00:00 2001 From: ssrq Date: Thu, 19 Nov 2020 19:33:47 +0100 Subject: [PATCH 13/52] __sshd config: New type --- cdist/conf/type/__sshd_config/explorer/state | 121 ++++++++ .../files/update_sshd_config.awk | 293 ++++++++++++++++++ cdist/conf/type/__sshd_config/gencode-remote | 97 ++++++ cdist/conf/type/__sshd_config/man.rst | 94 ++++++ cdist/conf/type/__sshd_config/manifest | 48 +++ .../type/__sshd_config/parameter/default/file | 1 + .../__sshd_config/parameter/default/state | 1 + .../type/__sshd_config/parameter/optional | 4 + .../__sshd_config/parameter/optional_multiple | 1 + 9 files changed, 660 insertions(+) create mode 100644 cdist/conf/type/__sshd_config/explorer/state create mode 100644 cdist/conf/type/__sshd_config/files/update_sshd_config.awk create mode 100755 cdist/conf/type/__sshd_config/gencode-remote create mode 100644 cdist/conf/type/__sshd_config/man.rst create mode 100755 cdist/conf/type/__sshd_config/manifest create mode 100644 cdist/conf/type/__sshd_config/parameter/default/file create mode 100644 cdist/conf/type/__sshd_config/parameter/default/state create mode 100644 cdist/conf/type/__sshd_config/parameter/optional create mode 100644 cdist/conf/type/__sshd_config/parameter/optional_multiple diff --git a/cdist/conf/type/__sshd_config/explorer/state b/cdist/conf/type/__sshd_config/explorer/state new file mode 100644 index 00000000..75c68b8a --- /dev/null +++ b/cdist/conf/type/__sshd_config/explorer/state @@ -0,0 +1,121 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Determines the current state of the config option. +# Possible output: +# - present: "should" option present in config file +# - default: the "should" option is the default -> don’t know if present +# - absent: no such option present in config file +# + +joinlines() { sed -n -e H -e "\${x;s/^\\n//;s/\\n/${1:?}/g;p;}"; } +trlower() { tr '[:upper:]' '[:lower:]'; } +tolower() { printf '%s' "$*" | trlower; } + +default_value() { + sshd -T -f /dev/null -C "$(make_conn_spec)" \ + | sed -n -e 's/^'"$(tolower "${1:?}")"'[[:blank:]]\{1,\}//p' +} + +make_conn_spec() { + if test -s "${__object:?}/parameter/match" + then + _match_file="${__object:?}/parameter/match" + else + _match_file='/dev/null' + fi + + for _kw in \ + addr=Address \ + user=User \ + host=Host \ + laddr=LocalAddress \ + lport=LocalPort \ + rdomain=RDomain + do + _specname=${_kw%%=*} + _confname=$(tolower "${_kw#*=}") + while read -r _k _v + do + if test "$(tolower "${_k}")" = "${_confname}" + then + printf '%s=%s\n' "${_specname}" "${_v}" + continue 2 + fi + done <"${_match_file}" + + # NOTE: Print test spec even for empty keys to suppress errors like: + # 'Match User' in configuration but 'user' not in connection test specification. + # except lport: + # Invalid port '' in test mode specification lport= + test "${_specname}" = 'lport' || printf '%s=\n' "${_specname}" + done \ + | joinlines ',' + unset _match_file +} + +sshd_config_file=$(cat "${__object:?}/parameter/file") +state_should=$(cat "${__object:?}/parameter/state") + +if test -s "${__object:?}/parameter/option" +then + option_name=$(cat "${__object:?}/parameter/option") +else + option_name=${__object_id:?} +fi + +value_should=$(cat "${__object:?}/parameter/value" 2>/dev/null) \ +|| test "${state_should}" = absent || exit 0 # param optional if --state absent + +command -v sshd >/dev/null 2>&1 || { + echo 'Cannot find sshd.' >&2 + exit 1 +} + +test -e "${sshd_config_file}" || { + echo 'absent' + exit 0 +} + +value_is=$( + sshd -T -f "${sshd_config_file}" -C "$(make_conn_spec)" \ + | sed -n -e 's/^'"$(tolower "${option_name}")"'[[:blank:]]\{1,\}//p') + +if printf '%s\n' "${value_is}" | { + if test -n "${value_should}" + then + grep -q -x -F "${value_should}" + else + # if no value provided, assume "any" value + grep -q -e . + fi + } +then + if default_value "${option_name}" | grep -q -x -F "${value_is}" + then + # Might produce false positives for default values. + # TODO: Manual checking should be done, but for simplicity, this case is + # currently ignored here. + echo default + else + echo present + fi +else + echo absent +fi diff --git a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk new file mode 100644 index 00000000..d0bc2b4b --- /dev/null +++ b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk @@ -0,0 +1,293 @@ +# -*- mode: awk; indent-tabs-mode: t -*- + +function usage() { + print_err("Usage: awk -f update_sshd_config.awk -- -o set|unset [-m 'User git'] -l 'X11Forwarding no' /etc/ssh/sshd_config") +} + +function print_err(s) { print s | "cat >&2" } + +function alength(a, i) { + for (i = 0; (i + 1) in a; ++i); + return i +} + +function join(sep, a, i, s) { + for (i = i ? i : 1; i in a; i++) + s = s sep a[i] + return substr(s, 2) +} + +function getopt(opts, argv, target, files, i, c, lv, idx, nf) { + # trivial getopt(3) implementation; only basic functionality + if (argv[1] == "--") i++ + for (i += 1; i in argv; i++) { + if (lv) { target[c] = argv[i]; lv = 0; continue } + if (argv[i] ~ /^-/) { + c = substr(argv[i], 2, 1) + idx = index(opts, c) + if (!idx) { + print_err(sprintf("invalid option -%c\n", c)) + continue + } + if (substr(opts, idx + 1, 1) == ":") { + # option takes argument + if (length(argv[i]) > 2) + target[c] = substr(argv[i], 3) + else + lv = 1 + } else { + target[c] = 1 + } + } else + files[++nf] = argv[i] + } +} + +# tokenise configuration line +# this function mimics the counterpart in OpenSSH (misc.c) +# but it returns two (next token SUBSEP rest) because I didn’t want to have to +# simulate any pointer magic. +function strdelim_internal(s, split_equals, old) { + if (!s) + return "" + + old = s + + if (!match(s, WHITESPACE "|" QUOTE "" (split_equals ? "|" EQUALS : ""))) + return s + + s = substr(s, RSTART) + old = substr(old, 1, RSTART - 1) + + if (s ~ "^" QUOTE) { + old = substr(old, 2) + + # Find matching quote + if (match(s, QUOTE)) { + old = substr(old, 1, RSTART) + # s = substr() + if (match(s, "^" WHITESPACE "*")) + s = substr(s, RLENGTH) + return old + } else { + # no matching quote + return "" + } + } + + if (match(s, "^" WHITESPACE "+")) { + sub("^" WHITESPACE "+", "", s) + if (split_equals) + sub(EQUALS WHITESPACE "*", "", s) + } else if (s ~ "^" EQUALS) { + s = substr(s, 2) + } + + return old SUBSEP s +} +function strdelim(s) { return strdelim_internal(s, 1) } +function strdelimw(s) { return strdelim_internal(s, 0) } + +function singleton_option(opt) { + return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|authenticationmethods|authorizedkeysfile|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/ +} + +function print_update() { + if (mode) { + if (match_only) printf "\t" + printf "%s\n", line_should + updated = 1 + } +} + +BEGIN { + FS = "\n" # disable field splitting + + WHITESPACE = "[ \t]" # servconf.c, misc.c:strdelim_internal (without line breaks, cf. bugs) + QUOTE = "[\"]" # misc.c:strdelim_internal + EQUALS = "[=]" + + split("", opts) + split("", files) + getopt("ho:l:m:", ARGV, opts, files) + + if (opts["h"]) { usage(); exit (e="0") } + + line_should = opts["l"] + match_only = opts["m"] + num_files = alength(files) + + if (num_files != 1 || !opts["o"] || !line_should) { + usage() + exit (e=126) + } + + if (opts["o"] == "set") { + mode = 1 + } else if (opts["o"] == "unset") { + mode = 0 + } else { + print_err(sprintf("invalid mode %s\n", mode)) + exit (e=1) + } + + if (mode) { + # loop over sshd_config twice! + ARGV[2] = ARGV[1] = files[1] + ARGC = 3 + } else { + # only loop once + ARGV[1] = files[1] + ARGC = 2 + } + + split(strdelim(line_should), should, SUBSEP) + option_should = tolower(should[1]) + value_should = should[2] +} + +{ + line = $0 + + # Strip trailing whitespace. Allow \f (form feed) at EOL only + sub("(" WHITESPACE "|\f)*$", "", line) + + # Strip leading whitespace + sub("^" WHITESPACE "*", "", line) + + if (match(line, "^#" WHITESPACE "*")) { + prefix = substr(line, RSTART, RLENGTH) + line = substr(line, RSTART + RLENGTH) + } else { + prefix = "" + } + + line_type = "invalid" + option_is = value_is = "" + + if (line) { + split(strdelim(line), toks, SUBSEP) + + if (tolower(toks[1]) == "match") { + MATCH = (prefix ~ /^#/ ? "#" : "") join(" ", toks, 2) + line_type = "match" + } else if (toks[1] ~ /^[A-Za-z][A-Za-z0-9]+$/) { + # This could be an option line + line_type = "option" + option_is = tolower(toks[1]) + value_is = toks[2] + } + } else { + line_type = "empty" + } +} + +# mode: unset + +!mode { + # delete matching config + if (prefix !~ /^#/) + if (MATCH == match_only && option_is == option_should) + if (!value_should || value_should == value_is) + next + + print + next +} + + +# mode: set + +mode && NR == FNR { + if (line_type == "option") { + if (MATCH !~ /^#/) { + if (prefix ~ /^#/) { + # comment line + last_occ[MATCH, "#" option_is] = FNR + } else { + # option line + last_occ[MATCH, option_is] = FNR + } + last_occ[MATCH] = FNR + } + } else if (line_type == "invalid" && !prefix) { + # INVALID LINE + print_err(sprintf("%s: syntax error on line %u\n", ARGV[0], FNR)) + } + + next +} + +# before second pass prepare hashes containing location information to be used +# in the second pass. +mode && NR > FNR && FNR == 1 { + # First we drop the locations of commented-out options if a non-commented + # option is available. If a non-commented option is available, we will + # append new config options there to have them all at one place. + for (k in last_occ) { + if (k ~ /^#/) { + # delete entries of commented out match blocks + delete last_occ[k] + continue + } + + split(k, parts, SUBSEP) + + if (parts[2] ~ /^#/ && ((parts[1], substr(parts[2], 2)) in last_occ)) + delete last_occ[k] + } + + # Reverse the option => line mapping. The line_map allows for easier lookups + # in the second pass. + # We only keep options, not top-level keywords, because we can only have + # one entry per line and there are conflicts with last lines of "sections". + for (k in last_occ) { + if (!index(k, SUBSEP)) continue + line_map[last_occ[k]] = k + } +} + +# Second pass +mode && line_map[FNR] == match_only SUBSEP option_should && !updated { + split(line_map[FNR], parts, SUBSEP) + + # If option allows multiple values, print current value + if (!singleton_option(parts[2])) { + if (value_should != value_is) + print + } + + print_update() + + next +} + +mode { print } + +# Is a comment option +mode && line_map[FNR] == match_only SUBSEP "#" option_should && !updated { + print_update() +} + +# Last line of the should match section +mode && last_occ[match_only] == FNR && !updated { + # NOTE: Inserting empty lines is only cosmetic. It is only done if + # different options are next to each other and not in a match block + # (match blocks are usually not in the default config and thus don’t + # contain commented blocks.) + if (line && option_is != option_should && !MATCH) + print "" + print_update() +} + +END { + if (e) exit e + + if (mode && !updated) { + if (match_only && MATCH != match_only) { + printf "\nMatch %s\n", match_only + } + + print_update() + } +} diff --git a/cdist/conf/type/__sshd_config/gencode-remote b/cdist/conf/type/__sshd_config/gencode-remote new file mode 100755 index 00000000..0b44dfa7 --- /dev/null +++ b/cdist/conf/type/__sshd_config/gencode-remote @@ -0,0 +1,97 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +joinlines() { sed -n -e H -e "\${x;s/^\\n//;s/\\n/${1:?}/g;p;}"; } + +state_is=$(cat "${__object:?}/explorer/state") +state_should=$(cat "${__object:?}/parameter/state") + +if test "${state_is}" = "${state_should}" -o "${state_is}" = 'default' +then + # nothing to do (if the value is the default, ignore its state) + exit 0 +fi + +case ${state_should} +in + (present) + mode='set' + ;; + (absent) + mode='unset' + ;; + (*) + printf 'Invalid --state: %s\n' "${state_should}" >&2 + exit 1 + ;; +esac + +sshd_config_file=$(cat "${__object:?}/parameter/file") + +quote() { printf "'%s'" "$(printf '%s' "$*" | sed -e "s/'/'\\\\''/g")"; } +drop_awk_comments() { quote "$(sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@")"; } + +# Ensure the sshd_config file is there +cat <$(quote "${sshd_config_file}") + chown 0:0 $(quote "${sshd_config_file}") + chmod 0644 $(quote "${sshd_config_file}") +} + +EOF + +match_only= +if test -s "${__object:?}/parameter/match" +then + match_only=$(joinlines ' ' <"${__object:?}/parameter/match") +fi + +if test -s "${__object:?}/parameter/option" +then + option_line=$(cat "${__object:?}/parameter/option") +else + option_line=${__object_id:?} +fi + +if test -s "${__object:?}/parameter/value" +then + option_line="${option_line} $(cat "${__object:?}/parameter/value")" +fi + +# Send message on config update +printf '%s%s %s\n' "${mode}" "${match_only:+ [${match_only}]}" \ + "${option_line}" >>"${__messages_out:?}" + +# Update sshd_config (remote code) +cat <$(quote "${sshd_config_file}.tmp") \\ +|| exit + +cmp -s $(quote "${sshd_config_file}") $(quote "${sshd_config_file}.tmp") || { + sshd -t -f $(quote "${sshd_config_file}.tmp") \\ + && cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}") +} +rm -f $(quote "${sshd_config_file}.tmp") +EOF diff --git a/cdist/conf/type/__sshd_config/man.rst b/cdist/conf/type/__sshd_config/man.rst new file mode 100644 index 00000000..8b0069ac --- /dev/null +++ b/cdist/conf/type/__sshd_config/man.rst @@ -0,0 +1,94 @@ +cdist-type__sshd_config(7) +========================== + +NAME +---- +cdist-type__sshd_config - Manage options in sshd_config + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +file + The path to the sshd_config file to edit. + Defaults to ``/etc/ssh/sshd_config``. +match + Restrict this option to apply only for certain connections. + Allowed values are what would be allowed to be written after a ``Match`` + keyword in ``sshd_config``, e.g. ``--match 'User anoncvs'``. + + Can be used multiple times. All of the values are ANDed together. +option + The name of the option to manipulate. Defaults to ``__object_id``. +state + Can be: + + - ``present``: ensure a matching config line is present (or the default + value). + - ``absent``: ensure no matching config line is present. +value + The option's value to be assigned to the option (if ``--state present``) or + removed (if ``--state absent``). + + This option is required if ``--state present``. If not specified and + ``--state absent``, all values for the given option are removed. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # Disallow root logins with password + __sshd_config PermitRootLogin --value without-password + + # Disallow password-based authentication + __sshd_config PasswordAuthentication --value no + + # Accept the EDITOR environment variable + __sshd_config AcceptEnv:EDITOR --option AcceptEnv --value EDITOR + + # Force command for connections as git user + __sshd_config git@ForceCommand --match 'User git' --option ForceCommand \ + --value 'cd ~git && exec git-shell ${SSH_ORIGINAL_COMMAND:+-c "${SSH_ORIGINAL_COMMAND}"}' + + +SEE ALSO +-------- +:strong:`sshd_config`\ (5) + + +BUGS +---- +- This type assumes a nicely formatted config file, + i.e. no config options spanning multiple lines. +- ``Include`` directives are ignored. +- Config options are not added/removed to/from the config file if their value is + the default value. + + +AUTHORS +------- +Dennis Camera + + +COPYING +------- +Copyright \(C) 2020 Dennis Camera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest new file mode 100755 index 00000000..566bde90 --- /dev/null +++ b/cdist/conf/type/__sshd_config/manifest @@ -0,0 +1,48 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$(cat "${__global:?}/explorer/os") + +state_should=$(cat "${__object:?}/parameter/state") + +case ${os} +in + (alpine|centos|fedora|redhat|scientific|debian|devuan|ubuntu) + if test "${state_should}" != 'absent' + then + __package openssh-server --state present + fi + ;; + (archlinux|gentoo|slackware|suse) + if test "${state_should}" != 'absent' + then + __package openssh --state present + fi + ;; + (freebsd|netbsd|openbsd) + # whitelist + ;; + (*) + printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \ + "${os}" "${__type##*/}" >&2 + printf 'Please contribute an implementation for it if you can.\n' >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__sshd_config/parameter/default/file b/cdist/conf/type/__sshd_config/parameter/default/file new file mode 100644 index 00000000..d8ea5dfc --- /dev/null +++ b/cdist/conf/type/__sshd_config/parameter/default/file @@ -0,0 +1 @@ +/etc/ssh/sshd_config diff --git a/cdist/conf/type/__sshd_config/parameter/default/state b/cdist/conf/type/__sshd_config/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__sshd_config/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__sshd_config/parameter/optional b/cdist/conf/type/__sshd_config/parameter/optional new file mode 100644 index 00000000..922ab093 --- /dev/null +++ b/cdist/conf/type/__sshd_config/parameter/optional @@ -0,0 +1,4 @@ +file +option +state +value diff --git a/cdist/conf/type/__sshd_config/parameter/optional_multiple b/cdist/conf/type/__sshd_config/parameter/optional_multiple new file mode 100644 index 00000000..02b1d1a9 --- /dev/null +++ b/cdist/conf/type/__sshd_config/parameter/optional_multiple @@ -0,0 +1 @@ +match From 82eadb69948e631041263938994be5aeae161ab0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 19 Nov 2020 19:34:43 +0100 Subject: [PATCH 14/52] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index ff411a46..331acbeb 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,8 @@ next: * Type __locale: Add state explorer (Matthias Stecher) * Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera) * New type: __hwclock (Dennis Camera) + * Type __hostname: Fix guessing SuSE OS version (Dennis Camera) + * New type: __sshd_config (Dennis Camera) 6.9.1: 2020-11-08 * Type __file: Fix state pre-exists (Dennis Camera) From 803a9d62a7a6b7fd11844f6ea273f4b692c2334e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 20 Nov 2020 19:46:03 +0100 Subject: [PATCH 15/52] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 331acbeb..cd5dbb4f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,8 @@ next: * New type: __hwclock (Dennis Camera) * Type __hostname: Fix guessing SuSE OS version (Dennis Camera) * New type: __sshd_config (Dennis Camera) + * New type: __localedef (Dennis Camera) + * Type __locale: Deprecate in favor of __localedef (Dennis Camera) 6.9.1: 2020-11-08 * Type __file: Fix state pre-exists (Dennis Camera) From 23e0da521c785715357b27d66007bd03fe173bf3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 20 Nov 2020 19:46:55 +0100 Subject: [PATCH 16/52] Release 6.9.2 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index cd5dbb4f..9a91bf57 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.2: 2020-11-20 * Documentation: Fix examples in best practice (Dennis Camera) * Type __locale: Add state explorer (Matthias Stecher) * Core: Reorganize scripts, version generation (Ander Punnar, Dennis Camera) From 84172550df9e1800bc795dc51da01c9d3a0be9e0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 30 Nov 2020 20:29:51 +0100 Subject: [PATCH 17/52] __iptables*: add IPv6 support Because it currently only support IPv4. To implement this, it falls back to IPv4 for backward compatibilty, but now supports rules for IPv6 and both protocols at the same time. --- .../type/__iptables_apply/files/init-script | 84 +++++++++++++++---- cdist/conf/type/__iptables_rule/man.rst | 38 ++++++++- cdist/conf/type/__iptables_rule/manifest | 40 ++++++++- .../type/__iptables_rule/parameter/boolean | 3 + 4 files changed, 140 insertions(+), 25 deletions(-) create mode 100644 cdist/conf/type/__iptables_rule/parameter/boolean diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script index d9c79ef7..196f019b 100644 --- a/cdist/conf/type/__iptables_apply/files/init-script +++ b/cdist/conf/type/__iptables_apply/files/init-script @@ -1,6 +1,27 @@ #!/bin/sh -# Nico Schottelius -# Zürisee, Mon Sep 2 18:38:27 CEST 2013 +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2020 Matthias Stecher (matthiasstecher at gmx.de) +# +# This file is distributed with cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Originally written by: +# Nico Schottelius +# Zürisee, Mon Sep 2 18:38:27 CEST 2013 # ### BEGIN INIT INFO # Provides: iptables @@ -15,33 +36,60 @@ ### END INIT INFO +# Read files and execute the content with the given commands +# +# Arguments: +# 1: Directory +# 2..n: Commands which should be used to execute the file content +gothrough() { + cd "$1" || return + shift + + # iterate through all rules and continue if it's not a file + for rule in *; do + [ -f "$rule" ] || continue + echo "Appling iptables rule $rule ..." + + # execute it with all commands specificed + ruleparam="$(cat "$rule")" + for cmd in "$@"; do + # Command and Rule should be split. + # shellcheck disable=SC2046 + command $cmd $ruleparam + done + done +} + +# Shortcut for iptables command to do IPv4 and v6 +iptables() { + command iptables "$@" + command ip6tables "$@" +} + basedir=/etc/iptables.d -status="${basedir}/.pre-start" +status4="${basedir}/.pre-start" +status6="${basedir}/.pre-start6" case $1 in start) # Save status - iptables-save > "$status" + iptables-save > "$status4" + ip6tables-save > "$status6" # Apply our ruleset - cd "$basedir" || exit - count="$(find . ! -name . -prune | wc -l)" - - # Only do something if there are rules - if [ "$count" -ge 1 ]; then - for rule in *; do - echo "Applying iptables rule $rule ..." - # Rule should be split. - # shellcheck disable=SC2046 - iptables $(cat "$rule") - done - fi + gothrough "$basedir" iptables + #gothrough "$basedir/v4" iptables # conflicts with $basedir + gothrough "$basedir/v6" ip6tables + gothrough "$basedir/all" iptables ip6tables ;; stop) # Restore from status before, if there is something to restore - if [ -f "$status" ]; then - iptables-restore < "$status" + if [ -f "$status4" ]; then + iptables-restore < "$status4" + fi + if [ -f "$status6" ]; then + ip6tables-restore < "$status6" fi ;; restart) diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst index 92d8859f..75d0740b 100644 --- a/cdist/conf/type/__iptables_rule/man.rst +++ b/cdist/conf/type/__iptables_rule/man.rst @@ -25,6 +25,24 @@ state 'present' or 'absent', defaults to 'present' +BOOLEAN PARAMETERS +------------------ +All rules without any of this parameter will be threaten like ``--v4`` because +of backward compatibility. + +v4 + Explicitly set it as rule for IPv4. If IPv6 is set, too, it will be + threaten like ``--all``. Will be the default if nothing else is set. + +v6 + Explicitly set it as rule for IPv6. If IPv4 is set, too, it will be + threaten like ``--all``. + +all + Set the rule for both IPv4 and IPv6. It will be saved separately from the + other rules. + + EXAMPLES -------- @@ -48,6 +66,16 @@ EXAMPLES --state absent + # IPv4-only rule for ICMPv4 + __iptables_rule icmp-v4 --v4 --rule "-A INPUT -p icmp -j ACCEPT" + # IPv6-only rule for ICMPv6 + __iptables_rule icmp-v6 --v6 --rule "-A INPUT -p icmpv6 -j ACCEPT" + + # doing something for the dual stack + __iptables_rule fwd-eth0-eth1 --v4 --v6 --rule "-A INPUT -i eth0 -o eth1 -j ACCEPT" + __iptables_rule fwd-eth1-eth0 --all --rule "-A -o eth1 -i eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT" + + SEE ALSO -------- :strong:`cdist-type__iptables_apply`\ (7), :strong:`iptables`\ (8) @@ -56,11 +84,13 @@ SEE ALSO AUTHORS ------- Nico Schottelius +Matthias Stecher COPYING ------- -Copyright \(C) 2013 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2013 Nico Schottelius. +Copyright \(C) 2020 Matthias Stecher. +You can redistribute it and/or modify it under the terms of the GNU +General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest index ed78787f..27d00024 100755 --- a/cdist/conf/type/__iptables_rule/manifest +++ b/cdist/conf/type/__iptables_rule/manifest @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2020 Matthias Stecher (matthiasstecher at gmx.de) # # This file is part of cdist. # @@ -24,12 +25,36 @@ base_dir=/etc/iptables.d name="$__object_id" state="$(cat "$__object/parameter/state")" +if [ -f "$__object/parameter/v4" ]; then + only_v4="yes" + # $specific_dir is $base_dir +fi +if [ -f "$__object/parameter/v6" ]; then + only_v6="yes" + specific_dir="$base_dir/v6" +fi +# If rules should be set for both protocols +if ([ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]) \ + || [ -f "$__object/parameter/all" ]; then + + # all to a specific directory + specific_dir="$base_dir/all" +fi + +# set rule directory based on if it's the base or subdirectory +rule_dir="${specific_dir:-$base_dir}" + ################################################################################ # Basic setup # __directory "$base_dir" --state present +# sub-directory if required +if [ "$specific_dir" ]; then + require="__directory/$base_dir" __directory "$specific_dir" --state present +fi + # Have apply do the real job require="$__object_name" __iptables_apply @@ -37,6 +62,15 @@ require="$__object_name" __iptables_apply # The rule # -require="__directory/$base_dir" __file "$base_dir/${name}" \ - --source "$__object/parameter/rule" \ - --state "$state" +for dir in "$base_dir" "$base_dir/v6" "$base_dir/all"; do + # defaults to absent except the directory that should contain the file + if [ "$rule_dir" = "$dir" ]; then + curr_state="$state" + else + curr_state="absent" + fi + + require="__directory/$rule_dir" __file "$dir/$name" \ + --source "$__object/parameter/rule" \ + --state "$curr_state" +done diff --git a/cdist/conf/type/__iptables_rule/parameter/boolean b/cdist/conf/type/__iptables_rule/parameter/boolean new file mode 100644 index 00000000..76882272 --- /dev/null +++ b/cdist/conf/type/__iptables_rule/parameter/boolean @@ -0,0 +1,3 @@ +all +v4 +v6 From f568462e4981c8b6437c29f96963c3d8e7bed742 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 2 Dec 2020 17:48:41 +0100 Subject: [PATCH 18/52] __iptables_rule: fix shellcheck SC2235 --- cdist/conf/type/__iptables_rule/manifest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest index 27d00024..d4394c25 100755 --- a/cdist/conf/type/__iptables_rule/manifest +++ b/cdist/conf/type/__iptables_rule/manifest @@ -34,8 +34,8 @@ if [ -f "$__object/parameter/v6" ]; then specific_dir="$base_dir/v6" fi # If rules should be set for both protocols -if ([ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]) \ - || [ -f "$__object/parameter/all" ]; then +if { [ "$only_v4" = "yes" ] && [ "$only_v6" = "yes" ]; } || + [ -f "$__object/parameter/all" ]; then # all to a specific directory specific_dir="$base_dir/all" From bee255c1ae0039e3d532317007f06fcd2e64e3bc Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 2 Dec 2020 18:04:50 +0100 Subject: [PATCH 19/52] __iptables_apply: man updates --- .../type/__iptables_apply/files/init-script | 1 + cdist/conf/type/__iptables_apply/man.rst | 21 +++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script index 196f019b..7faa2f92 100644 --- a/cdist/conf/type/__iptables_apply/files/init-script +++ b/cdist/conf/type/__iptables_apply/files/init-script @@ -61,6 +61,7 @@ gothrough() { } # Shortcut for iptables command to do IPv4 and v6 +# only applies to the "reset" target iptables() { command iptables "$@" command ip6tables "$@" diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst index 76e1f6bf..db0e7869 100644 --- a/cdist/conf/type/__iptables_apply/man.rst +++ b/cdist/conf/type/__iptables_apply/man.rst @@ -10,7 +10,14 @@ DESCRIPTION ----------- This cdist type deploys an init script that triggers the configured rules and also re-applies them on -configuration. +configuration. Rules are written from __iptables_rule +into the folder ``/etc/iptables.d/``. + +It reads all rules from the base folder as rules for IPv4. +Rules in the subfolder ``v6/`` are IPv6 rules. Rules in +the subfolder ``all/`` are applied to both rule tables. All +files contain the arguments for a single ``iptables`` and/or +``ip6tables`` command. REQUIRED PARAMETERS @@ -24,7 +31,7 @@ None EXAMPLES -------- -None (__iptables_apply is used by __iptables_rule) +None (__iptables_apply is used by __iptables_rule automaticly) SEE ALSO @@ -35,11 +42,13 @@ SEE ALSO AUTHORS ------- Nico Schottelius +Matthias Stecher COPYING ------- -Copyright \(C) 2013 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2013 Nico Schottelius. +Copyright \(C) 2020 Matthias Stecher. +You can redistribute it and/or modify it under the terms of the GNU +General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later version. From a1db5c3d0e7b5899f1a877c5bab28ac4d4796f8c Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 2 Dec 2020 18:22:31 +0100 Subject: [PATCH 20/52] __iptables*: Update manpages for execution order To make some thinks clear if someone needs it .. --- cdist/conf/type/__iptables_apply/man.rst | 10 ++++++++++ cdist/conf/type/__iptables_rule/man.rst | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst index db0e7869..4109e789 100644 --- a/cdist/conf/type/__iptables_apply/man.rst +++ b/cdist/conf/type/__iptables_apply/man.rst @@ -19,6 +19,16 @@ the subfolder ``all/`` are applied to both rule tables. All files contain the arguments for a single ``iptables`` and/or ``ip6tables`` command. +Rules are applied in the following order: +1. All IPv4 rules +2. All IPv6 rules +2. All rules that should be applied to both tables + +The order of the rules that will be applied are definite +from the result the shell glob returns, which should be +alphabetical. If rules must be applied in a special order, +prefix them with a number like ``02-some-rule``. + REQUIRED PARAMETERS ------------------- diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst index 75d0740b..86d38a34 100644 --- a/cdist/conf/type/__iptables_rule/man.rst +++ b/cdist/conf/type/__iptables_rule/man.rst @@ -11,6 +11,10 @@ DESCRIPTION This cdist type allows you to manage iptable rules in a distribution independent manner. +See :strong:`cdist-type__iptables_apply`\ (7) for the +execution order of these rules. It will be executed +automaticly to apply all rules non-volaite. + REQUIRED PARAMETERS ------------------- From 1055e92545e74cf7845ede8094e783b4103efbcc Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 2 Dec 2020 19:54:41 +0100 Subject: [PATCH 21/52] [setup.py] Add cdist.scan to packages --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 858c2c17..bfc8b495 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ os.chdir(cur) setup( name="cdist", - packages=["cdist", "cdist.core", "cdist.exec", "cdist.util", ], + packages=["cdist", "cdist.core", "cdist.exec", "cdist.scan", "cdist.util"], package_data={'cdist': package_data}, scripts=["bin/cdist", "bin/cdist-dump", "bin/cdist-new-type"], version=cdist.version.VERSION, From c7fa2efe6b780d63a779eed310bfa4ebf926d27e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 4 Dec 2020 15:30:08 +0100 Subject: [PATCH 22/52] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 9a91bf57..ad0aa75d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * pip install: Add cdist.scan to packages in setup.py (Dennis Camera) + 6.9.2: 2020-11-20 * Documentation: Fix examples in best practice (Dennis Camera) * Type __locale: Add state explorer (Matthias Stecher) From d44b5cfdc990ce020ffe3af5085bbb93b8ce64a8 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 4 Dec 2020 15:31:35 +0100 Subject: [PATCH 23/52] Release 6.9.3 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index ad0aa75d..6c272fd7 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.3: 2020-12-04 * pip install: Add cdist.scan to packages in setup.py (Dennis Camera) 6.9.2: 2020-11-20 From ba7d16a155cef46230d1dc650119f3d542a7f7f4 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 4 Dec 2020 17:57:55 +0100 Subject: [PATCH 24/52] __iptables_*: correct manpage spelling --- cdist/conf/type/__iptables_apply/man.rst | 2 +- cdist/conf/type/__iptables_rule/man.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__iptables_apply/man.rst b/cdist/conf/type/__iptables_apply/man.rst index 4109e789..3bef92cc 100644 --- a/cdist/conf/type/__iptables_apply/man.rst +++ b/cdist/conf/type/__iptables_apply/man.rst @@ -41,7 +41,7 @@ None EXAMPLES -------- -None (__iptables_apply is used by __iptables_rule automaticly) +None (__iptables_apply is used by __iptables_rule automatically) SEE ALSO diff --git a/cdist/conf/type/__iptables_rule/man.rst b/cdist/conf/type/__iptables_rule/man.rst index 86d38a34..afb71e01 100644 --- a/cdist/conf/type/__iptables_rule/man.rst +++ b/cdist/conf/type/__iptables_rule/man.rst @@ -31,7 +31,7 @@ state BOOLEAN PARAMETERS ------------------ -All rules without any of this parameter will be threaten like ``--v4`` because +All rules without any of these parameters will be treated like ``--v4`` because of backward compatibility. v4 From 2d19856840400af57bd6667cc868ab314a3e07c2 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 4 Dec 2020 18:26:03 +0100 Subject: [PATCH 25/52] [type/__package_pkgng_freebsd] Set ASSUME_ALWAYS_YES instead of -y --- cdist/conf/type/__package_pkgng_freebsd/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote index b5944177..05ba4cb2 100755 --- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote @@ -75,7 +75,7 @@ execcmd(){ esac if [ -z "${pkg_bootstrapped}" ]; then - echo "pkg bootstrap -y >/dev/null 2>&1" + echo "ASSUME_ALWAYS_YES=yes pkg bootstrap >/dev/null 2>&1" fi echo "$_cmd >/dev/null 2>&1" # Silence the output of the command From 087be130fa67d3fe195387ae0ab079f39c5066e1 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 4 Dec 2020 19:23:49 +0100 Subject: [PATCH 26/52] __iptables_apply: shorten copyright header Do we need all the copyright header or is this sufficient? The licence is given for cdist, but not on the target host. But it should be clear anyway. --- .../type/__iptables_apply/files/init-script | 34 ++++++------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script index 7faa2f92..e42017ae 100644 --- a/cdist/conf/type/__iptables_apply/files/init-script +++ b/cdist/conf/type/__iptables_apply/files/init-script @@ -1,28 +1,4 @@ #!/bin/sh -# -# 2013 Nico Schottelius (nico-cdist at schottelius.org) -# 2020 Matthias Stecher (matthiasstecher at gmx.de) -# -# This file is distributed with cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Originally written by: -# Nico Schottelius -# Zürisee, Mon Sep 2 18:38:27 CEST 2013 -# ### BEGIN INIT INFO # Provides: iptables # Required-Start: $local_fs $remote_fs @@ -35,6 +11,16 @@ # and saves/restores previous status ### END INIT INFO +# Originally written by: +# Nico Schottelius +# Zürisee, Mon Sep 2 18:38:27 CEST 2013 +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2020 Matthias Stecher (matthiasstecher at gmx.de) +# +# This file is distributed with cdist and licenced under the +# GNU GPLv3+ WITHOUT ANY WARRANTY. + # Read files and execute the content with the given commands # From 3930f69456fd5a0d108a107e4ee61d87c9a73a56 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 6 Dec 2020 16:45:58 +0100 Subject: [PATCH 27/52] __block: fix escaping in here-doc This changes the here-document to do not interpret any shell-things. It also single-quotes some more strings that are printed to code-remote. Fixes #838 --- cdist/conf/type/__block/gencode-remote | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__block/gencode-remote b/cdist/conf/type/__block/gencode-remote index 1f5cc033..acdb3286 100755 --- a/cdist/conf/type/__block/gencode-remote +++ b/cdist/conf/type/__block/gencode-remote @@ -46,10 +46,10 @@ fi remove_block() { cat << DONE -tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX) +tmpfile=\$(mktemp '${file}.cdist.XXXXXXXXXX') # preserve ownership and permissions of existing file -if [ -f "$file" ]; then - cp -p "$file" "\$tmpfile" +if [ -f '$file' ]; then + cp -p '$file' "\$tmpfile" fi awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ ' { @@ -63,8 +63,8 @@ awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ ' } else { print } -}' "$file" > "\$tmpfile" -mv -f "\$tmpfile" "$file" +}' '$file' > "\$tmpfile" +mv -f "\$tmpfile" '$file' DONE } @@ -77,7 +77,7 @@ case "$state_should" in echo add >> "$__messages_out" fi cat << DONE -cat >> "$file" << ${__type##*/}_DONE +cat >> '$file' << '${__type##*/}_DONE' $(cat "$block") ${__type##*/}_DONE DONE From bed08c2c5c71ec4252743e0b64b75e7d8fedbd92 Mon Sep 17 00:00:00 2001 From: Evilham Date: Sun, 6 Dec 2020 20:24:00 +0100 Subject: [PATCH 28/52] Deal with deprecation of imp module. importlib has been a thing since Python 3.1, and imp has been deprecated since 3.4. Insert random complaint here about not being able to use f-strings because they were introduced in Python 3.6 and apparently we support Python 3.5 >,<. Output diff before to after for ./bin/cdist-build-helper test (on heavy load): ``` 1,2d0 < /usr/home/evilham/s/cdist/cdist/cdist/test/__main__.py:23: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses < import imp 72c70 < ERROR: cdisttesthost: __file/tmp/foobar requires object __file without object id. Defined at /tmp/tmp.cdist.test.g87lx7c8/tmp.cdist.test.6ramsakx --- > ERROR: cdisttesthost: __file/tmp/foobar requires object __file without object id. Defined at /tmp/tmp.cdist.test.aqdf6vjz/tmp.cdist.test.jgv3udel 76c74 < test_nonexistent_type_requirement (cdist.test.emulator.EmulatorTestCase) ... ERROR: cdisttesthost: __file/tmp/foobar requires object __does-not-exist/some-id, but type __does-not-exist does not exist. Defined at /tmp/tmp.cdist.test.mma5j8ln/tmp.cdist.test.3zg4by4d --- > test_nonexistent_type_requirement (cdist.test.emulator.EmulatorTestCase) ... ERROR: cdisttesthost: __file/tmp/foobar requires object __does-not-exist/some-id, but type __does-not-exist does not exist. Defined at /tmp/tmp.cdist.test.t8d6ockr/tmp.cdist.test.uimxurg9 86c84 < test_initial_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running initial manifest /tmp/tmp.cdist.test.uvid60ij/759547ff4356de6e3d9e08522b0d0807/data/conf/manifest/dump_environment --- > test_initial_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running initial manifest /tmp/tmp.cdist.test._cttcnrj/759547ff4356de6e3d9e08522b0d0807/data/conf/manifest/dump_environment 89c87 < test_type_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running type manifest /tmp/tmp.cdist.test.k1i2onpb/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__dump_environment/manifest for object __dump_environment/whatever --- > test_type_manifest_environment (cdist.test.manifest.ManifestTestCase) ... VERBOSE: cdisttesthost: Running type manifest /tmp/tmp.cdist.test.ukr7lrzd/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__dump_environment/manifest for object __dump_environment/whatever 272c270 < Ran 225 tests in 44.457s --- > Ran 225 tests in 43.750s ``` --- cdist/test/__main__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cdist/test/__main__.py b/cdist/test/__main__.py index c8c7df3b..8049c752 100644 --- a/cdist/test/__main__.py +++ b/cdist/test/__main__.py @@ -20,7 +20,7 @@ # # -import imp +import importlib import os import sys import unittest @@ -37,8 +37,9 @@ for possible_test in os.listdir(base_dir): suites = [] for test_module in test_modules: - module_parameters = imp.find_module(test_module, [base_dir]) - module = imp.load_module("cdist.test." + test_module, *module_parameters) + module_spec = importlib.util.find_spec("cdist.test.{}".format(test_module)) + module = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(module) suite = unittest.defaultTestLoader.loadTestsFromModule(module) # print("Got suite: " + suite.__str__()) From 29662961731ef07fbc20213d0d87da1469f0d27c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 7 Dec 2020 19:47:52 +0100 Subject: [PATCH 29/52] ++changelog --- docs/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog b/docs/changelog index 6c272fd7..8b35901b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,10 @@ Changelog --------- +next: + * __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) + * Core: Deal with deprecated imp in unit tests (Evil Ham) + 6.9.3: 2020-12-04 * pip install: Add cdist.scan to packages in setup.py (Dennis Camera) From c5ca4cd2e13516dfb55371c1600e32297c3343e9 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 7 Dec 2020 19:59:05 +0100 Subject: [PATCH 30/52] __block: securly quote via the quote function Because the function already exists, it will be used for the file to be changed, too. Therefor, no quotes are required for that value. The prefix and suffix match was also improved: There is no regex check any more (the regex did checked the whole line); instead it will do it simple. --- cdist/conf/type/__block/gencode-remote | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/cdist/conf/type/__block/gencode-remote b/cdist/conf/type/__block/gencode-remote index acdb3286..7a1f4064 100755 --- a/cdist/conf/type/__block/gencode-remote +++ b/cdist/conf/type/__block/gencode-remote @@ -46,28 +46,29 @@ fi remove_block() { cat << DONE -tmpfile=\$(mktemp '${file}.cdist.XXXXXXXXXX') +tmpfile=\$(mktemp ${quoted_file}.cdist.XXXXXXXXXX) # preserve ownership and permissions of existing file -if [ -f '$file' ]; then - cp -p '$file' "\$tmpfile" +if [ -f $quoted_file ]; then + cp -p $quoted_file "\$tmpfile" fi -awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ ' +awk -v prefix=$(quote "$prefix") -v suffix=$(quote "$suffix") ' { - if (match(\$0,prefix)) { + if (\$0 == prefix) { triggered=1 } if (triggered) { - if (match(\$0,suffix)) { + if (\$0 == suffix) { triggered=0 } } else { print } -}' '$file' > "\$tmpfile" -mv -f "\$tmpfile" '$file' +}' $quoted_file > "\$tmpfile" +mv -f "\$tmpfile" $quoted_file DONE } +quoted_file="$(quote "$file")" case "$state_should" in present) if [ "$state_is" = "changed" ]; then @@ -77,7 +78,7 @@ case "$state_should" in echo add >> "$__messages_out" fi cat << DONE -cat >> '$file' << '${__type##*/}_DONE' +cat >> $quoted_file << '${__type##*/}_DONE' $(cat "$block") ${__type##*/}_DONE DONE From 14c81d6c7e20a2ed04c7318f6e8ecb51098390e8 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 8 Dec 2020 07:16:26 +0100 Subject: [PATCH 31/52] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 8b35901b..4be08ae9 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) * Core: Deal with deprecated imp in unit tests (Evil Ham) + * Type __iptables: Add IPv6 support (Matthias Stecher) 6.9.3: 2020-12-04 * pip install: Add cdist.scan to packages in setup.py (Dennis Camera) From a58f5ffa7f59a7e28ba0a414097f154e4a497e88 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 8 Dec 2020 19:36:44 +0100 Subject: [PATCH 32/52] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 4be08ae9..3d6084f1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) * Core: Deal with deprecated imp in unit tests (Evil Ham) * Type __iptables: Add IPv6 support (Matthias Stecher) + * Type __block: Fix escaping in here-doc (Matthias Stecher) 6.9.3: 2020-12-04 * pip install: Add cdist.scan to packages in setup.py (Dennis Camera) From a5169ad858a4d6e9c378184db5736f55a86306e0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 10 Dec 2020 21:24:26 +0100 Subject: [PATCH 33/52] new type __debian_backports This new type will setup the backports distribution for the current Debian release. --- cdist/conf/type/__debian_backports/man.rst | 90 +++++++++++++++++++ cdist/conf/type/__debian_backports/manifest | 59 ++++++++++++ .../parameter/default/mirror | 1 + .../parameter/default/state | 1 + .../__debian_backports/parameter/optional | 2 + cdist/conf/type/__debian_backports/singleton | 0 6 files changed, 153 insertions(+) create mode 100644 cdist/conf/type/__debian_backports/man.rst create mode 100755 cdist/conf/type/__debian_backports/manifest create mode 100644 cdist/conf/type/__debian_backports/parameter/default/mirror create mode 100644 cdist/conf/type/__debian_backports/parameter/default/state create mode 100644 cdist/conf/type/__debian_backports/parameter/optional create mode 100644 cdist/conf/type/__debian_backports/singleton diff --git a/cdist/conf/type/__debian_backports/man.rst b/cdist/conf/type/__debian_backports/man.rst new file mode 100644 index 00000000..ba353f4e --- /dev/null +++ b/cdist/conf/type/__debian_backports/man.rst @@ -0,0 +1,90 @@ +cdist-type__debian_backports(7) +=============================== + +NAME +---- +cdist-type__debian_backports - Install backports for Debain systems + + +DESCRIPTION +----------- +This singleton type installs backports for the current Debian version. +It aborts if backports are not supported for the specified os or no +version codename could be fetched (like Debian unstable). + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +state + Represents the state of the backports repository. ``present`` or + ``absent``, defaults to ``present``. + + Will be directly passed to :strong:`cdist-type__apt_source`\ (7). + +mirror + The mirror to fetch the backports from. Will defaults to the Debian default + ``_. + + Will be directly passed to :strong:`cdist-type__apt_source`\ (7). + + +BOOLEAN PARAMETERS +------------------ +None. + + +MESSAGES +-------- +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # setup the backports + __debian_backports + __debian_backports --state absent + __debian_backports --state present --mirror "http://ftp.de.debian.org/debian/" + + # update + require="__debian_backports" __apt_update_index + + # install a backports package + # currently for the buster release backports + require="__apt_update_index" __package_apt wireguard \ + --target-release buster-backports + + +ABORTS +------ +Aborts if the detected os is not Debian. + +Aborts if no distribuition codename could be detected. This is common for the +unstable distribution, but there is no backports repository for it already. + + +SEE ALSO +-------- +`Official Debian Backports site `_ + +:strong:`cdist-type__apt_source`\ (7) + + +AUTHORS +------- +Matthias Stecher + + +COPYING +------- +Copyright \(C) 2020 Matthias Stecher. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__debian_backports/manifest new file mode 100755 index 00000000..29bf9a43 --- /dev/null +++ b/cdist/conf/type/__debian_backports/manifest @@ -0,0 +1,59 @@ +#!/bin/sh -e +# __debian_backports/manifest +# +# 2020 Matthias Stecher (matthiasstecher at gmx.de) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Enables/disables backports repository. Utilies __apt_source for it. +# + + +# detect backport distribution +os="$(cat "$__global/explorer/os")" +case "$os" in + debian) + # distribution codename from /etc/os-release + # lsb_release may not be given in all debian installations + dist="$( + . "$__global/explorer/os-release" + printf "%s" "$VERSION_CODENAME" + )" + ;; + *) + printf "Backports for %s are not supported!\n" "$os" >&2 + exit 1 + ;; +esac + +# error if no codename given (e.g. on Debian unstable) +if [ -z "$dist" ]; then + printf "No backports for unkown version of distribution %s!\n" "$os" >&2 + exit 1 +fi + + +# parameters +state="$(cat "$__object/parameter/state")" +mirror="$(cat "$__object/parameter/mirror")" + +# install the given backports repository +__apt_source "${dist}-backports" \ + --state "$state" \ + --distribution "${dist}-backports" \ + --component main \ + --uri "$mirror" diff --git a/cdist/conf/type/__debian_backports/parameter/default/mirror b/cdist/conf/type/__debian_backports/parameter/default/mirror new file mode 100644 index 00000000..0965ef04 --- /dev/null +++ b/cdist/conf/type/__debian_backports/parameter/default/mirror @@ -0,0 +1 @@ +http://deb.debian.org/debian/ diff --git a/cdist/conf/type/__debian_backports/parameter/default/state b/cdist/conf/type/__debian_backports/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__debian_backports/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__debian_backports/parameter/optional b/cdist/conf/type/__debian_backports/parameter/optional new file mode 100644 index 00000000..4b05c235 --- /dev/null +++ b/cdist/conf/type/__debian_backports/parameter/optional @@ -0,0 +1,2 @@ +state +mirror diff --git a/cdist/conf/type/__debian_backports/singleton b/cdist/conf/type/__debian_backports/singleton new file mode 100644 index 00000000..e69de29b From 0d96b31b5696f95b559450e17c71914d0746c0ce Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 11 Dec 2020 18:13:44 +0100 Subject: [PATCH 34/52] __debian_backports: pass shellcheck for sourced file Because the sourced explorer can't be detected by shellcheck, it will be completely disabled. Changing the path to /etc/os-release isn't deterministic either. The shellcheck wiki page suggests to use `source=/dev/null` instead of `disable=SC1090`, but it was choosen to completely avoid that check .. --- cdist/conf/type/__debian_backports/manifest | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__debian_backports/manifest index 29bf9a43..661e5281 100755 --- a/cdist/conf/type/__debian_backports/manifest +++ b/cdist/conf/type/__debian_backports/manifest @@ -30,6 +30,7 @@ case "$os" in # distribution codename from /etc/os-release # lsb_release may not be given in all debian installations dist="$( + # shellcheck disable=SC1090 . "$__global/explorer/os-release" printf "%s" "$VERSION_CODENAME" )" From bc2948a8a552e3bda3ab9ae8e8e199e831d81af2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 11 Dec 2020 19:37:53 +0100 Subject: [PATCH 35/52] ++scan stuff --- cdist/scan/scan.py | 15 +++++++++++++++ docs/dev/logs/2020-10-29.org | 10 +++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index fcbf1899..00858581 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -56,6 +56,8 @@ from scapy.all import * # Datetime overwrites scapy.all.datetime - needs to be imported AFTER import datetime +import cdist.config + log = logging.getLogger("scan") class Trigger(object): @@ -120,6 +122,19 @@ class Scanner(object): with open(fname, "w") as fd: fd.write(f"{now}\n") + def config(self): + """ + Configure a host + + - Assume we are only called if necessary + - However we need to ensure to not run in parallel + - Maybe keep dict storing per host processes + - Save the result + - Save the output -> probably aligned to config mode + + """ + + def start(self): self.process = Process(target=self.scan) self.process.start() diff --git a/docs/dev/logs/2020-10-29.org b/docs/dev/logs/2020-10-29.org index 4461be8c..03d6b3f4 100644 --- a/docs/dev/logs/2020-10-29.org +++ b/docs/dev/logs/2020-10-29.org @@ -54,4 +54,12 @@ VERBOSE: scan: Host fe80::f29f:c2ff:fe7c:275e is alive VERBOSE: scan: Host fe80::ba69:f4ff:fec5:8db7 is alive VERBOSE: scan: Host fe80::42b0:34ff:fe6f:f863 is alive VERBOSE: scan: Host fe80::21b:fcff:feee:f4bc is alive -... +** Better usage -> saving the env + sudo -E cdist scan -b -I wlan0 -vv +** TODO Implement actual configuration step + - Also serves as a nice PoC + - Might need to escape literal IPv6 addresses for scp +** TODO Define how to map link local address to something useful + - via reverse DNS? + - via link local in manifest? +** TODO define ignorehosts? From c4d19a23193ec13eda96a3d7536ba338d0801825 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 12 Dec 2020 09:36:17 +0100 Subject: [PATCH 36/52] __debian_backports -> __apt_backports; add wider os support As discussed in the chat, this type now supports a broader list of OSes which it supports backports for. Because of this, it was renamed to something more generic. "apt" should fit in. --- .../man.rst | 27 ++++++++---- .../manifest | 41 ++++++++++++++----- .../parameter/default/state | 0 .../parameter/optional | 0 .../singleton | 0 .../parameter/default/mirror | 1 - 6 files changed, 49 insertions(+), 20 deletions(-) rename cdist/conf/type/{__debian_backports => __apt_backports}/man.rst (66%) rename cdist/conf/type/{__debian_backports => __apt_backports}/manifest (59%) rename cdist/conf/type/{__debian_backports => __apt_backports}/parameter/default/state (100%) rename cdist/conf/type/{__debian_backports => __apt_backports}/parameter/optional (100%) rename cdist/conf/type/{__debian_backports => __apt_backports}/singleton (100%) delete mode 100644 cdist/conf/type/__debian_backports/parameter/default/mirror diff --git a/cdist/conf/type/__debian_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst similarity index 66% rename from cdist/conf/type/__debian_backports/man.rst rename to cdist/conf/type/__apt_backports/man.rst index ba353f4e..7d269fbb 100644 --- a/cdist/conf/type/__debian_backports/man.rst +++ b/cdist/conf/type/__apt_backports/man.rst @@ -3,13 +3,13 @@ cdist-type__debian_backports(7) NAME ---- -cdist-type__debian_backports - Install backports for Debain systems +cdist-type__apt_backports - Install backports DESCRIPTION ----------- -This singleton type installs backports for the current Debian version. -It aborts if backports are not supported for the specified os or no +This singleton type installs backports for the current OS release. +It aborts if backports are not supported for the specified OS or no version codename could be fetched (like Debian unstable). @@ -27,8 +27,8 @@ state Will be directly passed to :strong:`cdist-type__apt_source`\ (7). mirror - The mirror to fetch the backports from. Will defaults to the Debian default - ``_. + The mirror to fetch the backports from. Will defaults to the generic + mirror of the current OS. Will be directly passed to :strong:`cdist-type__apt_source`\ (7). @@ -49,12 +49,12 @@ EXAMPLES .. code-block:: sh # setup the backports - __debian_backports - __debian_backports --state absent - __debian_backports --state present --mirror "http://ftp.de.debian.org/debian/" + __apt_backports + __apt_backports --state absent + __apt_backports --state present --mirror "http://ftp.de.debian.org/debian/" # update - require="__debian_backports" __apt_update_index + require="__apt_backports" __apt_update_index # install a backports package # currently for the buster release backports @@ -70,6 +70,15 @@ Aborts if no distribuition codename could be detected. This is common for the unstable distribution, but there is no backports repository for it already. +CAVEATS +------- +For Ubuntu, it setup all componenents for the backports repository: ``main``, +``restricted``, ``universe`` and ``multiverse``. The user may not want to +install proprietary packages, which will only be installed if the user +explicitly uses the backports target-release. The user may change this behavior +to install backports packages without the need of explicitly select it. + + SEE ALSO -------- `Official Debian Backports site `_ diff --git a/cdist/conf/type/__debian_backports/manifest b/cdist/conf/type/__apt_backports/manifest similarity index 59% rename from cdist/conf/type/__debian_backports/manifest rename to cdist/conf/type/__apt_backports/manifest index 661e5281..e5358dea 100755 --- a/cdist/conf/type/__debian_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -1,5 +1,5 @@ #!/bin/sh -e -# __debian_backports/manifest +# __apt_backports/manifest # # 2020 Matthias Stecher (matthiasstecher at gmx.de) # @@ -23,18 +23,34 @@ # +# Get the distribution codename by /etc/os-release. +# is already executed in a subshell by string substitution +# lsb_release may not be given in all installations +codename_os_release() { + # shellcheck disable=SC1090 + . "$__global/explorer/os-release" + printf "%s" "$VERSION_CODENAME" +} + # detect backport distribution os="$(cat "$__global/explorer/os")" case "$os" in debian) - # distribution codename from /etc/os-release - # lsb_release may not be given in all debian installations - dist="$( - # shellcheck disable=SC1090 - . "$__global/explorer/os-release" - printf "%s" "$VERSION_CODENAME" - )" + dist="$( codename_os_release )" + components="main" + mirror="http://deb.debian.org/debian/" ;; + devuan) + dist="$( codename_os_release )" + components="main" + mirror="http://deb.devuan.org/merged" + ;; + ubuntu) + dist="$( codename_os_release )" + components="main restricted universe multiverse" + mirror="http://archive.ubuntu.com/ubuntu" + ;; + *) printf "Backports for %s are not supported!\n" "$os" >&2 exit 1 @@ -50,11 +66,16 @@ fi # parameters state="$(cat "$__object/parameter/state")" -mirror="$(cat "$__object/parameter/mirror")" + +# mirror already set for the os, only override user-values +if [ -f "$__object/parameter/mirror" ]; then + mirror="$(cat "$__object/parameter/mirror")" +fi + # install the given backports repository __apt_source "${dist}-backports" \ --state "$state" \ --distribution "${dist}-backports" \ - --component main \ + --component "$components" \ --uri "$mirror" diff --git a/cdist/conf/type/__debian_backports/parameter/default/state b/cdist/conf/type/__apt_backports/parameter/default/state similarity index 100% rename from cdist/conf/type/__debian_backports/parameter/default/state rename to cdist/conf/type/__apt_backports/parameter/default/state diff --git a/cdist/conf/type/__debian_backports/parameter/optional b/cdist/conf/type/__apt_backports/parameter/optional similarity index 100% rename from cdist/conf/type/__debian_backports/parameter/optional rename to cdist/conf/type/__apt_backports/parameter/optional diff --git a/cdist/conf/type/__debian_backports/singleton b/cdist/conf/type/__apt_backports/singleton similarity index 100% rename from cdist/conf/type/__debian_backports/singleton rename to cdist/conf/type/__apt_backports/singleton diff --git a/cdist/conf/type/__debian_backports/parameter/default/mirror b/cdist/conf/type/__debian_backports/parameter/default/mirror deleted file mode 100644 index 0965ef04..00000000 --- a/cdist/conf/type/__debian_backports/parameter/default/mirror +++ /dev/null @@ -1 +0,0 @@ -http://deb.debian.org/debian/ From 49aec0b5e40ba05e7703565731aae6b0a9e5936a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 12 Dec 2020 09:40:47 +0100 Subject: [PATCH 37/52] __apt_backports: list supported OSes The manpage now lists all OSes where this type supports backports. --- cdist/conf/type/__apt_backports/man.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst index 7d269fbb..fd311361 100644 --- a/cdist/conf/type/__apt_backports/man.rst +++ b/cdist/conf/type/__apt_backports/man.rst @@ -12,6 +12,12 @@ This singleton type installs backports for the current OS release. It aborts if backports are not supported for the specified OS or no version codename could be fetched (like Debian unstable). +It supports backports from following OSes: + +- Debian +- Devuan +- Ubuntu + REQUIRED PARAMETERS ------------------- From fafa3d9ea55f38ae0a350a463d64b7ef1d70c7bb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 12 Dec 2020 10:00:23 +0100 Subject: [PATCH 38/52] __apt_backports: update index if required This type now automatically calls the type __apt_update_index to update the package index if something changed. --- cdist/conf/type/__apt_backports/man.rst | 13 +++++++------ cdist/conf/type/__apt_backports/manifest | 5 ++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst index fd311361..c578ce6b 100644 --- a/cdist/conf/type/__apt_backports/man.rst +++ b/cdist/conf/type/__apt_backports/man.rst @@ -9,8 +9,11 @@ cdist-type__apt_backports - Install backports DESCRIPTION ----------- This singleton type installs backports for the current OS release. -It aborts if backports are not supported for the specified OS or no -version codename could be fetched (like Debian unstable). +It aborts if backports are not supported for the specified OS or +no version codename could be fetched (like Debian unstable). + +The package index will be automatically updated by the type +:strong:`cdist-type__apt_update_index`\ (7) if required. It supports backports from following OSes: @@ -59,12 +62,9 @@ EXAMPLES __apt_backports --state absent __apt_backports --state present --mirror "http://ftp.de.debian.org/debian/" - # update - require="__apt_backports" __apt_update_index - # install a backports package # currently for the buster release backports - require="__apt_update_index" __package_apt wireguard \ + require="__apt_backports" __package_apt wireguard \ --target-release buster-backports @@ -90,6 +90,7 @@ SEE ALSO `Official Debian Backports site `_ :strong:`cdist-type__apt_source`\ (7) +:strong:`cdist-type__apt_update_index`\ (7) AUTHORS diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index e5358dea..c490a103 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -19,7 +19,7 @@ # along with cdist. If not, see . # # -# Enables/disables backports repository. Utilies __apt_source for it. +# Enables/disables backports repository. Utilises __apt_source for it. # @@ -79,3 +79,6 @@ __apt_source "${dist}-backports" \ --distribution "${dist}-backports" \ --component "$components" \ --uri "$mirror" + +# update the index if the source changed +require="__apt_source/${dist}-backports" __apt_update_index From 645734c62959c694597826598be3b34c0edc79d2 Mon Sep 17 00:00:00 2001 From: Evilham Date: Sat, 12 Dec 2020 12:15:17 +0100 Subject: [PATCH 39/52] [explorer/os_version] Improve FreeBSD support. It looks like uname -r is not the most reliable way to get the target patch level for the target system. For more information see: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743 --- cdist/conf/explorer/os_version | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index a7b1d3bc..3b02dedd 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -70,6 +70,11 @@ case "$("$__explorer/os")" in macosx) sw_vers -productVersion ;; + freebsd) + # Apparently uname -r is not a reliable way to get the patch level. + # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743 + freebsd-version + ;; *bsd|solaris) uname -r ;; From fca35fc858d09fc649dc5e7f0964cef4af9e09f0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 12 Dec 2020 17:29:58 +0100 Subject: [PATCH 40/52] __apt_backports: fix explorer call s/-/_/ because the explorers are following an other convention :-) --- cdist/conf/type/__apt_backports/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index c490a103..c2943725 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -28,7 +28,7 @@ # lsb_release may not be given in all installations codename_os_release() { # shellcheck disable=SC1090 - . "$__global/explorer/os-release" + . "$__global/explorer/os_release" printf "%s" "$VERSION_CODENAME" } From 27aca06fb893c84601e14bc30890ea6be9300dcd Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 12 Dec 2020 17:34:51 +0100 Subject: [PATCH 41/52] __apt_backports: undo __apt_update_index call Becuase it is already done by __apt_source. --- cdist/conf/type/__apt_backports/man.rst | 4 +--- cdist/conf/type/__apt_backports/manifest | 3 --- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cdist/conf/type/__apt_backports/man.rst b/cdist/conf/type/__apt_backports/man.rst index c578ce6b..7036fb84 100644 --- a/cdist/conf/type/__apt_backports/man.rst +++ b/cdist/conf/type/__apt_backports/man.rst @@ -12,8 +12,7 @@ This singleton type installs backports for the current OS release. It aborts if backports are not supported for the specified OS or no version codename could be fetched (like Debian unstable). -The package index will be automatically updated by the type -:strong:`cdist-type__apt_update_index`\ (7) if required. +The package index will be automatically updated if required. It supports backports from following OSes: @@ -90,7 +89,6 @@ SEE ALSO `Official Debian Backports site `_ :strong:`cdist-type__apt_source`\ (7) -:strong:`cdist-type__apt_update_index`\ (7) AUTHORS diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index c2943725..bc47d8de 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -79,6 +79,3 @@ __apt_source "${dist}-backports" \ --distribution "${dist}-backports" \ --component "$components" \ --uri "$mirror" - -# update the index if the source changed -require="__apt_source/${dist}-backports" __apt_update_index From 71f22831175ec41e9348a759fcdaf0eceb1a2a52 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 13 Dec 2020 16:03:39 +0100 Subject: [PATCH 42/52] ++changelog --- docs/changelog | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 3d6084f1..290f94d0 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,10 +2,12 @@ Changelog --------- next: - * __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) + * Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) * Core: Deal with deprecated imp in unit tests (Evil Ham) * Type __iptables: Add IPv6 support (Matthias Stecher) * Type __block: Fix escaping in here-doc (Matthias Stecher) + * Explorer os_version: Improve FreeBSD support (Evil Ham) + * New type: __apt_backports (Matthias Stecher) 6.9.3: 2020-12-04 * pip install: Add cdist.scan to packages in setup.py (Dennis Camera) From 8dc2c4207cdd826b9e64cc91cfc7c918cddd5f35 Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Fri, 18 Dec 2020 11:16:28 +0100 Subject: [PATCH 43/52] Added optional dirmode parameter to set the mode of (optional) the directory. --- cdist/conf/type/__dot_file/man.rst | 3 +++ cdist/conf/type/__dot_file/manifest | 2 ++ cdist/conf/type/__dot_file/parameter/default/dirmode | 1 + cdist/conf/type/__dot_file/parameter/optional | 1 + 4 files changed, 7 insertions(+) create mode 100644 cdist/conf/type/__dot_file/parameter/default/dirmode diff --git a/cdist/conf/type/__dot_file/man.rst b/cdist/conf/type/__dot_file/man.rst index ae65eb95..ba7621a1 100644 --- a/cdist/conf/type/__dot_file/man.rst +++ b/cdist/conf/type/__dot_file/man.rst @@ -25,6 +25,9 @@ user OPTIONAL PARAMETERS ------------------- +dirmode + forwarded to :strong:`__directory` type as mode + mode forwarded to :strong:`__file` type diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest index 5e4957e5..02dadf05 100755 --- a/cdist/conf/type/__dot_file/manifest +++ b/cdist/conf/type/__dot_file/manifest @@ -19,6 +19,7 @@ set -eu user="$(cat "${__object}/parameter/user")" home="$(cat "${__object}/explorer/home")" primary_group="$(cat "${__object}/explorer/primary_group")" +dirmode="$(cat "${__object}/parameter/dirmode")" # Create parent directory. Type __directory has flag 'parents', but it # will leave us with root-owned directory in user home, which is not @@ -36,6 +37,7 @@ export CDIST_ORDER_DEPENDENCY for dir ; do __directory "${home}/${dir}" \ --group "${primary_group}" \ + --mode "${dirmode}" \ --owner "${user}" done diff --git a/cdist/conf/type/__dot_file/parameter/default/dirmode b/cdist/conf/type/__dot_file/parameter/default/dirmode new file mode 100644 index 00000000..e9745d1f --- /dev/null +++ b/cdist/conf/type/__dot_file/parameter/default/dirmode @@ -0,0 +1 @@ +0700 diff --git a/cdist/conf/type/__dot_file/parameter/optional b/cdist/conf/type/__dot_file/parameter/optional index ccab9fa6..9f7f83fb 100644 --- a/cdist/conf/type/__dot_file/parameter/optional +++ b/cdist/conf/type/__dot_file/parameter/optional @@ -1,3 +1,4 @@ state mode source +dirmode From 4bae2863dbc1f6b60b8e797c2f121e1616a94c94 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 18 Dec 2020 12:54:33 +0100 Subject: [PATCH 44/52] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 290f94d0..3a623236 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * Type __block: Fix escaping in here-doc (Matthias Stecher) * Explorer os_version: Improve FreeBSD support (Evil Ham) * New type: __apt_backports (Matthias Stecher) + * Type __dot_file: Add dirmode parameter (Mark Verboom) 6.9.3: 2020-12-04 * pip install: Add cdist.scan to packages in setup.py (Dennis Camera) From 7cf85c465980c74049f8e0258758c0fd8ea178b2 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 21 Dec 2020 19:21:51 +0100 Subject: [PATCH 45/52] Release 6.9.4 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 3a623236..35953a88 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.4: 2020-12-21 * Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) * Core: Deal with deprecated imp in unit tests (Evil Ham) * Type __iptables: Add IPv6 support (Matthias Stecher) From 766198912d558e4630b258bba62e78587b3f0efe Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 5 Jan 2021 15:50:21 +0100 Subject: [PATCH 46/52] [type/__sshd_config] Produce error if invalid config file is generated Previously, cdist would silently swallow the error (no invalid config file was generated). Reason: `set -e` does not exit if a command in a sub-command group fails, it merely returns with a non-zero exit status. e.g. the following snippet does not abort the script if sshd -t returns with a non-zero exit status: set -e cmp -s old new || { # check config file and update it sshd -t -f new \ && cat new >old } or compressed: set -e false || { false && true; } echo $? # prints 1 --- cdist/conf/type/__sshd_config/gencode-remote | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__sshd_config/gencode-remote b/cdist/conf/type/__sshd_config/gencode-remote index 0b44dfa7..275db4aa 100755 --- a/cdist/conf/type/__sshd_config/gencode-remote +++ b/cdist/conf/type/__sshd_config/gencode-remote @@ -91,7 +91,8 @@ awk $(drop_awk_comments "${__type:?}/files/update_sshd_config.awk") \\ cmp -s $(quote "${sshd_config_file}") $(quote "${sshd_config_file}.tmp") || { sshd -t -f $(quote "${sshd_config_file}.tmp") \\ - && cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}") + && cat $(quote "${sshd_config_file}.tmp") >$(quote "${sshd_config_file}") \\ + || exit # stop if sshd_config file check fails } rm -f $(quote "${sshd_config_file}.tmp") EOF From 8753b7eedf022a052ddc0b27fbc58ebf8fd638e1 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 5 Jan 2021 15:57:39 +0100 Subject: [PATCH 47/52] [type/__sshd_config] Make AuthenticationMethods and AuthorizedKeysFile singleton options They were incorrectly treated as non-singleton options before. cf. https://github.com/openssh/openssh-portable/blob/V_8_4/servconf.c#L2273 and https://github.com/openssh/openssh-portable/blob/V_8_4/servconf.c#L1899 resp. --- cdist/conf/type/__sshd_config/files/update_sshd_config.awk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk index d0bc2b4b..f7f30e87 100644 --- a/cdist/conf/type/__sshd_config/files/update_sshd_config.awk +++ b/cdist/conf/type/__sshd_config/files/update_sshd_config.awk @@ -89,7 +89,7 @@ function strdelim(s) { return strdelim_internal(s, 1) } function strdelimw(s) { return strdelim_internal(s, 0) } function singleton_option(opt) { - return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|authenticationmethods|authorizedkeysfile|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/ + return tolower(opt) !~ /^(acceptenv|allowgroups|allowusers|denygroups|denyusers|hostcertificate|hostkey|listenaddress|logverbose|permitlisten|permitopen|port|setenv|subsystem)$/ } function print_update() { From bd8ab8f26fdc242f2eb77e58049d640f96d51a69 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 5 Jan 2021 17:00:55 +0100 Subject: [PATCH 48/52] [type/__sshd_config] Document "bug" in state explorer --- cdist/conf/type/__sshd_config/man.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cdist/conf/type/__sshd_config/man.rst b/cdist/conf/type/__sshd_config/man.rst index 8b0069ac..c8e6b8ad 100644 --- a/cdist/conf/type/__sshd_config/man.rst +++ b/cdist/conf/type/__sshd_config/man.rst @@ -79,6 +79,10 @@ BUGS - ``Include`` directives are ignored. - Config options are not added/removed to/from the config file if their value is the default value. +- | The explorer will incorrectly report ``absent`` if OpenSSH internally + transforms one value to another (e.g. ``permitrootlogin prohibit-password`` + is transformed to ``permitrootlogin without-password``). + | Workaround: Use the value that OpenSSH uses internally. AUTHORS From c819548343ad7bb24d023f94a32575e1b2520328 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 11 Jan 2021 09:50:12 +0100 Subject: [PATCH 49/52] Fix debug parameter -d was removed from cdist in favor of mulitple -v and -l parameters, but -d was not removed from preos. Resolve #849. --- cdist/preos/debootstrap/files/code | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/preos/debootstrap/files/code b/cdist/preos/debootstrap/files/code index 9e37003b..d836848c 100755 --- a/cdist/preos/debootstrap/files/code +++ b/cdist/preos/debootstrap/files/code @@ -22,7 +22,7 @@ set -e if [ "${debug}" ] then set -x - cdist_params="${cdist_params} -d" + cdist_params="${cdist_params} -l 3" fi bootstrap_dir="${target_dir}" From 6e9b13d94962e1bfb365efa849ebefc32854364b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 18 Jan 2021 06:22:32 +0100 Subject: [PATCH 50/52] ++changelog --- docs/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog b/docs/changelog index 35953a88..b2b35616 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,10 @@ Changelog --------- +next: + * Core: preos: Fix passing cdist debug parameter (Darko Poljak) + * Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera) + 6.9.4: 2020-12-21 * Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) * Core: Deal with deprecated imp in unit tests (Evil Ham) From 92a50da4873ff4bfa85dde3b85e272704cda02c7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 18 Jan 2021 06:28:09 +0100 Subject: [PATCH 51/52] Fix pycodestyle issues --- cdist/scan/scan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 5ca8fae2..b1d0e9e1 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -139,7 +139,6 @@ class Scanner(object): """ - def start(self): self.process = Process(target=self.scan) self.process.start() From 92432c5d9a625a07a3fa3230788662099cf7e1cb Mon Sep 17 00:00:00 2001 From: Alexander Sieg Date: Wed, 20 Jan 2021 20:18:38 +0100 Subject: [PATCH 52/52] cconn a transparent connection wrapper cconn used the inventory system to dynamically connect to FreeBSD jail via the host system and/or using sudo to get root access This is more of a prove of concept then a finish idea. --- other/examples/remote/cconn/cconn | 199 ++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100755 other/examples/remote/cconn/cconn diff --git a/other/examples/remote/cconn/cconn b/other/examples/remote/cconn/cconn new file mode 100755 index 00000000..bc712a0d --- /dev/null +++ b/other/examples/remote/cconn/cconn @@ -0,0 +1,199 @@ +#!/bin/sh -ex + +# Copyright (c) 2021 Alexander Sieg + +# cconn is a transparent wrapper that allows seamless usage of sudo/rsync and +# FreeBSD jails, this makes it possible to configure a FreeBSD by just using +# the host system sshd. +# Configuration is done by throw the cdist invertory system. +# +# +# Installation: +# +# Simply set this script as remote_exec and remote_copy implementation in your +# cdist.cfg. You MUST always pass either copy or exec as the first parameter +# to this script +# +# remote_exec = path/to/cconn exec +# remote_copy = path/to/cconn copy +# +# As cconn uses the cdist inventory system for host specific configuration, it +# ether need to be executed from the directory the contains the inventory +# directory or be setting the INVENTORY environment variable to path were the +# inventory entries are kept. +# +# Usage: +# +# To setup a host to use some form of connection "bending" (e.g. sudo) you need +# to add a single __cconn_options tag to the host inventory file. Options are always a +# key value pair separated by a '='. All options are passed in a single tag and +# a separated by a space. +# +# __cconn_options options: +# +# NOTE: jail_host and iocage_jail can not be used at the same time and will +# lead to a error +# +# jail_host: +# hostname on which the FreeBSD jail resides on. +# +# iocage_host: +# Same as jail_host, but for jails managed by iocage(8) +# +# jail_name: +# By default cconn will use the __target_host as the FreeBSD jail name, set +# this option to override this name +# +# sudo_user: +# Username used to connect to the __target_host, all commands are then +# prefixed with sudo and copy operations are done with rsync +# +# For this work you need to be able to execute all command without password entry. +# sudoers(5) +# %wheel ALL=(ALL) NOPASSWD: ALL +# +#Examples: +# +# inventory/jail.example.com: +# some_other_tag +# __cconn_options jail_host=example.com sudo_user=ada +# + + +#TODO: add sudo_pass and sudo_passfile option to support use without NOPASSWD +#TODO: support SSH connection multiplexing. This requieres a patch to cdist, as +# we need path to the tmp dir. + +log() { + # Uncomment this for debugging + echo "$@" | logger -t "cdist-cconn-$COMMAND" + : +} + +COMMAND="$1"; shift + +if [ -z "$INVENTORY" ]; then + INVENTORY="inventory" +fi + +# shellcheck disable=SC2154 +options="$(sed -n 's/^__cconn_options\(.*\)$/\1/p' "$INVENTORY/$__target_host" | cut -d' ' -f2-)" +tmpcmd=$* +for option in $options; do + # shellcheck disable=SC2046 + set -- $(echo "$option" | tr '=' ' ') + key="$1" + value="$2" + case "$1" in + "jail_host") + JAIL_HOST="$value" + ;; + "iocage_host") + IOCAGE_HOST="$value" + ;; + "jail_name") + JAIL_NAME="$value" + ;; + "sudo_user") + SUDO_USER="$value" + ;; + *) + log "unknown option $key=$value found" + ;; + esac +done +# shellcheck disable=SC2086 +set -- $tmpcmd + +if [ -n "$IOCAGE_HOST" ] && [ -n "$JAIL_HOST" ]; then + echo "WARING: jail_host and iocage_host can't be used at the same time" + log "WARING: jail_host and iocage_host can't be used at the same time" + exit 1 +fi + +TARGET_HOST="$__target_host" +SSH_USER="root" + +if [ -n "$IOCAGE_HOST" ]; then + JAIL_HOST="$IOCAGE_HOST" + + if [ -z "$JAIL_NAME" ]; then + JAIL_NAME="ioc-$(echo "$__target_host" | tr '.' '_')" + else + JAIL_NAME="ioc-$(echo "$JAIL_NAME" | tr '.' '_')" + fi +else + if [ -z "$JAIL_NAME" ]; then + JAIL_NAME="$TARGET_HOST" + fi +fi + +if [ -n "$JAIL_HOST" ]; then + log "INSIDE_JAIL: TRUE" + TARGET_HOST="$JAIL_HOST" + WRAPPER="jexec $JAIL_NAME" +fi + +if [ -n "$SUDO_USER" ]; then + log "SUDO_USER: $SUDO_USER" + WRAPPER="sudo -- $WRAPPER" + SSH_USER="$SUDO_USER" +fi + +log "TARGET_HOST: $TARGET_HOST" +log "@:" "$@" +if [ -n "$JAIL_HOST" ]; then + log "IOCAGE_HOST: $IOCAGE_HOST" + log "JAIL_HOST: $JAIL_HOST" + log "JAIL_NAME: $JAIL_NAME" + log "WRAPPER: $WRAPPER" +fi + +case "$COMMAND" in + "exec") + shift; # remove the jail host name from $@ + ssh -o User="$SSH_USER" -q "$TARGET_HOST" "$WRAPPER $*" + ;; + "copy") + if [ -n "$JAIL_HOST" ]; then + # jls(8) dosen't need root to print this information + jail_root=$(ssh -q "$TARGET_HOST" -- jls -j "$JAIL_NAME" path) + log "JAIL_ROOT: $jail_root" + fi + + if [ -n "$JAIL_HOST" ]; then + set -- "$(echo "$@" | sed "s|$__target_host:|$JAIL_HOST:$jail_root|g")" + fi + + if [ -n "$SUDO_USER" ]; then + # For rsync to do the right thing, the source has to end with "/" if it is + # a directory. The below preprocessor loop takes care of that. + + # second last argument is the source + source_index=$(($#-1)) + index=0 + for arg in "$@"; do + if [ $index -eq 0 ]; then + # reset $@ + set -- + fi + index=$((index+=1)) + if [ $index -eq $source_index ] && [ -d "$arg" ]; then + arg="${arg%/}/" + fi + set -- "$@" "$arg" + done + + rsync --copy-links -e "ssh -o User=$SSH_USER" --rsync-path='sudo rsync' "$@" + else + #shellcheck disable=SC2068 + scp -o "User=$SSH_USER" -q $@ + fi + ;; + *) + echo "unkown command - $COMMAND" + exit 1 + ;; +esac + +log "----"