From c51d68a7375915154a90c22e716e832f635ca878 Mon Sep 17 00:00:00 2001 From: Beni Ruef Date: Thu, 3 Dec 2020 18:48:04 +0100 Subject: [PATCH 001/128] [type/__postgres_conf] New type based on ALTER SYSTEM command --- .../conf/type/__postgres_conf/gencode-remote | 92 +++++++++++++++++++ cdist/conf/type/__postgres_conf/man.rst | 55 +++++++++++ .../__postgres_conf/parameter/default/state | 1 + .../type/__postgres_conf/parameter/optional | 2 + 4 files changed, 150 insertions(+) create mode 100755 cdist/conf/type/__postgres_conf/gencode-remote create mode 100644 cdist/conf/type/__postgres_conf/man.rst create mode 100644 cdist/conf/type/__postgres_conf/parameter/default/state create mode 100644 cdist/conf/type/__postgres_conf/parameter/optional diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote new file mode 100755 index 00000000..a09e1873 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -0,0 +1,92 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$(cat "${__global}/explorer/os") +state_should=$(cat "${__object}/parameter/state") + +if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ] +then + echo "Invalid state '${state_should}'." \ + 'Only "present" and "absent" are acceptable' >&2 + exit 1 +fi + +if [ "${state_should}" = 'present' ] && [ ! -f "${__object}/parameter/value" ] +then + echo 'Missing required parameter "value"' >&2 + exit 1 +fi + +# Parameters +conf_name="${__object_id}" +if [ -f "${__object}/parameter/value" ] +then + conf_value=$(cat "${__object}/parameter/value") +fi + +if [ "${state_should}" = 'present' ] +then + set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'" + check_command="SHOW ${conf_name}" +else + set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT" +fi + +case $os +in + openbsd|devuan) + case $os + in + openbsd) + postgres_user='_postgresql' + restart_command='/etc/rc.d/postgresql restart' + ;; + devuan) + postgres_user='postgres' + restart_command='/etc/init.d/postgresql restart' + ;; + esac + # needs two separate psql commands because ALTER SYSTEM + # cannot run inside a transaction block + cat <<-EOF + su - ${postgres_user} -c "psql postgres -twAc \ + \"${set_command}\"" + su - ${postgres_user} -c "psql postgres -twAc \ + \"SELECT pg_reload_conf()\"" + EOF + # check success (makes only sense if setting to a non-default value) + # and restart server if needed + if [ "${state_should}" = 'present' ] + then + cat <<-EOF + if [ \$(su - ${postgres_user} -c "psql postgres -twAc \"SHOW ${conf_name}\"") != '${conf_value}' ] + then + ${restart_command} + fi + EOF + fi + ;; + *) + echo "Unsupported OS: ${os}" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst new file mode 100644 index 00000000..76016c6c --- /dev/null +++ b/cdist/conf/type/__postgres_conf/man.rst @@ -0,0 +1,55 @@ +cdist-type__postgres_conf(7) +============================ + +Configure a PostgreSQL server. + +NOTE: This type might need to be run multiple times to apply all bits of the +configuration due to ordering requirements. + +SSRQ + + +DESCRIPTION +----------- +Configure a PostgreSQL server using ALTER SYSTEM. + + +REQUIRED PARAMETERS +------------------- +value + The value to setup (can be omitted when state is set to "absent"). + + +OPTIONAL PARAMETERS +------------------- +state + "present" or "absent". Defaults to "present". + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # set timezone + __postgres_conf timezone --value Europe/Zurich + + # reset maximum number of concurrent connections to default (normally 100) + __postgres_conf max_connections --state absent + + +SEE ALSO +-------- +- `cdist-type(7) `_ + + +COPYING +------- +Copyright \(C) 2020 SSRQ (www.ssrq-sds-fds.ch). +Free use of this software is granted under the terms +of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__postgres_conf/parameter/default/state b/cdist/conf/type/__postgres_conf/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__postgres_conf/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__postgres_conf/parameter/optional b/cdist/conf/type/__postgres_conf/parameter/optional new file mode 100644 index 00000000..d0460d86 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/parameter/optional @@ -0,0 +1,2 @@ +state +value From 534d5f6bb5dd3d8e9cb2256e75ed182d6530a892 Mon Sep 17 00:00:00 2001 From: Beni Ruef Date: Fri, 4 Dec 2020 14:31:04 +0100 Subject: [PATCH 002/128] [type/__postgres_conf] Fix errors found by ShellCheck --- cdist/conf/type/__postgres_conf/gencode-remote | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index a09e1873..e25514d0 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -20,8 +20,8 @@ # along with cdist. If not, see . # -os=$(cat "${__global}/explorer/os") -state_should=$(cat "${__object}/parameter/state") +os=$(cat "${__global:?}/explorer/os") +state_should=$(cat "${__object:?}/parameter/state") if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ] then @@ -37,7 +37,7 @@ then fi # Parameters -conf_name="${__object_id}" +conf_name="${__object_id:?}" if [ -f "${__object}/parameter/value" ] then conf_value=$(cat "${__object}/parameter/value") @@ -46,7 +46,6 @@ fi if [ "${state_should}" = 'present' ] then set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'" - check_command="SHOW ${conf_name}" else set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT" fi From 50bcd951055d7599ec340b4dea80a3a5d11dd9b0 Mon Sep 17 00:00:00 2001 From: Beni Ruef Date: Thu, 10 Dec 2020 11:45:32 +0100 Subject: [PATCH 003/128] [type/__postgres_conf] Remove faulty quotes --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index e25514d0..8ccb3b42 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -45,7 +45,7 @@ fi if [ "${state_should}" = 'present' ] then - set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'" + set_command="ALTER SYSTEM SET ${conf_name} = ${conf_value}" else set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT" fi From b4060720dc3c7cc58aa992285877e4c009caa6be Mon Sep 17 00:00:00 2001 From: Beni Ruef Date: Fri, 11 Dec 2020 11:12:16 +0100 Subject: [PATCH 004/128] [type/__postgres_conf] Fix psql options for ALTER command --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 8ccb3b42..7d86028e 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -67,7 +67,7 @@ in # needs two separate psql commands because ALTER SYSTEM # cannot run inside a transaction block cat <<-EOF - su - ${postgres_user} -c "psql postgres -twAc \ + su - ${postgres_user} -c "psql postgres -qwc \ \"${set_command}\"" su - ${postgres_user} -c "psql postgres -twAc \ \"SELECT pg_reload_conf()\"" From 1b49fec972ce5f486f2794571c8e892d1fa6cb51 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 18 Jan 2021 19:15:49 +0100 Subject: [PATCH 005/128] [type/__postgres_conf] Refactor --- .../type/__postgres_conf/explorer/context | 42 ++++++ .../conf/type/__postgres_conf/explorer/state | 60 ++++++++ .../conf/type/__postgres_conf/gencode-remote | 135 ++++++++++-------- 3 files changed, 180 insertions(+), 57 deletions(-) create mode 100644 cdist/conf/type/__postgres_conf/explorer/context create mode 100644 cdist/conf/type/__postgres_conf/explorer/state diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context new file mode 100644 index 00000000..3a2fa504 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/explorer/context @@ -0,0 +1,42 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Returns the "context" of the configuration setting. +# cf. also https://www.postgresql.org/docs/10/view-pg-settings.html + +os=$("${__explorer:?}/os") + +case ${os} +in + (openbsd) + postgres_user='_postgresql' + ;; + (devuan) + postgres_user='postgres' + ;; + (*) + echo "Unsupported OS: ${os}" >&2 + exit 1 + ;; +esac + +conf_name=${__object_id:?} + +su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE name = '${conf_name}'\"" diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state new file mode 100644 index 00000000..da904b56 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -0,0 +1,60 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$("${__explorer:?}/os") + +case ${os} +in + (openbsd) + postgres_user='_postgresql' + ;; + (devuan) + postgres_user='postgres' + ;; + (*) + echo "Unsupported OS: ${os}" >&2 + exit 1 + ;; +esac + +conf_name=${__object_id:?} + +if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ + | cmp -s "${__object:?}/parameter/value" - +then + echo present +else + case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT source FROM pg_settings WHERE name = '${conf_name}'\"") + in + ('') + # invalid configuration parameter + # (error message was already printed by SHOW command above. + # Yes, it's a hack) + exit 1 + ;; + (default) + echo absent + ;; + (*) + echo different + ;; + esac +fi diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 7d86028e..998b8582 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -1,7 +1,7 @@ #!/bin/sh -e # -*- mode: sh; indent-tabs-mode: t -*- # -# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# 2019-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch) # # This file is part of cdist. @@ -21,71 +21,92 @@ # os=$(cat "${__global:?}/explorer/os") +state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") -if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ] +conf_name=${__object_id:?} + +if test "${state_is}" = "${state_should}" then - echo "Invalid state '${state_should}'." \ - 'Only "present" and "absent" are acceptable' >&2 - exit 1 + exit 0 fi -if [ "${state_should}" = 'present' ] && [ ! -f "${__object}/parameter/value" ] -then - echo 'Missing required parameter "value"' >&2 - exit 1 -fi - -# Parameters -conf_name="${__object_id:?}" -if [ -f "${__object}/parameter/value" ] -then - conf_value=$(cat "${__object}/parameter/value") -fi - -if [ "${state_should}" = 'present' ] -then - set_command="ALTER SYSTEM SET ${conf_name} = ${conf_value}" -else - set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT" -fi - -case $os -in - openbsd|devuan) - case $os - in - openbsd) - postgres_user='_postgresql' - restart_command='/etc/rc.d/postgresql restart' - ;; - devuan) - postgres_user='postgres' - restart_command='/etc/init.d/postgresql restart' - ;; - esac - # needs two separate psql commands because ALTER SYSTEM - # cannot run inside a transaction block - cat <<-EOF - su - ${postgres_user} -c "psql postgres -qwc \ - \"${set_command}\"" - su - ${postgres_user} -c "psql postgres -twAc \ - \"SELECT pg_reload_conf()\"" - EOF - # check success (makes only sense if setting to a non-default value) - # and restart server if needed - if [ "${state_should}" = 'present' ] +quote() { + for _arg + do + shift + if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')" then - cat <<-EOF - if [ \$(su - ${postgres_user} -c "psql postgres -twAc \"SHOW ${conf_name}\"") != '${conf_value}' ] - then - ${restart_command} - fi - EOF + # needs quoting + set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'" + else + set -- "$@" "${_arg}" fi + done + unset _arg + + # NOTE: Use printf because POSIX echo interprets escape sequences + printf '%s' "$*" +} + +case ${os} +in + (openbsd) + postgres_user='_postgresql' + restart_command='/etc/rc.d/postgresql restart' ;; - *) + (devuan) + postgres_user='postgres' + restart_command='/etc/init.d/postgresql restart' + ;; + (*) echo "Unsupported OS: ${os}" >&2 exit 1 ;; esac + + +psql_cmd() { + printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")" +} + +case ${state_should} +in + (present) + test -s "${__object:?}/parameter/value" || { + echo 'Missing required parameter --value' >&2 + exit 1 + } + + cat <<-EOF + exec 3< "\${__object:?}/parameter/value" + $(psql_cmd postgres -tAw) <<'SQL' + \\set conf_value \`cat <&3\` + ALTER SYSTEM SET ${conf_name} = :'conf_value'; + SELECT pg_reload_conf(); + SQL + exec 3<&- + EOF + ;; + (absent) + psql_cmd postgres -qwc "ALTER SYSTEM SET ${conf_name} TO DEFAULT" + ;; + (*) + printf 'Invalid --state: %s\n' "${state_should}" >&2 + printf 'Only "present" and "absent" are acceptable.\n' >&2 + exit 1 + ;; +esac + +# check success (makes only sense if setting to a non-default value) +# and restart server if needed +if test "${state_should}" = 'present' +then + cat <<-EOF + + $(psql_cmd postgres -twAc "SHOW ${conf_name}") \\ + | cmp -s "\${__object:?}/parameter/value" - || { + ${restart_command} + } + EOF +fi From 803367b316f5c72c51153172a41b7a745f2d2d83 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 18 Jan 2021 19:29:42 +0100 Subject: [PATCH 006/128] [type/__postgres_conf] Fix default detection when default is also set in config file e.g. port is usually also set to the default value in postgresql.conf --- cdist/conf/type/__postgres_conf/explorer/state | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index da904b56..400b27e8 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -42,7 +42,7 @@ if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ then echo present else - case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT source FROM pg_settings WHERE name = '${conf_name}'\"") + case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE name = '${conf_name}'\"") in ('') # invalid configuration parameter From 891c98567e1d94577dc4f20d3ec9f73f4927fd24 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 18 Jan 2021 19:39:56 +0100 Subject: [PATCH 007/128] [type/__postgres_conf] Compare configuration parameter names case insensitively --- cdist/conf/type/__postgres_conf/explorer/context | 3 ++- cdist/conf/type/__postgres_conf/explorer/state | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context index 3a2fa504..def5f676 100644 --- a/cdist/conf/type/__postgres_conf/explorer/context +++ b/cdist/conf/type/__postgres_conf/explorer/context @@ -39,4 +39,5 @@ esac conf_name=${__object_id:?} -su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE name = '${conf_name}'\"" +# NOTE: SHOW/SET are case-insentitive, so this command should also be. +su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"" diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index 400b27e8..a4930296 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -42,7 +42,8 @@ if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ then echo present else - case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE name = '${conf_name}'\"") + # NOTE: SHOW/SET are case-insentitive, so this command should also be. + case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"") in ('') # invalid configuration parameter From 5051d4f40b7097f58431ce334f9ddf5b171dbc7f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 19 Jan 2021 16:36:44 +0100 Subject: [PATCH 008/128] [type/__postgres_conf] Catch invalid values --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 998b8582..cc9b4e99 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -80,7 +80,7 @@ in cat <<-EOF exec 3< "\${__object:?}/parameter/value" - $(psql_cmd postgres -tAw) <<'SQL' + $(psql_cmd postgres -tAw -v ON_ERROR_STOP=on) <<'SQL' \\set conf_value \`cat <&3\` ALTER SYSTEM SET ${conf_name} = :'conf_value'; SELECT pg_reload_conf(); From 0f2ff477381d67de27149a92744f873322e44fd7 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 19 Jan 2021 16:37:43 +0100 Subject: [PATCH 009/128] [type/__postgres_conf] Restart PostgreSQL server based on pending_restart column of pg_settings --- .../type/__postgres_conf/explorer/context | 43 ------------------- .../conf/type/__postgres_conf/gencode-remote | 16 +++---- 2 files changed, 6 insertions(+), 53 deletions(-) delete mode 100644 cdist/conf/type/__postgres_conf/explorer/context diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context deleted file mode 100644 index def5f676..00000000 --- a/cdist/conf/type/__postgres_conf/explorer/context +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -e -# -*- mode: sh; indent-tabs-mode: t -*- -# -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# Returns the "context" of the configuration setting. -# cf. also https://www.postgresql.org/docs/10/view-pg-settings.html - -os=$("${__explorer:?}/os") - -case ${os} -in - (openbsd) - postgres_user='_postgresql' - ;; - (devuan) - postgres_user='postgres' - ;; - (*) - echo "Unsupported OS: ${os}" >&2 - exit 1 - ;; -esac - -conf_name=${__object_id:?} - -# NOTE: SHOW/SET are case-insentitive, so this command should also be. -su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"" diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index cc9b4e99..fa930cc4 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -98,15 +98,11 @@ in ;; esac -# check success (makes only sense if setting to a non-default value) -# and restart server if needed -if test "${state_should}" = 'present' -then - cat <<-EOF +# Restart PostgreSQL server if required to apply new configuration value +cat < Date: Tue, 19 Jan 2021 17:18:27 +0100 Subject: [PATCH 010/128] [type/__postgres_conf] Add support for more init systems to restart service --- .../conf/type/__postgres_conf/gencode-remote | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index fa930cc4..15d5bb3e 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -53,11 +53,9 @@ case ${os} in (openbsd) postgres_user='_postgresql' - restart_command='/etc/rc.d/postgresql restart' ;; (devuan) postgres_user='postgres' - restart_command='/etc/init.d/postgresql restart' ;; (*) echo "Unsupported OS: ${os}" >&2 @@ -103,6 +101,37 @@ cat <&2 + exit 1 + esac + ;; + (*) + printf "Don't know how to restart services with your init (%s)\n" "${init}" >&2 + exit 1 + esac + ) fi EOF From 4967c7ebbb7e98af6aa2c7f8bc91511bcbbf9820 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 19 Jan 2021 17:20:27 +0100 Subject: [PATCH 011/128] [type/__postgres_conf] Silence psql output --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 15d5bb3e..9b8227ee 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -78,7 +78,7 @@ in cat <<-EOF exec 3< "\${__object:?}/parameter/value" - $(psql_cmd postgres -tAw -v ON_ERROR_STOP=on) <<'SQL' + $(psql_cmd postgres -tAwq -o /dev/null -v ON_ERROR_STOP=on) <<'SQL' \\set conf_value \`cat <&3\` ALTER SYSTEM SET ${conf_name} = :'conf_value'; SELECT pg_reload_conf(); From f9ebb4333c85e41439635cb00a40c3e07d8a3683 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 19 Jan 2021 17:21:50 +0100 Subject: [PATCH 012/128] [type/__postgres_conf] Add NetBSD PostgreSQL UNIX user --- cdist/conf/type/__postgres_conf/gencode-remote | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 9b8227ee..8f8a2bfe 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -51,15 +51,14 @@ quote() { case ${os} in + (netbsd) + postgres_user='pgsql' + ;; (openbsd) postgres_user='_postgresql' ;; - (devuan) - postgres_user='postgres' - ;; (*) - echo "Unsupported OS: ${os}" >&2 - exit 1 + postgres_user='postgres' ;; esac From 6b18cace759d1345cec6c2168c3bd8f238f23bca Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 21 Jan 2021 19:29:07 +0100 Subject: [PATCH 013/128] [type/__postgres_conf] Catch connection errors early --- cdist/conf/type/__postgres_conf/explorer/state | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index a4930296..589925de 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -37,6 +37,11 @@ esac conf_name=${__object_id:?} +su - "${postgres_user}" -c 'psql postgres -c "SELECT 1"' >/dev/null || { + echo 'Connection to PostgreSQL server failed' >&2 + exit 1 +} + if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ | cmp -s "${__object:?}/parameter/value" - then From 0835f414a5b2ce41309ee9b265cb80abb1362563 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 16 Feb 2021 16:03:23 +0100 Subject: [PATCH 014/128] [type/__postgres_conf] Extract PostgreSQL service user detection to separate explorer --- .../__postgres_conf/explorer/postgres_user | 64 +++++++++++++++++++ .../conf/type/__postgres_conf/explorer/state | 16 +---- .../conf/type/__postgres_conf/gencode-remote | 17 +---- 3 files changed, 67 insertions(+), 30 deletions(-) create mode 100644 cdist/conf/type/__postgres_conf/explorer/postgres_user diff --git a/cdist/conf/type/__postgres_conf/explorer/postgres_user b/cdist/conf/type/__postgres_conf/explorer/postgres_user new file mode 100644 index 00000000..c6582dc4 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/explorer/postgres_user @@ -0,0 +1,64 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$("${__explorer:?}/os") + +case ${os} +in + (alpine) + echo 'postgres' + ;; + (centos|rhel|scientific) + echo 'postgres' + ;; + (debian|devuan|ubuntu) + echo 'postgres' + ;; + (freebsd) + test -x /usr/local/etc/rc.d/postgresql || { + printf 'could not find postgresql rc script./n' >&2 + exit 1 + } + pg_status=$(/usr/local/etc/rc.d/postgresql onestatus) || { + printf 'postgresql daemon is not running.\n' >&2 + exit 1 + } + pg_pid=$(printf '%s\n' "${pg_status}" \ + | sed -n 's/^pg_ctl:.*(PID: *\([0-9]*\))$/\1/p') + + # PostgreSQL < 9.6: pgsql + # PostgreSQL >= 9.6: postgres + ps -o user -p "${pg_pid}" | sed -n '2p' + ;; + (netbsd) + echo 'pgsql' + ;; + (openbsd) + echo '_postgresql' + ;; + (suse) + echo 'postgres' + ;; + (*) + echo "Unsupported OS: ${os}" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index 589925de..de6e8aa0 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -19,21 +19,7 @@ # along with cdist. If not, see . # -os=$("${__explorer:?}/os") - -case ${os} -in - (openbsd) - postgres_user='_postgresql' - ;; - (devuan) - postgres_user='postgres' - ;; - (*) - echo "Unsupported OS: ${os}" >&2 - exit 1 - ;; -esac +postgres_user=$("${__type_explorer:?}/postgres_user") conf_name=${__object_id:?} diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 8f8a2bfe..d0d247b4 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -20,9 +20,9 @@ # along with cdist. If not, see . # -os=$(cat "${__global:?}/explorer/os") state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") +postgres_user=$(cat "${__object:?}/explorer/postgres_user") conf_name=${__object_id:?} @@ -49,19 +49,6 @@ quote() { printf '%s' "$*" } -case ${os} -in - (netbsd) - postgres_user='pgsql' - ;; - (openbsd) - postgres_user='_postgresql' - ;; - (*) - postgres_user='postgres' - ;; -esac - psql_cmd() { printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")" @@ -117,7 +104,7 @@ then case $(cat "${__global:?}/explorer/kernel_name") in (FreeBSD) - echo 'service postgresql restart' + echo '/usr/local/etc/rc.d/postgresql restart' ;; (OpenBSD|NetBSD) echo '/etc/rc.d/postgresql restart' From 0734288483700e8e10cebd87c797b625aa83d55e Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Sun, 21 Feb 2021 19:59:57 +0000 Subject: [PATCH 015/128] First draft of __apt_pin --- cdist/conf/type/__apt_pin/man.rst | 53 ++++++++++++++++ cdist/conf/type/__apt_pin/manifest | 63 +++++++++++++++++++ .../type/__apt_pin/parameter/default/package | 1 + .../type/__apt_pin/parameter/default/state | 1 + cdist/conf/type/__apt_pin/parameter/optional | 2 + cdist/conf/type/__apt_pin/parameter/required | 2 + 6 files changed, 122 insertions(+) create mode 100644 cdist/conf/type/__apt_pin/man.rst create mode 100755 cdist/conf/type/__apt_pin/manifest create mode 100644 cdist/conf/type/__apt_pin/parameter/default/package create mode 100644 cdist/conf/type/__apt_pin/parameter/default/state create mode 100644 cdist/conf/type/__apt_pin/parameter/optional create mode 100644 cdist/conf/type/__apt_pin/parameter/required diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst new file mode 100644 index 00000000..7fcae6f8 --- /dev/null +++ b/cdist/conf/type/__apt_pin/man.rst @@ -0,0 +1,53 @@ +cdist-type__apt_pin(7) +====================== + +NAME +---- +cdist-type__apt_pin - TODO + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # TODO + __apt_pin + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Daniel Fancsali + + +COPYING +------- +Copyright \(C) 2021 Daniel Fancsali. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest new file mode 100755 index 00000000..8dd9770d --- /dev/null +++ b/cdist/conf/type/__apt_pin/manifest @@ -0,0 +1,63 @@ +#!/bin/sh -e +# +# 2021 Daniel Fancsali (fancsali@gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") +state="$(cat "$__object/parameter/state")" +package="$(cat "$__object/parameter/package")" +distribution="$(cat "$__object/parameter/distribution")" +priority="$(cat "$__object/parameter/priority")" + + +case "$os" in + debian|ubuntu|devuan) + ;; + *) + printf "This type is specific to Debian and it's derivatives" >&2 + printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2 + exit 1 + ;; +esac + +if [ "$package" = "*" ]; then + name="default" + +else + name="$__object_id" +fi + +case $distribution in + stabletesting|unsatbel|experimental) + pin="release a=$distribution" + ;; + *) + pin="release n=$distribution" + ;; +esac + + +__file /etc/apt/preferences.d/$name \ + --owner root --group root --mode 0644 \ + --state "$state" \ + --source - << EOF +Package: $package +Pin: $pin +Pin-Priority: $priority +EOF diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package new file mode 100644 index 00000000..72e8ffc0 --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/default/package @@ -0,0 +1 @@ +* diff --git a/cdist/conf/type/__apt_pin/parameter/default/state b/cdist/conf/type/__apt_pin/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional new file mode 100644 index 00000000..52f01fd2 --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/optional @@ -0,0 +1,2 @@ +state +package diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required new file mode 100644 index 00000000..4b4e9741 --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/required @@ -0,0 +1,2 @@ +distribution +priority From 1a74470c4d9b30e41a72fbfc084dc54ea44e643b Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Tue, 23 Feb 2021 09:37:36 +0000 Subject: [PATCH 016/128] __apt_pin: Always use $__object_id as preferences.d filename --- cdist/conf/type/__apt_pin/manifest | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index 8dd9770d..162b523f 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -36,12 +36,7 @@ case "$os" in ;; esac -if [ "$package" = "*" ]; then - name="default" - -else - name="$__object_id" -fi +name="$__object_id" case $distribution in stabletesting|unsatbel|experimental) From dc66efa690e15ff32d6836e23baa1cbec5eee1ed Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Tue, 23 Feb 2021 11:59:09 +0000 Subject: [PATCH 017/128] Fix shellcheck issues --- cdist/conf/type/__apt_pin/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index 162b523f..b1372ad0 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -48,7 +48,7 @@ case $distribution in esac -__file /etc/apt/preferences.d/$name \ +__file "/etc/apt/preferences.d/$name" \ --owner root --group root --mode 0644 \ --state "$state" \ --source - << EOF From 60fd7ba1f38382761c366cbc0425ddaec62f1164 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 28 Feb 2021 13:37:23 +0100 Subject: [PATCH 018/128] Release 6.9.5 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 88dda0aa..95e36b88 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) * Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera) * Explorer memory: Fix result units; support Solaris (Dennis Camera) From 8ef19d47f6b5fb6cdcba7a9a3b8890388dbd7023 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 1 Mar 2021 17:59:25 +0100 Subject: [PATCH 019/128] [type/__pyvenv] Fix example (--user -> --owner) --- cdist/conf/type/__pyvenv/man.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__pyvenv/man.rst b/cdist/conf/type/__pyvenv/man.rst index 8085ff12..e2e4a1e6 100644 --- a/cdist/conf/type/__pyvenv/man.rst +++ b/cdist/conf/type/__pyvenv/man.rst @@ -61,7 +61,7 @@ EXAMPLES __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4 # Create python virtualenv for user foo. - __pyvenv /home/foo/fooenv --group foo --user foo + __pyvenv /home/foo/fooenv --group foo --owner foo # Create python virtualenv with specific parameters. __pyvenv /home/services/djangoenv --venvparams "--copies --system-site-packages" From e7d33891df945831d54d164eb13f2a9ae4f9dd8e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 2 Mar 2021 09:29:33 +0100 Subject: [PATCH 020/128] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 95e36b88..072fcb6b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __pyvenv: Fix user example in man page (Dennis Camera) + 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) * Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera) From ea0126dd8117c051612e6b6abfe895f5f722c584 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 5 Mar 2021 16:11:49 +0100 Subject: [PATCH 021/128] Make local state dir available to custom remote scripts Signed-off-by: Steven Armstrong --- cdist/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cdist/config.py b/cdist/config.py index e84f6f84..19d5bd70 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -420,6 +420,9 @@ class Config: exec_path=sys.argv[0], save_output_streams=args.save_output_streams) + # Make __global state dir available to custom remote scripts. + os.environ['__global'] = local.base_path + remote = cdist.exec.remote.Remote( target_host=target_host, remote_exec=remote_exec, From ecba284fc8ce68486c5f1f87a863409fcad0f106 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 5 Mar 2021 16:13:02 +0100 Subject: [PATCH 022/128] changelog++ Signed-off-by: Steven Armstrong --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 072fcb6b..ad93a595 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Type __pyvenv: Fix user example in man page (Dennis Camera) + * Core: config: Make local state directory available to custom remotes (Steven Armstrong 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) From fb19f342669d18e6decb21ee4cfb2b22e46748d9 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 9 Mar 2021 21:15:26 +0100 Subject: [PATCH 023/128] [type/__ssh_authorized_key] Only grep if file exists --- cdist/conf/type/__ssh_authorized_key/explorer/entry | 1 + cdist/conf/type/__ssh_authorized_key/gencode-remote | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__ssh_authorized_key/explorer/entry b/cdist/conf/type/__ssh_authorized_key/explorer/entry index ccab0afc..aca0f2b9 100755 --- a/cdist/conf/type/__ssh_authorized_key/explorer/entry +++ b/cdist/conf/type/__ssh_authorized_key/explorer/entry @@ -25,6 +25,7 @@ type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ] if [ -n "${type_and_key}" ] then file="$(cat "$__object/parameter/file")" + test -e "$file" || exit 0 # get any entries that match the type and key diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index f37aa565..61c77fb9 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -37,9 +37,9 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX) # preserve ownership and permissions of existing file if [ -f "$file" ]; then cp -p "$file" "\$tmpfile" + grep -v -F -x '$line' '$file' >\$tmpfile fi -grep -v -F -x '$line' '$file' > \$tmpfile || true -mv -f "\$tmpfile" "$file" +cat "\$tmpfile" >"$file" DONE } From 31cc592aa10902c3e83357ee4a0b9300b5e34efa Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 10 Mar 2021 19:25:04 +0100 Subject: [PATCH 024/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index ad93a595..c63b901a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __pyvenv: Fix user example in man page (Dennis Camera) * Core: config: Make local state directory available to custom remotes (Steven Armstrong + * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) From e47c4dd8a4cc441b94fe73d362d79525d01a4bb1 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 9 Mar 2021 19:59:05 +0100 Subject: [PATCH 025/128] [type/__sshd_config] Whitelist OpenBMC in manifest --- cdist/conf/type/__sshd_config/manifest | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest index 566bde90..e37afebb 100755 --- a/cdist/conf/type/__sshd_config/manifest +++ b/cdist/conf/type/__sshd_config/manifest @@ -39,7 +39,14 @@ in (freebsd|netbsd|openbsd) # whitelist ;; + (openbmc-phosphor) + # whitelist + # OpenBMC can be configured with dropbear and OpenSSH. + # If dropbear is used, the state explorer will already fail because it + # cannot find the sshd binary. + ;; (*) + : "${__type:?}" # make shellcheck happy printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \ "${os}" "${__type##*/}" >&2 printf 'Please contribute an implementation for it if you can.\n' >&2 From 10ca1c12fd25a9f2b92cebbc5d375a1723ed5e2f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 12 Mar 2021 08:21:03 +0100 Subject: [PATCH 026/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index c63b901a..42a74d04 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * Type __pyvenv: Fix user example in man page (Dennis Camera) * Core: config: Make local state directory available to custom remotes (Steven Armstrong * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) + * Type __sshd_config: Whitelist OpenBMC (Dennis Camera) 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) From 7a0b697f4c394f1ae8a8941a59937007cfe474aa Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 21 Mar 2021 18:10:06 +0100 Subject: [PATCH 027/128] Implement maintaining object relationship graph For each object maintain parent-child relationship graph, i.e. list of parent objects ('parents' property) and list of children objects ('children' property). Objects without parent(s) are objects specified in init manifest. Objects without children are object of types that do not reuse other types. --- cdist/core/cdist_object.py | 7 +++++++ cdist/emulator.py | 19 +++++++++++++++++++ docs/src/cdist-cache.rst | 17 +++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 51d61e04..eea2c255 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -247,6 +247,13 @@ class CdistObject: lambda obj: os.path.join(obj.absolute_path, 'typeorder')) typeorder_dep = fsproperty.FileListProperty( lambda obj: os.path.join(obj.absolute_path, 'typeorder_dep')) + # objects without parents are objects specified in init manifest + parents = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, 'parents')) + # objects without children are object of types that do not reuse other + # types + children = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, 'children')) def cleanup(self): try: diff --git a/cdist/emulator.py b/cdist/emulator.py index a2bdc3d4..f1db862e 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -106,6 +106,7 @@ class Emulator: self.save_stdin() self.record_requirements() self.record_auto_requirements() + self.record_parent_child_relationships() self.log.trace("Finished %s %s" % ( self.cdist_object.path, self.parameters)) @@ -420,3 +421,21 @@ class Emulator: self.log.debug("Recording autorequirement %s for %s", current_object.name, parent.name) parent.autorequire.append(current_object.name) + + def record_parent_child_relationships(self): + # __object_name is the name of the object whose type manifest is + # currently executed + __object_name = self.env.get('__object_name', None) + if __object_name: + # The object whose type manifest is currently run + parent = self.cdist_object.object_from_name(__object_name) + # The object currently being defined + current_object = self.cdist_object + if current_object.name not in parent.children: + self.log.debug("Recording child %s for %s", + current_object.name, parent.name) + parent.children.append(current_object.name) + if parent.name not in current_object.parents: + self.log.debug("Recording parent %s for %s", + parent.name, current_object.name) + current_object.parents.append(parent.name) diff --git a/docs/src/cdist-cache.rst b/docs/src/cdist-cache.rst index d2d2d56c..d4159e77 100644 --- a/docs/src/cdist-cache.rst +++ b/docs/src/cdist-cache.rst @@ -61,6 +61,14 @@ Object cache overview ~~~~~~~~~~~~~~~~~~~~~ Each object under :strong:`object` directory has its own structure. +autorequire + file containing a list of object auto requirements + +children + file containing a list of object children, i.e. objects of types that this + type reuses (along with 'parents' it is used for maintaining parent-child + relationship graph) + code-local code generated from gencode-local, present only if something is generated @@ -80,6 +88,15 @@ parameter directory containing type parameter named files containing parameter values +parents + file containing a list of object parents, i.e. objects of types that reuse + this type (along with 'children' it is used for maintaining parent-child + relationship graph); objects without parents are objects specified in init + manifest + +require + file containing a list of object requirements + source this type's source (init manifest) From 167c2ad7ea2b895cdc7d89c72968d88c7d1ab34f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 30 Mar 2021 13:24:56 +0200 Subject: [PATCH 028/128] [type/__git] Fix if --owner / --group is numeric Before, if --owner and/or --group was numeric, gencode-remote would generate `chown` code every time. --- cdist/conf/type/__git/explorer/group | 20 +++++++++++++++++--- cdist/conf/type/__git/explorer/owner | 20 +++++++++++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group index 3ddf9656..1365c60d 100644 --- a/cdist/conf/type/__git/explorer/group +++ b/cdist/conf/type/__git/explorer/group @@ -1,5 +1,19 @@ -#!/bin/sh +#!/bin/sh -e -destination="/$__object_id/.git" +destination="/${__object_id:?}/.git" -stat --print "%G" "${destination}" 2>/dev/null || exit 0 +# shellcheck disable=SC2012 +group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }') + +# NOTE: +1 because $((notanum)) prints 0. +if test $((group_gid + 1)) -ge 0 +then + group_should=$(cat "${__object:?}/parameter/group") + + if expr "${group_should}" : '[0-9]*$' >/dev/null + then + printf '%u\n' "${group_gid}" + else + printf '%s\n' "$(id -u -n "${group_gid}")" + fi +fi diff --git a/cdist/conf/type/__git/explorer/owner b/cdist/conf/type/__git/explorer/owner index 4c3cd431..4a4d0d13 100644 --- a/cdist/conf/type/__git/explorer/owner +++ b/cdist/conf/type/__git/explorer/owner @@ -1,5 +1,19 @@ -#!/bin/sh +#!/bin/sh -e -destination="/$__object_id/.git" +destination="/${__object_id:?}/.git" -stat --print "%U" "${destination}" 2>/dev/null || exit 0 +# shellcheck disable=SC2012 +owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }') + +# NOTE: +1 because $((notanum)) prints 0. +if test $((owner_uid + 1)) -ge 0 +then + owner_should=$(cat "${__object:?}/parameter/owner") + + if expr "${owner_should}" : '[0-9]*$' >/dev/null + then + printf '%u\n' "${owner_uid}" + else + printf '%s\n' "$(id -u -n "${owner_uid}")" + fi +fi From 985252585ce83605787d2612fdc2a84b9788d943 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 30 Mar 2021 13:26:21 +0200 Subject: [PATCH 029/128] [type/__pyvenv] Fix if --owner / --group is numeric Before, if --owner and/or --group was numeric, gencode-remote would generate `chown` code every time. --- cdist/conf/type/__pyvenv/explorer/group | 20 +++++++++++++++++--- cdist/conf/type/__pyvenv/explorer/owner | 20 +++++++++++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group index a655bda7..f31a1cb7 100755 --- a/cdist/conf/type/__pyvenv/explorer/group +++ b/cdist/conf/type/__pyvenv/explorer/group @@ -1,5 +1,19 @@ -#!/bin/sh +#!/bin/sh -e -destination="/$__object_id" +destination="/${__object_id:?}" -stat --print "%G" "${destination}" 2>/dev/null || exit 0 +# shellcheck disable=SC2012 +group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }') + +# NOTE: +1 because $((notanum)) prints 0. +if test $((group_gid + 1)) -ge 0 +then + group_should=$(cat "${__object:?}/parameter/group") + + if expr "${group_should}" : '[0-9]*$' >/dev/null + then + printf '%u\n' "${group_gid}" + else + printf '%s\n' "$(id -u -n "${group_gid}")" + fi +fi diff --git a/cdist/conf/type/__pyvenv/explorer/owner b/cdist/conf/type/__pyvenv/explorer/owner index 8b3c7f8e..ebec751f 100755 --- a/cdist/conf/type/__pyvenv/explorer/owner +++ b/cdist/conf/type/__pyvenv/explorer/owner @@ -1,5 +1,19 @@ -#!/bin/sh +#!/bin/sh -e -destination="/$__object_id" +destination="/${__object_id:?}" -stat --print "%U" "${destination}" 2>/dev/null || exit 0 +# shellcheck disable=SC2012 +owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }') + +# NOTE: +1 because $((notanum)) prints 0. +if test $((owner_uid + 1)) -ge 0 +then + owner_should=$(cat "${__object:?}/parameter/owner") + + if expr "${owner_should}" : '[0-9]*$' >/dev/null + then + printf '%u\n' "${owner_uid}" + else + printf '%s\n' "$(id -u -n "${owner_uid}")" + fi +fi From 1e765fcab7bae02d6b380ca02e7cb04ec69ebdc5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 31 Mar 2021 07:54:00 +0200 Subject: [PATCH 030/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 42a74d04..03ae7566 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * Core: config: Make local state directory available to custom remotes (Steven Armstrong * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) * Type __sshd_config: Whitelist OpenBMC (Dennis Camera) + * Core: Maintain object relationship graph in cdist cache (Darko Poljak) 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) From f984a918b959a46e49fe8456a327eca8d44313de Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 30 Mar 2021 07:56:38 +0200 Subject: [PATCH 031/128] Fix log message string formatting Use logging message format with args, instead of direct `%` or `str.format`. Resolve #855. --- cdist/argparse.py | 8 +-- cdist/config.py | 87 ++++++++++++-------------- cdist/core/cdist_type.py | 6 +- cdist/core/explorer.py | 26 ++++---- cdist/core/manifest.py | 2 +- cdist/emulator.py | 37 +++++------ cdist/exec/local.py | 16 +++-- cdist/exec/remote.py | 14 ++--- cdist/install.py | 2 +- cdist/inventory.py | 36 +++++------ cdist/preos.py | 6 +- cdist/preos/debootstrap/debootstrap.py | 41 ++++++------ cdist/scan/scan.py | 4 +- cdist/util/ipaddr.py | 6 +- 14 files changed, 140 insertions(+), 151 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 88759d7b..cadac39a 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -533,10 +533,10 @@ def parse_and_configure(argv, singleton=True): log = logging.getLogger("cdist") - log.verbose("version %s" % cdist.VERSION) - log.trace('command line args: {}'.format(cfg.command_line_args)) - log.trace('configuration: {}'.format(cfg.get_config())) - log.trace('configured args: {}'.format(args)) + log.verbose("version %s", cdist.VERSION) + log.trace('command line args: %s', cfg.command_line_args) + log.trace('configuration: %s', cfg.get_config()) + log.trace('configured args: %s', args) check_beta(vars(args)) diff --git a/cdist/config.py b/cdist/config.py index 19d5bd70..d6fec55f 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -273,15 +273,15 @@ class Config: host_tags = None host_base_path, hostdir = cls.create_host_base_dirs( host, base_root_path) - log.debug("Base root path for target host \"{}\" is \"{}\"".format( - host, host_base_path)) + log.debug("Base root path for target host \"%s\" is \"%s\"", + host, host_base_path) hostcnt += 1 if args.parallel: pargs = (host, host_tags, host_base_path, hostdir, args, True, configuration) - log.trace(("Args for multiprocessing operation " - "for host {}: {}".format(host, pargs))) + log.trace("Args for multiprocessing operation for host %s: %s", + host, pargs) process_args.append(pargs) else: try: @@ -298,10 +298,10 @@ class Config: except cdist.Error: failed_hosts.append(host) elif args.parallel: - log.trace("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - log.trace(("Starting multiprocessing Pool for {} " - "parallel host operation".format(args.parallel))) + log.trace("Multiprocessing start method is %s", + multiprocessing.get_start_method()) + log.trace("Starting multiprocessing Pool for %d parallel host" + " operation", args.parallel) results = mp_pool_run(cls.onehost, process_args, @@ -396,16 +396,13 @@ class Config: remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds( args) - log.debug("remote_exec for host \"{}\": {}".format( - host, remote_exec)) - log.debug("remote_copy for host \"{}\": {}".format( - host, remote_copy)) + log.debug("remote_exec for host \"%s\": %s", host, remote_exec) + log.debug("remote_copy for host \"%s\": %s", host, remote_copy) family = cls._address_family(args) - log.debug("address family: {}".format(family)) + log.debug("address family: %s", family) target_host = cls.resolve_target_addresses(host, family) - log.debug("target_host for host \"{}\": {}".format( - host, target_host)) + log.debug("target_host for host \"%s\": %s", host, target_host) local = cdist.exec.local.Local( target_host=target_host, @@ -474,8 +471,8 @@ class Config: """Do what is most often done: deploy & cleanup""" start_time = time.time() - self.log.info("Starting {} run".format( - 'dry' if self.dry_run else 'configuration')) + self.log.info("Starting %s run", + 'dry' if self.dry_run else 'configuration') self._init_files_dirs() @@ -493,9 +490,9 @@ class Config: self._remove_files_dirs() self.local.save_cache(start_time) - self.log.info("Finished {} run in {:.2f} seconds".format( + self.log.info("Finished %s run in %.2f seconds", 'dry' if self.dry_run else 'successful', - time.time() - start_time)) + time.time() - start_time) def cleanup(self): self.log.debug("Running cleanup commands") @@ -519,8 +516,8 @@ class Config: self.local.object_path, self.local.type_path, self.local.object_marker_name): if cdist_object.cdist_type.is_install: - self.log.debug(("Running in config mode, ignoring install " - "object: {0}").format(cdist_object)) + self.log.debug("Running in config mode, ignoring install " + "object: %s", cdist_object) else: yield cdist_object @@ -565,8 +562,7 @@ class Config: return objects_changed def _iterate_once_parallel(self): - self.log.debug("Iteration in parallel mode in {} jobs".format( - self.jobs)) + self.log.debug("Iteration in parallel mode in %d jobs", self.jobs) objects_changed = False cargo = [] @@ -588,8 +584,8 @@ class Config: self.object_prepare(cargo[0]) objects_changed = True elif cargo: - self.log.trace("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) + self.log.trace("Multiprocessing start method is %s", + multiprocessing.get_start_method()) self.log.trace("Multiprocessing cargo: %s", cargo) @@ -603,9 +599,8 @@ class Config: "sequentially")) self.explorer.transfer_type_explorers(cargo_types.pop()) else: - self.log.trace(("Starting multiprocessing Pool for {} " - "parallel types explorers transferring".format( - nt))) + self.log.trace("Starting multiprocessing Pool for %d " + "parallel types explorers transferring", nt) args = [ (ct, ) for ct in cargo_types ] @@ -614,8 +609,8 @@ class Config: self.log.trace(("Multiprocessing for parallel transferring " "types' explorers finished")) - self.log.trace(("Starting multiprocessing Pool for {} parallel " - "objects preparation".format(n))) + self.log.trace("Starting multiprocessing Pool for %d parallel " + "objects preparation", n) args = [ (c, False, ) for c in cargo ] @@ -667,10 +662,10 @@ class Config: self.object_run(chunk[0]) objects_changed = True elif chunk: - self.log.trace("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - self.log.trace(("Starting multiprocessing Pool for {} " - "parallel object run".format(n))) + self.log.trace("Multiprocessing start method is %s", + multiprocessing.get_start_method()) + self.log.trace("Starting multiprocessing Pool for %d " + "parallel object run", n) args = [ (c, ) for c in chunk ] @@ -794,9 +789,9 @@ class Config: def object_prepare(self, cdist_object, transfer_type_explorers=True): """Prepare object: Run type explorer + manifest""" self._handle_deprecation(cdist_object) - self.log.verbose("Preparing object {}".format(cdist_object.name)) - self.log.verbose( - "Running manifest and explorers for " + cdist_object.name) + self.log.verbose("Preparing object %s", cdist_object.name) + self.log.verbose("Running manifest and explorers for %s", + cdist_object.name) self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) try: self.manifest.run_type_manifest(cdist_object) @@ -810,13 +805,13 @@ class Config: def object_run(self, cdist_object): """Run gencode and code for an object""" try: - self.log.verbose("Running object " + cdist_object.name) + self.log.verbose("Running object %s", cdist_object.name) if cdist_object.state == core.CdistObject.STATE_DONE: raise cdist.Error(("Attempting to run an already finished " - "object: %s"), cdist_object) + "object: {}").format(cdist_object)) # Generate - self.log.debug("Generating code for %s" % (cdist_object.name)) + self.log.debug("Generating code for %s", cdist_object.name) cdist_object.code_local = self.code.run_gencode_local(cdist_object) cdist_object.code_remote = self.code.run_gencode_remote( cdist_object) @@ -825,20 +820,20 @@ class Config: # Execute if cdist_object.code_local or cdist_object.code_remote: - self.log.info("Processing %s" % (cdist_object.name)) + self.log.info("Processing %s", cdist_object.name) if not self.dry_run: if cdist_object.code_local: - self.log.trace("Executing local code for %s" - % (cdist_object.name)) + self.log.trace("Executing local code for %s", + cdist_object.name) self.code.run_code_local(cdist_object) if cdist_object.code_remote: - self.log.trace("Executing remote code for %s" - % (cdist_object.name)) + self.log.trace("Executing remote code for %s", + cdist_object.name) self.code.transfer_code_remote(cdist_object) self.code.run_code_remote(cdist_object) # Mark this object as done - self.log.trace("Finishing run of " + cdist_object.name) + self.log.trace("Finishing run of %s", cdist_object.name) cdist_object.state = core.CdistObject.STATE_DONE except cdist.Error as e: raise cdist.CdistObjectError(cdist_object, e) diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index c0329c8a..274de989 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -82,9 +82,9 @@ class CdistType: yield cls(base_path, name) except InvalidTypeError as e: # ignore invalid type, log warning and continue - msg = "Ignoring invalid type '%s' at '%s' defined at '%s'" % ( - e.type_path, e.type_absolute_path, e.source_path) - cls.log.warning(msg) + cls.log.warning("Ignoring invalid type '%s' at '%s' defined" + " at '%s'", e.type_path, e.type_absolute_path, + e.source_path) # remove invalid from runtime conf dir os.remove(e.type_absolute_path) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index a3baa959..caa12a7d 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -131,18 +131,17 @@ class Explorer: self._run_global_explorer(explorer, out_path) def _run_global_explorers_parallel(self, out_path): - self.log.debug("Running global explorers in {} parallel jobs".format( - self.jobs)) - self.log.trace("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - self.log.trace(("Starting multiprocessing Pool for global " - "explorers run")) + self.log.debug("Running global explorers in %s parallel jobs", + self.jobs) + self.log.trace("Multiprocessing start method is %s", + multiprocessing.get_start_method()) + self.log.trace("Starting multiprocessing Pool for global explorers" + " run") args = [ (e, out_path, ) for e in self.list_global_explorer_names() ] mp_pool_run(self._run_global_explorer, args, jobs=self.jobs) - self.log.trace(("Multiprocessing run for global explorers " - "finished")) + self.log.trace("Multiprocessing run for global explorers finished") # logger is not pickable, so remove it when we pickle def __getstate__(self): @@ -184,15 +183,14 @@ class Explorer: in the object. """ - self.log.verbose("Running type explorers for {}".format( - cdist_object.cdist_type)) + self.log.verbose("Running type explorers for %s", + cdist_object.cdist_type) if transfer_type_explorers: self.log.trace("Transferring type explorers for type: %s", cdist_object.cdist_type) self.transfer_type_explorers(cdist_object.cdist_type) else: - self.log.trace(("No need for transferring type explorers for " - "type: %s"), + self.log.trace("No need for transferring type explorers for %s", cdist_object.cdist_type) self.log.trace("Transferring object parameters for object: %s", cdist_object.name) @@ -236,8 +234,8 @@ class Explorer: remote side.""" if cdist_type.explorers: if cdist_type.name in self._type_explorers_transferred: - self.log.trace(("Skipping retransfer of type explorers " - "for: %s"), cdist_type) + self.log.trace("Skipping retransfer of type explorers for: %s", + cdist_type) else: source = os.path.join(self.local.type_path, cdist_type.explorer_path) diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 390340d4..3148d66c 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -161,7 +161,7 @@ class Manifest: raise NoInitialManifestError(initial_manifest, user_supplied) message_prefix = "initialmanifest" - self.log.verbose("Running initial manifest " + initial_manifest) + self.log.verbose("Running initial manifest %s", initial_manifest) which = "init" if self.local.save_output_streams: stderr_path = os.path.join(self.local.stderr_base_path, which) diff --git a/cdist/emulator.py b/cdist/emulator.py index f1db862e..f09c282d 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -107,8 +107,8 @@ class Emulator: self.record_requirements() self.record_auto_requirements() self.record_parent_child_relationships() - self.log.trace("Finished %s %s" % ( - self.cdist_object.path, self.parameters)) + self.log.trace("Finished %s %s", self.cdist_object.path, + self.parameters) def __init_log(self): """Setup logging facility""" @@ -170,7 +170,7 @@ class Emulator: # And finally parse/verify parameter self.args = parser.parse_args(self.argv[1:]) - self.log.trace('Args: %s' % self.args) + self.log.trace('Args: %s', self.args) def init_object(self): # Initialize object - and ensure it is not in args @@ -241,8 +241,8 @@ class Emulator: raise cdist.Error(errmsg) else: if self.cdist_object.exists: - self.log.debug(('Object %s override forced with ' - 'CDIST_OVERRIDE'), self.cdist_object.name) + self.log.debug('Object %s override forced with CDIST_OVERRIDE', + self.cdist_object.name) self.cdist_object.create(True) else: self.cdist_object.create() @@ -260,8 +260,8 @@ class Emulator: parent = self.cdist_object.object_from_name(__object_name) parent.typeorder.append(self.cdist_object.name) if self._order_dep_on(): - self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep' - ' for %s'), depname, parent.name) + self.log.trace('[ORDER_DEP] Adding %s to typeorder dep for %s', + depname, parent.name) parent.typeorder_dep.append(depname) elif self._order_dep_on(): self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep', @@ -301,16 +301,14 @@ class Emulator: try: cdist_object = self.cdist_object.object_from_name(requirement) except core.cdist_type.InvalidTypeError as e: - self.log.error(("%s requires object %s, but type %s does not" - " exist. Defined at %s" % ( - self.cdist_object.name, - requirement, e.name, self.object_source))) + self.log.error("%s requires object %s, but type %s does not" + " exist. Defined at %s", self.cdist_object.name, + requirement, e.name, self.object_source) raise except core.cdist_object.MissingObjectIdError: - self.log.error(("%s requires object %s without object id." - " Defined at %s" % (self.cdist_object.name, - requirement, - self.object_source))) + self.log.error("%s requires object %s without object id." + " Defined at %s", self.cdist_object.name, + requirement, self.object_source) raise self.log.debug("Recording requirement %s for %s", @@ -380,10 +378,9 @@ class Emulator: self.env['require'] += " " + lastcreatedtype else: self.env['require'] = lastcreatedtype - self.log.debug(("Injecting require for " - "CDIST_ORDER_DEPENDENCY: %s for %s"), - lastcreatedtype, - self.cdist_object.name) + self.log.debug("Injecting require for" + " CDIST_ORDER_DEPENDENCY: %s for %s", + lastcreatedtype, self.cdist_object.name) except IndexError: # if no second last line, we are on the first type, # so do not set a requirement @@ -391,7 +388,7 @@ class Emulator: if "require" in self.env: requirements = self.env['require'] - self.log.debug("reqs = " + requirements) + self.log.debug("reqs = %s", requirements) for requirement in self._parse_require(requirements): # Ignore empty fields - probably the only field anyway if len(requirement) == 0: diff --git a/cdist/exec/local.py b/cdist/exec/local.py index e0aab190..6713cd13 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -154,8 +154,8 @@ class Local: with open(self.object_marker_file, 'w') as fd: fd.write("%s\n" % self.object_marker_name) - self.log.trace("Object marker %s saved in %s" % ( - self.object_marker_name, self.object_marker_file)) + self.log.trace("Object marker %s saved in %s", + self.object_marker_name, self.object_marker_file) def _init_cache_dir(self, cache_dir): home_dir = cdist.home_dir() @@ -289,14 +289,12 @@ class Local: return cache_subpath def save_cache(self, start_time=time.time()): - self.log.trace("cache subpath pattern: {}".format( - self.cache_path_pattern)) + self.log.trace("cache subpath pattern: %s", self.cache_path_pattern) cache_subpath = self._cache_subpath(start_time, self.cache_path_pattern) - self.log.debug("cache subpath: {}".format(cache_subpath)) + self.log.debug("cache subpath: %s", cache_subpath) destination = os.path.join(self.cache_path, cache_subpath) - self.log.trace(("Saving cache: " + self.base_path + " to " + - destination)) + self.log.trace("Saving cache %s to %s", self.base_path, destination) if not os.path.exists(destination): shutil.move(self.base_path, destination) @@ -332,7 +330,7 @@ class Local: # Iterate over all directories and link the to the output dir for conf_dir in self.conf_dirs: - self.log.debug("Checking conf_dir %s ..." % (conf_dir)) + self.log.debug("Checking conf_dir %s ...", conf_dir) for sub_dir in CONF_SUBDIRS_LINKED: current_dir = os.path.join(conf_dir, sub_dir) @@ -350,7 +348,7 @@ class Local: if os.path.exists(dst): os.unlink(dst) - self.log.trace("Linking %s to %s ..." % (src, dst)) + self.log.trace("Linking %s to %s ...", src, dst) try: os.symlink(src, dst) except OSError as e: diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index e5af2f34..ea85da4c 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -176,19 +176,19 @@ class Remote: # create archive tarpath, fcnt = autil.tar(source, self.archiving_mode) if tarpath is None: - self.log.trace(("Files count {} is lower than {} limit, " - "skipping archiving").format( - fcnt, autil.FILES_LIMIT)) + self.log.trace("Files count %d is lower than %d limit, " + "skipping archiving", + fcnt, autil.FILES_LIMIT) else: - self.log.trace(("Archiving mode, tarpath: %s, file count: " - "%s"), tarpath, fcnt) + self.log.trace("Archiving mode, tarpath: %s, file count: " + "%s", tarpath, fcnt) # get archive name tarname = os.path.basename(tarpath) self.log.trace("Archiving mode tarname: %s", tarname) # archive path at the remote desttarpath = os.path.join(destination, tarname) - self.log.trace( - "Archiving mode desttarpath: %s", desttarpath) + self.log.trace("Archiving mode desttarpath: %s", + desttarpath) # transfer archive to the remote side self.log.trace("Archiving mode: transferring") self._transfer_file(tarpath, desttarpath) diff --git a/cdist/install.py b/cdist/install.py index 7c894fe5..d077baef 100644 --- a/cdist/install.py +++ b/cdist/install.py @@ -47,4 +47,4 @@ class Install(cdist.config.Config): yield cdist_object else: self.log.debug("Running in install mode, ignoring non install" - "object: {0}".format(cdist_object)) + "object: %s", cdist_object) diff --git a/cdist/inventory.py b/cdist/inventory.py index 0387f326..106052a2 100644 --- a/cdist/inventory.py +++ b/cdist/inventory.py @@ -92,7 +92,7 @@ class Inventory: self.init_db() def init_db(self): - self.log.trace("Init db: {}".format(self.db_basedir)) + self.log.trace("Init db: %s", self.db_basedir) if not os.path.exists(self.db_basedir): os.makedirs(self.db_basedir, exist_ok=True) elif not os.path.isdir(self.db_basedir): @@ -182,9 +182,9 @@ class Inventory: configuration = cfg.get_config(section='GLOBAL') determine_default_inventory_dir(args, configuration) - log.debug("Using inventory: {}".format(args.inventory_dir)) - log.trace("Inventory args: {}".format(vars(args))) - log.trace("Inventory command: {}".format(args.subcommand)) + log.debug("Using inventory: %s", args.inventory_dir) + log.trace("Inventory args: %s", vars(args)) + log.trace("Inventory command: %s", args.subcommand) if args.subcommand == "list": c = InventoryList(hosts=args.host, istag=args.tag, @@ -237,16 +237,16 @@ class InventoryList(Inventory): def _do_list(self, it_tags, it_hosts, check_func): if (it_tags is not None): param_tags = set(it_tags) - self.log.trace("param_tags: {}".format(param_tags)) + self.log.trace("param_tags: %s", param_tags) else: param_tags = set() for host in it_hosts: - self.log.trace("host: {}".format(host)) + self.log.trace("host: %s", host) tags = self._get_host_tags(host) if tags is None: - self.log.debug("Host \'{}\' not found, skipped".format(host)) + self.log.debug("Host \'%s\' not found, skipped", host) continue - self.log.trace("tags: {}".format(tags)) + self.log.trace("tags: %s", tags) if check_func(tags, param_tags): yield host, tags @@ -308,11 +308,11 @@ class InventoryHost(Inventory): def _action(self, host): if self.action == "add": - self.log.debug("Adding host \'{}\'".format(host)) + self.log.debug("Adding host \'%s\'", host) elif self.action == "del": - self.log.debug("Deleting host \'{}\'".format(host)) + self.log.debug("Deleting host \'%s\'", host) hostpath = self._host_path(host) - self.log.trace("hostpath: {}".format(hostpath)) + self.log.trace("hostpath: %s", hostpath) if self.action == "add" and not os.path.exists(hostpath): self._new_hostpath(hostpath) else: @@ -372,23 +372,23 @@ class InventoryTag(Inventory): print("Host \'{}\' does not exist, skipping".format(host), file=sys.stderr) return - self.log.trace("existing host_tags: {}".format(host_tags)) + self.log.trace("existing host_tags: %s", host_tags) if self.action == "del" and self.all: host_tags = set() else: for tag in self.input_tags: if self.action == "add": - self.log.debug("Adding tag \'{}\' for host \'{}\'".format( - tag, host)) + self.log.debug("Adding tag \'%s\' for host \'%s\'", + tag, host) host_tags.add(tag) elif self.action == "del": - self.log.debug("Deleting tag \'{}\' for host " - "\'{}\'".format(tag, host)) + self.log.debug("Deleting tag \'%s\' for host \'%s\'", + tag, host) if tag in host_tags: host_tags.remove(tag) - self.log.trace("new host tags: {}".format(host_tags)) + self.log.trace("new host tags: %s", host_tags) if not self._write_host_tags(host, host_tags): - self.log.trace("{} does not exist, skipped".format(host)) + self.log.trace("%s does not exist, skipped", host) def run(self): if self.allhosts: diff --git a/cdist/preos.py b/cdist/preos.py index f8a5dd67..45711a41 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -49,7 +49,7 @@ def scan_preos_dir_plugins(dir): c = cm[1] yield from preos_plugin(c) except ImportError as e: - log.warning("Cannot import '{}': {}".format(module_name, e)) + log.warning("Cannot import '%s': %s", module_name, e) def find_preos_plugins(): @@ -102,7 +102,7 @@ class PreOS: parser.add_argument('remainder_args', nargs=argparse.REMAINDER) args = parser.parse_args(argv[1:]) cdist.argparse.handle_loglevel(args) - log.debug("preos args : {}".format(args)) + log.debug("preos args : %s", args) conf_dirs = util.resolve_conf_dirs_from_config_and_args(args) @@ -122,7 +122,7 @@ class PreOS: func_args = [preos, args.remainder_args, ] else: func_args = [args.remainder_args, ] - log.info("Running preos : {}".format(preos_name)) + log.info("Running preos : %s", preos_name) func(*func_args) else: raise cdist.Error( diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py index a20cdb9c..f1f750ee 100644 --- a/cdist/preos/debootstrap/debootstrap.py +++ b/cdist/preos/debootstrap/debootstrap.py @@ -166,7 +166,7 @@ class Debian: args.pxe_boot_dir = os.path.realpath(args.pxe_boot_dir) cdist.argparse.handle_loglevel(args) - log.debug("preos: {}, args: {}".format(cls._preos_name, args)) + log.debug("preos: %s, args: %s", cls._preos_name, args) try: env = vars(args) new_env = {} @@ -190,27 +190,30 @@ class Debian: env = new_env env.update(os.environ) cls.update_env(env) - log.debug("preos: {} env: {}".format(cls._preos_name, env)) + log.debug("preos: %s env: %s", cls._preos_name, env) + + if log.getEffectiveLevel() <= logging.INFO: + info_msg = ["Running preos: {}, suite: {}, arch: {}".format( + cls._preos_name, args.suite, args.arch), ] + if args.mirror: + info_msg.append("mirror: {}".format(args.mirror)) + if args.script: + info_msg.append("script: {}".format(args.script)) + if args.bootstrap: + info_msg.append("bootstrapping") + if args.configure: + info_msg.append("configuring") + if args.pxe_boot_dir: + info_msg.append("creating PXE") + if args.drive: + info_msg.append("creating bootable drive") + log.info(info_msg) + cmd = os.path.join(cls._files_dir, "code") - info_msg = ["Running preos: {}, suite: {}, arch: {}".format( - cls._preos_name, args.suite, args.arch), ] - if args.mirror: - info_msg.append("mirror: {}".format(args.mirror)) - if args.script: - info_msg.append("script: {}".format(args.script)) - if args.bootstrap: - info_msg.append("bootstrapping") - if args.configure: - info_msg.append("configuring") - if args.pxe_boot_dir: - info_msg.append("creating PXE") - if args.drive: - info_msg.append("creating bootable drive") - log.info(info_msg) - log.debug("cmd={}".format(cmd)) + log.debug("cmd=%s", cmd) subprocess.check_call(cmd, env=env, shell=True) except subprocess.CalledProcessError as e: - log.error("preos {} failed: {}".format(cls._preos_name, e)) + log.error("preos %s failed: %s", cls._preos_name, e) class Ubuntu(Debian): diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index b1d0e9e1..9a0bcc52 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -94,7 +94,7 @@ class Trigger(object): def trigger(self, interface): packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest() - log.debug(f"Sending request on {interface}") + log.debug("Sending request on %s", interface) send(packet, verbose=self.verbose) @@ -114,7 +114,7 @@ class Scanner(object): def handle_pkg(self, pkg): if ICMPv6EchoReply in pkg: host = pkg['IPv6'].src - log.verbose(f"Host {host} is alive") + log.verbose("Host %s is alive", host) dir = os.path.join(self.outdir, host) fname = os.path.join(dir, "last_seen") diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py index 95ca74ee..d9e5f498 100644 --- a/cdist/util/ipaddr.py +++ b/cdist/util/ipaddr.py @@ -42,8 +42,7 @@ def resolve_target_host_name(host, family=0): # gethostbyaddr returns triple # (hostname, aliaslist, ipaddrlist) host_name = socket.gethostbyaddr(ip_addr)[0] - log.debug("derived host_name for host \"{}\": {}".format( - host, host_name)) + log.debug("derived host_name for host \"%s\": %s", host, host_name) except (socket.gaierror, socket.herror) as e: # in case of error provide empty value host_name = '' @@ -54,8 +53,7 @@ def resolve_target_fqdn(host): log = logging.getLogger(host) try: host_fqdn = socket.getfqdn(host) - log.debug("derived host_fqdn for host \"{}\": {}".format( - host, host_fqdn)) + log.debug("derived host_fqdn for host \"%s\": %s", host, host_fqdn) except socket.herror as e: # in case of error provide empty value host_fqdn = '' From 4c2d273f07e6cedb3dcb539d132cb0c3f8176d42 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 30 Mar 2021 07:56:38 +0200 Subject: [PATCH 032/128] Unify string formatting Use one way of string formatting: replace old `%` style with new `str.format`. Resolve #855. --- cdist/config.py | 4 ++-- cdist/core/cdist_object.py | 14 +++++++------- cdist/core/cdist_type.py | 4 ++-- cdist/core/code.py | 8 ++++---- cdist/core/explorer.py | 6 +++--- cdist/core/manifest.py | 11 +++++------ cdist/emulator.py | 12 ++++++------ cdist/exec/local.py | 11 ++++++----- cdist/exec/remote.py | 11 +++++------ cdist/message.py | 2 +- cdist/scan/scan.py | 2 +- cdist/shell.py | 2 +- cdist/test/cdist_object/__init__.py | 27 +++++++++++++++------------ cdist/test/emulator/__init__.py | 4 ++-- cdist/test/exec/remote.py | 8 ++++---- cdist/test/message/__init__.py | 2 +- cdist/util/fsproperty.py | 4 ++-- 17 files changed, 67 insertions(+), 65 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index d6fec55f..adc460de 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -190,7 +190,7 @@ class Config: fd.write(sys.stdin.read()) except (IOError, OSError) as e: raise cdist.Error(("Creating tempfile for stdin data " - "failed: %s" % e)) + "failed: {}").format(e)) args.manifest = initial_manifest_temp_path atexit.register(lambda: os.remove(initial_manifest_temp_path)) @@ -764,7 +764,7 @@ class Config: raise cdist.UnresolvableRequirementsError( ("The requirements of the following objects could not be " - "resolved:\n%s") % ("\n".join(info_string))) + "resolved:\n{}").format("\n".join(info_string))) def _handle_deprecation(self, cdist_object): cdist_type = cdist_object.cdist_type diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index eea2c255..ccf7392d 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -34,17 +34,17 @@ class IllegalObjectIdError(cdist.Error): self.message = message or 'Illegal object id' def __str__(self): - return '%s: %s' % (self.message, self.object_id) + return '{}: {}'.format(self.message, self.object_id) class MissingObjectIdError(cdist.Error): def __init__(self, type_name): self.type_name = type_name - self.message = ("Type %s requires object id (is not a " - "singleton type)") % self.type_name + self.message = ("Type {} requires object id (is not a " + "singleton type)").format(self.type_name) def __str__(self): - return '%s' % (self.message) + return '{}'.format(self.message) class CdistObject: @@ -142,7 +142,7 @@ class CdistObject: if self.object_marker in self.object_id.split(os.sep): raise IllegalObjectIdError( self.object_id, ('object_id may not contain ' - '\'%s\'') % self.object_marker) + '\'{}\'').format(self.object_marker)) if '//' in self.object_id: raise IllegalObjectIdError( self.object_id, 'object_id may not contain //') @@ -189,7 +189,7 @@ class CdistObject: object_id=object_id) def __repr__(self): - return '' % self.name + return ''.format(self.name) def __eq__(self, other): """define equality as 'name is the same'""" @@ -277,7 +277,7 @@ class CdistObject: os.makedirs(path, exist_ok=allow_overwrite) except EnvironmentError as error: raise cdist.Error(('Error creating directories for cdist object: ' - '%s: %s') % (self, error)) + '{}: {}').format(self, error)) def requirements_unfinished(self, requirements): """Return state whether requirements are satisfied""" diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index 274de989..b68448bc 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -34,7 +34,7 @@ class InvalidTypeError(cdist.Error): self.source_path = os.path.realpath(self.type_absolute_path) def __str__(self): - return "Invalid type '%s' at '%s' defined at '%s'" % ( + return "Invalid type '{}' at '{}' defined at '{}'".format( self.type_path, self.type_absolute_path, self.source_path) @@ -109,7 +109,7 @@ class CdistType: return cls._instances[name] def __repr__(self): - return '' % self.name + return ''.format(self.name) def __eq__(self, other): return isinstance(other, self.__class__) and self.name == other.name diff --git a/cdist/core/code.py b/cdist/core/code.py index 1e9b4f80..12888ba4 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -122,8 +122,8 @@ class Code: def _run_gencode(self, cdist_object, which): cdist_type = cdist_object.cdist_type - script = os.path.join(self.local.type_path, - getattr(cdist_type, 'gencode_%s_path' % which)) + gencode_attr = getattr(cdist_type, 'gencode_{}_path'.format(which)) + script = os.path.join(self.local.type_path, gencode_attr) if os.path.isfile(script): env = os.environ.copy() env.update(self.env) @@ -167,8 +167,8 @@ class Code: def _run_code(self, cdist_object, which, env=None): which_exec = getattr(self, which) - script = os.path.join(which_exec.object_path, - getattr(cdist_object, 'code_%s_path' % which)) + code_attr = getattr(cdist_object, 'code_{}_path'.format(which)) + script = os.path.join(which_exec.object_path, code_attr) if which_exec.save_output_streams: stderr_path = os.path.join(cdist_object.stderr_path, 'code-' + which) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index caa12a7d..48414ade 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -160,8 +160,8 @@ class Explorer: self.remote.transfer(self.local.global_explorer_path, self.remote.global_explorer_path, self.jobs) - self.remote.run(["chmod", "0700", - "%s/*" % (self.remote.global_explorer_path)]) + self.remote.run(["chmod", "0700", "{}/*".format( + self.remote.global_explorer_path)]) def run_global_explorer(self, explorer): """Run the given global explorer and return it's output.""" @@ -242,7 +242,7 @@ class Explorer: destination = os.path.join(self.remote.type_path, cdist_type.explorer_path) self.remote.transfer(source, destination) - self.remote.run(["chmod", "0700", "%s/*" % (destination)]) + self.remote.run(["chmod", "0700", "{}/*".format(destination)]) self._type_explorers_transferred.append(cdist_type.name) def transfer_object_parameters(self, cdist_object): diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 3148d66c..09e74dac 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -80,13 +80,12 @@ class NoInitialManifestError(cdist.Error): if user_supplied: if os.path.islink(manifest_path): - self.message = "%s: %s -> %s" % ( - msg_header, manifest_path, - os.path.realpath(manifest_path)) + self.message = "{}: {} -> {}".format( + msg_header, manifest_path, os.path.realpath(manifest_path)) else: - self.message = "%s: %s" % (msg_header, manifest_path) + self.message = "{}: {}".format(msg_header, manifest_path) else: - self.message = "%s" % (msg_header) + self.message = "{}".format(msg_header) def __str__(self): return repr(self.message) @@ -107,7 +106,7 @@ class Manifest: self._open_logger() self.env = { - 'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']), + 'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']), # for use in type emulator '__cdist_type_base_path': self.local.type_path, '__global': self.local.base_path, diff --git a/cdist/emulator.py b/cdist/emulator.py index f09c282d..c55b47d2 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -36,8 +36,8 @@ from cdist.core.manifest import Manifest class MissingRequiredEnvironmentVariableError(cdist.Error): def __init__(self, name): self.name = name - self.message = ("Emulator requires the environment variable %s to be " - "setup" % self.name) + self.message = ("Emulator requires the environment variable {} to be " + "setup").format(self.name) def __str__(self): return self.message @@ -231,13 +231,13 @@ class Emulator: if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env: obj_params = self._object_params_in_context() if obj_params != self.parameters: - errmsg = ("Object %s already exists with conflicting " - "parameters:\n%s: %s\n%s: %s" % ( + errmsg = ("Object {} already exists with conflicting " + "parameters:\n{}: {}\n{}: {}").format( self.cdist_object.name, " ".join(self.cdist_object.source), obj_params, self.object_source, - self.parameters)) + self.parameters) raise cdist.Error(errmsg) else: if self.cdist_object.exists: @@ -292,7 +292,7 @@ class Emulator: fd.write(chunk) chunk = self._read_stdin() except EnvironmentError as e: - raise cdist.Error('Failed to read from stdin: %s' % e) + raise cdist.Error('Failed to read from stdin: {}'.format(e)) def record_requirement(self, requirement): """record requirement and return recorded requirement""" diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 6713cd13..370336ad 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -152,7 +152,7 @@ class Local: def _setup_object_marker_file(self): with open(self.object_marker_file, 'w') as fd: - fd.write("%s\n" % self.object_marker_name) + fd.write("{}\n".format(self.object_marker_name)) self.log.trace("Object marker %s saved in %s", self.object_marker_name, self.object_marker_file) @@ -184,7 +184,7 @@ class Local: """ assert isinstance(command, (list, tuple)), ( - "list or tuple argument expected, got: %s" % command) + "list or tuple argument expected, got: {}".format(command)) quiet = self.quiet_mode or quiet_mode do_save_output = save_output and not quiet and self.save_output_streams @@ -352,8 +352,9 @@ class Local: try: os.symlink(src, dst) except OSError as e: - raise cdist.Error("Linking %s %s to %s failed: %s" % ( - sub_dir, src, dst, e.__str__())) + raise cdist.Error( + "Linking {} {} to {} failed: {}".format( + sub_dir, src, dst, e.__str__())) def _link_types_for_emulator(self): """Link emulator to types""" @@ -366,5 +367,5 @@ class Local: os.symlink(src, dst) except OSError as e: raise cdist.Error( - "Linking emulator from %s to %s failed: %s" % ( + "Linking emulator from {} to {} failed: {}".format( src, dst, e.__str__())) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index ea85da4c..af9e70cb 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -24,12 +24,10 @@ import os import glob import subprocess import logging -import multiprocessing import cdist import cdist.exec.util as util import cdist.util.ipaddr as ipaddr -from cdist.mputil import mp_pool_run def _wrap_addr(addr): @@ -262,9 +260,10 @@ class Remote: # remotely in e.g. csh and setting up CDIST_REMOTE_SHELL to e.g. # /bin/csh will execute this script in the right way. if env: - remote_env = [" export %s=%s;" % item for item in env.items()] - string_cmd = ("/bin/sh -c '" + " ".join(remote_env) + - " ".join(command) + "'") + remote_env = [" export {env[0]}={env[1]};".format(env=item) + for item in env.items()] + string_cmd = ("/bin/sh -c '{}{}'").format(" ".join(remote_env), + " ".join(command)) cmd.append(string_cmd) else: cmd.extend(command) @@ -278,7 +277,7 @@ class Remote: """ assert isinstance(command, (list, tuple)), ( - "list or tuple argument expected, got: %s" % command) + "list or tuple argument expected, got: {}".format(command)) close_stdout = False close_stderr = False diff --git a/cdist/message.py b/cdist/message.py index ffa8c2bb..0c4e21a6 100644 --- a/cdist/message.py +++ b/cdist/message.py @@ -70,7 +70,7 @@ class Message: with open(self.global_messages, 'a') as fd: for line in content: - fd.write("%s:%s" % (self.prefix, line)) + fd.write("{}:{}".format(self.prefix, line)) def merge_messages(self): self._merge_messages() diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 9a0bcc52..faee8a56 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -93,7 +93,7 @@ class Trigger(object): time.sleep(self.sleeptime) def trigger(self, interface): - packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest() + packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest() log.debug("Sending request on %s", interface) send(packet, verbose=self.verbose) diff --git a/cdist/shell.py b/cdist/shell.py index 04a68937..05803556 100644 --- a/cdist/shell.py +++ b/cdist/shell.py @@ -65,7 +65,7 @@ class Shell: def _init_environment(self): self.env = os.environ.copy() additional_env = { - 'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']), + 'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']), # for use in type emulator '__cdist_type_base_path': self.local.type_path, '__cdist_manifest': "cdist shell", diff --git a/cdist/test/cdist_object/__init__.py b/cdist/test/cdist_object/__init__.py index a9c20cd3..11619e96 100644 --- a/cdist/test/cdist_object/__init__.py +++ b/cdist/test/cdist_object/__init__.py @@ -86,8 +86,7 @@ class ObjectClassTestCase(test.CdistTestCase): def test_create_singleton(self): """Check whether creating an object without id (singleton) works""" - singleton = self.expected_objects[0].object_from_name( - "__test_singleton") + self.expected_objects[0].object_from_name("__test_singleton") # came here - everything fine def test_create_singleton_not_singleton_type(self): @@ -126,16 +125,16 @@ class ObjectIdTestCase(test.CdistTestCase): def test_object_id_contains_object_marker(self): cdist_type = core.CdistType(type_base_path, '__third') - illegal_object_id = ( - 'object_id/may/not/contain/%s/anywhere' % OBJECT_MARKER_NAME) + illegal_object_id = 'object_id/may/not/contain/{}/anywhere'.format( + OBJECT_MARKER_NAME) with self.assertRaises(core.IllegalObjectIdError): core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) def test_object_id_contains_object_marker_string(self): cdist_type = core.CdistType(type_base_path, '__third') - illegal_object_id = ( - 'object_id/may/contain_%s_in_filename' % OBJECT_MARKER_NAME) + illegal_object_id = 'object_id/may/contain_{}_in_filename'.format( + OBJECT_MARKER_NAME) core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) # if we get here, the test passed @@ -195,28 +194,32 @@ class ObjectTestCase(test.CdistTestCase): def test_path(self): self.assertEqual(self.cdist_object.path, - "__third/moon/%s" % OBJECT_MARKER_NAME) + "__third/moon/{}".format(OBJECT_MARKER_NAME)) def test_absolute_path(self): self.assertEqual(self.cdist_object.absolute_path, os.path.join(self.object_base_path, - "__third/moon/%s" % OBJECT_MARKER_NAME)) + "__third/moon/{}".format( + OBJECT_MARKER_NAME))) def test_code_local_path(self): self.assertEqual(self.cdist_object.code_local_path, - "__third/moon/%s/code-local" % OBJECT_MARKER_NAME) + "__third/moon/{}/code-local".format( + OBJECT_MARKER_NAME)) def test_code_remote_path(self): self.assertEqual(self.cdist_object.code_remote_path, - "__third/moon/%s/code-remote" % OBJECT_MARKER_NAME) + "__third/moon/{}/code-remote".format( + OBJECT_MARKER_NAME)) def test_parameter_path(self): self.assertEqual(self.cdist_object.parameter_path, - "__third/moon/%s/parameter" % OBJECT_MARKER_NAME) + "__third/moon/{}/parameter".format( + OBJECT_MARKER_NAME)) def test_explorer_path(self): self.assertEqual(self.cdist_object.explorer_path, - "__third/moon/%s/explorer" % OBJECT_MARKER_NAME) + "__third/moon/{}/explorer".format(OBJECT_MARKER_NAME)) def test_parameters(self): expected_parameters = {'planet': 'Saturn', 'name': 'Prometheus'} diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index befd7b57..4b2bc3ba 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -84,8 +84,8 @@ class EmulatorTestCase(test.CdistTestCase): def test_illegal_object_id_requirement(self): argv = ['__file', '/tmp/foobar'] - self.env['require'] = ( - "__file/bad/id/with/%s/inside") % self.local.object_marker_name + self.env['require'] = "__file/bad/id/with/{}/inside".format( + self.local.object_marker_name) emu = emulator.Emulator(argv, env=self.env) self.assertRaises(core.IllegalObjectIdError, emu.run) diff --git a/cdist/test/exec/remote.py b/cdist/test/exec/remote.py index a7fe384d..b23f8447 100644 --- a/cdist/test/exec/remote.py +++ b/cdist/test/exec/remote.py @@ -47,8 +47,8 @@ class RemoteTestCase(test.CdistTestCase): args = (self.target_host,) kwargs.setdefault('base_path', self.base_path) user = getpass.getuser() - kwargs.setdefault('remote_exec', 'ssh -o User=%s -q' % user) - kwargs.setdefault('remote_copy', 'scp -o User=%s -q' % user) + kwargs.setdefault('remote_exec', 'ssh -o User={} -q'.format(user)) + kwargs.setdefault('remote_copy', 'scp -o User={} -q'.format(user)) if 'stdout_base_path' not in kwargs: stdout_path = os.path.join(self.temp_dir, 'stdout') os.makedirs(stdout_path, exist_ok=True) @@ -170,7 +170,7 @@ class RemoteTestCase(test.CdistTestCase): r = self.create_remote(remote_exec=remote_exec, remote_copy=remote_copy) self.assertEqual(r.run('true', return_output=True), - "%s\n" % self.target_host[0]) + "{}\n".format(self.target_host[0])) def test_run_script_target_host_in_env(self): handle, remote_exec_path = self.mkstemp(dir=self.temp_dir) @@ -185,7 +185,7 @@ class RemoteTestCase(test.CdistTestCase): with os.fdopen(handle, "w") as fd: fd.writelines(["#!/bin/sh\n", "true"]) self.assertEqual(r.run_script(script, return_output=True), - "%s\n" % self.target_host[0]) + "{}\n".format(self.target_host[0])) def test_run_script_with_env_target_host_in_env(self): handle, script = self.mkstemp(dir=self.temp_dir) diff --git a/cdist/test/message/__init__.py b/cdist/test/message/__init__.py index 61cd5d97..55040fc9 100644 --- a/cdist/test/message/__init__.py +++ b/cdist/test/message/__init__.py @@ -67,7 +67,7 @@ class MessageTestCase(test.CdistTestCase): def test_message_merge_prefix(self): """Ensure messages are merged and are prefixed""" - expectedcontent = "%s:%s" % (self.prefix, self.content) + expectedcontent = "{}:{}".format(self.prefix, self.content) out = self.message.env['__messages_out'] diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py index 1d76fd76..09e9cc19 100644 --- a/cdist/util/fsproperty.py +++ b/cdist/util/fsproperty.py @@ -30,7 +30,7 @@ class AbsolutePathRequiredError(cdist.Error): self.path = path def __str__(self): - return 'Absolute path required, got: %s' % self.path + return 'Absolute path required, got: {}'.format(self.path) class FileList(collections.MutableSequence): @@ -218,7 +218,7 @@ class FileBasedProperty: def _get_attribute(self, instance, owner): name = self._get_property_name(owner) - attribute_name = '__%s' % name + attribute_name = '__{}'.format(name) if not hasattr(instance, attribute_name): path = self._get_path(instance) attribute_instance = self.attribute_class(path) From ab811ad282882558a4639986e75c800e1b63c2e0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 1 Apr 2021 15:37:11 +0200 Subject: [PATCH 033/128] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 03ae7566..db60c0c4 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,8 @@ next: * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) * Type __sshd_config: Whitelist OpenBMC (Dennis Camera) * Core: Maintain object relationship graph in cdist cache (Darko Poljak) + * Type __git: Fix numeric owner and group handling (Dennis Camera) + * Type __pyvenv: Fix numeric owner and group handling (Dennis Camera) 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) From 199effb7ef23980bc69f0233ff6d7ee465623f40 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 6 Apr 2021 19:35:14 +0200 Subject: [PATCH 034/128] Improve unfinished object requirements bool check When we need only boolean value for unfinished object requirements then we don't need to determine the whole list of unfinished objects. --- cdist/config.py | 12 +++++++----- cdist/core/cdist_object.py | 13 ++++++++++++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index adc460de..f7da9b1d 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -537,7 +537,7 @@ class Config: objects_changed = False for cdist_object in self.object_list(): - if cdist_object.requirements_unfinished( + if cdist_object.has_requirements_unfinished( cdist_object.requirements): """We cannot do anything for this poor object""" continue @@ -548,7 +548,7 @@ class Config: self.object_prepare(cdist_object) objects_changed = True - if cdist_object.requirements_unfinished( + if cdist_object.has_requirements_unfinished( cdist_object.autorequire): """The previous step created objects we depend on - wait for them @@ -567,7 +567,8 @@ class Config: cargo = [] for cdist_object in self.object_list(): - if cdist_object.requirements_unfinished(cdist_object.requirements): + if cdist_object.has_requirements_unfinished( + cdist_object.requirements): """We cannot do anything for this poor object""" continue @@ -621,12 +622,13 @@ class Config: del cargo[:] for cdist_object in self.object_list(): - if cdist_object.requirements_unfinished(cdist_object.requirements): + if cdist_object.has_requirements_unfinished( + cdist_object.requirements): """We cannot do anything for this poor object""" continue if cdist_object.state == core.CdistObject.STATE_PREPARED: - if cdist_object.requirements_unfinished( + if cdist_object.has_requirements_unfinished( cdist_object.autorequire): """The previous step created objects we depend on - wait for them diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index ccf7392d..bb3a65bd 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -280,7 +280,7 @@ class CdistObject: '{}: {}').format(self, error)) def requirements_unfinished(self, requirements): - """Return state whether requirements are satisfied""" + """Return unsatisfied requirements""" object_list = [] @@ -291,3 +291,14 @@ class CdistObject: object_list.append(cdist_object) return object_list + + def has_requirements_unfinished(self, requirements): + """Return whether requirements are satisfied""" + + for requirement in requirements: + cdist_object = self.object_from_name(requirement) + + if cdist_object.state != self.STATE_DONE: + return True + + return False From 750c71fb5a9fcf4a30af0700b7a09e3501988a27 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 7 Apr 2021 09:07:29 +0200 Subject: [PATCH 035/128] Minor refactoring and remove code duplication --- cdist/config.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index f7da9b1d..638fdf0e 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -699,20 +699,22 @@ class Config: check for cycles. ''' graph = {} - for cdist_object in self.object_list(): + + def _add_requirements(cdist_object, requirements): obj_name = cdist_object.name if obj_name not in graph: graph[obj_name] = [] + + for requirement in cdist_object.requirements_unfinished( + requirements): + graph[obj_name].append(requirement.name) + + for cdist_object in self.object_list(): if cdist_object.state == cdist_object.STATE_DONE: continue - for requirement in cdist_object.requirements_unfinished( - cdist_object.requirements): - graph[obj_name].append(requirement.name) - - for requirement in cdist_object.requirements_unfinished( - cdist_object.autorequire): - graph[obj_name].append(requirement.name) + _add_requirements(cdist_object, cdist_object.requirements) + _add_requirements(cdist_object, cdist_object.autorequire) return graph_check_cycle(graph) def iterate_until_finished(self): From d2eec6066876eeb4a295b81a670c70b0ff8ad2a7 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 11 Apr 2021 23:05:48 +0300 Subject: [PATCH 036/128] [__download] make --sum optional --- cdist/conf/type/__download/explorer/state | 6 ++++++ cdist/conf/type/__download/man.rst | 15 ++++++--------- cdist/conf/type/__download/parameter/optional | 1 + cdist/conf/type/__download/parameter/required | 1 - 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 00362545..68b517c5 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -8,6 +8,12 @@ then exit 0 fi +if [ ! -f "$__object/parameter/sum" ] +then + echo 'present' + exit 0 +fi + sum_should="$( cat "$__object/parameter/sum" )" if [ -f "$__object/parameter/cmd-sum" ] diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index 54503470..a1278cfb 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -8,9 +8,6 @@ cdist-type__download - Download a file DESCRIPTION ----------- -Destination (``$__object_id``) in target host must be persistent storage -in order to calculate checksum and decide if file must be (re-)downloaded. - By default type will try to use ``wget``, ``curl`` or ``fetch``. If download happens in target (see ``--download``) then type will fallback to (and install) ``wget``. @@ -25,14 +22,14 @@ REQUIRED PARAMETERS url File's URL. -sum - Checksum of file going to be downloaded. - By default output of ``cksum`` without filename is expected. - Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. - OPTIONAL PARAMETERS ------------------- +sum + Checksum is used to decide if existing destination file must be redownloaded. + By default output of ``cksum`` without filename is expected. + Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. + download If ``local`` (default), then download file to local storage and copy it to target host. If ``remote``, then download happens in target. @@ -81,7 +78,7 @@ Ander Punnar COPYING ------- -Copyright \(C) 2020 Ander Punnar. You can redistribute it +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional index 838e2fbf..d69e083e 100644 --- a/cdist/conf/type/__download/parameter/optional +++ b/cdist/conf/type/__download/parameter/optional @@ -1,3 +1,4 @@ +sum cmd-get cmd-sum download diff --git a/cdist/conf/type/__download/parameter/required b/cdist/conf/type/__download/parameter/required index 6ea4c38f..96cdd3b9 100644 --- a/cdist/conf/type/__download/parameter/required +++ b/cdist/conf/type/__download/parameter/required @@ -1,2 +1 @@ url -sum From 9ec01d9f971339ee67c866c4a248047966f73ce1 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 13 Apr 2021 12:22:45 +0200 Subject: [PATCH 037/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index db60c0c4..c8cb44da 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ next: * Core: Maintain object relationship graph in cdist cache (Darko Poljak) * Type __git: Fix numeric owner and group handling (Dennis Camera) * Type __pyvenv: Fix numeric owner and group handling (Dennis Camera) + * Type __download: Make sum parameter optional (Ander Punnar) 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) From 92b8942a8c37bb985ef42991f3b7aaef0cd8073f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 09:00:32 +0200 Subject: [PATCH 038/128] [type/__postgres_conf] Add psql_exec function to state explorer --- cdist/conf/type/__postgres_conf/explorer/state | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index de6e8aa0..1a58751a 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -20,21 +20,24 @@ # postgres_user=$("${__type_explorer:?}/postgres_user") - conf_name=${__object_id:?} -su - "${postgres_user}" -c 'psql postgres -c "SELECT 1"' >/dev/null || { +quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } +psql_exec() { + su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" +} + +psql_exec 'SELECT 1' >/dev/null || { echo 'Connection to PostgreSQL server failed' >&2 exit 1 } -if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ - | cmp -s "${__object:?}/parameter/value" - +if psql_exec "SHOW ${conf_name}" | cmp -s "${__object:?}/parameter/value" - then echo present else # NOTE: SHOW/SET are case-insentitive, so this command should also be. - case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"") + case $(psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')") in ('') # invalid configuration parameter From 2ccc03fef1e28c759a7918fa00ffcb45c76dc57a Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 09:02:40 +0200 Subject: [PATCH 039/128] [type/__postgres_conf] Add psql_conf_cmp function to state explorer --- cdist/conf/type/__postgres_conf/explorer/state | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index 1a58751a..be881be6 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -27,12 +27,16 @@ psql_exec() { su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" } +psql_conf_cmp() { + test "$(psql_exec "SHOW $1")" = "$2" +} + psql_exec 'SELECT 1' >/dev/null || { echo 'Connection to PostgreSQL server failed' >&2 exit 1 } -if psql_exec "SHOW ${conf_name}" | cmp -s "${__object:?}/parameter/value" - +if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" then echo present else From e0416403c4862cb8bc0f41a7c6f8a33d1a8656a5 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 09:04:07 +0200 Subject: [PATCH 040/128] [type/__postgres_conf] Add psql_conf_source function to state explorer --- cdist/conf/type/__postgres_conf/explorer/state | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index be881be6..9f54724a 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -27,6 +27,10 @@ psql_exec() { su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" } +psql_conf_source() { + # NOTE: SHOW/SET are case-insentitive, so this command should also be. + psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')" +} psql_conf_cmp() { test "$(psql_exec "SHOW $1")" = "$2" } @@ -40,8 +44,7 @@ if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" then echo present else - # NOTE: SHOW/SET are case-insentitive, so this command should also be. - case $(psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')") + case $(psql_conf_source "${conf_name}") in ('') # invalid configuration parameter From 12c2995494ce582cb3adf1d942b36f6df035370a Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 14:46:36 +0200 Subject: [PATCH 041/128] [type/__postgres_conf] Implement complex state compare logic --- .../conf/type/__postgres_conf/explorer/state | 166 +++++++++++++++++- 1 file changed, 163 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index 9f54724a..e76540c5 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -22,6 +22,52 @@ postgres_user=$("${__type_explorer:?}/postgres_user") conf_name=${__object_id:?} +tolower() { printf '%s' "$*" | tr '[:upper:]' '[:lower:]'; } + +tobytes() { + # NOTE: This function treats everything as base 2. + # It is not compatible with SI units. + awk 'BEGIN { FS = "\n" } + /TB$/ { $0 = ($0 * 1024) "GB" } + /GB$/ { $0 = ($0 * 1024) "MB" } + /MB$/ { $0 = ($0 * 1024) "kB" } + /kB$/ { $0 = ($0 * 1024) "B" } + /B?$/ { sub(/ *B?$/, "") } + ($0*1) == $0 # is number + ' <<-EOF + $1 + EOF +} + +tomillisecs() { + awk 'BEGIN { FS = "\n" } + /d$/ { $0 = ($0 * 24) "h" } + /h$/ { $0 = ($0 * 60) "min" } + /min$/ { $0 = ($0 * 60) "s" } + /[^m]s$/ { $0 = ($0 * 1000) "ms" } + /ms$/ { $0 *= 1 } + ($0*1) == $0 # is number + ' <<-EOF + $1 + EOF +} + +tobool() { + # prints either 'on' or 'off' + case $(tolower "$1") + in + (t|true|y|yes|on|1) + echo 'on' ;; + (f|false|n|no|off|0) + echo 'off' ;; + (*) + printf 'Inavlid bool value: %s\n' "$2" >&2 + return 1 + ;; + esac + return 0 +} + quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } psql_exec() { su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" @@ -31,9 +77,123 @@ psql_conf_source() { # NOTE: SHOW/SET are case-insentitive, so this command should also be. psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')" } -psql_conf_cmp() { - test "$(psql_exec "SHOW $1")" = "$2" -} +psql_conf_cmp() ( + IFS='|' read -r lower_name vartype setting unit <<-EOF + $(psql_exec "SELECT lower(name), vartype, setting, unit FROM pg_settings WHERE lower(name) = lower('$1')") + EOF + + should_value=$2 + is_value=${setting} + + # The following case contains special cases for special settings. + case ${lower_name} + in + (archive_command) + if test "${setting}" = '(disabled)' + then + # DAFUQ PostgreSQL?! + # PostgreSQL returns (disabled) if the feature is inactive. + # We cannot compare the values unless it is enabled, first. + return 0 + fi + ;; + (archive_mode|backslash_quote|constraint_exclusion|force_parallel_mode|huge_pages|synchronous_commit) + # Although only 'on', 'off' are documented, PostgreSQL accepts all + # the "likely" variants of "on" and "off". + case $(tolower "${should_value}") + in + (on|off|true|false|yes|no|1|0) + should_value=$(tobool "${should_value}") + ;; + esac + ;; + esac + + case ${vartype} + in + (bool) + test -z "${unit}" || { + # please fix the explorer if this error occurs. + printf 'units are not supported for vartype: %s\n' "${vartype}" >&2 + exit 1 + } + + should_value=$(tobool "${should_value}") + + test "${is_value}" = "${should_value}" + ;; + (enum) + test -z "${unit}" || { + # please fix the explorer if this error occurs. + printf 'units are not supported with vartype: %s\n' "${vartype}" >&2 + exit 1 + } + + # NOTE: All enums that are currently defined are lower case, but + # PostgreSQL also accepts upper case spelling. + should_value=$(tolower "$2") + + test "${is_value}" = "${should_value}" + ;; + (integer) + # split multiples from unit, first (e.g. 8kB -> 8, kB) + case ${unit} + in + ([0-9]*) + multiple=${unit%%[!0-9]*} + unit=${unit##*[0-9 ]} + ;; + (*) multiple=1 ;; + esac + + is_value=$((setting * multiple))${unit} + + if expr "${should_value}" : '-\{0,1\}[0-9]*$' >/dev/null + then + # default unit + should_value=$((should_value * multiple))${unit} + fi + + # then, do conversion + # NOTE: these conversions work for integers only! + case ${unit} + in + (B|[kMGT]B) + # bytes + is_bytes=$(tobytes "${is_value}") + should_bytes=$(tobytes "${should_value}") + + test $((is_bytes)) -eq $((should_bytes)) + ;; + (ms|s|min|h|d) + # seconds + is_ms=$(tomillisecs "${is_value}") + should_ms=$(tomillisecs "${should_value}") + + test $((is_ms)) -eq $((should_ms)) + ;; + ('') + # no unit + is_int=${is_value} + should_int=${should_value} + + test $((is_int)) -eq $((should_int)) + ;; + esac + ;; + (real|string) + # NOTE: reals could possibly have units, but currently there none. + + test -z "${unit}" || { + # please fix the explorer if this error occurs. + printf 'units are not supported with vartype: %s\n' "${vartype}" >&2 + exit 1 + } + + test "${is_value}" = "${should_value}" + ;; + esac +) psql_exec 'SELECT 1' >/dev/null || { echo 'Connection to PostgreSQL server failed' >&2 From bef1433ba3adb022f8b50eb9df0fbbbf90757d76 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 14:46:50 +0200 Subject: [PATCH 042/128] [type/__postgres_conf] Accept empty values --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index d0d247b4..27651600 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -57,7 +57,7 @@ psql_cmd() { case ${state_should} in (present) - test -s "${__object:?}/parameter/value" || { + test -n "${__object:?}/parameter/value" || { echo 'Missing required parameter --value' >&2 exit 1 } From 686e4f0f2d498ee39a69e43f7535171f16f5fcbc Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 15:01:38 +0200 Subject: [PATCH 043/128] [type/__postgres_conf] Reverse state logic (decide based on source first) --- .../conf/type/__postgres_conf/explorer/state | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index e76540c5..4b7b0a43 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -200,23 +200,24 @@ psql_exec 'SELECT 1' >/dev/null || { exit 1 } -if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" -then - echo present -else - case $(psql_conf_source "${conf_name}") - in - ('') - # invalid configuration parameter - # (error message was already printed by SHOW command above. - # Yes, it's a hack) - exit 1 - ;; - (default) - echo absent - ;; - (*) +case $(psql_conf_source "${conf_name}") +in + ('') + printf 'Invalid configuration parameter: %s\n' "${conf_name}" >&2 + exit 1 + ;; + (default) + echo absent + ;; + (*) + if ! test -f "${__object:?}/parameter/value" + then + echo present + elif psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" + then + echo present + else echo different - ;; - esac -fi + fi + ;; +esac From 19bf37be1a8b0b099a7199d1565b74598010f614 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 15:55:52 +0200 Subject: [PATCH 044/128] [type/__postgres_conf] Update man.rst --- cdist/conf/type/__postgres_conf/man.rst | 39 ++++++++++++++----------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst index 76016c6c..e035f080 100644 --- a/cdist/conf/type/__postgres_conf/man.rst +++ b/cdist/conf/type/__postgres_conf/man.rst @@ -1,29 +1,27 @@ cdist-type__postgres_conf(7) ============================ -Configure a PostgreSQL server. - -NOTE: This type might need to be run multiple times to apply all bits of the -configuration due to ordering requirements. - -SSRQ +NAME +---- +cdist-type__postgres_conf - Alter PostgreSQL configuration DESCRIPTION ----------- -Configure a PostgreSQL server using ALTER SYSTEM. +Configure a running PostgreSQL server using ``ALTER SYSTEM``. REQUIRED PARAMETERS ------------------- value - The value to setup (can be omitted when state is set to "absent"). + The value to set (can be omitted if ``--state`` is set to ``absent``). OPTIONAL PARAMETERS ------------------- state - "present" or "absent". Defaults to "present". + ``present`` or ``absent``. + Defaults to ``present``. BOOLEAN PARAMETERS @@ -36,20 +34,27 @@ EXAMPLES .. code-block:: sh - # set timezone - __postgres_conf timezone --value Europe/Zurich + # set timezone + __postgres_conf timezone --value Europe/Zurich - # reset maximum number of concurrent connections to default (normally 100) - __postgres_conf max_connections --state absent + # reset maximum number of concurrent connections to default (normally 100) + __postgres_conf max_connections --state absent SEE ALSO -------- -- `cdist-type(7) `_ +None. + + +AUTHORS +------- +Beni Ruef (bernhard.ruef--@--ssrq-sds-fds.ch) +Dennis Camera (dennis.camera--@--ssrq-sds-fds.ch) COPYING ------- -Copyright \(C) 2020 SSRQ (www.ssrq-sds-fds.ch). -Free use of this software is granted under the terms -of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2019-2021 SSRQ (www.ssrq-sds-fds.ch). +You can redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. From 1c047353a95e7ac079ccf89af8fc232451b8f891 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 17 Apr 2021 09:57:10 +0200 Subject: [PATCH 045/128] [bin/cdist] Fix Python version check --- bin/cdist | 8 +++++--- cdist/__init__.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index ddaffa7f..adb06a8d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -72,9 +72,11 @@ def commandline(): if __name__ == "__main__": - if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION: - print('Python >= {} is required on the source host.'.format( - cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr) + if sys.version_info[:3] < cdist.MIN_SUPPORTED_PYTHON_VERSION: + print( + 'Python >= {} is required on the source host.'.format( + ".".join(map(str, cdist.MIN_SUPPORTED_PYTHON_VERSION))), + file=sys.stderr) sys.exit(1) exit_code = 0 diff --git a/cdist/__init__.py b/cdist/__init__.py index 44366cd0..31d49889 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -64,7 +64,7 @@ REMOTE_EXEC = "ssh -o User=root" REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}" -MIN_SUPPORTED_PYTHON_VERSION = '3.5' +MIN_SUPPORTED_PYTHON_VERSION = (3, 5) class Error(Exception): From 1bb696a4100f161afada9b1e8d8ad2b8ab190f54 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 20 Apr 2021 07:33:07 +0200 Subject: [PATCH 046/128] Release 6.9.6 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index c8cb44da..70d0b960 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) * Core: config: Make local state directory available to custom remotes (Steven Armstrong * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) From acf9bf91f17fb4ad0e5813c6e4b0b40fc7e027b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 22 Apr 2021 08:55:14 +0200 Subject: [PATCH 047/128] [scanner] error to stderr and exit when scapy is not available --- cdist/scan/commandline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index eca4cf13..0d7fb0ca 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -20,6 +20,7 @@ # import logging +import sys log = logging.getLogger("scan") @@ -31,7 +32,9 @@ def commandline(args): try: import cdist.scan.scan as scan except ModuleNotFoundError: - print('cdist scan requires scapy to be installed') + print('cdist scan requires scapy to be installed! Exiting.', + file=sys.stderr) + sys.exit(1) processes = [] From a4464209b6741f2c1764aa432c56cfa8128852a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 22 Apr 2021 09:29:53 +0200 Subject: [PATCH 048/128] [scanner] add minimal error handling, consolidate CLI args processing --- cdist/argparse.py | 10 ++++-- cdist/scan/commandline.py | 19 ++++++----- cdist/scan/scan.py | 71 +++++++++++---------------------------- 3 files changed, 37 insertions(+), 63 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index cadac39a..153e7864 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -492,12 +492,16 @@ def get_parsers(): help='Try to configure detected hosts') parser['scan'].add_argument( '-I', '--interfaces', - action='append', default=[], + action='append', default=[], required=True, help='On which interfaces to scan/trigger') parser['scan'].add_argument( '-d', '--delay', - action='store', default=3600, - help='How long to wait before reconfiguring after last try') + action='store', default=3600, type=int, + help='How long (seconds) to wait before reconfiguring after last try') + parser['scan'].add_argument( + '-t', '--trigger-delay', + action='store', default=5, type=int, + help='How long (seconds) to wait between ICMPv6 echo requests') parser['scan'].set_defaults(func=cdist.scan.commandline.commandline) for p in parser: diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 0d7fb0ca..dead5292 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -24,26 +24,29 @@ import sys log = logging.getLogger("scan") - -# define this outside of the class to not handle scapy import errors by default +# CLI processing is defined outside of the main scan class to handle +# non-available optional scapy dependency (instead of crashing mid-flight). def commandline(args): log.debug(args) + # Check if we have the optional scapy dependency available. try: import cdist.scan.scan as scan except ModuleNotFoundError: - print('cdist scan requires scapy to be installed! Exiting.', - file=sys.stderr) + log.error('cdist scan requires scapy to be installed. Exiting.') sys.exit(1) - processes = [] - + # Default operation mode. if not args.mode: - # By default scan and trigger, but do not call any action + # By default scan and trigger, but do not call any action. args.mode = ['scan', 'trigger', ] + # We run each component in a separate process since they + # must not block on each other. + processes = [] + if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interfaces) + t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay) t.start() processes.append(t) log.debug("Trigger started") diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index faee8a56..633a5c06 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -61,20 +61,22 @@ import datetime import cdist.config +logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") - class Trigger(object): """ Trigger an ICMPv6EchoReply from all hosts that are alive """ - def __init__(self, interfaces=None, verbose=False): + def __init__(self, interfaces, sleeptime, verbose=False): self.interfaces = interfaces + + # Used by scapy / send in trigger/2. self.verbose = verbose - # Wait 5 seconds before triggering again - FIXME: add parameter - self.sleeptime = 5 + # Delay in seconds between sent ICMPv6EchoRequests. + self.sleeptime = sleeptime def start(self): self.processes = [] @@ -93,9 +95,12 @@ class Trigger(object): time.sleep(self.sleeptime) def trigger(self, interface): - packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest() - log.debug("Sending request on %s", interface) - send(packet, verbose=self.verbose) + try: + log.debug("Sending ICMPv6EchoRequest on %s", interface) + packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest() + send(packet, verbose=self.verbose) + except Exception as e: + log.error( "Could not send ICMPv6EchoRequest: %s", e) class Scanner(object): @@ -103,7 +108,7 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces=None, args=None, outdir=None): + def __init__(self, interfaces, args=None, outdir=None): self.interfaces = interfaces if outdir: @@ -148,47 +153,9 @@ class Scanner(object): def scan(self): log.debug("Scanning - zzzzz") - sniff(iface=self.interfaces, - filter="icmp6", - prn=self.handle_pkg) - - -if __name__ == '__main__': - t = Trigger(interfaces=["wlan0"]) - t.start() - - # Scanner can listen on many interfaces at the same time - s = Scanner(interfaces=["wlan0"]) - s.scan() - - # Join back the trigger processes - t.join() - - # Test in my lan shows: - # [18:48] bridge:cdist% ls -1d fe80::* - # fe80::142d:f0a5:725b:1103 - # fe80::20d:b9ff:fe49:ac11 - # fe80::20d:b9ff:fe4c:547d - # fe80::219:d2ff:feb2:2e12 - # fe80::21b:fcff:feee:f446 - # fe80::21b:fcff:feee:f45c - # fe80::21b:fcff:feee:f4b1 - # fe80::21b:fcff:feee:f4ba - # fe80::21b:fcff:feee:f4bc - # fe80::21b:fcff:feee:f4c1 - # fe80::21d:72ff:fe86:46b - # fe80::42b0:34ff:fe6f:f6f0 - # fe80::42b0:34ff:fe6f:f863 - # fe80::42b0:34ff:fe6f:f9b2 - # fe80::4a5d:60ff:fea1:e55f - # fe80::77a3:5e3f:82cc:f2e5 - # fe80::9e93:4eff:fe6c:c1f4 - # fe80::ba69:f4ff:fec5:6041 - # fe80::ba69:f4ff:fec5:8db7 - # fe80::bad8:12ff:fe65:313d - # fe80::bad8:12ff:fe65:d9b1 - # fe80::ce2d:e0ff:fed4:2611 - # fe80::ce32:e5ff:fe79:7ea7 - # fe80::d66d:6dff:fe33:e00 - # fe80::e2ff:f7ff:fe00:20e6 - # fe80::f29f:c2ff:fe7c:275e + try: + sniff(iface=self.interfaces, + filter="icmp6", + prn=self.handle_pkg) + except Exception as e: + log.error( "Could not start listener: %s", e) From bb24d632d62da8390dd1a724fc325a57526da40d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 22 Apr 2021 10:20:49 +0200 Subject: [PATCH 049/128] [scanner] implement the --list flag --- cdist/argparse.py | 4 +++ cdist/scan/commandline.py | 63 ++++++++++++++++++++++++++++----------- cdist/scan/scan.py | 17 +++++++++++ 3 files changed, 67 insertions(+), 17 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 153e7864..f390a974 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -486,6 +486,10 @@ def get_parsers(): '-m', '--mode', help='Which modes should run', action='append', default=[], choices=['scan', 'trigger']) + parser['scan'].add_argument( + '--list', + action='store_true', + help='List the known hosts and exit') parser['scan'].add_argument( '--config', action='store_true', diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index dead5292..331694e4 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -21,26 +21,11 @@ import logging import sys +from datetime import datetime log = logging.getLogger("scan") -# CLI processing is defined outside of the main scan class to handle -# non-available optional scapy dependency (instead of crashing mid-flight). -def commandline(args): - log.debug(args) - - # Check if we have the optional scapy dependency available. - try: - import cdist.scan.scan as scan - except ModuleNotFoundError: - log.error('cdist scan requires scapy to be installed. Exiting.') - sys.exit(1) - - # Default operation mode. - if not args.mode: - # By default scan and trigger, but do not call any action. - args.mode = ['scan', 'trigger', ] - +def run(scan, args): # We run each component in a separate process since they # must not block on each other. processes = [] @@ -59,3 +44,47 @@ def commandline(args): for process in processes: process.join() + +def list(scan, args): + s = scan.Scanner(interfaces=args.interfaces, args=args) + hosts = s.list() + + # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + + # 6 colons. + ipv6_max_size = 8 * 4 + 10 + # We format dates as follow: YYYY-MM-DD HH:MM:SS + date_max_size = 8 + 2 + 6 + 2 + + print("{} | {}".format( + 'link-local address'.ljust(ipv6_max_size), + 'last seen'.ljust(date_max_size))) + print('=' * (ipv6_max_size + 3 + date_max_size)) + for addr in hosts: + last_seen = datetime.strftime( + datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'), + '%Y-%m-%d %H:%M:%S') + print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size))) + +# CLI processing is defined outside of the main scan class to handle +# non-available optional scapy dependency (instead of crashing mid-flight). +def commandline(args): + log.debug(args) + + # Check if we have the optional scapy dependency available. + try: + import cdist.scan.scan as scan + except ModuleNotFoundError: + log.error('cdist scan requires scapy to be installed. Exiting.') + sys.exit(1) + + # Set default operation mode. + if not args.mode: + # By default scan and trigger, but do not call any action. + args.mode = ['scan', 'trigger', ] + + # Print known hosts and exit is --list is specified - do not start + # the scanner. + if args.list: + list(scan, args) + else: + run(scan, args) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 633a5c06..f3976370 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -132,6 +132,23 @@ class Scanner(object): with open(fname, "w") as fd: fd.write(f"{now}\n") + def list(self): + hosts = dict() + for linklocal_addr in os.listdir(self.outdir): + workdir = os.path.join(self.outdir, linklocal_addr) + # We ignore any (unexpected) file in this directory. + if os.path.isdir(workdir): + last_seen='-' + last_seen_file = os.path.join(workdir, 'last_seen') + if os.path.isfile(last_seen_file): + with open(last_seen_file, "r") as fd: + last_seen = fd.readline() + + hosts[linklocal_addr] = {'last_seen': last_seen} + + return hosts + + def config(self): """ Configure a host From 13e2ad175f01b6cfbdb21ee50e85b6f3ba6fe750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 25 Apr 2021 12:45:34 +0200 Subject: [PATCH 050/128] [scanner] add host class, name mapper and pre-config logic --- cdist/argparse.py | 6 +- cdist/scan/commandline.py | 37 ++++++++---- cdist/scan/scan.py | 122 +++++++++++++++++++++++++++++--------- 3 files changed, 124 insertions(+), 41 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index f390a974..bedb23ac 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -485,7 +485,7 @@ def get_parsers(): parser['scan'].add_argument( '-m', '--mode', help='Which modes should run', action='append', default=[], - choices=['scan', 'trigger']) + choices=['scan', 'trigger', 'config']) parser['scan'].add_argument( '--list', action='store_true', @@ -498,6 +498,10 @@ def get_parsers(): '-I', '--interfaces', action='append', default=[], required=True, help='On which interfaces to scan/trigger') + parser['scan'].add_argument( + '--name-mapper', + action='store', default=None, + help='Map addresses to names, required for config mode') parser['scan'].add_argument( '-d', '--delay', action='store', default=3600, type=int, diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 331694e4..1a0ab0a7 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -37,7 +37,10 @@ def run(scan, args): log.debug("Trigger started") if 'scan' in args.mode: - s = scan.Scanner(interfaces=args.interfaces, args=args) + s = scan.Scanner( + autoconfigure='config' in args.mode, + interfaces=args.interfaces, + name_mapper=args.name_mapper) s.start() processes.append(s) log.debug("Scanner started") @@ -46,24 +49,27 @@ def run(scan, args): process.join() def list(scan, args): - s = scan.Scanner(interfaces=args.interfaces, args=args) + s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper) hosts = s.list() # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + # 6 colons. ipv6_max_size = 8 * 4 + 10 - # We format dates as follow: YYYY-MM-DD HH:MM:SS - date_max_size = 8 + 2 + 6 + 2 + date_max_size = len(datetime.now().strftime(scan.datetime_format)) + name_max_size = 25 - print("{} | {}".format( - 'link-local address'.ljust(ipv6_max_size), - 'last seen'.ljust(date_max_size))) - print('=' * (ipv6_max_size + 3 + date_max_size)) - for addr in hosts: - last_seen = datetime.strftime( - datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'), - '%Y-%m-%d %H:%M:%S') - print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size))) + print("{} | {} | {} | {}".format( + 'name'.ljust(name_max_size), + 'address'.ljust(ipv6_max_size), + 'last seen'.ljust(date_max_size), + 'last configured'.ljust(date_max_size))) + print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) + for host in hosts: + print("{} | {} | {} | {}".format( + host.name(default='-').ljust(name_max_size), + host.address().ljust(ipv6_max_size), + host.last_seen().ljust(date_max_size), + host.last_configured().ljust(date_max_size))) # CLI processing is defined outside of the main scan class to handle # non-available optional scapy dependency (instead of crashing mid-flight). @@ -82,6 +88,11 @@ def commandline(args): # By default scan and trigger, but do not call any action. args.mode = ['scan', 'trigger', ] + if 'config' in args.mode and args.name_mapper == None: + print('--name-mapper must be specified for scanner config mode.', + file=sys.stderr) + sys.exit(1) + # Print known hosts and exit is --list is specified - do not start # the scanner. if args.list: diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index f3976370..459138e2 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -63,6 +63,69 @@ import cdist.config logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") +datetime_format = '%Y-%m-%d %H:%M:%S' + +class Host(object): + def __init__(self, addr, outdir, name_mapper=None): + self.addr = addr + self.workdir = os.path.join(outdir, addr) + self.name_mapper = name_mapper + + os.makedirs(self.workdir, exist_ok=True) + + def __get(self, key, default=None): + fname = os.path.join(self.workdir, key) + value=default + if os.path.isfile(fname): + with open(fname, "r") as fd: + value = fd.readline() + return value + + def __set(self, key, value): + fname = os.path.join(self.workdir, key) + with open(fname, "w") as fd: + fd.write(f"{value}") + + def name(self, default=None): + if self.name_mapper == None: + return default + + fpath = os.path.join(os.getcwd(), self.name_mapper) + if os.path.isfile(fpath) and os.access(fpath, os.X_OK): + out = subprocess.run([fpath, self.addr], capture_output=True) + if out.returncode != 0: + return default + else: + value = out.stdout.decode() + return (None if len(value) == 0 else value) + else: + return default + + def address(self): + return self.addr + + def last_seen(self, default=None): + raw = self.__get('last_seen') + if raw: + return datetime.datetime.strptime(raw, datetime_format) + else: + return default + + def last_configured(self, default=None): + raw = self.__get('last_configured') + if raw: + return datetime.datetime.strptime(raw, datetime_format) + else: + return default + + def seen(self): + now = datetime.datetime.now().strftime(datetime_format) + self.__set('last_seen', now) + + def configure(self): + # TODO: configure. + now = datetime.datetime.now().strftime(datetime_format) + self.__set('last_configured', now) class Trigger(object): """ @@ -108,48 +171,43 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces, args=None, outdir=None): + def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None): self.interfaces = interfaces + self.autoconfigure=autoconfigure + self.name_mapper = name_mapper + self.config_delay = datetime.timedelta(seconds=3600) if outdir: self.outdir = outdir else: self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan') + os.makedirs(self.outdir, exist_ok=True) + + self.running_configs = {} def handle_pkg(self, pkg): if ICMPv6EchoReply in pkg: - host = pkg['IPv6'].src - log.verbose("Host %s is alive", host) + host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper) + if host.name(): + log.verbose("Host %s (%s) is alive", host.name(), host.address()) + else: + log.verbose("Host %s is alive", host.address()) + host.seen() - dir = os.path.join(self.outdir, host) - fname = os.path.join(dir, "last_seen") - - now = datetime.datetime.now() - - os.makedirs(dir, exist_ok=True) - - # FIXME: maybe adjust the format so we can easily parse again - with open(fname, "w") as fd: - fd.write(f"{now}\n") + # TODO check last config. + if self.autoconfigure and \ + host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now(): + self.config(host) def list(self): - hosts = dict() - for linklocal_addr in os.listdir(self.outdir): - workdir = os.path.join(self.outdir, linklocal_addr) - # We ignore any (unexpected) file in this directory. - if os.path.isdir(workdir): - last_seen='-' - last_seen_file = os.path.join(workdir, 'last_seen') - if os.path.isfile(last_seen_file): - with open(last_seen_file, "r") as fd: - last_seen = fd.readline() - - hosts[linklocal_addr] = {'last_seen': last_seen} + hosts = [] + for addr in os.listdir(self.outdir): + hosts.append(Host(addr, self.outdir, self.name_mapper)) return hosts - def config(self): + def config(self, host): """ Configure a host @@ -158,9 +216,19 @@ class Scanner(object): - Maybe keep dict storing per host processes - Save the result - Save the output -> probably aligned to config mode - """ + if host.name() == None: + log.debug("config - could not resolve name for %s, aborting.", host.address()) + return + + if self.running_configs.get(host.name()) != None: + log.debug("config - is already running for %s, aborting.", host.name()) + + log.info("config - running against host %s.", host.name()) + p = host.configure() + self.running_configs[host.name()] = p + def start(self): self.process = Process(target=self.scan) self.process.start() From 512e9b23c0035b09ac1e7e76f706fee2e2ec481f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 25 Apr 2021 15:53:40 +0200 Subject: [PATCH 051/128] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 70d0b960..c41743cd 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * New type: __postgres_conf (Beni Ruef, Dennis Camera) + 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) * Core: config: Make local state directory available to custom remotes (Steven Armstrong From 6ac8cbf98f5d5018373db6eb9c038ef1c599ff2f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 16:33:56 +0200 Subject: [PATCH 052/128] [type/__postgres_database] Include postgres_user explorer from __postgres_conf --- .../explorer/postgres_user | 1 + .../type/__postgres_database/explorer/state | 23 +++++-------------- .../type/__postgres_database/gencode-remote | 13 +---------- 3 files changed, 8 insertions(+), 29 deletions(-) create mode 120000 cdist/conf/type/__postgres_database/explorer/postgres_user diff --git a/cdist/conf/type/__postgres_database/explorer/postgres_user b/cdist/conf/type/__postgres_database/explorer/postgres_user new file mode 120000 index 00000000..714e7237 --- /dev/null +++ b/cdist/conf/type/__postgres_database/explorer/postgres_user @@ -0,0 +1 @@ +../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state index d68d4120..bb044e0e 100755 --- a/cdist/conf/type/__postgres_database/explorer/state +++ b/cdist/conf/type/__postgres_database/explorer/state @@ -1,6 +1,7 @@ #!/bin/sh # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,25 +19,13 @@ # along with cdist. If not, see . # -case "$("${__explorer}/os")" -in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; -esac +postgres_user=$("${__type_explorer:?}/postgres_user") +name=${__object_id:?} -name="$__object_id" - -if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='$name'\"")" +if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='${name}'\"")" then - echo 'present' + echo 'present' else - echo 'absent' + echo 'absent' fi diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote index 0f11cff4..d70fc436 100755 --- a/cdist/conf/type/__postgres_database/gencode-remote +++ b/cdist/conf/type/__postgres_database/gencode-remote @@ -18,23 +18,12 @@ # along with cdist. If not, see . # -case "$(cat "${__global}/explorer/os")" -in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; -esac name="$__object_id" state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" +postgres_user=$(cat "${__object:?}/explorer/postgres_user") if [ "$state_should" != "$state_is" ]; then case "$state_should" in From 58b279a8d0121dcbef05a4999d1d88208656aeed Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 16:34:17 +0200 Subject: [PATCH 053/128] [type/__postgres_database] Improve quoting --- .../type/__postgres_database/gencode-remote | 95 +++++++++++-------- 1 file changed, 55 insertions(+), 40 deletions(-) diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote index d70fc436..7d7d6fa2 100755 --- a/cdist/conf/type/__postgres_database/gencode-remote +++ b/cdist/conf/type/__postgres_database/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,49 +19,63 @@ # along with cdist. If not, see . # +quote() { + for _arg + do + shift + if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')" + then + # needs quoting + set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'" + else + set -- "$@" "${_arg}" + fi + done + unset _arg + # NOTE: Use printf because POSIX echo interprets escape sequences + printf '%s' "$*" +} -name="$__object_id" -state_should="$(cat "$__object/parameter/state")" -state_is="$(cat "$__object/explorer/state")" postgres_user=$(cat "${__object:?}/explorer/postgres_user") -if [ "$state_should" != "$state_is" ]; then - case "$state_should" in - present) - owner="" - if [ -f "$__object/parameter/owner" ]; then - owner="-O \"$(cat "$__object/parameter/owner")\"" - fi +dbname=${__object_id:?} +state_should=$(cat "${__object:?}/parameter/state") +state_is=$(cat "${__object:?}/explorer/state") - template="" - if [ -f "$__object/parameter/template" ]; then - template="--template \"$(cat "$__object/parameter/template")\"" - fi - - encoding="" - if [ -f "$__object/parameter/encoding" ]; then - encoding="--encoding \"$(cat "$__object/parameter/encoding")\"" - fi - - lc_collate="" - if [ -f "$__object/parameter/lc-collate" ]; then - lc_collate="--lc-collate \"$(cat "$__object/parameter/lc-collate")\"" - fi - - lc_ctype="" - if [ -f "$__object/parameter/lc-ctype" ]; then - lc_ctype="--lc-ctype \"$(cat "$__object/parameter/lc-ctype")\"" - fi - - cat << EOF -su - '$postgres_user' -c "createdb $owner \"$name\" $template $encoding $lc_collate $lc_ctype" -EOF - ;; - absent) - cat << EOF -su - '$postgres_user' -c "dropdb \"$name\"" -EOF - ;; - esac +if test "${state_should}" = "$state_is" +then + exit 0 fi + +case ${state_should} +in + (present) + set -- + + while read -r param_name opt + do + if test -f "${__object:?}/parameter/${param_name}" + then + set -- "$@" "${opt}" "$(cat "${__object:?}/parameter/${param_name}")" + fi + done <<-'EOF' + owner -O + template --template + encoding --encoding + lc_collate --lc-collate + lc_ctype --lc-ctype + EOF + + set -- "$@" "${dbname}" + + cat <<-EOF + su - $(quote "${postgres_user}") -c $(quote "$(quote createdb "$@")") + EOF + ;; + (absent) + cat <<-EOF + su - $(quote "${postgres_user}") -c $(quote "$(quote dropdb "${dbname}")") + EOF + ;; +esac From beb8da6d5fd141ac63eacfb526cfcc43cbf22d8f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 16:48:51 +0200 Subject: [PATCH 054/128] [type/__postgres_role] Include postgres_user explorer from __postgres_conf --- .../type/__postgres_role/explorer/postgres_user | 1 + cdist/conf/type/__postgres_role/explorer/state | 14 +------------- cdist/conf/type/__postgres_role/gencode-remote | 15 +-------------- 3 files changed, 3 insertions(+), 27 deletions(-) create mode 120000 cdist/conf/type/__postgres_role/explorer/postgres_user diff --git a/cdist/conf/type/__postgres_role/explorer/postgres_user b/cdist/conf/type/__postgres_role/explorer/postgres_user new file mode 120000 index 00000000..714e7237 --- /dev/null +++ b/cdist/conf/type/__postgres_role/explorer/postgres_user @@ -0,0 +1 @@ +../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 34069de9..0b264e6f 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -19,19 +19,7 @@ # along with cdist. If not, see . # -case $("${__explorer:?}/os") -in - (netbsd) - postgres_user='pgsql' - ;; - (openbsd) - postgres_user='_postgresql' - ;; - (*) - postgres_user='postgres' - ;; -esac - +postgres_user=$("${__type_explorer:?}/postgres_user") rolename=${__object_id:?} diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index d7631fbd..7324b80c 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -28,20 +28,7 @@ quote() { fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" } -case $(cat "${__global:?}/explorer/os") -in - (netbsd) - postgres_user='pgsql' - ;; - (openbsd) - postgres_user='_postgresql' - ;; - (*) - postgres_user='postgres' - ;; -esac - - +postgres_user=$(cat "${__object:?}/explorer/postgres_user") rolename=${__object_id:?} state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") From 3cf93249c38a6f27c097877add619b5d6499a049 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 16:55:11 +0200 Subject: [PATCH 055/128] [type/__postgres_extension] Include postgres_user explorer from __postgres_conf --- .../__postgres_extension/explorer/postgres_user | 1 + cdist/conf/type/__postgres_extension/gencode-remote | 13 +------------ 2 files changed, 2 insertions(+), 12 deletions(-) create mode 120000 cdist/conf/type/__postgres_extension/explorer/postgres_user diff --git a/cdist/conf/type/__postgres_extension/explorer/postgres_user b/cdist/conf/type/__postgres_extension/explorer/postgres_user new file mode 120000 index 00000000..714e7237 --- /dev/null +++ b/cdist/conf/type/__postgres_extension/explorer/postgres_user @@ -0,0 +1 @@ +../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote index af9c97f1..a1c84828 100755 --- a/cdist/conf/type/__postgres_extension/gencode-remote +++ b/cdist/conf/type/__postgres_extension/gencode-remote @@ -22,18 +22,7 @@ # along with cdist. If not, see . # -case "$(cat "${__global}/explorer/os")" -in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; -esac +postgres_user=$(cat "${__object:?}/explorer/postgres_user") dbname=$( echo "$__object_id" | cut -d":" -f1 ) From 8296051653a8187374ad51a3a487315767b73a19 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 16 Apr 2021 19:22:58 +0200 Subject: [PATCH 056/128] [type/__postgres_extension] Add state explorer --- .../type/__postgres_extension/explorer/state | 41 ++++++++++++++ .../type/__postgres_extension/gencode-remote | 44 ++++++++++----- cdist/conf/type/__postgres_extension/man.rst | 55 +++++++++++-------- 3 files changed, 104 insertions(+), 36 deletions(-) create mode 100644 cdist/conf/type/__postgres_extension/explorer/state diff --git a/cdist/conf/type/__postgres_extension/explorer/state b/cdist/conf/type/__postgres_extension/explorer/state new file mode 100644 index 00000000..9d156be7 --- /dev/null +++ b/cdist/conf/type/__postgres_extension/explorer/state @@ -0,0 +1,41 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Prints "present" if the extension is currently installed. +# "absent" otherwise. + +quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } + +postgres_user=$("${__type_explorer:?}/postgres_user") + +IFS=: read -r dbname extname <&2 + exit 1 + ;; esac diff --git a/cdist/conf/type/__postgres_extension/man.rst b/cdist/conf/type/__postgres_extension/man.rst index 79645b2b..442239f6 100644 --- a/cdist/conf/type/__postgres_extension/man.rst +++ b/cdist/conf/type/__postgres_extension/man.rst @@ -3,32 +3,36 @@ cdist-type__postgres_extension(7) NAME ---- -cdist-type__postgres_extension - manage postgres extensions +cdist-type__postgres_extension - Manage PostgreSQL extensions DESCRIPTION ----------- -This cdist type allows you to create or drop postgres extensions. +This cdist type allows you to manage PostgreSQL extensions. -The object you need to pass to __postgres_extension consists of -the database name and the extension name joined by a colon in the -following form: - -.. code-block:: sh - - dbname:extension - -f.ex. +The ``__object_id`` to pass to ``__postgres_extension`` is of the form +``dbname:extension``, e.g.: .. code-block:: sh rails_test:unaccent +**CAUTION!** Be careful when installing extensions from (untrusted) third-party +sources: + + | Installing an extension as superuser requires trusting that the extension's + author wrote the extension installation script in a secure fashion. It is + not terribly difficult for a malicious user to create trojan-horse objects + that will compromise later execution of a carelessly-written extension + script, allowing that user to acquire superuser privileges. + | – ``_ + + OPTIONAL PARAMETERS ------------------- state - either "present" or "absent", defaults to "present" + either ``present`` or ``absent``, defaults to ``present``. EXAMPLES @@ -36,24 +40,29 @@ EXAMPLES .. code-block:: sh - __postgres_extension rails_test:unaccent - __postgres_extension --present rails_test:unaccent - __postgres_extension --absent rails_test:unaccent + # Install extension unaccent into database rails_test + __postgres_extension rails_test:unaccent + + # Drop extension unaccent from database fails_test + __postgres_extension rails_test:unaccent --state absent SEE ALSO -------- -:strong:`cdist-type__postgre_database`\ (7) +- :strong:`cdist-type__postgres_database`\ (7) +- PostgreSQL "CREATE EXTENSION" documentation at: + ``_. -Postgres "Create Extension" documentation at: . -AUTHOR +AUTHORS ------- -Tomas Pospisek +| Tomas Pospisek +| Dennis Camera + COPYING ------- -Copyright \(C) 2014 Tomas Pospisek. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2014 Tomas Pospisek, 2021 Dennis Camera. +You can redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. From 0d33407b182964672b1cfc3ffc49c7fa7cf462a0 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 16 Apr 2021 19:35:26 +0200 Subject: [PATCH 057/128] [type/__postgres_database] Proper quoting in state explorer --- cdist/conf/type/__postgres_database/explorer/state | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state index bb044e0e..6a25df86 100755 --- a/cdist/conf/type/__postgres_database/explorer/state +++ b/cdist/conf/type/__postgres_database/explorer/state @@ -21,9 +21,14 @@ postgres_user=$("${__type_explorer:?}/postgres_user") -name=${__object_id:?} +dbname=${__object_id:?} -if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='${name}'\"")" +quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } +psql_exec() { + su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")" +} + +if psql_exec postgres "SELECT datname FROM pg_database" | grep -qFx "${dbname}" then echo 'present' else From 0f05f38384c94b79232ca0917d1323eeefd6e1ed Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 19 Apr 2021 19:09:46 +0200 Subject: [PATCH 058/128] [type/__postgres_role] Treat --password '' like no --password --- cdist/conf/type/__postgres_role/explorer/state | 11 ++++------- cdist/conf/type/__postgres_role/gencode-remote | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 0b264e6f..822816c1 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -43,8 +43,7 @@ role_properties=$( BEGIN { RS = "\036"; FS = "\034" } /^\([0-9]+ rows?\)/ { exit } NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next } - NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i } - ' + NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }' ) if test -n "${role_properties}" @@ -78,12 +77,10 @@ then # Check password passwd_stored=$( psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \ - | awk 'BEGIN { RS = "\036" } NR == 2' - printf . - ) - passwd_stored=${passwd_stored%?.} + | awk 'BEGIN { RS = "\036" } NR == 2 { printf "%s.", $0 }') + passwd_stored=${passwd_stored%.} - if test -f "${__object:?}/parameter/password" + if test -s "${__object:?}/parameter/password" then passwd_should=$(cat "${__object:?}/parameter/password"; printf .) fi diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 7324b80c..4cb78330 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -46,7 +46,7 @@ psql_query() { psql_set_password() { # NOTE: Always make sure that the password does not end up in psql_history! - # NOTE: Never set an empty string as the password, because they can be + # NOTE: Never set an empty string as the password, because it can be # interpreted differently by different tooling. if test -s "${__object:?}/parameter/password" then From 92fff7cb77271009f25d999b6451dc1e43e9bb6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 26 Apr 2021 12:09:20 +0200 Subject: [PATCH 059/128] [scanner] fix crash on --list with name mapper provided --- cdist/scan/commandline.py | 10 ++++++++-- cdist/scan/scan.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 1a0ab0a7..3eb7eec4 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -65,11 +65,17 @@ def list(scan, args): 'last configured'.ljust(date_max_size))) print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) for host in hosts: + last_seen = host.last_seen() + last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-' + + last_configured = host.last_configured() + last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-' + print("{} | {} | {} | {}".format( host.name(default='-').ljust(name_max_size), host.address().ljust(ipv6_max_size), - host.last_seen().ljust(date_max_size), - host.last_configured().ljust(date_max_size))) + last_seen.ljust(date_max_size), + last_configured.ljust(date_max_size))) # CLI processing is defined outside of the main scan class to handle # non-available optional scapy dependency (instead of crashing mid-flight). diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 459138e2..69a4121d 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -97,7 +97,7 @@ class Host(object): return default else: value = out.stdout.decode() - return (None if len(value) == 0 else value) + return (default if len(value) == 0 else value) else: return default From 3a9dd5b1669fa29e1713cdadec1596d859d377f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 26 Apr 2021 12:09:55 +0200 Subject: [PATCH 060/128] [scanner] add minimal (non-configurable) config mode --- cdist/scan/scan.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 69a4121d..152abb4e 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -122,8 +122,20 @@ class Host(object): now = datetime.datetime.now().strftime(datetime_format) self.__set('last_seen', now) + # XXX: There's no easy way to use the config module without feeding it with + # CLI args. Might as well call everything from scratch! def configure(self): - # TODO: configure. + target = self.name() or self.address() + cmd = ['cdist', 'config', '-v', target ] + + fname = os.path.join(self.workdir, 'last_configuration_log') + with open(fname, "w") as fd: + log.debug("Executing: %s", cmd) + completed_process = subprocess.run(cmd, stdout=fd, stderr=fd) + if completed_process.returncode != 0: + log.error("%s return with non-zero code %i - see %s for details.", + cmd, completed_process.returncode, fname) + now = datetime.datetime.now().strftime(datetime_format) self.__set('last_configured', now) @@ -194,7 +206,7 @@ class Scanner(object): log.verbose("Host %s is alive", host.address()) host.seen() - # TODO check last config. + # Configure if needed. if self.autoconfigure and \ host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now(): self.config(host) @@ -206,27 +218,18 @@ class Scanner(object): return hosts - def config(self, host): - """ - Configure a host - - - Assume we are only called if necessary - - However we need to ensure to not run in parallel - - Maybe keep dict storing per host processes - - Save the result - - Save the output -> probably aligned to config mode - """ - if host.name() == None: log.debug("config - could not resolve name for %s, aborting.", host.address()) return - if self.running_configs.get(host.name()) != None: + previous_config_process = self.running_configs.get(host.name()) + if previous_config_process != None and previous_config_process.is_alive(): log.debug("config - is already running for %s, aborting.", host.name()) - log.info("config - running against host %s.", host.name()) - p = host.configure() + log.info("config - running against host %s (%s).", host.name(), host.address()) + p = Process(target=host.configure()) + p.start() self.running_configs[host.name()] = p def start(self): From 2232435c2206b2b3a9a5f2b976df16540b3f1eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 26 Apr 2021 14:39:26 +0200 Subject: [PATCH 061/128] [scanner] initial documentation Note: still needs to patch main cdist(1) manpage --- cdist/scan/scan.py | 32 ------------- docs/src/cdist-scan.rst | 99 +++++++++++++++++++++++++++++++++++++++++ docs/src/index.rst | 1 + 3 files changed, 100 insertions(+), 32 deletions(-) create mode 100644 docs/src/cdist-scan.rst diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 152abb4e..2912dab3 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -19,38 +19,6 @@ # # -# -# Interface to be implemented: -# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated -# scan: scan / listen for icmp6 replies -# trigger: send trigger to multicast -# config: configure newly detected hosts -# install: install newly detected hosts -# -# Scanner logic -# - save results to configdir: -# basedir = ~/.cdist/scan/ -# last_seen = ~/.cdist/scan//last_seen -- record unix time -# or similar -# last_configured = ~/.cdist/scan//last_configured -- record -# unix time or similar -# last_installed = ~/.cdist/scan//last_configured -- record -# unix time or similar -# -# -# -# -# cdist scan --list -# Show all known hosts including last seen flag -# -# Logic for reconfiguration: -# -# - record when configured last time -# - introduce a parameter --reconfigure-after that takes time argument -# - reconfigure if a) host alive and b) reconfigure-after time passed -# - - from multiprocessing import Process import os import logging diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst new file mode 100644 index 00000000..02193456 --- /dev/null +++ b/docs/src/cdist-scan.rst @@ -0,0 +1,99 @@ +Scan +===== + +Description +----------- +Runs cdist as a daemon that discover/watch on hosts and reconfigure them +periodically. It is especially useful in netboot-based environment where hosts +boot unconfigured, and to ensure your infrastructure stays in sync with your +configuration. + +This feature is still consider to be in **beta** stage. + +Usage (Examples) +---------------- + +Discover hosts on local network and configure those whose name is resolved by +the name mapper script. + +.. code-block:: sh + + $ cdist scan --beta --interface eth0 \ + --mode scan --name-mapper path/to/script \ + --mode trigger --mode config + +List known hosts and exit. + +.. code-block:: sh + + $ cdist scan --beta --list --name-mapper path/to/script + +Please refer to `cdist(1)` for a detailed list of parameters. + +Modes +----- + +The scanner has 3 modes that can be independently toggled. If the `--mode` +parameter is not specified, only `tigger` and `scan` are enabled (= hosts are +not configured). + +trigger + Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to + trigger detection by the `scan` module. + +scan + Watch for incoming ICMPv6 replies and optionally configure detected hosts. + +config + Enable configuration of hosts detected by `scan`. + +Name Mapper Script +------------------ + +The name mapper script takes an IPv6 address as first argument and writes the +resolved name to stdout - if any. The script must be executable. + +Simplest script: + +.. code-block:: sh + #!/bin/sh + + case "$1" in + "fe80::20d:b9ff:fe57:3524") + printf "my-host-01" + ;; + "fe80::7603:bdff:fe05:89bb") + printf "my-host-02" + ;; + esac + +Resolving name from `PTR` DNS record: + +.. code-block:: sh + #!/bin/sh + + for cmd in dig sed; do + if ! command -v $cmd > /dev/null; then + exit 1 + fi + done + + dig +short -x "$1" | sed -e 's/.$//' + + +Trigger Source Script +--------------------- + +This script returns a list of addresses (separated by a newline) to be used by +`trigger` mode. It is not used to map names. The script must be executable. + +Simplest script: + +.. code-block:: sh + #!/bin/sh + + cat << EOF + server1.domain.tld + server2.domain.tld + server3.domain.tld + EOF diff --git a/docs/src/index.rst b/docs/src/index.rst index 31c044dc..369d5309 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -34,6 +34,7 @@ It natively supports IPv6 since the first release. cdist-parallelization cdist-inventory cdist-preos + cdist-scan cdist-integration cdist-reference cdist-best-practice From a4122882f2a14034d1a219bd0314892176c5cb64 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 26 Apr 2021 16:39:51 +0200 Subject: [PATCH 062/128] [type/__debconf_set_selections] Add state explorer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …and to make it work, replace --file with --line. --file is deprecated because it does not work with the state explorer as the contents of the file are not available on the target. --- .../__debconf_set_selections/explorer/state | 124 ++++++++++++++++++ .../__debconf_set_selections/gencode-remote | 37 ++++-- .../type/__debconf_set_selections/man.rst | 49 ++++--- .../type/__debconf_set_selections/manifest | 21 +++ .../parameter/{required => deprecated} | 0 .../parameter/optional_multiple | 1 + 6 files changed, 206 insertions(+), 26 deletions(-) create mode 100644 cdist/conf/type/__debconf_set_selections/explorer/state create mode 100755 cdist/conf/type/__debconf_set_selections/manifest rename cdist/conf/type/__debconf_set_selections/parameter/{required => deprecated} (100%) create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/optional_multiple diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state new file mode 100644 index 00000000..68d28941 --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/explorer/state @@ -0,0 +1,124 @@ +#!/bin/sh -e +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Determine current debconf selections' state. +# Prints one of: +# present: all selections are already set as they should. +# different: one or more of the selections have a different value. +# absent: one or more of the selections are not (currently) defined. +# + +test -x /usr/bin/perl || { + # cannot find perl (no perl ~ no debconf) + echo 'absent' + exit 0 +} + +linesfile="${__object:?}/parameter/line" +test -s "${linesfile}" || { + if test -s "${__object:?}/parameter/file" + then + echo absent + else + echo present + fi + exit 0 +} + +/usr/bin/perl -- - "${linesfile}" <<'EOF' +use strict; +use warnings "all"; + +use Debconf::Db; +use Debconf::Question; + +# Extract @known... arrays from debconf-set-selections +# These values are required to distinguish flags and values in the given lines. +# DC: I couldn't think of a more ugly solution to the problem… +my @knownflags; +my @knowntypes; +my $debconf_set_selections = '/usr/bin/debconf-set-selections'; +if (-e $debconf_set_selections) { + my $sed_known = 's/^my \(@known\(flags\|types\) = qw([a-z ]*);\).*$/\1/p'; + eval `sed -n '$sed_known' '$debconf_set_selections'`; +} + +sub mungeline ($) { + my $line = shift; + chomp $line; + $line =~ s/\r$//; + return $line; +} + +sub fatal { printf STDERR @_; exit 1; } + +my $state = 'present'; + +sub state { + my $new = shift; + if ($state eq 'present' + or ($state eq 'different' and $new eq 'absent')) { + $state = $new; + } +} + +Debconf::Db->load(readonly => 'true'); + +while (<>) { + # Read and process lines (taken from debconf-set-selections) + $_ = mungeline($_); + while (/\\$/ && ! eof) { + s/\\$//; + $_ .= mungeline(<>); + } + next if /^\s*$/ || /^\s*\#/; + + my ($owner, $label, $type, $content) = /^\s*(\S+)\s+(\S+)\s+(\S+)(?:\s(.*))?/ + or fatal "invalid line: %s\n", $_; + $content = '' unless defined $content; + + + # Compare is and should state + my $q = Debconf::Question->get($label); + + unless (defined $q) { + # probably a preseed + state 'absent'; + next; + } + + if (grep { $_ eq $q->type } @knownflags) { + # This line wants to set a flag, presumably. + if ($q->flag($q->type) ne $content) { + state 'different'; + } + } else { + # Otherwise, it's probably a value… + if ($q->value ne $content) { + state 'different'; + } + + unless (grep { $_ eq $owner } (split /, /, $q->owners)) { + state 'different'; + } + } +} + +printf "%s\n", $state; +EOF diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote index e99aef40..50d898c6 100755 --- a/cdist/conf/type/__debconf_set_selections/gencode-remote +++ b/cdist/conf/type/__debconf_set_selections/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org) +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,16 +18,32 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# -# Setup selections -# -filename="$(cat "$__object/parameter/file")" - -if [ "$filename" = "-" ]; then - filename="$__object/stdin" +if test -f "${__object:?}/parameter/line" +then + filename="${__object:?}/parameter/line" +elif test -s "${__object:?}/parameter/file" +then + filename=$(cat "${__object:?}/parameter/file") + if test "${filename}" = '-' + then + filename="${__object:?}/stdin" + fi +else + printf 'Neither --line nor --file set.\n' >&2 + exit 1 fi -echo "debconf-set-selections << __file-eof" -cat "$filename" -echo "__file-eof" +# setting no lines makes no sense +test -s "${filename}" || exit 0 + +state_is=$(cat "${__object:?}/explorer/state") + +if test "${state_is}" != 'present' +then + cat <<-CODE + debconf-set-selections <<'EOF' + $(cat "${filename}") + EOF + CODE +fi diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst index 58c25b81..690e3e49 100644 --- a/cdist/conf/type/__debconf_set_selections/man.rst +++ b/cdist/conf/type/__debconf_set_selections/man.rst @@ -8,15 +8,33 @@ cdist-type__debconf_set_selections - Setup debconf selections DESCRIPTION ----------- -On Debian and alike systems debconf-set-selections(1) can be used +On Debian and alike systems :strong:`debconf-set-selections`\ (1) can be used to setup configuration parameters. REQUIRED PARAMETERS ------------------- +cf. ``--line``. + + +OPTIONAL PARAMETERS +------------------- file - Use the given filename as input for debconf-set-selections(1) - If filename is "-", read from stdin. + Use the given filename as input for :strong:`debconf-set-selections`\ (1) + If filename is ``-``, read from stdin. + + **This parameter is deprecated, because it doesn't work with state detection.** +line + A line in :strong:`debconf-set-selections`\ (1) compatible format. + This parameter can be used multiple times to set multiple options. + + (This parameter is actually required, but marked optional because the + deprecated ``--file`` is still accepted.) + + +BOOLEAN PARAMETERS +------------------ +None. EXAMPLES @@ -24,30 +42,29 @@ EXAMPLES .. code-block:: sh - # Setup configuration for nslcd - __debconf_set_selections nslcd --file /path/to/file + # Setup gitolite's gituser + __debconf_set_selections nslcd --line 'gitolite gitolite/gituser string git' - # Setup configuration for nslcd from another type - __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd" - - __debconf_set_selections nslcd --file - << eof - gitolite gitolite/gituser string git - eof + # Setup configuration for nslcd from a file. + # NB: Multiple lines can be passed to --line, although this can be considered a hack. + __debconf_set_selections nslcd --line "$(cat "${__files:?}/preseed/nslcd.debconf")" SEE ALSO -------- -:strong:`debconf-set-selections`\ (1), :strong:`cdist-type__update_alternatives`\ (7) +- :strong:`cdist-type__update_alternatives`\ (7) +- :strong:`debconf-set-selections`\ (1) AUTHORS ------- Nico Schottelius +Dennis Camera COPYING ------- -Copyright \(C) 2011-2014 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2011-2014 Nico Schottelius, 2021 Dennis Camera. +You can redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. diff --git a/cdist/conf/type/__debconf_set_selections/manifest b/cdist/conf/type/__debconf_set_selections/manifest new file mode 100755 index 00000000..0f4fb2e2 --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/manifest @@ -0,0 +1,21 @@ +#!/bin/sh -e +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +__package_apt debconf diff --git a/cdist/conf/type/__debconf_set_selections/parameter/required b/cdist/conf/type/__debconf_set_selections/parameter/deprecated similarity index 100% rename from cdist/conf/type/__debconf_set_selections/parameter/required rename to cdist/conf/type/__debconf_set_selections/parameter/deprecated diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple new file mode 100644 index 00000000..a999a0c2 --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple @@ -0,0 +1 @@ +line From 9cf19388abe95a5f8df9f5c94d2e54fa1060b2ef Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 26 Apr 2021 16:47:44 +0200 Subject: [PATCH 063/128] [type/__debconf_set_selections] Send message about each debconf setting that is changed --- cdist/conf/type/__debconf_set_selections/gencode-remote | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote index 50d898c6..9ba28f09 100755 --- a/cdist/conf/type/__debconf_set_selections/gencode-remote +++ b/cdist/conf/type/__debconf_set_selections/gencode-remote @@ -46,4 +46,9 @@ then $(cat "${filename}") EOF CODE + + awk ' + { + printf "set %s %s %s %s\n", $1, $2, $3, $4 + }' "${filename}" >>"${__messages_out:?}" fi From 3a25b804664963e092d1f75c1fdded25d7016ef3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 26 Apr 2021 21:27:15 +0200 Subject: [PATCH 064/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index c41743cd..497ae91a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * New type: __postgres_conf (Beni Ruef, Dennis Camera) + * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From a42ebc7a78e09e138954b1376376f7cb643420e0 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 27 Apr 2021 19:41:10 +0200 Subject: [PATCH 065/128] [type/__debconf_set_selections] Synchronise objects Works around locking error: debconf: DbDriver "config": /var/cache/debconf/config.dat is locked by another process: Resource temporarily unavailable --- .../__debconf_set_selections/explorer/state | 20 ++++++++++++++++++- .../type/__debconf_set_selections/nonparallel | 0 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 cdist/conf/type/__debconf_set_selections/nonparallel diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state index 68d28941..f8a3f6c8 100644 --- a/cdist/conf/type/__debconf_set_selections/explorer/state +++ b/cdist/conf/type/__debconf_set_selections/explorer/state @@ -41,10 +41,15 @@ test -s "${linesfile}" || { exit 0 } +# assert __type_explorer is set (because it is used by the Perl script) +: "${__type_explorer:?}" + /usr/bin/perl -- - "${linesfile}" <<'EOF' use strict; use warnings "all"; +use Fcntl qw(:DEFAULT :flock); + use Debconf::Db; use Debconf::Question; @@ -78,7 +83,20 @@ sub state { } } -Debconf::Db->load(readonly => 'true'); + +# Load Debconf DB but manually lock on the state explorer script, +# because Debconf aborts immediately if executed concurrently. +# This is not really an ideal solution because the Debconf DB could be locked by +# another process (e.g. apt-get), but no way to achieve this could be found. +# If you know how to, please provide a patch. +my $lockfile = "%ENV{'__type_explorer'}/state"; +if (open my $lock_fh, '+<', $lockfile) { + flock $lock_fh, LOCK_EX or die "Cannot lock $lockfile"; +} +{ + Debconf::Db->load(readonly => 'true'); +} + while (<>) { # Read and process lines (taken from debconf-set-selections) diff --git a/cdist/conf/type/__debconf_set_selections/nonparallel b/cdist/conf/type/__debconf_set_selections/nonparallel new file mode 100644 index 00000000..e69de29b From c00c8c20127af9881b4e295b7732524e7e1c5031 Mon Sep 17 00:00:00 2001 From: Evil Ham Date: Mon, 10 May 2021 12:08:22 +0200 Subject: [PATCH 066/128] [__apt_key*] Deprecate __apt_key_uri and improve __apt_key Previously this type was falling back to using the deprecated apt-key(8) by checking for existence of files/directories on the controller host in gencode-remote. Adding `--use-deprecated-apt-key` as an explicit boolean serves two purposes: 1. It prevents fallbacks that might end up doing the wrong thing (as was the case) 2. It allows for a simple way to remove keys from the keyring that were previously added with apt-key(8) to /etc/apt/trusted.gpg This parameter is added marked as deprecated as is only intended use is to migrate to directory-based keyrings as recommended by Debian for a few releases. It will be removed when Debian 11 stops being supported. During the review process of this merge request, it was noted that the state of PGP Key Servers is somewhat suboptimal, that the examples encouraged bad practise (it is trivial to produce collisions for short key IDs), and that this use does not require the Web of Trust, but instead only the public key that is signing the repository. That is why this also adds `--source` as an argument allowing for in-type or in-manifest provision of such public keys by the type/manifest maintainer and the use of Key Servers is still supported, but discouraged. --- cdist/conf/type/__apt_key/explorer/state | 27 +++-- cdist/conf/type/__apt_key/gencode-remote | 53 ++++----- cdist/conf/type/__apt_key/man.rst | 84 ++++++++++---- cdist/conf/type/__apt_key/manifest | 104 +++++++++++++++++- cdist/conf/type/__apt_key/parameter/boolean | 1 + .../deprecated/use-deprecated-apt-key | 3 + cdist/conf/type/__apt_key/parameter/optional | 5 +- cdist/conf/type/__apt_key_uri/deprecated | 1 + 8 files changed, 209 insertions(+), 69 deletions(-) create mode 100644 cdist/conf/type/__apt_key/parameter/boolean create mode 100644 cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key create mode 100644 cdist/conf/type/__apt_key_uri/deprecated diff --git a/cdist/conf/type/__apt_key/explorer/state b/cdist/conf/type/__apt_key/explorer/state index 38f1bd3c..8ab268c1 100755 --- a/cdist/conf/type/__apt_key/explorer/state +++ b/cdist/conf/type/__apt_key/explorer/state @@ -27,18 +27,25 @@ else keyid="$__object_id" fi +# From apt-key(8): +# Use of apt-key is deprecated, except for the use of apt-key del in +# maintainer scripts to remove existing keys from the main keyring. +# If such usage of apt-key is desired the additional installation of +# the GNU Privacy Guard suite (packaged in gnupg) is required. +if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then + if apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" + then echo present + else echo absent + fi + exit +fi + keydir="$(cat "$__object/parameter/keydir")" keyfile="$keydir/$__object_id.gpg" -if [ -d "$keydir" ] +if [ -f "$keyfile" ] then - if [ -f "$keyfile" ] - then echo present - else echo absent - fi -else - # fallback to deprecated apt-key - apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \ - && echo present \ - || echo absent + echo present + exit fi +echo absent diff --git a/cdist/conf/type/__apt_key/gencode-remote b/cdist/conf/type/__apt_key/gencode-remote index 0c96ff67..17dc9bfc 100755 --- a/cdist/conf/type/__apt_key/gencode-remote +++ b/cdist/conf/type/__apt_key/gencode-remote @@ -25,11 +25,7 @@ else fi state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" - -if [ "$state_should" = "$state_is" ]; then - # nothing to do - exit 0 -fi +method="$(cat "$__object/key_method")" keydir="$(cat "$__object/parameter/keydir")" keyfile="$keydir/$__object_id.gpg" @@ -37,30 +33,18 @@ keyfile="$keydir/$__object_id.gpg" case "$state_should" in present) keyserver="$(cat "$__object/parameter/keyserver")" - - if [ -f "$__object/parameter/uri" ]; then - uri="$(cat "$__object/parameter/uri")" - - if [ -d "$keydir" ]; then - cat << EOF - -curl -s -L \\ - -o "$keyfile" \\ - "$uri" - -key="\$( cat "$keyfile" )" - -if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK' -then - echo "\$key" | gpg --dearmor > "$keyfile" -fi - -EOF - else - # fallback to deprecated apt-key - echo "curl -s -L '$uri' | apt-key add -" + # Using __download or __file as key source + # Propagate messages if needed + if [ "${method}" = "uri" ] || [ "${method}" = "source" ]; then + if grep -Eq "^__(file|download)$keyfile" "$__messages_in"; then + echo "added '$keyid'" >> "$__messages_out" fi - elif [ -d "$keydir" ]; then + exit 0 + elif [ "${state_is}" = "present" ]; then + exit 0 + fi + # Using key servers to fetch the key + if [ ! -f "$__object/parameter/use-deprecated-apt-key" ]; then # we need to kill gpg after 30 seconds, because gpg # can get stuck if keyserver is not responding. # exporting env var and not exit 1, @@ -100,13 +84,16 @@ EOF echo "added '$keyid'" >> "$__messages_out" ;; absent) - if [ -f "$keyfile" ]; then - echo "rm '$keyfile'" - else + # Removal for keys added from a keyserver without this flag + # is done in the manifest + if [ "$state_is" != "absent" ] && \ + [ -f "$__object/parameter/use-deprecated-apt-key" ]; then # fallback to deprecated apt-key echo "apt-key del \"$keyid\"" + echo "removed '$keyid'" >> "$__messages_out" + # Propagate messages if needed + elif grep -Eq "^__file$keyfile" "$__messages_in"; then + echo "removed '$keyid'" >> "$__messages_out" fi - - echo "removed '$keyid'" >> "$__messages_out" ;; esac diff --git a/cdist/conf/type/__apt_key/man.rst b/cdist/conf/type/__apt_key/man.rst index 234bc715..e35eaa0f 100644 --- a/cdist/conf/type/__apt_key/man.rst +++ b/cdist/conf/type/__apt_key/man.rst @@ -10,6 +10,14 @@ DESCRIPTION ----------- Manages the list of keys used by apt to authenticate packages. +This is done by placing the requested key in a file named +``$__object_id.gpg`` in the ``keydir`` directory. + +This is supported by modern releases of Debian-based distributions. + +In order of preference, exactly one of: ``source``, ``uri`` or ``keyid`` +must be specified. + REQUIRED PARAMETERS ------------------- @@ -18,21 +26,49 @@ None. OPTIONAL PARAMETERS ------------------- +keydir + keyring directory, defaults to ``/etc/apt/trusted.pgp.d``, which is + enabled system-wide by default. + +source + path to a file containing the GPG key of the repository. + Using this is recommended as it ensures that the manifest/type manintainer + has validated the key. + If ``-``, the GPG key is read from the type's stdin. + state 'present' or 'absent'. Defaults to 'present' +uri + the URI from which to download the key. + It is highly recommended that you only use protocols with TLS like HTTPS. + This uses ``__download`` but does not use checksums, if you want to ensure + that the key doesn't change, you are better off downloading it and using + ``--source``. + + +DEPRECATED OPTIONAL PARAMETERS +------------------------------ keyid - the id of the key to add. Defaults to __object_id + the id of the key to download from the ``keyserver``. + This is to be used in absence of ``--source`` and ``--uri`` or together + with ``--use-deprecated-apt-key`` for key removal. + Defaults to ``$__object_id``. keyserver - the keyserver from which to fetch the key. If omitted the default set - in ./parameter/default/keyserver is used. + the keyserver from which to fetch the key. + Defaults to ``pool.sks-keyservers.net``. -keydir - key save location, defaults to ``/etc/apt/trusted.pgp.d`` -uri - the URI from which to download the key +DEPRECATED BOOLEAN PARAMETERS +----------------------------- +use-deprecated-apt-key + ``apt-key(8)`` will last be available in Debian 11 and Ubuntu 22.04. + You can use this parameter to force usage of ``apt-key(8)``. + Please only use this parameter to *remove* keys from the keyring, + in order to prepare for removal of ``apt-key``. + Adding keys should be done without this parameter. + This parameter will be removed when Debian 11 stops being supported. EXAMPLES @@ -40,33 +76,39 @@ EXAMPLES .. code-block:: sh - # Add Ubuntu Archive Automatic Signing Key - __apt_key 437D05B5 - # Same thing - __apt_key 437D05B5 --state present - # Get rid of it - __apt_key 437D05B5 --state absent + # add a key that has been verified by a type maintainer + __apt_key jitsi_meet_2021 \ + --source cdist-contrib/type/__jitsi_meet/files/apt_2021.gpg - # same thing with human readable name and explicit keyid - __apt_key UbuntuArchiveKey --keyid 437D05B5 + # remove an old, deprecated or expired key + __apt_key jitsi_meet_2016 --state absent - # same thing with other keyserver - __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com + # Get rid of a key that might have been added to + # /etc/apt/trusted.gpg with apt-key + __apt_key 0x40976EAF437D05B5 --use-deprecated-apt-key --state absent - # download key from the internet - __apt_key rabbitmq \ - --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc + # add a key that we define in-line + __apt_key jitsi_meet_2021 --source '-' < Ander Punnar +Evilham COPYING ------- -Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can +Copyright \(C) 2011-2021 Steven Armstrong, Ander Punnar and Evilham. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_key/manifest b/cdist/conf/type/__apt_key/manifest index 010357cd..889a764a 100755 --- a/cdist/conf/type/__apt_key/manifest +++ b/cdist/conf/type/__apt_key/manifest @@ -2,7 +2,105 @@ __package gnupg -if [ -f "$__object/parameter/uri" ] -then __package curl -else __package dirmngr +state_should="$(cat "${__object}/parameter/state")" + +incompatible_args() +{ + cat >> /dev/stderr <<-EOF + This type does not support --${1} and --${method} simultaneously. + EOF + exit 1 +} + +if [ -f "${__object}/parameter/source" ]; then + method="source" + src="$(cat "${__object}/parameter/source")" + if [ "${src}" = "-" ]; then + src="${__object}/stdin" + fi +fi +if [ -f "${__object}/parameter/uri" ]; then + if [ -n "${method}" ]; then + incompatible_args uri + fi + method="uri" + src="$(cat "${__object}/parameter/uri")" +fi +if [ -f "${__object}/parameter/keyid" ]; then + if [ -n "${method}" ]; then + incompatible_args keyid + fi + method="keyid" +fi +# Keep old default +if [ -z "${method}" ]; then + method="keyid" +fi +# Save this for later in gencode-remote +echo "${method}" > "${__object}/key_method" + +# Required remotely (most likely already installed) +__package dirmngr +# We need this in case a key has to be dearmor'd +__package gnupg +export require="__package/gnupg" + +if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then + # This is required if apt-key(8) is to be used + if [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then + incompatible_args use-deprecated-apt-key + fi +else + if [ "${state_should}" = "absent" ] && \ + [ -f "${__object}/parameter/keyid" ]; then + cat >> /dev/stderr < Date: Mon, 10 May 2021 12:10:00 +0200 Subject: [PATCH 067/128] [__letsencrypt_cert] Revamp explorers, add locking. This would fix #839 Certbot uses locking [1] even for read-only operations and does not properly use exit codes, which means that sometimes it would print: "Another instance of Certbot is already running" and exit with success. However, the previous explorers would take that as the certificate being absent and would trigger code generation. The issue was made worse by having many explorers running certbot, so for N certificates, we'd run certbot N*4 times, potentially "in parallel". [1]: https://certbot.eff.org/docs/using.html#id5 This patch joins all explorers in one to avoid starting multiple remote python processes and uses a cdist-specific lock in /tmp/certbot.cdist.lock with a 60 seconds timeout. It has been tested with certbot 0.31.0 and 0.17 that the: from certbot.main import main trick works. It is somewhat well documented so it can be somewhat relied upon. --- .../__letsencrypt_cert/explorer/certbot-path | 3 - .../explorer/certificate-data | 78 +++++++++++++++++++ .../explorer/certificate-domains | 8 -- .../explorer/certificate-exists | 13 ---- .../explorer/certificate-is-test | 14 ---- .../type/__letsencrypt_cert/gencode-remote | 11 ++- cdist/conf/type/__letsencrypt_cert/manifest | 2 +- 7 files changed, 87 insertions(+), 42 deletions(-) delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certbot-path create mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-data delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path deleted file mode 100755 index 3c6076df..00000000 --- a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -e - -command -v certbot 2>/dev/null || true diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data new file mode 100755 index 00000000..ff62e742 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data @@ -0,0 +1,78 @@ +#!/bin/sh -e +certbot_path="$(command -v certbot 2>/dev/null || true)" +# Defaults +certificate_exists="no" +certificate_is_test="no" + +if [ -n "${certbot_path}" ]; then + # Find python executable that has access to certbot's module + python_path=$(sed -n '1s/^#! *//p' "${certbot_path}") + + # Use a lock for cdist due to certbot not exiting with failure + # or having any flags for concurrent use. + _certbot() { + ${python_path} - 2>/dev/null < "${existing_domains}" + certificate_is_test="$(_explorer_var certificate_is_test)" sort -uo "${requested_domains}" "${requested_domains}" sort -uo "${existing_domains}" "${existing_domains}" diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 1df3574a..6394f629 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -1,6 +1,6 @@ #!/bin/sh -certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")" +certbot_fullpath="$(grep "^certbot_path:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-)" state=$(cat "${__object}/parameter/state") os="$(cat "${__global:?}/explorer/os")" From f14623e45f56aec4bdce53bc2b367e11ff157865 Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 10 May 2021 12:17:08 +0200 Subject: [PATCH 068/128] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 497ae91a..5c3b547a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,8 @@ Changelog next: * New type: __postgres_conf (Beni Ruef, Dennis Camera) * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) + * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) + * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 6210cccb28ace8ec2807ffe6ca9f9e4d9007990b Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 10 May 2021 12:34:04 +0200 Subject: [PATCH 069/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 5c3b547a..a7310788 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) + * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking. 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 503a06ed28f743aad47797a0989be735c67b07a6 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 28 Apr 2021 13:32:10 +0300 Subject: [PATCH 070/128] [__git] fix group explorer group name from numberic id wasn't resolved correctly. try to use getent and fallback to reading /etc/group directly. --- cdist/conf/type/__git/explorer/group | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group index 1365c60d..ab4396b1 100644 --- a/cdist/conf/type/__git/explorer/group +++ b/cdist/conf/type/__git/explorer/group @@ -14,6 +14,11 @@ then then printf '%u\n' "${group_gid}" else - printf '%s\n' "$(id -u -n "${group_gid}")" + if command -v getent > /dev/null + then + getent group "${group_gid}" | cut -d : -f 1 + else + awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group + fi fi fi From 75c71f69c1fed4371c5891ddcca0eaf28ad928e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 10:17:48 +0200 Subject: [PATCH 071/128] [scanner] pycodestyle compliance --- cdist/scan/commandline.py | 20 +++++++++++---- cdist/scan/scan.py | 54 ++++++++++++++++++++++++--------------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 3eb7eec4..b42bc7b2 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -25,13 +25,15 @@ from datetime import datetime log = logging.getLogger("scan") + def run(scan, args): # We run each component in a separate process since they # must not block on each other. processes = [] if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay) + t = scan.Trigger(interfaces=args.interfaces, + sleeptime=args.trigger_delay) t.start() processes.append(t) log.debug("Trigger started") @@ -48,6 +50,7 @@ def run(scan, args): for process in processes: process.join() + def list(scan, args): s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper) hosts = s.list() @@ -66,10 +69,16 @@ def list(scan, args): print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) for host in hosts: last_seen = host.last_seen() - last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-' + if last_seen: + last_seen = last_seen.strftime(scan.datetime_format) + else: + last_seen = '-' last_configured = host.last_configured() - last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-' + if last_configured: + last_configured = last_configured.strftime(scan.datetime_format) + else: + '-' print("{} | {} | {} | {}".format( host.name(default='-').ljust(name_max_size), @@ -77,6 +86,7 @@ def list(scan, args): last_seen.ljust(date_max_size), last_configured.ljust(date_max_size))) + # CLI processing is defined outside of the main scan class to handle # non-available optional scapy dependency (instead of crashing mid-flight). def commandline(args): @@ -94,9 +104,9 @@ def commandline(args): # By default scan and trigger, but do not call any action. args.mode = ['scan', 'trigger', ] - if 'config' in args.mode and args.name_mapper == None: + if 'config' in args.mode and args.name_mapper is None: print('--name-mapper must be specified for scanner config mode.', - file=sys.stderr) + file=sys.stderr) sys.exit(1) # Print known hosts and exit is --list is specified - do not start diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 2912dab3..4a20f511 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -33,6 +33,7 @@ logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") datetime_format = '%Y-%m-%d %H:%M:%S' + class Host(object): def __init__(self, addr, outdir, name_mapper=None): self.addr = addr @@ -43,7 +44,7 @@ class Host(object): def __get(self, key, default=None): fname = os.path.join(self.workdir, key) - value=default + value = default if os.path.isfile(fname): with open(fname, "r") as fd: value = fd.readline() @@ -55,15 +56,15 @@ class Host(object): fd.write(f"{value}") def name(self, default=None): - if self.name_mapper == None: + if self.name_mapper is None: return default fpath = os.path.join(os.getcwd(), self.name_mapper) if os.path.isfile(fpath) and os.access(fpath, os.X_OK): - out = subprocess.run([fpath, self.addr], capture_output=True) - if out.returncode != 0: - return default - else: + out = subprocess.run([fpath, self.addr], capture_output=True) + if out.returncode != 0: + return default + else: value = out.stdout.decode() return (default if len(value) == 0 else value) else: @@ -94,19 +95,20 @@ class Host(object): # CLI args. Might as well call everything from scratch! def configure(self): target = self.name() or self.address() - cmd = ['cdist', 'config', '-v', target ] + cmd = ['cdist', 'config', '-v', target] fname = os.path.join(self.workdir, 'last_configuration_log') with open(fname, "w") as fd: log.debug("Executing: %s", cmd) completed_process = subprocess.run(cmd, stdout=fd, stderr=fd) if completed_process.returncode != 0: - log.error("%s return with non-zero code %i - see %s for details.", - cmd, completed_process.returncode, fname) + log.error("%s return with non-zero code %i - see %s for \ + details.", cmd, completed_process.returncode, fname) now = datetime.datetime.now().strftime(datetime_format) self.__set('last_configured', now) + class Trigger(object): """ Trigger an ICMPv6EchoReply from all hosts that are alive @@ -140,10 +142,12 @@ class Trigger(object): def trigger(self, interface): try: log.debug("Sending ICMPv6EchoRequest on %s", interface) - packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest() + packet = IPv6( + dst="ff02::1%{}".format(interface) + ) / ICMPv6EchoRequest() send(packet, verbose=self.verbose) except Exception as e: - log.error( "Could not send ICMPv6EchoRequest: %s", e) + log.error("Could not send ICMPv6EchoRequest: %s", e) class Scanner(object): @@ -151,9 +155,10 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None): + def __init__(self, interfaces, autoconfigure=False, outdir=None, + name_mapper=None): self.interfaces = interfaces - self.autoconfigure=autoconfigure + self.autoconfigure = autoconfigure self.name_mapper = name_mapper self.config_delay = datetime.timedelta(seconds=3600) @@ -169,14 +174,17 @@ class Scanner(object): if ICMPv6EchoReply in pkg: host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper) if host.name(): - log.verbose("Host %s (%s) is alive", host.name(), host.address()) + log.verbose("Host %s (%s) is alive", host.name(), + host.address()) else: log.verbose("Host %s is alive", host.address()) + host.seen() # Configure if needed. if self.autoconfigure and \ - host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now(): + host.last_configured(default=datetime.datetime.min) + \ + self.config_delay < datetime.datetime.now(): self.config(host) def list(self): @@ -187,15 +195,19 @@ class Scanner(object): return hosts def config(self, host): - if host.name() == None: - log.debug("config - could not resolve name for %s, aborting.", host.address()) + if host.name() is None: + log.debug("config - could not resolve name for %s, aborting.", + host.address()) return previous_config_process = self.running_configs.get(host.name()) - if previous_config_process != None and previous_config_process.is_alive(): - log.debug("config - is already running for %s, aborting.", host.name()) + if previous_config_process is not None and \ + previous_config_process.is_alive(): + log.debug("config - is already running for %s, aborting.", + host.name()) - log.info("config - running against host %s (%s).", host.name(), host.address()) + log.info("config - running against host %s (%s).", host.name(), + host.address()) p = Process(target=host.configure()) p.start() self.running_configs[host.name()] = p @@ -214,4 +226,4 @@ class Scanner(object): filter="icmp6", prn=self.handle_pkg) except Exception as e: - log.error( "Could not start listener: %s", e) + log.error("Could not start listener: %s", e) From ab10b453f275d4289f3d37988b3caec204227194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 11:15:41 +0200 Subject: [PATCH 072/128] [scanner] populate cdist(1) --- docs/src/man1/cdist.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 0ecb4a61..599ec3b7 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -88,6 +88,9 @@ SYNOPSIS cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] + cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list] + [-d CONFIG_DELAY] [-t TRIGGER_DELAY] + DESCRIPTION ----------- @@ -641,6 +644,31 @@ Display information for cdist (global explorers, types). **-t, --types** Display info for types. +SCAN +---- + +Runs cdist as a daemon that discover/watch on hosts and reconfigure them +periodically. + +**-I INTERFACE, --interfaces INTERFACE** + Interface to listen on. Can be specified multiple times. + +**-m MODE, --mode MODE** + Scanner components to enable. Can be specified multiple time to enable more + than one component. Supported modes are: scan, trigger and config. Defaults + to tiggger and scan. + +**--name-mapper PATH_TO_SCRIPT** + Path to script used to resolve a remote host name from an IPv6 address. + +**--list** + List known hosts and exit. + +**-d CONFIG_DELAY, --config-delay CONFIG_DELAY** + How long (seconds) to wait before reconfiguring after last try (config mode only). + +**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY** + How long (seconds) to wait between ICMPv6 echo requests (trigger mode only). CONFIGURATION ------------- From b8733c65f52776facce7a8de0265538a856ead64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 11:26:35 +0200 Subject: [PATCH 073/128] [scanner] fix minor CLI handling and --list bugs / typo --- cdist/argparse.py | 4 ++-- cdist/scan/commandline.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index bedb23ac..f17315e7 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -495,7 +495,7 @@ def get_parsers(): action='store_true', help='Try to configure detected hosts') parser['scan'].add_argument( - '-I', '--interfaces', + '-I', '--interface', action='append', default=[], required=True, help='On which interfaces to scan/trigger') parser['scan'].add_argument( @@ -503,7 +503,7 @@ def get_parsers(): action='store', default=None, help='Map addresses to names, required for config mode') parser['scan'].add_argument( - '-d', '--delay', + '-d', '--config-delay', action='store', default=3600, type=int, help='How long (seconds) to wait before reconfiguring after last try') parser['scan'].add_argument( diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index b42bc7b2..ddbe4933 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -32,7 +32,7 @@ def run(scan, args): processes = [] if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interfaces, + t = scan.Trigger(interfaces=args.interface, sleeptime=args.trigger_delay) t.start() processes.append(t) @@ -41,7 +41,7 @@ def run(scan, args): if 'scan' in args.mode: s = scan.Scanner( autoconfigure='config' in args.mode, - interfaces=args.interfaces, + interfaces=args.interface, name_mapper=args.name_mapper) s.start() processes.append(s) @@ -52,7 +52,7 @@ def run(scan, args): def list(scan, args): - s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper) + s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper) hosts = s.list() # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + @@ -75,10 +75,10 @@ def list(scan, args): last_seen = '-' last_configured = host.last_configured() - if last_configured: + if last_configured is not None: last_configured = last_configured.strftime(scan.datetime_format) else: - '-' + last_configured = '-' print("{} | {} | {} | {}".format( host.name(default='-').ljust(name_max_size), From e0c52d0e1dfbaa3814a2ff26482177c2909a15ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 11:27:11 +0200 Subject: [PATCH 074/128] [scanner] remove mention of non-implemented trigger soruce script --- docs/src/cdist-scan.rst | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst index 02193456..064e65ff 100644 --- a/docs/src/cdist-scan.rst +++ b/docs/src/cdist-scan.rst @@ -8,7 +8,8 @@ periodically. It is especially useful in netboot-based environment where hosts boot unconfigured, and to ensure your infrastructure stays in sync with your configuration. -This feature is still consider to be in **beta** stage. +This feature is still consider to be in **beta** stage, and only operate on +IPv6 (including link-local). Usage (Examples) ---------------- @@ -79,21 +80,3 @@ Resolving name from `PTR` DNS record: done dig +short -x "$1" | sed -e 's/.$//' - - -Trigger Source Script ---------------------- - -This script returns a list of addresses (separated by a newline) to be used by -`trigger` mode. It is not used to map names. The script must be executable. - -Simplest script: - -.. code-block:: sh - #!/bin/sh - - cat << EOF - server1.domain.tld - server2.domain.tld - server3.domain.tld - EOF From defa3c22eaef88652a2bec6135d038a84710c41e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 29 May 2021 11:21:34 +0200 Subject: [PATCH 075/128] ++changelog --- docs/changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index a7310788..403e7653 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,7 +6,8 @@ next: * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) - * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking. + * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham) + * Type __git: Fix group explorer (Ander Punnar) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From d596986af894749877fb3dcadf90628c9cfa9d13 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 31 May 2021 09:06:52 +0200 Subject: [PATCH 076/128] [type/__pyvenv] Fix group explorer --- cdist/conf/type/__pyvenv/explorer/group | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group index f31a1cb7..922ce3df 100755 --- a/cdist/conf/type/__pyvenv/explorer/group +++ b/cdist/conf/type/__pyvenv/explorer/group @@ -14,6 +14,11 @@ then then printf '%u\n' "${group_gid}" else - printf '%s\n' "$(id -u -n "${group_gid}")" + if command -v getent >/dev/null 2>&1 + then + getent group "${group_gid}" | cut -d : -f 1 + else + awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group + fi fi fi From 6ede76b08b96b7504c65ca2dc6737e31be7e7f24 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 8 Jun 2021 16:20:55 +0200 Subject: [PATCH 077/128] [type/__debconf_set_selections] man.rst: Fix line break in AUTHORS --- cdist/conf/type/__debconf_set_selections/man.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst index 690e3e49..fd0040ae 100644 --- a/cdist/conf/type/__debconf_set_selections/man.rst +++ b/cdist/conf/type/__debconf_set_selections/man.rst @@ -58,8 +58,8 @@ SEE ALSO AUTHORS ------- -Nico Schottelius -Dennis Camera +| Nico Schottelius +| Dennis Camera COPYING From c308a2896971ac4808be0152630ef41b7f96a2de Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 10 Jun 2021 06:39:55 +0200 Subject: [PATCH 078/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 403e7653..97ea204b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham) * Type __git: Fix group explorer (Ander Punnar) + * Type __pyvenv: Fix group explorer (Dennis Camera) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From a3102022e18a23ce9b54eeaf7415b55361f80bd1 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 11 Jun 2021 15:05:17 +0100 Subject: [PATCH 079/128] More sensible defaults; reword debian-only error message --- cdist/conf/type/__apt_pin/manifest | 13 +++++++++---- cdist/conf/type/__apt_pin/nonparallel | 0 cdist/conf/type/__apt_pin/parameter/default/package | 1 - 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 cdist/conf/type/__apt_pin/nonparallel delete mode 100644 cdist/conf/type/__apt_pin/parameter/default/package diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index b1372ad0..909bc80d 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -19,9 +19,17 @@ # +name="$__object_id" + os=$(cat "$__global/explorer/os") state="$(cat "$__object/parameter/state")" -package="$(cat "$__object/parameter/package")" + +if [ -f "$__object/parameter/package" ]; then + package="$(cat "$__object/parameter/package")" +else + package=$name +fi + distribution="$(cat "$__object/parameter/distribution")" priority="$(cat "$__object/parameter/priority")" @@ -31,13 +39,10 @@ case "$os" in ;; *) printf "This type is specific to Debian and it's derivatives" >&2 - printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2 exit 1 ;; esac -name="$__object_id" - case $distribution in stabletesting|unsatbel|experimental) pin="release a=$distribution" diff --git a/cdist/conf/type/__apt_pin/nonparallel b/cdist/conf/type/__apt_pin/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package deleted file mode 100644 index 72e8ffc0..00000000 --- a/cdist/conf/type/__apt_pin/parameter/default/package +++ /dev/null @@ -1 +0,0 @@ -* From b726697e070e5266eae38c7951ced14a2305acb2 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 11 Jun 2021 15:05:33 +0100 Subject: [PATCH 080/128] Add documentation --- cdist/conf/type/__apt_pin/man.rst | 37 ++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst index 7fcae6f8..0c91cdec 100644 --- a/cdist/conf/type/__apt_pin/man.rst +++ b/cdist/conf/type/__apt_pin/man.rst @@ -13,11 +13,21 @@ This space intentionally left blank. REQUIRED PARAMETERS ------------------- -None. +distribution + Specifies what distribution the package should be pinned to. Accepts both codenames (buster/bullseye/sid) and suite names (stable/testing/...). OPTIONAL PARAMETERS ------------------- +package + Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used. + +priority + The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority) + +state + Will be passed to underlying `__file` type; see there for valid values and defaults. + None. @@ -31,14 +41,31 @@ EXAMPLES .. code-block:: sh - # TODO - __apt_pin + # Add the bullseye repo to buster, but do not install any pacakges by default + # only if explicitely asked for + __apt_pin bullseye-default \ + --package "*" \ + --distribution bullseye \ + --priority -1 + + require="__apt_pin/bullseye-default" __apt_source bullseye \ + --uri http://deb.debian.org/debian/ \ + --distribution bullseye \ + --component main + # TODO + __apt_pin + + __apt_pin foo --package "foo foo-*" --distribution bullseye + + __foo # Installs the `foo` package internally + + __package foo-plugin-extras SEE ALSO -------- -:strong:`TODO`\ (7) - +:strong:`apt_preferences`\ (7) +:strong:`cdist-type__file`\ (7) AUTHORS ------- From 7b3f268df25922c515174862da95569aea547367 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 22 Jun 2021 16:36:30 +0300 Subject: [PATCH 081/128] [__download] improvements 1. post download checksum verification 2. detect hashes without prefix 3. add optional --destination 4. updated man --- .../conf/type/__download/explorer/remote_cmd | 19 --- .../type/__download/explorer/remote_cmd_get | 16 +++ .../type/__download/explorer/remote_cmd_sum | 82 +++++++++++ cdist/conf/type/__download/explorer/state | 60 ++------ cdist/conf/type/__download/gencode-local | 131 +++++++++++++++--- cdist/conf/type/__download/gencode-remote | 46 +++++- cdist/conf/type/__download/man.rst | 31 ++++- cdist/conf/type/__download/manifest | 2 +- cdist/conf/type/__download/parameter/optional | 3 +- 9 files changed, 292 insertions(+), 98 deletions(-) delete mode 100755 cdist/conf/type/__download/explorer/remote_cmd create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_get create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_sum diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd deleted file mode 100755 index e3e35b45..00000000 --- a/cdist/conf/type/__download/explorer/remote_cmd +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/cmd-get" ] -then - cmd="$( cat "$__object/parameter/cmd-get" )" - -elif command -v curl > /dev/null -then - cmd="curl -L -o - '%s'" - -elif command -v fetch > /dev/null -then - cmd="fetch -o - '%s'" - -else - cmd="wget -O - '%s'" -fi - -echo "$cmd" diff --git a/cdist/conf/type/__download/explorer/remote_cmd_get b/cdist/conf/type/__download/explorer/remote_cmd_get new file mode 100755 index 00000000..9f1cd59c --- /dev/null +++ b/cdist/conf/type/__download/explorer/remote_cmd_get @@ -0,0 +1,16 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/cmd-get" ] +then + cat "$__object/parameter/cmd-get" +elif + command -v curl > /dev/null +then + echo "curl -sSL -o - '%s'" +elif + command -v fetch > /dev/null +then + echo "fetch -o - '%s'" +else + echo "wget -O - '%s'" +fi diff --git a/cdist/conf/type/__download/explorer/remote_cmd_sum b/cdist/conf/type/__download/explorer/remote_cmd_sum new file mode 100755 index 00000000..84df663c --- /dev/null +++ b/cdist/conf/type/__download/explorer/remote_cmd_sum @@ -0,0 +1,82 @@ +#!/bin/sh -e + +if [ ! -f "$__object/parameter/sum" ] +then + exit 0 +fi + +if [ -f "$__object/parameter/cmd-sum" ] +then + cat "$__object/parameter/cmd-sum" + exit 0 +fi + +sum_should="$( cat "$__object/parameter/sum" )" + +if echo "$sum_should" | grep -Fq ':' +then + sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" +else + if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' + then + sum_hash='cksum' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' + then + sum_hash='md5' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' + then + sum_hash='sha1' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' + then + sum_hash='sha256' + else + echo 'hash format detection failed' >&2 + exit 1 + fi +fi + +os="$( "$__explorer/os" )" + +case "$sum_hash" in + cksum) + echo "cksum %s | awk '{print \$1\" \"\$2}'" + ;; + md5) + case "$os" in + freebsd) + echo "md5 -q %s" + ;; + *) + echo "md5sum %s | awk '{print \$1}'" + ;; + esac + ;; + sha1) + case "$os" in + freebsd) + echo "sha1 -q %s" + ;; + *) + echo "sha1sum %s | awk '{print \$1}'" + ;; + esac + ;; + sha256) + case "$os" in + freebsd) + echo "sha256 -q %s" + ;; + *) + echo "sha256sum %s | awk '{print \$1}'" + ;; + esac + ;; + *) + # we arrive here only if --sum is given with unknown format prefix + echo "unknown hash format: $sum_hash" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 68b517c5..881a1c09 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -1,6 +1,11 @@ #!/bin/sh -e -dst="/$__object_id" +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" +else + dst="/$__object_id" +fi if [ ! -f "$dst" ] then @@ -16,57 +21,18 @@ fi sum_should="$( cat "$__object/parameter/sum" )" -if [ -f "$__object/parameter/cmd-sum" ] +if echo "$sum_should" | grep -Fq ':' then - # shellcheck disable=SC2059 - sum_is="$( eval "$( printf \ - "$( cat "$__object/parameter/cmd-sum" )" \ - "$dst" )" )" -else - os="$( "$__explorer/os" )" - - if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' - then - sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )" - - elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$' - then - case "$os" in - freebsd) - sum_is="md5:$( md5 -q "$dst" )" - ;; - *) - sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )" - ;; - esac - - elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$' - then - case "$os" in - freebsd) - sum_is="sha1:$( sha1 -q "$dst" )" - ;; - *) - sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )" - ;; - esac - - elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$' - then - case "$os" in - freebsd) - sum_is="sha256:$( sha256 -q "$dst" )" - ;; - *) - sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )" - ;; - esac - fi + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" fi +sum_cmd="$( "$__type_explorer/remote_cmd_sum" )" + +sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )" + if [ -z "$sum_is" ] then - echo 'no checksum from target' >&2 + echo 'existing destination checksum failed' >&2 exit 1 fi diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local index 571d2c3c..d1b0d0d5 100755 --- a/cdist/conf/type/__download/gencode-local +++ b/cdist/conf/type/__download/gencode-local @@ -11,34 +11,133 @@ fi url="$( cat "$__object/parameter/url" )" -tmp="$( mktemp )" - -dst="/$__object_id" +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" +else + dst="/$__object_id" +fi if [ -f "$__object/parameter/cmd-get" ] then cmd="$( cat "$__object/parameter/cmd-get" )" -elif command -v wget > /dev/null -then - cmd="wget -O - '%s'" - elif command -v curl > /dev/null then - cmd="curl -L -o - '%s'" + cmd="curl -sSL -o - '%s'" elif command -v fetch > /dev/null then cmd="fetch -o - '%s'" +elif command -v wget > /dev/null +then + cmd="wget -O - '%s'" + else - echo 'no usable locally installed utility for downloading' >&2 + echo 'local download failed, no usable utility' >&2 exit 1 fi -printf "$cmd > %s\n" \ - "$url" \ - "$tmp" +echo "download_tmp=\"\$( mktemp )\"" + +# shellcheck disable=SC2059 +printf "$cmd > \"\$download_tmp\"\n" "$url" + +if [ -f "$__object/parameter/sum" ] +then + sum_should="$( cat "$__object/parameter/sum" )" + + if [ -f "$__object/parameter/cmd-sum" ] + then + local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" + else + if echo "$sum_should" | grep -Fq ':' + then + sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" + + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" + else + if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' + then + sum_hash='cksum' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' + then + sum_hash='md5' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' + then + sum_hash='sha1' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' + then + sum_hash='sha256' + else + echo 'hash format detection failed' >&2 + exit 1 + fi + fi + + case "$sum_hash" in + cksum) + local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'" + ;; + md5) + if command -v md5 > /dev/null + then + local_cmd_sum="md5 -q %s" + elif + command -v md5sum > /dev/null + then + local_cmd_sum="md5sum %s | awk '{print \$1}'" + fi + ;; + sha1) + if command -v sha1 > /dev/null + then + local_cmd_sum="sha1 -q %s" + elif + command -v sha1sum > /dev/null + then + local_cmd_sum="sha1sum %s | awk '{print \$1}'" + fi + ;; + sha256) + if command -v sha256 > /dev/null + then + local_cmd_sum="sha256 -q %s" + elif + command -v sha256sum > /dev/null + then + local_cmd_sum="sha256sum %s | awk '{print \$1}'" + fi + ;; + *) + # we arrive here only if --sum is given with unknown format prefix + echo "unknown hash format: $sum_hash" >&2 + exit 1 + ;; + esac + + if [ -z "$local_cmd_sum" ] + then + echo 'local checksum verification failed, no usable utility' >&2 + exit 1 + fi + fi + + # shellcheck disable=SC2059 + echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\"" + + echo "if [ \"\$sum_is\" != '$sum_should' ]; then" + + echo "echo 'local download checksum mismatch' >&2" + + echo "rm -f \"\$download_tmp\"" + + echo 'exit 1; fi' +fi if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$' then @@ -47,12 +146,10 @@ else target_host="$__target_host" fi -printf '%s %s %s:%s\n' \ +# shellcheck disable=SC2016 +printf '%s "$download_tmp" %s:%s\n' \ "$__remote_copy" \ - "$tmp" \ "$target_host" \ "$dst" -echo "rm -f '$tmp'" - -echo 'downloaded' > "$__messages_out" +echo "rm -f \"\$download_tmp\"" diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote index 029a0801..e49bcec3 100755 --- a/cdist/conf/type/__download/gencode-remote +++ b/cdist/conf/type/__download/gencode-remote @@ -6,17 +6,51 @@ state_is="$( cat "$__object/explorer/state" )" if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ] then - cmd="$( cat "$__object/explorer/remote_cmd" )" + cmd_get="$( cat "$__object/explorer/remote_cmd_get" )" url="$( cat "$__object/parameter/url" )" - dst="/$__object_id" + if [ -f "$__object/parameter/destination" ] + then + dst="$( cat "$__object/parameter/destination" )" + else + dst="/$__object_id" + fi - printf "$cmd > %s\n" \ - "$url" \ - "$dst" + echo "download_tmp=\"\$( mktemp )\"" - echo 'downloaded' > "$__messages_out" + # shellcheck disable=SC2059 + printf "$cmd_get > \"\$download_tmp\"\n" "$url" + + if [ -f "$__object/parameter/sum" ] + then + sum_should="$( cat "$__object/parameter/sum" )" + + if [ -f "$__object/parameter/cmd-sum" ] + then + remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" + else + remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )" + + if echo "$sum_should" | grep -Fq ':' + then + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" + fi + fi + + # shellcheck disable=SC2059 + echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\"" + + echo "if [ \"\$sum_is\" != '$sum_should' ]; then" + + echo "echo 'remote download checksum mismatch' >&2" + + echo "rm -f \"\$download_tmp\"" + + echo 'exit 1; fi' + fi + + echo "mv \"\$download_tmp\" '$dst'" fi if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ] diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index a1278cfb..c16510a9 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -8,7 +8,7 @@ cdist-type__download - Download a file DESCRIPTION ----------- -By default type will try to use ``wget``, ``curl`` or ``fetch``. +By default type will try to use ``curl``, ``fetch`` or ``wget``. If download happens in target (see ``--download``) then type will fallback to (and install) ``wget``. @@ -16,6 +16,8 @@ If download happens in local machine, then environment variables like ``{http,https,ftp}_proxy`` etc can be used on cdist execution (``http_proxy=foo cdist config ...``). +To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``. + REQUIRED PARAMETERS ------------------- @@ -25,14 +27,29 @@ url OPTIONAL PARAMETERS ------------------- +destination + Downloaded file's destination in target. If unset, ``$__object_id`` is used. + sum - Checksum is used to decide if existing destination file must be redownloaded. - By default output of ``cksum`` without filename is expected. - Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. + Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256. + + Type tries to detect hash format with regexes, but prefixes + ``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported. + + Checksum have two purposes - state check and post-download verification. + In state check, if destination checksum mismatches, then content of URL + will be downloaded to temporary file. If downloaded temporary file's + checksum matches, then it will be moved to destination (overwritten). + + For local downloads it is expected that usable utilities for checksum + calculation exist in the system. download - If ``local`` (default), then download file to local storage and copy - it to target host. If ``remote``, then download happens in target. + If ``local`` (default), then file is downloaded to local storage and copied + to target host. If ``remote``, then download happens in target. + + For local downloads it is expected that usable utilities for downloading + exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``. cmd-get Command used for downloading. @@ -62,7 +79,7 @@ EXAMPLES require='__directory/opt/cpma' \ __download /opt/cpma/cnq3.zip \ --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \ - --sum md5:46da3021ca9eace277115ec9106c5b46 + --sum 46da3021ca9eace277115ec9106c5b46 require='__download/opt/cpma/cnq3.zip' \ __unpack /opt/cpma/cnq3.zip \ diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest index 7ec8d86d..3d4c498b 100755 --- a/cdist/conf/type/__download/manifest +++ b/cdist/conf/type/__download/manifest @@ -1,6 +1,6 @@ #!/bin/sh -e -if grep -Eq '^wget' "$__object/explorer/remote_cmd" +if grep -Eq '^wget' "$__object/explorer/remote_cmd_get" then __package wget fi diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional index d69e083e..e809ef78 100644 --- a/cdist/conf/type/__download/parameter/optional +++ b/cdist/conf/type/__download/parameter/optional @@ -1,5 +1,6 @@ -sum cmd-get cmd-sum +destination download onchange +sum From 2db40d8d704b427768307fbea29384bd3dc8dbd7 Mon Sep 17 00:00:00 2001 From: fancsali Date: Mon, 28 Jun 2021 12:54:20 +0200 Subject: [PATCH 082/128] Use $__remote_exec and thus the ssh multiplexing --- cdist/conf/type/__rsync/gencode-local | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index e36ded2f..36addc36 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -36,4 +36,5 @@ fi echo rsync -a \ --no-owner --no-group \ + -e "$__remote_exec" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From d937d53f3dfd10830a07aee0450596eea62f2a1a Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Mon, 28 Jun 2021 18:09:35 +0100 Subject: [PATCH 083/128] Add quotes to rsync command --- cdist/conf/type/__rsync/gencode-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index 36addc36..f1bddc16 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -36,5 +36,5 @@ fi echo rsync -a \ --no-owner --no-group \ - -e "$__remote_exec" \ + -e \"$__remote_exec\" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From 60753ddfcc3ac49303d572da7f6a68f398c02227 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 1 Jul 2021 14:42:10 +0300 Subject: [PATCH 084/128] fix shellcheck --- cdist/conf/type/__download/explorer/state | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 881a1c09..8c5d5ce1 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -28,6 +28,7 @@ fi sum_cmd="$( "$__type_explorer/remote_cmd_sum" )" +# shellcheck disable=SC2059 sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )" if [ -z "$sum_is" ] From a90e642c1354cf01be7c9a1ba8a468ad624c4202 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 1 Jul 2021 14:50:40 +0300 Subject: [PATCH 085/128] update README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index de6901c7..a468dd86 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ For community-maintained types there is ## Participating -IRC: ``#cdist`` @ freenode +IRC: ``#cdist`` @ [libera](https://libera.chat) Matrix: ``#cdist:ungleich.ch`` -Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist +Matrix and IRC are bridged. From 243a4b904a2de638d00bac57f6e762a076f9ae54 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 2 Jul 2021 06:50:02 +0200 Subject: [PATCH 086/128] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 97ea204b..d48c657e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,8 @@ next: * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham) * Type __git: Fix group explorer (Ander Punnar) * Type __pyvenv: Fix group explorer (Dennis Camera) + * Type __download: Improve checksum verification, add optional --destination (Ander Punnar) + * Type __debconf_set_selections: Add state explorer (Dennis Camera) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 30ba796d060fd3d0c4affd4167f14784156b5354 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 1 Jul 2021 11:49:07 +0300 Subject: [PATCH 087/128] new type: __snakeoil_cert --- .../__snakeoil_cert/explorer/ssl-cert-group | 8 ++ .../conf/type/__snakeoil_cert/explorer/state | 24 ++++++ .../conf/type/__snakeoil_cert/gencode-remote | 73 +++++++++++++++++++ cdist/conf/type/__snakeoil_cert/man.rst | 60 +++++++++++++++ .../parameter/default/cert-path | 1 + .../parameter/default/key-path | 1 + .../parameter/default/key-type | 1 + .../type/__snakeoil_cert/parameter/optional | 4 + 8 files changed, 172 insertions(+) create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/state create mode 100755 cdist/conf/type/__snakeoil_cert/gencode-remote create mode 100644 cdist/conf/type/__snakeoil_cert/man.rst create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/cert-path create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-path create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-type create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/optional diff --git a/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group new file mode 100755 index 00000000..a6cb3dfd --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group @@ -0,0 +1,8 @@ +#!/bin/sh -e + +if grep -Eq '^ssl-cert:' /etc/group +then + echo 'present' +else + echo 'absent' +fi diff --git a/cdist/conf/type/__snakeoil_cert/explorer/state b/cdist/conf/type/__snakeoil_cert/explorer/state new file mode 100755 index 00000000..cc5aae0b --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/explorer/state @@ -0,0 +1,24 @@ +#!/bin/sh -e + +key_path="$( cat "$__object/parameter/key-path" )" + +if echo "$key_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + key_path="$( printf "$key_path" "$__object_id" )" +fi + +cert_path="$( cat "$__object/parameter/cert-path" )" + +if echo "$cert_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + cert_path="$( printf "$cert_path" "$__object_id" )" +fi + +if [ ! -f "$key_path" ] || [ ! -f "$cert_path" ] +then + echo 'absent' +else + echo 'present' +fi diff --git a/cdist/conf/type/__snakeoil_cert/gencode-remote b/cdist/conf/type/__snakeoil_cert/gencode-remote new file mode 100755 index 00000000..8ffbfad1 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/gencode-remote @@ -0,0 +1,73 @@ +#!/bin/sh -e + +state="$( cat "$__object/explorer/state" )" + +if [ "$state" = 'present' ] +then + exit 0 +fi + +if [ -f "$__object/parameter/common-name" ] +then + common_name="$( cat "$__object/parameter/common-name" )" +else + common_name="$__object_id" +fi + +key_path="$( cat "$__object/parameter/key-path" )" + +if echo "$key_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + key_path="$( printf "$key_path" "$__object_id" )" +fi + +cert_path="$( cat "$__object/parameter/cert-path" )" + +if echo "$cert_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + cert_path="$( printf "$cert_path" "$__object_id" )" +fi + +key_type="$( cat "$__object/parameter/key-type" )" + +key_type_arg="$( echo "$key_type" | cut -d : -f 2 )" + +case "$key_type" in + rsa:*) + echo "openssl genrsa -out '$key_path' $key_type_arg" + ;; + ec:*) + echo "openssl ecparam -name $key_type_arg -genkey -noout -out '$key_path'" + ;; +esac + +# shellcheck disable=SC2016 +echo 'csr_path="$( mktemp )"' + +echo "openssl req -new -subj '/CN=$common_name' -key '$key_path' -out \"\$csr_path\"" + +echo "openssl x509 -req -sha256 -days 3650 -in \"\$csr_path\" -signkey '$key_path' -out '$cert_path'" + +# shellcheck disable=SC2016 +echo 'rm -f "$csr_path"' + +if [ "$( cat "$__object/explorer/ssl-cert-group" )" = 'present' ] +then + key_group='ssl-cert' +else + key_group='root' +fi + +echo "chmod 640 '$key_path'" + +echo "chown root '$key_path'" + +echo "chgrp $key_group '$key_path'" + +echo "chmod 644 '$cert_path'" + +echo "chown root '$cert_path'" + +echo "chgrp root '$cert_path'" diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst new file mode 100644 index 00000000..0b547804 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/man.rst @@ -0,0 +1,60 @@ +cdist-type__snakeoil_cert(7) +============================ + +NAME +---- +cdist-type__snakeoil_cert - Generate self-signed certificate + + +DESCRIPTION +----------- +The purpose of this type is to generate **self-signed** certificate and private key +for **testing purposes**. Certificate will expire in 3650 days. + +Certificate's and key's access bits will be ``644`` and ``640`` respectively. +If target system has ``ssl-cert`` group, then it will be used as key's group. +Use ``require='__snakeoil_cert/...' __file ...`` to override. + + +OPTIONAL PARAMETERS +------------------- +common-name + Defaults to ``$__object_id``. + +key-path + ``%s`` in path will be replaced with ``$__object_id``. + Defaults to ``/etc/ssl/private/%s.pem``. + +key-type + Possible values are ``rsa:$bits`` and ``ec:$name``. + For possible EC names see ``openssl ecparam -list_curves``. + Defaults to ``rsa:2048``. + +cert-path + ``%s`` in path will be replaced with ``$__object_id``. + Defaults to ``/etc/ssl/certs/%s.pem``. + + +EXAMPLES +-------- +.. code-block:: sh + __snakeoil_cert localhost-rsa \ + --common-name localhost \ + --key-type rsa:4096 + + __snakeoil_cert localhost-ec \ + --common-name localhost \ + --key-type ec:prime256v1 + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path new file mode 100644 index 00000000..4bbae089 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path @@ -0,0 +1 @@ +/etc/ssl/certs/%s.pem diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-path b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path new file mode 100644 index 00000000..86eb9359 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path @@ -0,0 +1 @@ +/etc/ssl/private/%s.pem diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-type b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type new file mode 100644 index 00000000..f13f8ada --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type @@ -0,0 +1 @@ +rsa:2048 diff --git a/cdist/conf/type/__snakeoil_cert/parameter/optional b/cdist/conf/type/__snakeoil_cert/parameter/optional new file mode 100644 index 00000000..76d08c0a --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/optional @@ -0,0 +1,4 @@ +common-name +key-path +key-type +cert-path From 853e5cf7b4f615e1470c7ec2cfa8fe68cfa85ce7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 5 Jul 2021 09:07:06 +0200 Subject: [PATCH 088/128] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index d48c657e..0c3c64e1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -11,6 +11,8 @@ next: * Type __pyvenv: Fix group explorer (Dennis Camera) * Type __download: Improve checksum verification, add optional --destination (Ander Punnar) * Type __debconf_set_selections: Add state explorer (Dennis Camera) + * Core: Implement usable cdist scan (Timothée Floure) + * New type: __snakeoil_cert (Ander Punnar) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From be92731c5c8a8543448f0d87fafae67e22ac76a1 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Mon, 5 Jul 2021 12:38:26 +0100 Subject: [PATCH 089/128] Shell check quoting We're actually echo-ing the command, hence the escape in front of the quotes - the issue Shellcheck alludes too would actually occur, had the escaping bakcslashes been omitted. --- cdist/conf/type/__rsync/gencode-local | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index f1bddc16..be4feabb 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -34,7 +34,8 @@ if [ -f "$__object/parameter/rsync-opts" ]; then done < "$__object/parameter/rsync-opts" fi +# shellcheck disable=SC2086 echo rsync -a \ --no-owner --no-group \ - -e \"$__remote_exec\" \ + -e \"${__remote_exec}\" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From 521241d74102e37fae5f55552e1ef565d26ea9d2 Mon Sep 17 00:00:00 2001 From: fancsali Date: Mon, 5 Jul 2021 15:28:05 +0200 Subject: [PATCH 090/128] Refine docs even more --- cdist/conf/type/__apt_pin/man.rst | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst index 0c91cdec..4229c0cd 100644 --- a/cdist/conf/type/__apt_pin/man.rst +++ b/cdist/conf/type/__apt_pin/man.rst @@ -3,12 +3,12 @@ cdist-type__apt_pin(7) NAME ---- -cdist-type__apt_pin - TODO +cdist-type__apt_pin - Manage apt pinning rules DESCRIPTION ----------- -This space intentionally left blank. +Adds/removes/edits rules to pin some packages to a specific distribution. Useful if using multiple debian repositories at the same time. (Useful, if one wants to use a few specific packages from backports or perhaps Debain testing... or even sid.) REQUIRED PARAMETERS @@ -20,7 +20,7 @@ distribution OPTIONAL PARAMETERS ------------------- package - Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used. + Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used. priority The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority) @@ -28,7 +28,6 @@ priority state Will be passed to underlying `__file` type; see there for valid values and defaults. -None. BOOLEAN PARAMETERS @@ -41,8 +40,8 @@ EXAMPLES .. code-block:: sh - # Add the bullseye repo to buster, but do not install any pacakges by default - # only if explicitely asked for + # Add the bullseye repo to buster, but do not install any packages by default, + # only if explicitely asked for (-1 means "never" for apt) __apt_pin bullseye-default \ --package "*" \ --distribution bullseye \ @@ -52,19 +51,19 @@ EXAMPLES --uri http://deb.debian.org/debian/ \ --distribution bullseye \ --component main - # TODO - __apt_pin __apt_pin foo --package "foo foo-*" --distribution bullseye - __foo # Installs the `foo` package internally + __foo # Assuming, this installs the `foo` package internally - __package foo-plugin-extras + __package foo-plugin-extras # Assuming we also need some extra stuff SEE ALSO -------- -:strong:`apt_preferences`\ (7) +:strong:`apt_preferences`\ (5) +:strong:`cdist-type__apt_source`\ (7) +:strong:`cdist-type__apt_backports`\ (7) :strong:`cdist-type__file`\ (7) AUTHORS From 166b58aeea09da41086525849291c70a3c3a571c Mon Sep 17 00:00:00 2001 From: fancsali Date: Mon, 5 Jul 2021 15:32:27 +0200 Subject: [PATCH 091/128] Fix typo in distro names... --- cdist/conf/type/__apt_pin/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index 909bc80d..e72a8fdd 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -44,7 +44,7 @@ case "$os" in esac case $distribution in - stabletesting|unsatbel|experimental) + stable|testing|unstable|experimental) pin="release a=$distribution" ;; *) From 3e76d1cd3fbe53fa77c460cb2ce8416698b101bd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 8 Jul 2021 08:09:05 +0200 Subject: [PATCH 092/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 0c3c64e1..04f826f0 100644 --- a/docs/changelog +++ b/docs/changelog @@ -13,6 +13,7 @@ next: * Type __debconf_set_selections: Add state explorer (Dennis Camera) * Core: Implement usable cdist scan (Timothée Floure) * New type: __snakeoil_cert (Ander Punnar) + * Type __rsync: Honour $__remote_exec env var (Daniel Fancsali) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 77dab4c5c63070aef962875af7fe8b1565f5ba78 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 10 Jul 2021 20:37:02 +0200 Subject: [PATCH 093/128] Release 6.9.7 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 04f826f0..284293c1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) From 65c43d3c1db938ac0063f54b1cb6b5090bb0a665 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 10 Jul 2021 21:02:27 +0200 Subject: [PATCH 094/128] Fix docs code block errors --- cdist/conf/type/__snakeoil_cert/man.rst | 1 + docs/src/cdist-scan.rst | 2 ++ 2 files changed, 3 insertions(+) diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst index 0b547804..b0b0a2e9 100644 --- a/cdist/conf/type/__snakeoil_cert/man.rst +++ b/cdist/conf/type/__snakeoil_cert/man.rst @@ -38,6 +38,7 @@ cert-path EXAMPLES -------- .. code-block:: sh + __snakeoil_cert localhost-rsa \ --common-name localhost \ --key-type rsa:4096 diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst index 064e65ff..86b7fab6 100644 --- a/docs/src/cdist-scan.rst +++ b/docs/src/cdist-scan.rst @@ -57,6 +57,7 @@ resolved name to stdout - if any. The script must be executable. Simplest script: .. code-block:: sh + #!/bin/sh case "$1" in @@ -71,6 +72,7 @@ Simplest script: Resolving name from `PTR` DNS record: .. code-block:: sh + #!/bin/sh for cmd in dig sed; do From 0e611af2a6388572eef5112c2ffaed082803965c Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 13 Jul 2021 00:13:22 +0300 Subject: [PATCH 095/128] [__rsync] rewrite --- cdist/conf/type/__rsync/gencode-local | 123 +++++++++++++----- cdist/conf/type/__rsync/gencode-remote | 37 ------ cdist/conf/type/__rsync/man.rst | 107 +++++---------- cdist/conf/type/__rsync/manifest | 18 --- .../type/__rsync/parameter/default/options | 1 + cdist/conf/type/__rsync/parameter/optional | 4 +- .../type/__rsync/parameter/optional_multiple | 2 +- 7 files changed, 130 insertions(+), 162 deletions(-) delete mode 100755 cdist/conf/type/__rsync/gencode-remote create mode 100644 cdist/conf/type/__rsync/parameter/default/options diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index be4feabb..612d237e 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -1,41 +1,100 @@ #!/bin/sh -e -# -# 2015 Dominique Roux (dominique.roux4 at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -source=$(cat "$__object/parameter/source") -remote_user=$(cat "$__object/parameter/remote-user") +if ! command -v rsync > /dev/null +then + echo 'rsync is missing in local machine' >&2 + exit 1 +fi -if [ -f "$__object/parameter/destination" ]; then - destination=$(cat "$__object/parameter/destination") +src="$( cat "$__object/parameter/source" )" + +if [ ! -e "$src" ] +then + echo "$src not found" >&2 + exit 1 +fi + +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" else - destination="/$__object_id" + dst="/$__object_id" fi -set -- -if [ -f "$__object/parameter/rsync-opts" ]; then - while read -r opts; do - set -- "$@" "--$opts" - done < "$__object/parameter/rsync-opts" +# if source is directory, then make sure that +# source and destination are ending with slash, +# because this is what you almost always want when +# rsyncing two directories. + +if [ -d "$src" ] +then + if ! echo "$src" | grep -Eq '/$' + then + src="$src/" + fi + + if ! echo "$dst" | grep -Eq '/$' + then + dst="$dst/" + fi fi +remote_user="$( cat "$__object/parameter/remote-user" )" + +options="$( cat "$__object/parameter/options" )" + +if [ -f "$__object/parameter/option" ] +then + while read -r l + do + # there's a limitation in argparse: value can't begin with '-'. + # to workaround this, let's prefix opts with '\' in manifest and remove here. + # read more about argparse issue: https://bugs.python.org/issue9334 + + options="$options $( echo "$l" | sed 's/\\//g' )" + done \ + < "$__object/parameter/option" +fi + +if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ] +then + options="$options --chown=" + + if [ -f "$__object/parameter/owner" ] + then + owner="$( cat "$__object/parameter/owner" )" + options="$options$owner" + fi + + if [ -f "$__object/parameter/group" ] + then + group="$( cat "$__object/parameter/group" )" + options="$options:$group" + fi +fi + +if [ -f "$__object/parameter/mode" ] +then + mode="$( cat "$__object/parameter/mode" )" + options="$options --chmod=$mode" +fi + +# IMPORTANT +# +# 1. we first dry-run rsync with change summary to find out +# if there are any changes and code generation is needed. +# 2. normally, to get current state or target host, we run +# such operations in type explorers, but that's not +# possible due to how rsync works. +# 3. redirecting output of dry-run to stderr to ease debugging. +# 4. to understand how that cryptic regex works, please +# open rsync manpage and read about --itemize-changes. + # shellcheck disable=SC2086 -echo rsync -a \ - --no-owner --no-group \ - -e \"${__remote_exec}\" \ - -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" +if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \ + | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2 +then + exit 0 +fi + +echo "rsync $options $src $remote_user@$__target_host:$dst" diff --git a/cdist/conf/type/__rsync/gencode-remote b/cdist/conf/type/__rsync/gencode-remote deleted file mode 100755 index 074246af..00000000 --- a/cdist/conf/type/__rsync/gencode-remote +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -e -# -# 2015 Dominique Roux (dominique.roux4 at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -if [ -f "$__object/parameter/destination" ]; then - destination=$(cat "$__object/parameter/destination") -else - destination="/$__object_id" -fi - -ownergroup="" -if [ -f "$__object/parameter/owner" ]; then - ownergroup=$(cat "$__object/parameter/owner") -fi -if [ -f "$__object/parameter/group" ]; then - ownergroup="${ownergroup}:$(cat "$__object/parameter/group")" -fi - -if [ "$ownergroup" ]; then - echo chown -R "$ownergroup" "$destination" -fi diff --git a/cdist/conf/type/__rsync/man.rst b/cdist/conf/type/__rsync/man.rst index 94b06d63..88019c92 100644 --- a/cdist/conf/type/__rsync/man.rst +++ b/cdist/conf/type/__rsync/man.rst @@ -3,112 +3,73 @@ cdist-type__rsync(7) NAME ---- -cdist-type__rsync - Mirror directories using rsync +cdist-type__rsync - Mirror directories using ``rsync`` DESCRIPTION ----------- -WARNING: This type is of BETA quality: - -- it has not been tested widely -- interfaces *may* change -- if there is a better approach to solve the problem -> the type may even vanish - -If you are fine with these constraints, please read on. - - -This cdist type allows you to mirror local directories to the -target host using rsync. Rsync will be installed in the manifest of the type. -If group or owner are giveng, a recursive chown will be executed on the -target host. - -A slash will be appended to the source directory so that only the contents -of the directory are taken and not the directory name itself. +The purpose of this type is to bring power of ``rsync`` into ``cdist``. REQUIRED PARAMETERS ------------------- source - Where to take files from + Source directory in local machine. + If source is directory, slash (``/``) will be added to source and destination paths. OPTIONAL PARAMETERS ------------------- -group - Group to chgrp to. +destination + Destination directory. Defaults to ``$__object_id``. owner - User to chown to. + Will be passed to ``rsync`` as ``--chown=OWNER``. + Read ``rsync(1)`` for more details. -destination - Use this as the base destination instead of the object id +group + Will be passed to ``rsync`` as ``--chown=:GROUP``. + Read ``rsync(1)`` for more details. + +mode + Will be passed to ``rsync`` as ``--chmod=MODE``. + Read ``rsync(1)`` for more details. + +options + Defaults to ``--recursive --links --perms --times``. + Due to `bug in Python's argparse`_, value must be prefixed with ``\``. remote-user - Use this user instead of the default "root" for rsync operations. + Defaults to ``root``. OPTIONAL MULTIPLE PARAMETERS ---------------------------- -rsync-opts - Use this option to give rsync options with. - See rsync(1) for available options. - Only "--" options are supported. - Write the options without the beginning "--" - Can be specified multiple times. - - -MESSAGES --------- -NONE +option + Pass additional options to ``rsync``. + See ``rsync(1)`` for all possible options. + Due to `bug in Python's argparse`_, value must be prefixed with ``\``. EXAMPLES -------- - .. code-block:: sh - # You can use any source directory - __rsync /tmp/testdir \ - --source /etc - - # Use source from type - __rsync /etc \ - --source "$__type/files/package" - - # Allow multiple __rsync objects to write to the same dir - __rsync mystuff \ - --destination /usr/local/bin \ - --source "$__type/files/package" - - __rsync otherstuff \ - --destination /usr/local/bin \ - --source "$__type/files/package2" - - # Use rsync option --exclude - __rsync /tmp/testdir \ - --source /etc \ - --rsync-opts exclude=sshd_conf - - # Use rsync with multiple options --exclude --dry-run - __rsync /tmp/testing \ - --source /home/tester \ - --rsync-opts exclude=id_rsa \ - --rsync-opts dry-run - - -SEE ALSO --------- -:strong:`rsync`\ (1) + __rsync /var/www/example.com \ + --owner root \ + --group www-data \ + --mode 'D750,F640' \ + --source "$__files/example.com/www" AUTHORS ------- -Nico Schottelius +Ander Punnar COPYING ------- -Copyright \(C) 2015 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. diff --git a/cdist/conf/type/__rsync/manifest b/cdist/conf/type/__rsync/manifest index 9bd44c6d..64fa804e 100755 --- a/cdist/conf/type/__rsync/manifest +++ b/cdist/conf/type/__rsync/manifest @@ -1,21 +1,3 @@ #!/bin/sh -e -# -# 2015 Dominique Roux (dominique.roux4 at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# __package rsync diff --git a/cdist/conf/type/__rsync/parameter/default/options b/cdist/conf/type/__rsync/parameter/default/options new file mode 100644 index 00000000..d967b110 --- /dev/null +++ b/cdist/conf/type/__rsync/parameter/default/options @@ -0,0 +1 @@ +--recursive --links --perms --times diff --git a/cdist/conf/type/__rsync/parameter/optional b/cdist/conf/type/__rsync/parameter/optional index ac2b2390..833e9bbe 100644 --- a/cdist/conf/type/__rsync/parameter/optional +++ b/cdist/conf/type/__rsync/parameter/optional @@ -1,4 +1,6 @@ destination -owner group +mode +options +owner remote-user diff --git a/cdist/conf/type/__rsync/parameter/optional_multiple b/cdist/conf/type/__rsync/parameter/optional_multiple index fdb7cd88..01925a15 100644 --- a/cdist/conf/type/__rsync/parameter/optional_multiple +++ b/cdist/conf/type/__rsync/parameter/optional_multiple @@ -1 +1 @@ -rsync-opts +option From 46b5c24cd240cd9006d93cfcc12a4d81b46a5238 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 18 Jul 2021 16:25:00 +0300 Subject: [PATCH 096/128] use $__remote_exec for RSYNC_RSH --- cdist/conf/type/__rsync/gencode-local | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index 612d237e..e9f3c131 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -90,6 +90,8 @@ fi # 4. to understand how that cryptic regex works, please # open rsync manpage and read about --itemize-changes. +export RSYNC_RSH="$__remote_exec" + # shellcheck disable=SC2086 if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \ | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2 @@ -97,4 +99,6 @@ then exit 0 fi +echo "export RSYNC_RSH='$__remote_exec'" + echo "rsync $options $src $remote_user@$__target_host:$dst" From 5229337611b7e7afb2597729a786c637a8bec1f6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 18 Jul 2021 17:41:29 +0200 Subject: [PATCH 097/128] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 284293c1..f0746218 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __rsync: Rewrite (Ander Punnar) + 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) From de116661613002e91b54194ac8f0d5c3dd3eebbd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 18 Jul 2021 17:45:19 +0200 Subject: [PATCH 098/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index f0746218..ea51ac1b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Type __rsync: Rewrite (Ander Punnar) + * New type: __apt_pin (Daniel Fancsali) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 24c9406ea0f2c6c8edc87e5bbc1be25b5e8e1572 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 19 Jul 2021 12:13:23 +0200 Subject: [PATCH 099/128] [explorer/os_version] Convert Devuan ceres to version number Conversion of Devuan ceres to version numbers is done based on Devuan codenames. The version number is the version number of the final release - 0.01. Analogous to Debian. --- cdist/conf/explorer/os_version | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 3b02dedd..6c94915c 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -1,6 +1,7 @@ -#!/bin/sh +#!/bin/sh -e # # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2020-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,12 +18,11 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# # All os variables are lower case # -# -case "$("$__explorer/os")" in +case $("${__explorer:?}/os") +in amazon) cat /etc/system-release ;; @@ -59,7 +59,23 @@ case "$("$__explorer/os")" in esac ;; devuan) - cat /etc/devuan_version + devuan_version=$(cat /etc/devuan_version) + case ${devuan_version} + in + (*/ceres) + # ceres versions don't have a number, so we decode by codename: + case ${devuan_version} + in + (chimaera/ceres) echo 3.99 ;; + (beowulf/ceres) echo 2.99 ;; + (ascii/ceres) echo 1.99 ;; + (*) exit 1 + esac + ;; + (*) + echo "${devuan_version}" + ;; + esac ;; fedora) cat /etc/fedora-release From fbc9594729ece74b2f8612fe8046ddea966e8c05 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 20 Jul 2021 06:38:46 +0200 Subject: [PATCH 100/128] ++changelog --- docs/changelog | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/changelog b/docs/changelog index ea51ac1b..55d7cb73 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) + * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) @@ -146,7 +147,7 @@ next: * Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham) * Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham) * Configuration: Add notes to cdist.cfg.skeleton (Evil Ham) - * Explorers cpu_cores, memory: Improve *BSD support (Evil Ham) + * Explorers cpu_cores, memory: Improve BSD support (Evil Ham) * Core: Remove debug logging noise (Evil Ham) 6.5.4: 2020-04-11 @@ -211,7 +212,7 @@ next: * Documentation: PreOS english nitpicking (Evil Ham) * Documentation: Add installing from source with signature verification (Darko Poljak) * Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak) - * Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham) + * Type __start_on_boot: Docs: remove unsupported BSD claim (Evil Ham) * New type: __openldap_server (Evil Ham) 6.2.0: 2019-11-30 @@ -1070,9 +1071,9 @@ next: * Removed type __removeline (replaced by __line) (Nico Schottelius) * Type __directory: Parameter --parents and --recursive are now boolean (Nico Schottelius) * Type __package_apt, __package_luarocks, __package_opkg, - __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, - __package_rubygem, __package_yum, __process: - Parameter state accepts only "present" and "absent" (Nico Schottelius) + __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, + __package_rubygem, __package_yum, __process: + Parameter state accepts only "present" and "absent" (Nico Schottelius) * Dist: Initial support for pypi packaging (Nico Schottelius) 2.0.15: 2012-11-02 From c7daaabc6c28cbfaebaf6c620cc3cac130684d2c Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 20 Jul 2021 09:03:16 +0200 Subject: [PATCH 101/128] [docs] Bump copyright year to 2021 --- docs/src/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/conf.py b/docs/src/conf.py index 47765413..a3dfafca 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -56,7 +56,7 @@ master_doc = 'index' # General information about the project. project = 'cdist' -copyright = 'ungleich GmbH 2020' +copyright = 'ungleich GmbH 2021' # author = 'Darko Poljak' # The version info for the project you're documenting, acts as replacement for From fed01ded83b217a3ad147f2403ca863be373eeaa Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 22 Jul 2021 11:17:41 +0200 Subject: [PATCH 102/128] [cdist.log] Define custom log functions on logging.Logger Define out custom logger functions on logging.Logger so that they are passed on to all other loggers. Also, the logger functions need to take a self argument so that they can log on the corrent Logger. --- cdist/log.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cdist/log.py b/cdist/log.py index 113f3b4c..62e457fe 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -36,25 +36,27 @@ import threading logging.OFF = logging.CRITICAL + 10 # disable logging logging.addLevelName(logging.OFF, 'OFF') + logging.VERBOSE = logging.INFO - 5 logging.addLevelName(logging.VERBOSE, 'VERBOSE') -def _verbose(msg, *args, **kwargs): - logging.log(logging.VERBOSE, msg, *args, **kwargs) +def _verbose(self, msg, *args, **kwargs): + self.log(logging.VERBOSE, msg, args, **kwargs) -logging.verbose = _verbose +logging.Logger.verbose = _verbose + logging.TRACE = logging.DEBUG - 5 logging.addLevelName(logging.TRACE, 'TRACE') -def _trace(msg, *args, **kwargs): - logging.log(logging.TRACE, msg, *args, **kwargs) +def _trace(self, msg, *args, **kwargs): + self.log(logging.TRACE, msg, *args, **kwargs) -logging.trace = _trace +logging.Logger.trace = _trace class CdistFormatter(logging.Formatter): From 71fee1fd6b4874abffee5173a96c31842a9583bd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 23 Jul 2021 08:06:45 +0200 Subject: [PATCH 103/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 55d7cb73..1273e432 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) + * Core: Fix logging bug (Dennis Camera) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 67bcc6cae38e2d480c3336b6a4d59d01196190e3 Mon Sep 17 00:00:00 2001 From: Evilham Date: Sat, 24 Jul 2021 02:37:58 +0200 Subject: [PATCH 104/128] Improve Makefile compatibility and build docs We now use `$(MAKE)` for subsequent calls to `make`. This means that systems that do not default to GNU make can run `gmake man` and produce the man pages. While there also document a dependency on the rtd theme for sphinx. --- Makefile | 6 +++--- docs/src/cdist-install.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3712511c..89286310 100644 --- a/Makefile +++ b/Makefile @@ -35,9 +35,9 @@ DOCS_SRC_DIR=./docs/src SPEECHDIR=./docs/speeches TYPEDIR=./cdist/conf/type -SPHINXM=make -C $(DOCS_SRC_DIR) man -SPHINXH=make -C $(DOCS_SRC_DIR) html -SPHINXC=make -C $(DOCS_SRC_DIR) clean +SPHINXM=$(MAKE) -C $(DOCS_SRC_DIR) man +SPHINXH=$(MAKE) -C $(DOCS_SRC_DIR) html +SPHINXC=$(MAKE) -C $(DOCS_SRC_DIR) clean ################################################################################ # Manpages diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst index 18863145..390ab9ec 100644 --- a/docs/src/cdist-install.rst +++ b/docs/src/cdist-install.rst @@ -12,7 +12,7 @@ This is the machine from which you will configure target hosts. * /bin/sh: A POSIX like shell (for instance bash, dash, zsh) * Python >= 3.5 * SSH client - * sphinx (for building html docs and/or the man pages) + * sphinx with the rtd theme (for building html docs and/or the man pages) Target Hosts ~~~~~~~~~~~~ From cb8695cc88478640d7e76992438d90d3d6a68a90 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 24 Jul 2021 12:53:39 +0200 Subject: [PATCH 105/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 1273e432..181c4139 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * New type: __apt_pin (Daniel Fancsali) * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) * Core: Fix logging bug (Dennis Camera) + * Build: Improve Makefile compatibility (Evilham) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 4156fea9001aa267c8b173247cada2f919511c1b Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Wed, 28 Jul 2021 12:56:39 +0200 Subject: [PATCH 106/128] [filesystem] Add ubuntu as supported distribution. --- cdist/conf/type/__filesystem/explorer/lsblk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk index 9be3c575..d376c09f 100644 --- a/cdist/conf/type/__filesystem/explorer/lsblk +++ b/cdist/conf/type/__filesystem/explorer/lsblk @@ -27,7 +27,7 @@ else fi case "$os" in - alpine|centos|fedora|redhat|suse|gentoo) + alpine|centos|fedora|gentoo|redhat|suse|ubuntu) if [ ! -x "$(command -v lsblk)" ]; then echo "lsblk is required for __filesystem type" >&2 exit 1 From 542674dae81ab03a4de2c4ad0b2eaf264ee2c442 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 30 Jul 2021 10:30:33 +0200 Subject: [PATCH 107/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 181c4139..9fc10b20 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) * Core: Fix logging bug (Dennis Camera) * Build: Improve Makefile compatibility (Evilham) + * Type __filesystem: Support ubuntu (Joachim Desroches) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 53334fb4eb311550af7f5b73f279a2e86fa1c504 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 4 Aug 2021 19:50:10 +0200 Subject: [PATCH 108/128] [explorer/os_version] Fix for FreeBSD < 10.0 (again) --- cdist/conf/explorer/os_version | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 6c94915c..cc976608 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -89,7 +89,14 @@ in freebsd) # Apparently uname -r is not a reliable way to get the patch level. # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743 - freebsd-version + if command -v freebsd-version >/dev/null 2>&1 + then + # get userland version + freebsd-version -u + else + # fallback to kernel release for FreeBSD < 10.0 + uname -r + fi ;; *bsd|solaris) uname -r From e108cbc205cb5e7ac0d2e07b82cef4c83eaa285f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 3 Aug 2021 13:20:43 +0200 Subject: [PATCH 109/128] [explorer/os_version] Ubuntu: fall back to os-release/lsb-release files --- cdist/conf/explorer/os_version | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 6c94915c..e70fe7f4 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -21,6 +21,17 @@ # All os variables are lower case # +rc_getvar() { + awk -F= -v varname="$2" ' + function unquote(s) { + if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/) + return substr(s, 2, length(s) - 2) + else + return s + } + $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1" +} + case $("${__explorer:?}/os") in amazon) @@ -114,7 +125,20 @@ in fi ;; ubuntu) - lsb_release -sr + if command -v lsb_release >/dev/null 2>&1 + then + lsb_release -sr + elif test -r /usr/lib/os-release + then + # fallback to /usr/lib/os-release if lsb_release is not present (like + # on minimized Ubuntu installations) + rc_getvar /usr/lib/os-release VERSION_ID + elif test -r /etc/lsb-release + then + # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old + # versions without /usr/lib/os-release. + rc_getvar /etc/lsb-release DISTRIB_RELEASE + fi ;; alpine) cat /etc/alpine-release From 83fe6e9f5b2537db73d5c6a142b6d24eef75ac58 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 3 Aug 2021 19:26:55 +0200 Subject: [PATCH 110/128] [explorer/memory] Fix conversion of large numbers (>= 2GiB) At least mawk uses scientific notation when using print for numbers >=2^31 (INT_MAX of a signed 32-bit int). `printf "%.f\n"` works around this. --- cdist/conf/explorer/memory | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory index 63aba9c6..c6d113cf 100755 --- a/cdist/conf/explorer/memory +++ b/cdist/conf/explorer/memory @@ -27,19 +27,18 @@ str2bytes() { awk -F' ' ' $2 == "B" || !$2 { print $1 } - $2 == "kB" { print $1 * 1000 } - $2 == "MB" { print $1 * 1000 * 1000 } - $2 == "GB" { print $1 * 1000 * 1000 * 1000 } - $2 == "TB" { print $1 * 1000 * 1000 * 1000 * 1000 } - $2 == "kiB" { print $1 * 1024 } - $2 == "MiB" { print $1 * 1024 * 1024 } - $2 == "GiB" { print $1 * 1024 * 1024 * 1024 } - $2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }' + $2 == "kB" { printf "%.f\n", ($1 * 1000) } + $2 == "MB" { printf "%.f\n", ($1 * 1000 * 1000) } + $2 == "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) } + $2 == "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) } + $2 == "kiB" { printf "%.f\n", ($1 * 1024) } + $2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) } + $2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) } + $2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }' } bytes2kib() { - set -- "$(cat)" - test "$1" -gt 0 && echo $(($1 / 1024)) + awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }' } From a7d6481a7ddc7cb72b1a55bfca7fdfed20514a62 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 2 Aug 2021 21:23:50 +0200 Subject: [PATCH 111/128] [type/__update_alternatives] Secure cdist-defined environment variables with :? --- .../__update_alternatives/explorer/alternatives | 2 +- cdist/conf/type/__update_alternatives/explorer/link | 6 +++--- .../type/__update_alternatives/explorer/path_is | 4 ++-- .../explorer/path_should_state | 2 +- .../conf/type/__update_alternatives/gencode-remote | 13 ++++++------- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives index 34aaca56..ecc62f4b 100755 --- a/cdist/conf/type/__update_alternatives/explorer/alternatives +++ b/cdist/conf/type/__update_alternatives/explorer/alternatives @@ -1,4 +1,4 @@ #!/bin/sh -e -update-alternatives --display "$__object_id" 2>/dev/null \ +update-alternatives --display "${__object_id:?}" 2>/dev/null \ | awk -F ' - ' '/priority [0-9]+$/ { print $1 }' diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link index 6519e7c2..c6fd1c98 100755 --- a/cdist/conf/type/__update_alternatives/explorer/link +++ b/cdist/conf/type/__update_alternatives/explorer/link @@ -18,12 +18,12 @@ for altdir in \ /var/lib/dpkg/alternatives \ /var/lib/alternatives do - if [ ! -f "$altdir/$__object_id" ] + if [ ! -f "$altdir/${__object_id:?}" ] then continue fi - link="$( awk 'NR==2' "$altdir/$__object_id" )" + link="$( awk 'NR==2' "$altdir/${__object_id:?}" )" if [ -n "$link" ] then @@ -33,7 +33,7 @@ done if [ -z "$link" ] then - echo "unable to get link for $__object_id" >&2 + echo "unable to get link for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index fc304d5d..a24bd40e 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -1,11 +1,11 @@ #!/bin/sh -e -path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \ +path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \ | awk '/link currently points to/ {print $5}' )" if [ -z "$path_is" ] then - echo "unable to get current path for $__object_id" >&2 + echo "unable to get current path for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state index 59e015c5..b74a7ee8 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_should_state +++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state @@ -1,6 +1,6 @@ #!/bin/sh -e -if [ -f "$( cat "$__object/parameter/path" )" ] +if [ -f "$( cat "${__object:?}/parameter/path" )" ] then echo 'present' else diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index e393cdef..13666805 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -18,26 +18,25 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . -path_is="$( cat "$__object/explorer/path_is" )" +path_is="$( cat "${__object:?}/explorer/path_is" )" -path_should="$( cat "$__object/parameter/path" )" +path_should="$( cat "${__object:?}/parameter/path" )" if [ "$path_is" = "$path_should" ] then exit 0 fi -if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ] +if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \ + && [ -z "${__cdist_dry_run+dry run}" ] then echo "$path_should does not exist in target" >&2 exit 1 fi -name="$__object_id" +name=${__object_id:?} -alternatives="$( cat "$__object/explorer/alternatives" )" - -if ! echo "$alternatives" | grep -Fxq "$path_should" +if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives" then if [ ! -f "$__object/parameter/install" ] then From 0b3b47396f2aafa377e3d5d9a13f51ace2303d41 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 2 Aug 2021 21:25:08 +0200 Subject: [PATCH 112/128] [type/__update_alternatives] dry-run fixes --- cdist/conf/type/__update_alternatives/explorer/link | 5 ++++- .../type/__update_alternatives/explorer/path_is | 5 ++++- .../conf/type/__update_alternatives/gencode-remote | 13 ++++++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link index c6fd1c98..d1087c75 100755 --- a/cdist/conf/type/__update_alternatives/explorer/link +++ b/cdist/conf/type/__update_alternatives/explorer/link @@ -31,8 +31,11 @@ do fi done -if [ -z "$link" ] +if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ] then + # NOTE: ignore error for dry-runs because a package providing the link + # might be managed by another cdist object (which wasn't executed, + # because dry run…). echo "unable to get link for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index a24bd40e..9208df7b 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -3,8 +3,11 @@ path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \ | awk '/link currently points to/ {print $5}' )" -if [ -z "$path_is" ] +if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ] then + # NOTE: ignore error for dry-runs because a package providing the + # alternative might be managed by another cdist object (which + # wasn't executed, because dry run…). echo "unable to get current path for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index 13666805..e91ea78f 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -38,16 +38,19 @@ name=${__object_id:?} if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives" then - if [ ! -f "$__object/parameter/install" ] + if [ -f "${__object:?}/parameter/install" ] then + link="$( cat "${__object:?}/explorer/link" )" + echo "update-alternatives --install '$link' '$name' '$path_should' 1000" + elif [ -z "${__cdist_dry_run+dry run}" ] + then + # NOTE: ignore error for dry-runs because a package providing the link + # to be installed might be managed by another cdist object (which + # wasn't executed, because dry run…). echo "$path_should is not in $name alternatives." >&2 echo 'Please install missing packages or use --install to add path to alternatives.' >&2 exit 1 fi - - link="$( cat "$__object/explorer/link" )" - - echo "update-alternatives --install '$link' '$name' '$path_should' 1000" fi echo "update-alternatives --set '$name' '$path_should'" From bbcc81a9841f2619e1b9e13b25a941337489a681 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 4 Aug 2021 21:44:04 +0200 Subject: [PATCH 113/128] [type/__update_alternatives] Fix for non-English locales Since update-alternatives(1) is localized, screen scraping its output breaks if the locale is set to non-English. --- cdist/conf/type/__update_alternatives/explorer/alternatives | 4 ++-- cdist/conf/type/__update_alternatives/explorer/path_is | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives index ecc62f4b..bb1619a9 100755 --- a/cdist/conf/type/__update_alternatives/explorer/alternatives +++ b/cdist/conf/type/__update_alternatives/explorer/alternatives @@ -1,4 +1,4 @@ #!/bin/sh -e -update-alternatives --display "${__object_id:?}" 2>/dev/null \ - | awk -F ' - ' '/priority [0-9]+$/ { print $1 }' +LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \ +| awk -F ' - ' '/priority [0-9]+$/ { print $1 }' diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index 9208df7b..5cf4fa4b 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -1,7 +1,8 @@ #!/bin/sh -e -path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \ - | awk '/link currently points to/ {print $5}' )" +path_is=$( + LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \ + | awk '/link currently points to/ { print $5 }') if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ] then From 2a0c073d4021206c7459015cefaba218004235ce Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 4 Aug 2021 21:54:17 +0200 Subject: [PATCH 114/128] [explorer/os_version] Fix for legacy Mac OS X versions --- cdist/conf/explorer/os_version | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 3b02dedd..96eca1ee 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -68,7 +68,8 @@ case "$("$__explorer/os")" in cat /etc/gentoo-release ;; macosx) - sw_vers -productVersion + # NOTE: Legacy versions (< 10.3) do not support options + sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }' ;; freebsd) # Apparently uname -r is not a reliable way to get the patch level. From 3ae5a606ca7182ec7fe13134670400506e198ec1 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 5 Aug 2021 10:27:51 +0200 Subject: [PATCH 115/128] ++changelog --- docs/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog b/docs/changelog index 9fc10b20..f9409d7e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,10 @@ next: * Core: Fix logging bug (Dennis Camera) * Build: Improve Makefile compatibility (Evilham) * Type __filesystem: Support ubuntu (Joachim Desroches) + * Explorer os_version: Fall back to os-release/lsb-release file on Ubuntu (Dennis Camera) + * Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera) + * Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera) + * Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From edcac70b2a99ce532dd5bca9a0b8fc5bf6dc5148 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 21 Jul 2021 13:22:34 +0200 Subject: [PATCH 116/128] [explorer/machine_type] Reimplement --- cdist/conf/explorer/machine_type | 986 ++++++++++++++++++++++++++++--- 1 file changed, 904 insertions(+), 82 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 1c84f4d7..29f98849 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -1,8 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2014 Daniel Heule (hda at sfs.biz) -# 2014 Thomas Oettli (otho at sfs.biz) -# 2020 Evilham (contact at evilham.com) +# 2021 Dennis Camera (cdist at dtnr.ch) # # This file is part of cdist. # @@ -19,91 +17,915 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# This explorer tries to determine what type of machine the target to be +# configured is (container, virtual machine, bare-metal). +# +# It will print one line for each layer it can detect. +# The format of all lines is: TYPE[ VERB VENDOR] +# +# VERB does not have a special meaning, it is just for better readability. +# +# e.g. +# container on lxc +# virtual by kvm-spapr +# +# The third word of each line can be composed of different parts concatenated with a `-' +# (minus) character, with each component being a specification of the previous, +# e.g.: +# - lxc-libvirt (LXC container, managed by libvirt) +# - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) +# - xen-hvm / xen-pv (Xen HVM vs para-virtualization) +# +# If this explorer cannot determine any information it will print nothing. +# -os=$("$__explorer/os") +# Add /sbin and /usr/sbin to the path so we can find system +# binaries like dmidecode. +PATH=$(getconf PATH 2>/dev/null) || PATH='/usr/bin:/bin' +PATH="/sbin:/usr/sbin:${PATH}" +export PATH -vendor_string_to_machine_type() { - for vendor in vmware bochs kvm qemu virtualbox bhyve; do - if echo "${1}" | grep -q -i "${vendor}"; then - if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then - vendor="kvm" - fi - echo "virtual_by_${vendor}" - exit - fi - done +arch=$(uname -m | sed -e 's/i.86/i386/' -e 's/arm.*/arm/') +uname_s=$(uname -s) + + +is_command() { command -v "$1" >/dev/null 2>&1; } + +is_oneof() ( + x=$1; shift + for y + do + test "${x}" = "${y}" || continue + return 0 + done + return 1 +) + +tolower() { LC_ALL=C tr '[:upper:]' '[:lower:]'; } + +# shellcheck disable=SC2086 +glob_exists() { set -- $1; test -e "$1"; } + +get_dmi_field() { + if is_oneof "${uname_s}" NetBSD + then + case $1 + in + (system-manufacturer) _mib=machdep.dmi.system-vendor ;; + (system-product-name) _mib=machdep.dmi.system-product ;; + (system-version|system-uuid) _mib=machdep.dmi.$1 ;; + (bios-vendor|bios-version) _mib=machdep.dmi.$1 ;; + (biod-release-date) _mib=machdep.dmi.bios-date ;; + (*) _mib= ;; + esac + + test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return + fi + + if is_command dmidecode + then + dmidecode -s "$1" + elif test -d "${dmi_sysfs-}" + then + case $1 + in + (system-manufacturer) _filename=sys_vendor ;; + (system-product-name) _filename=product_name ;; + (*) _filename=$(echo "$1" | tr - _) ;; + esac + if test -r "${dmi_sysfs-}/${_filename}" + then + cat "${dmi_sysfs}/${_filename}" + fi + unset _filename + elif test "${uname_s}" = OpenBSD + then + # NOTE: something similar to system-manufacutrer and system-product-name + # is available on OpenBSD in sysctl + case $1 + in + (system-manufacturer) _mib=hw.vendor ;; + (system-product-name) _mib=hw.product ;; + (*) _mib= ;; + esac + + test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return + fi + + return 1 } -case "$os" in - "freebsd") - # FreeBSD does not have /proc/cpuinfo even when procfs is used. - # Instead there is a sysctl kern.vm_guest. - # Which is 'none' if physical, else the virtualisation. - vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)" - if [ -n "${vm_guest}" ]; then - if [ "${vm_guest}" = "none" ]; then - echo "physical" - exit - fi - echo "virtual_by_${vm_guest}" - exit - fi - ;; +has_cpuinfo() { test -e /proc/cpuinfo; } - "openbsd") - # OpenBSD can also use the sysctl's: hw.vendor or hw.product. - # Note we can be reasonably sure about a machine being virtualised - # as long as we can identify the virtualisation technology. - # But not so much about it being physical... - # Patches are welcome / reach out if you have better ideas. - for sysctl in hw.vendor hw.product; do - # This exits if we can make a reasonable judgement - vendor_string_to_machine_type "$(sysctl -n "${sysctl}")" - done - ;; +get_sysctl() { + is_command sysctl && sysctl -n "$1" 2>/dev/null +} - *) - # Defaulting to linux for compatibility with previous cdist behaviour - if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then - echo openvz - exit - fi +# Check for container - if [ -e "/proc/1/environ" ] && - tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then - echo lxc - exit - fi +has_ct_pid_1() { + test -r /run/systemd/container -o -r /proc/1/environ +} - if [ -r /proc/cpuinfo ]; then - # this should only exist on virtual guest machines, - # tested on vmware, xen, kvm, bhyve - if grep -q "hypervisor" /proc/cpuinfo; then - # this file is aviable in xen guest systems - if [ -r /sys/hypervisor/type ]; then - if grep -q -i "xen" /sys/hypervisor/type; then - echo virtual_by_xen - exit - fi - else - for vendor_file in /sys/class/dmi/id/product_name \ - /sys/class/dmi/id/sys_vendor \ - /sys/class/dmi/id/chasis_vendor; do - if [ -r ${vendor_file} ]; then - # This exits if we can make a reasonable judgement - vendor_string_to_machine_type "$(cat "${vendor_file}")" - fi - done - fi - echo "virtual_by_unknown" - exit - else - echo "physical" - exit - fi - fi - ;; -esac +translate_container_name() { + case $1 + in + ('lxc') + echo lxc ;; + ('lxc-libvirt') + echo lxc-libvirt ;; + ('podman') + echo podman ;; + ('systemd-nspawn') + echo systemd_nspawn ;; + (*) + return 1 ;; + esac + return 0 +} -echo "unknown" +check_ct_pid_1() { + if test -r /run/systemd/container + then + translate_container_name "$(head -n1 /run/systemd/container)" \ + && return 0 + fi + + if test -r /proc/1/environ + then + translate_container_name "$( + LC_ALL=C tr '\000' '\n' /dev/null + then + # https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 + echo wsl + elif test -d /var/.cagefs + then + # https://docs.cloudlinux.com/cloudlinux_os_components/#cagefs + # CageFS is not "really" a container, but it isn't a chroot either. + echo cagefs + elif test -e /proc/self/status && grep -q -e '^VxID: [0-9]\{1,\}' /proc/self/status + then + # Linux-VServer + if grep -q -x -F 'VxID: 0' /proc/self/status + then + # host + return 1 + else + # guest + echo linux_vserver + fi + else + return 1 + fi +} + +check_ct_os_specific() ( + if jailed=$(get_sysctl security.jail.jailed) && test "${jailed}" = 1 + then + # FreeBSD jail + echo jail + return 0 + fi + + if is_command zonename && test "$(zonename)" != global + then + # Solaris zone + echo zone + return 0 + fi + + return 1 +) + + +# Check for hypervisor + +guess_hypervisor_from_cpu_model() { + case $1 + in + (*\ KVM\ *) + echo kvm ;; + (*\ QEMU\ *|QEMU\ *) + echo qemu ;; + (*) + return 1 ;; + esac +} + +has_vm_cpuinfo() { has_cpuinfo; } + +check_vm_cpuinfo() { + if grep -q -F 'User Mode Linux' /proc/cpuinfo \ + || grep -q -F 'UML' /proc/cpuinfo + then + # User Mode Linux + echo uml + elif grep -q -e '^vendor_id.*: PowerVM Lx86' /proc/cpuinfo + then + # IBM PowerVM Lx86 (Linux/x86 emulator) + echo powervm_lx86 + elif grep -q -e '^vendor_id.*: IBM/S390' /proc/cpuinfo + then + # IBM SystemZ (S/390) + if test -f /proc/sysinfo + then + if grep -q -e '^VM[0-9]* Control Program: KVM/Linux' /proc/sysinfo + then + echo kvm-s390 + return 0 + elif grep -q -e '^VM[0-9]* Control Program: z/VM' /proc/sysinfo + then + echo zvm + return 0 + elif grep -q -e '^LPAR ' /proc/sysinfo + then + echo zvm-lpar + return 0 + fi + fi + return 1 + else + if grep -q -e '^model name.*:' /proc/cpuinfo + then + sed -n -e 's/^model name[^:]*: *//p' /proc/cpuinfo \ + | while read -r _cpu_model + do + guess_hypervisor_from_cpu_model "${_cpu_model}" + done \ + | sort \ + | uniq -c \ + | awk ' + { if ($1 > most_c) { most_c = $1; most_s = $2 } } + END { + if (most_s) print most_s + exit !most_s + }' \ + && return 0 + fi + return 1 + fi +} + +check_vm_arch_specific() { + case ${arch} + in + (ppc64|ppc64le) + # Check PPC64 LPAR, KVM + + # example /proc/cpuinfo line indicating 'not baremetal' + # platform : pSeries + # + # example /proc/ppc64/lparcfg systemtype line + # system_type=IBM pSeries (emulated by qemu) + + if has_cpuinfo && grep -q -e 'platform.**pSeries' /proc/cpuinfo + then + if test -e /proc/ppc64/lparcfg + then + # Assume LPAR, now detect shared or dedicated + if grep -q -x -F 'shared_processor_mode=1' /proc/ppc64/lparcfg + then + echo powervm-shared + return 0 + else + echo powervm-dedicated + return 0 + fi + fi + fi + ;; + (sparc*) + # Check for SPARC LDoms + + if test -e /dev/mdesc + then + if test -d /sys/class/vlds/ctrl -a -d /sys/class/vlds/sp + then + # control LDom + return 1 + else + # guest LDom + echo ldom-sparc + fi + + # MDPROP=/usr/lib/ldoms/mdprop.py + # if test -x "${MDPROP}" + # then + # if test -n "$("${MDPROP}" -v iodevice device-type=pciex)" + # then + # echo ldoms-root + # echo ldoms-io + # elif test -n "$("${MDPROP}" -v iov-device vf-id=0)" + # then + # echo ldoms-io + # fi + # fi + return 0 + fi + ;; + (i?86|x86*|amd64|i86pc) + # Check CPUID + # + # Many fullvirt hypervisors give an indication through CPUID. Use + # the virt-what helper program to get this information if available. + + for CPUID_HELPER in \ + $(command -v virt-what-cpuid-helper 2>/dev/null) \ + /usr/lib/x86_64-*/virt-what-cpuid-helper \ + /usr/lib/i?86-*/virt-what-cpuid-helper \ + /usr/lib/virt-what/virt-what-cpuid-helper + do + if test -x "${CPUID_HELPER:?}"; then break; fi + done + + if test -x "${CPUID_HELPER-}" + then + case $(command "${CPUID_HELPER}") + in + ('bhyve bhyve ') + echo bhyve + ;; + ('LKVMLKVMLKVM') + echo lkvm + ;; + ('KVMKVMKVM') + echo kvm + ;; + ('TCGTCGTCGTCG') + echo qemu-tcg + ;; + ('Microsoft Hv') + # http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx + echo hyperv + ;; + ('OpenBSDVMM58') + # OpenBSD/VMM + echo openbsd_vmm + ;; + ('VMwareVMware') + # check added by Chetan Loke. + echo vmware + ;; + ('XenVMMXenVMM') + if has dmi + then + # https://access.redhat.com/solutions/222903 + echo xen-hvm + else + echo xen-paravirt + fi + ;; + (*) + return 1 ;; + esac + return 0 + fi + + unset CPUID_HELPER + + # VMM CPUID flag denotes that this system is running under a VMM + if is_oneof "${uname_s}" Darwin + then + get_sysctl machdep.cpu.features | tr ' ' '\n' | grep -qixF VMM \ + && return 0 + fi + if has_cpuinfo \ + && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo + then + return 0 + fi + ;; + (ia64) + if test -d /sys/bus/xen -a ! -d /sys/bus/xen-backend + then + # PV-on-HVM drivers installed in a Xen guest + echo xen-hvm + return 0 + fi + ;; + esac + return 1 +} + +has_vm_dmi() { + # Check for various products in SMBIOS/DMI. + # Note that DMI doesn't exist on all architectures (only x86 and some ARM). + # On other architectures the $dmi variable will be empty. + + if test -d /sys/class/dmi/id/ + then + dmi_sysfs=/sys/class/dmi/id + elif test -d /sys/devices/virtual/dmi/id/ + then + dmi_sysfs=/sys/devices/virtual/dmi/id + fi + + # shellcheck disable=SC2015 + { + is_command dmidecode \ + && ( + # dmidecode needs to exit 0 and not print the No SMBIOS/DMI line + dmi_out=$(dmidecode 2>&1) \ + && ! printf '%s\n' "${dmi_out}" \ + | grep -qF 'No SMBIOS nor DMI entry point found, sorry.' + ) \ + || test -d "${dmi_sysfs}" + } +} + +check_vm_dmi() { + case $(get_dmi_field system-product-name) + in + (*.metal) + if test "$(get_dmi_field system-manufacturer)" = 'Amazon EC2' + then + # AWS EC2 bare metal -> no virtualisation + return 1 + fi + ;; + ('BHYVE') + echo bhyve + return 0 + ;; + ('Google Compute Engine') + echo gce + return 0 + ;; + ('RHEV Hypervisor') + # Red Hat Enterprise Virtualization + echo rhev + return 0 + ;; + ('KVM'|'Bochs'|'KVM Virtual Machine') + echo kvm + return 0 + ;; + ('Parallels Virtual Platform') + echo parallels + return 0 + ;; + ('VirtualBox') + echo virtualbox + return 0 + ;; + ('VMware Virtual Platform') + echo vmware + return 0 + ;; + esac + + case $(get_dmi_field system-manufacturer) + in + ('Alibaba'*) + case $(get_dmi_field system-product-name) + in + ('Alibaba Cloud ECS') + echo alibaba-ecs + ;; + (*) + echo alibaba + ;; + esac + return 0 + ;; + ('Amazon EC2') + # AWS on bare-metal or KVM + echo aws-ec2 + return 0 + ;; + ('innotek GmbH'|'Oracle Corporation') + echo virtualbox + return 0 + ;; + ('Joyent') + if test "$(get_dmi_field system-product-name)" = 'SmartDC HVM' + then + # SmartOS KVM + echo kvm-smartdc_hvm + return 0 + fi + ;; + ('Microsoft Corporation'*) + if test "$(get_dmi_field system-product-name)" = 'Virtual Machine' + then + if test -e /proc/irq/7/hyperv \ + || expr "$(get_dmi_field bios-version)" : 'VRTUAL.*' >/dev/null + then + echo hyperv + return 0 + fi + + case $(get_dmi_field system-version) + in + (VPC[0-9]*|VS2005*|*[Vv]irtual*[Pp][Cc]*) + echo virtualpc + return 0 + ;; + (*) + echo hyperv + return 0 + ;; + esac + fi + ;; + ('Nutanix') + # Nutanix AHV. Similar to KVM. + if test "$(get_dmi_field system-product-name)" = 'AHV' + then + echo nutanix_ahv + return 0 + fi + ;; + ('oVirt') + echo ovirt + return 0 + ;; + ('Parallels Software International Inc.') + echo parallels + return 0 + ;; + ('QEMU') + echo qemu + return 0 + ;; + ('VMware, Inc.') + echo vmware + return 0 + ;; + esac + + case $(get_dmi_field bios-vendor) + in + ('Amazon EC2') + # AWS on bare-metal or KVM + echo aws-ec2 + return 0 + ;; + ('BHYVE') + echo bhyve + return 0 + ;; + ('innotek GmbH') + echo virtualbox + return 0 + ;; + ('Parallels Software International Inc.') + echo parallels + return 0 + ;; + ('Xen') + if get_dmi_field bios-version | grep -q -e '\([0-9]\{1,\}\.\)\{2\}amazon' + then + # AWS on Xen + echo aws-xen + return 0 + fi + ;; + esac + + return 1 +} + +check_vm_hyp_specific() { + if is_command vmware-checkvm && vmware-checkvm >/dev/null + then + # vmware-checkvm is provided by VMware's open-vm-tools + echo vmware + return 0 + elif test -d /proc/xen + then + test -r /proc/xen/capabilities && + if grep -q -F 'control_d' /proc/xen/capabilities 2>/dev/null + then + # Xen dom0 + return 1 + else + # Xen domU + echo xen + return 0 + fi + fi + return 1 +} + +has_vm_dt() { + # OpenFirmware/Das U-Boot device-tree + test -d /proc/device-tree +} + +check_vm_dt() { + case ${arch} + in + (arm|aarch64) + if test -r /proc/device-tree/hypervisor/compatible + then + if grep -q -F 'xen' /proc/device-tree/hypervisor/compatible + then + echo xen + return 0 + elif grep -q -F 'vmware' /proc/device-tree/hypervisor/compatible + then + # e.g. VMware ESXi on ARM + echo vmware + return 0 + fi + fi + if glob_exists /proc/device-tree/fw-cfg@*/compatible + then + # qemu,fw-cfg-mmio + sed -e 's/,.*$//' /proc/device-tree/fw-cfg@*/compatible | head -n1 + return 0 + fi + if grep -q -F 'dummy-virt' /proc/device-tree/compatible + then + echo lkvm + return 0 + fi + ;; + (ppc64*) + if test -d /proc/device-tree/hypervisor \ + && grep -qF 'linux,kvm' /proc/device-tree/hypervisor/compatible + then + # We are running as a spapr KVM guest on ppc64 + echo kvm-spapr + return 0 + fi + if test -r /proc/device-tree/ibm,partition-name \ + && test -r /proc/device-tree/hmc-managed\? \ + && test -r /proc/device-tree/chosen/qemu,graphic-width + then + echo powervm + fi + ;; + esac + return 1 +} + +has_vm_sys_hypervisor() { + test -d /sys/hypervisor/ +} + +check_vm_sys_hypervisor() { + test -r /sys/hypervisor/type && + case $(head -n1 /sys/hypervisor/type) + in + (xen) + # Ordinary kernel with pv_ops. There does not seem to be + # enough information at present to tell whether this is dom0 + # or domU. + echo xen + return 0 + ;; + esac + return 1 +} + +check_vm_os_specific() { + _hyp_generic=false + + case ${uname_s} + in + (Darwin) + if hv_vmm_present=$(get_sysctl kern.hv_vmm_present) \ + && test "${hv_vmm_present}" -ne 0 + then + _hyp_generic=true + fi + ;; + (FreeBSD) + # FreeBSD does not have /proc/cpuinfo even when procfs is used. + # Instead there is a sysctl kern.vm_guest. + # Which is 'none' if physical, else the virtualisation. + vm_guest=$(get_sysctl kern.vm_guest | tolower) && + case ${vm_guest} + in + (none) ;; + (generic) _hyp_generic=true ;; + (*) + # kernel could detect hypervisor + case ${vm_guest} + in + (hv) echo hyperv ;; + (vbox) echo virtualbox ;; + (*) echo "${vm_guest}" ;; + esac + return 0 + ;; + esac + ;; + (NetBSD) + machdep_hv=$(get_sysctl machdep.hypervisor | tolower) && + case ${machdep_hv} + in + (none) ;; + (generic) _hyp_generic=true ;; + (*) + # kernel could detect hypervisor + case ${machdep_hv} + in + (hyper-v) echo hyperv ;; + (xenhvm*) echo xen-hvm ;; + (xenpv*) echo xen-pv ;; + (xen*) echo xen ;; + (*) echo "${machdep_hv}" ;; + esac + return 0 + ;; + esac + ;; + (OpenBSD) + if is_command hostctl && glob_exists /dev/pvbus[0-9]* + then + for _pvbus in /dev/pvbus[0-9]* + do + _h_out=$(hostctl -f "${_pvbus}" -t 2>/dev/null) || continue + case $(expr "${_h_out}" : '[^:]*: *\(.*\)$') + in + (KVM) echo kvm ;; + (Hyper-V) echo hyperv ;; + (VMware) echo vmware ;; + (Xen) echo xen ;; + (bhyve) echo bhyve ;; + (OpenBSD) echo openbsd_vmm ;; + esac + return 0 + done + fi + ;; + (SunOS) + diag_conf=$(prtdiag | sed -n -e 's/.*Configuration: *//p' -e '/^$/q') + # NOTE: Don't use -e or -F in Solaris grep + if printf '%s\n' "${diag_conf}" | grep -q -i QEMU + then + echo qemu + return 0 + elif printf '%s\n' "${diag_conf}" | grep -q -i VMware + then + echo vmware + return 0 + fi + ;; + (Linux) + if is_command dmesg + then + while read -r line + do + case ${line} + in + ('Booting paravirtualized kernel on ') + case $(expr "${line}" : '.* kernel on \(.*\)') + in + ('Xen') + echo xen-pv; return 0 ;; + ('bare hardware') + return 1 ;; + esac + ;; + ('Hypervisor detected') + case $(expr "${line}" : '.*: *\(.*\)') + in + ('ACRN') + echo acrn ;; + ('Jailhouse') + echo jailhouse ;; + ('KVM') + echo kvm ;; + ('Microsoft Hyper-V') + echo hyperv ;; + ('VMware') + echo vmware ;; + ('Xen HVM') + echo xen-hvm ;; + ('Xen PV') + echo xen-pv ;; + esac + return 0 + ;; + (lpar:*' under hypervisor') + return 0 ;; + esac + done <<-EOF + $(dmesg 2>/dev/null | awk ' + /Booting paravirtualized kernel on / + /Hypervisor detected: / + /lpar: .* under hypervisor/ + ') + EOF + fi + esac + + # Try to guess hypervisor based on CPU model (sysctl hw.model if available) + if cpu_model=$(get_sysctl hw.model) + then + guess_hypervisor_from_cpu_model "${cpu_model}" && return 0 + fi + + if ${_hyp_generic} + then + # cannot say which hypervisor, but one was detected + return 0 + else + return 1 + fi +} + +run_stage() { + if type "has_$1_$2" >/dev/null 2>&1 + then + "has_$1_$2" + else + true + fi \ + && "check_$1_$2" +} + + +# Execute container stages + +for stage in \ + pid_1 cgroup files os_specific +do + ctengine=$(run_stage ct ${stage}) || continue + is_contained=true + if test -n "${ctengine}" + then + echo container on "${ctengine}" + break + fi +done +if ${is_contained:-false} && test -z "${ctengine}" +then + # none of the stages could determine the specific container engine, but + # we are running in some container. + echo container +fi + + +# Execute virtual machine / hypervisor stages + +for stage in \ + os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific +do + hypervisor=$(run_stage vm ${stage}) || continue + is_virtual=true + if test -n "${hypervisor}" + then + echo virtual by "${hypervisor}" + break + fi +done +if ${is_virtual:-false} && test -z "${hypervisor}" +then + # none of the stages could determine the specific hypervisor, but + # we are virtual. + echo virtual +fi From abc6d009b21b0d1ce3fc5107201e30740f127200 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 19:29:41 +0200 Subject: [PATCH 117/128] [explorer/machine_type] Print top most machine layer as first line (fallback to physical) --- cdist/conf/explorer/machine_type | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 29f98849..90c441da 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -26,17 +26,19 @@ # VERB does not have a special meaning, it is just for better readability. # # e.g. +# container # container on lxc # virtual by kvm-spapr # -# The third word of each line can be composed of different parts concatenated with a `-' -# (minus) character, with each component being a specification of the previous, -# e.g.: +# The third word of each line (except the first) can be composed of different +# parts concatenated with a `-' (minus) character, with each component being +# a specification of the previous, e.g.: # - lxc-libvirt (LXC container, managed by libvirt) # - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) # - xen-hvm / xen-pv (Xen HVM vs para-virtualization) # -# If this explorer cannot determine any information it will print nothing. +# If this explorer cannot collect enough information about virtualization it +# will fall back to 'physical'. # # Add /sbin and /usr/sbin to the path so we can find system @@ -121,6 +123,10 @@ get_sysctl() { is_command sysctl && sysctl -n "$1" 2>/dev/null } +detected_layer() { + test -n "${_toplayer:-}" || echo "${_toplayer:=${1:?}}" +} + # Check for container @@ -895,6 +901,7 @@ for stage in \ pid_1 cgroup files os_specific do ctengine=$(run_stage ct ${stage}) || continue + detected_layer 'container' is_contained=true if test -n "${ctengine}" then @@ -916,6 +923,7 @@ for stage in \ os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific do hypervisor=$(run_stage vm ${stage}) || continue + detected_layer 'virtual machine' is_virtual=true if test -n "${hypervisor}" then @@ -929,3 +937,8 @@ then # we are virtual. echo virtual fi + + +# Fallback + +detected_layer physical From 2ffa895f578f12d615ed65b9a93f3e2ae07f1d07 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 19:57:24 +0200 Subject: [PATCH 118/128] [explorer/machine_type] Remove CPUID check it's a lot of code and depends on a binary helper unlikely to be installed. --- cdist/conf/explorer/machine_type | 59 -------------------------------- 1 file changed, 59 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 90c441da..7ce035e3 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -379,65 +379,6 @@ check_vm_arch_specific() { fi ;; (i?86|x86*|amd64|i86pc) - # Check CPUID - # - # Many fullvirt hypervisors give an indication through CPUID. Use - # the virt-what helper program to get this information if available. - - for CPUID_HELPER in \ - $(command -v virt-what-cpuid-helper 2>/dev/null) \ - /usr/lib/x86_64-*/virt-what-cpuid-helper \ - /usr/lib/i?86-*/virt-what-cpuid-helper \ - /usr/lib/virt-what/virt-what-cpuid-helper - do - if test -x "${CPUID_HELPER:?}"; then break; fi - done - - if test -x "${CPUID_HELPER-}" - then - case $(command "${CPUID_HELPER}") - in - ('bhyve bhyve ') - echo bhyve - ;; - ('LKVMLKVMLKVM') - echo lkvm - ;; - ('KVMKVMKVM') - echo kvm - ;; - ('TCGTCGTCGTCG') - echo qemu-tcg - ;; - ('Microsoft Hv') - # http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx - echo hyperv - ;; - ('OpenBSDVMM58') - # OpenBSD/VMM - echo openbsd_vmm - ;; - ('VMwareVMware') - # check added by Chetan Loke. - echo vmware - ;; - ('XenVMMXenVMM') - if has dmi - then - # https://access.redhat.com/solutions/222903 - echo xen-hvm - else - echo xen-paravirt - fi - ;; - (*) - return 1 ;; - esac - return 0 - fi - - unset CPUID_HELPER - # VMM CPUID flag denotes that this system is running under a VMM if is_oneof "${uname_s}" Darwin then From 23fbfaf0352eadaabd43504d3ee074bb9f696fcd Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 21:29:24 +0200 Subject: [PATCH 119/128] [explorer/machine_type] Use systemd-detect-virt (if available) to detect containers and VMs --- cdist/conf/explorer/machine_type | 49 ++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 7ce035e3..1a38fda0 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -130,6 +130,25 @@ detected_layer() { # Check for container +has_ct_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -c' +} + +check_ct_systemd() ( + _ctengine=$(systemd-detect-virt -c 2>/dev/null) && + case ${_ctengine} + in + (''|'none') + return 1 ;; + ('container-other') + return 0 ;; + ('systemd-nspawn') + echo systemd_nspawn ;; + (*) + echo "${_ctengine}" ;; + esac +) + has_ct_pid_1() { test -r /run/systemd/container -o -r /proc/1/environ } @@ -267,6 +286,32 @@ guess_hypervisor_from_cpu_model() { esac } +has_vm_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -v' +} + +check_vm_systemd() ( + _hypervisor=$(systemd-detect-virt -v 2>/dev/null) && + case ${_hypervisor} + in + (''|'none') + return 1 ;; + ('amazon') + echo aws ;; + ('bochs') + echo kvm ;; + ('microsoft') + # assumption + echo hyperv ;; + ('oracle') + echo virtualbox ;; + ('vm-other') + return 0 ;; + (*) + echo "${_hypervisor}" ;; + esac +) + has_vm_cpuinfo() { has_cpuinfo; } check_vm_cpuinfo() { @@ -839,7 +884,7 @@ run_stage() { # Execute container stages for stage in \ - pid_1 cgroup files os_specific + systemd pid_1 cgroup files os_specific do ctengine=$(run_stage ct ${stage}) || continue detected_layer 'container' @@ -861,7 +906,7 @@ fi # Execute virtual machine / hypervisor stages for stage in \ - os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific + systemd os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific do hypervisor=$(run_stage vm ${stage}) || continue detected_layer 'virtual machine' From 4a05669765c2c00c19af0ef1b607b0f2efa10f42 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 22:01:28 +0200 Subject: [PATCH 120/128] [explorer/machine_type] Implement chroot detection --- cdist/conf/explorer/machine_type | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 1a38fda0..10c914ba 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -53,6 +53,21 @@ uname_s=$(uname -s) is_command() { command -v "$1" >/dev/null 2>&1; } +files_same() { + # shellcheck disable=SC2012 + LC_ALL=C df -P "$1" "$2" 2>/dev/null | { + read -r _ # skip header line + read -r fs1 _ _ _ _ mp1 + read -r fs2 _ _ _ _ mp2 + test "${fs1}" = "${fs2}" -a "${mp1}" = "${mp2}" || return 1 + } && + ls -1Ldi "$1" "$2" 2>/dev/null | { + read -r ino1 _ + read -r ino2 _ + test "${ino1}" = "${ino2}" || return 1 + } +} + is_oneof() ( x=$1; shift for y @@ -128,6 +143,32 @@ detected_layer() { } +# Check for chroot + +has_chroot_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -r' +} + +check_chroot_systemd() { + systemd-detect-virt -r +} + +has_chroot_debian_ischroot() { + is_command ischroot +} + +check_chroot_debian_ischroot() { + ischroot --default-false +} + +has_chroot_procfs() { + test -d /proc/ +} + +check_chroot_procfs() { + test -e /proc/1/root && ! files_same /proc/1/root / +} + # Check for container has_ct_systemd() { @@ -881,6 +922,18 @@ run_stage() { } +# Execute chroot stages + +for stage in \ + procfs debian_ischroot systemd +do + run_stage chroot ${stage} || continue + detected_layer 'chroot' + echo chroot + break +done + + # Execute container stages for stage in \ From 5af1317c2969995871da1d17b951de60dbc3fd09 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 1 Aug 2021 16:40:20 +0200 Subject: [PATCH 121/128] [explorer/machine_type] Try to detect chroot path --- cdist/conf/explorer/machine_type | 38 ++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 10c914ba..fa68ec4d 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -166,7 +166,28 @@ has_chroot_procfs() { } check_chroot_procfs() { - test -e /proc/1/root && ! files_same /proc/1/root / + if test -e /proc/1/root && ! files_same /proc/1/root / + then + # try to determine where the chroot has been mounted + ( + rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') + + if test -e "${rootdev}" + then + # escape chroot to determine where the device containing the + # chroot's / is mounted + rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') + + # shellcheck disable=SC2012 + root_ino=$(ls -1di / | awk '{print $1}') + + # Get mount point + chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" + fi + ) + return 0 + fi + return 1 } # Check for container @@ -927,11 +948,20 @@ run_stage() { for stage in \ procfs debian_ischroot systemd do - run_stage chroot ${stage} || continue + chrootpnt=$(run_stage chroot ${stage}) || continue + is_chrooted=true detected_layer 'chroot' - echo chroot - break + if test -n "${chrootpnt}" + then + echo chroot at "${chrootpnt}" + break + fi done +if ${is_chrooted:-false} && test -z "${chrootpnt}" +then + # could determine chroot, but not its mount point + echo chroot +fi # Execute container stages From 05c2a62191f533ceff1e074df571278a103d75ab Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 1 Aug 2021 23:09:02 +0200 Subject: [PATCH 122/128] [explorer/machine_type] Implement chroot detection using /proc/.../mountinfo --- cdist/conf/explorer/machine_type | 64 +++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index fa68ec4d..00646c75 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -165,30 +165,52 @@ has_chroot_procfs() { test -d /proc/ } -check_chroot_procfs() { +check_chroot_procfs() ( + is_chroot=false # default if test -e /proc/1/root && ! files_same /proc/1/root / then - # try to determine where the chroot has been mounted - ( - rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') - - if test -e "${rootdev}" - then - # escape chroot to determine where the device containing the - # chroot's / is mounted - rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') - - # shellcheck disable=SC2012 - root_ino=$(ls -1di / | awk '{print $1}') - - # Get mount point - chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" - fi - ) - return 0 + is_chroot=true fi - return 1 -} + if test -e /proc/1/mountinfo -a -e /proc/self/mountinfo + then + has_mountinfo=true + cmp -s /proc/1/mountinfo /proc/self/mountinfo || is_chroot=true + fi + + if ${is_chroot} + then + # try to determine where the chroot has been mounted + rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') + + if test -e "${rootdev}" + then + # escape chroot to determine where the device containing the + # chroot's / is mounted + rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') + + # shellcheck disable=SC2012 + root_ino=$(ls -1di / | awk '{print $1}') + + # escape chroot and find mount point by inode + chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" + elif ${has_mountinfo} + then + while read -r mntid _ _ _ cmntpnt _ + do + read -r _ _ _ _ hmntpnt _ <<-EOF + $(grep -e "^$((mntid)) " /proc/1/mountinfo) + EOF + printf '%s\n' "${hmntpnt%${cmntpnt}}" + done Date: Mon, 23 Aug 2021 09:57:20 +0300 Subject: [PATCH 123/128] [explorer/os_version] add new debian code names: bookworm and trixie --- cdist/conf/explorer/os_version | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 7bc6dd6b..aea3c43f 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -54,6 +54,8 @@ in # sid versions don't have a number, so we decode by codename: case $(expr "$debian_version" : '\([a-z]\{1,\}\)/') in + trixie) echo 12.99 ;; + bookworm) echo 11.99 ;; bullseye) echo 10.99 ;; buster) echo 9.99 ;; stretch) echo 8.99 ;; From e1e134899811e51e4694e89e421790e41cb894bf Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Mon, 23 Aug 2021 10:44:48 +0300 Subject: [PATCH 124/128] [explorer/os_version] use 99.99 as fallback for unknown code names in */sid --- cdist/conf/explorer/os_version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index aea3c43f..bbc9e4f0 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -63,7 +63,7 @@ in wheezy) echo 6.99 ;; squeeze) echo 5.99 ;; lenny) echo 4.99 ;; - *) exit 1 + *) echo 99.99 ;; esac ;; *) From 46ed48d546af6ff663b131c803a9027b19798ba1 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 08:09:47 +0200 Subject: [PATCH 125/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index f9409d7e..8507c663 100644 --- a/docs/changelog +++ b/docs/changelog @@ -12,6 +12,7 @@ next: * Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera) * Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera) * Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera) + * Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 0546283d0ebc0058dbff68cec9d18b9281c26edd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:32:44 +0200 Subject: [PATCH 126/128] Update shellcheck disable --- cdist/conf/type/__apt_backports/manifest | 1 + cdist/conf/type/__package_pkg_freebsd/gencode-remote | 1 + cdist/conf/type/__ssh_authorized_keys/explorer/keys | 1 + 3 files changed, 3 insertions(+) diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index bc47d8de..6fcd9212 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -28,6 +28,7 @@ # lsb_release may not be given in all installations codename_os_release() { # shellcheck disable=SC1090 + # shellcheck disable=SC1091 . "$__global/explorer/os_release" printf "%s" "$VERSION_CODENAME" } diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote index 3f88f6bc..ca9aa45a 100755 --- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote @@ -37,6 +37,7 @@ assert () # If condition false, then echo "Assertion failed: \"$1\"" # shellcheck disable=SC2039 + # shellcheck disable=SC3044 echo "File \"$0\", line $lineno, called by $(caller 0)" exit $E_ASSERT_FAILED fi diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys index cec25746..9694a64b 100755 --- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys +++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys @@ -1,6 +1,7 @@ #!/bin/sh -e # shellcheck disable=SC1090 +# shellcheck disable=SC1091 file="$( . "$__type_explorer/file" )" if [ -f "$file" ] From 44741e714b16f7a00bf84bd54ae33eacc7593192 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:25:49 +0200 Subject: [PATCH 127/128] Release 6.9.8 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 8507c663..dcdc4b3d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) From b8eb6e984c1638e8e167394c6b3aa482cb8aad49 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:47:50 +0200 Subject: [PATCH 128/128] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index dcdc4b3d..693d028f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Explorer machine_type: Rewrite (Dennis Camera) + 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali)