From c51d68a7375915154a90c22e716e832f635ca878 Mon Sep 17 00:00:00 2001 From: Beni Ruef Date: Thu, 3 Dec 2020 18:48:04 +0100 Subject: [PATCH 01/64] [type/__postgres_conf] New type based on ALTER SYSTEM command --- .../conf/type/__postgres_conf/gencode-remote | 92 +++++++++++++++++++ cdist/conf/type/__postgres_conf/man.rst | 55 +++++++++++ .../__postgres_conf/parameter/default/state | 1 + .../type/__postgres_conf/parameter/optional | 2 + 4 files changed, 150 insertions(+) create mode 100755 cdist/conf/type/__postgres_conf/gencode-remote create mode 100644 cdist/conf/type/__postgres_conf/man.rst create mode 100644 cdist/conf/type/__postgres_conf/parameter/default/state create mode 100644 cdist/conf/type/__postgres_conf/parameter/optional diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote new file mode 100755 index 00000000..a09e1873 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -0,0 +1,92 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$(cat "${__global}/explorer/os") +state_should=$(cat "${__object}/parameter/state") + +if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ] +then + echo "Invalid state '${state_should}'." \ + 'Only "present" and "absent" are acceptable' >&2 + exit 1 +fi + +if [ "${state_should}" = 'present' ] && [ ! -f "${__object}/parameter/value" ] +then + echo 'Missing required parameter "value"' >&2 + exit 1 +fi + +# Parameters +conf_name="${__object_id}" +if [ -f "${__object}/parameter/value" ] +then + conf_value=$(cat "${__object}/parameter/value") +fi + +if [ "${state_should}" = 'present' ] +then + set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'" + check_command="SHOW ${conf_name}" +else + set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT" +fi + +case $os +in + openbsd|devuan) + case $os + in + openbsd) + postgres_user='_postgresql' + restart_command='/etc/rc.d/postgresql restart' + ;; + devuan) + postgres_user='postgres' + restart_command='/etc/init.d/postgresql restart' + ;; + esac + # needs two separate psql commands because ALTER SYSTEM + # cannot run inside a transaction block + cat <<-EOF + su - ${postgres_user} -c "psql postgres -twAc \ + \"${set_command}\"" + su - ${postgres_user} -c "psql postgres -twAc \ + \"SELECT pg_reload_conf()\"" + EOF + # check success (makes only sense if setting to a non-default value) + # and restart server if needed + if [ "${state_should}" = 'present' ] + then + cat <<-EOF + if [ \$(su - ${postgres_user} -c "psql postgres -twAc \"SHOW ${conf_name}\"") != '${conf_value}' ] + then + ${restart_command} + fi + EOF + fi + ;; + *) + echo "Unsupported OS: ${os}" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst new file mode 100644 index 00000000..76016c6c --- /dev/null +++ b/cdist/conf/type/__postgres_conf/man.rst @@ -0,0 +1,55 @@ +cdist-type__postgres_conf(7) +============================ + +Configure a PostgreSQL server. + +NOTE: This type might need to be run multiple times to apply all bits of the +configuration due to ordering requirements. + +SSRQ + + +DESCRIPTION +----------- +Configure a PostgreSQL server using ALTER SYSTEM. + + +REQUIRED PARAMETERS +------------------- +value + The value to setup (can be omitted when state is set to "absent"). + + +OPTIONAL PARAMETERS +------------------- +state + "present" or "absent". Defaults to "present". + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # set timezone + __postgres_conf timezone --value Europe/Zurich + + # reset maximum number of concurrent connections to default (normally 100) + __postgres_conf max_connections --state absent + + +SEE ALSO +-------- +- `cdist-type(7) `_ + + +COPYING +------- +Copyright \(C) 2020 SSRQ (www.ssrq-sds-fds.ch). +Free use of this software is granted under the terms +of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__postgres_conf/parameter/default/state b/cdist/conf/type/__postgres_conf/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__postgres_conf/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__postgres_conf/parameter/optional b/cdist/conf/type/__postgres_conf/parameter/optional new file mode 100644 index 00000000..d0460d86 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/parameter/optional @@ -0,0 +1,2 @@ +state +value From 534d5f6bb5dd3d8e9cb2256e75ed182d6530a892 Mon Sep 17 00:00:00 2001 From: Beni Ruef Date: Fri, 4 Dec 2020 14:31:04 +0100 Subject: [PATCH 02/64] [type/__postgres_conf] Fix errors found by ShellCheck --- cdist/conf/type/__postgres_conf/gencode-remote | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index a09e1873..e25514d0 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -20,8 +20,8 @@ # along with cdist. If not, see . # -os=$(cat "${__global}/explorer/os") -state_should=$(cat "${__object}/parameter/state") +os=$(cat "${__global:?}/explorer/os") +state_should=$(cat "${__object:?}/parameter/state") if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ] then @@ -37,7 +37,7 @@ then fi # Parameters -conf_name="${__object_id}" +conf_name="${__object_id:?}" if [ -f "${__object}/parameter/value" ] then conf_value=$(cat "${__object}/parameter/value") @@ -46,7 +46,6 @@ fi if [ "${state_should}" = 'present' ] then set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'" - check_command="SHOW ${conf_name}" else set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT" fi From 50bcd951055d7599ec340b4dea80a3a5d11dd9b0 Mon Sep 17 00:00:00 2001 From: Beni Ruef Date: Thu, 10 Dec 2020 11:45:32 +0100 Subject: [PATCH 03/64] [type/__postgres_conf] Remove faulty quotes --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index e25514d0..8ccb3b42 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -45,7 +45,7 @@ fi if [ "${state_should}" = 'present' ] then - set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'" + set_command="ALTER SYSTEM SET ${conf_name} = ${conf_value}" else set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT" fi From b4060720dc3c7cc58aa992285877e4c009caa6be Mon Sep 17 00:00:00 2001 From: Beni Ruef Date: Fri, 11 Dec 2020 11:12:16 +0100 Subject: [PATCH 04/64] [type/__postgres_conf] Fix psql options for ALTER command --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 8ccb3b42..7d86028e 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -67,7 +67,7 @@ in # needs two separate psql commands because ALTER SYSTEM # cannot run inside a transaction block cat <<-EOF - su - ${postgres_user} -c "psql postgres -twAc \ + su - ${postgres_user} -c "psql postgres -qwc \ \"${set_command}\"" su - ${postgres_user} -c "psql postgres -twAc \ \"SELECT pg_reload_conf()\"" From 1b49fec972ce5f486f2794571c8e892d1fa6cb51 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 18 Jan 2021 19:15:49 +0100 Subject: [PATCH 05/64] [type/__postgres_conf] Refactor --- .../type/__postgres_conf/explorer/context | 42 ++++++ .../conf/type/__postgres_conf/explorer/state | 60 ++++++++ .../conf/type/__postgres_conf/gencode-remote | 135 ++++++++++-------- 3 files changed, 180 insertions(+), 57 deletions(-) create mode 100644 cdist/conf/type/__postgres_conf/explorer/context create mode 100644 cdist/conf/type/__postgres_conf/explorer/state diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context new file mode 100644 index 00000000..3a2fa504 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/explorer/context @@ -0,0 +1,42 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Returns the "context" of the configuration setting. +# cf. also https://www.postgresql.org/docs/10/view-pg-settings.html + +os=$("${__explorer:?}/os") + +case ${os} +in + (openbsd) + postgres_user='_postgresql' + ;; + (devuan) + postgres_user='postgres' + ;; + (*) + echo "Unsupported OS: ${os}" >&2 + exit 1 + ;; +esac + +conf_name=${__object_id:?} + +su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE name = '${conf_name}'\"" diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state new file mode 100644 index 00000000..da904b56 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -0,0 +1,60 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$("${__explorer:?}/os") + +case ${os} +in + (openbsd) + postgres_user='_postgresql' + ;; + (devuan) + postgres_user='postgres' + ;; + (*) + echo "Unsupported OS: ${os}" >&2 + exit 1 + ;; +esac + +conf_name=${__object_id:?} + +if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ + | cmp -s "${__object:?}/parameter/value" - +then + echo present +else + case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT source FROM pg_settings WHERE name = '${conf_name}'\"") + in + ('') + # invalid configuration parameter + # (error message was already printed by SHOW command above. + # Yes, it's a hack) + exit 1 + ;; + (default) + echo absent + ;; + (*) + echo different + ;; + esac +fi diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 7d86028e..998b8582 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -1,7 +1,7 @@ #!/bin/sh -e # -*- mode: sh; indent-tabs-mode: t -*- # -# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# 2019-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch) # # This file is part of cdist. @@ -21,71 +21,92 @@ # os=$(cat "${__global:?}/explorer/os") +state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") -if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ] +conf_name=${__object_id:?} + +if test "${state_is}" = "${state_should}" then - echo "Invalid state '${state_should}'." \ - 'Only "present" and "absent" are acceptable' >&2 - exit 1 + exit 0 fi -if [ "${state_should}" = 'present' ] && [ ! -f "${__object}/parameter/value" ] -then - echo 'Missing required parameter "value"' >&2 - exit 1 -fi - -# Parameters -conf_name="${__object_id:?}" -if [ -f "${__object}/parameter/value" ] -then - conf_value=$(cat "${__object}/parameter/value") -fi - -if [ "${state_should}" = 'present' ] -then - set_command="ALTER SYSTEM SET ${conf_name} = ${conf_value}" -else - set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT" -fi - -case $os -in - openbsd|devuan) - case $os - in - openbsd) - postgres_user='_postgresql' - restart_command='/etc/rc.d/postgresql restart' - ;; - devuan) - postgres_user='postgres' - restart_command='/etc/init.d/postgresql restart' - ;; - esac - # needs two separate psql commands because ALTER SYSTEM - # cannot run inside a transaction block - cat <<-EOF - su - ${postgres_user} -c "psql postgres -qwc \ - \"${set_command}\"" - su - ${postgres_user} -c "psql postgres -twAc \ - \"SELECT pg_reload_conf()\"" - EOF - # check success (makes only sense if setting to a non-default value) - # and restart server if needed - if [ "${state_should}" = 'present' ] +quote() { + for _arg + do + shift + if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')" then - cat <<-EOF - if [ \$(su - ${postgres_user} -c "psql postgres -twAc \"SHOW ${conf_name}\"") != '${conf_value}' ] - then - ${restart_command} - fi - EOF + # needs quoting + set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'" + else + set -- "$@" "${_arg}" fi + done + unset _arg + + # NOTE: Use printf because POSIX echo interprets escape sequences + printf '%s' "$*" +} + +case ${os} +in + (openbsd) + postgres_user='_postgresql' + restart_command='/etc/rc.d/postgresql restart' ;; - *) + (devuan) + postgres_user='postgres' + restart_command='/etc/init.d/postgresql restart' + ;; + (*) echo "Unsupported OS: ${os}" >&2 exit 1 ;; esac + + +psql_cmd() { + printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")" +} + +case ${state_should} +in + (present) + test -s "${__object:?}/parameter/value" || { + echo 'Missing required parameter --value' >&2 + exit 1 + } + + cat <<-EOF + exec 3< "\${__object:?}/parameter/value" + $(psql_cmd postgres -tAw) <<'SQL' + \\set conf_value \`cat <&3\` + ALTER SYSTEM SET ${conf_name} = :'conf_value'; + SELECT pg_reload_conf(); + SQL + exec 3<&- + EOF + ;; + (absent) + psql_cmd postgres -qwc "ALTER SYSTEM SET ${conf_name} TO DEFAULT" + ;; + (*) + printf 'Invalid --state: %s\n' "${state_should}" >&2 + printf 'Only "present" and "absent" are acceptable.\n' >&2 + exit 1 + ;; +esac + +# check success (makes only sense if setting to a non-default value) +# and restart server if needed +if test "${state_should}" = 'present' +then + cat <<-EOF + + $(psql_cmd postgres -twAc "SHOW ${conf_name}") \\ + | cmp -s "\${__object:?}/parameter/value" - || { + ${restart_command} + } + EOF +fi From 803367b316f5c72c51153172a41b7a745f2d2d83 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 18 Jan 2021 19:29:42 +0100 Subject: [PATCH 06/64] [type/__postgres_conf] Fix default detection when default is also set in config file e.g. port is usually also set to the default value in postgresql.conf --- cdist/conf/type/__postgres_conf/explorer/state | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index da904b56..400b27e8 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -42,7 +42,7 @@ if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ then echo present else - case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT source FROM pg_settings WHERE name = '${conf_name}'\"") + case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE name = '${conf_name}'\"") in ('') # invalid configuration parameter From 891c98567e1d94577dc4f20d3ec9f73f4927fd24 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 18 Jan 2021 19:39:56 +0100 Subject: [PATCH 07/64] [type/__postgres_conf] Compare configuration parameter names case insensitively --- cdist/conf/type/__postgres_conf/explorer/context | 3 ++- cdist/conf/type/__postgres_conf/explorer/state | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context index 3a2fa504..def5f676 100644 --- a/cdist/conf/type/__postgres_conf/explorer/context +++ b/cdist/conf/type/__postgres_conf/explorer/context @@ -39,4 +39,5 @@ esac conf_name=${__object_id:?} -su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE name = '${conf_name}'\"" +# NOTE: SHOW/SET are case-insentitive, so this command should also be. +su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"" diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index 400b27e8..a4930296 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -42,7 +42,8 @@ if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ then echo present else - case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE name = '${conf_name}'\"") + # NOTE: SHOW/SET are case-insentitive, so this command should also be. + case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"") in ('') # invalid configuration parameter From 5051d4f40b7097f58431ce334f9ddf5b171dbc7f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 19 Jan 2021 16:36:44 +0100 Subject: [PATCH 08/64] [type/__postgres_conf] Catch invalid values --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 998b8582..cc9b4e99 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -80,7 +80,7 @@ in cat <<-EOF exec 3< "\${__object:?}/parameter/value" - $(psql_cmd postgres -tAw) <<'SQL' + $(psql_cmd postgres -tAw -v ON_ERROR_STOP=on) <<'SQL' \\set conf_value \`cat <&3\` ALTER SYSTEM SET ${conf_name} = :'conf_value'; SELECT pg_reload_conf(); From 0f2ff477381d67de27149a92744f873322e44fd7 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 19 Jan 2021 16:37:43 +0100 Subject: [PATCH 09/64] [type/__postgres_conf] Restart PostgreSQL server based on pending_restart column of pg_settings --- .../type/__postgres_conf/explorer/context | 43 ------------------- .../conf/type/__postgres_conf/gencode-remote | 16 +++---- 2 files changed, 6 insertions(+), 53 deletions(-) delete mode 100644 cdist/conf/type/__postgres_conf/explorer/context diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context deleted file mode 100644 index def5f676..00000000 --- a/cdist/conf/type/__postgres_conf/explorer/context +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -e -# -*- mode: sh; indent-tabs-mode: t -*- -# -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# Returns the "context" of the configuration setting. -# cf. also https://www.postgresql.org/docs/10/view-pg-settings.html - -os=$("${__explorer:?}/os") - -case ${os} -in - (openbsd) - postgres_user='_postgresql' - ;; - (devuan) - postgres_user='postgres' - ;; - (*) - echo "Unsupported OS: ${os}" >&2 - exit 1 - ;; -esac - -conf_name=${__object_id:?} - -# NOTE: SHOW/SET are case-insentitive, so this command should also be. -su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"" diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index cc9b4e99..fa930cc4 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -98,15 +98,11 @@ in ;; esac -# check success (makes only sense if setting to a non-default value) -# and restart server if needed -if test "${state_should}" = 'present' -then - cat <<-EOF +# Restart PostgreSQL server if required to apply new configuration value +cat < Date: Tue, 19 Jan 2021 17:18:27 +0100 Subject: [PATCH 10/64] [type/__postgres_conf] Add support for more init systems to restart service --- .../conf/type/__postgres_conf/gencode-remote | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index fa930cc4..15d5bb3e 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -53,11 +53,9 @@ case ${os} in (openbsd) postgres_user='_postgresql' - restart_command='/etc/rc.d/postgresql restart' ;; (devuan) postgres_user='postgres' - restart_command='/etc/init.d/postgresql restart' ;; (*) echo "Unsupported OS: ${os}" >&2 @@ -103,6 +101,37 @@ cat <&2 + exit 1 + esac + ;; + (*) + printf "Don't know how to restart services with your init (%s)\n" "${init}" >&2 + exit 1 + esac + ) fi EOF From 4967c7ebbb7e98af6aa2c7f8bc91511bcbbf9820 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 19 Jan 2021 17:20:27 +0100 Subject: [PATCH 11/64] [type/__postgres_conf] Silence psql output --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 15d5bb3e..9b8227ee 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -78,7 +78,7 @@ in cat <<-EOF exec 3< "\${__object:?}/parameter/value" - $(psql_cmd postgres -tAw -v ON_ERROR_STOP=on) <<'SQL' + $(psql_cmd postgres -tAwq -o /dev/null -v ON_ERROR_STOP=on) <<'SQL' \\set conf_value \`cat <&3\` ALTER SYSTEM SET ${conf_name} = :'conf_value'; SELECT pg_reload_conf(); From f9ebb4333c85e41439635cb00a40c3e07d8a3683 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 19 Jan 2021 17:21:50 +0100 Subject: [PATCH 12/64] [type/__postgres_conf] Add NetBSD PostgreSQL UNIX user --- cdist/conf/type/__postgres_conf/gencode-remote | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 9b8227ee..8f8a2bfe 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -51,15 +51,14 @@ quote() { case ${os} in + (netbsd) + postgres_user='pgsql' + ;; (openbsd) postgres_user='_postgresql' ;; - (devuan) - postgres_user='postgres' - ;; (*) - echo "Unsupported OS: ${os}" >&2 - exit 1 + postgres_user='postgres' ;; esac From 6b18cace759d1345cec6c2168c3bd8f238f23bca Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 21 Jan 2021 19:29:07 +0100 Subject: [PATCH 13/64] [type/__postgres_conf] Catch connection errors early --- cdist/conf/type/__postgres_conf/explorer/state | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index a4930296..589925de 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -37,6 +37,11 @@ esac conf_name=${__object_id:?} +su - "${postgres_user}" -c 'psql postgres -c "SELECT 1"' >/dev/null || { + echo 'Connection to PostgreSQL server failed' >&2 + exit 1 +} + if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ | cmp -s "${__object:?}/parameter/value" - then From 0835f414a5b2ce41309ee9b265cb80abb1362563 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 16 Feb 2021 16:03:23 +0100 Subject: [PATCH 14/64] [type/__postgres_conf] Extract PostgreSQL service user detection to separate explorer --- .../__postgres_conf/explorer/postgres_user | 64 +++++++++++++++++++ .../conf/type/__postgres_conf/explorer/state | 16 +---- .../conf/type/__postgres_conf/gencode-remote | 17 +---- 3 files changed, 67 insertions(+), 30 deletions(-) create mode 100644 cdist/conf/type/__postgres_conf/explorer/postgres_user diff --git a/cdist/conf/type/__postgres_conf/explorer/postgres_user b/cdist/conf/type/__postgres_conf/explorer/postgres_user new file mode 100644 index 00000000..c6582dc4 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/explorer/postgres_user @@ -0,0 +1,64 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$("${__explorer:?}/os") + +case ${os} +in + (alpine) + echo 'postgres' + ;; + (centos|rhel|scientific) + echo 'postgres' + ;; + (debian|devuan|ubuntu) + echo 'postgres' + ;; + (freebsd) + test -x /usr/local/etc/rc.d/postgresql || { + printf 'could not find postgresql rc script./n' >&2 + exit 1 + } + pg_status=$(/usr/local/etc/rc.d/postgresql onestatus) || { + printf 'postgresql daemon is not running.\n' >&2 + exit 1 + } + pg_pid=$(printf '%s\n' "${pg_status}" \ + | sed -n 's/^pg_ctl:.*(PID: *\([0-9]*\))$/\1/p') + + # PostgreSQL < 9.6: pgsql + # PostgreSQL >= 9.6: postgres + ps -o user -p "${pg_pid}" | sed -n '2p' + ;; + (netbsd) + echo 'pgsql' + ;; + (openbsd) + echo '_postgresql' + ;; + (suse) + echo 'postgres' + ;; + (*) + echo "Unsupported OS: ${os}" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index 589925de..de6e8aa0 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -19,21 +19,7 @@ # along with cdist. If not, see . # -os=$("${__explorer:?}/os") - -case ${os} -in - (openbsd) - postgres_user='_postgresql' - ;; - (devuan) - postgres_user='postgres' - ;; - (*) - echo "Unsupported OS: ${os}" >&2 - exit 1 - ;; -esac +postgres_user=$("${__type_explorer:?}/postgres_user") conf_name=${__object_id:?} diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 8f8a2bfe..d0d247b4 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -20,9 +20,9 @@ # along with cdist. If not, see . # -os=$(cat "${__global:?}/explorer/os") state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") +postgres_user=$(cat "${__object:?}/explorer/postgres_user") conf_name=${__object_id:?} @@ -49,19 +49,6 @@ quote() { printf '%s' "$*" } -case ${os} -in - (netbsd) - postgres_user='pgsql' - ;; - (openbsd) - postgres_user='_postgresql' - ;; - (*) - postgres_user='postgres' - ;; -esac - psql_cmd() { printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")" @@ -117,7 +104,7 @@ then case $(cat "${__global:?}/explorer/kernel_name") in (FreeBSD) - echo 'service postgresql restart' + echo '/usr/local/etc/rc.d/postgresql restart' ;; (OpenBSD|NetBSD) echo '/etc/rc.d/postgresql restart' From 92b8942a8c37bb985ef42991f3b7aaef0cd8073f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 09:00:32 +0200 Subject: [PATCH 15/64] [type/__postgres_conf] Add psql_exec function to state explorer --- cdist/conf/type/__postgres_conf/explorer/state | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index de6e8aa0..1a58751a 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -20,21 +20,24 @@ # postgres_user=$("${__type_explorer:?}/postgres_user") - conf_name=${__object_id:?} -su - "${postgres_user}" -c 'psql postgres -c "SELECT 1"' >/dev/null || { +quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } +psql_exec() { + su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" +} + +psql_exec 'SELECT 1' >/dev/null || { echo 'Connection to PostgreSQL server failed' >&2 exit 1 } -if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ - | cmp -s "${__object:?}/parameter/value" - +if psql_exec "SHOW ${conf_name}" | cmp -s "${__object:?}/parameter/value" - then echo present else # NOTE: SHOW/SET are case-insentitive, so this command should also be. - case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"") + case $(psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')") in ('') # invalid configuration parameter From 2ccc03fef1e28c759a7918fa00ffcb45c76dc57a Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 09:02:40 +0200 Subject: [PATCH 16/64] [type/__postgres_conf] Add psql_conf_cmp function to state explorer --- cdist/conf/type/__postgres_conf/explorer/state | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index 1a58751a..be881be6 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -27,12 +27,16 @@ psql_exec() { su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" } +psql_conf_cmp() { + test "$(psql_exec "SHOW $1")" = "$2" +} + psql_exec 'SELECT 1' >/dev/null || { echo 'Connection to PostgreSQL server failed' >&2 exit 1 } -if psql_exec "SHOW ${conf_name}" | cmp -s "${__object:?}/parameter/value" - +if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" then echo present else From e0416403c4862cb8bc0f41a7c6f8a33d1a8656a5 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 09:04:07 +0200 Subject: [PATCH 17/64] [type/__postgres_conf] Add psql_conf_source function to state explorer --- cdist/conf/type/__postgres_conf/explorer/state | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index be881be6..9f54724a 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -27,6 +27,10 @@ psql_exec() { su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" } +psql_conf_source() { + # NOTE: SHOW/SET are case-insentitive, so this command should also be. + psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')" +} psql_conf_cmp() { test "$(psql_exec "SHOW $1")" = "$2" } @@ -40,8 +44,7 @@ if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" then echo present else - # NOTE: SHOW/SET are case-insentitive, so this command should also be. - case $(psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')") + case $(psql_conf_source "${conf_name}") in ('') # invalid configuration parameter From 12c2995494ce582cb3adf1d942b36f6df035370a Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 14:46:36 +0200 Subject: [PATCH 18/64] [type/__postgres_conf] Implement complex state compare logic --- .../conf/type/__postgres_conf/explorer/state | 166 +++++++++++++++++- 1 file changed, 163 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index 9f54724a..e76540c5 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -22,6 +22,52 @@ postgres_user=$("${__type_explorer:?}/postgres_user") conf_name=${__object_id:?} +tolower() { printf '%s' "$*" | tr '[:upper:]' '[:lower:]'; } + +tobytes() { + # NOTE: This function treats everything as base 2. + # It is not compatible with SI units. + awk 'BEGIN { FS = "\n" } + /TB$/ { $0 = ($0 * 1024) "GB" } + /GB$/ { $0 = ($0 * 1024) "MB" } + /MB$/ { $0 = ($0 * 1024) "kB" } + /kB$/ { $0 = ($0 * 1024) "B" } + /B?$/ { sub(/ *B?$/, "") } + ($0*1) == $0 # is number + ' <<-EOF + $1 + EOF +} + +tomillisecs() { + awk 'BEGIN { FS = "\n" } + /d$/ { $0 = ($0 * 24) "h" } + /h$/ { $0 = ($0 * 60) "min" } + /min$/ { $0 = ($0 * 60) "s" } + /[^m]s$/ { $0 = ($0 * 1000) "ms" } + /ms$/ { $0 *= 1 } + ($0*1) == $0 # is number + ' <<-EOF + $1 + EOF +} + +tobool() { + # prints either 'on' or 'off' + case $(tolower "$1") + in + (t|true|y|yes|on|1) + echo 'on' ;; + (f|false|n|no|off|0) + echo 'off' ;; + (*) + printf 'Inavlid bool value: %s\n' "$2" >&2 + return 1 + ;; + esac + return 0 +} + quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } psql_exec() { su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" @@ -31,9 +77,123 @@ psql_conf_source() { # NOTE: SHOW/SET are case-insentitive, so this command should also be. psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')" } -psql_conf_cmp() { - test "$(psql_exec "SHOW $1")" = "$2" -} +psql_conf_cmp() ( + IFS='|' read -r lower_name vartype setting unit <<-EOF + $(psql_exec "SELECT lower(name), vartype, setting, unit FROM pg_settings WHERE lower(name) = lower('$1')") + EOF + + should_value=$2 + is_value=${setting} + + # The following case contains special cases for special settings. + case ${lower_name} + in + (archive_command) + if test "${setting}" = '(disabled)' + then + # DAFUQ PostgreSQL?! + # PostgreSQL returns (disabled) if the feature is inactive. + # We cannot compare the values unless it is enabled, first. + return 0 + fi + ;; + (archive_mode|backslash_quote|constraint_exclusion|force_parallel_mode|huge_pages|synchronous_commit) + # Although only 'on', 'off' are documented, PostgreSQL accepts all + # the "likely" variants of "on" and "off". + case $(tolower "${should_value}") + in + (on|off|true|false|yes|no|1|0) + should_value=$(tobool "${should_value}") + ;; + esac + ;; + esac + + case ${vartype} + in + (bool) + test -z "${unit}" || { + # please fix the explorer if this error occurs. + printf 'units are not supported for vartype: %s\n' "${vartype}" >&2 + exit 1 + } + + should_value=$(tobool "${should_value}") + + test "${is_value}" = "${should_value}" + ;; + (enum) + test -z "${unit}" || { + # please fix the explorer if this error occurs. + printf 'units are not supported with vartype: %s\n' "${vartype}" >&2 + exit 1 + } + + # NOTE: All enums that are currently defined are lower case, but + # PostgreSQL also accepts upper case spelling. + should_value=$(tolower "$2") + + test "${is_value}" = "${should_value}" + ;; + (integer) + # split multiples from unit, first (e.g. 8kB -> 8, kB) + case ${unit} + in + ([0-9]*) + multiple=${unit%%[!0-9]*} + unit=${unit##*[0-9 ]} + ;; + (*) multiple=1 ;; + esac + + is_value=$((setting * multiple))${unit} + + if expr "${should_value}" : '-\{0,1\}[0-9]*$' >/dev/null + then + # default unit + should_value=$((should_value * multiple))${unit} + fi + + # then, do conversion + # NOTE: these conversions work for integers only! + case ${unit} + in + (B|[kMGT]B) + # bytes + is_bytes=$(tobytes "${is_value}") + should_bytes=$(tobytes "${should_value}") + + test $((is_bytes)) -eq $((should_bytes)) + ;; + (ms|s|min|h|d) + # seconds + is_ms=$(tomillisecs "${is_value}") + should_ms=$(tomillisecs "${should_value}") + + test $((is_ms)) -eq $((should_ms)) + ;; + ('') + # no unit + is_int=${is_value} + should_int=${should_value} + + test $((is_int)) -eq $((should_int)) + ;; + esac + ;; + (real|string) + # NOTE: reals could possibly have units, but currently there none. + + test -z "${unit}" || { + # please fix the explorer if this error occurs. + printf 'units are not supported with vartype: %s\n' "${vartype}" >&2 + exit 1 + } + + test "${is_value}" = "${should_value}" + ;; + esac +) psql_exec 'SELECT 1' >/dev/null || { echo 'Connection to PostgreSQL server failed' >&2 From bef1433ba3adb022f8b50eb9df0fbbbf90757d76 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 14:46:50 +0200 Subject: [PATCH 19/64] [type/__postgres_conf] Accept empty values --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index d0d247b4..27651600 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -57,7 +57,7 @@ psql_cmd() { case ${state_should} in (present) - test -s "${__object:?}/parameter/value" || { + test -n "${__object:?}/parameter/value" || { echo 'Missing required parameter --value' >&2 exit 1 } From 686e4f0f2d498ee39a69e43f7535171f16f5fcbc Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 15:01:38 +0200 Subject: [PATCH 20/64] [type/__postgres_conf] Reverse state logic (decide based on source first) --- .../conf/type/__postgres_conf/explorer/state | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index e76540c5..4b7b0a43 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -200,23 +200,24 @@ psql_exec 'SELECT 1' >/dev/null || { exit 1 } -if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" -then - echo present -else - case $(psql_conf_source "${conf_name}") - in - ('') - # invalid configuration parameter - # (error message was already printed by SHOW command above. - # Yes, it's a hack) - exit 1 - ;; - (default) - echo absent - ;; - (*) +case $(psql_conf_source "${conf_name}") +in + ('') + printf 'Invalid configuration parameter: %s\n' "${conf_name}" >&2 + exit 1 + ;; + (default) + echo absent + ;; + (*) + if ! test -f "${__object:?}/parameter/value" + then + echo present + elif psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" + then + echo present + else echo different - ;; - esac -fi + fi + ;; +esac From 19bf37be1a8b0b099a7199d1565b74598010f614 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 15:55:52 +0200 Subject: [PATCH 21/64] [type/__postgres_conf] Update man.rst --- cdist/conf/type/__postgres_conf/man.rst | 39 ++++++++++++++----------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst index 76016c6c..e035f080 100644 --- a/cdist/conf/type/__postgres_conf/man.rst +++ b/cdist/conf/type/__postgres_conf/man.rst @@ -1,29 +1,27 @@ cdist-type__postgres_conf(7) ============================ -Configure a PostgreSQL server. - -NOTE: This type might need to be run multiple times to apply all bits of the -configuration due to ordering requirements. - -SSRQ +NAME +---- +cdist-type__postgres_conf - Alter PostgreSQL configuration DESCRIPTION ----------- -Configure a PostgreSQL server using ALTER SYSTEM. +Configure a running PostgreSQL server using ``ALTER SYSTEM``. REQUIRED PARAMETERS ------------------- value - The value to setup (can be omitted when state is set to "absent"). + The value to set (can be omitted if ``--state`` is set to ``absent``). OPTIONAL PARAMETERS ------------------- state - "present" or "absent". Defaults to "present". + ``present`` or ``absent``. + Defaults to ``present``. BOOLEAN PARAMETERS @@ -36,20 +34,27 @@ EXAMPLES .. code-block:: sh - # set timezone - __postgres_conf timezone --value Europe/Zurich + # set timezone + __postgres_conf timezone --value Europe/Zurich - # reset maximum number of concurrent connections to default (normally 100) - __postgres_conf max_connections --state absent + # reset maximum number of concurrent connections to default (normally 100) + __postgres_conf max_connections --state absent SEE ALSO -------- -- `cdist-type(7) `_ +None. + + +AUTHORS +------- +Beni Ruef (bernhard.ruef--@--ssrq-sds-fds.ch) +Dennis Camera (dennis.camera--@--ssrq-sds-fds.ch) COPYING ------- -Copyright \(C) 2020 SSRQ (www.ssrq-sds-fds.ch). -Free use of this software is granted under the terms -of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2019-2021 SSRQ (www.ssrq-sds-fds.ch). +You can redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. From acf9bf91f17fb4ad0e5813c6e4b0b40fc7e027b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 22 Apr 2021 08:55:14 +0200 Subject: [PATCH 22/64] [scanner] error to stderr and exit when scapy is not available --- cdist/scan/commandline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index eca4cf13..0d7fb0ca 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -20,6 +20,7 @@ # import logging +import sys log = logging.getLogger("scan") @@ -31,7 +32,9 @@ def commandline(args): try: import cdist.scan.scan as scan except ModuleNotFoundError: - print('cdist scan requires scapy to be installed') + print('cdist scan requires scapy to be installed! Exiting.', + file=sys.stderr) + sys.exit(1) processes = [] From a4464209b6741f2c1764aa432c56cfa8128852a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 22 Apr 2021 09:29:53 +0200 Subject: [PATCH 23/64] [scanner] add minimal error handling, consolidate CLI args processing --- cdist/argparse.py | 10 ++++-- cdist/scan/commandline.py | 19 ++++++----- cdist/scan/scan.py | 71 +++++++++++---------------------------- 3 files changed, 37 insertions(+), 63 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index cadac39a..153e7864 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -492,12 +492,16 @@ def get_parsers(): help='Try to configure detected hosts') parser['scan'].add_argument( '-I', '--interfaces', - action='append', default=[], + action='append', default=[], required=True, help='On which interfaces to scan/trigger') parser['scan'].add_argument( '-d', '--delay', - action='store', default=3600, - help='How long to wait before reconfiguring after last try') + action='store', default=3600, type=int, + help='How long (seconds) to wait before reconfiguring after last try') + parser['scan'].add_argument( + '-t', '--trigger-delay', + action='store', default=5, type=int, + help='How long (seconds) to wait between ICMPv6 echo requests') parser['scan'].set_defaults(func=cdist.scan.commandline.commandline) for p in parser: diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 0d7fb0ca..dead5292 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -24,26 +24,29 @@ import sys log = logging.getLogger("scan") - -# define this outside of the class to not handle scapy import errors by default +# CLI processing is defined outside of the main scan class to handle +# non-available optional scapy dependency (instead of crashing mid-flight). def commandline(args): log.debug(args) + # Check if we have the optional scapy dependency available. try: import cdist.scan.scan as scan except ModuleNotFoundError: - print('cdist scan requires scapy to be installed! Exiting.', - file=sys.stderr) + log.error('cdist scan requires scapy to be installed. Exiting.') sys.exit(1) - processes = [] - + # Default operation mode. if not args.mode: - # By default scan and trigger, but do not call any action + # By default scan and trigger, but do not call any action. args.mode = ['scan', 'trigger', ] + # We run each component in a separate process since they + # must not block on each other. + processes = [] + if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interfaces) + t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay) t.start() processes.append(t) log.debug("Trigger started") diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index faee8a56..633a5c06 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -61,20 +61,22 @@ import datetime import cdist.config +logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") - class Trigger(object): """ Trigger an ICMPv6EchoReply from all hosts that are alive """ - def __init__(self, interfaces=None, verbose=False): + def __init__(self, interfaces, sleeptime, verbose=False): self.interfaces = interfaces + + # Used by scapy / send in trigger/2. self.verbose = verbose - # Wait 5 seconds before triggering again - FIXME: add parameter - self.sleeptime = 5 + # Delay in seconds between sent ICMPv6EchoRequests. + self.sleeptime = sleeptime def start(self): self.processes = [] @@ -93,9 +95,12 @@ class Trigger(object): time.sleep(self.sleeptime) def trigger(self, interface): - packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest() - log.debug("Sending request on %s", interface) - send(packet, verbose=self.verbose) + try: + log.debug("Sending ICMPv6EchoRequest on %s", interface) + packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest() + send(packet, verbose=self.verbose) + except Exception as e: + log.error( "Could not send ICMPv6EchoRequest: %s", e) class Scanner(object): @@ -103,7 +108,7 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces=None, args=None, outdir=None): + def __init__(self, interfaces, args=None, outdir=None): self.interfaces = interfaces if outdir: @@ -148,47 +153,9 @@ class Scanner(object): def scan(self): log.debug("Scanning - zzzzz") - sniff(iface=self.interfaces, - filter="icmp6", - prn=self.handle_pkg) - - -if __name__ == '__main__': - t = Trigger(interfaces=["wlan0"]) - t.start() - - # Scanner can listen on many interfaces at the same time - s = Scanner(interfaces=["wlan0"]) - s.scan() - - # Join back the trigger processes - t.join() - - # Test in my lan shows: - # [18:48] bridge:cdist% ls -1d fe80::* - # fe80::142d:f0a5:725b:1103 - # fe80::20d:b9ff:fe49:ac11 - # fe80::20d:b9ff:fe4c:547d - # fe80::219:d2ff:feb2:2e12 - # fe80::21b:fcff:feee:f446 - # fe80::21b:fcff:feee:f45c - # fe80::21b:fcff:feee:f4b1 - # fe80::21b:fcff:feee:f4ba - # fe80::21b:fcff:feee:f4bc - # fe80::21b:fcff:feee:f4c1 - # fe80::21d:72ff:fe86:46b - # fe80::42b0:34ff:fe6f:f6f0 - # fe80::42b0:34ff:fe6f:f863 - # fe80::42b0:34ff:fe6f:f9b2 - # fe80::4a5d:60ff:fea1:e55f - # fe80::77a3:5e3f:82cc:f2e5 - # fe80::9e93:4eff:fe6c:c1f4 - # fe80::ba69:f4ff:fec5:6041 - # fe80::ba69:f4ff:fec5:8db7 - # fe80::bad8:12ff:fe65:313d - # fe80::bad8:12ff:fe65:d9b1 - # fe80::ce2d:e0ff:fed4:2611 - # fe80::ce32:e5ff:fe79:7ea7 - # fe80::d66d:6dff:fe33:e00 - # fe80::e2ff:f7ff:fe00:20e6 - # fe80::f29f:c2ff:fe7c:275e + try: + sniff(iface=self.interfaces, + filter="icmp6", + prn=self.handle_pkg) + except Exception as e: + log.error( "Could not start listener: %s", e) From bb24d632d62da8390dd1a724fc325a57526da40d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 22 Apr 2021 10:20:49 +0200 Subject: [PATCH 24/64] [scanner] implement the --list flag --- cdist/argparse.py | 4 +++ cdist/scan/commandline.py | 63 ++++++++++++++++++++++++++++----------- cdist/scan/scan.py | 17 +++++++++++ 3 files changed, 67 insertions(+), 17 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 153e7864..f390a974 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -486,6 +486,10 @@ def get_parsers(): '-m', '--mode', help='Which modes should run', action='append', default=[], choices=['scan', 'trigger']) + parser['scan'].add_argument( + '--list', + action='store_true', + help='List the known hosts and exit') parser['scan'].add_argument( '--config', action='store_true', diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index dead5292..331694e4 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -21,26 +21,11 @@ import logging import sys +from datetime import datetime log = logging.getLogger("scan") -# CLI processing is defined outside of the main scan class to handle -# non-available optional scapy dependency (instead of crashing mid-flight). -def commandline(args): - log.debug(args) - - # Check if we have the optional scapy dependency available. - try: - import cdist.scan.scan as scan - except ModuleNotFoundError: - log.error('cdist scan requires scapy to be installed. Exiting.') - sys.exit(1) - - # Default operation mode. - if not args.mode: - # By default scan and trigger, but do not call any action. - args.mode = ['scan', 'trigger', ] - +def run(scan, args): # We run each component in a separate process since they # must not block on each other. processes = [] @@ -59,3 +44,47 @@ def commandline(args): for process in processes: process.join() + +def list(scan, args): + s = scan.Scanner(interfaces=args.interfaces, args=args) + hosts = s.list() + + # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + + # 6 colons. + ipv6_max_size = 8 * 4 + 10 + # We format dates as follow: YYYY-MM-DD HH:MM:SS + date_max_size = 8 + 2 + 6 + 2 + + print("{} | {}".format( + 'link-local address'.ljust(ipv6_max_size), + 'last seen'.ljust(date_max_size))) + print('=' * (ipv6_max_size + 3 + date_max_size)) + for addr in hosts: + last_seen = datetime.strftime( + datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'), + '%Y-%m-%d %H:%M:%S') + print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size))) + +# CLI processing is defined outside of the main scan class to handle +# non-available optional scapy dependency (instead of crashing mid-flight). +def commandline(args): + log.debug(args) + + # Check if we have the optional scapy dependency available. + try: + import cdist.scan.scan as scan + except ModuleNotFoundError: + log.error('cdist scan requires scapy to be installed. Exiting.') + sys.exit(1) + + # Set default operation mode. + if not args.mode: + # By default scan and trigger, but do not call any action. + args.mode = ['scan', 'trigger', ] + + # Print known hosts and exit is --list is specified - do not start + # the scanner. + if args.list: + list(scan, args) + else: + run(scan, args) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 633a5c06..f3976370 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -132,6 +132,23 @@ class Scanner(object): with open(fname, "w") as fd: fd.write(f"{now}\n") + def list(self): + hosts = dict() + for linklocal_addr in os.listdir(self.outdir): + workdir = os.path.join(self.outdir, linklocal_addr) + # We ignore any (unexpected) file in this directory. + if os.path.isdir(workdir): + last_seen='-' + last_seen_file = os.path.join(workdir, 'last_seen') + if os.path.isfile(last_seen_file): + with open(last_seen_file, "r") as fd: + last_seen = fd.readline() + + hosts[linklocal_addr] = {'last_seen': last_seen} + + return hosts + + def config(self): """ Configure a host From 13e2ad175f01b6cfbdb21ee50e85b6f3ba6fe750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 25 Apr 2021 12:45:34 +0200 Subject: [PATCH 25/64] [scanner] add host class, name mapper and pre-config logic --- cdist/argparse.py | 6 +- cdist/scan/commandline.py | 37 ++++++++---- cdist/scan/scan.py | 122 +++++++++++++++++++++++++++++--------- 3 files changed, 124 insertions(+), 41 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index f390a974..bedb23ac 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -485,7 +485,7 @@ def get_parsers(): parser['scan'].add_argument( '-m', '--mode', help='Which modes should run', action='append', default=[], - choices=['scan', 'trigger']) + choices=['scan', 'trigger', 'config']) parser['scan'].add_argument( '--list', action='store_true', @@ -498,6 +498,10 @@ def get_parsers(): '-I', '--interfaces', action='append', default=[], required=True, help='On which interfaces to scan/trigger') + parser['scan'].add_argument( + '--name-mapper', + action='store', default=None, + help='Map addresses to names, required for config mode') parser['scan'].add_argument( '-d', '--delay', action='store', default=3600, type=int, diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 331694e4..1a0ab0a7 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -37,7 +37,10 @@ def run(scan, args): log.debug("Trigger started") if 'scan' in args.mode: - s = scan.Scanner(interfaces=args.interfaces, args=args) + s = scan.Scanner( + autoconfigure='config' in args.mode, + interfaces=args.interfaces, + name_mapper=args.name_mapper) s.start() processes.append(s) log.debug("Scanner started") @@ -46,24 +49,27 @@ def run(scan, args): process.join() def list(scan, args): - s = scan.Scanner(interfaces=args.interfaces, args=args) + s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper) hosts = s.list() # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + # 6 colons. ipv6_max_size = 8 * 4 + 10 - # We format dates as follow: YYYY-MM-DD HH:MM:SS - date_max_size = 8 + 2 + 6 + 2 + date_max_size = len(datetime.now().strftime(scan.datetime_format)) + name_max_size = 25 - print("{} | {}".format( - 'link-local address'.ljust(ipv6_max_size), - 'last seen'.ljust(date_max_size))) - print('=' * (ipv6_max_size + 3 + date_max_size)) - for addr in hosts: - last_seen = datetime.strftime( - datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'), - '%Y-%m-%d %H:%M:%S') - print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size))) + print("{} | {} | {} | {}".format( + 'name'.ljust(name_max_size), + 'address'.ljust(ipv6_max_size), + 'last seen'.ljust(date_max_size), + 'last configured'.ljust(date_max_size))) + print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) + for host in hosts: + print("{} | {} | {} | {}".format( + host.name(default='-').ljust(name_max_size), + host.address().ljust(ipv6_max_size), + host.last_seen().ljust(date_max_size), + host.last_configured().ljust(date_max_size))) # CLI processing is defined outside of the main scan class to handle # non-available optional scapy dependency (instead of crashing mid-flight). @@ -82,6 +88,11 @@ def commandline(args): # By default scan and trigger, but do not call any action. args.mode = ['scan', 'trigger', ] + if 'config' in args.mode and args.name_mapper == None: + print('--name-mapper must be specified for scanner config mode.', + file=sys.stderr) + sys.exit(1) + # Print known hosts and exit is --list is specified - do not start # the scanner. if args.list: diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index f3976370..459138e2 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -63,6 +63,69 @@ import cdist.config logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") +datetime_format = '%Y-%m-%d %H:%M:%S' + +class Host(object): + def __init__(self, addr, outdir, name_mapper=None): + self.addr = addr + self.workdir = os.path.join(outdir, addr) + self.name_mapper = name_mapper + + os.makedirs(self.workdir, exist_ok=True) + + def __get(self, key, default=None): + fname = os.path.join(self.workdir, key) + value=default + if os.path.isfile(fname): + with open(fname, "r") as fd: + value = fd.readline() + return value + + def __set(self, key, value): + fname = os.path.join(self.workdir, key) + with open(fname, "w") as fd: + fd.write(f"{value}") + + def name(self, default=None): + if self.name_mapper == None: + return default + + fpath = os.path.join(os.getcwd(), self.name_mapper) + if os.path.isfile(fpath) and os.access(fpath, os.X_OK): + out = subprocess.run([fpath, self.addr], capture_output=True) + if out.returncode != 0: + return default + else: + value = out.stdout.decode() + return (None if len(value) == 0 else value) + else: + return default + + def address(self): + return self.addr + + def last_seen(self, default=None): + raw = self.__get('last_seen') + if raw: + return datetime.datetime.strptime(raw, datetime_format) + else: + return default + + def last_configured(self, default=None): + raw = self.__get('last_configured') + if raw: + return datetime.datetime.strptime(raw, datetime_format) + else: + return default + + def seen(self): + now = datetime.datetime.now().strftime(datetime_format) + self.__set('last_seen', now) + + def configure(self): + # TODO: configure. + now = datetime.datetime.now().strftime(datetime_format) + self.__set('last_configured', now) class Trigger(object): """ @@ -108,48 +171,43 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces, args=None, outdir=None): + def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None): self.interfaces = interfaces + self.autoconfigure=autoconfigure + self.name_mapper = name_mapper + self.config_delay = datetime.timedelta(seconds=3600) if outdir: self.outdir = outdir else: self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan') + os.makedirs(self.outdir, exist_ok=True) + + self.running_configs = {} def handle_pkg(self, pkg): if ICMPv6EchoReply in pkg: - host = pkg['IPv6'].src - log.verbose("Host %s is alive", host) + host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper) + if host.name(): + log.verbose("Host %s (%s) is alive", host.name(), host.address()) + else: + log.verbose("Host %s is alive", host.address()) + host.seen() - dir = os.path.join(self.outdir, host) - fname = os.path.join(dir, "last_seen") - - now = datetime.datetime.now() - - os.makedirs(dir, exist_ok=True) - - # FIXME: maybe adjust the format so we can easily parse again - with open(fname, "w") as fd: - fd.write(f"{now}\n") + # TODO check last config. + if self.autoconfigure and \ + host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now(): + self.config(host) def list(self): - hosts = dict() - for linklocal_addr in os.listdir(self.outdir): - workdir = os.path.join(self.outdir, linklocal_addr) - # We ignore any (unexpected) file in this directory. - if os.path.isdir(workdir): - last_seen='-' - last_seen_file = os.path.join(workdir, 'last_seen') - if os.path.isfile(last_seen_file): - with open(last_seen_file, "r") as fd: - last_seen = fd.readline() - - hosts[linklocal_addr] = {'last_seen': last_seen} + hosts = [] + for addr in os.listdir(self.outdir): + hosts.append(Host(addr, self.outdir, self.name_mapper)) return hosts - def config(self): + def config(self, host): """ Configure a host @@ -158,9 +216,19 @@ class Scanner(object): - Maybe keep dict storing per host processes - Save the result - Save the output -> probably aligned to config mode - """ + if host.name() == None: + log.debug("config - could not resolve name for %s, aborting.", host.address()) + return + + if self.running_configs.get(host.name()) != None: + log.debug("config - is already running for %s, aborting.", host.name()) + + log.info("config - running against host %s.", host.name()) + p = host.configure() + self.running_configs[host.name()] = p + def start(self): self.process = Process(target=self.scan) self.process.start() From 512e9b23c0035b09ac1e7e76f706fee2e2ec481f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 25 Apr 2021 15:53:40 +0200 Subject: [PATCH 26/64] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 70d0b960..c41743cd 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * New type: __postgres_conf (Beni Ruef, Dennis Camera) + 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) * Core: config: Make local state directory available to custom remotes (Steven Armstrong From 6ac8cbf98f5d5018373db6eb9c038ef1c599ff2f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 16:33:56 +0200 Subject: [PATCH 27/64] [type/__postgres_database] Include postgres_user explorer from __postgres_conf --- .../explorer/postgres_user | 1 + .../type/__postgres_database/explorer/state | 23 +++++-------------- .../type/__postgres_database/gencode-remote | 13 +---------- 3 files changed, 8 insertions(+), 29 deletions(-) create mode 120000 cdist/conf/type/__postgres_database/explorer/postgres_user diff --git a/cdist/conf/type/__postgres_database/explorer/postgres_user b/cdist/conf/type/__postgres_database/explorer/postgres_user new file mode 120000 index 00000000..714e7237 --- /dev/null +++ b/cdist/conf/type/__postgres_database/explorer/postgres_user @@ -0,0 +1 @@ +../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state index d68d4120..bb044e0e 100755 --- a/cdist/conf/type/__postgres_database/explorer/state +++ b/cdist/conf/type/__postgres_database/explorer/state @@ -1,6 +1,7 @@ #!/bin/sh # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,25 +19,13 @@ # along with cdist. If not, see . # -case "$("${__explorer}/os")" -in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; -esac +postgres_user=$("${__type_explorer:?}/postgres_user") +name=${__object_id:?} -name="$__object_id" - -if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='$name'\"")" +if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='${name}'\"")" then - echo 'present' + echo 'present' else - echo 'absent' + echo 'absent' fi diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote index 0f11cff4..d70fc436 100755 --- a/cdist/conf/type/__postgres_database/gencode-remote +++ b/cdist/conf/type/__postgres_database/gencode-remote @@ -18,23 +18,12 @@ # along with cdist. If not, see . # -case "$(cat "${__global}/explorer/os")" -in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; -esac name="$__object_id" state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" +postgres_user=$(cat "${__object:?}/explorer/postgres_user") if [ "$state_should" != "$state_is" ]; then case "$state_should" in From 58b279a8d0121dcbef05a4999d1d88208656aeed Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 16:34:17 +0200 Subject: [PATCH 28/64] [type/__postgres_database] Improve quoting --- .../type/__postgres_database/gencode-remote | 95 +++++++++++-------- 1 file changed, 55 insertions(+), 40 deletions(-) diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote index d70fc436..7d7d6fa2 100755 --- a/cdist/conf/type/__postgres_database/gencode-remote +++ b/cdist/conf/type/__postgres_database/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,49 +19,63 @@ # along with cdist. If not, see . # +quote() { + for _arg + do + shift + if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')" + then + # needs quoting + set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'" + else + set -- "$@" "${_arg}" + fi + done + unset _arg + # NOTE: Use printf because POSIX echo interprets escape sequences + printf '%s' "$*" +} -name="$__object_id" -state_should="$(cat "$__object/parameter/state")" -state_is="$(cat "$__object/explorer/state")" postgres_user=$(cat "${__object:?}/explorer/postgres_user") -if [ "$state_should" != "$state_is" ]; then - case "$state_should" in - present) - owner="" - if [ -f "$__object/parameter/owner" ]; then - owner="-O \"$(cat "$__object/parameter/owner")\"" - fi +dbname=${__object_id:?} +state_should=$(cat "${__object:?}/parameter/state") +state_is=$(cat "${__object:?}/explorer/state") - template="" - if [ -f "$__object/parameter/template" ]; then - template="--template \"$(cat "$__object/parameter/template")\"" - fi - - encoding="" - if [ -f "$__object/parameter/encoding" ]; then - encoding="--encoding \"$(cat "$__object/parameter/encoding")\"" - fi - - lc_collate="" - if [ -f "$__object/parameter/lc-collate" ]; then - lc_collate="--lc-collate \"$(cat "$__object/parameter/lc-collate")\"" - fi - - lc_ctype="" - if [ -f "$__object/parameter/lc-ctype" ]; then - lc_ctype="--lc-ctype \"$(cat "$__object/parameter/lc-ctype")\"" - fi - - cat << EOF -su - '$postgres_user' -c "createdb $owner \"$name\" $template $encoding $lc_collate $lc_ctype" -EOF - ;; - absent) - cat << EOF -su - '$postgres_user' -c "dropdb \"$name\"" -EOF - ;; - esac +if test "${state_should}" = "$state_is" +then + exit 0 fi + +case ${state_should} +in + (present) + set -- + + while read -r param_name opt + do + if test -f "${__object:?}/parameter/${param_name}" + then + set -- "$@" "${opt}" "$(cat "${__object:?}/parameter/${param_name}")" + fi + done <<-'EOF' + owner -O + template --template + encoding --encoding + lc_collate --lc-collate + lc_ctype --lc-ctype + EOF + + set -- "$@" "${dbname}" + + cat <<-EOF + su - $(quote "${postgres_user}") -c $(quote "$(quote createdb "$@")") + EOF + ;; + (absent) + cat <<-EOF + su - $(quote "${postgres_user}") -c $(quote "$(quote dropdb "${dbname}")") + EOF + ;; +esac From beb8da6d5fd141ac63eacfb526cfcc43cbf22d8f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 16:48:51 +0200 Subject: [PATCH 29/64] [type/__postgres_role] Include postgres_user explorer from __postgres_conf --- .../type/__postgres_role/explorer/postgres_user | 1 + cdist/conf/type/__postgres_role/explorer/state | 14 +------------- cdist/conf/type/__postgres_role/gencode-remote | 15 +-------------- 3 files changed, 3 insertions(+), 27 deletions(-) create mode 120000 cdist/conf/type/__postgres_role/explorer/postgres_user diff --git a/cdist/conf/type/__postgres_role/explorer/postgres_user b/cdist/conf/type/__postgres_role/explorer/postgres_user new file mode 120000 index 00000000..714e7237 --- /dev/null +++ b/cdist/conf/type/__postgres_role/explorer/postgres_user @@ -0,0 +1 @@ +../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 34069de9..0b264e6f 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -19,19 +19,7 @@ # along with cdist. If not, see . # -case $("${__explorer:?}/os") -in - (netbsd) - postgres_user='pgsql' - ;; - (openbsd) - postgres_user='_postgresql' - ;; - (*) - postgres_user='postgres' - ;; -esac - +postgres_user=$("${__type_explorer:?}/postgres_user") rolename=${__object_id:?} diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index d7631fbd..7324b80c 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -28,20 +28,7 @@ quote() { fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" } -case $(cat "${__global:?}/explorer/os") -in - (netbsd) - postgres_user='pgsql' - ;; - (openbsd) - postgres_user='_postgresql' - ;; - (*) - postgres_user='postgres' - ;; -esac - - +postgres_user=$(cat "${__object:?}/explorer/postgres_user") rolename=${__object_id:?} state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") From 3cf93249c38a6f27c097877add619b5d6499a049 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 16:55:11 +0200 Subject: [PATCH 30/64] [type/__postgres_extension] Include postgres_user explorer from __postgres_conf --- .../__postgres_extension/explorer/postgres_user | 1 + cdist/conf/type/__postgres_extension/gencode-remote | 13 +------------ 2 files changed, 2 insertions(+), 12 deletions(-) create mode 120000 cdist/conf/type/__postgres_extension/explorer/postgres_user diff --git a/cdist/conf/type/__postgres_extension/explorer/postgres_user b/cdist/conf/type/__postgres_extension/explorer/postgres_user new file mode 120000 index 00000000..714e7237 --- /dev/null +++ b/cdist/conf/type/__postgres_extension/explorer/postgres_user @@ -0,0 +1 @@ +../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote index af9c97f1..a1c84828 100755 --- a/cdist/conf/type/__postgres_extension/gencode-remote +++ b/cdist/conf/type/__postgres_extension/gencode-remote @@ -22,18 +22,7 @@ # along with cdist. If not, see . # -case "$(cat "${__global}/explorer/os")" -in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; -esac +postgres_user=$(cat "${__object:?}/explorer/postgres_user") dbname=$( echo "$__object_id" | cut -d":" -f1 ) From 8296051653a8187374ad51a3a487315767b73a19 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 16 Apr 2021 19:22:58 +0200 Subject: [PATCH 31/64] [type/__postgres_extension] Add state explorer --- .../type/__postgres_extension/explorer/state | 41 ++++++++++++++ .../type/__postgres_extension/gencode-remote | 44 ++++++++++----- cdist/conf/type/__postgres_extension/man.rst | 55 +++++++++++-------- 3 files changed, 104 insertions(+), 36 deletions(-) create mode 100644 cdist/conf/type/__postgres_extension/explorer/state diff --git a/cdist/conf/type/__postgres_extension/explorer/state b/cdist/conf/type/__postgres_extension/explorer/state new file mode 100644 index 00000000..9d156be7 --- /dev/null +++ b/cdist/conf/type/__postgres_extension/explorer/state @@ -0,0 +1,41 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Prints "present" if the extension is currently installed. +# "absent" otherwise. + +quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } + +postgres_user=$("${__type_explorer:?}/postgres_user") + +IFS=: read -r dbname extname <&2 + exit 1 + ;; esac diff --git a/cdist/conf/type/__postgres_extension/man.rst b/cdist/conf/type/__postgres_extension/man.rst index 79645b2b..442239f6 100644 --- a/cdist/conf/type/__postgres_extension/man.rst +++ b/cdist/conf/type/__postgres_extension/man.rst @@ -3,32 +3,36 @@ cdist-type__postgres_extension(7) NAME ---- -cdist-type__postgres_extension - manage postgres extensions +cdist-type__postgres_extension - Manage PostgreSQL extensions DESCRIPTION ----------- -This cdist type allows you to create or drop postgres extensions. +This cdist type allows you to manage PostgreSQL extensions. -The object you need to pass to __postgres_extension consists of -the database name and the extension name joined by a colon in the -following form: - -.. code-block:: sh - - dbname:extension - -f.ex. +The ``__object_id`` to pass to ``__postgres_extension`` is of the form +``dbname:extension``, e.g.: .. code-block:: sh rails_test:unaccent +**CAUTION!** Be careful when installing extensions from (untrusted) third-party +sources: + + | Installing an extension as superuser requires trusting that the extension's + author wrote the extension installation script in a secure fashion. It is + not terribly difficult for a malicious user to create trojan-horse objects + that will compromise later execution of a carelessly-written extension + script, allowing that user to acquire superuser privileges. + | – ``_ + + OPTIONAL PARAMETERS ------------------- state - either "present" or "absent", defaults to "present" + either ``present`` or ``absent``, defaults to ``present``. EXAMPLES @@ -36,24 +40,29 @@ EXAMPLES .. code-block:: sh - __postgres_extension rails_test:unaccent - __postgres_extension --present rails_test:unaccent - __postgres_extension --absent rails_test:unaccent + # Install extension unaccent into database rails_test + __postgres_extension rails_test:unaccent + + # Drop extension unaccent from database fails_test + __postgres_extension rails_test:unaccent --state absent SEE ALSO -------- -:strong:`cdist-type__postgre_database`\ (7) +- :strong:`cdist-type__postgres_database`\ (7) +- PostgreSQL "CREATE EXTENSION" documentation at: + ``_. -Postgres "Create Extension" documentation at: . -AUTHOR +AUTHORS ------- -Tomas Pospisek +| Tomas Pospisek +| Dennis Camera + COPYING ------- -Copyright \(C) 2014 Tomas Pospisek. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2014 Tomas Pospisek, 2021 Dennis Camera. +You can redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. From 0d33407b182964672b1cfc3ffc49c7fa7cf462a0 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 16 Apr 2021 19:35:26 +0200 Subject: [PATCH 32/64] [type/__postgres_database] Proper quoting in state explorer --- cdist/conf/type/__postgres_database/explorer/state | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state index bb044e0e..6a25df86 100755 --- a/cdist/conf/type/__postgres_database/explorer/state +++ b/cdist/conf/type/__postgres_database/explorer/state @@ -21,9 +21,14 @@ postgres_user=$("${__type_explorer:?}/postgres_user") -name=${__object_id:?} +dbname=${__object_id:?} -if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='${name}'\"")" +quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } +psql_exec() { + su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")" +} + +if psql_exec postgres "SELECT datname FROM pg_database" | grep -qFx "${dbname}" then echo 'present' else From 0f05f38384c94b79232ca0917d1323eeefd6e1ed Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 19 Apr 2021 19:09:46 +0200 Subject: [PATCH 33/64] [type/__postgres_role] Treat --password '' like no --password --- cdist/conf/type/__postgres_role/explorer/state | 11 ++++------- cdist/conf/type/__postgres_role/gencode-remote | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 0b264e6f..822816c1 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -43,8 +43,7 @@ role_properties=$( BEGIN { RS = "\036"; FS = "\034" } /^\([0-9]+ rows?\)/ { exit } NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next } - NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i } - ' + NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }' ) if test -n "${role_properties}" @@ -78,12 +77,10 @@ then # Check password passwd_stored=$( psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \ - | awk 'BEGIN { RS = "\036" } NR == 2' - printf . - ) - passwd_stored=${passwd_stored%?.} + | awk 'BEGIN { RS = "\036" } NR == 2 { printf "%s.", $0 }') + passwd_stored=${passwd_stored%.} - if test -f "${__object:?}/parameter/password" + if test -s "${__object:?}/parameter/password" then passwd_should=$(cat "${__object:?}/parameter/password"; printf .) fi diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 7324b80c..4cb78330 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -46,7 +46,7 @@ psql_query() { psql_set_password() { # NOTE: Always make sure that the password does not end up in psql_history! - # NOTE: Never set an empty string as the password, because they can be + # NOTE: Never set an empty string as the password, because it can be # interpreted differently by different tooling. if test -s "${__object:?}/parameter/password" then From 92fff7cb77271009f25d999b6451dc1e43e9bb6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 26 Apr 2021 12:09:20 +0200 Subject: [PATCH 34/64] [scanner] fix crash on --list with name mapper provided --- cdist/scan/commandline.py | 10 ++++++++-- cdist/scan/scan.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 1a0ab0a7..3eb7eec4 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -65,11 +65,17 @@ def list(scan, args): 'last configured'.ljust(date_max_size))) print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) for host in hosts: + last_seen = host.last_seen() + last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-' + + last_configured = host.last_configured() + last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-' + print("{} | {} | {} | {}".format( host.name(default='-').ljust(name_max_size), host.address().ljust(ipv6_max_size), - host.last_seen().ljust(date_max_size), - host.last_configured().ljust(date_max_size))) + last_seen.ljust(date_max_size), + last_configured.ljust(date_max_size))) # CLI processing is defined outside of the main scan class to handle # non-available optional scapy dependency (instead of crashing mid-flight). diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 459138e2..69a4121d 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -97,7 +97,7 @@ class Host(object): return default else: value = out.stdout.decode() - return (None if len(value) == 0 else value) + return (default if len(value) == 0 else value) else: return default From 3a9dd5b1669fa29e1713cdadec1596d859d377f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 26 Apr 2021 12:09:55 +0200 Subject: [PATCH 35/64] [scanner] add minimal (non-configurable) config mode --- cdist/scan/scan.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 69a4121d..152abb4e 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -122,8 +122,20 @@ class Host(object): now = datetime.datetime.now().strftime(datetime_format) self.__set('last_seen', now) + # XXX: There's no easy way to use the config module without feeding it with + # CLI args. Might as well call everything from scratch! def configure(self): - # TODO: configure. + target = self.name() or self.address() + cmd = ['cdist', 'config', '-v', target ] + + fname = os.path.join(self.workdir, 'last_configuration_log') + with open(fname, "w") as fd: + log.debug("Executing: %s", cmd) + completed_process = subprocess.run(cmd, stdout=fd, stderr=fd) + if completed_process.returncode != 0: + log.error("%s return with non-zero code %i - see %s for details.", + cmd, completed_process.returncode, fname) + now = datetime.datetime.now().strftime(datetime_format) self.__set('last_configured', now) @@ -194,7 +206,7 @@ class Scanner(object): log.verbose("Host %s is alive", host.address()) host.seen() - # TODO check last config. + # Configure if needed. if self.autoconfigure and \ host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now(): self.config(host) @@ -206,27 +218,18 @@ class Scanner(object): return hosts - def config(self, host): - """ - Configure a host - - - Assume we are only called if necessary - - However we need to ensure to not run in parallel - - Maybe keep dict storing per host processes - - Save the result - - Save the output -> probably aligned to config mode - """ - if host.name() == None: log.debug("config - could not resolve name for %s, aborting.", host.address()) return - if self.running_configs.get(host.name()) != None: + previous_config_process = self.running_configs.get(host.name()) + if previous_config_process != None and previous_config_process.is_alive(): log.debug("config - is already running for %s, aborting.", host.name()) - log.info("config - running against host %s.", host.name()) - p = host.configure() + log.info("config - running against host %s (%s).", host.name(), host.address()) + p = Process(target=host.configure()) + p.start() self.running_configs[host.name()] = p def start(self): From 2232435c2206b2b3a9a5f2b976df16540b3f1eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 26 Apr 2021 14:39:26 +0200 Subject: [PATCH 36/64] [scanner] initial documentation Note: still needs to patch main cdist(1) manpage --- cdist/scan/scan.py | 32 ------------- docs/src/cdist-scan.rst | 99 +++++++++++++++++++++++++++++++++++++++++ docs/src/index.rst | 1 + 3 files changed, 100 insertions(+), 32 deletions(-) create mode 100644 docs/src/cdist-scan.rst diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 152abb4e..2912dab3 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -19,38 +19,6 @@ # # -# -# Interface to be implemented: -# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated -# scan: scan / listen for icmp6 replies -# trigger: send trigger to multicast -# config: configure newly detected hosts -# install: install newly detected hosts -# -# Scanner logic -# - save results to configdir: -# basedir = ~/.cdist/scan/ -# last_seen = ~/.cdist/scan//last_seen -- record unix time -# or similar -# last_configured = ~/.cdist/scan//last_configured -- record -# unix time or similar -# last_installed = ~/.cdist/scan//last_configured -- record -# unix time or similar -# -# -# -# -# cdist scan --list -# Show all known hosts including last seen flag -# -# Logic for reconfiguration: -# -# - record when configured last time -# - introduce a parameter --reconfigure-after that takes time argument -# - reconfigure if a) host alive and b) reconfigure-after time passed -# - - from multiprocessing import Process import os import logging diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst new file mode 100644 index 00000000..02193456 --- /dev/null +++ b/docs/src/cdist-scan.rst @@ -0,0 +1,99 @@ +Scan +===== + +Description +----------- +Runs cdist as a daemon that discover/watch on hosts and reconfigure them +periodically. It is especially useful in netboot-based environment where hosts +boot unconfigured, and to ensure your infrastructure stays in sync with your +configuration. + +This feature is still consider to be in **beta** stage. + +Usage (Examples) +---------------- + +Discover hosts on local network and configure those whose name is resolved by +the name mapper script. + +.. code-block:: sh + + $ cdist scan --beta --interface eth0 \ + --mode scan --name-mapper path/to/script \ + --mode trigger --mode config + +List known hosts and exit. + +.. code-block:: sh + + $ cdist scan --beta --list --name-mapper path/to/script + +Please refer to `cdist(1)` for a detailed list of parameters. + +Modes +----- + +The scanner has 3 modes that can be independently toggled. If the `--mode` +parameter is not specified, only `tigger` and `scan` are enabled (= hosts are +not configured). + +trigger + Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to + trigger detection by the `scan` module. + +scan + Watch for incoming ICMPv6 replies and optionally configure detected hosts. + +config + Enable configuration of hosts detected by `scan`. + +Name Mapper Script +------------------ + +The name mapper script takes an IPv6 address as first argument and writes the +resolved name to stdout - if any. The script must be executable. + +Simplest script: + +.. code-block:: sh + #!/bin/sh + + case "$1" in + "fe80::20d:b9ff:fe57:3524") + printf "my-host-01" + ;; + "fe80::7603:bdff:fe05:89bb") + printf "my-host-02" + ;; + esac + +Resolving name from `PTR` DNS record: + +.. code-block:: sh + #!/bin/sh + + for cmd in dig sed; do + if ! command -v $cmd > /dev/null; then + exit 1 + fi + done + + dig +short -x "$1" | sed -e 's/.$//' + + +Trigger Source Script +--------------------- + +This script returns a list of addresses (separated by a newline) to be used by +`trigger` mode. It is not used to map names. The script must be executable. + +Simplest script: + +.. code-block:: sh + #!/bin/sh + + cat << EOF + server1.domain.tld + server2.domain.tld + server3.domain.tld + EOF diff --git a/docs/src/index.rst b/docs/src/index.rst index 31c044dc..369d5309 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -34,6 +34,7 @@ It natively supports IPv6 since the first release. cdist-parallelization cdist-inventory cdist-preos + cdist-scan cdist-integration cdist-reference cdist-best-practice From a4122882f2a14034d1a219bd0314892176c5cb64 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 26 Apr 2021 16:39:51 +0200 Subject: [PATCH 37/64] [type/__debconf_set_selections] Add state explorer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …and to make it work, replace --file with --line. --file is deprecated because it does not work with the state explorer as the contents of the file are not available on the target. --- .../__debconf_set_selections/explorer/state | 124 ++++++++++++++++++ .../__debconf_set_selections/gencode-remote | 37 ++++-- .../type/__debconf_set_selections/man.rst | 49 ++++--- .../type/__debconf_set_selections/manifest | 21 +++ .../parameter/{required => deprecated} | 0 .../parameter/optional_multiple | 1 + 6 files changed, 206 insertions(+), 26 deletions(-) create mode 100644 cdist/conf/type/__debconf_set_selections/explorer/state create mode 100755 cdist/conf/type/__debconf_set_selections/manifest rename cdist/conf/type/__debconf_set_selections/parameter/{required => deprecated} (100%) create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/optional_multiple diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state new file mode 100644 index 00000000..68d28941 --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/explorer/state @@ -0,0 +1,124 @@ +#!/bin/sh -e +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Determine current debconf selections' state. +# Prints one of: +# present: all selections are already set as they should. +# different: one or more of the selections have a different value. +# absent: one or more of the selections are not (currently) defined. +# + +test -x /usr/bin/perl || { + # cannot find perl (no perl ~ no debconf) + echo 'absent' + exit 0 +} + +linesfile="${__object:?}/parameter/line" +test -s "${linesfile}" || { + if test -s "${__object:?}/parameter/file" + then + echo absent + else + echo present + fi + exit 0 +} + +/usr/bin/perl -- - "${linesfile}" <<'EOF' +use strict; +use warnings "all"; + +use Debconf::Db; +use Debconf::Question; + +# Extract @known... arrays from debconf-set-selections +# These values are required to distinguish flags and values in the given lines. +# DC: I couldn't think of a more ugly solution to the problem… +my @knownflags; +my @knowntypes; +my $debconf_set_selections = '/usr/bin/debconf-set-selections'; +if (-e $debconf_set_selections) { + my $sed_known = 's/^my \(@known\(flags\|types\) = qw([a-z ]*);\).*$/\1/p'; + eval `sed -n '$sed_known' '$debconf_set_selections'`; +} + +sub mungeline ($) { + my $line = shift; + chomp $line; + $line =~ s/\r$//; + return $line; +} + +sub fatal { printf STDERR @_; exit 1; } + +my $state = 'present'; + +sub state { + my $new = shift; + if ($state eq 'present' + or ($state eq 'different' and $new eq 'absent')) { + $state = $new; + } +} + +Debconf::Db->load(readonly => 'true'); + +while (<>) { + # Read and process lines (taken from debconf-set-selections) + $_ = mungeline($_); + while (/\\$/ && ! eof) { + s/\\$//; + $_ .= mungeline(<>); + } + next if /^\s*$/ || /^\s*\#/; + + my ($owner, $label, $type, $content) = /^\s*(\S+)\s+(\S+)\s+(\S+)(?:\s(.*))?/ + or fatal "invalid line: %s\n", $_; + $content = '' unless defined $content; + + + # Compare is and should state + my $q = Debconf::Question->get($label); + + unless (defined $q) { + # probably a preseed + state 'absent'; + next; + } + + if (grep { $_ eq $q->type } @knownflags) { + # This line wants to set a flag, presumably. + if ($q->flag($q->type) ne $content) { + state 'different'; + } + } else { + # Otherwise, it's probably a value… + if ($q->value ne $content) { + state 'different'; + } + + unless (grep { $_ eq $owner } (split /, /, $q->owners)) { + state 'different'; + } + } +} + +printf "%s\n", $state; +EOF diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote index e99aef40..50d898c6 100755 --- a/cdist/conf/type/__debconf_set_selections/gencode-remote +++ b/cdist/conf/type/__debconf_set_selections/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org) +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,16 +18,32 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# -# Setup selections -# -filename="$(cat "$__object/parameter/file")" - -if [ "$filename" = "-" ]; then - filename="$__object/stdin" +if test -f "${__object:?}/parameter/line" +then + filename="${__object:?}/parameter/line" +elif test -s "${__object:?}/parameter/file" +then + filename=$(cat "${__object:?}/parameter/file") + if test "${filename}" = '-' + then + filename="${__object:?}/stdin" + fi +else + printf 'Neither --line nor --file set.\n' >&2 + exit 1 fi -echo "debconf-set-selections << __file-eof" -cat "$filename" -echo "__file-eof" +# setting no lines makes no sense +test -s "${filename}" || exit 0 + +state_is=$(cat "${__object:?}/explorer/state") + +if test "${state_is}" != 'present' +then + cat <<-CODE + debconf-set-selections <<'EOF' + $(cat "${filename}") + EOF + CODE +fi diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst index 58c25b81..690e3e49 100644 --- a/cdist/conf/type/__debconf_set_selections/man.rst +++ b/cdist/conf/type/__debconf_set_selections/man.rst @@ -8,15 +8,33 @@ cdist-type__debconf_set_selections - Setup debconf selections DESCRIPTION ----------- -On Debian and alike systems debconf-set-selections(1) can be used +On Debian and alike systems :strong:`debconf-set-selections`\ (1) can be used to setup configuration parameters. REQUIRED PARAMETERS ------------------- +cf. ``--line``. + + +OPTIONAL PARAMETERS +------------------- file - Use the given filename as input for debconf-set-selections(1) - If filename is "-", read from stdin. + Use the given filename as input for :strong:`debconf-set-selections`\ (1) + If filename is ``-``, read from stdin. + + **This parameter is deprecated, because it doesn't work with state detection.** +line + A line in :strong:`debconf-set-selections`\ (1) compatible format. + This parameter can be used multiple times to set multiple options. + + (This parameter is actually required, but marked optional because the + deprecated ``--file`` is still accepted.) + + +BOOLEAN PARAMETERS +------------------ +None. EXAMPLES @@ -24,30 +42,29 @@ EXAMPLES .. code-block:: sh - # Setup configuration for nslcd - __debconf_set_selections nslcd --file /path/to/file + # Setup gitolite's gituser + __debconf_set_selections nslcd --line 'gitolite gitolite/gituser string git' - # Setup configuration for nslcd from another type - __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd" - - __debconf_set_selections nslcd --file - << eof - gitolite gitolite/gituser string git - eof + # Setup configuration for nslcd from a file. + # NB: Multiple lines can be passed to --line, although this can be considered a hack. + __debconf_set_selections nslcd --line "$(cat "${__files:?}/preseed/nslcd.debconf")" SEE ALSO -------- -:strong:`debconf-set-selections`\ (1), :strong:`cdist-type__update_alternatives`\ (7) +- :strong:`cdist-type__update_alternatives`\ (7) +- :strong:`debconf-set-selections`\ (1) AUTHORS ------- Nico Schottelius +Dennis Camera COPYING ------- -Copyright \(C) 2011-2014 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2011-2014 Nico Schottelius, 2021 Dennis Camera. +You can redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. diff --git a/cdist/conf/type/__debconf_set_selections/manifest b/cdist/conf/type/__debconf_set_selections/manifest new file mode 100755 index 00000000..0f4fb2e2 --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/manifest @@ -0,0 +1,21 @@ +#!/bin/sh -e +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +__package_apt debconf diff --git a/cdist/conf/type/__debconf_set_selections/parameter/required b/cdist/conf/type/__debconf_set_selections/parameter/deprecated similarity index 100% rename from cdist/conf/type/__debconf_set_selections/parameter/required rename to cdist/conf/type/__debconf_set_selections/parameter/deprecated diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple new file mode 100644 index 00000000..a999a0c2 --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple @@ -0,0 +1 @@ +line From 9cf19388abe95a5f8df9f5c94d2e54fa1060b2ef Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 26 Apr 2021 16:47:44 +0200 Subject: [PATCH 38/64] [type/__debconf_set_selections] Send message about each debconf setting that is changed --- cdist/conf/type/__debconf_set_selections/gencode-remote | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote index 50d898c6..9ba28f09 100755 --- a/cdist/conf/type/__debconf_set_selections/gencode-remote +++ b/cdist/conf/type/__debconf_set_selections/gencode-remote @@ -46,4 +46,9 @@ then $(cat "${filename}") EOF CODE + + awk ' + { + printf "set %s %s %s %s\n", $1, $2, $3, $4 + }' "${filename}" >>"${__messages_out:?}" fi From 3a25b804664963e092d1f75c1fdded25d7016ef3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 26 Apr 2021 21:27:15 +0200 Subject: [PATCH 39/64] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index c41743cd..497ae91a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * New type: __postgres_conf (Beni Ruef, Dennis Camera) + * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From a42ebc7a78e09e138954b1376376f7cb643420e0 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 27 Apr 2021 19:41:10 +0200 Subject: [PATCH 40/64] [type/__debconf_set_selections] Synchronise objects Works around locking error: debconf: DbDriver "config": /var/cache/debconf/config.dat is locked by another process: Resource temporarily unavailable --- .../__debconf_set_selections/explorer/state | 20 ++++++++++++++++++- .../type/__debconf_set_selections/nonparallel | 0 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 cdist/conf/type/__debconf_set_selections/nonparallel diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state index 68d28941..f8a3f6c8 100644 --- a/cdist/conf/type/__debconf_set_selections/explorer/state +++ b/cdist/conf/type/__debconf_set_selections/explorer/state @@ -41,10 +41,15 @@ test -s "${linesfile}" || { exit 0 } +# assert __type_explorer is set (because it is used by the Perl script) +: "${__type_explorer:?}" + /usr/bin/perl -- - "${linesfile}" <<'EOF' use strict; use warnings "all"; +use Fcntl qw(:DEFAULT :flock); + use Debconf::Db; use Debconf::Question; @@ -78,7 +83,20 @@ sub state { } } -Debconf::Db->load(readonly => 'true'); + +# Load Debconf DB but manually lock on the state explorer script, +# because Debconf aborts immediately if executed concurrently. +# This is not really an ideal solution because the Debconf DB could be locked by +# another process (e.g. apt-get), but no way to achieve this could be found. +# If you know how to, please provide a patch. +my $lockfile = "%ENV{'__type_explorer'}/state"; +if (open my $lock_fh, '+<', $lockfile) { + flock $lock_fh, LOCK_EX or die "Cannot lock $lockfile"; +} +{ + Debconf::Db->load(readonly => 'true'); +} + while (<>) { # Read and process lines (taken from debconf-set-selections) diff --git a/cdist/conf/type/__debconf_set_selections/nonparallel b/cdist/conf/type/__debconf_set_selections/nonparallel new file mode 100644 index 00000000..e69de29b From c00c8c20127af9881b4e295b7732524e7e1c5031 Mon Sep 17 00:00:00 2001 From: Evil Ham Date: Mon, 10 May 2021 12:08:22 +0200 Subject: [PATCH 41/64] [__apt_key*] Deprecate __apt_key_uri and improve __apt_key Previously this type was falling back to using the deprecated apt-key(8) by checking for existence of files/directories on the controller host in gencode-remote. Adding `--use-deprecated-apt-key` as an explicit boolean serves two purposes: 1. It prevents fallbacks that might end up doing the wrong thing (as was the case) 2. It allows for a simple way to remove keys from the keyring that were previously added with apt-key(8) to /etc/apt/trusted.gpg This parameter is added marked as deprecated as is only intended use is to migrate to directory-based keyrings as recommended by Debian for a few releases. It will be removed when Debian 11 stops being supported. During the review process of this merge request, it was noted that the state of PGP Key Servers is somewhat suboptimal, that the examples encouraged bad practise (it is trivial to produce collisions for short key IDs), and that this use does not require the Web of Trust, but instead only the public key that is signing the repository. That is why this also adds `--source` as an argument allowing for in-type or in-manifest provision of such public keys by the type/manifest maintainer and the use of Key Servers is still supported, but discouraged. --- cdist/conf/type/__apt_key/explorer/state | 27 +++-- cdist/conf/type/__apt_key/gencode-remote | 53 ++++----- cdist/conf/type/__apt_key/man.rst | 84 ++++++++++---- cdist/conf/type/__apt_key/manifest | 104 +++++++++++++++++- cdist/conf/type/__apt_key/parameter/boolean | 1 + .../deprecated/use-deprecated-apt-key | 3 + cdist/conf/type/__apt_key/parameter/optional | 5 +- cdist/conf/type/__apt_key_uri/deprecated | 1 + 8 files changed, 209 insertions(+), 69 deletions(-) create mode 100644 cdist/conf/type/__apt_key/parameter/boolean create mode 100644 cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key create mode 100644 cdist/conf/type/__apt_key_uri/deprecated diff --git a/cdist/conf/type/__apt_key/explorer/state b/cdist/conf/type/__apt_key/explorer/state index 38f1bd3c..8ab268c1 100755 --- a/cdist/conf/type/__apt_key/explorer/state +++ b/cdist/conf/type/__apt_key/explorer/state @@ -27,18 +27,25 @@ else keyid="$__object_id" fi +# From apt-key(8): +# Use of apt-key is deprecated, except for the use of apt-key del in +# maintainer scripts to remove existing keys from the main keyring. +# If such usage of apt-key is desired the additional installation of +# the GNU Privacy Guard suite (packaged in gnupg) is required. +if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then + if apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" + then echo present + else echo absent + fi + exit +fi + keydir="$(cat "$__object/parameter/keydir")" keyfile="$keydir/$__object_id.gpg" -if [ -d "$keydir" ] +if [ -f "$keyfile" ] then - if [ -f "$keyfile" ] - then echo present - else echo absent - fi -else - # fallback to deprecated apt-key - apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \ - && echo present \ - || echo absent + echo present + exit fi +echo absent diff --git a/cdist/conf/type/__apt_key/gencode-remote b/cdist/conf/type/__apt_key/gencode-remote index 0c96ff67..17dc9bfc 100755 --- a/cdist/conf/type/__apt_key/gencode-remote +++ b/cdist/conf/type/__apt_key/gencode-remote @@ -25,11 +25,7 @@ else fi state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" - -if [ "$state_should" = "$state_is" ]; then - # nothing to do - exit 0 -fi +method="$(cat "$__object/key_method")" keydir="$(cat "$__object/parameter/keydir")" keyfile="$keydir/$__object_id.gpg" @@ -37,30 +33,18 @@ keyfile="$keydir/$__object_id.gpg" case "$state_should" in present) keyserver="$(cat "$__object/parameter/keyserver")" - - if [ -f "$__object/parameter/uri" ]; then - uri="$(cat "$__object/parameter/uri")" - - if [ -d "$keydir" ]; then - cat << EOF - -curl -s -L \\ - -o "$keyfile" \\ - "$uri" - -key="\$( cat "$keyfile" )" - -if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK' -then - echo "\$key" | gpg --dearmor > "$keyfile" -fi - -EOF - else - # fallback to deprecated apt-key - echo "curl -s -L '$uri' | apt-key add -" + # Using __download or __file as key source + # Propagate messages if needed + if [ "${method}" = "uri" ] || [ "${method}" = "source" ]; then + if grep -Eq "^__(file|download)$keyfile" "$__messages_in"; then + echo "added '$keyid'" >> "$__messages_out" fi - elif [ -d "$keydir" ]; then + exit 0 + elif [ "${state_is}" = "present" ]; then + exit 0 + fi + # Using key servers to fetch the key + if [ ! -f "$__object/parameter/use-deprecated-apt-key" ]; then # we need to kill gpg after 30 seconds, because gpg # can get stuck if keyserver is not responding. # exporting env var and not exit 1, @@ -100,13 +84,16 @@ EOF echo "added '$keyid'" >> "$__messages_out" ;; absent) - if [ -f "$keyfile" ]; then - echo "rm '$keyfile'" - else + # Removal for keys added from a keyserver without this flag + # is done in the manifest + if [ "$state_is" != "absent" ] && \ + [ -f "$__object/parameter/use-deprecated-apt-key" ]; then # fallback to deprecated apt-key echo "apt-key del \"$keyid\"" + echo "removed '$keyid'" >> "$__messages_out" + # Propagate messages if needed + elif grep -Eq "^__file$keyfile" "$__messages_in"; then + echo "removed '$keyid'" >> "$__messages_out" fi - - echo "removed '$keyid'" >> "$__messages_out" ;; esac diff --git a/cdist/conf/type/__apt_key/man.rst b/cdist/conf/type/__apt_key/man.rst index 234bc715..e35eaa0f 100644 --- a/cdist/conf/type/__apt_key/man.rst +++ b/cdist/conf/type/__apt_key/man.rst @@ -10,6 +10,14 @@ DESCRIPTION ----------- Manages the list of keys used by apt to authenticate packages. +This is done by placing the requested key in a file named +``$__object_id.gpg`` in the ``keydir`` directory. + +This is supported by modern releases of Debian-based distributions. + +In order of preference, exactly one of: ``source``, ``uri`` or ``keyid`` +must be specified. + REQUIRED PARAMETERS ------------------- @@ -18,21 +26,49 @@ None. OPTIONAL PARAMETERS ------------------- +keydir + keyring directory, defaults to ``/etc/apt/trusted.pgp.d``, which is + enabled system-wide by default. + +source + path to a file containing the GPG key of the repository. + Using this is recommended as it ensures that the manifest/type manintainer + has validated the key. + If ``-``, the GPG key is read from the type's stdin. + state 'present' or 'absent'. Defaults to 'present' +uri + the URI from which to download the key. + It is highly recommended that you only use protocols with TLS like HTTPS. + This uses ``__download`` but does not use checksums, if you want to ensure + that the key doesn't change, you are better off downloading it and using + ``--source``. + + +DEPRECATED OPTIONAL PARAMETERS +------------------------------ keyid - the id of the key to add. Defaults to __object_id + the id of the key to download from the ``keyserver``. + This is to be used in absence of ``--source`` and ``--uri`` or together + with ``--use-deprecated-apt-key`` for key removal. + Defaults to ``$__object_id``. keyserver - the keyserver from which to fetch the key. If omitted the default set - in ./parameter/default/keyserver is used. + the keyserver from which to fetch the key. + Defaults to ``pool.sks-keyservers.net``. -keydir - key save location, defaults to ``/etc/apt/trusted.pgp.d`` -uri - the URI from which to download the key +DEPRECATED BOOLEAN PARAMETERS +----------------------------- +use-deprecated-apt-key + ``apt-key(8)`` will last be available in Debian 11 and Ubuntu 22.04. + You can use this parameter to force usage of ``apt-key(8)``. + Please only use this parameter to *remove* keys from the keyring, + in order to prepare for removal of ``apt-key``. + Adding keys should be done without this parameter. + This parameter will be removed when Debian 11 stops being supported. EXAMPLES @@ -40,33 +76,39 @@ EXAMPLES .. code-block:: sh - # Add Ubuntu Archive Automatic Signing Key - __apt_key 437D05B5 - # Same thing - __apt_key 437D05B5 --state present - # Get rid of it - __apt_key 437D05B5 --state absent + # add a key that has been verified by a type maintainer + __apt_key jitsi_meet_2021 \ + --source cdist-contrib/type/__jitsi_meet/files/apt_2021.gpg - # same thing with human readable name and explicit keyid - __apt_key UbuntuArchiveKey --keyid 437D05B5 + # remove an old, deprecated or expired key + __apt_key jitsi_meet_2016 --state absent - # same thing with other keyserver - __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com + # Get rid of a key that might have been added to + # /etc/apt/trusted.gpg with apt-key + __apt_key 0x40976EAF437D05B5 --use-deprecated-apt-key --state absent - # download key from the internet - __apt_key rabbitmq \ - --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc + # add a key that we define in-line + __apt_key jitsi_meet_2021 --source '-' < Ander Punnar +Evilham COPYING ------- -Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can +Copyright \(C) 2011-2021 Steven Armstrong, Ander Punnar and Evilham. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_key/manifest b/cdist/conf/type/__apt_key/manifest index 010357cd..889a764a 100755 --- a/cdist/conf/type/__apt_key/manifest +++ b/cdist/conf/type/__apt_key/manifest @@ -2,7 +2,105 @@ __package gnupg -if [ -f "$__object/parameter/uri" ] -then __package curl -else __package dirmngr +state_should="$(cat "${__object}/parameter/state")" + +incompatible_args() +{ + cat >> /dev/stderr <<-EOF + This type does not support --${1} and --${method} simultaneously. + EOF + exit 1 +} + +if [ -f "${__object}/parameter/source" ]; then + method="source" + src="$(cat "${__object}/parameter/source")" + if [ "${src}" = "-" ]; then + src="${__object}/stdin" + fi +fi +if [ -f "${__object}/parameter/uri" ]; then + if [ -n "${method}" ]; then + incompatible_args uri + fi + method="uri" + src="$(cat "${__object}/parameter/uri")" +fi +if [ -f "${__object}/parameter/keyid" ]; then + if [ -n "${method}" ]; then + incompatible_args keyid + fi + method="keyid" +fi +# Keep old default +if [ -z "${method}" ]; then + method="keyid" +fi +# Save this for later in gencode-remote +echo "${method}" > "${__object}/key_method" + +# Required remotely (most likely already installed) +__package dirmngr +# We need this in case a key has to be dearmor'd +__package gnupg +export require="__package/gnupg" + +if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then + # This is required if apt-key(8) is to be used + if [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then + incompatible_args use-deprecated-apt-key + fi +else + if [ "${state_should}" = "absent" ] && \ + [ -f "${__object}/parameter/keyid" ]; then + cat >> /dev/stderr < Date: Mon, 10 May 2021 12:10:00 +0200 Subject: [PATCH 42/64] [__letsencrypt_cert] Revamp explorers, add locking. This would fix #839 Certbot uses locking [1] even for read-only operations and does not properly use exit codes, which means that sometimes it would print: "Another instance of Certbot is already running" and exit with success. However, the previous explorers would take that as the certificate being absent and would trigger code generation. The issue was made worse by having many explorers running certbot, so for N certificates, we'd run certbot N*4 times, potentially "in parallel". [1]: https://certbot.eff.org/docs/using.html#id5 This patch joins all explorers in one to avoid starting multiple remote python processes and uses a cdist-specific lock in /tmp/certbot.cdist.lock with a 60 seconds timeout. It has been tested with certbot 0.31.0 and 0.17 that the: from certbot.main import main trick works. It is somewhat well documented so it can be somewhat relied upon. --- .../__letsencrypt_cert/explorer/certbot-path | 3 - .../explorer/certificate-data | 78 +++++++++++++++++++ .../explorer/certificate-domains | 8 -- .../explorer/certificate-exists | 13 ---- .../explorer/certificate-is-test | 14 ---- .../type/__letsencrypt_cert/gencode-remote | 11 ++- cdist/conf/type/__letsencrypt_cert/manifest | 2 +- 7 files changed, 87 insertions(+), 42 deletions(-) delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certbot-path create mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-data delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path deleted file mode 100755 index 3c6076df..00000000 --- a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -e - -command -v certbot 2>/dev/null || true diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data new file mode 100755 index 00000000..ff62e742 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data @@ -0,0 +1,78 @@ +#!/bin/sh -e +certbot_path="$(command -v certbot 2>/dev/null || true)" +# Defaults +certificate_exists="no" +certificate_is_test="no" + +if [ -n "${certbot_path}" ]; then + # Find python executable that has access to certbot's module + python_path=$(sed -n '1s/^#! *//p' "${certbot_path}") + + # Use a lock for cdist due to certbot not exiting with failure + # or having any flags for concurrent use. + _certbot() { + ${python_path} - 2>/dev/null < "${existing_domains}" + certificate_is_test="$(_explorer_var certificate_is_test)" sort -uo "${requested_domains}" "${requested_domains}" sort -uo "${existing_domains}" "${existing_domains}" diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 1df3574a..6394f629 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -1,6 +1,6 @@ #!/bin/sh -certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")" +certbot_fullpath="$(grep "^certbot_path:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-)" state=$(cat "${__object}/parameter/state") os="$(cat "${__global:?}/explorer/os")" From f14623e45f56aec4bdce53bc2b367e11ff157865 Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 10 May 2021 12:17:08 +0200 Subject: [PATCH 43/64] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 497ae91a..5c3b547a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,8 @@ Changelog next: * New type: __postgres_conf (Beni Ruef, Dennis Camera) * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) + * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) + * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 6210cccb28ace8ec2807ffe6ca9f9e4d9007990b Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 10 May 2021 12:34:04 +0200 Subject: [PATCH 44/64] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 5c3b547a..a7310788 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) + * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking. 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 503a06ed28f743aad47797a0989be735c67b07a6 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 28 Apr 2021 13:32:10 +0300 Subject: [PATCH 45/64] [__git] fix group explorer group name from numberic id wasn't resolved correctly. try to use getent and fallback to reading /etc/group directly. --- cdist/conf/type/__git/explorer/group | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group index 1365c60d..ab4396b1 100644 --- a/cdist/conf/type/__git/explorer/group +++ b/cdist/conf/type/__git/explorer/group @@ -14,6 +14,11 @@ then then printf '%u\n' "${group_gid}" else - printf '%s\n' "$(id -u -n "${group_gid}")" + if command -v getent > /dev/null + then + getent group "${group_gid}" | cut -d : -f 1 + else + awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group + fi fi fi From 75c71f69c1fed4371c5891ddcca0eaf28ad928e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 10:17:48 +0200 Subject: [PATCH 46/64] [scanner] pycodestyle compliance --- cdist/scan/commandline.py | 20 +++++++++++---- cdist/scan/scan.py | 54 ++++++++++++++++++++++++--------------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 3eb7eec4..b42bc7b2 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -25,13 +25,15 @@ from datetime import datetime log = logging.getLogger("scan") + def run(scan, args): # We run each component in a separate process since they # must not block on each other. processes = [] if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay) + t = scan.Trigger(interfaces=args.interfaces, + sleeptime=args.trigger_delay) t.start() processes.append(t) log.debug("Trigger started") @@ -48,6 +50,7 @@ def run(scan, args): for process in processes: process.join() + def list(scan, args): s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper) hosts = s.list() @@ -66,10 +69,16 @@ def list(scan, args): print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) for host in hosts: last_seen = host.last_seen() - last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-' + if last_seen: + last_seen = last_seen.strftime(scan.datetime_format) + else: + last_seen = '-' last_configured = host.last_configured() - last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-' + if last_configured: + last_configured = last_configured.strftime(scan.datetime_format) + else: + '-' print("{} | {} | {} | {}".format( host.name(default='-').ljust(name_max_size), @@ -77,6 +86,7 @@ def list(scan, args): last_seen.ljust(date_max_size), last_configured.ljust(date_max_size))) + # CLI processing is defined outside of the main scan class to handle # non-available optional scapy dependency (instead of crashing mid-flight). def commandline(args): @@ -94,9 +104,9 @@ def commandline(args): # By default scan and trigger, but do not call any action. args.mode = ['scan', 'trigger', ] - if 'config' in args.mode and args.name_mapper == None: + if 'config' in args.mode and args.name_mapper is None: print('--name-mapper must be specified for scanner config mode.', - file=sys.stderr) + file=sys.stderr) sys.exit(1) # Print known hosts and exit is --list is specified - do not start diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 2912dab3..4a20f511 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -33,6 +33,7 @@ logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") datetime_format = '%Y-%m-%d %H:%M:%S' + class Host(object): def __init__(self, addr, outdir, name_mapper=None): self.addr = addr @@ -43,7 +44,7 @@ class Host(object): def __get(self, key, default=None): fname = os.path.join(self.workdir, key) - value=default + value = default if os.path.isfile(fname): with open(fname, "r") as fd: value = fd.readline() @@ -55,15 +56,15 @@ class Host(object): fd.write(f"{value}") def name(self, default=None): - if self.name_mapper == None: + if self.name_mapper is None: return default fpath = os.path.join(os.getcwd(), self.name_mapper) if os.path.isfile(fpath) and os.access(fpath, os.X_OK): - out = subprocess.run([fpath, self.addr], capture_output=True) - if out.returncode != 0: - return default - else: + out = subprocess.run([fpath, self.addr], capture_output=True) + if out.returncode != 0: + return default + else: value = out.stdout.decode() return (default if len(value) == 0 else value) else: @@ -94,19 +95,20 @@ class Host(object): # CLI args. Might as well call everything from scratch! def configure(self): target = self.name() or self.address() - cmd = ['cdist', 'config', '-v', target ] + cmd = ['cdist', 'config', '-v', target] fname = os.path.join(self.workdir, 'last_configuration_log') with open(fname, "w") as fd: log.debug("Executing: %s", cmd) completed_process = subprocess.run(cmd, stdout=fd, stderr=fd) if completed_process.returncode != 0: - log.error("%s return with non-zero code %i - see %s for details.", - cmd, completed_process.returncode, fname) + log.error("%s return with non-zero code %i - see %s for \ + details.", cmd, completed_process.returncode, fname) now = datetime.datetime.now().strftime(datetime_format) self.__set('last_configured', now) + class Trigger(object): """ Trigger an ICMPv6EchoReply from all hosts that are alive @@ -140,10 +142,12 @@ class Trigger(object): def trigger(self, interface): try: log.debug("Sending ICMPv6EchoRequest on %s", interface) - packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest() + packet = IPv6( + dst="ff02::1%{}".format(interface) + ) / ICMPv6EchoRequest() send(packet, verbose=self.verbose) except Exception as e: - log.error( "Could not send ICMPv6EchoRequest: %s", e) + log.error("Could not send ICMPv6EchoRequest: %s", e) class Scanner(object): @@ -151,9 +155,10 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None): + def __init__(self, interfaces, autoconfigure=False, outdir=None, + name_mapper=None): self.interfaces = interfaces - self.autoconfigure=autoconfigure + self.autoconfigure = autoconfigure self.name_mapper = name_mapper self.config_delay = datetime.timedelta(seconds=3600) @@ -169,14 +174,17 @@ class Scanner(object): if ICMPv6EchoReply in pkg: host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper) if host.name(): - log.verbose("Host %s (%s) is alive", host.name(), host.address()) + log.verbose("Host %s (%s) is alive", host.name(), + host.address()) else: log.verbose("Host %s is alive", host.address()) + host.seen() # Configure if needed. if self.autoconfigure and \ - host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now(): + host.last_configured(default=datetime.datetime.min) + \ + self.config_delay < datetime.datetime.now(): self.config(host) def list(self): @@ -187,15 +195,19 @@ class Scanner(object): return hosts def config(self, host): - if host.name() == None: - log.debug("config - could not resolve name for %s, aborting.", host.address()) + if host.name() is None: + log.debug("config - could not resolve name for %s, aborting.", + host.address()) return previous_config_process = self.running_configs.get(host.name()) - if previous_config_process != None and previous_config_process.is_alive(): - log.debug("config - is already running for %s, aborting.", host.name()) + if previous_config_process is not None and \ + previous_config_process.is_alive(): + log.debug("config - is already running for %s, aborting.", + host.name()) - log.info("config - running against host %s (%s).", host.name(), host.address()) + log.info("config - running against host %s (%s).", host.name(), + host.address()) p = Process(target=host.configure()) p.start() self.running_configs[host.name()] = p @@ -214,4 +226,4 @@ class Scanner(object): filter="icmp6", prn=self.handle_pkg) except Exception as e: - log.error( "Could not start listener: %s", e) + log.error("Could not start listener: %s", e) From ab10b453f275d4289f3d37988b3caec204227194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 11:15:41 +0200 Subject: [PATCH 47/64] [scanner] populate cdist(1) --- docs/src/man1/cdist.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 0ecb4a61..599ec3b7 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -88,6 +88,9 @@ SYNOPSIS cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] + cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list] + [-d CONFIG_DELAY] [-t TRIGGER_DELAY] + DESCRIPTION ----------- @@ -641,6 +644,31 @@ Display information for cdist (global explorers, types). **-t, --types** Display info for types. +SCAN +---- + +Runs cdist as a daemon that discover/watch on hosts and reconfigure them +periodically. + +**-I INTERFACE, --interfaces INTERFACE** + Interface to listen on. Can be specified multiple times. + +**-m MODE, --mode MODE** + Scanner components to enable. Can be specified multiple time to enable more + than one component. Supported modes are: scan, trigger and config. Defaults + to tiggger and scan. + +**--name-mapper PATH_TO_SCRIPT** + Path to script used to resolve a remote host name from an IPv6 address. + +**--list** + List known hosts and exit. + +**-d CONFIG_DELAY, --config-delay CONFIG_DELAY** + How long (seconds) to wait before reconfiguring after last try (config mode only). + +**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY** + How long (seconds) to wait between ICMPv6 echo requests (trigger mode only). CONFIGURATION ------------- From b8733c65f52776facce7a8de0265538a856ead64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 11:26:35 +0200 Subject: [PATCH 48/64] [scanner] fix minor CLI handling and --list bugs / typo --- cdist/argparse.py | 4 ++-- cdist/scan/commandline.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index bedb23ac..f17315e7 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -495,7 +495,7 @@ def get_parsers(): action='store_true', help='Try to configure detected hosts') parser['scan'].add_argument( - '-I', '--interfaces', + '-I', '--interface', action='append', default=[], required=True, help='On which interfaces to scan/trigger') parser['scan'].add_argument( @@ -503,7 +503,7 @@ def get_parsers(): action='store', default=None, help='Map addresses to names, required for config mode') parser['scan'].add_argument( - '-d', '--delay', + '-d', '--config-delay', action='store', default=3600, type=int, help='How long (seconds) to wait before reconfiguring after last try') parser['scan'].add_argument( diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index b42bc7b2..ddbe4933 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -32,7 +32,7 @@ def run(scan, args): processes = [] if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interfaces, + t = scan.Trigger(interfaces=args.interface, sleeptime=args.trigger_delay) t.start() processes.append(t) @@ -41,7 +41,7 @@ def run(scan, args): if 'scan' in args.mode: s = scan.Scanner( autoconfigure='config' in args.mode, - interfaces=args.interfaces, + interfaces=args.interface, name_mapper=args.name_mapper) s.start() processes.append(s) @@ -52,7 +52,7 @@ def run(scan, args): def list(scan, args): - s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper) + s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper) hosts = s.list() # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + @@ -75,10 +75,10 @@ def list(scan, args): last_seen = '-' last_configured = host.last_configured() - if last_configured: + if last_configured is not None: last_configured = last_configured.strftime(scan.datetime_format) else: - '-' + last_configured = '-' print("{} | {} | {} | {}".format( host.name(default='-').ljust(name_max_size), From e0c52d0e1dfbaa3814a2ff26482177c2909a15ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 11:27:11 +0200 Subject: [PATCH 49/64] [scanner] remove mention of non-implemented trigger soruce script --- docs/src/cdist-scan.rst | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst index 02193456..064e65ff 100644 --- a/docs/src/cdist-scan.rst +++ b/docs/src/cdist-scan.rst @@ -8,7 +8,8 @@ periodically. It is especially useful in netboot-based environment where hosts boot unconfigured, and to ensure your infrastructure stays in sync with your configuration. -This feature is still consider to be in **beta** stage. +This feature is still consider to be in **beta** stage, and only operate on +IPv6 (including link-local). Usage (Examples) ---------------- @@ -79,21 +80,3 @@ Resolving name from `PTR` DNS record: done dig +short -x "$1" | sed -e 's/.$//' - - -Trigger Source Script ---------------------- - -This script returns a list of addresses (separated by a newline) to be used by -`trigger` mode. It is not used to map names. The script must be executable. - -Simplest script: - -.. code-block:: sh - #!/bin/sh - - cat << EOF - server1.domain.tld - server2.domain.tld - server3.domain.tld - EOF From defa3c22eaef88652a2bec6135d038a84710c41e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 29 May 2021 11:21:34 +0200 Subject: [PATCH 50/64] ++changelog --- docs/changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index a7310788..403e7653 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,7 +6,8 @@ next: * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) - * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking. + * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham) + * Type __git: Fix group explorer (Ander Punnar) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From d596986af894749877fb3dcadf90628c9cfa9d13 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 31 May 2021 09:06:52 +0200 Subject: [PATCH 51/64] [type/__pyvenv] Fix group explorer --- cdist/conf/type/__pyvenv/explorer/group | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group index f31a1cb7..922ce3df 100755 --- a/cdist/conf/type/__pyvenv/explorer/group +++ b/cdist/conf/type/__pyvenv/explorer/group @@ -14,6 +14,11 @@ then then printf '%u\n' "${group_gid}" else - printf '%s\n' "$(id -u -n "${group_gid}")" + if command -v getent >/dev/null 2>&1 + then + getent group "${group_gid}" | cut -d : -f 1 + else + awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group + fi fi fi From 6ede76b08b96b7504c65ca2dc6737e31be7e7f24 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 8 Jun 2021 16:20:55 +0200 Subject: [PATCH 52/64] [type/__debconf_set_selections] man.rst: Fix line break in AUTHORS --- cdist/conf/type/__debconf_set_selections/man.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst index 690e3e49..fd0040ae 100644 --- a/cdist/conf/type/__debconf_set_selections/man.rst +++ b/cdist/conf/type/__debconf_set_selections/man.rst @@ -58,8 +58,8 @@ SEE ALSO AUTHORS ------- -Nico Schottelius -Dennis Camera +| Nico Schottelius +| Dennis Camera COPYING From c308a2896971ac4808be0152630ef41b7f96a2de Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 10 Jun 2021 06:39:55 +0200 Subject: [PATCH 53/64] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 403e7653..97ea204b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham) * Type __git: Fix group explorer (Ander Punnar) + * Type __pyvenv: Fix group explorer (Dennis Camera) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 7b3f268df25922c515174862da95569aea547367 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 22 Jun 2021 16:36:30 +0300 Subject: [PATCH 54/64] [__download] improvements 1. post download checksum verification 2. detect hashes without prefix 3. add optional --destination 4. updated man --- .../conf/type/__download/explorer/remote_cmd | 19 --- .../type/__download/explorer/remote_cmd_get | 16 +++ .../type/__download/explorer/remote_cmd_sum | 82 +++++++++++ cdist/conf/type/__download/explorer/state | 60 ++------ cdist/conf/type/__download/gencode-local | 131 +++++++++++++++--- cdist/conf/type/__download/gencode-remote | 46 +++++- cdist/conf/type/__download/man.rst | 31 ++++- cdist/conf/type/__download/manifest | 2 +- cdist/conf/type/__download/parameter/optional | 3 +- 9 files changed, 292 insertions(+), 98 deletions(-) delete mode 100755 cdist/conf/type/__download/explorer/remote_cmd create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_get create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_sum diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd deleted file mode 100755 index e3e35b45..00000000 --- a/cdist/conf/type/__download/explorer/remote_cmd +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/cmd-get" ] -then - cmd="$( cat "$__object/parameter/cmd-get" )" - -elif command -v curl > /dev/null -then - cmd="curl -L -o - '%s'" - -elif command -v fetch > /dev/null -then - cmd="fetch -o - '%s'" - -else - cmd="wget -O - '%s'" -fi - -echo "$cmd" diff --git a/cdist/conf/type/__download/explorer/remote_cmd_get b/cdist/conf/type/__download/explorer/remote_cmd_get new file mode 100755 index 00000000..9f1cd59c --- /dev/null +++ b/cdist/conf/type/__download/explorer/remote_cmd_get @@ -0,0 +1,16 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/cmd-get" ] +then + cat "$__object/parameter/cmd-get" +elif + command -v curl > /dev/null +then + echo "curl -sSL -o - '%s'" +elif + command -v fetch > /dev/null +then + echo "fetch -o - '%s'" +else + echo "wget -O - '%s'" +fi diff --git a/cdist/conf/type/__download/explorer/remote_cmd_sum b/cdist/conf/type/__download/explorer/remote_cmd_sum new file mode 100755 index 00000000..84df663c --- /dev/null +++ b/cdist/conf/type/__download/explorer/remote_cmd_sum @@ -0,0 +1,82 @@ +#!/bin/sh -e + +if [ ! -f "$__object/parameter/sum" ] +then + exit 0 +fi + +if [ -f "$__object/parameter/cmd-sum" ] +then + cat "$__object/parameter/cmd-sum" + exit 0 +fi + +sum_should="$( cat "$__object/parameter/sum" )" + +if echo "$sum_should" | grep -Fq ':' +then + sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" +else + if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' + then + sum_hash='cksum' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' + then + sum_hash='md5' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' + then + sum_hash='sha1' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' + then + sum_hash='sha256' + else + echo 'hash format detection failed' >&2 + exit 1 + fi +fi + +os="$( "$__explorer/os" )" + +case "$sum_hash" in + cksum) + echo "cksum %s | awk '{print \$1\" \"\$2}'" + ;; + md5) + case "$os" in + freebsd) + echo "md5 -q %s" + ;; + *) + echo "md5sum %s | awk '{print \$1}'" + ;; + esac + ;; + sha1) + case "$os" in + freebsd) + echo "sha1 -q %s" + ;; + *) + echo "sha1sum %s | awk '{print \$1}'" + ;; + esac + ;; + sha256) + case "$os" in + freebsd) + echo "sha256 -q %s" + ;; + *) + echo "sha256sum %s | awk '{print \$1}'" + ;; + esac + ;; + *) + # we arrive here only if --sum is given with unknown format prefix + echo "unknown hash format: $sum_hash" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 68b517c5..881a1c09 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -1,6 +1,11 @@ #!/bin/sh -e -dst="/$__object_id" +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" +else + dst="/$__object_id" +fi if [ ! -f "$dst" ] then @@ -16,57 +21,18 @@ fi sum_should="$( cat "$__object/parameter/sum" )" -if [ -f "$__object/parameter/cmd-sum" ] +if echo "$sum_should" | grep -Fq ':' then - # shellcheck disable=SC2059 - sum_is="$( eval "$( printf \ - "$( cat "$__object/parameter/cmd-sum" )" \ - "$dst" )" )" -else - os="$( "$__explorer/os" )" - - if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' - then - sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )" - - elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$' - then - case "$os" in - freebsd) - sum_is="md5:$( md5 -q "$dst" )" - ;; - *) - sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )" - ;; - esac - - elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$' - then - case "$os" in - freebsd) - sum_is="sha1:$( sha1 -q "$dst" )" - ;; - *) - sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )" - ;; - esac - - elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$' - then - case "$os" in - freebsd) - sum_is="sha256:$( sha256 -q "$dst" )" - ;; - *) - sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )" - ;; - esac - fi + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" fi +sum_cmd="$( "$__type_explorer/remote_cmd_sum" )" + +sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )" + if [ -z "$sum_is" ] then - echo 'no checksum from target' >&2 + echo 'existing destination checksum failed' >&2 exit 1 fi diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local index 571d2c3c..d1b0d0d5 100755 --- a/cdist/conf/type/__download/gencode-local +++ b/cdist/conf/type/__download/gencode-local @@ -11,34 +11,133 @@ fi url="$( cat "$__object/parameter/url" )" -tmp="$( mktemp )" - -dst="/$__object_id" +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" +else + dst="/$__object_id" +fi if [ -f "$__object/parameter/cmd-get" ] then cmd="$( cat "$__object/parameter/cmd-get" )" -elif command -v wget > /dev/null -then - cmd="wget -O - '%s'" - elif command -v curl > /dev/null then - cmd="curl -L -o - '%s'" + cmd="curl -sSL -o - '%s'" elif command -v fetch > /dev/null then cmd="fetch -o - '%s'" +elif command -v wget > /dev/null +then + cmd="wget -O - '%s'" + else - echo 'no usable locally installed utility for downloading' >&2 + echo 'local download failed, no usable utility' >&2 exit 1 fi -printf "$cmd > %s\n" \ - "$url" \ - "$tmp" +echo "download_tmp=\"\$( mktemp )\"" + +# shellcheck disable=SC2059 +printf "$cmd > \"\$download_tmp\"\n" "$url" + +if [ -f "$__object/parameter/sum" ] +then + sum_should="$( cat "$__object/parameter/sum" )" + + if [ -f "$__object/parameter/cmd-sum" ] + then + local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" + else + if echo "$sum_should" | grep -Fq ':' + then + sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" + + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" + else + if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' + then + sum_hash='cksum' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' + then + sum_hash='md5' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' + then + sum_hash='sha1' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' + then + sum_hash='sha256' + else + echo 'hash format detection failed' >&2 + exit 1 + fi + fi + + case "$sum_hash" in + cksum) + local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'" + ;; + md5) + if command -v md5 > /dev/null + then + local_cmd_sum="md5 -q %s" + elif + command -v md5sum > /dev/null + then + local_cmd_sum="md5sum %s | awk '{print \$1}'" + fi + ;; + sha1) + if command -v sha1 > /dev/null + then + local_cmd_sum="sha1 -q %s" + elif + command -v sha1sum > /dev/null + then + local_cmd_sum="sha1sum %s | awk '{print \$1}'" + fi + ;; + sha256) + if command -v sha256 > /dev/null + then + local_cmd_sum="sha256 -q %s" + elif + command -v sha256sum > /dev/null + then + local_cmd_sum="sha256sum %s | awk '{print \$1}'" + fi + ;; + *) + # we arrive here only if --sum is given with unknown format prefix + echo "unknown hash format: $sum_hash" >&2 + exit 1 + ;; + esac + + if [ -z "$local_cmd_sum" ] + then + echo 'local checksum verification failed, no usable utility' >&2 + exit 1 + fi + fi + + # shellcheck disable=SC2059 + echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\"" + + echo "if [ \"\$sum_is\" != '$sum_should' ]; then" + + echo "echo 'local download checksum mismatch' >&2" + + echo "rm -f \"\$download_tmp\"" + + echo 'exit 1; fi' +fi if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$' then @@ -47,12 +146,10 @@ else target_host="$__target_host" fi -printf '%s %s %s:%s\n' \ +# shellcheck disable=SC2016 +printf '%s "$download_tmp" %s:%s\n' \ "$__remote_copy" \ - "$tmp" \ "$target_host" \ "$dst" -echo "rm -f '$tmp'" - -echo 'downloaded' > "$__messages_out" +echo "rm -f \"\$download_tmp\"" diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote index 029a0801..e49bcec3 100755 --- a/cdist/conf/type/__download/gencode-remote +++ b/cdist/conf/type/__download/gencode-remote @@ -6,17 +6,51 @@ state_is="$( cat "$__object/explorer/state" )" if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ] then - cmd="$( cat "$__object/explorer/remote_cmd" )" + cmd_get="$( cat "$__object/explorer/remote_cmd_get" )" url="$( cat "$__object/parameter/url" )" - dst="/$__object_id" + if [ -f "$__object/parameter/destination" ] + then + dst="$( cat "$__object/parameter/destination" )" + else + dst="/$__object_id" + fi - printf "$cmd > %s\n" \ - "$url" \ - "$dst" + echo "download_tmp=\"\$( mktemp )\"" - echo 'downloaded' > "$__messages_out" + # shellcheck disable=SC2059 + printf "$cmd_get > \"\$download_tmp\"\n" "$url" + + if [ -f "$__object/parameter/sum" ] + then + sum_should="$( cat "$__object/parameter/sum" )" + + if [ -f "$__object/parameter/cmd-sum" ] + then + remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" + else + remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )" + + if echo "$sum_should" | grep -Fq ':' + then + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" + fi + fi + + # shellcheck disable=SC2059 + echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\"" + + echo "if [ \"\$sum_is\" != '$sum_should' ]; then" + + echo "echo 'remote download checksum mismatch' >&2" + + echo "rm -f \"\$download_tmp\"" + + echo 'exit 1; fi' + fi + + echo "mv \"\$download_tmp\" '$dst'" fi if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ] diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index a1278cfb..c16510a9 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -8,7 +8,7 @@ cdist-type__download - Download a file DESCRIPTION ----------- -By default type will try to use ``wget``, ``curl`` or ``fetch``. +By default type will try to use ``curl``, ``fetch`` or ``wget``. If download happens in target (see ``--download``) then type will fallback to (and install) ``wget``. @@ -16,6 +16,8 @@ If download happens in local machine, then environment variables like ``{http,https,ftp}_proxy`` etc can be used on cdist execution (``http_proxy=foo cdist config ...``). +To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``. + REQUIRED PARAMETERS ------------------- @@ -25,14 +27,29 @@ url OPTIONAL PARAMETERS ------------------- +destination + Downloaded file's destination in target. If unset, ``$__object_id`` is used. + sum - Checksum is used to decide if existing destination file must be redownloaded. - By default output of ``cksum`` without filename is expected. - Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. + Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256. + + Type tries to detect hash format with regexes, but prefixes + ``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported. + + Checksum have two purposes - state check and post-download verification. + In state check, if destination checksum mismatches, then content of URL + will be downloaded to temporary file. If downloaded temporary file's + checksum matches, then it will be moved to destination (overwritten). + + For local downloads it is expected that usable utilities for checksum + calculation exist in the system. download - If ``local`` (default), then download file to local storage and copy - it to target host. If ``remote``, then download happens in target. + If ``local`` (default), then file is downloaded to local storage and copied + to target host. If ``remote``, then download happens in target. + + For local downloads it is expected that usable utilities for downloading + exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``. cmd-get Command used for downloading. @@ -62,7 +79,7 @@ EXAMPLES require='__directory/opt/cpma' \ __download /opt/cpma/cnq3.zip \ --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \ - --sum md5:46da3021ca9eace277115ec9106c5b46 + --sum 46da3021ca9eace277115ec9106c5b46 require='__download/opt/cpma/cnq3.zip' \ __unpack /opt/cpma/cnq3.zip \ diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest index 7ec8d86d..3d4c498b 100755 --- a/cdist/conf/type/__download/manifest +++ b/cdist/conf/type/__download/manifest @@ -1,6 +1,6 @@ #!/bin/sh -e -if grep -Eq '^wget' "$__object/explorer/remote_cmd" +if grep -Eq '^wget' "$__object/explorer/remote_cmd_get" then __package wget fi diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional index d69e083e..e809ef78 100644 --- a/cdist/conf/type/__download/parameter/optional +++ b/cdist/conf/type/__download/parameter/optional @@ -1,5 +1,6 @@ -sum cmd-get cmd-sum +destination download onchange +sum From 2db40d8d704b427768307fbea29384bd3dc8dbd7 Mon Sep 17 00:00:00 2001 From: fancsali Date: Mon, 28 Jun 2021 12:54:20 +0200 Subject: [PATCH 55/64] Use $__remote_exec and thus the ssh multiplexing --- cdist/conf/type/__rsync/gencode-local | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index e36ded2f..36addc36 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -36,4 +36,5 @@ fi echo rsync -a \ --no-owner --no-group \ + -e "$__remote_exec" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From d937d53f3dfd10830a07aee0450596eea62f2a1a Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Mon, 28 Jun 2021 18:09:35 +0100 Subject: [PATCH 56/64] Add quotes to rsync command --- cdist/conf/type/__rsync/gencode-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index 36addc36..f1bddc16 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -36,5 +36,5 @@ fi echo rsync -a \ --no-owner --no-group \ - -e "$__remote_exec" \ + -e \"$__remote_exec\" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From 60753ddfcc3ac49303d572da7f6a68f398c02227 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 1 Jul 2021 14:42:10 +0300 Subject: [PATCH 57/64] fix shellcheck --- cdist/conf/type/__download/explorer/state | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 881a1c09..8c5d5ce1 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -28,6 +28,7 @@ fi sum_cmd="$( "$__type_explorer/remote_cmd_sum" )" +# shellcheck disable=SC2059 sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )" if [ -z "$sum_is" ] From a90e642c1354cf01be7c9a1ba8a468ad624c4202 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 1 Jul 2021 14:50:40 +0300 Subject: [PATCH 58/64] update README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index de6901c7..a468dd86 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ For community-maintained types there is ## Participating -IRC: ``#cdist`` @ freenode +IRC: ``#cdist`` @ [libera](https://libera.chat) Matrix: ``#cdist:ungleich.ch`` -Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist +Matrix and IRC are bridged. From 243a4b904a2de638d00bac57f6e762a076f9ae54 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 2 Jul 2021 06:50:02 +0200 Subject: [PATCH 59/64] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 97ea204b..d48c657e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,8 @@ next: * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham) * Type __git: Fix group explorer (Ander Punnar) * Type __pyvenv: Fix group explorer (Dennis Camera) + * Type __download: Improve checksum verification, add optional --destination (Ander Punnar) + * Type __debconf_set_selections: Add state explorer (Dennis Camera) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 30ba796d060fd3d0c4affd4167f14784156b5354 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 1 Jul 2021 11:49:07 +0300 Subject: [PATCH 60/64] new type: __snakeoil_cert --- .../__snakeoil_cert/explorer/ssl-cert-group | 8 ++ .../conf/type/__snakeoil_cert/explorer/state | 24 ++++++ .../conf/type/__snakeoil_cert/gencode-remote | 73 +++++++++++++++++++ cdist/conf/type/__snakeoil_cert/man.rst | 60 +++++++++++++++ .../parameter/default/cert-path | 1 + .../parameter/default/key-path | 1 + .../parameter/default/key-type | 1 + .../type/__snakeoil_cert/parameter/optional | 4 + 8 files changed, 172 insertions(+) create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/state create mode 100755 cdist/conf/type/__snakeoil_cert/gencode-remote create mode 100644 cdist/conf/type/__snakeoil_cert/man.rst create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/cert-path create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-path create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-type create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/optional diff --git a/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group new file mode 100755 index 00000000..a6cb3dfd --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group @@ -0,0 +1,8 @@ +#!/bin/sh -e + +if grep -Eq '^ssl-cert:' /etc/group +then + echo 'present' +else + echo 'absent' +fi diff --git a/cdist/conf/type/__snakeoil_cert/explorer/state b/cdist/conf/type/__snakeoil_cert/explorer/state new file mode 100755 index 00000000..cc5aae0b --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/explorer/state @@ -0,0 +1,24 @@ +#!/bin/sh -e + +key_path="$( cat "$__object/parameter/key-path" )" + +if echo "$key_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + key_path="$( printf "$key_path" "$__object_id" )" +fi + +cert_path="$( cat "$__object/parameter/cert-path" )" + +if echo "$cert_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + cert_path="$( printf "$cert_path" "$__object_id" )" +fi + +if [ ! -f "$key_path" ] || [ ! -f "$cert_path" ] +then + echo 'absent' +else + echo 'present' +fi diff --git a/cdist/conf/type/__snakeoil_cert/gencode-remote b/cdist/conf/type/__snakeoil_cert/gencode-remote new file mode 100755 index 00000000..8ffbfad1 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/gencode-remote @@ -0,0 +1,73 @@ +#!/bin/sh -e + +state="$( cat "$__object/explorer/state" )" + +if [ "$state" = 'present' ] +then + exit 0 +fi + +if [ -f "$__object/parameter/common-name" ] +then + common_name="$( cat "$__object/parameter/common-name" )" +else + common_name="$__object_id" +fi + +key_path="$( cat "$__object/parameter/key-path" )" + +if echo "$key_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + key_path="$( printf "$key_path" "$__object_id" )" +fi + +cert_path="$( cat "$__object/parameter/cert-path" )" + +if echo "$cert_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + cert_path="$( printf "$cert_path" "$__object_id" )" +fi + +key_type="$( cat "$__object/parameter/key-type" )" + +key_type_arg="$( echo "$key_type" | cut -d : -f 2 )" + +case "$key_type" in + rsa:*) + echo "openssl genrsa -out '$key_path' $key_type_arg" + ;; + ec:*) + echo "openssl ecparam -name $key_type_arg -genkey -noout -out '$key_path'" + ;; +esac + +# shellcheck disable=SC2016 +echo 'csr_path="$( mktemp )"' + +echo "openssl req -new -subj '/CN=$common_name' -key '$key_path' -out \"\$csr_path\"" + +echo "openssl x509 -req -sha256 -days 3650 -in \"\$csr_path\" -signkey '$key_path' -out '$cert_path'" + +# shellcheck disable=SC2016 +echo 'rm -f "$csr_path"' + +if [ "$( cat "$__object/explorer/ssl-cert-group" )" = 'present' ] +then + key_group='ssl-cert' +else + key_group='root' +fi + +echo "chmod 640 '$key_path'" + +echo "chown root '$key_path'" + +echo "chgrp $key_group '$key_path'" + +echo "chmod 644 '$cert_path'" + +echo "chown root '$cert_path'" + +echo "chgrp root '$cert_path'" diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst new file mode 100644 index 00000000..0b547804 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/man.rst @@ -0,0 +1,60 @@ +cdist-type__snakeoil_cert(7) +============================ + +NAME +---- +cdist-type__snakeoil_cert - Generate self-signed certificate + + +DESCRIPTION +----------- +The purpose of this type is to generate **self-signed** certificate and private key +for **testing purposes**. Certificate will expire in 3650 days. + +Certificate's and key's access bits will be ``644`` and ``640`` respectively. +If target system has ``ssl-cert`` group, then it will be used as key's group. +Use ``require='__snakeoil_cert/...' __file ...`` to override. + + +OPTIONAL PARAMETERS +------------------- +common-name + Defaults to ``$__object_id``. + +key-path + ``%s`` in path will be replaced with ``$__object_id``. + Defaults to ``/etc/ssl/private/%s.pem``. + +key-type + Possible values are ``rsa:$bits`` and ``ec:$name``. + For possible EC names see ``openssl ecparam -list_curves``. + Defaults to ``rsa:2048``. + +cert-path + ``%s`` in path will be replaced with ``$__object_id``. + Defaults to ``/etc/ssl/certs/%s.pem``. + + +EXAMPLES +-------- +.. code-block:: sh + __snakeoil_cert localhost-rsa \ + --common-name localhost \ + --key-type rsa:4096 + + __snakeoil_cert localhost-ec \ + --common-name localhost \ + --key-type ec:prime256v1 + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path new file mode 100644 index 00000000..4bbae089 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path @@ -0,0 +1 @@ +/etc/ssl/certs/%s.pem diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-path b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path new file mode 100644 index 00000000..86eb9359 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path @@ -0,0 +1 @@ +/etc/ssl/private/%s.pem diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-type b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type new file mode 100644 index 00000000..f13f8ada --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type @@ -0,0 +1 @@ +rsa:2048 diff --git a/cdist/conf/type/__snakeoil_cert/parameter/optional b/cdist/conf/type/__snakeoil_cert/parameter/optional new file mode 100644 index 00000000..76d08c0a --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/optional @@ -0,0 +1,4 @@ +common-name +key-path +key-type +cert-path From 853e5cf7b4f615e1470c7ec2cfa8fe68cfa85ce7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 5 Jul 2021 09:07:06 +0200 Subject: [PATCH 61/64] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index d48c657e..0c3c64e1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -11,6 +11,8 @@ next: * Type __pyvenv: Fix group explorer (Dennis Camera) * Type __download: Improve checksum verification, add optional --destination (Ander Punnar) * Type __debconf_set_selections: Add state explorer (Dennis Camera) + * Core: Implement usable cdist scan (Timothée Floure) + * New type: __snakeoil_cert (Ander Punnar) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From be92731c5c8a8543448f0d87fafae67e22ac76a1 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Mon, 5 Jul 2021 12:38:26 +0100 Subject: [PATCH 62/64] Shell check quoting We're actually echo-ing the command, hence the escape in front of the quotes - the issue Shellcheck alludes too would actually occur, had the escaping bakcslashes been omitted. --- cdist/conf/type/__rsync/gencode-local | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index f1bddc16..be4feabb 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -34,7 +34,8 @@ if [ -f "$__object/parameter/rsync-opts" ]; then done < "$__object/parameter/rsync-opts" fi +# shellcheck disable=SC2086 echo rsync -a \ --no-owner --no-group \ - -e \"$__remote_exec\" \ + -e \"${__remote_exec}\" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From 3e76d1cd3fbe53fa77c460cb2ce8416698b101bd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 8 Jul 2021 08:09:05 +0200 Subject: [PATCH 63/64] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 0c3c64e1..04f826f0 100644 --- a/docs/changelog +++ b/docs/changelog @@ -13,6 +13,7 @@ next: * Type __debconf_set_selections: Add state explorer (Dennis Camera) * Core: Implement usable cdist scan (Timothée Floure) * New type: __snakeoil_cert (Ander Punnar) + * Type __rsync: Honour $__remote_exec env var (Daniel Fancsali) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 77dab4c5c63070aef962875af7fe8b1565f5ba78 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 10 Jul 2021 20:37:02 +0200 Subject: [PATCH 64/64] Release 6.9.7 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 04f826f0..284293c1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham)