From 932e2496ed87b830a468a263cf1b1f5e6a88e2ef Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 15 Dec 2020 18:40:39 +0100 Subject: [PATCH 001/206] [type/__postgres_role] Lint --- .../conf/type/__postgres_role/explorer/state | 32 ++++---- .../conf/type/__postgres_role/gencode-remote | 78 ++++++++++--------- 2 files changed, 58 insertions(+), 52 deletions(-) diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index c8e1fa9d..110d29d5 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -11,32 +11,32 @@ # # cdist is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -case "$("${__explorer}/os")" +case $("${__explorer:?}/os") in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; + (netbsd) + postgres_user='pgsql' + ;; + (openbsd) + postgres_user='_postgresql' + ;; + (*) + postgres_user='postgres' + ;; esac -name="$__object_id" +rolename=${__object_id:?} -if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='$name'\"")" +if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='${rolename}'\"")" then - echo 'present' + echo 'present' else - echo 'absent' + echo 'absent' fi diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 282294c9..4fb761ed 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -11,55 +11,61 @@ # # cdist is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -case "$(cat "${__global}/explorer/os")" +case $(cat "${__global:?}/explorer/os") in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; + (netbsd) + postgres_user='pgsql' + ;; + (openbsd) + postgres_user='_postgresql' + ;; + (*) + postgres_user='postgres' + ;; esac -name="$__object_id" -state_is="$(cat "$__object/explorer/state")" -state_should="$(cat "$__object/parameter/state")" +rolename=${__object_id:?} +state_is=$(cat "${__object:?}/explorer/state") +state_should=$(cat "${__object:?}/parameter/state") -[ "$state_is" = "$state_should" ] && exit 0 +if test "${state_is}" = "${state_should}" +then + exit 0 +fi -case "$state_should" in - present) - if [ -f "$__object/parameter/password" ]; then - password="$(cat "$__object/parameter/password")" - fi - booleans="" - for boolean in login createdb createrole superuser; do - if [ ! -f "$__object/parameter/$boolean" ]; then - boolean="no${boolean}" - fi - upper=$(echo $boolean | tr '[:lower:]' '[:upper:]') - booleans="$booleans $upper" - done +case ${state_should} +in + (present) + if test -f "${__object:?}/parameter/password" + then + password=$(cat "${__object:?}/parameter/password") + fi + booleans= + for boolean in login createdb createrole superuser + do + if test ! -f "${__object:?}/parameter/${boolean}" + then + boolean="no${boolean}" + fi + booleans="${booleans} $(echo ${boolean} | tr '[:lower:]' '[:upper:]')" + done - [ -n "$password" ] && password="PASSWORD '$password'" - cat << EOF -su - '$postgres_user' -c "psql postgres -wc \"CREATE ROLE \\\\\"$name\\\\\" WITH $password $booleans;\"" + [ -n "${password}" ] && password="PASSWORD '${password}'" + cat << EOF +su - '${postgres_user}' -c "psql postgres -wc 'CREATE ROLE \\"${rolename}\\" WITH ${password} ${booleans};'" EOF - ;; - absent) - cat << EOF -su - '$postgres_user' -c "dropuser \"$name\"" + ;; + (absent) + cat << EOF +su - '${postgres_user}' -c "dropuser '${rolename}'" EOF - ;; + ;; esac From c36df82882fedb6ce3630e4718c46992a7c153a0 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 15 Dec 2020 21:05:55 +0100 Subject: [PATCH 002/206] [type/__postgres_role] ALTER ROLE when parameters change --- .../conf/type/__postgres_role/explorer/state | 42 +++++++++++- .../conf/type/__postgres_role/gencode-remote | 65 +++++++++++++++---- 2 files changed, 91 insertions(+), 16 deletions(-) diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 110d29d5..033afcb2 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -31,11 +32,48 @@ in ;; esac - rolename=${__object_id:?} -if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='${rolename}'\"")" +role_properties=$( + cmd=$(printf "psql -F '\034' -R '\036' -wAc \"SELECT * FROM pg_roles WHERE rolname='%s'\"" "${rolename}") + su -l "${postgres_user}" -c "${cmd}" \ + | awk ' + BEGIN { RS = "\036"; FS = "\034" } + /^\([0-9]+ rows?\)/ { exit } + NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next } + NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i } + ' +) + +if test -n "${role_properties}" then + # Check if the user's properties match the parameters + for prop in login createdb createrole superuser + do + bool_should=$(test -f "${__object:?}/parameter/${prop}" && echo 't' || echo 'f') + bool_is=$( + printf '%s\n' "${role_properties}" | + awk -F '=' -v key="${prop}" ' + BEGIN { + if (key == "login") + key = "canlogin" + else if (key == "superuser") + key = "super" + key = "rol" key + } + $1 == key { + sub(/^[^=]*=/, "") + print + } + ' + ) + + test "${bool_is}" = "${bool_should}" || { + echo 'different' + exit 0 + } + done + echo 'present' else echo 'absent' diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 4fb761ed..7837976c 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,6 +19,15 @@ # along with cdist. If not, see . # +quote() { + if test $# -gt 0 + then + printf '%s' "$*" + else + cat - + fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" +} + case $(cat "${__global:?}/explorer/os") in (netbsd) @@ -44,28 +54,55 @@ fi case ${state_should} in (present) - if test -f "${__object:?}/parameter/password" + if test -s "${__object:?}/parameter/password" then - password=$(cat "${__object:?}/parameter/password") + quoted_password=$( + delim='$$' + while grep -q -F "${delim}" "${__object:?}/parameter/password" + do + delim="\$$(LC_ALL=C tr -cd '[:alpha:]' /dev/null)$" + done + + raw_passwd=$(cat "${__object:?}/parameter/password"; printf .) + # shellcheck disable=SC2016 + printf '%s%s%s' "${delim}" "${raw_passwd%?.}" "${delim}" + ) fi + booleans= for boolean in login createdb createrole superuser do - if test ! -f "${__object:?}/parameter/${boolean}" - then - boolean="no${boolean}" - fi - booleans="${booleans} $(echo ${boolean} | tr '[:lower:]' '[:upper:]')" + booleans="${booleans}${booleans:+ }$( + if test -f "${__object:?}/parameter/${boolean}" + then + echo "${boolean}" + else + echo "no${boolean}" + fi \ + | tr '[:lower:]' '[:upper:]')" done - [ -n "${password}" ] && password="PASSWORD '${password}'" - cat << EOF -su - '${postgres_user}' -c "psql postgres -wc 'CREATE ROLE \\"${rolename}\\" WITH ${password} ${booleans};'" -EOF + case ${state_is} + in + (absent) + query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \ + "${rolename}" "${booleans}" "${quoted_password:-NULL}") + ;; + (different) + query=$(printf 'ALTER ROLE "%s" WITH %s PASSWORD %s;' \ + "${rolename}" "${booleans}" "${quoted_password:-NULL}") + ;; + (*) + exit 1 # TODO: error msg + ;; + esac + + psql_cmd=$(printf 'psql postgres -wc %s' "$(quote "${query}")" | quote) + printf "su -l '%s' -c %s\\n" "${postgres_user}" "${psql_cmd}" ;; (absent) - cat << EOF -su - '${postgres_user}' -c "dropuser '${rolename}'" -EOF + printf "su -l '%s' -c 'dropuser '\\\\'%s\\\\'\\n" \ + "${postgres_user}" \ + "$(quote "${rolename}")" ;; esac From 7b7ca4d385ccd025cbefced58419468651652da8 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 16 Dec 2020 19:06:50 +0100 Subject: [PATCH 003/206] [type/__postgres_role] Handle password changes --- .../conf/type/__postgres_role/explorer/state | 76 +++++++++++++++++-- .../conf/type/__postgres_role/gencode-remote | 23 ++++-- 2 files changed, 88 insertions(+), 11 deletions(-) diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 033afcb2..4aadbdff 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -34,9 +34,23 @@ esac rolename=${__object_id:?} + +psql_query() { + su -l "${postgres_user}" -c "$( + printf "psql -F '\034' -R '\036' -wAc '%s'" \ + "$(printf %s "$*" | sed "s/'/'\\\\''/g")" + )" +} + +password_check_login() ( + PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .) + PGPASSWORD=${PGPASSWORD%?.} + export PGPASSWORD + psql -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1 +) + role_properties=$( - cmd=$(printf "psql -F '\034' -R '\036' -wAc \"SELECT * FROM pg_roles WHERE rolname='%s'\"" "${rolename}") - su -l "${postgres_user}" -c "${cmd}" \ + psql_query "SELECT * FROM pg_roles WHERE rolname = '${rolename}'" \ | awk ' BEGIN { RS = "\036"; FS = "\034" } /^\([0-9]+ rows?\)/ { exit } @@ -69,12 +83,62 @@ then ) test "${bool_is}" = "${bool_should}" || { - echo 'different' - exit 0 + state='different properties' } done - echo 'present' + # Check password + passwd_stored=$( + psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \ + | awk 'BEGIN { RS = "\036" } NR == 2' + printf . + ) + passwd_stored=${passwd_stored%?.} + + passwd_should=$(cat "${__object}/parameter/password"; printf .) + passwd_should=${passwd_should%?.} + + if expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null + then + # SCRAM-SHA-256 "encrypted" password + # NOTE: There is currently no easy way to check SCRAM passwords + password_check_login || state="${state:-different} password" + elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null + then + # MD5 "encrypted" password + if command -v md5sum >/dev/null 2>&1 + then + should_md5=$( + printf '%s%s' "${passwd_should}" "${rolename}" \ + | md5sum - | sed -e 's/[^0-9a-f]*$//') + elif command -v gmd5sum >/dev/null 2>&1 + then + should_md5=$( + printf '%s%s' "${passwd_should}" "${rolename}" \ + | gmd5sum - | sed -e 's/[^0-9a-f]*$//') + elif command -v openssl >/dev/null 2>&1 + then + should_md5=$( + printf '%s%s' "${passwd_should}" "${rolename}" \ + | openssl dgst -md5 | sed 's/^.* //') + fi + + if test -n "${should_md5}" + then + test "${passwd_stored}" = "md5${should_md5}" \ + || state="${state:-different} password" + else + password_check_login || state="${state:-different} password" + fi + else + # unencrypted password (unsupported since PostgreSQL 10) + test "${passwd_stored}" = "${passwd_should}" \ + || state="${state:-different} password" + fi + + test -n "${state}" || state='present' else - echo 'absent' + state='absent' fi + +echo "${state}" diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 7837976c..fd590a4b 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -58,7 +58,9 @@ in then quoted_password=$( delim='$$' - while grep -q -F "${delim}" "${__object:?}/parameter/password" + # NOTE: Strip away trailing $ because with it the check breaks + # if the password ends with $ + random value. + while grep -q -F "${delim%$}" "${__object:?}/parameter/password" do delim="\$$(LC_ALL=C tr -cd '[:alpha:]' /dev/null)$" done @@ -88,12 +90,23 @@ in query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \ "${rolename}" "${booleans}" "${quoted_password:-NULL}") ;; - (different) - query=$(printf 'ALTER ROLE "%s" WITH %s PASSWORD %s;' \ - "${rolename}" "${booleans}" "${quoted_password:-NULL}") + (different*) + query="ALTER ROLE \"${rolename}\" WITH" + + if expr "${state_is}" : 'different.*properties' >/dev/null + then + query="${query} ${booleans}" + fi + if expr "${state_is}" : 'different.*password' >/dev/null + then + query="${query} PASSWORD ${quoted_password:-NULL}" + fi + + query="${query};" ;; (*) - exit 1 # TODO: error msg + printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2 + exit 1 ;; esac From 4859c27900d125841c703e0270dd8c0eae05d504 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 17 Dec 2020 16:57:03 +0100 Subject: [PATCH 004/206] [type/__postgres_role] Refactor gencode-remote --- .../conf/type/__postgres_role/gencode-remote | 65 ++++++++++--------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index fd590a4b..540eb606 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -51,9 +51,6 @@ then exit 0 fi -case ${state_should} -in - (present) if test -s "${__object:?}/parameter/password" then quoted_password=$( @@ -64,6 +61,11 @@ in do delim="\$$(LC_ALL=C tr -cd '[:alpha:]' /dev/null)$" done +psql_query() { + printf 'su -l %s -c %s\n' \ + "$(quote "${postgres_user}")" \ + "$(quote "psql postgres -wc $(quote "$1")")" +} raw_passwd=$(cat "${__object:?}/parameter/password"; printf .) # shellcheck disable=SC2016 @@ -71,51 +73,52 @@ in ) fi - booleans= - for boolean in login createdb createrole superuser - do - booleans="${booleans}${booleans:+ }$( - if test -f "${__object:?}/parameter/${boolean}" - then - echo "${boolean}" - else - echo "no${boolean}" - fi \ - | tr '[:lower:]' '[:upper:]')" - done +role_properties_should() { + _props= + for _prop in login createdb createrole superuser + do + _props="${_props}${_props:+ }$( + if test -f "${__object:?}/parameter/${_prop}" + then + echo "${_prop}" + else + echo "no${_prop}" + fi \ + | tr '[:lower:]' '[:upper:]')" + done + printf '%s\n' "${_props}" + unset _prop _props +} +case ${state_should} +in + (present) case ${state_is} in (absent) - query=$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \ - "${rolename}" "${booleans}" "${quoted_password:-NULL}") + psql_query "$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \ + "${rolename}" "$(role_properties_should)" "${quoted_password:-NULL}")" ;; (different*) - query="ALTER ROLE \"${rolename}\" WITH" - if expr "${state_is}" : 'different.*properties' >/dev/null then - query="${query} ${booleans}" - fi - if expr "${state_is}" : 'different.*password' >/dev/null - then - query="${query} PASSWORD ${quoted_password:-NULL}" + psql_query "ALTER ROLE \"${rolename}\" WITH $(role_properties_should);" fi - query="${query};" + if expr "${state_is}" : 'different.*password' >/dev/null + then + psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD ${quoted_password:-NULL};" + fi ;; (*) printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2 exit 1 ;; esac - - psql_cmd=$(printf 'psql postgres -wc %s' "$(quote "${query}")" | quote) - printf "su -l '%s' -c %s\\n" "${postgres_user}" "${psql_cmd}" ;; (absent) - printf "su -l '%s' -c 'dropuser '\\\\'%s\\\\'\\n" \ - "${postgres_user}" \ - "$(quote "${rolename}")" + printf 'su -l %s -c %s\n' \ + "$(quote "${postgres_user}")" \ + "$(quote "dropuser $(quote "${rolename}")")" ;; esac From 1180f13ed6c12e0277bebdf32bdd9840320d0fb9 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 17 Dec 2020 16:58:32 +0100 Subject: [PATCH 005/206] [type/__postgres_role] Fix setting password We need to make sure that the password does not end up in ~/.psql_history. --- .../conf/type/__postgres_role/gencode-remote | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 540eb606..15c3e692 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -51,27 +51,29 @@ then exit 0 fi - if test -s "${__object:?}/parameter/password" - then - quoted_password=$( - delim='$$' - # NOTE: Strip away trailing $ because with it the check breaks - # if the password ends with $ + random value. - while grep -q -F "${delim%$}" "${__object:?}/parameter/password" - do - delim="\$$(LC_ALL=C tr -cd '[:alpha:]' /dev/null)$" - done psql_query() { printf 'su -l %s -c %s\n' \ "$(quote "${postgres_user}")" \ "$(quote "psql postgres -wc $(quote "$1")")" } - raw_passwd=$(cat "${__object:?}/parameter/password"; printf .) - # shellcheck disable=SC2016 - printf '%s%s%s' "${delim}" "${raw_passwd%?.}" "${delim}" - ) - fi +psql_set_password() { + # NOTE: Always make sure that the password does not end up in psql_history! + if test -s "${__object:?}/parameter/password" + then + cat <<-EOF + exec 3< "\${__object:?}/parameter/password" + su -l '${postgres_user}' -c 'psql -q postgres -w' <<'SQL' + \set HISTFILE /dev/null + \set pw \`cat <&3\` + ALTER ROLE "${rolename}" WITH PASSWORD :'pw'; + SQL + exec 3<&- + EOF + else + psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD NULL;" + fi +} role_properties_should() { _props= @@ -96,8 +98,8 @@ in case ${state_is} in (absent) - psql_query "$(printf 'CREATE ROLE "%s" WITH %s PASSWORD %s;' \ - "${rolename}" "$(role_properties_should)" "${quoted_password:-NULL}")" + psql_query "CREATE ROLE \"${rolename}\" WITH $(role_properties_should);" + psql_set_password ;; (different*) if expr "${state_is}" : 'different.*properties' >/dev/null @@ -107,7 +109,7 @@ in if expr "${state_is}" : 'different.*password' >/dev/null then - psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD ${quoted_password:-NULL};" + psql_set_password fi ;; (*) From 99d82fd0d5ab6d6ccd95fe8a7d3b24f5d82a4cce Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 17 Dec 2020 17:05:58 +0100 Subject: [PATCH 006/206] [type/__postgres_role] Always set psql -q --- cdist/conf/type/__postgres_role/explorer/state | 4 ++-- cdist/conf/type/__postgres_role/gencode-remote | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 4aadbdff..77779662 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -37,7 +37,7 @@ rolename=${__object_id:?} psql_query() { su -l "${postgres_user}" -c "$( - printf "psql -F '\034' -R '\036' -wAc '%s'" \ + printf "psql -q -F '\034' -R '\036' -wAc '%s'" \ "$(printf %s "$*" | sed "s/'/'\\\\''/g")" )" } @@ -46,7 +46,7 @@ password_check_login() ( PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .) PGPASSWORD=${PGPASSWORD%?.} export PGPASSWORD - psql -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1 + psql -q -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1 ) role_properties=$( diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 15c3e692..877c8135 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -54,7 +54,7 @@ fi psql_query() { printf 'su -l %s -c %s\n' \ "$(quote "${postgres_user}")" \ - "$(quote "psql postgres -wc $(quote "$1")")" + "$(quote "psql postgres -q -w -c $(quote "$1")")" } psql_set_password() { @@ -63,7 +63,7 @@ psql_set_password() { then cat <<-EOF exec 3< "\${__object:?}/parameter/password" - su -l '${postgres_user}' -c 'psql -q postgres -w' <<'SQL' + su -l '${postgres_user}' -c 'psql -q -w postgres' <<'SQL' \set HISTFILE /dev/null \set pw \`cat <&3\` ALTER ROLE "${rolename}" WITH PASSWORD :'pw'; From 2954347771e00610a084c7b033016765f7a3d6d7 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 14 Jan 2021 13:46:40 +0100 Subject: [PATCH 007/206] [type/__postgres_role] Add note regarding empty passwords --- cdist/conf/type/__postgres_role/gencode-remote | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 877c8135..d7631fbd 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -59,6 +59,8 @@ psql_query() { psql_set_password() { # NOTE: Always make sure that the password does not end up in psql_history! + # NOTE: Never set an empty string as the password, because they can be + # interpreted differently by different tooling. if test -s "${__object:?}/parameter/password" then cat <<-EOF From 35cde3e666c229ceb94a1c5074083f7ba905768a Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 18 Jan 2021 13:09:29 +0100 Subject: [PATCH 008/206] [type/__postgres_role] Fix state explorer when stored password is empty --- cdist/conf/type/__postgres_role/explorer/state | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 77779662..34069de9 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -95,13 +95,20 @@ then ) passwd_stored=${passwd_stored%?.} - passwd_should=$(cat "${__object}/parameter/password"; printf .) + if test -f "${__object:?}/parameter/password" + then + passwd_should=$(cat "${__object:?}/parameter/password"; printf .) + fi passwd_should=${passwd_should%?.} - if expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null + if test -z "${passwd_stored}" + then + test -z "${passwd_should}" || state="${state:-different} password" + elif expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null then # SCRAM-SHA-256 "encrypted" password - # NOTE: There is currently no easy way to check SCRAM passwords + # NOTE: There is currently no easy way to check SCRAM passwords without + # logging in password_check_login || state="${state:-different} password" elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null then From c51d68a7375915154a90c22e716e832f635ca878 Mon Sep 17 00:00:00 2001 From: Beni Ruef Date: Thu, 3 Dec 2020 18:48:04 +0100 Subject: [PATCH 009/206] [type/__postgres_conf] New type based on ALTER SYSTEM command --- .../conf/type/__postgres_conf/gencode-remote | 92 +++++++++++++++++++ cdist/conf/type/__postgres_conf/man.rst | 55 +++++++++++ .../__postgres_conf/parameter/default/state | 1 + .../type/__postgres_conf/parameter/optional | 2 + 4 files changed, 150 insertions(+) create mode 100755 cdist/conf/type/__postgres_conf/gencode-remote create mode 100644 cdist/conf/type/__postgres_conf/man.rst create mode 100644 cdist/conf/type/__postgres_conf/parameter/default/state create mode 100644 cdist/conf/type/__postgres_conf/parameter/optional diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote new file mode 100755 index 00000000..a09e1873 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -0,0 +1,92 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$(cat "${__global}/explorer/os") +state_should=$(cat "${__object}/parameter/state") + +if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ] +then + echo "Invalid state '${state_should}'." \ + 'Only "present" and "absent" are acceptable' >&2 + exit 1 +fi + +if [ "${state_should}" = 'present' ] && [ ! -f "${__object}/parameter/value" ] +then + echo 'Missing required parameter "value"' >&2 + exit 1 +fi + +# Parameters +conf_name="${__object_id}" +if [ -f "${__object}/parameter/value" ] +then + conf_value=$(cat "${__object}/parameter/value") +fi + +if [ "${state_should}" = 'present' ] +then + set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'" + check_command="SHOW ${conf_name}" +else + set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT" +fi + +case $os +in + openbsd|devuan) + case $os + in + openbsd) + postgres_user='_postgresql' + restart_command='/etc/rc.d/postgresql restart' + ;; + devuan) + postgres_user='postgres' + restart_command='/etc/init.d/postgresql restart' + ;; + esac + # needs two separate psql commands because ALTER SYSTEM + # cannot run inside a transaction block + cat <<-EOF + su - ${postgres_user} -c "psql postgres -twAc \ + \"${set_command}\"" + su - ${postgres_user} -c "psql postgres -twAc \ + \"SELECT pg_reload_conf()\"" + EOF + # check success (makes only sense if setting to a non-default value) + # and restart server if needed + if [ "${state_should}" = 'present' ] + then + cat <<-EOF + if [ \$(su - ${postgres_user} -c "psql postgres -twAc \"SHOW ${conf_name}\"") != '${conf_value}' ] + then + ${restart_command} + fi + EOF + fi + ;; + *) + echo "Unsupported OS: ${os}" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst new file mode 100644 index 00000000..76016c6c --- /dev/null +++ b/cdist/conf/type/__postgres_conf/man.rst @@ -0,0 +1,55 @@ +cdist-type__postgres_conf(7) +============================ + +Configure a PostgreSQL server. + +NOTE: This type might need to be run multiple times to apply all bits of the +configuration due to ordering requirements. + +SSRQ + + +DESCRIPTION +----------- +Configure a PostgreSQL server using ALTER SYSTEM. + + +REQUIRED PARAMETERS +------------------- +value + The value to setup (can be omitted when state is set to "absent"). + + +OPTIONAL PARAMETERS +------------------- +state + "present" or "absent". Defaults to "present". + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # set timezone + __postgres_conf timezone --value Europe/Zurich + + # reset maximum number of concurrent connections to default (normally 100) + __postgres_conf max_connections --state absent + + +SEE ALSO +-------- +- `cdist-type(7) `_ + + +COPYING +------- +Copyright \(C) 2020 SSRQ (www.ssrq-sds-fds.ch). +Free use of this software is granted under the terms +of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__postgres_conf/parameter/default/state b/cdist/conf/type/__postgres_conf/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__postgres_conf/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__postgres_conf/parameter/optional b/cdist/conf/type/__postgres_conf/parameter/optional new file mode 100644 index 00000000..d0460d86 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/parameter/optional @@ -0,0 +1,2 @@ +state +value From 534d5f6bb5dd3d8e9cb2256e75ed182d6530a892 Mon Sep 17 00:00:00 2001 From: Beni Ruef Date: Fri, 4 Dec 2020 14:31:04 +0100 Subject: [PATCH 010/206] [type/__postgres_conf] Fix errors found by ShellCheck --- cdist/conf/type/__postgres_conf/gencode-remote | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index a09e1873..e25514d0 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -20,8 +20,8 @@ # along with cdist. If not, see . # -os=$(cat "${__global}/explorer/os") -state_should=$(cat "${__object}/parameter/state") +os=$(cat "${__global:?}/explorer/os") +state_should=$(cat "${__object:?}/parameter/state") if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ] then @@ -37,7 +37,7 @@ then fi # Parameters -conf_name="${__object_id}" +conf_name="${__object_id:?}" if [ -f "${__object}/parameter/value" ] then conf_value=$(cat "${__object}/parameter/value") @@ -46,7 +46,6 @@ fi if [ "${state_should}" = 'present' ] then set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'" - check_command="SHOW ${conf_name}" else set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT" fi From 50bcd951055d7599ec340b4dea80a3a5d11dd9b0 Mon Sep 17 00:00:00 2001 From: Beni Ruef Date: Thu, 10 Dec 2020 11:45:32 +0100 Subject: [PATCH 011/206] [type/__postgres_conf] Remove faulty quotes --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index e25514d0..8ccb3b42 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -45,7 +45,7 @@ fi if [ "${state_should}" = 'present' ] then - set_command="ALTER SYSTEM SET ${conf_name} = '${conf_value}'" + set_command="ALTER SYSTEM SET ${conf_name} = ${conf_value}" else set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT" fi From b4060720dc3c7cc58aa992285877e4c009caa6be Mon Sep 17 00:00:00 2001 From: Beni Ruef Date: Fri, 11 Dec 2020 11:12:16 +0100 Subject: [PATCH 012/206] [type/__postgres_conf] Fix psql options for ALTER command --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 8ccb3b42..7d86028e 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -67,7 +67,7 @@ in # needs two separate psql commands because ALTER SYSTEM # cannot run inside a transaction block cat <<-EOF - su - ${postgres_user} -c "psql postgres -twAc \ + su - ${postgres_user} -c "psql postgres -qwc \ \"${set_command}\"" su - ${postgres_user} -c "psql postgres -twAc \ \"SELECT pg_reload_conf()\"" From 1b49fec972ce5f486f2794571c8e892d1fa6cb51 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 18 Jan 2021 19:15:49 +0100 Subject: [PATCH 013/206] [type/__postgres_conf] Refactor --- .../type/__postgres_conf/explorer/context | 42 ++++++ .../conf/type/__postgres_conf/explorer/state | 60 ++++++++ .../conf/type/__postgres_conf/gencode-remote | 135 ++++++++++-------- 3 files changed, 180 insertions(+), 57 deletions(-) create mode 100644 cdist/conf/type/__postgres_conf/explorer/context create mode 100644 cdist/conf/type/__postgres_conf/explorer/state diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context new file mode 100644 index 00000000..3a2fa504 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/explorer/context @@ -0,0 +1,42 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Returns the "context" of the configuration setting. +# cf. also https://www.postgresql.org/docs/10/view-pg-settings.html + +os=$("${__explorer:?}/os") + +case ${os} +in + (openbsd) + postgres_user='_postgresql' + ;; + (devuan) + postgres_user='postgres' + ;; + (*) + echo "Unsupported OS: ${os}" >&2 + exit 1 + ;; +esac + +conf_name=${__object_id:?} + +su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE name = '${conf_name}'\"" diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state new file mode 100644 index 00000000..da904b56 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -0,0 +1,60 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$("${__explorer:?}/os") + +case ${os} +in + (openbsd) + postgres_user='_postgresql' + ;; + (devuan) + postgres_user='postgres' + ;; + (*) + echo "Unsupported OS: ${os}" >&2 + exit 1 + ;; +esac + +conf_name=${__object_id:?} + +if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ + | cmp -s "${__object:?}/parameter/value" - +then + echo present +else + case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT source FROM pg_settings WHERE name = '${conf_name}'\"") + in + ('') + # invalid configuration parameter + # (error message was already printed by SHOW command above. + # Yes, it's a hack) + exit 1 + ;; + (default) + echo absent + ;; + (*) + echo different + ;; + esac +fi diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 7d86028e..998b8582 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -1,7 +1,7 @@ #!/bin/sh -e # -*- mode: sh; indent-tabs-mode: t -*- # -# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# 2019-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch) # # This file is part of cdist. @@ -21,71 +21,92 @@ # os=$(cat "${__global:?}/explorer/os") +state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") -if [ "${state_should}" != 'present' ] && [ "${state_should}" != 'absent' ] +conf_name=${__object_id:?} + +if test "${state_is}" = "${state_should}" then - echo "Invalid state '${state_should}'." \ - 'Only "present" and "absent" are acceptable' >&2 - exit 1 + exit 0 fi -if [ "${state_should}" = 'present' ] && [ ! -f "${__object}/parameter/value" ] -then - echo 'Missing required parameter "value"' >&2 - exit 1 -fi - -# Parameters -conf_name="${__object_id:?}" -if [ -f "${__object}/parameter/value" ] -then - conf_value=$(cat "${__object}/parameter/value") -fi - -if [ "${state_should}" = 'present' ] -then - set_command="ALTER SYSTEM SET ${conf_name} = ${conf_value}" -else - set_command="ALTER SYSTEM SET ${conf_name} = DEFAULT" -fi - -case $os -in - openbsd|devuan) - case $os - in - openbsd) - postgres_user='_postgresql' - restart_command='/etc/rc.d/postgresql restart' - ;; - devuan) - postgres_user='postgres' - restart_command='/etc/init.d/postgresql restart' - ;; - esac - # needs two separate psql commands because ALTER SYSTEM - # cannot run inside a transaction block - cat <<-EOF - su - ${postgres_user} -c "psql postgres -qwc \ - \"${set_command}\"" - su - ${postgres_user} -c "psql postgres -twAc \ - \"SELECT pg_reload_conf()\"" - EOF - # check success (makes only sense if setting to a non-default value) - # and restart server if needed - if [ "${state_should}" = 'present' ] +quote() { + for _arg + do + shift + if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')" then - cat <<-EOF - if [ \$(su - ${postgres_user} -c "psql postgres -twAc \"SHOW ${conf_name}\"") != '${conf_value}' ] - then - ${restart_command} - fi - EOF + # needs quoting + set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'" + else + set -- "$@" "${_arg}" fi + done + unset _arg + + # NOTE: Use printf because POSIX echo interprets escape sequences + printf '%s' "$*" +} + +case ${os} +in + (openbsd) + postgres_user='_postgresql' + restart_command='/etc/rc.d/postgresql restart' ;; - *) + (devuan) + postgres_user='postgres' + restart_command='/etc/init.d/postgresql restart' + ;; + (*) echo "Unsupported OS: ${os}" >&2 exit 1 ;; esac + + +psql_cmd() { + printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")" +} + +case ${state_should} +in + (present) + test -s "${__object:?}/parameter/value" || { + echo 'Missing required parameter --value' >&2 + exit 1 + } + + cat <<-EOF + exec 3< "\${__object:?}/parameter/value" + $(psql_cmd postgres -tAw) <<'SQL' + \\set conf_value \`cat <&3\` + ALTER SYSTEM SET ${conf_name} = :'conf_value'; + SELECT pg_reload_conf(); + SQL + exec 3<&- + EOF + ;; + (absent) + psql_cmd postgres -qwc "ALTER SYSTEM SET ${conf_name} TO DEFAULT" + ;; + (*) + printf 'Invalid --state: %s\n' "${state_should}" >&2 + printf 'Only "present" and "absent" are acceptable.\n' >&2 + exit 1 + ;; +esac + +# check success (makes only sense if setting to a non-default value) +# and restart server if needed +if test "${state_should}" = 'present' +then + cat <<-EOF + + $(psql_cmd postgres -twAc "SHOW ${conf_name}") \\ + | cmp -s "\${__object:?}/parameter/value" - || { + ${restart_command} + } + EOF +fi From 803367b316f5c72c51153172a41b7a745f2d2d83 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 18 Jan 2021 19:29:42 +0100 Subject: [PATCH 014/206] [type/__postgres_conf] Fix default detection when default is also set in config file e.g. port is usually also set to the default value in postgresql.conf --- cdist/conf/type/__postgres_conf/explorer/state | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index da904b56..400b27e8 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -42,7 +42,7 @@ if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ then echo present else - case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT source FROM pg_settings WHERE name = '${conf_name}'\"") + case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE name = '${conf_name}'\"") in ('') # invalid configuration parameter From 891c98567e1d94577dc4f20d3ec9f73f4927fd24 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 18 Jan 2021 19:39:56 +0100 Subject: [PATCH 015/206] [type/__postgres_conf] Compare configuration parameter names case insensitively --- cdist/conf/type/__postgres_conf/explorer/context | 3 ++- cdist/conf/type/__postgres_conf/explorer/state | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context index 3a2fa504..def5f676 100644 --- a/cdist/conf/type/__postgres_conf/explorer/context +++ b/cdist/conf/type/__postgres_conf/explorer/context @@ -39,4 +39,5 @@ esac conf_name=${__object_id:?} -su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE name = '${conf_name}'\"" +# NOTE: SHOW/SET are case-insentitive, so this command should also be. +su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"" diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index 400b27e8..a4930296 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -42,7 +42,8 @@ if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ then echo present else - case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE name = '${conf_name}'\"") + # NOTE: SHOW/SET are case-insentitive, so this command should also be. + case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"") in ('') # invalid configuration parameter From 5051d4f40b7097f58431ce334f9ddf5b171dbc7f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 19 Jan 2021 16:36:44 +0100 Subject: [PATCH 016/206] [type/__postgres_conf] Catch invalid values --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 998b8582..cc9b4e99 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -80,7 +80,7 @@ in cat <<-EOF exec 3< "\${__object:?}/parameter/value" - $(psql_cmd postgres -tAw) <<'SQL' + $(psql_cmd postgres -tAw -v ON_ERROR_STOP=on) <<'SQL' \\set conf_value \`cat <&3\` ALTER SYSTEM SET ${conf_name} = :'conf_value'; SELECT pg_reload_conf(); From 0f2ff477381d67de27149a92744f873322e44fd7 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 19 Jan 2021 16:37:43 +0100 Subject: [PATCH 017/206] [type/__postgres_conf] Restart PostgreSQL server based on pending_restart column of pg_settings --- .../type/__postgres_conf/explorer/context | 43 ------------------- .../conf/type/__postgres_conf/gencode-remote | 16 +++---- 2 files changed, 6 insertions(+), 53 deletions(-) delete mode 100644 cdist/conf/type/__postgres_conf/explorer/context diff --git a/cdist/conf/type/__postgres_conf/explorer/context b/cdist/conf/type/__postgres_conf/explorer/context deleted file mode 100644 index def5f676..00000000 --- a/cdist/conf/type/__postgres_conf/explorer/context +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -e -# -*- mode: sh; indent-tabs-mode: t -*- -# -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# Returns the "context" of the configuration setting. -# cf. also https://www.postgresql.org/docs/10/view-pg-settings.html - -os=$("${__explorer:?}/os") - -case ${os} -in - (openbsd) - postgres_user='_postgresql' - ;; - (devuan) - postgres_user='postgres' - ;; - (*) - echo "Unsupported OS: ${os}" >&2 - exit 1 - ;; -esac - -conf_name=${__object_id:?} - -# NOTE: SHOW/SET are case-insentitive, so this command should also be. -su - "${postgres_user}" -c "psql postgres -twAc \"SELECT context FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"" diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index cc9b4e99..fa930cc4 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -98,15 +98,11 @@ in ;; esac -# check success (makes only sense if setting to a non-default value) -# and restart server if needed -if test "${state_should}" = 'present' -then - cat <<-EOF +# Restart PostgreSQL server if required to apply new configuration value +cat < Date: Tue, 19 Jan 2021 17:18:27 +0100 Subject: [PATCH 018/206] [type/__postgres_conf] Add support for more init systems to restart service --- .../conf/type/__postgres_conf/gencode-remote | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index fa930cc4..15d5bb3e 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -53,11 +53,9 @@ case ${os} in (openbsd) postgres_user='_postgresql' - restart_command='/etc/rc.d/postgresql restart' ;; (devuan) postgres_user='postgres' - restart_command='/etc/init.d/postgresql restart' ;; (*) echo "Unsupported OS: ${os}" >&2 @@ -103,6 +101,37 @@ cat <&2 + exit 1 + esac + ;; + (*) + printf "Don't know how to restart services with your init (%s)\n" "${init}" >&2 + exit 1 + esac + ) fi EOF From 4967c7ebbb7e98af6aa2c7f8bc91511bcbbf9820 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 19 Jan 2021 17:20:27 +0100 Subject: [PATCH 019/206] [type/__postgres_conf] Silence psql output --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 15d5bb3e..9b8227ee 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -78,7 +78,7 @@ in cat <<-EOF exec 3< "\${__object:?}/parameter/value" - $(psql_cmd postgres -tAw -v ON_ERROR_STOP=on) <<'SQL' + $(psql_cmd postgres -tAwq -o /dev/null -v ON_ERROR_STOP=on) <<'SQL' \\set conf_value \`cat <&3\` ALTER SYSTEM SET ${conf_name} = :'conf_value'; SELECT pg_reload_conf(); From f9ebb4333c85e41439635cb00a40c3e07d8a3683 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 19 Jan 2021 17:21:50 +0100 Subject: [PATCH 020/206] [type/__postgres_conf] Add NetBSD PostgreSQL UNIX user --- cdist/conf/type/__postgres_conf/gencode-remote | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 9b8227ee..8f8a2bfe 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -51,15 +51,14 @@ quote() { case ${os} in + (netbsd) + postgres_user='pgsql' + ;; (openbsd) postgres_user='_postgresql' ;; - (devuan) - postgres_user='postgres' - ;; (*) - echo "Unsupported OS: ${os}" >&2 - exit 1 + postgres_user='postgres' ;; esac From 6b18cace759d1345cec6c2168c3bd8f238f23bca Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 21 Jan 2021 19:29:07 +0100 Subject: [PATCH 021/206] [type/__postgres_conf] Catch connection errors early --- cdist/conf/type/__postgres_conf/explorer/state | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index a4930296..589925de 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -37,6 +37,11 @@ esac conf_name=${__object_id:?} +su - "${postgres_user}" -c 'psql postgres -c "SELECT 1"' >/dev/null || { + echo 'Connection to PostgreSQL server failed' >&2 + exit 1 +} + if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ | cmp -s "${__object:?}/parameter/value" - then From 8eccacec59b0badd7a6e2a40a27f02f0ff355ccf Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 4 Feb 2021 19:09:26 +0100 Subject: [PATCH 022/206] __package_pip: add optional dependencies This is a poor implementation of optional dependencies for pip packages. It ensures to install them if the package will be installed, but does not take into account if they must be added/removed after the package is already installed. Also, it will not be autoremoved, as all dependencies will not be removed. --- cdist/conf/type/__package_pip/gencode-remote | 13 +++++++++++++ cdist/conf/type/__package_pip/man.rst | 13 +++++++++++++ .../type/__package_pip/parameter/optional_multiple | 1 + 3 files changed, 27 insertions(+) create mode 100644 cdist/conf/type/__package_pip/parameter/optional_multiple diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index a1375c2d..58c89040 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -56,6 +56,19 @@ fi case "$state_should" in present) + if [ -f "$__object/parameter/extra" ] + then + while read extra + do + if [ "$extras" ]; then + extras="$extras,$extra" + else + extras="$extra" + fi + done < "$__object/parameter/extra" + name="${name}[${extras}]" + fi + if [ "$runas" ] then echo "su -c '$pip install -q $name' $runas" diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst index 234ceee2..64ad0358 100644 --- a/cdist/conf/type/__package_pip/man.rst +++ b/cdist/conf/type/__package_pip/man.rst @@ -22,6 +22,11 @@ OPTIONAL PARAMETERS name If supplied, use the name and not the object id as the package name. +extra + Extra optional dependencies which should be installed along the selected + package. Can be specified multiple times. Will only be applied if the + package actually will be installed, but will not explicitly checked. + pip Instead of using pip from PATH, use the specific pip path. @@ -46,6 +51,14 @@ EXAMPLES # Use pip in a virtualenv located at /foo/shinken_virtualenv as user foo __package_pip pyro --state present --pip /foo/shinken_virtualenv/bin/pip --runas foo + # Install package with optional dependencies + __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails + # or do a little cheating + __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails + + # or take all extras + __package_pip mautrix-telegram --extra all + SEE ALSO -------- diff --git a/cdist/conf/type/__package_pip/parameter/optional_multiple b/cdist/conf/type/__package_pip/parameter/optional_multiple new file mode 100644 index 00000000..0f228715 --- /dev/null +++ b/cdist/conf/type/__package_pip/parameter/optional_multiple @@ -0,0 +1 @@ +extra From 73a03d75d7d23fea292b5fd60aeb059d8c72c4ba Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 4 Feb 2021 19:18:02 +0100 Subject: [PATCH 023/206] __package_pip: fix shellcheck --- cdist/conf/type/__package_pip/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index 58c89040..925eaf24 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -58,7 +58,7 @@ case "$state_should" in present) if [ -f "$__object/parameter/extra" ] then - while read extra + while read -r extra do if [ "$extras" ]; then extras="$extras,$extra" From cda17be38a1030742ed224b2536fca517bf8c09b Mon Sep 17 00:00:00 2001 From: ssrq Date: Mon, 8 Feb 2021 08:27:03 +0100 Subject: [PATCH 024/206] [explorer/memory] Clean up, return kiB for all systems, add SunOS BSDs were MiB before. --- cdist/conf/explorer/memory | 85 ++++++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 17 deletions(-) diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory index 5ea15ada..63aba9c6 100755 --- a/cdist/conf/explorer/memory +++ b/cdist/conf/explorer/memory @@ -1,8 +1,9 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Daniel Heule (hda at sfs.biz) # 2014 Thomas Oettli (otho at sfs.biz) # Copyright 2017, Philippe Gregoire +# 2020 Dennis Camera # # This file is part of cdist. # @@ -19,24 +20,74 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# +# Returns the amount of memory physically installed in the system, or if that +# cannot be determined the amount available to the operating system kernel, +# in kibibytes (kiB). -# FIXME: other system types (not linux ...) +str2bytes() { + awk -F' ' ' + $2 == "B" || !$2 { print $1 } + $2 == "kB" { print $1 * 1000 } + $2 == "MB" { print $1 * 1000 * 1000 } + $2 == "GB" { print $1 * 1000 * 1000 * 1000 } + $2 == "TB" { print $1 * 1000 * 1000 * 1000 * 1000 } + $2 == "kiB" { print $1 * 1024 } + $2 == "MiB" { print $1 * 1024 * 1024 } + $2 == "GiB" { print $1 * 1024 * 1024 * 1024 } + $2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }' +} -os=$("$__explorer/os") -case "$os" in - "macosx") - echo "$(sysctl -n hw.memsize)/1024" | bc - ;; +bytes2kib() { + set -- "$(cat)" + test "$1" -gt 0 && echo $(($1 / 1024)) +} - *"bsd") - PATH=$(getconf PATH) - echo "$(sysctl -n hw.physmem) / 1048576" | bc - ;; - *) - if [ -r /proc/meminfo ]; then - grep "MemTotal:" /proc/meminfo | awk '{print $2}' - fi - ;; +case $(uname -s) +in + (Darwin) + sysctl -n hw.memsize | bytes2kib + ;; + (FreeBSD) + sysctl -n hw.realmem | bytes2kib + ;; + (NetBSD|OpenBSD) + # NOTE: This reports "usable" memory, not physically installed memory. + command -p sysctl -n hw.physmem | bytes2kib + ;; + (SunOS) + # Make sure that awk from xpg4 is used for the scripts to work + export PATH="/usr/xpg4/bin:${PATH}" + prtconf \ + | awk -F ': ' ' + $1 == "Memory size" { sub(/Megabytes/, "MiB", $2); print $2 } + /^$/ { exit }' \ + | str2bytes \ + | bytes2kib + ;; + (Linux) + if test -d /sys/devices/system/memory + then + # Use memory blocks if the architecture (e.g. x86, PPC64, s390) + # supports them (they denote physical memory) + num_mem_blocks=$(cat /sys/devices/system/memory/memory[0-9]*/state | grep -cxF online) + mem_block_size=$(cat /sys/devices/system/memory/block_size_bytes) + + echo $((num_mem_blocks * 0x$mem_block_size)) | bytes2kib && exit + fi + if test -r /proc/meminfo + then + # Fall back to meminfo file on other architectures (e.g. ARM, MIPS, + # PowerPC) + # NOTE: This is "usable" memory, not physically installed memory. + awk -F ': +' '$1 == "MemTotal" { sub(/B$/, "iB", $2); print $2 }' /proc/meminfo \ + | str2bytes \ + | bytes2kib + fi + ;; + (*) + printf "Your kernel (%s) is currently not supported by the memory explorer\n" "$(uname -s)" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; esac From 65a6a2ed52c56f553d65ad0aa0aaba3ee4579e3c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 8 Feb 2021 08:28:31 +0100 Subject: [PATCH 025/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index b2b35616..52686617 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Core: preos: Fix passing cdist debug parameter (Darko Poljak) * Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera) + * Explorer memory: Fix result units; support Solaris (Dennis Camera) 6.9.4: 2020-12-21 * Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) From bc145bbc27a45e382adb744e6774fb803f4fda0a Mon Sep 17 00:00:00 2001 From: Evilham Date: Tue, 9 Feb 2021 19:58:47 +0100 Subject: [PATCH 026/206] [__letsencrypt_cert] Fix various issues with hooks. Closes #853, see issue for full description / discussion. Short summary: - There was about 6.53% chances of `--renewal-hook` not being applied - Using --automatic-renewal in one cert and not in another was an error. - It was not possible to use different hooks for different certificates. - FreeBSD support was utterly broken. --- cdist/conf/type/__letsencrypt_cert/man.rst | 84 +++++++-- cdist/conf/type/__letsencrypt_cert/manifest | 171 ++++++++++++++---- .../parameter/deprecated/automatic-renewal | 2 + .../parameter/deprecated/renew-hook | 2 + .../parameter/optional_multiple | 3 + 5 files changed, 220 insertions(+), 42 deletions(-) mode change 100755 => 100644 cdist/conf/type/__letsencrypt_cert/manifest create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/deprecated/automatic-renewal create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/deprecated/renew-hook diff --git a/cdist/conf/type/__letsencrypt_cert/man.rst b/cdist/conf/type/__letsencrypt_cert/man.rst index 85eb88ea..43be8424 100644 --- a/cdist/conf/type/__letsencrypt_cert/man.rst +++ b/cdist/conf/type/__letsencrypt_cert/man.rst @@ -1,16 +1,33 @@ cdist-type__letsencrypt_cert(7) =============================== + NAME ---- cdist-type__letsencrypt_cert - Get an SSL certificate from Let's Encrypt + DESCRIPTION ----------- Automatically obtain a Let's Encrypt SSL certificate using Certbot. +This type attempts to setup automatic renewals always. In many Linux +distributions, that is the case out of the box, see: +https://certbot.eff.org/docs/using.html#automated-renewals + +For Alpine Linux and Arch Linux, we setup a system-wide cronjob that +attempts to renew certificates daily. + +If you are using FreeBSD, we configure periodic(8) as recommended by +the port mantainer, so there will be a weekly attempt at renewal. + +If your OS is not mentioned here or on Certbot's docs as having +support for automated renewals, please make sure you check your OS +and possibly patch this type so the system-wide cronjob is installed. + + REQUIRED PARAMETERS ------------------- @@ -21,6 +38,7 @@ object id admin-email Where to send Let's Encrypt emails like "certificate needs renewal". + OPTIONAL PARAMETERS ------------------- @@ -36,25 +54,68 @@ webroot The path to your webroot, as set up in your webserver config. If this parameter is not present, Certbot will be run in standalone mode. + OPTIONAL MULTIPLE PARAMETERS ---------------------------- -renew-hook - Renew hook command directly passed to Certbot in cron job. - domain Domains to be included in the certificate. When specified then object id is not used as a domain. +deploy-hook + Command to be executed only when the certificate associated with this + ``$__object_id`` is issued or renewed. + You can specify it multiple times, but any failure will prevent further + commands from being executed. + + For this command, the + shell variable ``$RENEWED_LINEAGE`` will point to the + config live subdirectory (for example, + ``/etc/letsencrypt/live/${__object_id}``) containing the + new certificates and keys; the shell variable + ``$RENEWED_DOMAINS`` will contain a space-delimited list + of renewed certificate domains (for example, + ``example.com www.example.com``) + +pre-hook + Command to be run in a shell before obtaining any + certificates. + You can specify it multiple times, but any failure will prevent further + commands from being executed. + + Note these run regardless of which certificate is attempted, you may want to + manage these system-wide hooks with ``__file`` in + ``/etc/letsencrypt/renewal-hooks/pre/``. + + Intended primarily for renewal, where it + can be used to temporarily shut down a webserver that + might conflict with the standalone plugin. This will + only be called if a certificate is actually to be + obtained/renewed. + +post-hook + Command to be run in a shell after attempting to + obtain/renew certificates. + You can specify it multiple times, but any failure will prevent further + commands from being executed. + + Note these run regardless of which certificate was attempted, you may want to + manage these system-wide hooks with ``__file`` in + ``/etc/letsencrypt/renewal-hooks/post/``. + + Can be used to deploy + renewed certificates, or to restart any servers that + were stopped by --pre-hook. This is only run if an + attempt was made to obtain/renew a certificate. + + BOOLEAN PARAMETERS ------------------ -automatic-renewal - Install a cron job, which attempts to renew certificates daily. - staging Obtain a test certificate from a staging server. + MESSAGES -------- @@ -67,6 +128,7 @@ create remove Certificate was removed. + EXAMPLES -------- @@ -75,8 +137,7 @@ EXAMPLES # use object id as domain __letsencrypt_cert example.com \ --admin-email root@example.com \ - --automatic-renewal \ - --renew-hook "service nginx reload" \ + --deploy-hook "service nginx reload" \ --webroot /data/letsencrypt/root .. code-block:: sh @@ -85,11 +146,10 @@ EXAMPLES # and example.com needs to be included again with domain parameter __letsencrypt_cert example.com \ --admin-email root@example.com \ - --automatic-renewal \ --domain example.com \ --domain foo.example.com \ --domain bar.example.com \ - --renew-hook "service nginx reload" \ + --deploy-hook "service nginx reload" \ --webroot /data/letsencrypt/root AUTHORS @@ -99,11 +159,13 @@ AUTHORS | Kamila Součková | Darko Poljak | Ľubomír Kučera +| Evilham + COPYING ------- -Copyright \(C) 2017-2018 Nico Schottelius, Kamila Součková, Darko Poljak and +Copyright \(C) 2017-2021 Nico Schottelius, Kamila Součková, Darko Poljak and Ľubomír Kučera. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest old mode 100755 new mode 100644 index b4464366..9c8cc043 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -1,18 +1,20 @@ #!/bin/sh certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")" +state=$(cat "${__object}/parameter/state") +os="$(cat "${__global:?}/explorer/os")" if [ -z "${certbot_fullpath}" ]; then - os="$(cat "${__global:?}/explorer/os")" os_version="$(cat "${__global}/explorer/os_version")" - + # Use this, very common value, as a default. It is OS-dependent + certbot_fullpath="/usr/bin/certbot" case "$os" in - archlinux) - __package certbot - ;; - alpine) - __package certbot - ;; + archlinux) + __package certbot + ;; + alpine) + __package certbot + ;; debian) case "$os_version" in 8*) @@ -48,9 +50,7 @@ if [ -z "${certbot_fullpath}" ]; then exit 1 ;; esac - - certbot_fullpath=/usr/bin/certbot - ;; + ;; devuan) case "$os_version" in jessie) @@ -83,17 +83,14 @@ if [ -z "${certbot_fullpath}" ]; then exit 1 ;; esac - - certbot_fullpath=/usr/bin/certbot ;; freebsd) - __package py27-certbot - - certbot_fullpath=/usr/local/bin/certbot + __package py37-certbot + certbot_fullpath="/usr/local/bin/certbot" ;; ubuntu) - __package certbot - ;; + __package certbot + ;; *) echo "Unsupported os: $os" >&2 exit 1 @@ -101,18 +98,130 @@ if [ -z "${certbot_fullpath}" ]; then esac fi -if [ -f "${__object}/parameter/automatic-renewal" ]; then - renew_hook_param="${__object}/parameter/renew-hook" - renew_hook="" - if [ -f "${renew_hook_param}" ]; then - while read -r hook; do - renew_hook="${renew_hook} --renew-hook \"${hook}\"" - done < "${renew_hook_param}" - fi +# Other OS-dependent values that we want to set every time +LE_DIR="/etc/letsencrypt" +certbot_cronjob_state="absent" +case "$os" in + archlinux|alpine) + certbot_cronjob_state="present" + ;; + freebsd) + LE_DIR="/usr/local/etc/letsencrypt" + # FreeBSD uses periodic(8) instead of crontabs for this + __line "periodic.conf_weekly_certbot" \ + --file "/etc/periodic.conf" \ + --regex "^(#[[:space:]]*)?weekly_certbot_enable=.*" \ + --state "replace" \ + --line 'weekly_certbot_enable="YES"' + ;; + *) + ;; +esac - __cron letsencrypt-certbot \ - --user root \ - --command "${certbot_fullpath} renew -q ${renew_hook}" \ - --hour 0 \ - --minute 47 +# This is only necessary in certain OS +__cron letsencrypt-certbot \ + --user root \ + --command "${certbot_fullpath} renew -q" \ + --hour 0 \ + --minute 47 \ + --state "${certbot_cronjob_state}" + +# Ensure hook directories +HOOKS_DIR="${LE_DIR}/renewal-hooks" +__directory "${LE_DIR}" --mode 0755 +require="__directory/${LE_DIR}" __directory "${HOOKS_DIR}" --mode 0755 + +if [ -f "${__object}/parameter/domain" ]; then + domains="$(sort "${__object}/parameter/domain")" +else + domains="${__object_id}" fi + +# Install hooks as needed +for hook in deploy pre post; do + # Using something unique and specific to this object + hook_file="${HOOKS_DIR}/${hook}/${__object_id}.cdist.sh" + # Reasonable defaults + hook_source="${__object}/parameter/${hook}-hook" + hook_state="absent" + hook_contents_head="#!/bin/sh -e" + hook_contents_logic="" + hook_contents_tail="" + + # Backwards compatibility + # Remove this when renew-hook is removed + # Falling back to renew-hook if deploy-hook is not passed + if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then + hook_source="${__object}/parameter/renew-hook" + fi + if [ "${state}" = "present" ] && \ + [ -f "${hook_source}" ]; then + # This hook is to be installed, let's generate it with some + # safety boilerplate + # Since certbot runs all hooks for all renewal processes + # (at each state for deploy, pre, post), it is up to us to + # differentiate whether or not the hook must run + hook_state="present" + hook_contents_head="$(cat <> /dev/stderr + exit 1 + ;; + esac + + hook_contents_tail="$(cat < Date: Tue, 9 Feb 2021 20:29:17 +0100 Subject: [PATCH 027/206] [__letsencrypt_cert] Remove problematic trailing slash in sed. Happy fingers are happy and like adding slashes places. --- cdist/conf/type/__letsencrypt_cert/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 9c8cc043..04edea1e 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -206,7 +206,7 @@ EOF hook_contents_tail="$(cat < Date: Tue, 9 Feb 2021 20:53:58 +0100 Subject: [PATCH 028/206] [__letsencrypt_cert] Don't mess with user script indentation This could break in odd ways if they passed sth like: cat < Date: Wed, 10 Feb 2021 10:10:21 +0100 Subject: [PATCH 029/206] [__letsencrypt_cert] Move hook contents generation out of manifest While there address some minor issues in the comments in the hook contents. --- .../type/__letsencrypt_cert/files/gen_hook.sh | 84 +++++++++++++++++++ cdist/conf/type/__letsencrypt_cert/manifest | 78 +---------------- 2 files changed, 88 insertions(+), 74 deletions(-) create mode 100644 cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh diff --git a/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh b/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh new file mode 100644 index 00000000..81ea4856 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh @@ -0,0 +1,84 @@ +#!/bin/sh -e + +# It is expected that this defines hook_contents + +# Reasonable defaults +hook_source="${__object}/parameter/${hook}-hook" +hook_state="absent" +hook_contents_head="#!/bin/sh -e" +hook_contents_logic="" +hook_contents_tail="" + +# Backwards compatibility +# Remove this when renew-hook is removed +# Falling back to renew-hook if deploy-hook is not passed +if [ "${hook}" = "deploy" ] && [ ! -f "${hook_source}" ]; then + hook_source="${__object}/parameter/renew-hook" +fi +if [ "${state}" = "present" ] && \ + [ -f "${hook_source}" ]; then + # This hook is to be installed, let's generate it with some + # safety boilerplate + # Since certbot runs all hooks for all renewal processes + # (at each state for deploy, pre, post), it is up to us to + # differentiate whether or not the hook must run + hook_state="present" + hook_contents_head="$(cat <> /dev/stderr + exit 1 + ;; + esac + + hook_contents_tail="$(cat <> /dev/stderr - exit 1 - ;; - esac + # This defines hook_contents + # shellcheck source=cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh + . "${__type}/files/gen_hook.sh" - hook_contents_tail="$(cat < Date: Thu, 11 Feb 2021 10:31:07 +0100 Subject: [PATCH 030/206] __package_pip: add extras explorer The two new explorers detect all installed extras for this package. --- .../type/__package_pip/explorer/distinfo-dir | 45 +++++++++++++++++++ cdist/conf/type/__package_pip/explorer/extras | 26 +++++++++++ cdist/conf/type/__package_pip/explorer/state | 0 3 files changed, 71 insertions(+) create mode 100755 cdist/conf/type/__package_pip/explorer/distinfo-dir create mode 100755 cdist/conf/type/__package_pip/explorer/extras mode change 100644 => 100755 cdist/conf/type/__package_pip/explorer/state diff --git a/cdist/conf/type/__package_pip/explorer/distinfo-dir b/cdist/conf/type/__package_pip/explorer/distinfo-dir new file mode 100755 index 00000000..18e169ae --- /dev/null +++ b/cdist/conf/type/__package_pip/explorer/distinfo-dir @@ -0,0 +1,45 @@ +#!/bin/sh +# +# 2021 Matthias Stecher (matthiasstecher at gmx.de) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +nameparam="$__object/parameter/name" +if [ -f "$nameparam" ]; then + name=$(cat "$nameparam") +else + name="$__object_id" +fi + +pipparam="$__object/parameter/pip" +if [ -f "$pipparam" ]; then + pip=$(cat "$pipparam") +else + pip="$( "$__type_explorer/pip" )" +fi + + +if command -v "$pip" >/dev/null 2>&1; then + # assemble the path where pip stores all pip package info + "$pip" show "$name" \ + | awk -F': ' ' + $1 == "Name" {name=$2; gsub(/-/,"_",name); next} + $1 == "Version" {version=$2; next} + $1 == "Location" {location=$2; next} + END {if (version != "") printf "%s/%s-%s.dist-info", location, name, version}' +fi diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras new file mode 100755 index 00000000..92fc4732 --- /dev/null +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -0,0 +1,26 @@ +#!/bin/sh +# +# 2021 Matthias Stecher (matthiasstecher at gmx.de) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +distinfo_dir="$("$__type_explorer/distinfo-dir")" +if [ "$distinfo_dir" ]; then + # output all extras that are installed + awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA" +fi diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state old mode 100644 new mode 100755 From 8dc6ab97384f2240e9218fbd4463ba12b004f44c Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 11 Feb 2021 13:49:53 +0100 Subject: [PATCH 031/206] __package_pip: install not found extras Compares the explorer against the parameters and install those extras that are not already installed. --- cdist/conf/type/__package_pip/gencode-remote | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index 925eaf24..ad327e37 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -2,6 +2,7 @@ # # 2012 Nico Schottelius (nico-cdist at schottelius.org) # 2016 Darko Poljak (darko.poljak at gmail.com) +# 2021 Matthias Stecher (matthiasstecher at gmx.de) # # This file is part of cdist. # @@ -58,14 +59,17 @@ case "$state_should" in present) if [ -f "$__object/parameter/extra" ] then - while read -r extra - do - if [ "$extras" ]; then - extras="$extras,$extra" - else - extras="$extra" - fi - done < "$__object/parameter/extra" + mkdir "$__object/files" + # sort and generalize all extras + sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params" + sort "$__object/explorer/extras" > "$__object/files/extras-explorer" + + # assemble extras + # all extras are passed to pip in a comma-separated list + extras="$(comm -23 \ + "$__object/files/extras-params" "$__object/files/extras-explorer" \ + | sed ':a; $!N; s/\n/,/; ta' # loops through all input lines and add commas between them + )" name="${name}[${extras}]" fi From 2db0ef7c98226661e5e4b55c89180b31ba65b8bc Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 11 Feb 2021 22:53:26 +0100 Subject: [PATCH 032/206] __package_pip: updating real detection of extras As the previous detection took the wrong values, this explorer now checks if packages for an extra are installed or not. If not, the extra is not installed. Based on the information of the explorer, it will install the package again with the absent extras. --- cdist/conf/type/__package_pip/explorer/extras | 37 +++++++++++++++++-- cdist/conf/type/__package_pip/gencode-remote | 21 ++++------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index 92fc4732..2f5fcca6 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -17,10 +17,41 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# +# Checks if the given extras are really installed or not. It will be +# done by querring all dependencies for that extra and return it as +# "to be installed" if no dependency was found. +# distinfo_dir="$("$__type_explorer/distinfo-dir")" -if [ "$distinfo_dir" ]; then - # output all extras that are installed - awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA" + +# check if we have something to check +if [ "$distinfo_dir" ] && [ -s "$__object/parameter/extra" ] +then + # save cause freezing is slow + mkdir "$__object/files" + pip_freeze="$__object/files/pip-freeze.tmp" + pip3 freeze > "$pip_freeze" + + for extra in $(cat "$__object/parameter/extra" | tr ',' '\n') + do + # create a grep BRE pattern to search all packages + grep_pattern="$( + awk -F'(: | ; )' -v check="$extra" ' + $1 == "Requires-Dist" { + split($2, r, " "); + sub("extra == ", "", $3); gsub("'"'"'", "", $3); + if($3 == check) print r[1] + }' "$distinfo_dir/METADATA" \ + | sed ':a; $!N; s/\n/\\|/; ta' + )" + + # echo the extra if no packages where found for it + # if there is no pattern, we don't need to search ;-) + if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze" + then + echo "$extra" + fi + done fi diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index ad327e37..9abe28bf 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -26,7 +26,10 @@ state_is=$(cat "$__object/explorer/state") state_should="$(cat "$__object/parameter/state")" -[ "$state_is" = "$state_should" ] && exit 0 +# short circuit if state is the same and no extras to install +[ "$state_is" = "$state_should" ] && ! [ -s "$__object/explorer/extras" ] \ + && exit 0 + nameparam="$__object/parameter/name" if [ -f "$nameparam" ]; then @@ -57,19 +60,11 @@ fi case "$state_should" in present) - if [ -f "$__object/parameter/extra" ] + if [ -s "$__object/explorer/extras" ] then - mkdir "$__object/files" - # sort and generalize all extras - sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params" - sort "$__object/explorer/extras" > "$__object/files/extras-explorer" - - # assemble extras - # all extras are passed to pip in a comma-separated list - extras="$(comm -23 \ - "$__object/files/extras-params" "$__object/files/extras-explorer" \ - | sed ':a; $!N; s/\n/,/; ta' # loops through all input lines and add commas between them - )" + # all extras are passed to pip in a comma-separated list in the name + # sed loops through all input lines and add commas between them + extras="$(sed ':a; $!N; s/\n/,/; ta' "$__object/explorer/extras")" name="${name}[${extras}]" fi From 7398382890a4368e588e700183e6c777e90fb069 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 11 Feb 2021 23:12:10 +0100 Subject: [PATCH 033/206] __package_pip: fix shellcheck Useless `cat $file`, use `< $file` instead. --- cdist/conf/type/__package_pip/explorer/extras | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index 2f5fcca6..d57c69db 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -34,7 +34,7 @@ then pip_freeze="$__object/files/pip-freeze.tmp" pip3 freeze > "$pip_freeze" - for extra in $(cat "$__object/parameter/extra" | tr ',' '\n') + for extra in $(tr ',' '\n' < "$__object/parameter/extra") do # create a grep BRE pattern to search all packages grep_pattern="$( From a9d7dfb2ed40d0a61978cc706ff469c354a6c8f5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 12 Feb 2021 09:17:02 +0100 Subject: [PATCH 034/206] __package_pip: split extra 'all' to a list of all extras This will fix if a package will be upgraded from some extras to all extras. Previously, it will not work because some dependencies of 'all' are already installed, so the feature 'all' is already installed. Now, it will use a list of all extras to iterate over them separatly. This will result it will never install all extras via `[all]`, but rather `[foo,bar]`. --- cdist/conf/type/__package_pip/explorer/extras | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index d57c69db..be945e36 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -34,7 +34,14 @@ then pip_freeze="$__object/files/pip-freeze.tmp" pip3 freeze > "$pip_freeze" - for extra in $(tr ',' '\n' < "$__object/parameter/extra") + # If all is set, it searches all available extras to separatly check them. + # It would work with just 'all' (cause dependencies are given), but will + # not update if one extra is already present. Side effect is that it will + # not use [all] but instead name all extras seperatly. + for extra in $(if grep -qFx all "$__object/parameter/extra"; + then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA"; + else tr ',' '\n' < "$__object/parameter/extra"; + fi) do # create a grep BRE pattern to search all packages grep_pattern="$( From 951712740f95f8f1277f1295bcd97c0b9bafdc94 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 12 Feb 2021 13:42:51 +0100 Subject: [PATCH 035/206] __package_pip: update man.rst Adjusted comments for `explorer/extras` and updated the man page for the new behaviour of updating the extras. --- cdist/conf/type/__package_pip/explorer/extras | 7 +++--- cdist/conf/type/__package_pip/man.rst | 25 ++++++++++++------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index be945e36..8104ed47 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -35,15 +35,16 @@ then pip3 freeze > "$pip_freeze" # If all is set, it searches all available extras to separatly check them. - # It would work with just 'all' (cause dependencies are given), but will - # not update if one extra is already present. Side effect is that it will - # not use [all] but instead name all extras seperatly. + # It would work with just 'all' (cause dependencies are specified for + # 'all'), but will not update if one extra is already present. Side effect + # is that it will not use [all] but instead name all extras seperatly. for extra in $(if grep -qFx all "$__object/parameter/extra"; then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA"; else tr ',' '\n' < "$__object/parameter/extra"; fi) do # create a grep BRE pattern to search all packages + # maybe a file full of patterns for -F could be written grep_pattern="$( awk -F'(: | ; )' -v check="$extra" ' $1 == "Requires-Dist" { diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst index 64ad0358..c013695c 100644 --- a/cdist/conf/type/__package_pip/man.rst +++ b/cdist/conf/type/__package_pip/man.rst @@ -24,8 +24,14 @@ name extra Extra optional dependencies which should be installed along the selected - package. Can be specified multiple times. Will only be applied if the - package actually will be installed, but will not explicitly checked. + package. Can be specified multiple times. Multiple extra optional + dependencies can also be specified in a comma-separated list to provide + a more pip-natvie style. + + Extra optional dependencies will be installed even when the base package + is already installed. Notice that the type will not remove installed extras + that are not explicitly named for the type because pip does not offer a + management for orphaned packages and they may be used by other packages. pip Instead of using pip from PATH, use the specific pip path. @@ -53,8 +59,8 @@ EXAMPLES # Install package with optional dependencies __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails - # or do a little cheating - __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails + # the extras can also be specified comma-separated + __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails --extra postgres # or take all extras __package_pip mautrix-telegram --extra all @@ -67,12 +73,13 @@ SEE ALSO AUTHORS ------- -Nico Schottelius +| Nico Schottelius +| Matthias Stecher COPYING ------- -Copyright \(C) 2012 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2012 Nico Schottelius, 2021 Matthias Stecher. You can +redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either +version 3 of the License, or (at your option) any later version. From 2ce1fce7677b64a8d17a86f29f676b1ba8475db2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 15 Feb 2021 16:17:46 +0100 Subject: [PATCH 036/206] __package_pip: match package names case insensitive Pip matches them insensitive, so we need to do the same to avoid problems by saying extras are not installed but already is there in place. --- cdist/conf/type/__package_pip/explorer/extras | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index 8104ed47..bbdc17ab 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -57,7 +57,8 @@ then # echo the extra if no packages where found for it # if there is no pattern, we don't need to search ;-) - if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze" + # pip matches packages case-insensetive, we need to do that, too + if [ "$grep_pattern" ] && ! grep -qi "$grep_pattern" "$pip_freeze" then echo "$extra" fi From 0835f414a5b2ce41309ee9b265cb80abb1362563 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 16 Feb 2021 16:03:23 +0100 Subject: [PATCH 037/206] [type/__postgres_conf] Extract PostgreSQL service user detection to separate explorer --- .../__postgres_conf/explorer/postgres_user | 64 +++++++++++++++++++ .../conf/type/__postgres_conf/explorer/state | 16 +---- .../conf/type/__postgres_conf/gencode-remote | 17 +---- 3 files changed, 67 insertions(+), 30 deletions(-) create mode 100644 cdist/conf/type/__postgres_conf/explorer/postgres_user diff --git a/cdist/conf/type/__postgres_conf/explorer/postgres_user b/cdist/conf/type/__postgres_conf/explorer/postgres_user new file mode 100644 index 00000000..c6582dc4 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/explorer/postgres_user @@ -0,0 +1,64 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$("${__explorer:?}/os") + +case ${os} +in + (alpine) + echo 'postgres' + ;; + (centos|rhel|scientific) + echo 'postgres' + ;; + (debian|devuan|ubuntu) + echo 'postgres' + ;; + (freebsd) + test -x /usr/local/etc/rc.d/postgresql || { + printf 'could not find postgresql rc script./n' >&2 + exit 1 + } + pg_status=$(/usr/local/etc/rc.d/postgresql onestatus) || { + printf 'postgresql daemon is not running.\n' >&2 + exit 1 + } + pg_pid=$(printf '%s\n' "${pg_status}" \ + | sed -n 's/^pg_ctl:.*(PID: *\([0-9]*\))$/\1/p') + + # PostgreSQL < 9.6: pgsql + # PostgreSQL >= 9.6: postgres + ps -o user -p "${pg_pid}" | sed -n '2p' + ;; + (netbsd) + echo 'pgsql' + ;; + (openbsd) + echo '_postgresql' + ;; + (suse) + echo 'postgres' + ;; + (*) + echo "Unsupported OS: ${os}" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index 589925de..de6e8aa0 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -19,21 +19,7 @@ # along with cdist. If not, see . # -os=$("${__explorer:?}/os") - -case ${os} -in - (openbsd) - postgres_user='_postgresql' - ;; - (devuan) - postgres_user='postgres' - ;; - (*) - echo "Unsupported OS: ${os}" >&2 - exit 1 - ;; -esac +postgres_user=$("${__type_explorer:?}/postgres_user") conf_name=${__object_id:?} diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index 8f8a2bfe..d0d247b4 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -20,9 +20,9 @@ # along with cdist. If not, see . # -os=$(cat "${__global:?}/explorer/os") state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") +postgres_user=$(cat "${__object:?}/explorer/postgres_user") conf_name=${__object_id:?} @@ -49,19 +49,6 @@ quote() { printf '%s' "$*" } -case ${os} -in - (netbsd) - postgres_user='pgsql' - ;; - (openbsd) - postgres_user='_postgresql' - ;; - (*) - postgres_user='postgres' - ;; -esac - psql_cmd() { printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")" @@ -117,7 +104,7 @@ then case $(cat "${__global:?}/explorer/kernel_name") in (FreeBSD) - echo 'service postgresql restart' + echo '/usr/local/etc/rc.d/postgresql restart' ;; (OpenBSD|NetBSD) echo '/etc/rc.d/postgresql restart' From d1f45d3524f8a025488d0d2f020433b6efb7edb6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 19 Feb 2021 09:03:56 +0100 Subject: [PATCH 038/206] __package_pip: corrected typo in man .. by fully replacing it with a smaller sentence. --- cdist/conf/type/__package_pip/man.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst index c013695c..5a2bc673 100644 --- a/cdist/conf/type/__package_pip/man.rst +++ b/cdist/conf/type/__package_pip/man.rst @@ -24,9 +24,8 @@ name extra Extra optional dependencies which should be installed along the selected - package. Can be specified multiple times. Multiple extra optional - dependencies can also be specified in a comma-separated list to provide - a more pip-natvie style. + package. Can be specified multiple times. Multiple extras can be passed + in one `--extra` as a comma-separated list. Extra optional dependencies will be installed even when the base package is already installed. Notice that the type will not remove installed extras From 5e0572189fb930764318227fa9fd9f4966b0da99 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 22 Feb 2021 09:11:22 +0100 Subject: [PATCH 039/206] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 52686617..59c58e65 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,8 @@ next: * Core: preos: Fix passing cdist debug parameter (Darko Poljak) * Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera) * Explorer memory: Fix result units; support Solaris (Dennis Camera) + * Type __postgres_role: Implement modification of roles (Dennis Camera) + * Type __letsencrypt_cert: Fix issues with hooks (Evil Ham) 6.9.4: 2020-12-21 * Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) From 22f637c15b6616899c848e2a08c4e1c1b13f0f0b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 23 Feb 2021 06:29:24 +0100 Subject: [PATCH 040/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 59c58e65..88dda0aa 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * Explorer memory: Fix result units; support Solaris (Dennis Camera) * Type __postgres_role: Implement modification of roles (Dennis Camera) * Type __letsencrypt_cert: Fix issues with hooks (Evil Ham) + * Type __package_pip: Add optional extra dependencies param (Matthias Stecher) 6.9.4: 2020-12-21 * Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) From 0734288483700e8e10cebd87c797b625aa83d55e Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Sun, 21 Feb 2021 19:59:57 +0000 Subject: [PATCH 041/206] First draft of __apt_pin --- cdist/conf/type/__apt_pin/man.rst | 53 ++++++++++++++++ cdist/conf/type/__apt_pin/manifest | 63 +++++++++++++++++++ .../type/__apt_pin/parameter/default/package | 1 + .../type/__apt_pin/parameter/default/state | 1 + cdist/conf/type/__apt_pin/parameter/optional | 2 + cdist/conf/type/__apt_pin/parameter/required | 2 + 6 files changed, 122 insertions(+) create mode 100644 cdist/conf/type/__apt_pin/man.rst create mode 100755 cdist/conf/type/__apt_pin/manifest create mode 100644 cdist/conf/type/__apt_pin/parameter/default/package create mode 100644 cdist/conf/type/__apt_pin/parameter/default/state create mode 100644 cdist/conf/type/__apt_pin/parameter/optional create mode 100644 cdist/conf/type/__apt_pin/parameter/required diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst new file mode 100644 index 00000000..7fcae6f8 --- /dev/null +++ b/cdist/conf/type/__apt_pin/man.rst @@ -0,0 +1,53 @@ +cdist-type__apt_pin(7) +====================== + +NAME +---- +cdist-type__apt_pin - TODO + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # TODO + __apt_pin + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Daniel Fancsali + + +COPYING +------- +Copyright \(C) 2021 Daniel Fancsali. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest new file mode 100755 index 00000000..8dd9770d --- /dev/null +++ b/cdist/conf/type/__apt_pin/manifest @@ -0,0 +1,63 @@ +#!/bin/sh -e +# +# 2021 Daniel Fancsali (fancsali@gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") +state="$(cat "$__object/parameter/state")" +package="$(cat "$__object/parameter/package")" +distribution="$(cat "$__object/parameter/distribution")" +priority="$(cat "$__object/parameter/priority")" + + +case "$os" in + debian|ubuntu|devuan) + ;; + *) + printf "This type is specific to Debian and it's derivatives" >&2 + printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2 + exit 1 + ;; +esac + +if [ "$package" = "*" ]; then + name="default" + +else + name="$__object_id" +fi + +case $distribution in + stabletesting|unsatbel|experimental) + pin="release a=$distribution" + ;; + *) + pin="release n=$distribution" + ;; +esac + + +__file /etc/apt/preferences.d/$name \ + --owner root --group root --mode 0644 \ + --state "$state" \ + --source - << EOF +Package: $package +Pin: $pin +Pin-Priority: $priority +EOF diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package new file mode 100644 index 00000000..72e8ffc0 --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/default/package @@ -0,0 +1 @@ +* diff --git a/cdist/conf/type/__apt_pin/parameter/default/state b/cdist/conf/type/__apt_pin/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional new file mode 100644 index 00000000..52f01fd2 --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/optional @@ -0,0 +1,2 @@ +state +package diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required new file mode 100644 index 00000000..4b4e9741 --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/required @@ -0,0 +1,2 @@ +distribution +priority From 1a74470c4d9b30e41a72fbfc084dc54ea44e643b Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Tue, 23 Feb 2021 09:37:36 +0000 Subject: [PATCH 042/206] __apt_pin: Always use $__object_id as preferences.d filename --- cdist/conf/type/__apt_pin/manifest | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index 8dd9770d..162b523f 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -36,12 +36,7 @@ case "$os" in ;; esac -if [ "$package" = "*" ]; then - name="default" - -else - name="$__object_id" -fi +name="$__object_id" case $distribution in stabletesting|unsatbel|experimental) From dc66efa690e15ff32d6836e23baa1cbec5eee1ed Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Tue, 23 Feb 2021 11:59:09 +0000 Subject: [PATCH 043/206] Fix shellcheck issues --- cdist/conf/type/__apt_pin/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index 162b523f..b1372ad0 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -48,7 +48,7 @@ case $distribution in esac -__file /etc/apt/preferences.d/$name \ +__file "/etc/apt/preferences.d/$name" \ --owner root --group root --mode 0644 \ --state "$state" \ --source - << EOF From 60fd7ba1f38382761c366cbc0425ddaec62f1164 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 28 Feb 2021 13:37:23 +0100 Subject: [PATCH 044/206] Release 6.9.5 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 88dda0aa..95e36b88 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) * Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera) * Explorer memory: Fix result units; support Solaris (Dennis Camera) From 8ef19d47f6b5fb6cdcba7a9a3b8890388dbd7023 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 1 Mar 2021 17:59:25 +0100 Subject: [PATCH 045/206] [type/__pyvenv] Fix example (--user -> --owner) --- cdist/conf/type/__pyvenv/man.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__pyvenv/man.rst b/cdist/conf/type/__pyvenv/man.rst index 8085ff12..e2e4a1e6 100644 --- a/cdist/conf/type/__pyvenv/man.rst +++ b/cdist/conf/type/__pyvenv/man.rst @@ -61,7 +61,7 @@ EXAMPLES __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4 # Create python virtualenv for user foo. - __pyvenv /home/foo/fooenv --group foo --user foo + __pyvenv /home/foo/fooenv --group foo --owner foo # Create python virtualenv with specific parameters. __pyvenv /home/services/djangoenv --venvparams "--copies --system-site-packages" From e7d33891df945831d54d164eb13f2a9ae4f9dd8e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 2 Mar 2021 09:29:33 +0100 Subject: [PATCH 046/206] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 95e36b88..072fcb6b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __pyvenv: Fix user example in man page (Dennis Camera) + 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) * Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera) From ea0126dd8117c051612e6b6abfe895f5f722c584 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 5 Mar 2021 16:11:49 +0100 Subject: [PATCH 047/206] Make local state dir available to custom remote scripts Signed-off-by: Steven Armstrong --- cdist/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cdist/config.py b/cdist/config.py index e84f6f84..19d5bd70 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -420,6 +420,9 @@ class Config: exec_path=sys.argv[0], save_output_streams=args.save_output_streams) + # Make __global state dir available to custom remote scripts. + os.environ['__global'] = local.base_path + remote = cdist.exec.remote.Remote( target_host=target_host, remote_exec=remote_exec, From ecba284fc8ce68486c5f1f87a863409fcad0f106 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 5 Mar 2021 16:13:02 +0100 Subject: [PATCH 048/206] changelog++ Signed-off-by: Steven Armstrong --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 072fcb6b..ad93a595 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Type __pyvenv: Fix user example in man page (Dennis Camera) + * Core: config: Make local state directory available to custom remotes (Steven Armstrong 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) From fb19f342669d18e6decb21ee4cfb2b22e46748d9 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 9 Mar 2021 21:15:26 +0100 Subject: [PATCH 049/206] [type/__ssh_authorized_key] Only grep if file exists --- cdist/conf/type/__ssh_authorized_key/explorer/entry | 1 + cdist/conf/type/__ssh_authorized_key/gencode-remote | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__ssh_authorized_key/explorer/entry b/cdist/conf/type/__ssh_authorized_key/explorer/entry index ccab0afc..aca0f2b9 100755 --- a/cdist/conf/type/__ssh_authorized_key/explorer/entry +++ b/cdist/conf/type/__ssh_authorized_key/explorer/entry @@ -25,6 +25,7 @@ type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ] if [ -n "${type_and_key}" ] then file="$(cat "$__object/parameter/file")" + test -e "$file" || exit 0 # get any entries that match the type and key diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index f37aa565..61c77fb9 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -37,9 +37,9 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX) # preserve ownership and permissions of existing file if [ -f "$file" ]; then cp -p "$file" "\$tmpfile" + grep -v -F -x '$line' '$file' >\$tmpfile fi -grep -v -F -x '$line' '$file' > \$tmpfile || true -mv -f "\$tmpfile" "$file" +cat "\$tmpfile" >"$file" DONE } From 31cc592aa10902c3e83357ee4a0b9300b5e34efa Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 10 Mar 2021 19:25:04 +0100 Subject: [PATCH 050/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index ad93a595..c63b901a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __pyvenv: Fix user example in man page (Dennis Camera) * Core: config: Make local state directory available to custom remotes (Steven Armstrong + * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) From e47c4dd8a4cc441b94fe73d362d79525d01a4bb1 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 9 Mar 2021 19:59:05 +0100 Subject: [PATCH 051/206] [type/__sshd_config] Whitelist OpenBMC in manifest --- cdist/conf/type/__sshd_config/manifest | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest index 566bde90..e37afebb 100755 --- a/cdist/conf/type/__sshd_config/manifest +++ b/cdist/conf/type/__sshd_config/manifest @@ -39,7 +39,14 @@ in (freebsd|netbsd|openbsd) # whitelist ;; + (openbmc-phosphor) + # whitelist + # OpenBMC can be configured with dropbear and OpenSSH. + # If dropbear is used, the state explorer will already fail because it + # cannot find the sshd binary. + ;; (*) + : "${__type:?}" # make shellcheck happy printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \ "${os}" "${__type##*/}" >&2 printf 'Please contribute an implementation for it if you can.\n' >&2 From 10ca1c12fd25a9f2b92cebbc5d375a1723ed5e2f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 12 Mar 2021 08:21:03 +0100 Subject: [PATCH 052/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index c63b901a..42a74d04 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * Type __pyvenv: Fix user example in man page (Dennis Camera) * Core: config: Make local state directory available to custom remotes (Steven Armstrong * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) + * Type __sshd_config: Whitelist OpenBMC (Dennis Camera) 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) From 7a0b697f4c394f1ae8a8941a59937007cfe474aa Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 21 Mar 2021 18:10:06 +0100 Subject: [PATCH 053/206] Implement maintaining object relationship graph For each object maintain parent-child relationship graph, i.e. list of parent objects ('parents' property) and list of children objects ('children' property). Objects without parent(s) are objects specified in init manifest. Objects without children are object of types that do not reuse other types. --- cdist/core/cdist_object.py | 7 +++++++ cdist/emulator.py | 19 +++++++++++++++++++ docs/src/cdist-cache.rst | 17 +++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 51d61e04..eea2c255 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -247,6 +247,13 @@ class CdistObject: lambda obj: os.path.join(obj.absolute_path, 'typeorder')) typeorder_dep = fsproperty.FileListProperty( lambda obj: os.path.join(obj.absolute_path, 'typeorder_dep')) + # objects without parents are objects specified in init manifest + parents = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, 'parents')) + # objects without children are object of types that do not reuse other + # types + children = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, 'children')) def cleanup(self): try: diff --git a/cdist/emulator.py b/cdist/emulator.py index a2bdc3d4..f1db862e 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -106,6 +106,7 @@ class Emulator: self.save_stdin() self.record_requirements() self.record_auto_requirements() + self.record_parent_child_relationships() self.log.trace("Finished %s %s" % ( self.cdist_object.path, self.parameters)) @@ -420,3 +421,21 @@ class Emulator: self.log.debug("Recording autorequirement %s for %s", current_object.name, parent.name) parent.autorequire.append(current_object.name) + + def record_parent_child_relationships(self): + # __object_name is the name of the object whose type manifest is + # currently executed + __object_name = self.env.get('__object_name', None) + if __object_name: + # The object whose type manifest is currently run + parent = self.cdist_object.object_from_name(__object_name) + # The object currently being defined + current_object = self.cdist_object + if current_object.name not in parent.children: + self.log.debug("Recording child %s for %s", + current_object.name, parent.name) + parent.children.append(current_object.name) + if parent.name not in current_object.parents: + self.log.debug("Recording parent %s for %s", + parent.name, current_object.name) + current_object.parents.append(parent.name) diff --git a/docs/src/cdist-cache.rst b/docs/src/cdist-cache.rst index d2d2d56c..d4159e77 100644 --- a/docs/src/cdist-cache.rst +++ b/docs/src/cdist-cache.rst @@ -61,6 +61,14 @@ Object cache overview ~~~~~~~~~~~~~~~~~~~~~ Each object under :strong:`object` directory has its own structure. +autorequire + file containing a list of object auto requirements + +children + file containing a list of object children, i.e. objects of types that this + type reuses (along with 'parents' it is used for maintaining parent-child + relationship graph) + code-local code generated from gencode-local, present only if something is generated @@ -80,6 +88,15 @@ parameter directory containing type parameter named files containing parameter values +parents + file containing a list of object parents, i.e. objects of types that reuse + this type (along with 'children' it is used for maintaining parent-child + relationship graph); objects without parents are objects specified in init + manifest + +require + file containing a list of object requirements + source this type's source (init manifest) From 167c2ad7ea2b895cdc7d89c72968d88c7d1ab34f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 30 Mar 2021 13:24:56 +0200 Subject: [PATCH 054/206] [type/__git] Fix if --owner / --group is numeric Before, if --owner and/or --group was numeric, gencode-remote would generate `chown` code every time. --- cdist/conf/type/__git/explorer/group | 20 +++++++++++++++++--- cdist/conf/type/__git/explorer/owner | 20 +++++++++++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group index 3ddf9656..1365c60d 100644 --- a/cdist/conf/type/__git/explorer/group +++ b/cdist/conf/type/__git/explorer/group @@ -1,5 +1,19 @@ -#!/bin/sh +#!/bin/sh -e -destination="/$__object_id/.git" +destination="/${__object_id:?}/.git" -stat --print "%G" "${destination}" 2>/dev/null || exit 0 +# shellcheck disable=SC2012 +group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }') + +# NOTE: +1 because $((notanum)) prints 0. +if test $((group_gid + 1)) -ge 0 +then + group_should=$(cat "${__object:?}/parameter/group") + + if expr "${group_should}" : '[0-9]*$' >/dev/null + then + printf '%u\n' "${group_gid}" + else + printf '%s\n' "$(id -u -n "${group_gid}")" + fi +fi diff --git a/cdist/conf/type/__git/explorer/owner b/cdist/conf/type/__git/explorer/owner index 4c3cd431..4a4d0d13 100644 --- a/cdist/conf/type/__git/explorer/owner +++ b/cdist/conf/type/__git/explorer/owner @@ -1,5 +1,19 @@ -#!/bin/sh +#!/bin/sh -e -destination="/$__object_id/.git" +destination="/${__object_id:?}/.git" -stat --print "%U" "${destination}" 2>/dev/null || exit 0 +# shellcheck disable=SC2012 +owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }') + +# NOTE: +1 because $((notanum)) prints 0. +if test $((owner_uid + 1)) -ge 0 +then + owner_should=$(cat "${__object:?}/parameter/owner") + + if expr "${owner_should}" : '[0-9]*$' >/dev/null + then + printf '%u\n' "${owner_uid}" + else + printf '%s\n' "$(id -u -n "${owner_uid}")" + fi +fi From 985252585ce83605787d2612fdc2a84b9788d943 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 30 Mar 2021 13:26:21 +0200 Subject: [PATCH 055/206] [type/__pyvenv] Fix if --owner / --group is numeric Before, if --owner and/or --group was numeric, gencode-remote would generate `chown` code every time. --- cdist/conf/type/__pyvenv/explorer/group | 20 +++++++++++++++++--- cdist/conf/type/__pyvenv/explorer/owner | 20 +++++++++++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group index a655bda7..f31a1cb7 100755 --- a/cdist/conf/type/__pyvenv/explorer/group +++ b/cdist/conf/type/__pyvenv/explorer/group @@ -1,5 +1,19 @@ -#!/bin/sh +#!/bin/sh -e -destination="/$__object_id" +destination="/${__object_id:?}" -stat --print "%G" "${destination}" 2>/dev/null || exit 0 +# shellcheck disable=SC2012 +group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }') + +# NOTE: +1 because $((notanum)) prints 0. +if test $((group_gid + 1)) -ge 0 +then + group_should=$(cat "${__object:?}/parameter/group") + + if expr "${group_should}" : '[0-9]*$' >/dev/null + then + printf '%u\n' "${group_gid}" + else + printf '%s\n' "$(id -u -n "${group_gid}")" + fi +fi diff --git a/cdist/conf/type/__pyvenv/explorer/owner b/cdist/conf/type/__pyvenv/explorer/owner index 8b3c7f8e..ebec751f 100755 --- a/cdist/conf/type/__pyvenv/explorer/owner +++ b/cdist/conf/type/__pyvenv/explorer/owner @@ -1,5 +1,19 @@ -#!/bin/sh +#!/bin/sh -e -destination="/$__object_id" +destination="/${__object_id:?}" -stat --print "%U" "${destination}" 2>/dev/null || exit 0 +# shellcheck disable=SC2012 +owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }') + +# NOTE: +1 because $((notanum)) prints 0. +if test $((owner_uid + 1)) -ge 0 +then + owner_should=$(cat "${__object:?}/parameter/owner") + + if expr "${owner_should}" : '[0-9]*$' >/dev/null + then + printf '%u\n' "${owner_uid}" + else + printf '%s\n' "$(id -u -n "${owner_uid}")" + fi +fi From 1e765fcab7bae02d6b380ca02e7cb04ec69ebdc5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 31 Mar 2021 07:54:00 +0200 Subject: [PATCH 056/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 42a74d04..03ae7566 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * Core: config: Make local state directory available to custom remotes (Steven Armstrong * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) * Type __sshd_config: Whitelist OpenBMC (Dennis Camera) + * Core: Maintain object relationship graph in cdist cache (Darko Poljak) 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) From f984a918b959a46e49fe8456a327eca8d44313de Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 30 Mar 2021 07:56:38 +0200 Subject: [PATCH 057/206] Fix log message string formatting Use logging message format with args, instead of direct `%` or `str.format`. Resolve #855. --- cdist/argparse.py | 8 +-- cdist/config.py | 87 ++++++++++++-------------- cdist/core/cdist_type.py | 6 +- cdist/core/explorer.py | 26 ++++---- cdist/core/manifest.py | 2 +- cdist/emulator.py | 37 +++++------ cdist/exec/local.py | 16 +++-- cdist/exec/remote.py | 14 ++--- cdist/install.py | 2 +- cdist/inventory.py | 36 +++++------ cdist/preos.py | 6 +- cdist/preos/debootstrap/debootstrap.py | 41 ++++++------ cdist/scan/scan.py | 4 +- cdist/util/ipaddr.py | 6 +- 14 files changed, 140 insertions(+), 151 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 88759d7b..cadac39a 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -533,10 +533,10 @@ def parse_and_configure(argv, singleton=True): log = logging.getLogger("cdist") - log.verbose("version %s" % cdist.VERSION) - log.trace('command line args: {}'.format(cfg.command_line_args)) - log.trace('configuration: {}'.format(cfg.get_config())) - log.trace('configured args: {}'.format(args)) + log.verbose("version %s", cdist.VERSION) + log.trace('command line args: %s', cfg.command_line_args) + log.trace('configuration: %s', cfg.get_config()) + log.trace('configured args: %s', args) check_beta(vars(args)) diff --git a/cdist/config.py b/cdist/config.py index 19d5bd70..d6fec55f 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -273,15 +273,15 @@ class Config: host_tags = None host_base_path, hostdir = cls.create_host_base_dirs( host, base_root_path) - log.debug("Base root path for target host \"{}\" is \"{}\"".format( - host, host_base_path)) + log.debug("Base root path for target host \"%s\" is \"%s\"", + host, host_base_path) hostcnt += 1 if args.parallel: pargs = (host, host_tags, host_base_path, hostdir, args, True, configuration) - log.trace(("Args for multiprocessing operation " - "for host {}: {}".format(host, pargs))) + log.trace("Args for multiprocessing operation for host %s: %s", + host, pargs) process_args.append(pargs) else: try: @@ -298,10 +298,10 @@ class Config: except cdist.Error: failed_hosts.append(host) elif args.parallel: - log.trace("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - log.trace(("Starting multiprocessing Pool for {} " - "parallel host operation".format(args.parallel))) + log.trace("Multiprocessing start method is %s", + multiprocessing.get_start_method()) + log.trace("Starting multiprocessing Pool for %d parallel host" + " operation", args.parallel) results = mp_pool_run(cls.onehost, process_args, @@ -396,16 +396,13 @@ class Config: remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds( args) - log.debug("remote_exec for host \"{}\": {}".format( - host, remote_exec)) - log.debug("remote_copy for host \"{}\": {}".format( - host, remote_copy)) + log.debug("remote_exec for host \"%s\": %s", host, remote_exec) + log.debug("remote_copy for host \"%s\": %s", host, remote_copy) family = cls._address_family(args) - log.debug("address family: {}".format(family)) + log.debug("address family: %s", family) target_host = cls.resolve_target_addresses(host, family) - log.debug("target_host for host \"{}\": {}".format( - host, target_host)) + log.debug("target_host for host \"%s\": %s", host, target_host) local = cdist.exec.local.Local( target_host=target_host, @@ -474,8 +471,8 @@ class Config: """Do what is most often done: deploy & cleanup""" start_time = time.time() - self.log.info("Starting {} run".format( - 'dry' if self.dry_run else 'configuration')) + self.log.info("Starting %s run", + 'dry' if self.dry_run else 'configuration') self._init_files_dirs() @@ -493,9 +490,9 @@ class Config: self._remove_files_dirs() self.local.save_cache(start_time) - self.log.info("Finished {} run in {:.2f} seconds".format( + self.log.info("Finished %s run in %.2f seconds", 'dry' if self.dry_run else 'successful', - time.time() - start_time)) + time.time() - start_time) def cleanup(self): self.log.debug("Running cleanup commands") @@ -519,8 +516,8 @@ class Config: self.local.object_path, self.local.type_path, self.local.object_marker_name): if cdist_object.cdist_type.is_install: - self.log.debug(("Running in config mode, ignoring install " - "object: {0}").format(cdist_object)) + self.log.debug("Running in config mode, ignoring install " + "object: %s", cdist_object) else: yield cdist_object @@ -565,8 +562,7 @@ class Config: return objects_changed def _iterate_once_parallel(self): - self.log.debug("Iteration in parallel mode in {} jobs".format( - self.jobs)) + self.log.debug("Iteration in parallel mode in %d jobs", self.jobs) objects_changed = False cargo = [] @@ -588,8 +584,8 @@ class Config: self.object_prepare(cargo[0]) objects_changed = True elif cargo: - self.log.trace("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) + self.log.trace("Multiprocessing start method is %s", + multiprocessing.get_start_method()) self.log.trace("Multiprocessing cargo: %s", cargo) @@ -603,9 +599,8 @@ class Config: "sequentially")) self.explorer.transfer_type_explorers(cargo_types.pop()) else: - self.log.trace(("Starting multiprocessing Pool for {} " - "parallel types explorers transferring".format( - nt))) + self.log.trace("Starting multiprocessing Pool for %d " + "parallel types explorers transferring", nt) args = [ (ct, ) for ct in cargo_types ] @@ -614,8 +609,8 @@ class Config: self.log.trace(("Multiprocessing for parallel transferring " "types' explorers finished")) - self.log.trace(("Starting multiprocessing Pool for {} parallel " - "objects preparation".format(n))) + self.log.trace("Starting multiprocessing Pool for %d parallel " + "objects preparation", n) args = [ (c, False, ) for c in cargo ] @@ -667,10 +662,10 @@ class Config: self.object_run(chunk[0]) objects_changed = True elif chunk: - self.log.trace("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - self.log.trace(("Starting multiprocessing Pool for {} " - "parallel object run".format(n))) + self.log.trace("Multiprocessing start method is %s", + multiprocessing.get_start_method()) + self.log.trace("Starting multiprocessing Pool for %d " + "parallel object run", n) args = [ (c, ) for c in chunk ] @@ -794,9 +789,9 @@ class Config: def object_prepare(self, cdist_object, transfer_type_explorers=True): """Prepare object: Run type explorer + manifest""" self._handle_deprecation(cdist_object) - self.log.verbose("Preparing object {}".format(cdist_object.name)) - self.log.verbose( - "Running manifest and explorers for " + cdist_object.name) + self.log.verbose("Preparing object %s", cdist_object.name) + self.log.verbose("Running manifest and explorers for %s", + cdist_object.name) self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) try: self.manifest.run_type_manifest(cdist_object) @@ -810,13 +805,13 @@ class Config: def object_run(self, cdist_object): """Run gencode and code for an object""" try: - self.log.verbose("Running object " + cdist_object.name) + self.log.verbose("Running object %s", cdist_object.name) if cdist_object.state == core.CdistObject.STATE_DONE: raise cdist.Error(("Attempting to run an already finished " - "object: %s"), cdist_object) + "object: {}").format(cdist_object)) # Generate - self.log.debug("Generating code for %s" % (cdist_object.name)) + self.log.debug("Generating code for %s", cdist_object.name) cdist_object.code_local = self.code.run_gencode_local(cdist_object) cdist_object.code_remote = self.code.run_gencode_remote( cdist_object) @@ -825,20 +820,20 @@ class Config: # Execute if cdist_object.code_local or cdist_object.code_remote: - self.log.info("Processing %s" % (cdist_object.name)) + self.log.info("Processing %s", cdist_object.name) if not self.dry_run: if cdist_object.code_local: - self.log.trace("Executing local code for %s" - % (cdist_object.name)) + self.log.trace("Executing local code for %s", + cdist_object.name) self.code.run_code_local(cdist_object) if cdist_object.code_remote: - self.log.trace("Executing remote code for %s" - % (cdist_object.name)) + self.log.trace("Executing remote code for %s", + cdist_object.name) self.code.transfer_code_remote(cdist_object) self.code.run_code_remote(cdist_object) # Mark this object as done - self.log.trace("Finishing run of " + cdist_object.name) + self.log.trace("Finishing run of %s", cdist_object.name) cdist_object.state = core.CdistObject.STATE_DONE except cdist.Error as e: raise cdist.CdistObjectError(cdist_object, e) diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index c0329c8a..274de989 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -82,9 +82,9 @@ class CdistType: yield cls(base_path, name) except InvalidTypeError as e: # ignore invalid type, log warning and continue - msg = "Ignoring invalid type '%s' at '%s' defined at '%s'" % ( - e.type_path, e.type_absolute_path, e.source_path) - cls.log.warning(msg) + cls.log.warning("Ignoring invalid type '%s' at '%s' defined" + " at '%s'", e.type_path, e.type_absolute_path, + e.source_path) # remove invalid from runtime conf dir os.remove(e.type_absolute_path) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index a3baa959..caa12a7d 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -131,18 +131,17 @@ class Explorer: self._run_global_explorer(explorer, out_path) def _run_global_explorers_parallel(self, out_path): - self.log.debug("Running global explorers in {} parallel jobs".format( - self.jobs)) - self.log.trace("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - self.log.trace(("Starting multiprocessing Pool for global " - "explorers run")) + self.log.debug("Running global explorers in %s parallel jobs", + self.jobs) + self.log.trace("Multiprocessing start method is %s", + multiprocessing.get_start_method()) + self.log.trace("Starting multiprocessing Pool for global explorers" + " run") args = [ (e, out_path, ) for e in self.list_global_explorer_names() ] mp_pool_run(self._run_global_explorer, args, jobs=self.jobs) - self.log.trace(("Multiprocessing run for global explorers " - "finished")) + self.log.trace("Multiprocessing run for global explorers finished") # logger is not pickable, so remove it when we pickle def __getstate__(self): @@ -184,15 +183,14 @@ class Explorer: in the object. """ - self.log.verbose("Running type explorers for {}".format( - cdist_object.cdist_type)) + self.log.verbose("Running type explorers for %s", + cdist_object.cdist_type) if transfer_type_explorers: self.log.trace("Transferring type explorers for type: %s", cdist_object.cdist_type) self.transfer_type_explorers(cdist_object.cdist_type) else: - self.log.trace(("No need for transferring type explorers for " - "type: %s"), + self.log.trace("No need for transferring type explorers for %s", cdist_object.cdist_type) self.log.trace("Transferring object parameters for object: %s", cdist_object.name) @@ -236,8 +234,8 @@ class Explorer: remote side.""" if cdist_type.explorers: if cdist_type.name in self._type_explorers_transferred: - self.log.trace(("Skipping retransfer of type explorers " - "for: %s"), cdist_type) + self.log.trace("Skipping retransfer of type explorers for: %s", + cdist_type) else: source = os.path.join(self.local.type_path, cdist_type.explorer_path) diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 390340d4..3148d66c 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -161,7 +161,7 @@ class Manifest: raise NoInitialManifestError(initial_manifest, user_supplied) message_prefix = "initialmanifest" - self.log.verbose("Running initial manifest " + initial_manifest) + self.log.verbose("Running initial manifest %s", initial_manifest) which = "init" if self.local.save_output_streams: stderr_path = os.path.join(self.local.stderr_base_path, which) diff --git a/cdist/emulator.py b/cdist/emulator.py index f1db862e..f09c282d 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -107,8 +107,8 @@ class Emulator: self.record_requirements() self.record_auto_requirements() self.record_parent_child_relationships() - self.log.trace("Finished %s %s" % ( - self.cdist_object.path, self.parameters)) + self.log.trace("Finished %s %s", self.cdist_object.path, + self.parameters) def __init_log(self): """Setup logging facility""" @@ -170,7 +170,7 @@ class Emulator: # And finally parse/verify parameter self.args = parser.parse_args(self.argv[1:]) - self.log.trace('Args: %s' % self.args) + self.log.trace('Args: %s', self.args) def init_object(self): # Initialize object - and ensure it is not in args @@ -241,8 +241,8 @@ class Emulator: raise cdist.Error(errmsg) else: if self.cdist_object.exists: - self.log.debug(('Object %s override forced with ' - 'CDIST_OVERRIDE'), self.cdist_object.name) + self.log.debug('Object %s override forced with CDIST_OVERRIDE', + self.cdist_object.name) self.cdist_object.create(True) else: self.cdist_object.create() @@ -260,8 +260,8 @@ class Emulator: parent = self.cdist_object.object_from_name(__object_name) parent.typeorder.append(self.cdist_object.name) if self._order_dep_on(): - self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep' - ' for %s'), depname, parent.name) + self.log.trace('[ORDER_DEP] Adding %s to typeorder dep for %s', + depname, parent.name) parent.typeorder_dep.append(depname) elif self._order_dep_on(): self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep', @@ -301,16 +301,14 @@ class Emulator: try: cdist_object = self.cdist_object.object_from_name(requirement) except core.cdist_type.InvalidTypeError as e: - self.log.error(("%s requires object %s, but type %s does not" - " exist. Defined at %s" % ( - self.cdist_object.name, - requirement, e.name, self.object_source))) + self.log.error("%s requires object %s, but type %s does not" + " exist. Defined at %s", self.cdist_object.name, + requirement, e.name, self.object_source) raise except core.cdist_object.MissingObjectIdError: - self.log.error(("%s requires object %s without object id." - " Defined at %s" % (self.cdist_object.name, - requirement, - self.object_source))) + self.log.error("%s requires object %s without object id." + " Defined at %s", self.cdist_object.name, + requirement, self.object_source) raise self.log.debug("Recording requirement %s for %s", @@ -380,10 +378,9 @@ class Emulator: self.env['require'] += " " + lastcreatedtype else: self.env['require'] = lastcreatedtype - self.log.debug(("Injecting require for " - "CDIST_ORDER_DEPENDENCY: %s for %s"), - lastcreatedtype, - self.cdist_object.name) + self.log.debug("Injecting require for" + " CDIST_ORDER_DEPENDENCY: %s for %s", + lastcreatedtype, self.cdist_object.name) except IndexError: # if no second last line, we are on the first type, # so do not set a requirement @@ -391,7 +388,7 @@ class Emulator: if "require" in self.env: requirements = self.env['require'] - self.log.debug("reqs = " + requirements) + self.log.debug("reqs = %s", requirements) for requirement in self._parse_require(requirements): # Ignore empty fields - probably the only field anyway if len(requirement) == 0: diff --git a/cdist/exec/local.py b/cdist/exec/local.py index e0aab190..6713cd13 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -154,8 +154,8 @@ class Local: with open(self.object_marker_file, 'w') as fd: fd.write("%s\n" % self.object_marker_name) - self.log.trace("Object marker %s saved in %s" % ( - self.object_marker_name, self.object_marker_file)) + self.log.trace("Object marker %s saved in %s", + self.object_marker_name, self.object_marker_file) def _init_cache_dir(self, cache_dir): home_dir = cdist.home_dir() @@ -289,14 +289,12 @@ class Local: return cache_subpath def save_cache(self, start_time=time.time()): - self.log.trace("cache subpath pattern: {}".format( - self.cache_path_pattern)) + self.log.trace("cache subpath pattern: %s", self.cache_path_pattern) cache_subpath = self._cache_subpath(start_time, self.cache_path_pattern) - self.log.debug("cache subpath: {}".format(cache_subpath)) + self.log.debug("cache subpath: %s", cache_subpath) destination = os.path.join(self.cache_path, cache_subpath) - self.log.trace(("Saving cache: " + self.base_path + " to " + - destination)) + self.log.trace("Saving cache %s to %s", self.base_path, destination) if not os.path.exists(destination): shutil.move(self.base_path, destination) @@ -332,7 +330,7 @@ class Local: # Iterate over all directories and link the to the output dir for conf_dir in self.conf_dirs: - self.log.debug("Checking conf_dir %s ..." % (conf_dir)) + self.log.debug("Checking conf_dir %s ...", conf_dir) for sub_dir in CONF_SUBDIRS_LINKED: current_dir = os.path.join(conf_dir, sub_dir) @@ -350,7 +348,7 @@ class Local: if os.path.exists(dst): os.unlink(dst) - self.log.trace("Linking %s to %s ..." % (src, dst)) + self.log.trace("Linking %s to %s ...", src, dst) try: os.symlink(src, dst) except OSError as e: diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index e5af2f34..ea85da4c 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -176,19 +176,19 @@ class Remote: # create archive tarpath, fcnt = autil.tar(source, self.archiving_mode) if tarpath is None: - self.log.trace(("Files count {} is lower than {} limit, " - "skipping archiving").format( - fcnt, autil.FILES_LIMIT)) + self.log.trace("Files count %d is lower than %d limit, " + "skipping archiving", + fcnt, autil.FILES_LIMIT) else: - self.log.trace(("Archiving mode, tarpath: %s, file count: " - "%s"), tarpath, fcnt) + self.log.trace("Archiving mode, tarpath: %s, file count: " + "%s", tarpath, fcnt) # get archive name tarname = os.path.basename(tarpath) self.log.trace("Archiving mode tarname: %s", tarname) # archive path at the remote desttarpath = os.path.join(destination, tarname) - self.log.trace( - "Archiving mode desttarpath: %s", desttarpath) + self.log.trace("Archiving mode desttarpath: %s", + desttarpath) # transfer archive to the remote side self.log.trace("Archiving mode: transferring") self._transfer_file(tarpath, desttarpath) diff --git a/cdist/install.py b/cdist/install.py index 7c894fe5..d077baef 100644 --- a/cdist/install.py +++ b/cdist/install.py @@ -47,4 +47,4 @@ class Install(cdist.config.Config): yield cdist_object else: self.log.debug("Running in install mode, ignoring non install" - "object: {0}".format(cdist_object)) + "object: %s", cdist_object) diff --git a/cdist/inventory.py b/cdist/inventory.py index 0387f326..106052a2 100644 --- a/cdist/inventory.py +++ b/cdist/inventory.py @@ -92,7 +92,7 @@ class Inventory: self.init_db() def init_db(self): - self.log.trace("Init db: {}".format(self.db_basedir)) + self.log.trace("Init db: %s", self.db_basedir) if not os.path.exists(self.db_basedir): os.makedirs(self.db_basedir, exist_ok=True) elif not os.path.isdir(self.db_basedir): @@ -182,9 +182,9 @@ class Inventory: configuration = cfg.get_config(section='GLOBAL') determine_default_inventory_dir(args, configuration) - log.debug("Using inventory: {}".format(args.inventory_dir)) - log.trace("Inventory args: {}".format(vars(args))) - log.trace("Inventory command: {}".format(args.subcommand)) + log.debug("Using inventory: %s", args.inventory_dir) + log.trace("Inventory args: %s", vars(args)) + log.trace("Inventory command: %s", args.subcommand) if args.subcommand == "list": c = InventoryList(hosts=args.host, istag=args.tag, @@ -237,16 +237,16 @@ class InventoryList(Inventory): def _do_list(self, it_tags, it_hosts, check_func): if (it_tags is not None): param_tags = set(it_tags) - self.log.trace("param_tags: {}".format(param_tags)) + self.log.trace("param_tags: %s", param_tags) else: param_tags = set() for host in it_hosts: - self.log.trace("host: {}".format(host)) + self.log.trace("host: %s", host) tags = self._get_host_tags(host) if tags is None: - self.log.debug("Host \'{}\' not found, skipped".format(host)) + self.log.debug("Host \'%s\' not found, skipped", host) continue - self.log.trace("tags: {}".format(tags)) + self.log.trace("tags: %s", tags) if check_func(tags, param_tags): yield host, tags @@ -308,11 +308,11 @@ class InventoryHost(Inventory): def _action(self, host): if self.action == "add": - self.log.debug("Adding host \'{}\'".format(host)) + self.log.debug("Adding host \'%s\'", host) elif self.action == "del": - self.log.debug("Deleting host \'{}\'".format(host)) + self.log.debug("Deleting host \'%s\'", host) hostpath = self._host_path(host) - self.log.trace("hostpath: {}".format(hostpath)) + self.log.trace("hostpath: %s", hostpath) if self.action == "add" and not os.path.exists(hostpath): self._new_hostpath(hostpath) else: @@ -372,23 +372,23 @@ class InventoryTag(Inventory): print("Host \'{}\' does not exist, skipping".format(host), file=sys.stderr) return - self.log.trace("existing host_tags: {}".format(host_tags)) + self.log.trace("existing host_tags: %s", host_tags) if self.action == "del" and self.all: host_tags = set() else: for tag in self.input_tags: if self.action == "add": - self.log.debug("Adding tag \'{}\' for host \'{}\'".format( - tag, host)) + self.log.debug("Adding tag \'%s\' for host \'%s\'", + tag, host) host_tags.add(tag) elif self.action == "del": - self.log.debug("Deleting tag \'{}\' for host " - "\'{}\'".format(tag, host)) + self.log.debug("Deleting tag \'%s\' for host \'%s\'", + tag, host) if tag in host_tags: host_tags.remove(tag) - self.log.trace("new host tags: {}".format(host_tags)) + self.log.trace("new host tags: %s", host_tags) if not self._write_host_tags(host, host_tags): - self.log.trace("{} does not exist, skipped".format(host)) + self.log.trace("%s does not exist, skipped", host) def run(self): if self.allhosts: diff --git a/cdist/preos.py b/cdist/preos.py index f8a5dd67..45711a41 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -49,7 +49,7 @@ def scan_preos_dir_plugins(dir): c = cm[1] yield from preos_plugin(c) except ImportError as e: - log.warning("Cannot import '{}': {}".format(module_name, e)) + log.warning("Cannot import '%s': %s", module_name, e) def find_preos_plugins(): @@ -102,7 +102,7 @@ class PreOS: parser.add_argument('remainder_args', nargs=argparse.REMAINDER) args = parser.parse_args(argv[1:]) cdist.argparse.handle_loglevel(args) - log.debug("preos args : {}".format(args)) + log.debug("preos args : %s", args) conf_dirs = util.resolve_conf_dirs_from_config_and_args(args) @@ -122,7 +122,7 @@ class PreOS: func_args = [preos, args.remainder_args, ] else: func_args = [args.remainder_args, ] - log.info("Running preos : {}".format(preos_name)) + log.info("Running preos : %s", preos_name) func(*func_args) else: raise cdist.Error( diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py index a20cdb9c..f1f750ee 100644 --- a/cdist/preos/debootstrap/debootstrap.py +++ b/cdist/preos/debootstrap/debootstrap.py @@ -166,7 +166,7 @@ class Debian: args.pxe_boot_dir = os.path.realpath(args.pxe_boot_dir) cdist.argparse.handle_loglevel(args) - log.debug("preos: {}, args: {}".format(cls._preos_name, args)) + log.debug("preos: %s, args: %s", cls._preos_name, args) try: env = vars(args) new_env = {} @@ -190,27 +190,30 @@ class Debian: env = new_env env.update(os.environ) cls.update_env(env) - log.debug("preos: {} env: {}".format(cls._preos_name, env)) + log.debug("preos: %s env: %s", cls._preos_name, env) + + if log.getEffectiveLevel() <= logging.INFO: + info_msg = ["Running preos: {}, suite: {}, arch: {}".format( + cls._preos_name, args.suite, args.arch), ] + if args.mirror: + info_msg.append("mirror: {}".format(args.mirror)) + if args.script: + info_msg.append("script: {}".format(args.script)) + if args.bootstrap: + info_msg.append("bootstrapping") + if args.configure: + info_msg.append("configuring") + if args.pxe_boot_dir: + info_msg.append("creating PXE") + if args.drive: + info_msg.append("creating bootable drive") + log.info(info_msg) + cmd = os.path.join(cls._files_dir, "code") - info_msg = ["Running preos: {}, suite: {}, arch: {}".format( - cls._preos_name, args.suite, args.arch), ] - if args.mirror: - info_msg.append("mirror: {}".format(args.mirror)) - if args.script: - info_msg.append("script: {}".format(args.script)) - if args.bootstrap: - info_msg.append("bootstrapping") - if args.configure: - info_msg.append("configuring") - if args.pxe_boot_dir: - info_msg.append("creating PXE") - if args.drive: - info_msg.append("creating bootable drive") - log.info(info_msg) - log.debug("cmd={}".format(cmd)) + log.debug("cmd=%s", cmd) subprocess.check_call(cmd, env=env, shell=True) except subprocess.CalledProcessError as e: - log.error("preos {} failed: {}".format(cls._preos_name, e)) + log.error("preos %s failed: %s", cls._preos_name, e) class Ubuntu(Debian): diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index b1d0e9e1..9a0bcc52 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -94,7 +94,7 @@ class Trigger(object): def trigger(self, interface): packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest() - log.debug(f"Sending request on {interface}") + log.debug("Sending request on %s", interface) send(packet, verbose=self.verbose) @@ -114,7 +114,7 @@ class Scanner(object): def handle_pkg(self, pkg): if ICMPv6EchoReply in pkg: host = pkg['IPv6'].src - log.verbose(f"Host {host} is alive") + log.verbose("Host %s is alive", host) dir = os.path.join(self.outdir, host) fname = os.path.join(dir, "last_seen") diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py index 95ca74ee..d9e5f498 100644 --- a/cdist/util/ipaddr.py +++ b/cdist/util/ipaddr.py @@ -42,8 +42,7 @@ def resolve_target_host_name(host, family=0): # gethostbyaddr returns triple # (hostname, aliaslist, ipaddrlist) host_name = socket.gethostbyaddr(ip_addr)[0] - log.debug("derived host_name for host \"{}\": {}".format( - host, host_name)) + log.debug("derived host_name for host \"%s\": %s", host, host_name) except (socket.gaierror, socket.herror) as e: # in case of error provide empty value host_name = '' @@ -54,8 +53,7 @@ def resolve_target_fqdn(host): log = logging.getLogger(host) try: host_fqdn = socket.getfqdn(host) - log.debug("derived host_fqdn for host \"{}\": {}".format( - host, host_fqdn)) + log.debug("derived host_fqdn for host \"%s\": %s", host, host_fqdn) except socket.herror as e: # in case of error provide empty value host_fqdn = '' From 4c2d273f07e6cedb3dcb539d132cb0c3f8176d42 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 30 Mar 2021 07:56:38 +0200 Subject: [PATCH 058/206] Unify string formatting Use one way of string formatting: replace old `%` style with new `str.format`. Resolve #855. --- cdist/config.py | 4 ++-- cdist/core/cdist_object.py | 14 +++++++------- cdist/core/cdist_type.py | 4 ++-- cdist/core/code.py | 8 ++++---- cdist/core/explorer.py | 6 +++--- cdist/core/manifest.py | 11 +++++------ cdist/emulator.py | 12 ++++++------ cdist/exec/local.py | 11 ++++++----- cdist/exec/remote.py | 11 +++++------ cdist/message.py | 2 +- cdist/scan/scan.py | 2 +- cdist/shell.py | 2 +- cdist/test/cdist_object/__init__.py | 27 +++++++++++++++------------ cdist/test/emulator/__init__.py | 4 ++-- cdist/test/exec/remote.py | 8 ++++---- cdist/test/message/__init__.py | 2 +- cdist/util/fsproperty.py | 4 ++-- 17 files changed, 67 insertions(+), 65 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index d6fec55f..adc460de 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -190,7 +190,7 @@ class Config: fd.write(sys.stdin.read()) except (IOError, OSError) as e: raise cdist.Error(("Creating tempfile for stdin data " - "failed: %s" % e)) + "failed: {}").format(e)) args.manifest = initial_manifest_temp_path atexit.register(lambda: os.remove(initial_manifest_temp_path)) @@ -764,7 +764,7 @@ class Config: raise cdist.UnresolvableRequirementsError( ("The requirements of the following objects could not be " - "resolved:\n%s") % ("\n".join(info_string))) + "resolved:\n{}").format("\n".join(info_string))) def _handle_deprecation(self, cdist_object): cdist_type = cdist_object.cdist_type diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index eea2c255..ccf7392d 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -34,17 +34,17 @@ class IllegalObjectIdError(cdist.Error): self.message = message or 'Illegal object id' def __str__(self): - return '%s: %s' % (self.message, self.object_id) + return '{}: {}'.format(self.message, self.object_id) class MissingObjectIdError(cdist.Error): def __init__(self, type_name): self.type_name = type_name - self.message = ("Type %s requires object id (is not a " - "singleton type)") % self.type_name + self.message = ("Type {} requires object id (is not a " + "singleton type)").format(self.type_name) def __str__(self): - return '%s' % (self.message) + return '{}'.format(self.message) class CdistObject: @@ -142,7 +142,7 @@ class CdistObject: if self.object_marker in self.object_id.split(os.sep): raise IllegalObjectIdError( self.object_id, ('object_id may not contain ' - '\'%s\'') % self.object_marker) + '\'{}\'').format(self.object_marker)) if '//' in self.object_id: raise IllegalObjectIdError( self.object_id, 'object_id may not contain //') @@ -189,7 +189,7 @@ class CdistObject: object_id=object_id) def __repr__(self): - return '' % self.name + return ''.format(self.name) def __eq__(self, other): """define equality as 'name is the same'""" @@ -277,7 +277,7 @@ class CdistObject: os.makedirs(path, exist_ok=allow_overwrite) except EnvironmentError as error: raise cdist.Error(('Error creating directories for cdist object: ' - '%s: %s') % (self, error)) + '{}: {}').format(self, error)) def requirements_unfinished(self, requirements): """Return state whether requirements are satisfied""" diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index 274de989..b68448bc 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -34,7 +34,7 @@ class InvalidTypeError(cdist.Error): self.source_path = os.path.realpath(self.type_absolute_path) def __str__(self): - return "Invalid type '%s' at '%s' defined at '%s'" % ( + return "Invalid type '{}' at '{}' defined at '{}'".format( self.type_path, self.type_absolute_path, self.source_path) @@ -109,7 +109,7 @@ class CdistType: return cls._instances[name] def __repr__(self): - return '' % self.name + return ''.format(self.name) def __eq__(self, other): return isinstance(other, self.__class__) and self.name == other.name diff --git a/cdist/core/code.py b/cdist/core/code.py index 1e9b4f80..12888ba4 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -122,8 +122,8 @@ class Code: def _run_gencode(self, cdist_object, which): cdist_type = cdist_object.cdist_type - script = os.path.join(self.local.type_path, - getattr(cdist_type, 'gencode_%s_path' % which)) + gencode_attr = getattr(cdist_type, 'gencode_{}_path'.format(which)) + script = os.path.join(self.local.type_path, gencode_attr) if os.path.isfile(script): env = os.environ.copy() env.update(self.env) @@ -167,8 +167,8 @@ class Code: def _run_code(self, cdist_object, which, env=None): which_exec = getattr(self, which) - script = os.path.join(which_exec.object_path, - getattr(cdist_object, 'code_%s_path' % which)) + code_attr = getattr(cdist_object, 'code_{}_path'.format(which)) + script = os.path.join(which_exec.object_path, code_attr) if which_exec.save_output_streams: stderr_path = os.path.join(cdist_object.stderr_path, 'code-' + which) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index caa12a7d..48414ade 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -160,8 +160,8 @@ class Explorer: self.remote.transfer(self.local.global_explorer_path, self.remote.global_explorer_path, self.jobs) - self.remote.run(["chmod", "0700", - "%s/*" % (self.remote.global_explorer_path)]) + self.remote.run(["chmod", "0700", "{}/*".format( + self.remote.global_explorer_path)]) def run_global_explorer(self, explorer): """Run the given global explorer and return it's output.""" @@ -242,7 +242,7 @@ class Explorer: destination = os.path.join(self.remote.type_path, cdist_type.explorer_path) self.remote.transfer(source, destination) - self.remote.run(["chmod", "0700", "%s/*" % (destination)]) + self.remote.run(["chmod", "0700", "{}/*".format(destination)]) self._type_explorers_transferred.append(cdist_type.name) def transfer_object_parameters(self, cdist_object): diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 3148d66c..09e74dac 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -80,13 +80,12 @@ class NoInitialManifestError(cdist.Error): if user_supplied: if os.path.islink(manifest_path): - self.message = "%s: %s -> %s" % ( - msg_header, manifest_path, - os.path.realpath(manifest_path)) + self.message = "{}: {} -> {}".format( + msg_header, manifest_path, os.path.realpath(manifest_path)) else: - self.message = "%s: %s" % (msg_header, manifest_path) + self.message = "{}: {}".format(msg_header, manifest_path) else: - self.message = "%s" % (msg_header) + self.message = "{}".format(msg_header) def __str__(self): return repr(self.message) @@ -107,7 +106,7 @@ class Manifest: self._open_logger() self.env = { - 'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']), + 'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']), # for use in type emulator '__cdist_type_base_path': self.local.type_path, '__global': self.local.base_path, diff --git a/cdist/emulator.py b/cdist/emulator.py index f09c282d..c55b47d2 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -36,8 +36,8 @@ from cdist.core.manifest import Manifest class MissingRequiredEnvironmentVariableError(cdist.Error): def __init__(self, name): self.name = name - self.message = ("Emulator requires the environment variable %s to be " - "setup" % self.name) + self.message = ("Emulator requires the environment variable {} to be " + "setup").format(self.name) def __str__(self): return self.message @@ -231,13 +231,13 @@ class Emulator: if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env: obj_params = self._object_params_in_context() if obj_params != self.parameters: - errmsg = ("Object %s already exists with conflicting " - "parameters:\n%s: %s\n%s: %s" % ( + errmsg = ("Object {} already exists with conflicting " + "parameters:\n{}: {}\n{}: {}").format( self.cdist_object.name, " ".join(self.cdist_object.source), obj_params, self.object_source, - self.parameters)) + self.parameters) raise cdist.Error(errmsg) else: if self.cdist_object.exists: @@ -292,7 +292,7 @@ class Emulator: fd.write(chunk) chunk = self._read_stdin() except EnvironmentError as e: - raise cdist.Error('Failed to read from stdin: %s' % e) + raise cdist.Error('Failed to read from stdin: {}'.format(e)) def record_requirement(self, requirement): """record requirement and return recorded requirement""" diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 6713cd13..370336ad 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -152,7 +152,7 @@ class Local: def _setup_object_marker_file(self): with open(self.object_marker_file, 'w') as fd: - fd.write("%s\n" % self.object_marker_name) + fd.write("{}\n".format(self.object_marker_name)) self.log.trace("Object marker %s saved in %s", self.object_marker_name, self.object_marker_file) @@ -184,7 +184,7 @@ class Local: """ assert isinstance(command, (list, tuple)), ( - "list or tuple argument expected, got: %s" % command) + "list or tuple argument expected, got: {}".format(command)) quiet = self.quiet_mode or quiet_mode do_save_output = save_output and not quiet and self.save_output_streams @@ -352,8 +352,9 @@ class Local: try: os.symlink(src, dst) except OSError as e: - raise cdist.Error("Linking %s %s to %s failed: %s" % ( - sub_dir, src, dst, e.__str__())) + raise cdist.Error( + "Linking {} {} to {} failed: {}".format( + sub_dir, src, dst, e.__str__())) def _link_types_for_emulator(self): """Link emulator to types""" @@ -366,5 +367,5 @@ class Local: os.symlink(src, dst) except OSError as e: raise cdist.Error( - "Linking emulator from %s to %s failed: %s" % ( + "Linking emulator from {} to {} failed: {}".format( src, dst, e.__str__())) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index ea85da4c..af9e70cb 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -24,12 +24,10 @@ import os import glob import subprocess import logging -import multiprocessing import cdist import cdist.exec.util as util import cdist.util.ipaddr as ipaddr -from cdist.mputil import mp_pool_run def _wrap_addr(addr): @@ -262,9 +260,10 @@ class Remote: # remotely in e.g. csh and setting up CDIST_REMOTE_SHELL to e.g. # /bin/csh will execute this script in the right way. if env: - remote_env = [" export %s=%s;" % item for item in env.items()] - string_cmd = ("/bin/sh -c '" + " ".join(remote_env) + - " ".join(command) + "'") + remote_env = [" export {env[0]}={env[1]};".format(env=item) + for item in env.items()] + string_cmd = ("/bin/sh -c '{}{}'").format(" ".join(remote_env), + " ".join(command)) cmd.append(string_cmd) else: cmd.extend(command) @@ -278,7 +277,7 @@ class Remote: """ assert isinstance(command, (list, tuple)), ( - "list or tuple argument expected, got: %s" % command) + "list or tuple argument expected, got: {}".format(command)) close_stdout = False close_stderr = False diff --git a/cdist/message.py b/cdist/message.py index ffa8c2bb..0c4e21a6 100644 --- a/cdist/message.py +++ b/cdist/message.py @@ -70,7 +70,7 @@ class Message: with open(self.global_messages, 'a') as fd: for line in content: - fd.write("%s:%s" % (self.prefix, line)) + fd.write("{}:{}".format(self.prefix, line)) def merge_messages(self): self._merge_messages() diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 9a0bcc52..faee8a56 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -93,7 +93,7 @@ class Trigger(object): time.sleep(self.sleeptime) def trigger(self, interface): - packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest() + packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest() log.debug("Sending request on %s", interface) send(packet, verbose=self.verbose) diff --git a/cdist/shell.py b/cdist/shell.py index 04a68937..05803556 100644 --- a/cdist/shell.py +++ b/cdist/shell.py @@ -65,7 +65,7 @@ class Shell: def _init_environment(self): self.env = os.environ.copy() additional_env = { - 'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']), + 'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']), # for use in type emulator '__cdist_type_base_path': self.local.type_path, '__cdist_manifest': "cdist shell", diff --git a/cdist/test/cdist_object/__init__.py b/cdist/test/cdist_object/__init__.py index a9c20cd3..11619e96 100644 --- a/cdist/test/cdist_object/__init__.py +++ b/cdist/test/cdist_object/__init__.py @@ -86,8 +86,7 @@ class ObjectClassTestCase(test.CdistTestCase): def test_create_singleton(self): """Check whether creating an object without id (singleton) works""" - singleton = self.expected_objects[0].object_from_name( - "__test_singleton") + self.expected_objects[0].object_from_name("__test_singleton") # came here - everything fine def test_create_singleton_not_singleton_type(self): @@ -126,16 +125,16 @@ class ObjectIdTestCase(test.CdistTestCase): def test_object_id_contains_object_marker(self): cdist_type = core.CdistType(type_base_path, '__third') - illegal_object_id = ( - 'object_id/may/not/contain/%s/anywhere' % OBJECT_MARKER_NAME) + illegal_object_id = 'object_id/may/not/contain/{}/anywhere'.format( + OBJECT_MARKER_NAME) with self.assertRaises(core.IllegalObjectIdError): core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) def test_object_id_contains_object_marker_string(self): cdist_type = core.CdistType(type_base_path, '__third') - illegal_object_id = ( - 'object_id/may/contain_%s_in_filename' % OBJECT_MARKER_NAME) + illegal_object_id = 'object_id/may/contain_{}_in_filename'.format( + OBJECT_MARKER_NAME) core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) # if we get here, the test passed @@ -195,28 +194,32 @@ class ObjectTestCase(test.CdistTestCase): def test_path(self): self.assertEqual(self.cdist_object.path, - "__third/moon/%s" % OBJECT_MARKER_NAME) + "__third/moon/{}".format(OBJECT_MARKER_NAME)) def test_absolute_path(self): self.assertEqual(self.cdist_object.absolute_path, os.path.join(self.object_base_path, - "__third/moon/%s" % OBJECT_MARKER_NAME)) + "__third/moon/{}".format( + OBJECT_MARKER_NAME))) def test_code_local_path(self): self.assertEqual(self.cdist_object.code_local_path, - "__third/moon/%s/code-local" % OBJECT_MARKER_NAME) + "__third/moon/{}/code-local".format( + OBJECT_MARKER_NAME)) def test_code_remote_path(self): self.assertEqual(self.cdist_object.code_remote_path, - "__third/moon/%s/code-remote" % OBJECT_MARKER_NAME) + "__third/moon/{}/code-remote".format( + OBJECT_MARKER_NAME)) def test_parameter_path(self): self.assertEqual(self.cdist_object.parameter_path, - "__third/moon/%s/parameter" % OBJECT_MARKER_NAME) + "__third/moon/{}/parameter".format( + OBJECT_MARKER_NAME)) def test_explorer_path(self): self.assertEqual(self.cdist_object.explorer_path, - "__third/moon/%s/explorer" % OBJECT_MARKER_NAME) + "__third/moon/{}/explorer".format(OBJECT_MARKER_NAME)) def test_parameters(self): expected_parameters = {'planet': 'Saturn', 'name': 'Prometheus'} diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index befd7b57..4b2bc3ba 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -84,8 +84,8 @@ class EmulatorTestCase(test.CdistTestCase): def test_illegal_object_id_requirement(self): argv = ['__file', '/tmp/foobar'] - self.env['require'] = ( - "__file/bad/id/with/%s/inside") % self.local.object_marker_name + self.env['require'] = "__file/bad/id/with/{}/inside".format( + self.local.object_marker_name) emu = emulator.Emulator(argv, env=self.env) self.assertRaises(core.IllegalObjectIdError, emu.run) diff --git a/cdist/test/exec/remote.py b/cdist/test/exec/remote.py index a7fe384d..b23f8447 100644 --- a/cdist/test/exec/remote.py +++ b/cdist/test/exec/remote.py @@ -47,8 +47,8 @@ class RemoteTestCase(test.CdistTestCase): args = (self.target_host,) kwargs.setdefault('base_path', self.base_path) user = getpass.getuser() - kwargs.setdefault('remote_exec', 'ssh -o User=%s -q' % user) - kwargs.setdefault('remote_copy', 'scp -o User=%s -q' % user) + kwargs.setdefault('remote_exec', 'ssh -o User={} -q'.format(user)) + kwargs.setdefault('remote_copy', 'scp -o User={} -q'.format(user)) if 'stdout_base_path' not in kwargs: stdout_path = os.path.join(self.temp_dir, 'stdout') os.makedirs(stdout_path, exist_ok=True) @@ -170,7 +170,7 @@ class RemoteTestCase(test.CdistTestCase): r = self.create_remote(remote_exec=remote_exec, remote_copy=remote_copy) self.assertEqual(r.run('true', return_output=True), - "%s\n" % self.target_host[0]) + "{}\n".format(self.target_host[0])) def test_run_script_target_host_in_env(self): handle, remote_exec_path = self.mkstemp(dir=self.temp_dir) @@ -185,7 +185,7 @@ class RemoteTestCase(test.CdistTestCase): with os.fdopen(handle, "w") as fd: fd.writelines(["#!/bin/sh\n", "true"]) self.assertEqual(r.run_script(script, return_output=True), - "%s\n" % self.target_host[0]) + "{}\n".format(self.target_host[0])) def test_run_script_with_env_target_host_in_env(self): handle, script = self.mkstemp(dir=self.temp_dir) diff --git a/cdist/test/message/__init__.py b/cdist/test/message/__init__.py index 61cd5d97..55040fc9 100644 --- a/cdist/test/message/__init__.py +++ b/cdist/test/message/__init__.py @@ -67,7 +67,7 @@ class MessageTestCase(test.CdistTestCase): def test_message_merge_prefix(self): """Ensure messages are merged and are prefixed""" - expectedcontent = "%s:%s" % (self.prefix, self.content) + expectedcontent = "{}:{}".format(self.prefix, self.content) out = self.message.env['__messages_out'] diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py index 1d76fd76..09e9cc19 100644 --- a/cdist/util/fsproperty.py +++ b/cdist/util/fsproperty.py @@ -30,7 +30,7 @@ class AbsolutePathRequiredError(cdist.Error): self.path = path def __str__(self): - return 'Absolute path required, got: %s' % self.path + return 'Absolute path required, got: {}'.format(self.path) class FileList(collections.MutableSequence): @@ -218,7 +218,7 @@ class FileBasedProperty: def _get_attribute(self, instance, owner): name = self._get_property_name(owner) - attribute_name = '__%s' % name + attribute_name = '__{}'.format(name) if not hasattr(instance, attribute_name): path = self._get_path(instance) attribute_instance = self.attribute_class(path) From ab811ad282882558a4639986e75c800e1b63c2e0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 1 Apr 2021 15:37:11 +0200 Subject: [PATCH 059/206] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 03ae7566..db60c0c4 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,8 @@ next: * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) * Type __sshd_config: Whitelist OpenBMC (Dennis Camera) * Core: Maintain object relationship graph in cdist cache (Darko Poljak) + * Type __git: Fix numeric owner and group handling (Dennis Camera) + * Type __pyvenv: Fix numeric owner and group handling (Dennis Camera) 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) From 199effb7ef23980bc69f0233ff6d7ee465623f40 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 6 Apr 2021 19:35:14 +0200 Subject: [PATCH 060/206] Improve unfinished object requirements bool check When we need only boolean value for unfinished object requirements then we don't need to determine the whole list of unfinished objects. --- cdist/config.py | 12 +++++++----- cdist/core/cdist_object.py | 13 ++++++++++++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index adc460de..f7da9b1d 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -537,7 +537,7 @@ class Config: objects_changed = False for cdist_object in self.object_list(): - if cdist_object.requirements_unfinished( + if cdist_object.has_requirements_unfinished( cdist_object.requirements): """We cannot do anything for this poor object""" continue @@ -548,7 +548,7 @@ class Config: self.object_prepare(cdist_object) objects_changed = True - if cdist_object.requirements_unfinished( + if cdist_object.has_requirements_unfinished( cdist_object.autorequire): """The previous step created objects we depend on - wait for them @@ -567,7 +567,8 @@ class Config: cargo = [] for cdist_object in self.object_list(): - if cdist_object.requirements_unfinished(cdist_object.requirements): + if cdist_object.has_requirements_unfinished( + cdist_object.requirements): """We cannot do anything for this poor object""" continue @@ -621,12 +622,13 @@ class Config: del cargo[:] for cdist_object in self.object_list(): - if cdist_object.requirements_unfinished(cdist_object.requirements): + if cdist_object.has_requirements_unfinished( + cdist_object.requirements): """We cannot do anything for this poor object""" continue if cdist_object.state == core.CdistObject.STATE_PREPARED: - if cdist_object.requirements_unfinished( + if cdist_object.has_requirements_unfinished( cdist_object.autorequire): """The previous step created objects we depend on - wait for them diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index ccf7392d..bb3a65bd 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -280,7 +280,7 @@ class CdistObject: '{}: {}').format(self, error)) def requirements_unfinished(self, requirements): - """Return state whether requirements are satisfied""" + """Return unsatisfied requirements""" object_list = [] @@ -291,3 +291,14 @@ class CdistObject: object_list.append(cdist_object) return object_list + + def has_requirements_unfinished(self, requirements): + """Return whether requirements are satisfied""" + + for requirement in requirements: + cdist_object = self.object_from_name(requirement) + + if cdist_object.state != self.STATE_DONE: + return True + + return False From 750c71fb5a9fcf4a30af0700b7a09e3501988a27 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 7 Apr 2021 09:07:29 +0200 Subject: [PATCH 061/206] Minor refactoring and remove code duplication --- cdist/config.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index f7da9b1d..638fdf0e 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -699,20 +699,22 @@ class Config: check for cycles. ''' graph = {} - for cdist_object in self.object_list(): + + def _add_requirements(cdist_object, requirements): obj_name = cdist_object.name if obj_name not in graph: graph[obj_name] = [] + + for requirement in cdist_object.requirements_unfinished( + requirements): + graph[obj_name].append(requirement.name) + + for cdist_object in self.object_list(): if cdist_object.state == cdist_object.STATE_DONE: continue - for requirement in cdist_object.requirements_unfinished( - cdist_object.requirements): - graph[obj_name].append(requirement.name) - - for requirement in cdist_object.requirements_unfinished( - cdist_object.autorequire): - graph[obj_name].append(requirement.name) + _add_requirements(cdist_object, cdist_object.requirements) + _add_requirements(cdist_object, cdist_object.autorequire) return graph_check_cycle(graph) def iterate_until_finished(self): From d2eec6066876eeb4a295b81a670c70b0ff8ad2a7 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 11 Apr 2021 23:05:48 +0300 Subject: [PATCH 062/206] [__download] make --sum optional --- cdist/conf/type/__download/explorer/state | 6 ++++++ cdist/conf/type/__download/man.rst | 15 ++++++--------- cdist/conf/type/__download/parameter/optional | 1 + cdist/conf/type/__download/parameter/required | 1 - 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 00362545..68b517c5 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -8,6 +8,12 @@ then exit 0 fi +if [ ! -f "$__object/parameter/sum" ] +then + echo 'present' + exit 0 +fi + sum_should="$( cat "$__object/parameter/sum" )" if [ -f "$__object/parameter/cmd-sum" ] diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index 54503470..a1278cfb 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -8,9 +8,6 @@ cdist-type__download - Download a file DESCRIPTION ----------- -Destination (``$__object_id``) in target host must be persistent storage -in order to calculate checksum and decide if file must be (re-)downloaded. - By default type will try to use ``wget``, ``curl`` or ``fetch``. If download happens in target (see ``--download``) then type will fallback to (and install) ``wget``. @@ -25,14 +22,14 @@ REQUIRED PARAMETERS url File's URL. -sum - Checksum of file going to be downloaded. - By default output of ``cksum`` without filename is expected. - Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. - OPTIONAL PARAMETERS ------------------- +sum + Checksum is used to decide if existing destination file must be redownloaded. + By default output of ``cksum`` without filename is expected. + Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. + download If ``local`` (default), then download file to local storage and copy it to target host. If ``remote``, then download happens in target. @@ -81,7 +78,7 @@ Ander Punnar COPYING ------- -Copyright \(C) 2020 Ander Punnar. You can redistribute it +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional index 838e2fbf..d69e083e 100644 --- a/cdist/conf/type/__download/parameter/optional +++ b/cdist/conf/type/__download/parameter/optional @@ -1,3 +1,4 @@ +sum cmd-get cmd-sum download diff --git a/cdist/conf/type/__download/parameter/required b/cdist/conf/type/__download/parameter/required index 6ea4c38f..96cdd3b9 100644 --- a/cdist/conf/type/__download/parameter/required +++ b/cdist/conf/type/__download/parameter/required @@ -1,2 +1 @@ url -sum From 9ec01d9f971339ee67c866c4a248047966f73ce1 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 13 Apr 2021 12:22:45 +0200 Subject: [PATCH 063/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index db60c0c4..c8cb44da 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ next: * Core: Maintain object relationship graph in cdist cache (Darko Poljak) * Type __git: Fix numeric owner and group handling (Dennis Camera) * Type __pyvenv: Fix numeric owner and group handling (Dennis Camera) + * Type __download: Make sum parameter optional (Ander Punnar) 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) From 92b8942a8c37bb985ef42991f3b7aaef0cd8073f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 09:00:32 +0200 Subject: [PATCH 064/206] [type/__postgres_conf] Add psql_exec function to state explorer --- cdist/conf/type/__postgres_conf/explorer/state | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index de6e8aa0..1a58751a 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -20,21 +20,24 @@ # postgres_user=$("${__type_explorer:?}/postgres_user") - conf_name=${__object_id:?} -su - "${postgres_user}" -c 'psql postgres -c "SELECT 1"' >/dev/null || { +quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } +psql_exec() { + su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" +} + +psql_exec 'SELECT 1' >/dev/null || { echo 'Connection to PostgreSQL server failed' >&2 exit 1 } -if su - "${postgres_user}" -c "psql postgres -twAc 'SHOW ${conf_name}'" \ - | cmp -s "${__object:?}/parameter/value" - +if psql_exec "SHOW ${conf_name}" | cmp -s "${__object:?}/parameter/value" - then echo present else # NOTE: SHOW/SET are case-insentitive, so this command should also be. - case $(su - "${postgres_user}" -c "psql postgres -tAwc \"SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')\"") + case $(psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')") in ('') # invalid configuration parameter From 2ccc03fef1e28c759a7918fa00ffcb45c76dc57a Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 09:02:40 +0200 Subject: [PATCH 065/206] [type/__postgres_conf] Add psql_conf_cmp function to state explorer --- cdist/conf/type/__postgres_conf/explorer/state | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index 1a58751a..be881be6 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -27,12 +27,16 @@ psql_exec() { su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" } +psql_conf_cmp() { + test "$(psql_exec "SHOW $1")" = "$2" +} + psql_exec 'SELECT 1' >/dev/null || { echo 'Connection to PostgreSQL server failed' >&2 exit 1 } -if psql_exec "SHOW ${conf_name}" | cmp -s "${__object:?}/parameter/value" - +if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" then echo present else From e0416403c4862cb8bc0f41a7c6f8a33d1a8656a5 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 09:04:07 +0200 Subject: [PATCH 066/206] [type/__postgres_conf] Add psql_conf_source function to state explorer --- cdist/conf/type/__postgres_conf/explorer/state | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index be881be6..9f54724a 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -27,6 +27,10 @@ psql_exec() { su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" } +psql_conf_source() { + # NOTE: SHOW/SET are case-insentitive, so this command should also be. + psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')" +} psql_conf_cmp() { test "$(psql_exec "SHOW $1")" = "$2" } @@ -40,8 +44,7 @@ if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" then echo present else - # NOTE: SHOW/SET are case-insentitive, so this command should also be. - case $(psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('${conf_name}')") + case $(psql_conf_source "${conf_name}") in ('') # invalid configuration parameter From 12c2995494ce582cb3adf1d942b36f6df035370a Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 14:46:36 +0200 Subject: [PATCH 067/206] [type/__postgres_conf] Implement complex state compare logic --- .../conf/type/__postgres_conf/explorer/state | 166 +++++++++++++++++- 1 file changed, 163 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index 9f54724a..e76540c5 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -22,6 +22,52 @@ postgres_user=$("${__type_explorer:?}/postgres_user") conf_name=${__object_id:?} +tolower() { printf '%s' "$*" | tr '[:upper:]' '[:lower:]'; } + +tobytes() { + # NOTE: This function treats everything as base 2. + # It is not compatible with SI units. + awk 'BEGIN { FS = "\n" } + /TB$/ { $0 = ($0 * 1024) "GB" } + /GB$/ { $0 = ($0 * 1024) "MB" } + /MB$/ { $0 = ($0 * 1024) "kB" } + /kB$/ { $0 = ($0 * 1024) "B" } + /B?$/ { sub(/ *B?$/, "") } + ($0*1) == $0 # is number + ' <<-EOF + $1 + EOF +} + +tomillisecs() { + awk 'BEGIN { FS = "\n" } + /d$/ { $0 = ($0 * 24) "h" } + /h$/ { $0 = ($0 * 60) "min" } + /min$/ { $0 = ($0 * 60) "s" } + /[^m]s$/ { $0 = ($0 * 1000) "ms" } + /ms$/ { $0 *= 1 } + ($0*1) == $0 # is number + ' <<-EOF + $1 + EOF +} + +tobool() { + # prints either 'on' or 'off' + case $(tolower "$1") + in + (t|true|y|yes|on|1) + echo 'on' ;; + (f|false|n|no|off|0) + echo 'off' ;; + (*) + printf 'Inavlid bool value: %s\n' "$2" >&2 + return 1 + ;; + esac + return 0 +} + quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } psql_exec() { su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" @@ -31,9 +77,123 @@ psql_conf_source() { # NOTE: SHOW/SET are case-insentitive, so this command should also be. psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')" } -psql_conf_cmp() { - test "$(psql_exec "SHOW $1")" = "$2" -} +psql_conf_cmp() ( + IFS='|' read -r lower_name vartype setting unit <<-EOF + $(psql_exec "SELECT lower(name), vartype, setting, unit FROM pg_settings WHERE lower(name) = lower('$1')") + EOF + + should_value=$2 + is_value=${setting} + + # The following case contains special cases for special settings. + case ${lower_name} + in + (archive_command) + if test "${setting}" = '(disabled)' + then + # DAFUQ PostgreSQL?! + # PostgreSQL returns (disabled) if the feature is inactive. + # We cannot compare the values unless it is enabled, first. + return 0 + fi + ;; + (archive_mode|backslash_quote|constraint_exclusion|force_parallel_mode|huge_pages|synchronous_commit) + # Although only 'on', 'off' are documented, PostgreSQL accepts all + # the "likely" variants of "on" and "off". + case $(tolower "${should_value}") + in + (on|off|true|false|yes|no|1|0) + should_value=$(tobool "${should_value}") + ;; + esac + ;; + esac + + case ${vartype} + in + (bool) + test -z "${unit}" || { + # please fix the explorer if this error occurs. + printf 'units are not supported for vartype: %s\n' "${vartype}" >&2 + exit 1 + } + + should_value=$(tobool "${should_value}") + + test "${is_value}" = "${should_value}" + ;; + (enum) + test -z "${unit}" || { + # please fix the explorer if this error occurs. + printf 'units are not supported with vartype: %s\n' "${vartype}" >&2 + exit 1 + } + + # NOTE: All enums that are currently defined are lower case, but + # PostgreSQL also accepts upper case spelling. + should_value=$(tolower "$2") + + test "${is_value}" = "${should_value}" + ;; + (integer) + # split multiples from unit, first (e.g. 8kB -> 8, kB) + case ${unit} + in + ([0-9]*) + multiple=${unit%%[!0-9]*} + unit=${unit##*[0-9 ]} + ;; + (*) multiple=1 ;; + esac + + is_value=$((setting * multiple))${unit} + + if expr "${should_value}" : '-\{0,1\}[0-9]*$' >/dev/null + then + # default unit + should_value=$((should_value * multiple))${unit} + fi + + # then, do conversion + # NOTE: these conversions work for integers only! + case ${unit} + in + (B|[kMGT]B) + # bytes + is_bytes=$(tobytes "${is_value}") + should_bytes=$(tobytes "${should_value}") + + test $((is_bytes)) -eq $((should_bytes)) + ;; + (ms|s|min|h|d) + # seconds + is_ms=$(tomillisecs "${is_value}") + should_ms=$(tomillisecs "${should_value}") + + test $((is_ms)) -eq $((should_ms)) + ;; + ('') + # no unit + is_int=${is_value} + should_int=${should_value} + + test $((is_int)) -eq $((should_int)) + ;; + esac + ;; + (real|string) + # NOTE: reals could possibly have units, but currently there none. + + test -z "${unit}" || { + # please fix the explorer if this error occurs. + printf 'units are not supported with vartype: %s\n' "${vartype}" >&2 + exit 1 + } + + test "${is_value}" = "${should_value}" + ;; + esac +) psql_exec 'SELECT 1' >/dev/null || { echo 'Connection to PostgreSQL server failed' >&2 From bef1433ba3adb022f8b50eb9df0fbbbf90757d76 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 14:46:50 +0200 Subject: [PATCH 068/206] [type/__postgres_conf] Accept empty values --- cdist/conf/type/__postgres_conf/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote index d0d247b4..27651600 100755 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -57,7 +57,7 @@ psql_cmd() { case ${state_should} in (present) - test -s "${__object:?}/parameter/value" || { + test -n "${__object:?}/parameter/value" || { echo 'Missing required parameter --value' >&2 exit 1 } From 686e4f0f2d498ee39a69e43f7535171f16f5fcbc Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 15:01:38 +0200 Subject: [PATCH 069/206] [type/__postgres_conf] Reverse state logic (decide based on source first) --- .../conf/type/__postgres_conf/explorer/state | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state index e76540c5..4b7b0a43 100644 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -200,23 +200,24 @@ psql_exec 'SELECT 1' >/dev/null || { exit 1 } -if psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" -then - echo present -else - case $(psql_conf_source "${conf_name}") - in - ('') - # invalid configuration parameter - # (error message was already printed by SHOW command above. - # Yes, it's a hack) - exit 1 - ;; - (default) - echo absent - ;; - (*) +case $(psql_conf_source "${conf_name}") +in + ('') + printf 'Invalid configuration parameter: %s\n' "${conf_name}" >&2 + exit 1 + ;; + (default) + echo absent + ;; + (*) + if ! test -f "${__object:?}/parameter/value" + then + echo present + elif psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" + then + echo present + else echo different - ;; - esac -fi + fi + ;; +esac From 19bf37be1a8b0b099a7199d1565b74598010f614 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 15:55:52 +0200 Subject: [PATCH 070/206] [type/__postgres_conf] Update man.rst --- cdist/conf/type/__postgres_conf/man.rst | 39 ++++++++++++++----------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst index 76016c6c..e035f080 100644 --- a/cdist/conf/type/__postgres_conf/man.rst +++ b/cdist/conf/type/__postgres_conf/man.rst @@ -1,29 +1,27 @@ cdist-type__postgres_conf(7) ============================ -Configure a PostgreSQL server. - -NOTE: This type might need to be run multiple times to apply all bits of the -configuration due to ordering requirements. - -SSRQ +NAME +---- +cdist-type__postgres_conf - Alter PostgreSQL configuration DESCRIPTION ----------- -Configure a PostgreSQL server using ALTER SYSTEM. +Configure a running PostgreSQL server using ``ALTER SYSTEM``. REQUIRED PARAMETERS ------------------- value - The value to setup (can be omitted when state is set to "absent"). + The value to set (can be omitted if ``--state`` is set to ``absent``). OPTIONAL PARAMETERS ------------------- state - "present" or "absent". Defaults to "present". + ``present`` or ``absent``. + Defaults to ``present``. BOOLEAN PARAMETERS @@ -36,20 +34,27 @@ EXAMPLES .. code-block:: sh - # set timezone - __postgres_conf timezone --value Europe/Zurich + # set timezone + __postgres_conf timezone --value Europe/Zurich - # reset maximum number of concurrent connections to default (normally 100) - __postgres_conf max_connections --state absent + # reset maximum number of concurrent connections to default (normally 100) + __postgres_conf max_connections --state absent SEE ALSO -------- -- `cdist-type(7) `_ +None. + + +AUTHORS +------- +Beni Ruef (bernhard.ruef--@--ssrq-sds-fds.ch) +Dennis Camera (dennis.camera--@--ssrq-sds-fds.ch) COPYING ------- -Copyright \(C) 2020 SSRQ (www.ssrq-sds-fds.ch). -Free use of this software is granted under the terms -of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2019-2021 SSRQ (www.ssrq-sds-fds.ch). +You can redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. From 1c047353a95e7ac079ccf89af8fc232451b8f891 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 17 Apr 2021 09:57:10 +0200 Subject: [PATCH 071/206] [bin/cdist] Fix Python version check --- bin/cdist | 8 +++++--- cdist/__init__.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index ddaffa7f..adb06a8d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -72,9 +72,11 @@ def commandline(): if __name__ == "__main__": - if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION: - print('Python >= {} is required on the source host.'.format( - cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr) + if sys.version_info[:3] < cdist.MIN_SUPPORTED_PYTHON_VERSION: + print( + 'Python >= {} is required on the source host.'.format( + ".".join(map(str, cdist.MIN_SUPPORTED_PYTHON_VERSION))), + file=sys.stderr) sys.exit(1) exit_code = 0 diff --git a/cdist/__init__.py b/cdist/__init__.py index 44366cd0..31d49889 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -64,7 +64,7 @@ REMOTE_EXEC = "ssh -o User=root" REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}" -MIN_SUPPORTED_PYTHON_VERSION = '3.5' +MIN_SUPPORTED_PYTHON_VERSION = (3, 5) class Error(Exception): From 1bb696a4100f161afada9b1e8d8ad2b8ab190f54 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 20 Apr 2021 07:33:07 +0200 Subject: [PATCH 072/206] Release 6.9.6 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index c8cb44da..70d0b960 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) * Core: config: Make local state directory available to custom remotes (Steven Armstrong * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) From acf9bf91f17fb4ad0e5813c6e4b0b40fc7e027b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 22 Apr 2021 08:55:14 +0200 Subject: [PATCH 073/206] [scanner] error to stderr and exit when scapy is not available --- cdist/scan/commandline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index eca4cf13..0d7fb0ca 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -20,6 +20,7 @@ # import logging +import sys log = logging.getLogger("scan") @@ -31,7 +32,9 @@ def commandline(args): try: import cdist.scan.scan as scan except ModuleNotFoundError: - print('cdist scan requires scapy to be installed') + print('cdist scan requires scapy to be installed! Exiting.', + file=sys.stderr) + sys.exit(1) processes = [] From a4464209b6741f2c1764aa432c56cfa8128852a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 22 Apr 2021 09:29:53 +0200 Subject: [PATCH 074/206] [scanner] add minimal error handling, consolidate CLI args processing --- cdist/argparse.py | 10 ++++-- cdist/scan/commandline.py | 19 ++++++----- cdist/scan/scan.py | 71 +++++++++++---------------------------- 3 files changed, 37 insertions(+), 63 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index cadac39a..153e7864 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -492,12 +492,16 @@ def get_parsers(): help='Try to configure detected hosts') parser['scan'].add_argument( '-I', '--interfaces', - action='append', default=[], + action='append', default=[], required=True, help='On which interfaces to scan/trigger') parser['scan'].add_argument( '-d', '--delay', - action='store', default=3600, - help='How long to wait before reconfiguring after last try') + action='store', default=3600, type=int, + help='How long (seconds) to wait before reconfiguring after last try') + parser['scan'].add_argument( + '-t', '--trigger-delay', + action='store', default=5, type=int, + help='How long (seconds) to wait between ICMPv6 echo requests') parser['scan'].set_defaults(func=cdist.scan.commandline.commandline) for p in parser: diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 0d7fb0ca..dead5292 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -24,26 +24,29 @@ import sys log = logging.getLogger("scan") - -# define this outside of the class to not handle scapy import errors by default +# CLI processing is defined outside of the main scan class to handle +# non-available optional scapy dependency (instead of crashing mid-flight). def commandline(args): log.debug(args) + # Check if we have the optional scapy dependency available. try: import cdist.scan.scan as scan except ModuleNotFoundError: - print('cdist scan requires scapy to be installed! Exiting.', - file=sys.stderr) + log.error('cdist scan requires scapy to be installed. Exiting.') sys.exit(1) - processes = [] - + # Default operation mode. if not args.mode: - # By default scan and trigger, but do not call any action + # By default scan and trigger, but do not call any action. args.mode = ['scan', 'trigger', ] + # We run each component in a separate process since they + # must not block on each other. + processes = [] + if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interfaces) + t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay) t.start() processes.append(t) log.debug("Trigger started") diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index faee8a56..633a5c06 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -61,20 +61,22 @@ import datetime import cdist.config +logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") - class Trigger(object): """ Trigger an ICMPv6EchoReply from all hosts that are alive """ - def __init__(self, interfaces=None, verbose=False): + def __init__(self, interfaces, sleeptime, verbose=False): self.interfaces = interfaces + + # Used by scapy / send in trigger/2. self.verbose = verbose - # Wait 5 seconds before triggering again - FIXME: add parameter - self.sleeptime = 5 + # Delay in seconds between sent ICMPv6EchoRequests. + self.sleeptime = sleeptime def start(self): self.processes = [] @@ -93,9 +95,12 @@ class Trigger(object): time.sleep(self.sleeptime) def trigger(self, interface): - packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest() - log.debug("Sending request on %s", interface) - send(packet, verbose=self.verbose) + try: + log.debug("Sending ICMPv6EchoRequest on %s", interface) + packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest() + send(packet, verbose=self.verbose) + except Exception as e: + log.error( "Could not send ICMPv6EchoRequest: %s", e) class Scanner(object): @@ -103,7 +108,7 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces=None, args=None, outdir=None): + def __init__(self, interfaces, args=None, outdir=None): self.interfaces = interfaces if outdir: @@ -148,47 +153,9 @@ class Scanner(object): def scan(self): log.debug("Scanning - zzzzz") - sniff(iface=self.interfaces, - filter="icmp6", - prn=self.handle_pkg) - - -if __name__ == '__main__': - t = Trigger(interfaces=["wlan0"]) - t.start() - - # Scanner can listen on many interfaces at the same time - s = Scanner(interfaces=["wlan0"]) - s.scan() - - # Join back the trigger processes - t.join() - - # Test in my lan shows: - # [18:48] bridge:cdist% ls -1d fe80::* - # fe80::142d:f0a5:725b:1103 - # fe80::20d:b9ff:fe49:ac11 - # fe80::20d:b9ff:fe4c:547d - # fe80::219:d2ff:feb2:2e12 - # fe80::21b:fcff:feee:f446 - # fe80::21b:fcff:feee:f45c - # fe80::21b:fcff:feee:f4b1 - # fe80::21b:fcff:feee:f4ba - # fe80::21b:fcff:feee:f4bc - # fe80::21b:fcff:feee:f4c1 - # fe80::21d:72ff:fe86:46b - # fe80::42b0:34ff:fe6f:f6f0 - # fe80::42b0:34ff:fe6f:f863 - # fe80::42b0:34ff:fe6f:f9b2 - # fe80::4a5d:60ff:fea1:e55f - # fe80::77a3:5e3f:82cc:f2e5 - # fe80::9e93:4eff:fe6c:c1f4 - # fe80::ba69:f4ff:fec5:6041 - # fe80::ba69:f4ff:fec5:8db7 - # fe80::bad8:12ff:fe65:313d - # fe80::bad8:12ff:fe65:d9b1 - # fe80::ce2d:e0ff:fed4:2611 - # fe80::ce32:e5ff:fe79:7ea7 - # fe80::d66d:6dff:fe33:e00 - # fe80::e2ff:f7ff:fe00:20e6 - # fe80::f29f:c2ff:fe7c:275e + try: + sniff(iface=self.interfaces, + filter="icmp6", + prn=self.handle_pkg) + except Exception as e: + log.error( "Could not start listener: %s", e) From bb24d632d62da8390dd1a724fc325a57526da40d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 22 Apr 2021 10:20:49 +0200 Subject: [PATCH 075/206] [scanner] implement the --list flag --- cdist/argparse.py | 4 +++ cdist/scan/commandline.py | 63 ++++++++++++++++++++++++++++----------- cdist/scan/scan.py | 17 +++++++++++ 3 files changed, 67 insertions(+), 17 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 153e7864..f390a974 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -486,6 +486,10 @@ def get_parsers(): '-m', '--mode', help='Which modes should run', action='append', default=[], choices=['scan', 'trigger']) + parser['scan'].add_argument( + '--list', + action='store_true', + help='List the known hosts and exit') parser['scan'].add_argument( '--config', action='store_true', diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index dead5292..331694e4 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -21,26 +21,11 @@ import logging import sys +from datetime import datetime log = logging.getLogger("scan") -# CLI processing is defined outside of the main scan class to handle -# non-available optional scapy dependency (instead of crashing mid-flight). -def commandline(args): - log.debug(args) - - # Check if we have the optional scapy dependency available. - try: - import cdist.scan.scan as scan - except ModuleNotFoundError: - log.error('cdist scan requires scapy to be installed. Exiting.') - sys.exit(1) - - # Default operation mode. - if not args.mode: - # By default scan and trigger, but do not call any action. - args.mode = ['scan', 'trigger', ] - +def run(scan, args): # We run each component in a separate process since they # must not block on each other. processes = [] @@ -59,3 +44,47 @@ def commandline(args): for process in processes: process.join() + +def list(scan, args): + s = scan.Scanner(interfaces=args.interfaces, args=args) + hosts = s.list() + + # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + + # 6 colons. + ipv6_max_size = 8 * 4 + 10 + # We format dates as follow: YYYY-MM-DD HH:MM:SS + date_max_size = 8 + 2 + 6 + 2 + + print("{} | {}".format( + 'link-local address'.ljust(ipv6_max_size), + 'last seen'.ljust(date_max_size))) + print('=' * (ipv6_max_size + 3 + date_max_size)) + for addr in hosts: + last_seen = datetime.strftime( + datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'), + '%Y-%m-%d %H:%M:%S') + print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size))) + +# CLI processing is defined outside of the main scan class to handle +# non-available optional scapy dependency (instead of crashing mid-flight). +def commandline(args): + log.debug(args) + + # Check if we have the optional scapy dependency available. + try: + import cdist.scan.scan as scan + except ModuleNotFoundError: + log.error('cdist scan requires scapy to be installed. Exiting.') + sys.exit(1) + + # Set default operation mode. + if not args.mode: + # By default scan and trigger, but do not call any action. + args.mode = ['scan', 'trigger', ] + + # Print known hosts and exit is --list is specified - do not start + # the scanner. + if args.list: + list(scan, args) + else: + run(scan, args) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 633a5c06..f3976370 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -132,6 +132,23 @@ class Scanner(object): with open(fname, "w") as fd: fd.write(f"{now}\n") + def list(self): + hosts = dict() + for linklocal_addr in os.listdir(self.outdir): + workdir = os.path.join(self.outdir, linklocal_addr) + # We ignore any (unexpected) file in this directory. + if os.path.isdir(workdir): + last_seen='-' + last_seen_file = os.path.join(workdir, 'last_seen') + if os.path.isfile(last_seen_file): + with open(last_seen_file, "r") as fd: + last_seen = fd.readline() + + hosts[linklocal_addr] = {'last_seen': last_seen} + + return hosts + + def config(self): """ Configure a host From 13e2ad175f01b6cfbdb21ee50e85b6f3ba6fe750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 25 Apr 2021 12:45:34 +0200 Subject: [PATCH 076/206] [scanner] add host class, name mapper and pre-config logic --- cdist/argparse.py | 6 +- cdist/scan/commandline.py | 37 ++++++++---- cdist/scan/scan.py | 122 +++++++++++++++++++++++++++++--------- 3 files changed, 124 insertions(+), 41 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index f390a974..bedb23ac 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -485,7 +485,7 @@ def get_parsers(): parser['scan'].add_argument( '-m', '--mode', help='Which modes should run', action='append', default=[], - choices=['scan', 'trigger']) + choices=['scan', 'trigger', 'config']) parser['scan'].add_argument( '--list', action='store_true', @@ -498,6 +498,10 @@ def get_parsers(): '-I', '--interfaces', action='append', default=[], required=True, help='On which interfaces to scan/trigger') + parser['scan'].add_argument( + '--name-mapper', + action='store', default=None, + help='Map addresses to names, required for config mode') parser['scan'].add_argument( '-d', '--delay', action='store', default=3600, type=int, diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 331694e4..1a0ab0a7 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -37,7 +37,10 @@ def run(scan, args): log.debug("Trigger started") if 'scan' in args.mode: - s = scan.Scanner(interfaces=args.interfaces, args=args) + s = scan.Scanner( + autoconfigure='config' in args.mode, + interfaces=args.interfaces, + name_mapper=args.name_mapper) s.start() processes.append(s) log.debug("Scanner started") @@ -46,24 +49,27 @@ def run(scan, args): process.join() def list(scan, args): - s = scan.Scanner(interfaces=args.interfaces, args=args) + s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper) hosts = s.list() # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + # 6 colons. ipv6_max_size = 8 * 4 + 10 - # We format dates as follow: YYYY-MM-DD HH:MM:SS - date_max_size = 8 + 2 + 6 + 2 + date_max_size = len(datetime.now().strftime(scan.datetime_format)) + name_max_size = 25 - print("{} | {}".format( - 'link-local address'.ljust(ipv6_max_size), - 'last seen'.ljust(date_max_size))) - print('=' * (ipv6_max_size + 3 + date_max_size)) - for addr in hosts: - last_seen = datetime.strftime( - datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'), - '%Y-%m-%d %H:%M:%S') - print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size))) + print("{} | {} | {} | {}".format( + 'name'.ljust(name_max_size), + 'address'.ljust(ipv6_max_size), + 'last seen'.ljust(date_max_size), + 'last configured'.ljust(date_max_size))) + print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) + for host in hosts: + print("{} | {} | {} | {}".format( + host.name(default='-').ljust(name_max_size), + host.address().ljust(ipv6_max_size), + host.last_seen().ljust(date_max_size), + host.last_configured().ljust(date_max_size))) # CLI processing is defined outside of the main scan class to handle # non-available optional scapy dependency (instead of crashing mid-flight). @@ -82,6 +88,11 @@ def commandline(args): # By default scan and trigger, but do not call any action. args.mode = ['scan', 'trigger', ] + if 'config' in args.mode and args.name_mapper == None: + print('--name-mapper must be specified for scanner config mode.', + file=sys.stderr) + sys.exit(1) + # Print known hosts and exit is --list is specified - do not start # the scanner. if args.list: diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index f3976370..459138e2 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -63,6 +63,69 @@ import cdist.config logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") +datetime_format = '%Y-%m-%d %H:%M:%S' + +class Host(object): + def __init__(self, addr, outdir, name_mapper=None): + self.addr = addr + self.workdir = os.path.join(outdir, addr) + self.name_mapper = name_mapper + + os.makedirs(self.workdir, exist_ok=True) + + def __get(self, key, default=None): + fname = os.path.join(self.workdir, key) + value=default + if os.path.isfile(fname): + with open(fname, "r") as fd: + value = fd.readline() + return value + + def __set(self, key, value): + fname = os.path.join(self.workdir, key) + with open(fname, "w") as fd: + fd.write(f"{value}") + + def name(self, default=None): + if self.name_mapper == None: + return default + + fpath = os.path.join(os.getcwd(), self.name_mapper) + if os.path.isfile(fpath) and os.access(fpath, os.X_OK): + out = subprocess.run([fpath, self.addr], capture_output=True) + if out.returncode != 0: + return default + else: + value = out.stdout.decode() + return (None if len(value) == 0 else value) + else: + return default + + def address(self): + return self.addr + + def last_seen(self, default=None): + raw = self.__get('last_seen') + if raw: + return datetime.datetime.strptime(raw, datetime_format) + else: + return default + + def last_configured(self, default=None): + raw = self.__get('last_configured') + if raw: + return datetime.datetime.strptime(raw, datetime_format) + else: + return default + + def seen(self): + now = datetime.datetime.now().strftime(datetime_format) + self.__set('last_seen', now) + + def configure(self): + # TODO: configure. + now = datetime.datetime.now().strftime(datetime_format) + self.__set('last_configured', now) class Trigger(object): """ @@ -108,48 +171,43 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces, args=None, outdir=None): + def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None): self.interfaces = interfaces + self.autoconfigure=autoconfigure + self.name_mapper = name_mapper + self.config_delay = datetime.timedelta(seconds=3600) if outdir: self.outdir = outdir else: self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan') + os.makedirs(self.outdir, exist_ok=True) + + self.running_configs = {} def handle_pkg(self, pkg): if ICMPv6EchoReply in pkg: - host = pkg['IPv6'].src - log.verbose("Host %s is alive", host) + host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper) + if host.name(): + log.verbose("Host %s (%s) is alive", host.name(), host.address()) + else: + log.verbose("Host %s is alive", host.address()) + host.seen() - dir = os.path.join(self.outdir, host) - fname = os.path.join(dir, "last_seen") - - now = datetime.datetime.now() - - os.makedirs(dir, exist_ok=True) - - # FIXME: maybe adjust the format so we can easily parse again - with open(fname, "w") as fd: - fd.write(f"{now}\n") + # TODO check last config. + if self.autoconfigure and \ + host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now(): + self.config(host) def list(self): - hosts = dict() - for linklocal_addr in os.listdir(self.outdir): - workdir = os.path.join(self.outdir, linklocal_addr) - # We ignore any (unexpected) file in this directory. - if os.path.isdir(workdir): - last_seen='-' - last_seen_file = os.path.join(workdir, 'last_seen') - if os.path.isfile(last_seen_file): - with open(last_seen_file, "r") as fd: - last_seen = fd.readline() - - hosts[linklocal_addr] = {'last_seen': last_seen} + hosts = [] + for addr in os.listdir(self.outdir): + hosts.append(Host(addr, self.outdir, self.name_mapper)) return hosts - def config(self): + def config(self, host): """ Configure a host @@ -158,9 +216,19 @@ class Scanner(object): - Maybe keep dict storing per host processes - Save the result - Save the output -> probably aligned to config mode - """ + if host.name() == None: + log.debug("config - could not resolve name for %s, aborting.", host.address()) + return + + if self.running_configs.get(host.name()) != None: + log.debug("config - is already running for %s, aborting.", host.name()) + + log.info("config - running against host %s.", host.name()) + p = host.configure() + self.running_configs[host.name()] = p + def start(self): self.process = Process(target=self.scan) self.process.start() From 512e9b23c0035b09ac1e7e76f706fee2e2ec481f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 25 Apr 2021 15:53:40 +0200 Subject: [PATCH 077/206] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 70d0b960..c41743cd 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * New type: __postgres_conf (Beni Ruef, Dennis Camera) + 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) * Core: config: Make local state directory available to custom remotes (Steven Armstrong From 6ac8cbf98f5d5018373db6eb9c038ef1c599ff2f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 16:33:56 +0200 Subject: [PATCH 078/206] [type/__postgres_database] Include postgres_user explorer from __postgres_conf --- .../explorer/postgres_user | 1 + .../type/__postgres_database/explorer/state | 23 +++++-------------- .../type/__postgres_database/gencode-remote | 13 +---------- 3 files changed, 8 insertions(+), 29 deletions(-) create mode 120000 cdist/conf/type/__postgres_database/explorer/postgres_user diff --git a/cdist/conf/type/__postgres_database/explorer/postgres_user b/cdist/conf/type/__postgres_database/explorer/postgres_user new file mode 120000 index 00000000..714e7237 --- /dev/null +++ b/cdist/conf/type/__postgres_database/explorer/postgres_user @@ -0,0 +1 @@ +../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state index d68d4120..bb044e0e 100755 --- a/cdist/conf/type/__postgres_database/explorer/state +++ b/cdist/conf/type/__postgres_database/explorer/state @@ -1,6 +1,7 @@ #!/bin/sh # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,25 +19,13 @@ # along with cdist. If not, see . # -case "$("${__explorer}/os")" -in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; -esac +postgres_user=$("${__type_explorer:?}/postgres_user") +name=${__object_id:?} -name="$__object_id" - -if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='$name'\"")" +if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='${name}'\"")" then - echo 'present' + echo 'present' else - echo 'absent' + echo 'absent' fi diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote index 0f11cff4..d70fc436 100755 --- a/cdist/conf/type/__postgres_database/gencode-remote +++ b/cdist/conf/type/__postgres_database/gencode-remote @@ -18,23 +18,12 @@ # along with cdist. If not, see . # -case "$(cat "${__global}/explorer/os")" -in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; -esac name="$__object_id" state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" +postgres_user=$(cat "${__object:?}/explorer/postgres_user") if [ "$state_should" != "$state_is" ]; then case "$state_should" in From 58b279a8d0121dcbef05a4999d1d88208656aeed Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 16:34:17 +0200 Subject: [PATCH 079/206] [type/__postgres_database] Improve quoting --- .../type/__postgres_database/gencode-remote | 95 +++++++++++-------- 1 file changed, 55 insertions(+), 40 deletions(-) diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote index d70fc436..7d7d6fa2 100755 --- a/cdist/conf/type/__postgres_database/gencode-remote +++ b/cdist/conf/type/__postgres_database/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,49 +19,63 @@ # along with cdist. If not, see . # +quote() { + for _arg + do + shift + if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')" + then + # needs quoting + set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'" + else + set -- "$@" "${_arg}" + fi + done + unset _arg + # NOTE: Use printf because POSIX echo interprets escape sequences + printf '%s' "$*" +} -name="$__object_id" -state_should="$(cat "$__object/parameter/state")" -state_is="$(cat "$__object/explorer/state")" postgres_user=$(cat "${__object:?}/explorer/postgres_user") -if [ "$state_should" != "$state_is" ]; then - case "$state_should" in - present) - owner="" - if [ -f "$__object/parameter/owner" ]; then - owner="-O \"$(cat "$__object/parameter/owner")\"" - fi +dbname=${__object_id:?} +state_should=$(cat "${__object:?}/parameter/state") +state_is=$(cat "${__object:?}/explorer/state") - template="" - if [ -f "$__object/parameter/template" ]; then - template="--template \"$(cat "$__object/parameter/template")\"" - fi - - encoding="" - if [ -f "$__object/parameter/encoding" ]; then - encoding="--encoding \"$(cat "$__object/parameter/encoding")\"" - fi - - lc_collate="" - if [ -f "$__object/parameter/lc-collate" ]; then - lc_collate="--lc-collate \"$(cat "$__object/parameter/lc-collate")\"" - fi - - lc_ctype="" - if [ -f "$__object/parameter/lc-ctype" ]; then - lc_ctype="--lc-ctype \"$(cat "$__object/parameter/lc-ctype")\"" - fi - - cat << EOF -su - '$postgres_user' -c "createdb $owner \"$name\" $template $encoding $lc_collate $lc_ctype" -EOF - ;; - absent) - cat << EOF -su - '$postgres_user' -c "dropdb \"$name\"" -EOF - ;; - esac +if test "${state_should}" = "$state_is" +then + exit 0 fi + +case ${state_should} +in + (present) + set -- + + while read -r param_name opt + do + if test -f "${__object:?}/parameter/${param_name}" + then + set -- "$@" "${opt}" "$(cat "${__object:?}/parameter/${param_name}")" + fi + done <<-'EOF' + owner -O + template --template + encoding --encoding + lc_collate --lc-collate + lc_ctype --lc-ctype + EOF + + set -- "$@" "${dbname}" + + cat <<-EOF + su - $(quote "${postgres_user}") -c $(quote "$(quote createdb "$@")") + EOF + ;; + (absent) + cat <<-EOF + su - $(quote "${postgres_user}") -c $(quote "$(quote dropdb "${dbname}")") + EOF + ;; +esac From beb8da6d5fd141ac63eacfb526cfcc43cbf22d8f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 16:48:51 +0200 Subject: [PATCH 080/206] [type/__postgres_role] Include postgres_user explorer from __postgres_conf --- .../type/__postgres_role/explorer/postgres_user | 1 + cdist/conf/type/__postgres_role/explorer/state | 14 +------------- cdist/conf/type/__postgres_role/gencode-remote | 15 +-------------- 3 files changed, 3 insertions(+), 27 deletions(-) create mode 120000 cdist/conf/type/__postgres_role/explorer/postgres_user diff --git a/cdist/conf/type/__postgres_role/explorer/postgres_user b/cdist/conf/type/__postgres_role/explorer/postgres_user new file mode 120000 index 00000000..714e7237 --- /dev/null +++ b/cdist/conf/type/__postgres_role/explorer/postgres_user @@ -0,0 +1 @@ +../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 34069de9..0b264e6f 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -19,19 +19,7 @@ # along with cdist. If not, see . # -case $("${__explorer:?}/os") -in - (netbsd) - postgres_user='pgsql' - ;; - (openbsd) - postgres_user='_postgresql' - ;; - (*) - postgres_user='postgres' - ;; -esac - +postgres_user=$("${__type_explorer:?}/postgres_user") rolename=${__object_id:?} diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index d7631fbd..7324b80c 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -28,20 +28,7 @@ quote() { fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" } -case $(cat "${__global:?}/explorer/os") -in - (netbsd) - postgres_user='pgsql' - ;; - (openbsd) - postgres_user='_postgresql' - ;; - (*) - postgres_user='postgres' - ;; -esac - - +postgres_user=$(cat "${__object:?}/explorer/postgres_user") rolename=${__object_id:?} state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") From 3cf93249c38a6f27c097877add619b5d6499a049 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 15 Apr 2021 16:55:11 +0200 Subject: [PATCH 081/206] [type/__postgres_extension] Include postgres_user explorer from __postgres_conf --- .../__postgres_extension/explorer/postgres_user | 1 + cdist/conf/type/__postgres_extension/gencode-remote | 13 +------------ 2 files changed, 2 insertions(+), 12 deletions(-) create mode 120000 cdist/conf/type/__postgres_extension/explorer/postgres_user diff --git a/cdist/conf/type/__postgres_extension/explorer/postgres_user b/cdist/conf/type/__postgres_extension/explorer/postgres_user new file mode 120000 index 00000000..714e7237 --- /dev/null +++ b/cdist/conf/type/__postgres_extension/explorer/postgres_user @@ -0,0 +1 @@ +../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote index af9c97f1..a1c84828 100755 --- a/cdist/conf/type/__postgres_extension/gencode-remote +++ b/cdist/conf/type/__postgres_extension/gencode-remote @@ -22,18 +22,7 @@ # along with cdist. If not, see . # -case "$(cat "${__global}/explorer/os")" -in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; -esac +postgres_user=$(cat "${__object:?}/explorer/postgres_user") dbname=$( echo "$__object_id" | cut -d":" -f1 ) From 8296051653a8187374ad51a3a487315767b73a19 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 16 Apr 2021 19:22:58 +0200 Subject: [PATCH 082/206] [type/__postgres_extension] Add state explorer --- .../type/__postgres_extension/explorer/state | 41 ++++++++++++++ .../type/__postgres_extension/gencode-remote | 44 ++++++++++----- cdist/conf/type/__postgres_extension/man.rst | 55 +++++++++++-------- 3 files changed, 104 insertions(+), 36 deletions(-) create mode 100644 cdist/conf/type/__postgres_extension/explorer/state diff --git a/cdist/conf/type/__postgres_extension/explorer/state b/cdist/conf/type/__postgres_extension/explorer/state new file mode 100644 index 00000000..9d156be7 --- /dev/null +++ b/cdist/conf/type/__postgres_extension/explorer/state @@ -0,0 +1,41 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Prints "present" if the extension is currently installed. +# "absent" otherwise. + +quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } + +postgres_user=$("${__type_explorer:?}/postgres_user") + +IFS=: read -r dbname extname <&2 + exit 1 + ;; esac diff --git a/cdist/conf/type/__postgres_extension/man.rst b/cdist/conf/type/__postgres_extension/man.rst index 79645b2b..442239f6 100644 --- a/cdist/conf/type/__postgres_extension/man.rst +++ b/cdist/conf/type/__postgres_extension/man.rst @@ -3,32 +3,36 @@ cdist-type__postgres_extension(7) NAME ---- -cdist-type__postgres_extension - manage postgres extensions +cdist-type__postgres_extension - Manage PostgreSQL extensions DESCRIPTION ----------- -This cdist type allows you to create or drop postgres extensions. +This cdist type allows you to manage PostgreSQL extensions. -The object you need to pass to __postgres_extension consists of -the database name and the extension name joined by a colon in the -following form: - -.. code-block:: sh - - dbname:extension - -f.ex. +The ``__object_id`` to pass to ``__postgres_extension`` is of the form +``dbname:extension``, e.g.: .. code-block:: sh rails_test:unaccent +**CAUTION!** Be careful when installing extensions from (untrusted) third-party +sources: + + | Installing an extension as superuser requires trusting that the extension's + author wrote the extension installation script in a secure fashion. It is + not terribly difficult for a malicious user to create trojan-horse objects + that will compromise later execution of a carelessly-written extension + script, allowing that user to acquire superuser privileges. + | – ``_ + + OPTIONAL PARAMETERS ------------------- state - either "present" or "absent", defaults to "present" + either ``present`` or ``absent``, defaults to ``present``. EXAMPLES @@ -36,24 +40,29 @@ EXAMPLES .. code-block:: sh - __postgres_extension rails_test:unaccent - __postgres_extension --present rails_test:unaccent - __postgres_extension --absent rails_test:unaccent + # Install extension unaccent into database rails_test + __postgres_extension rails_test:unaccent + + # Drop extension unaccent from database fails_test + __postgres_extension rails_test:unaccent --state absent SEE ALSO -------- -:strong:`cdist-type__postgre_database`\ (7) +- :strong:`cdist-type__postgres_database`\ (7) +- PostgreSQL "CREATE EXTENSION" documentation at: + ``_. -Postgres "Create Extension" documentation at: . -AUTHOR +AUTHORS ------- -Tomas Pospisek +| Tomas Pospisek +| Dennis Camera + COPYING ------- -Copyright \(C) 2014 Tomas Pospisek. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2014 Tomas Pospisek, 2021 Dennis Camera. +You can redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. From 0d33407b182964672b1cfc3ffc49c7fa7cf462a0 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 16 Apr 2021 19:35:26 +0200 Subject: [PATCH 083/206] [type/__postgres_database] Proper quoting in state explorer --- cdist/conf/type/__postgres_database/explorer/state | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state index bb044e0e..6a25df86 100755 --- a/cdist/conf/type/__postgres_database/explorer/state +++ b/cdist/conf/type/__postgres_database/explorer/state @@ -21,9 +21,14 @@ postgres_user=$("${__type_explorer:?}/postgres_user") -name=${__object_id:?} +dbname=${__object_id:?} -if test -n "$(su - "${postgres_user}" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='${name}'\"")" +quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } +psql_exec() { + su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")" +} + +if psql_exec postgres "SELECT datname FROM pg_database" | grep -qFx "${dbname}" then echo 'present' else From 0f05f38384c94b79232ca0917d1323eeefd6e1ed Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 19 Apr 2021 19:09:46 +0200 Subject: [PATCH 084/206] [type/__postgres_role] Treat --password '' like no --password --- cdist/conf/type/__postgres_role/explorer/state | 11 ++++------- cdist/conf/type/__postgres_role/gencode-remote | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 0b264e6f..822816c1 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -43,8 +43,7 @@ role_properties=$( BEGIN { RS = "\036"; FS = "\034" } /^\([0-9]+ rows?\)/ { exit } NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next } - NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i } - ' + NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }' ) if test -n "${role_properties}" @@ -78,12 +77,10 @@ then # Check password passwd_stored=$( psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \ - | awk 'BEGIN { RS = "\036" } NR == 2' - printf . - ) - passwd_stored=${passwd_stored%?.} + | awk 'BEGIN { RS = "\036" } NR == 2 { printf "%s.", $0 }') + passwd_stored=${passwd_stored%.} - if test -f "${__object:?}/parameter/password" + if test -s "${__object:?}/parameter/password" then passwd_should=$(cat "${__object:?}/parameter/password"; printf .) fi diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 7324b80c..4cb78330 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -46,7 +46,7 @@ psql_query() { psql_set_password() { # NOTE: Always make sure that the password does not end up in psql_history! - # NOTE: Never set an empty string as the password, because they can be + # NOTE: Never set an empty string as the password, because it can be # interpreted differently by different tooling. if test -s "${__object:?}/parameter/password" then From 92fff7cb77271009f25d999b6451dc1e43e9bb6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 26 Apr 2021 12:09:20 +0200 Subject: [PATCH 085/206] [scanner] fix crash on --list with name mapper provided --- cdist/scan/commandline.py | 10 ++++++++-- cdist/scan/scan.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 1a0ab0a7..3eb7eec4 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -65,11 +65,17 @@ def list(scan, args): 'last configured'.ljust(date_max_size))) print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) for host in hosts: + last_seen = host.last_seen() + last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-' + + last_configured = host.last_configured() + last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-' + print("{} | {} | {} | {}".format( host.name(default='-').ljust(name_max_size), host.address().ljust(ipv6_max_size), - host.last_seen().ljust(date_max_size), - host.last_configured().ljust(date_max_size))) + last_seen.ljust(date_max_size), + last_configured.ljust(date_max_size))) # CLI processing is defined outside of the main scan class to handle # non-available optional scapy dependency (instead of crashing mid-flight). diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 459138e2..69a4121d 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -97,7 +97,7 @@ class Host(object): return default else: value = out.stdout.decode() - return (None if len(value) == 0 else value) + return (default if len(value) == 0 else value) else: return default From 3a9dd5b1669fa29e1713cdadec1596d859d377f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 26 Apr 2021 12:09:55 +0200 Subject: [PATCH 086/206] [scanner] add minimal (non-configurable) config mode --- cdist/scan/scan.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 69a4121d..152abb4e 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -122,8 +122,20 @@ class Host(object): now = datetime.datetime.now().strftime(datetime_format) self.__set('last_seen', now) + # XXX: There's no easy way to use the config module without feeding it with + # CLI args. Might as well call everything from scratch! def configure(self): - # TODO: configure. + target = self.name() or self.address() + cmd = ['cdist', 'config', '-v', target ] + + fname = os.path.join(self.workdir, 'last_configuration_log') + with open(fname, "w") as fd: + log.debug("Executing: %s", cmd) + completed_process = subprocess.run(cmd, stdout=fd, stderr=fd) + if completed_process.returncode != 0: + log.error("%s return with non-zero code %i - see %s for details.", + cmd, completed_process.returncode, fname) + now = datetime.datetime.now().strftime(datetime_format) self.__set('last_configured', now) @@ -194,7 +206,7 @@ class Scanner(object): log.verbose("Host %s is alive", host.address()) host.seen() - # TODO check last config. + # Configure if needed. if self.autoconfigure and \ host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now(): self.config(host) @@ -206,27 +218,18 @@ class Scanner(object): return hosts - def config(self, host): - """ - Configure a host - - - Assume we are only called if necessary - - However we need to ensure to not run in parallel - - Maybe keep dict storing per host processes - - Save the result - - Save the output -> probably aligned to config mode - """ - if host.name() == None: log.debug("config - could not resolve name for %s, aborting.", host.address()) return - if self.running_configs.get(host.name()) != None: + previous_config_process = self.running_configs.get(host.name()) + if previous_config_process != None and previous_config_process.is_alive(): log.debug("config - is already running for %s, aborting.", host.name()) - log.info("config - running against host %s.", host.name()) - p = host.configure() + log.info("config - running against host %s (%s).", host.name(), host.address()) + p = Process(target=host.configure()) + p.start() self.running_configs[host.name()] = p def start(self): From 2232435c2206b2b3a9a5f2b976df16540b3f1eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 26 Apr 2021 14:39:26 +0200 Subject: [PATCH 087/206] [scanner] initial documentation Note: still needs to patch main cdist(1) manpage --- cdist/scan/scan.py | 32 ------------- docs/src/cdist-scan.rst | 99 +++++++++++++++++++++++++++++++++++++++++ docs/src/index.rst | 1 + 3 files changed, 100 insertions(+), 32 deletions(-) create mode 100644 docs/src/cdist-scan.rst diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 152abb4e..2912dab3 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -19,38 +19,6 @@ # # -# -# Interface to be implemented: -# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated -# scan: scan / listen for icmp6 replies -# trigger: send trigger to multicast -# config: configure newly detected hosts -# install: install newly detected hosts -# -# Scanner logic -# - save results to configdir: -# basedir = ~/.cdist/scan/ -# last_seen = ~/.cdist/scan//last_seen -- record unix time -# or similar -# last_configured = ~/.cdist/scan//last_configured -- record -# unix time or similar -# last_installed = ~/.cdist/scan//last_configured -- record -# unix time or similar -# -# -# -# -# cdist scan --list -# Show all known hosts including last seen flag -# -# Logic for reconfiguration: -# -# - record when configured last time -# - introduce a parameter --reconfigure-after that takes time argument -# - reconfigure if a) host alive and b) reconfigure-after time passed -# - - from multiprocessing import Process import os import logging diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst new file mode 100644 index 00000000..02193456 --- /dev/null +++ b/docs/src/cdist-scan.rst @@ -0,0 +1,99 @@ +Scan +===== + +Description +----------- +Runs cdist as a daemon that discover/watch on hosts and reconfigure them +periodically. It is especially useful in netboot-based environment where hosts +boot unconfigured, and to ensure your infrastructure stays in sync with your +configuration. + +This feature is still consider to be in **beta** stage. + +Usage (Examples) +---------------- + +Discover hosts on local network and configure those whose name is resolved by +the name mapper script. + +.. code-block:: sh + + $ cdist scan --beta --interface eth0 \ + --mode scan --name-mapper path/to/script \ + --mode trigger --mode config + +List known hosts and exit. + +.. code-block:: sh + + $ cdist scan --beta --list --name-mapper path/to/script + +Please refer to `cdist(1)` for a detailed list of parameters. + +Modes +----- + +The scanner has 3 modes that can be independently toggled. If the `--mode` +parameter is not specified, only `tigger` and `scan` are enabled (= hosts are +not configured). + +trigger + Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to + trigger detection by the `scan` module. + +scan + Watch for incoming ICMPv6 replies and optionally configure detected hosts. + +config + Enable configuration of hosts detected by `scan`. + +Name Mapper Script +------------------ + +The name mapper script takes an IPv6 address as first argument and writes the +resolved name to stdout - if any. The script must be executable. + +Simplest script: + +.. code-block:: sh + #!/bin/sh + + case "$1" in + "fe80::20d:b9ff:fe57:3524") + printf "my-host-01" + ;; + "fe80::7603:bdff:fe05:89bb") + printf "my-host-02" + ;; + esac + +Resolving name from `PTR` DNS record: + +.. code-block:: sh + #!/bin/sh + + for cmd in dig sed; do + if ! command -v $cmd > /dev/null; then + exit 1 + fi + done + + dig +short -x "$1" | sed -e 's/.$//' + + +Trigger Source Script +--------------------- + +This script returns a list of addresses (separated by a newline) to be used by +`trigger` mode. It is not used to map names. The script must be executable. + +Simplest script: + +.. code-block:: sh + #!/bin/sh + + cat << EOF + server1.domain.tld + server2.domain.tld + server3.domain.tld + EOF diff --git a/docs/src/index.rst b/docs/src/index.rst index 31c044dc..369d5309 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -34,6 +34,7 @@ It natively supports IPv6 since the first release. cdist-parallelization cdist-inventory cdist-preos + cdist-scan cdist-integration cdist-reference cdist-best-practice From a4122882f2a14034d1a219bd0314892176c5cb64 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 26 Apr 2021 16:39:51 +0200 Subject: [PATCH 088/206] [type/__debconf_set_selections] Add state explorer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …and to make it work, replace --file with --line. --file is deprecated because it does not work with the state explorer as the contents of the file are not available on the target. --- .../__debconf_set_selections/explorer/state | 124 ++++++++++++++++++ .../__debconf_set_selections/gencode-remote | 37 ++++-- .../type/__debconf_set_selections/man.rst | 49 ++++--- .../type/__debconf_set_selections/manifest | 21 +++ .../parameter/{required => deprecated} | 0 .../parameter/optional_multiple | 1 + 6 files changed, 206 insertions(+), 26 deletions(-) create mode 100644 cdist/conf/type/__debconf_set_selections/explorer/state create mode 100755 cdist/conf/type/__debconf_set_selections/manifest rename cdist/conf/type/__debconf_set_selections/parameter/{required => deprecated} (100%) create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/optional_multiple diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state new file mode 100644 index 00000000..68d28941 --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/explorer/state @@ -0,0 +1,124 @@ +#!/bin/sh -e +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Determine current debconf selections' state. +# Prints one of: +# present: all selections are already set as they should. +# different: one or more of the selections have a different value. +# absent: one or more of the selections are not (currently) defined. +# + +test -x /usr/bin/perl || { + # cannot find perl (no perl ~ no debconf) + echo 'absent' + exit 0 +} + +linesfile="${__object:?}/parameter/line" +test -s "${linesfile}" || { + if test -s "${__object:?}/parameter/file" + then + echo absent + else + echo present + fi + exit 0 +} + +/usr/bin/perl -- - "${linesfile}" <<'EOF' +use strict; +use warnings "all"; + +use Debconf::Db; +use Debconf::Question; + +# Extract @known... arrays from debconf-set-selections +# These values are required to distinguish flags and values in the given lines. +# DC: I couldn't think of a more ugly solution to the problem… +my @knownflags; +my @knowntypes; +my $debconf_set_selections = '/usr/bin/debconf-set-selections'; +if (-e $debconf_set_selections) { + my $sed_known = 's/^my \(@known\(flags\|types\) = qw([a-z ]*);\).*$/\1/p'; + eval `sed -n '$sed_known' '$debconf_set_selections'`; +} + +sub mungeline ($) { + my $line = shift; + chomp $line; + $line =~ s/\r$//; + return $line; +} + +sub fatal { printf STDERR @_; exit 1; } + +my $state = 'present'; + +sub state { + my $new = shift; + if ($state eq 'present' + or ($state eq 'different' and $new eq 'absent')) { + $state = $new; + } +} + +Debconf::Db->load(readonly => 'true'); + +while (<>) { + # Read and process lines (taken from debconf-set-selections) + $_ = mungeline($_); + while (/\\$/ && ! eof) { + s/\\$//; + $_ .= mungeline(<>); + } + next if /^\s*$/ || /^\s*\#/; + + my ($owner, $label, $type, $content) = /^\s*(\S+)\s+(\S+)\s+(\S+)(?:\s(.*))?/ + or fatal "invalid line: %s\n", $_; + $content = '' unless defined $content; + + + # Compare is and should state + my $q = Debconf::Question->get($label); + + unless (defined $q) { + # probably a preseed + state 'absent'; + next; + } + + if (grep { $_ eq $q->type } @knownflags) { + # This line wants to set a flag, presumably. + if ($q->flag($q->type) ne $content) { + state 'different'; + } + } else { + # Otherwise, it's probably a value… + if ($q->value ne $content) { + state 'different'; + } + + unless (grep { $_ eq $owner } (split /, /, $q->owners)) { + state 'different'; + } + } +} + +printf "%s\n", $state; +EOF diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote index e99aef40..50d898c6 100755 --- a/cdist/conf/type/__debconf_set_selections/gencode-remote +++ b/cdist/conf/type/__debconf_set_selections/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org) +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,16 +18,32 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# -# Setup selections -# -filename="$(cat "$__object/parameter/file")" - -if [ "$filename" = "-" ]; then - filename="$__object/stdin" +if test -f "${__object:?}/parameter/line" +then + filename="${__object:?}/parameter/line" +elif test -s "${__object:?}/parameter/file" +then + filename=$(cat "${__object:?}/parameter/file") + if test "${filename}" = '-' + then + filename="${__object:?}/stdin" + fi +else + printf 'Neither --line nor --file set.\n' >&2 + exit 1 fi -echo "debconf-set-selections << __file-eof" -cat "$filename" -echo "__file-eof" +# setting no lines makes no sense +test -s "${filename}" || exit 0 + +state_is=$(cat "${__object:?}/explorer/state") + +if test "${state_is}" != 'present' +then + cat <<-CODE + debconf-set-selections <<'EOF' + $(cat "${filename}") + EOF + CODE +fi diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst index 58c25b81..690e3e49 100644 --- a/cdist/conf/type/__debconf_set_selections/man.rst +++ b/cdist/conf/type/__debconf_set_selections/man.rst @@ -8,15 +8,33 @@ cdist-type__debconf_set_selections - Setup debconf selections DESCRIPTION ----------- -On Debian and alike systems debconf-set-selections(1) can be used +On Debian and alike systems :strong:`debconf-set-selections`\ (1) can be used to setup configuration parameters. REQUIRED PARAMETERS ------------------- +cf. ``--line``. + + +OPTIONAL PARAMETERS +------------------- file - Use the given filename as input for debconf-set-selections(1) - If filename is "-", read from stdin. + Use the given filename as input for :strong:`debconf-set-selections`\ (1) + If filename is ``-``, read from stdin. + + **This parameter is deprecated, because it doesn't work with state detection.** +line + A line in :strong:`debconf-set-selections`\ (1) compatible format. + This parameter can be used multiple times to set multiple options. + + (This parameter is actually required, but marked optional because the + deprecated ``--file`` is still accepted.) + + +BOOLEAN PARAMETERS +------------------ +None. EXAMPLES @@ -24,30 +42,29 @@ EXAMPLES .. code-block:: sh - # Setup configuration for nslcd - __debconf_set_selections nslcd --file /path/to/file + # Setup gitolite's gituser + __debconf_set_selections nslcd --line 'gitolite gitolite/gituser string git' - # Setup configuration for nslcd from another type - __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd" - - __debconf_set_selections nslcd --file - << eof - gitolite gitolite/gituser string git - eof + # Setup configuration for nslcd from a file. + # NB: Multiple lines can be passed to --line, although this can be considered a hack. + __debconf_set_selections nslcd --line "$(cat "${__files:?}/preseed/nslcd.debconf")" SEE ALSO -------- -:strong:`debconf-set-selections`\ (1), :strong:`cdist-type__update_alternatives`\ (7) +- :strong:`cdist-type__update_alternatives`\ (7) +- :strong:`debconf-set-selections`\ (1) AUTHORS ------- Nico Schottelius +Dennis Camera COPYING ------- -Copyright \(C) 2011-2014 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2011-2014 Nico Schottelius, 2021 Dennis Camera. +You can redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. diff --git a/cdist/conf/type/__debconf_set_selections/manifest b/cdist/conf/type/__debconf_set_selections/manifest new file mode 100755 index 00000000..0f4fb2e2 --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/manifest @@ -0,0 +1,21 @@ +#!/bin/sh -e +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +__package_apt debconf diff --git a/cdist/conf/type/__debconf_set_selections/parameter/required b/cdist/conf/type/__debconf_set_selections/parameter/deprecated similarity index 100% rename from cdist/conf/type/__debconf_set_selections/parameter/required rename to cdist/conf/type/__debconf_set_selections/parameter/deprecated diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple new file mode 100644 index 00000000..a999a0c2 --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple @@ -0,0 +1 @@ +line From 9cf19388abe95a5f8df9f5c94d2e54fa1060b2ef Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 26 Apr 2021 16:47:44 +0200 Subject: [PATCH 089/206] [type/__debconf_set_selections] Send message about each debconf setting that is changed --- cdist/conf/type/__debconf_set_selections/gencode-remote | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote index 50d898c6..9ba28f09 100755 --- a/cdist/conf/type/__debconf_set_selections/gencode-remote +++ b/cdist/conf/type/__debconf_set_selections/gencode-remote @@ -46,4 +46,9 @@ then $(cat "${filename}") EOF CODE + + awk ' + { + printf "set %s %s %s %s\n", $1, $2, $3, $4 + }' "${filename}" >>"${__messages_out:?}" fi From 3a25b804664963e092d1f75c1fdded25d7016ef3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 26 Apr 2021 21:27:15 +0200 Subject: [PATCH 090/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index c41743cd..497ae91a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * New type: __postgres_conf (Beni Ruef, Dennis Camera) + * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From a42ebc7a78e09e138954b1376376f7cb643420e0 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 27 Apr 2021 19:41:10 +0200 Subject: [PATCH 091/206] [type/__debconf_set_selections] Synchronise objects Works around locking error: debconf: DbDriver "config": /var/cache/debconf/config.dat is locked by another process: Resource temporarily unavailable --- .../__debconf_set_selections/explorer/state | 20 ++++++++++++++++++- .../type/__debconf_set_selections/nonparallel | 0 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 cdist/conf/type/__debconf_set_selections/nonparallel diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state index 68d28941..f8a3f6c8 100644 --- a/cdist/conf/type/__debconf_set_selections/explorer/state +++ b/cdist/conf/type/__debconf_set_selections/explorer/state @@ -41,10 +41,15 @@ test -s "${linesfile}" || { exit 0 } +# assert __type_explorer is set (because it is used by the Perl script) +: "${__type_explorer:?}" + /usr/bin/perl -- - "${linesfile}" <<'EOF' use strict; use warnings "all"; +use Fcntl qw(:DEFAULT :flock); + use Debconf::Db; use Debconf::Question; @@ -78,7 +83,20 @@ sub state { } } -Debconf::Db->load(readonly => 'true'); + +# Load Debconf DB but manually lock on the state explorer script, +# because Debconf aborts immediately if executed concurrently. +# This is not really an ideal solution because the Debconf DB could be locked by +# another process (e.g. apt-get), but no way to achieve this could be found. +# If you know how to, please provide a patch. +my $lockfile = "%ENV{'__type_explorer'}/state"; +if (open my $lock_fh, '+<', $lockfile) { + flock $lock_fh, LOCK_EX or die "Cannot lock $lockfile"; +} +{ + Debconf::Db->load(readonly => 'true'); +} + while (<>) { # Read and process lines (taken from debconf-set-selections) diff --git a/cdist/conf/type/__debconf_set_selections/nonparallel b/cdist/conf/type/__debconf_set_selections/nonparallel new file mode 100644 index 00000000..e69de29b From b276bf874199b842a8e7fd2ce25b9103c39b61ae Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Tue, 4 May 2021 17:28:03 +0100 Subject: [PATCH 092/206] Types to install fail2ban - Install fail2ban server and client - Configure path overrides - Enable and configure jails --- cdist/conf/type/__fail2ban/gencode-remote | 24 +++++++++ cdist/conf/type/__fail2ban/man.rst | 53 +++++++++++++++++++ cdist/conf/type/__fail2ban/manifest | 43 +++++++++++++++ .../__fail2ban/parameter/optional_multiple | 1 + cdist/conf/type/__fail2ban/singleton | 0 .../conf/type/__fail2ban_jail/gencode-remote | 22 ++++++++ cdist/conf/type/__fail2ban_jail/man.rst | 53 +++++++++++++++++++ cdist/conf/type/__fail2ban_jail/manifest | 46 ++++++++++++++++ cdist/conf/type/__fail2ban_jail/nonparallel | 0 .../parameter/optional_multiple | 1 + 10 files changed, 243 insertions(+) create mode 100755 cdist/conf/type/__fail2ban/gencode-remote create mode 100644 cdist/conf/type/__fail2ban/man.rst create mode 100755 cdist/conf/type/__fail2ban/manifest create mode 100644 cdist/conf/type/__fail2ban/parameter/optional_multiple create mode 100644 cdist/conf/type/__fail2ban/singleton create mode 100755 cdist/conf/type/__fail2ban_jail/gencode-remote create mode 100644 cdist/conf/type/__fail2ban_jail/man.rst create mode 100755 cdist/conf/type/__fail2ban_jail/manifest create mode 100644 cdist/conf/type/__fail2ban_jail/nonparallel create mode 100644 cdist/conf/type/__fail2ban_jail/parameter/optional_multiple diff --git a/cdist/conf/type/__fail2ban/gencode-remote b/cdist/conf/type/__fail2ban/gencode-remote new file mode 100755 index 00000000..900d054e --- /dev/null +++ b/cdist/conf/type/__fail2ban/gencode-remote @@ -0,0 +1,24 @@ +#!/bin/sh -e +# +# 2021 Daniel Fancsali (fancsali@gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +if [ -f "$__object/parameter/path-override" ]; then + echo "fail2ban-client reload" +fi diff --git a/cdist/conf/type/__fail2ban/man.rst b/cdist/conf/type/__fail2ban/man.rst new file mode 100644 index 00000000..81743d15 --- /dev/null +++ b/cdist/conf/type/__fail2ban/man.rst @@ -0,0 +1,53 @@ +cdist-type__fail2ban(7) +======================= + +NAME +---- +cdist-type__fail2ban - TODO + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # TODO + __fail2ban + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Daniel Fancsali + + +COPYING +------- +Copyright \(C) 2021 Daniel Fancsali. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__fail2ban/manifest b/cdist/conf/type/__fail2ban/manifest new file mode 100755 index 00000000..2a954743 --- /dev/null +++ b/cdist/conf/type/__fail2ban/manifest @@ -0,0 +1,43 @@ +#!/bin/sh -e +# +# 2021 Daniel Fancsali (fancsali@gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + debian|ubuntu|devuan) + : + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + + +__package_apt fail2ban + +if [ -f "$__object/parameter/path-override" ]; then + ( + echo [DEFAULT] + cat "$__object/parameter/path-override" + ) | __file /etc/fail2ban/paths-overrides.local --mode 644 --source - +fi diff --git a/cdist/conf/type/__fail2ban/parameter/optional_multiple b/cdist/conf/type/__fail2ban/parameter/optional_multiple new file mode 100644 index 00000000..660fbeff --- /dev/null +++ b/cdist/conf/type/__fail2ban/parameter/optional_multiple @@ -0,0 +1 @@ +path-override diff --git a/cdist/conf/type/__fail2ban/singleton b/cdist/conf/type/__fail2ban/singleton new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__fail2ban_jail/gencode-remote b/cdist/conf/type/__fail2ban_jail/gencode-remote new file mode 100755 index 00000000..08d873ae --- /dev/null +++ b/cdist/conf/type/__fail2ban_jail/gencode-remote @@ -0,0 +1,22 @@ +#!/bin/sh -e +# +# 2021 Daniel Fancsali (fancsali@gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +echo "fail2ban-client reload" diff --git a/cdist/conf/type/__fail2ban_jail/man.rst b/cdist/conf/type/__fail2ban_jail/man.rst new file mode 100644 index 00000000..27276990 --- /dev/null +++ b/cdist/conf/type/__fail2ban_jail/man.rst @@ -0,0 +1,53 @@ +cdist-type__fail2ban_jail(7) +============================ + +NAME +---- +cdist-type__fail2ban_jail - TODO + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # TODO + __fail2ban_jail + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Daniel Fancsali + + +COPYING +------- +Copyright \(C) 2021 Daniel Fancsali. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__fail2ban_jail/manifest b/cdist/conf/type/__fail2ban_jail/manifest new file mode 100755 index 00000000..cc222310 --- /dev/null +++ b/cdist/conf/type/__fail2ban_jail/manifest @@ -0,0 +1,46 @@ +#!/bin/sh -e +# +# 2021 Daniel Fancsali (fancsali@gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + debian|ubuntu|devuan) + : + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + +export require="__fail2ban" + +__file /etc/fail2ban/jail.local --mode 644 + +export require="__file/etc/fail2ban/jail.local" +( + echo [$__object_id] + echo "enabled = true" + if [ -f "$__object/parameter/override" ]; then + cat "$__object/parameter/override" + fi +) | __block /etc/fail2ban/jail.local:$__object_id --file /etc/fail2ban/jail.local --text - diff --git a/cdist/conf/type/__fail2ban_jail/nonparallel b/cdist/conf/type/__fail2ban_jail/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__fail2ban_jail/parameter/optional_multiple b/cdist/conf/type/__fail2ban_jail/parameter/optional_multiple new file mode 100644 index 00000000..9ab91d29 --- /dev/null +++ b/cdist/conf/type/__fail2ban_jail/parameter/optional_multiple @@ -0,0 +1 @@ +override From 6a3ba5a140b17c9add73212af60df7deb6c64c81 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Tue, 4 May 2021 17:48:15 +0100 Subject: [PATCH 093/206] Fix shellcheck issues --- cdist/conf/type/__fail2ban_jail/manifest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__fail2ban_jail/manifest b/cdist/conf/type/__fail2ban_jail/manifest index cc222310..47b8a2d6 100755 --- a/cdist/conf/type/__fail2ban_jail/manifest +++ b/cdist/conf/type/__fail2ban_jail/manifest @@ -38,9 +38,9 @@ __file /etc/fail2ban/jail.local --mode 644 export require="__file/etc/fail2ban/jail.local" ( - echo [$__object_id] + echo "[$__object_id]" echo "enabled = true" if [ -f "$__object/parameter/override" ]; then cat "$__object/parameter/override" fi -) | __block /etc/fail2ban/jail.local:$__object_id --file /etc/fail2ban/jail.local --text - +) | __block "/etc/fail2ban/jail.local:$__object_id" --file /etc/fail2ban/jail.local --text - From c00c8c20127af9881b4e295b7732524e7e1c5031 Mon Sep 17 00:00:00 2001 From: Evil Ham Date: Mon, 10 May 2021 12:08:22 +0200 Subject: [PATCH 094/206] [__apt_key*] Deprecate __apt_key_uri and improve __apt_key Previously this type was falling back to using the deprecated apt-key(8) by checking for existence of files/directories on the controller host in gencode-remote. Adding `--use-deprecated-apt-key` as an explicit boolean serves two purposes: 1. It prevents fallbacks that might end up doing the wrong thing (as was the case) 2. It allows for a simple way to remove keys from the keyring that were previously added with apt-key(8) to /etc/apt/trusted.gpg This parameter is added marked as deprecated as is only intended use is to migrate to directory-based keyrings as recommended by Debian for a few releases. It will be removed when Debian 11 stops being supported. During the review process of this merge request, it was noted that the state of PGP Key Servers is somewhat suboptimal, that the examples encouraged bad practise (it is trivial to produce collisions for short key IDs), and that this use does not require the Web of Trust, but instead only the public key that is signing the repository. That is why this also adds `--source` as an argument allowing for in-type or in-manifest provision of such public keys by the type/manifest maintainer and the use of Key Servers is still supported, but discouraged. --- cdist/conf/type/__apt_key/explorer/state | 27 +++-- cdist/conf/type/__apt_key/gencode-remote | 53 ++++----- cdist/conf/type/__apt_key/man.rst | 84 ++++++++++---- cdist/conf/type/__apt_key/manifest | 104 +++++++++++++++++- cdist/conf/type/__apt_key/parameter/boolean | 1 + .../deprecated/use-deprecated-apt-key | 3 + cdist/conf/type/__apt_key/parameter/optional | 5 +- cdist/conf/type/__apt_key_uri/deprecated | 1 + 8 files changed, 209 insertions(+), 69 deletions(-) create mode 100644 cdist/conf/type/__apt_key/parameter/boolean create mode 100644 cdist/conf/type/__apt_key/parameter/deprecated/use-deprecated-apt-key create mode 100644 cdist/conf/type/__apt_key_uri/deprecated diff --git a/cdist/conf/type/__apt_key/explorer/state b/cdist/conf/type/__apt_key/explorer/state index 38f1bd3c..8ab268c1 100755 --- a/cdist/conf/type/__apt_key/explorer/state +++ b/cdist/conf/type/__apt_key/explorer/state @@ -27,18 +27,25 @@ else keyid="$__object_id" fi +# From apt-key(8): +# Use of apt-key is deprecated, except for the use of apt-key del in +# maintainer scripts to remove existing keys from the main keyring. +# If such usage of apt-key is desired the additional installation of +# the GNU Privacy Guard suite (packaged in gnupg) is required. +if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then + if apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" + then echo present + else echo absent + fi + exit +fi + keydir="$(cat "$__object/parameter/keydir")" keyfile="$keydir/$__object_id.gpg" -if [ -d "$keydir" ] +if [ -f "$keyfile" ] then - if [ -f "$keyfile" ] - then echo present - else echo absent - fi -else - # fallback to deprecated apt-key - apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \ - && echo present \ - || echo absent + echo present + exit fi +echo absent diff --git a/cdist/conf/type/__apt_key/gencode-remote b/cdist/conf/type/__apt_key/gencode-remote index 0c96ff67..17dc9bfc 100755 --- a/cdist/conf/type/__apt_key/gencode-remote +++ b/cdist/conf/type/__apt_key/gencode-remote @@ -25,11 +25,7 @@ else fi state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" - -if [ "$state_should" = "$state_is" ]; then - # nothing to do - exit 0 -fi +method="$(cat "$__object/key_method")" keydir="$(cat "$__object/parameter/keydir")" keyfile="$keydir/$__object_id.gpg" @@ -37,30 +33,18 @@ keyfile="$keydir/$__object_id.gpg" case "$state_should" in present) keyserver="$(cat "$__object/parameter/keyserver")" - - if [ -f "$__object/parameter/uri" ]; then - uri="$(cat "$__object/parameter/uri")" - - if [ -d "$keydir" ]; then - cat << EOF - -curl -s -L \\ - -o "$keyfile" \\ - "$uri" - -key="\$( cat "$keyfile" )" - -if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK' -then - echo "\$key" | gpg --dearmor > "$keyfile" -fi - -EOF - else - # fallback to deprecated apt-key - echo "curl -s -L '$uri' | apt-key add -" + # Using __download or __file as key source + # Propagate messages if needed + if [ "${method}" = "uri" ] || [ "${method}" = "source" ]; then + if grep -Eq "^__(file|download)$keyfile" "$__messages_in"; then + echo "added '$keyid'" >> "$__messages_out" fi - elif [ -d "$keydir" ]; then + exit 0 + elif [ "${state_is}" = "present" ]; then + exit 0 + fi + # Using key servers to fetch the key + if [ ! -f "$__object/parameter/use-deprecated-apt-key" ]; then # we need to kill gpg after 30 seconds, because gpg # can get stuck if keyserver is not responding. # exporting env var and not exit 1, @@ -100,13 +84,16 @@ EOF echo "added '$keyid'" >> "$__messages_out" ;; absent) - if [ -f "$keyfile" ]; then - echo "rm '$keyfile'" - else + # Removal for keys added from a keyserver without this flag + # is done in the manifest + if [ "$state_is" != "absent" ] && \ + [ -f "$__object/parameter/use-deprecated-apt-key" ]; then # fallback to deprecated apt-key echo "apt-key del \"$keyid\"" + echo "removed '$keyid'" >> "$__messages_out" + # Propagate messages if needed + elif grep -Eq "^__file$keyfile" "$__messages_in"; then + echo "removed '$keyid'" >> "$__messages_out" fi - - echo "removed '$keyid'" >> "$__messages_out" ;; esac diff --git a/cdist/conf/type/__apt_key/man.rst b/cdist/conf/type/__apt_key/man.rst index 234bc715..e35eaa0f 100644 --- a/cdist/conf/type/__apt_key/man.rst +++ b/cdist/conf/type/__apt_key/man.rst @@ -10,6 +10,14 @@ DESCRIPTION ----------- Manages the list of keys used by apt to authenticate packages. +This is done by placing the requested key in a file named +``$__object_id.gpg`` in the ``keydir`` directory. + +This is supported by modern releases of Debian-based distributions. + +In order of preference, exactly one of: ``source``, ``uri`` or ``keyid`` +must be specified. + REQUIRED PARAMETERS ------------------- @@ -18,21 +26,49 @@ None. OPTIONAL PARAMETERS ------------------- +keydir + keyring directory, defaults to ``/etc/apt/trusted.pgp.d``, which is + enabled system-wide by default. + +source + path to a file containing the GPG key of the repository. + Using this is recommended as it ensures that the manifest/type manintainer + has validated the key. + If ``-``, the GPG key is read from the type's stdin. + state 'present' or 'absent'. Defaults to 'present' +uri + the URI from which to download the key. + It is highly recommended that you only use protocols with TLS like HTTPS. + This uses ``__download`` but does not use checksums, if you want to ensure + that the key doesn't change, you are better off downloading it and using + ``--source``. + + +DEPRECATED OPTIONAL PARAMETERS +------------------------------ keyid - the id of the key to add. Defaults to __object_id + the id of the key to download from the ``keyserver``. + This is to be used in absence of ``--source`` and ``--uri`` or together + with ``--use-deprecated-apt-key`` for key removal. + Defaults to ``$__object_id``. keyserver - the keyserver from which to fetch the key. If omitted the default set - in ./parameter/default/keyserver is used. + the keyserver from which to fetch the key. + Defaults to ``pool.sks-keyservers.net``. -keydir - key save location, defaults to ``/etc/apt/trusted.pgp.d`` -uri - the URI from which to download the key +DEPRECATED BOOLEAN PARAMETERS +----------------------------- +use-deprecated-apt-key + ``apt-key(8)`` will last be available in Debian 11 and Ubuntu 22.04. + You can use this parameter to force usage of ``apt-key(8)``. + Please only use this parameter to *remove* keys from the keyring, + in order to prepare for removal of ``apt-key``. + Adding keys should be done without this parameter. + This parameter will be removed when Debian 11 stops being supported. EXAMPLES @@ -40,33 +76,39 @@ EXAMPLES .. code-block:: sh - # Add Ubuntu Archive Automatic Signing Key - __apt_key 437D05B5 - # Same thing - __apt_key 437D05B5 --state present - # Get rid of it - __apt_key 437D05B5 --state absent + # add a key that has been verified by a type maintainer + __apt_key jitsi_meet_2021 \ + --source cdist-contrib/type/__jitsi_meet/files/apt_2021.gpg - # same thing with human readable name and explicit keyid - __apt_key UbuntuArchiveKey --keyid 437D05B5 + # remove an old, deprecated or expired key + __apt_key jitsi_meet_2016 --state absent - # same thing with other keyserver - __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com + # Get rid of a key that might have been added to + # /etc/apt/trusted.gpg with apt-key + __apt_key 0x40976EAF437D05B5 --use-deprecated-apt-key --state absent - # download key from the internet - __apt_key rabbitmq \ - --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc + # add a key that we define in-line + __apt_key jitsi_meet_2021 --source '-' < Ander Punnar +Evilham COPYING ------- -Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can +Copyright \(C) 2011-2021 Steven Armstrong, Ander Punnar and Evilham. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_key/manifest b/cdist/conf/type/__apt_key/manifest index 010357cd..889a764a 100755 --- a/cdist/conf/type/__apt_key/manifest +++ b/cdist/conf/type/__apt_key/manifest @@ -2,7 +2,105 @@ __package gnupg -if [ -f "$__object/parameter/uri" ] -then __package curl -else __package dirmngr +state_should="$(cat "${__object}/parameter/state")" + +incompatible_args() +{ + cat >> /dev/stderr <<-EOF + This type does not support --${1} and --${method} simultaneously. + EOF + exit 1 +} + +if [ -f "${__object}/parameter/source" ]; then + method="source" + src="$(cat "${__object}/parameter/source")" + if [ "${src}" = "-" ]; then + src="${__object}/stdin" + fi +fi +if [ -f "${__object}/parameter/uri" ]; then + if [ -n "${method}" ]; then + incompatible_args uri + fi + method="uri" + src="$(cat "${__object}/parameter/uri")" +fi +if [ -f "${__object}/parameter/keyid" ]; then + if [ -n "${method}" ]; then + incompatible_args keyid + fi + method="keyid" +fi +# Keep old default +if [ -z "${method}" ]; then + method="keyid" +fi +# Save this for later in gencode-remote +echo "${method}" > "${__object}/key_method" + +# Required remotely (most likely already installed) +__package dirmngr +# We need this in case a key has to be dearmor'd +__package gnupg +export require="__package/gnupg" + +if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then + # This is required if apt-key(8) is to be used + if [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then + incompatible_args use-deprecated-apt-key + fi +else + if [ "${state_should}" = "absent" ] && \ + [ -f "${__object}/parameter/keyid" ]; then + cat >> /dev/stderr < Date: Mon, 10 May 2021 12:10:00 +0200 Subject: [PATCH 095/206] [__letsencrypt_cert] Revamp explorers, add locking. This would fix #839 Certbot uses locking [1] even for read-only operations and does not properly use exit codes, which means that sometimes it would print: "Another instance of Certbot is already running" and exit with success. However, the previous explorers would take that as the certificate being absent and would trigger code generation. The issue was made worse by having many explorers running certbot, so for N certificates, we'd run certbot N*4 times, potentially "in parallel". [1]: https://certbot.eff.org/docs/using.html#id5 This patch joins all explorers in one to avoid starting multiple remote python processes and uses a cdist-specific lock in /tmp/certbot.cdist.lock with a 60 seconds timeout. It has been tested with certbot 0.31.0 and 0.17 that the: from certbot.main import main trick works. It is somewhat well documented so it can be somewhat relied upon. --- .../__letsencrypt_cert/explorer/certbot-path | 3 - .../explorer/certificate-data | 78 +++++++++++++++++++ .../explorer/certificate-domains | 8 -- .../explorer/certificate-exists | 13 ---- .../explorer/certificate-is-test | 14 ---- .../type/__letsencrypt_cert/gencode-remote | 11 ++- cdist/conf/type/__letsencrypt_cert/manifest | 2 +- 7 files changed, 87 insertions(+), 42 deletions(-) delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certbot-path create mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-data delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists delete mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path deleted file mode 100755 index 3c6076df..00000000 --- a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -e - -command -v certbot 2>/dev/null || true diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data new file mode 100755 index 00000000..ff62e742 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data @@ -0,0 +1,78 @@ +#!/bin/sh -e +certbot_path="$(command -v certbot 2>/dev/null || true)" +# Defaults +certificate_exists="no" +certificate_is_test="no" + +if [ -n "${certbot_path}" ]; then + # Find python executable that has access to certbot's module + python_path=$(sed -n '1s/^#! *//p' "${certbot_path}") + + # Use a lock for cdist due to certbot not exiting with failure + # or having any flags for concurrent use. + _certbot() { + ${python_path} - 2>/dev/null < "${existing_domains}" + certificate_is_test="$(_explorer_var certificate_is_test)" sort -uo "${requested_domains}" "${requested_domains}" sort -uo "${existing_domains}" "${existing_domains}" diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 1df3574a..6394f629 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -1,6 +1,6 @@ #!/bin/sh -certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")" +certbot_fullpath="$(grep "^certbot_path:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-)" state=$(cat "${__object}/parameter/state") os="$(cat "${__global:?}/explorer/os")" From f14623e45f56aec4bdce53bc2b367e11ff157865 Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 10 May 2021 12:17:08 +0200 Subject: [PATCH 096/206] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 497ae91a..5c3b547a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,8 @@ Changelog next: * New type: __postgres_conf (Beni Ruef, Dennis Camera) * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) + * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) + * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 6210cccb28ace8ec2807ffe6ca9f9e4d9007990b Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 10 May 2021 12:34:04 +0200 Subject: [PATCH 097/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 5c3b547a..a7310788 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) + * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking. 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From a2eeac6fe41bc8cb8b99a0867a299ef2f6bd23a5 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Tue, 18 May 2021 17:21:44 +0100 Subject: [PATCH 098/206] Fix missing dependency for fail2ban override files --- cdist/conf/type/__fail2ban/manifest | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/conf/type/__fail2ban/manifest b/cdist/conf/type/__fail2ban/manifest index 2a954743..a557eafb 100755 --- a/cdist/conf/type/__fail2ban/manifest +++ b/cdist/conf/type/__fail2ban/manifest @@ -35,6 +35,8 @@ esac __package_apt fail2ban +export require="__package_apt/fail2ban" + if [ -f "$__object/parameter/path-override" ]; then ( echo [DEFAULT] From 503a06ed28f743aad47797a0989be735c67b07a6 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 28 Apr 2021 13:32:10 +0300 Subject: [PATCH 099/206] [__git] fix group explorer group name from numberic id wasn't resolved correctly. try to use getent and fallback to reading /etc/group directly. --- cdist/conf/type/__git/explorer/group | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group index 1365c60d..ab4396b1 100644 --- a/cdist/conf/type/__git/explorer/group +++ b/cdist/conf/type/__git/explorer/group @@ -14,6 +14,11 @@ then then printf '%u\n' "${group_gid}" else - printf '%s\n' "$(id -u -n "${group_gid}")" + if command -v getent > /dev/null + then + getent group "${group_gid}" | cut -d : -f 1 + else + awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group + fi fi fi From 75c71f69c1fed4371c5891ddcca0eaf28ad928e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 10:17:48 +0200 Subject: [PATCH 100/206] [scanner] pycodestyle compliance --- cdist/scan/commandline.py | 20 +++++++++++---- cdist/scan/scan.py | 54 ++++++++++++++++++++++++--------------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 3eb7eec4..b42bc7b2 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -25,13 +25,15 @@ from datetime import datetime log = logging.getLogger("scan") + def run(scan, args): # We run each component in a separate process since they # must not block on each other. processes = [] if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay) + t = scan.Trigger(interfaces=args.interfaces, + sleeptime=args.trigger_delay) t.start() processes.append(t) log.debug("Trigger started") @@ -48,6 +50,7 @@ def run(scan, args): for process in processes: process.join() + def list(scan, args): s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper) hosts = s.list() @@ -66,10 +69,16 @@ def list(scan, args): print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) for host in hosts: last_seen = host.last_seen() - last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-' + if last_seen: + last_seen = last_seen.strftime(scan.datetime_format) + else: + last_seen = '-' last_configured = host.last_configured() - last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-' + if last_configured: + last_configured = last_configured.strftime(scan.datetime_format) + else: + '-' print("{} | {} | {} | {}".format( host.name(default='-').ljust(name_max_size), @@ -77,6 +86,7 @@ def list(scan, args): last_seen.ljust(date_max_size), last_configured.ljust(date_max_size))) + # CLI processing is defined outside of the main scan class to handle # non-available optional scapy dependency (instead of crashing mid-flight). def commandline(args): @@ -94,9 +104,9 @@ def commandline(args): # By default scan and trigger, but do not call any action. args.mode = ['scan', 'trigger', ] - if 'config' in args.mode and args.name_mapper == None: + if 'config' in args.mode and args.name_mapper is None: print('--name-mapper must be specified for scanner config mode.', - file=sys.stderr) + file=sys.stderr) sys.exit(1) # Print known hosts and exit is --list is specified - do not start diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 2912dab3..4a20f511 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -33,6 +33,7 @@ logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") datetime_format = '%Y-%m-%d %H:%M:%S' + class Host(object): def __init__(self, addr, outdir, name_mapper=None): self.addr = addr @@ -43,7 +44,7 @@ class Host(object): def __get(self, key, default=None): fname = os.path.join(self.workdir, key) - value=default + value = default if os.path.isfile(fname): with open(fname, "r") as fd: value = fd.readline() @@ -55,15 +56,15 @@ class Host(object): fd.write(f"{value}") def name(self, default=None): - if self.name_mapper == None: + if self.name_mapper is None: return default fpath = os.path.join(os.getcwd(), self.name_mapper) if os.path.isfile(fpath) and os.access(fpath, os.X_OK): - out = subprocess.run([fpath, self.addr], capture_output=True) - if out.returncode != 0: - return default - else: + out = subprocess.run([fpath, self.addr], capture_output=True) + if out.returncode != 0: + return default + else: value = out.stdout.decode() return (default if len(value) == 0 else value) else: @@ -94,19 +95,20 @@ class Host(object): # CLI args. Might as well call everything from scratch! def configure(self): target = self.name() or self.address() - cmd = ['cdist', 'config', '-v', target ] + cmd = ['cdist', 'config', '-v', target] fname = os.path.join(self.workdir, 'last_configuration_log') with open(fname, "w") as fd: log.debug("Executing: %s", cmd) completed_process = subprocess.run(cmd, stdout=fd, stderr=fd) if completed_process.returncode != 0: - log.error("%s return with non-zero code %i - see %s for details.", - cmd, completed_process.returncode, fname) + log.error("%s return with non-zero code %i - see %s for \ + details.", cmd, completed_process.returncode, fname) now = datetime.datetime.now().strftime(datetime_format) self.__set('last_configured', now) + class Trigger(object): """ Trigger an ICMPv6EchoReply from all hosts that are alive @@ -140,10 +142,12 @@ class Trigger(object): def trigger(self, interface): try: log.debug("Sending ICMPv6EchoRequest on %s", interface) - packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest() + packet = IPv6( + dst="ff02::1%{}".format(interface) + ) / ICMPv6EchoRequest() send(packet, verbose=self.verbose) except Exception as e: - log.error( "Could not send ICMPv6EchoRequest: %s", e) + log.error("Could not send ICMPv6EchoRequest: %s", e) class Scanner(object): @@ -151,9 +155,10 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None): + def __init__(self, interfaces, autoconfigure=False, outdir=None, + name_mapper=None): self.interfaces = interfaces - self.autoconfigure=autoconfigure + self.autoconfigure = autoconfigure self.name_mapper = name_mapper self.config_delay = datetime.timedelta(seconds=3600) @@ -169,14 +174,17 @@ class Scanner(object): if ICMPv6EchoReply in pkg: host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper) if host.name(): - log.verbose("Host %s (%s) is alive", host.name(), host.address()) + log.verbose("Host %s (%s) is alive", host.name(), + host.address()) else: log.verbose("Host %s is alive", host.address()) + host.seen() # Configure if needed. if self.autoconfigure and \ - host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now(): + host.last_configured(default=datetime.datetime.min) + \ + self.config_delay < datetime.datetime.now(): self.config(host) def list(self): @@ -187,15 +195,19 @@ class Scanner(object): return hosts def config(self, host): - if host.name() == None: - log.debug("config - could not resolve name for %s, aborting.", host.address()) + if host.name() is None: + log.debug("config - could not resolve name for %s, aborting.", + host.address()) return previous_config_process = self.running_configs.get(host.name()) - if previous_config_process != None and previous_config_process.is_alive(): - log.debug("config - is already running for %s, aborting.", host.name()) + if previous_config_process is not None and \ + previous_config_process.is_alive(): + log.debug("config - is already running for %s, aborting.", + host.name()) - log.info("config - running against host %s (%s).", host.name(), host.address()) + log.info("config - running against host %s (%s).", host.name(), + host.address()) p = Process(target=host.configure()) p.start() self.running_configs[host.name()] = p @@ -214,4 +226,4 @@ class Scanner(object): filter="icmp6", prn=self.handle_pkg) except Exception as e: - log.error( "Could not start listener: %s", e) + log.error("Could not start listener: %s", e) From ab10b453f275d4289f3d37988b3caec204227194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 11:15:41 +0200 Subject: [PATCH 101/206] [scanner] populate cdist(1) --- docs/src/man1/cdist.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 0ecb4a61..599ec3b7 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -88,6 +88,9 @@ SYNOPSIS cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] + cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list] + [-d CONFIG_DELAY] [-t TRIGGER_DELAY] + DESCRIPTION ----------- @@ -641,6 +644,31 @@ Display information for cdist (global explorers, types). **-t, --types** Display info for types. +SCAN +---- + +Runs cdist as a daemon that discover/watch on hosts and reconfigure them +periodically. + +**-I INTERFACE, --interfaces INTERFACE** + Interface to listen on. Can be specified multiple times. + +**-m MODE, --mode MODE** + Scanner components to enable. Can be specified multiple time to enable more + than one component. Supported modes are: scan, trigger and config. Defaults + to tiggger and scan. + +**--name-mapper PATH_TO_SCRIPT** + Path to script used to resolve a remote host name from an IPv6 address. + +**--list** + List known hosts and exit. + +**-d CONFIG_DELAY, --config-delay CONFIG_DELAY** + How long (seconds) to wait before reconfiguring after last try (config mode only). + +**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY** + How long (seconds) to wait between ICMPv6 echo requests (trigger mode only). CONFIGURATION ------------- From b8733c65f52776facce7a8de0265538a856ead64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 11:26:35 +0200 Subject: [PATCH 102/206] [scanner] fix minor CLI handling and --list bugs / typo --- cdist/argparse.py | 4 ++-- cdist/scan/commandline.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index bedb23ac..f17315e7 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -495,7 +495,7 @@ def get_parsers(): action='store_true', help='Try to configure detected hosts') parser['scan'].add_argument( - '-I', '--interfaces', + '-I', '--interface', action='append', default=[], required=True, help='On which interfaces to scan/trigger') parser['scan'].add_argument( @@ -503,7 +503,7 @@ def get_parsers(): action='store', default=None, help='Map addresses to names, required for config mode') parser['scan'].add_argument( - '-d', '--delay', + '-d', '--config-delay', action='store', default=3600, type=int, help='How long (seconds) to wait before reconfiguring after last try') parser['scan'].add_argument( diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index b42bc7b2..ddbe4933 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -32,7 +32,7 @@ def run(scan, args): processes = [] if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interfaces, + t = scan.Trigger(interfaces=args.interface, sleeptime=args.trigger_delay) t.start() processes.append(t) @@ -41,7 +41,7 @@ def run(scan, args): if 'scan' in args.mode: s = scan.Scanner( autoconfigure='config' in args.mode, - interfaces=args.interfaces, + interfaces=args.interface, name_mapper=args.name_mapper) s.start() processes.append(s) @@ -52,7 +52,7 @@ def run(scan, args): def list(scan, args): - s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper) + s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper) hosts = s.list() # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + @@ -75,10 +75,10 @@ def list(scan, args): last_seen = '-' last_configured = host.last_configured() - if last_configured: + if last_configured is not None: last_configured = last_configured.strftime(scan.datetime_format) else: - '-' + last_configured = '-' print("{} | {} | {} | {}".format( host.name(default='-').ljust(name_max_size), From e0c52d0e1dfbaa3814a2ff26482177c2909a15ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 11:27:11 +0200 Subject: [PATCH 103/206] [scanner] remove mention of non-implemented trigger soruce script --- docs/src/cdist-scan.rst | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst index 02193456..064e65ff 100644 --- a/docs/src/cdist-scan.rst +++ b/docs/src/cdist-scan.rst @@ -8,7 +8,8 @@ periodically. It is especially useful in netboot-based environment where hosts boot unconfigured, and to ensure your infrastructure stays in sync with your configuration. -This feature is still consider to be in **beta** stage. +This feature is still consider to be in **beta** stage, and only operate on +IPv6 (including link-local). Usage (Examples) ---------------- @@ -79,21 +80,3 @@ Resolving name from `PTR` DNS record: done dig +short -x "$1" | sed -e 's/.$//' - - -Trigger Source Script ---------------------- - -This script returns a list of addresses (separated by a newline) to be used by -`trigger` mode. It is not used to map names. The script must be executable. - -Simplest script: - -.. code-block:: sh - #!/bin/sh - - cat << EOF - server1.domain.tld - server2.domain.tld - server3.domain.tld - EOF From defa3c22eaef88652a2bec6135d038a84710c41e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 29 May 2021 11:21:34 +0200 Subject: [PATCH 104/206] ++changelog --- docs/changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index a7310788..403e7653 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,7 +6,8 @@ next: * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) - * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking. + * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham) + * Type __git: Fix group explorer (Ander Punnar) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From d596986af894749877fb3dcadf90628c9cfa9d13 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 31 May 2021 09:06:52 +0200 Subject: [PATCH 105/206] [type/__pyvenv] Fix group explorer --- cdist/conf/type/__pyvenv/explorer/group | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group index f31a1cb7..922ce3df 100755 --- a/cdist/conf/type/__pyvenv/explorer/group +++ b/cdist/conf/type/__pyvenv/explorer/group @@ -14,6 +14,11 @@ then then printf '%u\n' "${group_gid}" else - printf '%s\n' "$(id -u -n "${group_gid}")" + if command -v getent >/dev/null 2>&1 + then + getent group "${group_gid}" | cut -d : -f 1 + else + awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group + fi fi fi From 6ede76b08b96b7504c65ca2dc6737e31be7e7f24 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 8 Jun 2021 16:20:55 +0200 Subject: [PATCH 106/206] [type/__debconf_set_selections] man.rst: Fix line break in AUTHORS --- cdist/conf/type/__debconf_set_selections/man.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst index 690e3e49..fd0040ae 100644 --- a/cdist/conf/type/__debconf_set_selections/man.rst +++ b/cdist/conf/type/__debconf_set_selections/man.rst @@ -58,8 +58,8 @@ SEE ALSO AUTHORS ------- -Nico Schottelius -Dennis Camera +| Nico Schottelius +| Dennis Camera COPYING From c308a2896971ac4808be0152630ef41b7f96a2de Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 10 Jun 2021 06:39:55 +0200 Subject: [PATCH 107/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 403e7653..97ea204b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham) * Type __git: Fix group explorer (Ander Punnar) + * Type __pyvenv: Fix group explorer (Dennis Camera) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From a3102022e18a23ce9b54eeaf7415b55361f80bd1 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 11 Jun 2021 15:05:17 +0100 Subject: [PATCH 108/206] More sensible defaults; reword debian-only error message --- cdist/conf/type/__apt_pin/manifest | 13 +++++++++---- cdist/conf/type/__apt_pin/nonparallel | 0 cdist/conf/type/__apt_pin/parameter/default/package | 1 - 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 cdist/conf/type/__apt_pin/nonparallel delete mode 100644 cdist/conf/type/__apt_pin/parameter/default/package diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index b1372ad0..909bc80d 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -19,9 +19,17 @@ # +name="$__object_id" + os=$(cat "$__global/explorer/os") state="$(cat "$__object/parameter/state")" -package="$(cat "$__object/parameter/package")" + +if [ -f "$__object/parameter/package" ]; then + package="$(cat "$__object/parameter/package")" +else + package=$name +fi + distribution="$(cat "$__object/parameter/distribution")" priority="$(cat "$__object/parameter/priority")" @@ -31,13 +39,10 @@ case "$os" in ;; *) printf "This type is specific to Debian and it's derivatives" >&2 - printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2 exit 1 ;; esac -name="$__object_id" - case $distribution in stabletesting|unsatbel|experimental) pin="release a=$distribution" diff --git a/cdist/conf/type/__apt_pin/nonparallel b/cdist/conf/type/__apt_pin/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package deleted file mode 100644 index 72e8ffc0..00000000 --- a/cdist/conf/type/__apt_pin/parameter/default/package +++ /dev/null @@ -1 +0,0 @@ -* From b726697e070e5266eae38c7951ced14a2305acb2 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 11 Jun 2021 15:05:33 +0100 Subject: [PATCH 109/206] Add documentation --- cdist/conf/type/__apt_pin/man.rst | 37 ++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst index 7fcae6f8..0c91cdec 100644 --- a/cdist/conf/type/__apt_pin/man.rst +++ b/cdist/conf/type/__apt_pin/man.rst @@ -13,11 +13,21 @@ This space intentionally left blank. REQUIRED PARAMETERS ------------------- -None. +distribution + Specifies what distribution the package should be pinned to. Accepts both codenames (buster/bullseye/sid) and suite names (stable/testing/...). OPTIONAL PARAMETERS ------------------- +package + Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used. + +priority + The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority) + +state + Will be passed to underlying `__file` type; see there for valid values and defaults. + None. @@ -31,14 +41,31 @@ EXAMPLES .. code-block:: sh - # TODO - __apt_pin + # Add the bullseye repo to buster, but do not install any pacakges by default + # only if explicitely asked for + __apt_pin bullseye-default \ + --package "*" \ + --distribution bullseye \ + --priority -1 + + require="__apt_pin/bullseye-default" __apt_source bullseye \ + --uri http://deb.debian.org/debian/ \ + --distribution bullseye \ + --component main + # TODO + __apt_pin + + __apt_pin foo --package "foo foo-*" --distribution bullseye + + __foo # Installs the `foo` package internally + + __package foo-plugin-extras SEE ALSO -------- -:strong:`TODO`\ (7) - +:strong:`apt_preferences`\ (7) +:strong:`cdist-type__file`\ (7) AUTHORS ------- From 7b3f268df25922c515174862da95569aea547367 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 22 Jun 2021 16:36:30 +0300 Subject: [PATCH 110/206] [__download] improvements 1. post download checksum verification 2. detect hashes without prefix 3. add optional --destination 4. updated man --- .../conf/type/__download/explorer/remote_cmd | 19 --- .../type/__download/explorer/remote_cmd_get | 16 +++ .../type/__download/explorer/remote_cmd_sum | 82 +++++++++++ cdist/conf/type/__download/explorer/state | 60 ++------ cdist/conf/type/__download/gencode-local | 131 +++++++++++++++--- cdist/conf/type/__download/gencode-remote | 46 +++++- cdist/conf/type/__download/man.rst | 31 ++++- cdist/conf/type/__download/manifest | 2 +- cdist/conf/type/__download/parameter/optional | 3 +- 9 files changed, 292 insertions(+), 98 deletions(-) delete mode 100755 cdist/conf/type/__download/explorer/remote_cmd create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_get create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_sum diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd deleted file mode 100755 index e3e35b45..00000000 --- a/cdist/conf/type/__download/explorer/remote_cmd +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/cmd-get" ] -then - cmd="$( cat "$__object/parameter/cmd-get" )" - -elif command -v curl > /dev/null -then - cmd="curl -L -o - '%s'" - -elif command -v fetch > /dev/null -then - cmd="fetch -o - '%s'" - -else - cmd="wget -O - '%s'" -fi - -echo "$cmd" diff --git a/cdist/conf/type/__download/explorer/remote_cmd_get b/cdist/conf/type/__download/explorer/remote_cmd_get new file mode 100755 index 00000000..9f1cd59c --- /dev/null +++ b/cdist/conf/type/__download/explorer/remote_cmd_get @@ -0,0 +1,16 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/cmd-get" ] +then + cat "$__object/parameter/cmd-get" +elif + command -v curl > /dev/null +then + echo "curl -sSL -o - '%s'" +elif + command -v fetch > /dev/null +then + echo "fetch -o - '%s'" +else + echo "wget -O - '%s'" +fi diff --git a/cdist/conf/type/__download/explorer/remote_cmd_sum b/cdist/conf/type/__download/explorer/remote_cmd_sum new file mode 100755 index 00000000..84df663c --- /dev/null +++ b/cdist/conf/type/__download/explorer/remote_cmd_sum @@ -0,0 +1,82 @@ +#!/bin/sh -e + +if [ ! -f "$__object/parameter/sum" ] +then + exit 0 +fi + +if [ -f "$__object/parameter/cmd-sum" ] +then + cat "$__object/parameter/cmd-sum" + exit 0 +fi + +sum_should="$( cat "$__object/parameter/sum" )" + +if echo "$sum_should" | grep -Fq ':' +then + sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" +else + if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' + then + sum_hash='cksum' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' + then + sum_hash='md5' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' + then + sum_hash='sha1' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' + then + sum_hash='sha256' + else + echo 'hash format detection failed' >&2 + exit 1 + fi +fi + +os="$( "$__explorer/os" )" + +case "$sum_hash" in + cksum) + echo "cksum %s | awk '{print \$1\" \"\$2}'" + ;; + md5) + case "$os" in + freebsd) + echo "md5 -q %s" + ;; + *) + echo "md5sum %s | awk '{print \$1}'" + ;; + esac + ;; + sha1) + case "$os" in + freebsd) + echo "sha1 -q %s" + ;; + *) + echo "sha1sum %s | awk '{print \$1}'" + ;; + esac + ;; + sha256) + case "$os" in + freebsd) + echo "sha256 -q %s" + ;; + *) + echo "sha256sum %s | awk '{print \$1}'" + ;; + esac + ;; + *) + # we arrive here only if --sum is given with unknown format prefix + echo "unknown hash format: $sum_hash" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 68b517c5..881a1c09 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -1,6 +1,11 @@ #!/bin/sh -e -dst="/$__object_id" +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" +else + dst="/$__object_id" +fi if [ ! -f "$dst" ] then @@ -16,57 +21,18 @@ fi sum_should="$( cat "$__object/parameter/sum" )" -if [ -f "$__object/parameter/cmd-sum" ] +if echo "$sum_should" | grep -Fq ':' then - # shellcheck disable=SC2059 - sum_is="$( eval "$( printf \ - "$( cat "$__object/parameter/cmd-sum" )" \ - "$dst" )" )" -else - os="$( "$__explorer/os" )" - - if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' - then - sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )" - - elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$' - then - case "$os" in - freebsd) - sum_is="md5:$( md5 -q "$dst" )" - ;; - *) - sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )" - ;; - esac - - elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$' - then - case "$os" in - freebsd) - sum_is="sha1:$( sha1 -q "$dst" )" - ;; - *) - sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )" - ;; - esac - - elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$' - then - case "$os" in - freebsd) - sum_is="sha256:$( sha256 -q "$dst" )" - ;; - *) - sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )" - ;; - esac - fi + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" fi +sum_cmd="$( "$__type_explorer/remote_cmd_sum" )" + +sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )" + if [ -z "$sum_is" ] then - echo 'no checksum from target' >&2 + echo 'existing destination checksum failed' >&2 exit 1 fi diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local index 571d2c3c..d1b0d0d5 100755 --- a/cdist/conf/type/__download/gencode-local +++ b/cdist/conf/type/__download/gencode-local @@ -11,34 +11,133 @@ fi url="$( cat "$__object/parameter/url" )" -tmp="$( mktemp )" - -dst="/$__object_id" +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" +else + dst="/$__object_id" +fi if [ -f "$__object/parameter/cmd-get" ] then cmd="$( cat "$__object/parameter/cmd-get" )" -elif command -v wget > /dev/null -then - cmd="wget -O - '%s'" - elif command -v curl > /dev/null then - cmd="curl -L -o - '%s'" + cmd="curl -sSL -o - '%s'" elif command -v fetch > /dev/null then cmd="fetch -o - '%s'" +elif command -v wget > /dev/null +then + cmd="wget -O - '%s'" + else - echo 'no usable locally installed utility for downloading' >&2 + echo 'local download failed, no usable utility' >&2 exit 1 fi -printf "$cmd > %s\n" \ - "$url" \ - "$tmp" +echo "download_tmp=\"\$( mktemp )\"" + +# shellcheck disable=SC2059 +printf "$cmd > \"\$download_tmp\"\n" "$url" + +if [ -f "$__object/parameter/sum" ] +then + sum_should="$( cat "$__object/parameter/sum" )" + + if [ -f "$__object/parameter/cmd-sum" ] + then + local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" + else + if echo "$sum_should" | grep -Fq ':' + then + sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" + + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" + else + if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' + then + sum_hash='cksum' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' + then + sum_hash='md5' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' + then + sum_hash='sha1' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' + then + sum_hash='sha256' + else + echo 'hash format detection failed' >&2 + exit 1 + fi + fi + + case "$sum_hash" in + cksum) + local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'" + ;; + md5) + if command -v md5 > /dev/null + then + local_cmd_sum="md5 -q %s" + elif + command -v md5sum > /dev/null + then + local_cmd_sum="md5sum %s | awk '{print \$1}'" + fi + ;; + sha1) + if command -v sha1 > /dev/null + then + local_cmd_sum="sha1 -q %s" + elif + command -v sha1sum > /dev/null + then + local_cmd_sum="sha1sum %s | awk '{print \$1}'" + fi + ;; + sha256) + if command -v sha256 > /dev/null + then + local_cmd_sum="sha256 -q %s" + elif + command -v sha256sum > /dev/null + then + local_cmd_sum="sha256sum %s | awk '{print \$1}'" + fi + ;; + *) + # we arrive here only if --sum is given with unknown format prefix + echo "unknown hash format: $sum_hash" >&2 + exit 1 + ;; + esac + + if [ -z "$local_cmd_sum" ] + then + echo 'local checksum verification failed, no usable utility' >&2 + exit 1 + fi + fi + + # shellcheck disable=SC2059 + echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\"" + + echo "if [ \"\$sum_is\" != '$sum_should' ]; then" + + echo "echo 'local download checksum mismatch' >&2" + + echo "rm -f \"\$download_tmp\"" + + echo 'exit 1; fi' +fi if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$' then @@ -47,12 +146,10 @@ else target_host="$__target_host" fi -printf '%s %s %s:%s\n' \ +# shellcheck disable=SC2016 +printf '%s "$download_tmp" %s:%s\n' \ "$__remote_copy" \ - "$tmp" \ "$target_host" \ "$dst" -echo "rm -f '$tmp'" - -echo 'downloaded' > "$__messages_out" +echo "rm -f \"\$download_tmp\"" diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote index 029a0801..e49bcec3 100755 --- a/cdist/conf/type/__download/gencode-remote +++ b/cdist/conf/type/__download/gencode-remote @@ -6,17 +6,51 @@ state_is="$( cat "$__object/explorer/state" )" if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ] then - cmd="$( cat "$__object/explorer/remote_cmd" )" + cmd_get="$( cat "$__object/explorer/remote_cmd_get" )" url="$( cat "$__object/parameter/url" )" - dst="/$__object_id" + if [ -f "$__object/parameter/destination" ] + then + dst="$( cat "$__object/parameter/destination" )" + else + dst="/$__object_id" + fi - printf "$cmd > %s\n" \ - "$url" \ - "$dst" + echo "download_tmp=\"\$( mktemp )\"" - echo 'downloaded' > "$__messages_out" + # shellcheck disable=SC2059 + printf "$cmd_get > \"\$download_tmp\"\n" "$url" + + if [ -f "$__object/parameter/sum" ] + then + sum_should="$( cat "$__object/parameter/sum" )" + + if [ -f "$__object/parameter/cmd-sum" ] + then + remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" + else + remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )" + + if echo "$sum_should" | grep -Fq ':' + then + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" + fi + fi + + # shellcheck disable=SC2059 + echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\"" + + echo "if [ \"\$sum_is\" != '$sum_should' ]; then" + + echo "echo 'remote download checksum mismatch' >&2" + + echo "rm -f \"\$download_tmp\"" + + echo 'exit 1; fi' + fi + + echo "mv \"\$download_tmp\" '$dst'" fi if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ] diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index a1278cfb..c16510a9 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -8,7 +8,7 @@ cdist-type__download - Download a file DESCRIPTION ----------- -By default type will try to use ``wget``, ``curl`` or ``fetch``. +By default type will try to use ``curl``, ``fetch`` or ``wget``. If download happens in target (see ``--download``) then type will fallback to (and install) ``wget``. @@ -16,6 +16,8 @@ If download happens in local machine, then environment variables like ``{http,https,ftp}_proxy`` etc can be used on cdist execution (``http_proxy=foo cdist config ...``). +To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``. + REQUIRED PARAMETERS ------------------- @@ -25,14 +27,29 @@ url OPTIONAL PARAMETERS ------------------- +destination + Downloaded file's destination in target. If unset, ``$__object_id`` is used. + sum - Checksum is used to decide if existing destination file must be redownloaded. - By default output of ``cksum`` without filename is expected. - Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. + Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256. + + Type tries to detect hash format with regexes, but prefixes + ``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported. + + Checksum have two purposes - state check and post-download verification. + In state check, if destination checksum mismatches, then content of URL + will be downloaded to temporary file. If downloaded temporary file's + checksum matches, then it will be moved to destination (overwritten). + + For local downloads it is expected that usable utilities for checksum + calculation exist in the system. download - If ``local`` (default), then download file to local storage and copy - it to target host. If ``remote``, then download happens in target. + If ``local`` (default), then file is downloaded to local storage and copied + to target host. If ``remote``, then download happens in target. + + For local downloads it is expected that usable utilities for downloading + exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``. cmd-get Command used for downloading. @@ -62,7 +79,7 @@ EXAMPLES require='__directory/opt/cpma' \ __download /opt/cpma/cnq3.zip \ --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \ - --sum md5:46da3021ca9eace277115ec9106c5b46 + --sum 46da3021ca9eace277115ec9106c5b46 require='__download/opt/cpma/cnq3.zip' \ __unpack /opt/cpma/cnq3.zip \ diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest index 7ec8d86d..3d4c498b 100755 --- a/cdist/conf/type/__download/manifest +++ b/cdist/conf/type/__download/manifest @@ -1,6 +1,6 @@ #!/bin/sh -e -if grep -Eq '^wget' "$__object/explorer/remote_cmd" +if grep -Eq '^wget' "$__object/explorer/remote_cmd_get" then __package wget fi diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional index d69e083e..e809ef78 100644 --- a/cdist/conf/type/__download/parameter/optional +++ b/cdist/conf/type/__download/parameter/optional @@ -1,5 +1,6 @@ -sum cmd-get cmd-sum +destination download onchange +sum From 2db40d8d704b427768307fbea29384bd3dc8dbd7 Mon Sep 17 00:00:00 2001 From: fancsali Date: Mon, 28 Jun 2021 12:54:20 +0200 Subject: [PATCH 111/206] Use $__remote_exec and thus the ssh multiplexing --- cdist/conf/type/__rsync/gencode-local | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index e36ded2f..36addc36 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -36,4 +36,5 @@ fi echo rsync -a \ --no-owner --no-group \ + -e "$__remote_exec" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From d937d53f3dfd10830a07aee0450596eea62f2a1a Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Mon, 28 Jun 2021 18:09:35 +0100 Subject: [PATCH 112/206] Add quotes to rsync command --- cdist/conf/type/__rsync/gencode-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index 36addc36..f1bddc16 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -36,5 +36,5 @@ fi echo rsync -a \ --no-owner --no-group \ - -e "$__remote_exec" \ + -e \"$__remote_exec\" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From 60753ddfcc3ac49303d572da7f6a68f398c02227 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 1 Jul 2021 14:42:10 +0300 Subject: [PATCH 113/206] fix shellcheck --- cdist/conf/type/__download/explorer/state | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 881a1c09..8c5d5ce1 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -28,6 +28,7 @@ fi sum_cmd="$( "$__type_explorer/remote_cmd_sum" )" +# shellcheck disable=SC2059 sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )" if [ -z "$sum_is" ] From a90e642c1354cf01be7c9a1ba8a468ad624c4202 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 1 Jul 2021 14:50:40 +0300 Subject: [PATCH 114/206] update README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index de6901c7..a468dd86 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ For community-maintained types there is ## Participating -IRC: ``#cdist`` @ freenode +IRC: ``#cdist`` @ [libera](https://libera.chat) Matrix: ``#cdist:ungleich.ch`` -Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist +Matrix and IRC are bridged. From 243a4b904a2de638d00bac57f6e762a076f9ae54 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 2 Jul 2021 06:50:02 +0200 Subject: [PATCH 115/206] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 97ea204b..d48c657e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,8 @@ next: * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham) * Type __git: Fix group explorer (Ander Punnar) * Type __pyvenv: Fix group explorer (Dennis Camera) + * Type __download: Improve checksum verification, add optional --destination (Ander Punnar) + * Type __debconf_set_selections: Add state explorer (Dennis Camera) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 30ba796d060fd3d0c4affd4167f14784156b5354 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 1 Jul 2021 11:49:07 +0300 Subject: [PATCH 116/206] new type: __snakeoil_cert --- .../__snakeoil_cert/explorer/ssl-cert-group | 8 ++ .../conf/type/__snakeoil_cert/explorer/state | 24 ++++++ .../conf/type/__snakeoil_cert/gencode-remote | 73 +++++++++++++++++++ cdist/conf/type/__snakeoil_cert/man.rst | 60 +++++++++++++++ .../parameter/default/cert-path | 1 + .../parameter/default/key-path | 1 + .../parameter/default/key-type | 1 + .../type/__snakeoil_cert/parameter/optional | 4 + 8 files changed, 172 insertions(+) create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/state create mode 100755 cdist/conf/type/__snakeoil_cert/gencode-remote create mode 100644 cdist/conf/type/__snakeoil_cert/man.rst create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/cert-path create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-path create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-type create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/optional diff --git a/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group new file mode 100755 index 00000000..a6cb3dfd --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group @@ -0,0 +1,8 @@ +#!/bin/sh -e + +if grep -Eq '^ssl-cert:' /etc/group +then + echo 'present' +else + echo 'absent' +fi diff --git a/cdist/conf/type/__snakeoil_cert/explorer/state b/cdist/conf/type/__snakeoil_cert/explorer/state new file mode 100755 index 00000000..cc5aae0b --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/explorer/state @@ -0,0 +1,24 @@ +#!/bin/sh -e + +key_path="$( cat "$__object/parameter/key-path" )" + +if echo "$key_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + key_path="$( printf "$key_path" "$__object_id" )" +fi + +cert_path="$( cat "$__object/parameter/cert-path" )" + +if echo "$cert_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + cert_path="$( printf "$cert_path" "$__object_id" )" +fi + +if [ ! -f "$key_path" ] || [ ! -f "$cert_path" ] +then + echo 'absent' +else + echo 'present' +fi diff --git a/cdist/conf/type/__snakeoil_cert/gencode-remote b/cdist/conf/type/__snakeoil_cert/gencode-remote new file mode 100755 index 00000000..8ffbfad1 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/gencode-remote @@ -0,0 +1,73 @@ +#!/bin/sh -e + +state="$( cat "$__object/explorer/state" )" + +if [ "$state" = 'present' ] +then + exit 0 +fi + +if [ -f "$__object/parameter/common-name" ] +then + common_name="$( cat "$__object/parameter/common-name" )" +else + common_name="$__object_id" +fi + +key_path="$( cat "$__object/parameter/key-path" )" + +if echo "$key_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + key_path="$( printf "$key_path" "$__object_id" )" +fi + +cert_path="$( cat "$__object/parameter/cert-path" )" + +if echo "$cert_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + cert_path="$( printf "$cert_path" "$__object_id" )" +fi + +key_type="$( cat "$__object/parameter/key-type" )" + +key_type_arg="$( echo "$key_type" | cut -d : -f 2 )" + +case "$key_type" in + rsa:*) + echo "openssl genrsa -out '$key_path' $key_type_arg" + ;; + ec:*) + echo "openssl ecparam -name $key_type_arg -genkey -noout -out '$key_path'" + ;; +esac + +# shellcheck disable=SC2016 +echo 'csr_path="$( mktemp )"' + +echo "openssl req -new -subj '/CN=$common_name' -key '$key_path' -out \"\$csr_path\"" + +echo "openssl x509 -req -sha256 -days 3650 -in \"\$csr_path\" -signkey '$key_path' -out '$cert_path'" + +# shellcheck disable=SC2016 +echo 'rm -f "$csr_path"' + +if [ "$( cat "$__object/explorer/ssl-cert-group" )" = 'present' ] +then + key_group='ssl-cert' +else + key_group='root' +fi + +echo "chmod 640 '$key_path'" + +echo "chown root '$key_path'" + +echo "chgrp $key_group '$key_path'" + +echo "chmod 644 '$cert_path'" + +echo "chown root '$cert_path'" + +echo "chgrp root '$cert_path'" diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst new file mode 100644 index 00000000..0b547804 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/man.rst @@ -0,0 +1,60 @@ +cdist-type__snakeoil_cert(7) +============================ + +NAME +---- +cdist-type__snakeoil_cert - Generate self-signed certificate + + +DESCRIPTION +----------- +The purpose of this type is to generate **self-signed** certificate and private key +for **testing purposes**. Certificate will expire in 3650 days. + +Certificate's and key's access bits will be ``644`` and ``640`` respectively. +If target system has ``ssl-cert`` group, then it will be used as key's group. +Use ``require='__snakeoil_cert/...' __file ...`` to override. + + +OPTIONAL PARAMETERS +------------------- +common-name + Defaults to ``$__object_id``. + +key-path + ``%s`` in path will be replaced with ``$__object_id``. + Defaults to ``/etc/ssl/private/%s.pem``. + +key-type + Possible values are ``rsa:$bits`` and ``ec:$name``. + For possible EC names see ``openssl ecparam -list_curves``. + Defaults to ``rsa:2048``. + +cert-path + ``%s`` in path will be replaced with ``$__object_id``. + Defaults to ``/etc/ssl/certs/%s.pem``. + + +EXAMPLES +-------- +.. code-block:: sh + __snakeoil_cert localhost-rsa \ + --common-name localhost \ + --key-type rsa:4096 + + __snakeoil_cert localhost-ec \ + --common-name localhost \ + --key-type ec:prime256v1 + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path new file mode 100644 index 00000000..4bbae089 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path @@ -0,0 +1 @@ +/etc/ssl/certs/%s.pem diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-path b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path new file mode 100644 index 00000000..86eb9359 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path @@ -0,0 +1 @@ +/etc/ssl/private/%s.pem diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-type b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type new file mode 100644 index 00000000..f13f8ada --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type @@ -0,0 +1 @@ +rsa:2048 diff --git a/cdist/conf/type/__snakeoil_cert/parameter/optional b/cdist/conf/type/__snakeoil_cert/parameter/optional new file mode 100644 index 00000000..76d08c0a --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/optional @@ -0,0 +1,4 @@ +common-name +key-path +key-type +cert-path From 853e5cf7b4f615e1470c7ec2cfa8fe68cfa85ce7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 5 Jul 2021 09:07:06 +0200 Subject: [PATCH 117/206] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index d48c657e..0c3c64e1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -11,6 +11,8 @@ next: * Type __pyvenv: Fix group explorer (Dennis Camera) * Type __download: Improve checksum verification, add optional --destination (Ander Punnar) * Type __debconf_set_selections: Add state explorer (Dennis Camera) + * Core: Implement usable cdist scan (Timothée Floure) + * New type: __snakeoil_cert (Ander Punnar) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From be92731c5c8a8543448f0d87fafae67e22ac76a1 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Mon, 5 Jul 2021 12:38:26 +0100 Subject: [PATCH 118/206] Shell check quoting We're actually echo-ing the command, hence the escape in front of the quotes - the issue Shellcheck alludes too would actually occur, had the escaping bakcslashes been omitted. --- cdist/conf/type/__rsync/gencode-local | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index f1bddc16..be4feabb 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -34,7 +34,8 @@ if [ -f "$__object/parameter/rsync-opts" ]; then done < "$__object/parameter/rsync-opts" fi +# shellcheck disable=SC2086 echo rsync -a \ --no-owner --no-group \ - -e \"$__remote_exec\" \ + -e \"${__remote_exec}\" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From 521241d74102e37fae5f55552e1ef565d26ea9d2 Mon Sep 17 00:00:00 2001 From: fancsali Date: Mon, 5 Jul 2021 15:28:05 +0200 Subject: [PATCH 119/206] Refine docs even more --- cdist/conf/type/__apt_pin/man.rst | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst index 0c91cdec..4229c0cd 100644 --- a/cdist/conf/type/__apt_pin/man.rst +++ b/cdist/conf/type/__apt_pin/man.rst @@ -3,12 +3,12 @@ cdist-type__apt_pin(7) NAME ---- -cdist-type__apt_pin - TODO +cdist-type__apt_pin - Manage apt pinning rules DESCRIPTION ----------- -This space intentionally left blank. +Adds/removes/edits rules to pin some packages to a specific distribution. Useful if using multiple debian repositories at the same time. (Useful, if one wants to use a few specific packages from backports or perhaps Debain testing... or even sid.) REQUIRED PARAMETERS @@ -20,7 +20,7 @@ distribution OPTIONAL PARAMETERS ------------------- package - Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used. + Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used. priority The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority) @@ -28,7 +28,6 @@ priority state Will be passed to underlying `__file` type; see there for valid values and defaults. -None. BOOLEAN PARAMETERS @@ -41,8 +40,8 @@ EXAMPLES .. code-block:: sh - # Add the bullseye repo to buster, but do not install any pacakges by default - # only if explicitely asked for + # Add the bullseye repo to buster, but do not install any packages by default, + # only if explicitely asked for (-1 means "never" for apt) __apt_pin bullseye-default \ --package "*" \ --distribution bullseye \ @@ -52,19 +51,19 @@ EXAMPLES --uri http://deb.debian.org/debian/ \ --distribution bullseye \ --component main - # TODO - __apt_pin __apt_pin foo --package "foo foo-*" --distribution bullseye - __foo # Installs the `foo` package internally + __foo # Assuming, this installs the `foo` package internally - __package foo-plugin-extras + __package foo-plugin-extras # Assuming we also need some extra stuff SEE ALSO -------- -:strong:`apt_preferences`\ (7) +:strong:`apt_preferences`\ (5) +:strong:`cdist-type__apt_source`\ (7) +:strong:`cdist-type__apt_backports`\ (7) :strong:`cdist-type__file`\ (7) AUTHORS From 166b58aeea09da41086525849291c70a3c3a571c Mon Sep 17 00:00:00 2001 From: fancsali Date: Mon, 5 Jul 2021 15:32:27 +0200 Subject: [PATCH 120/206] Fix typo in distro names... --- cdist/conf/type/__apt_pin/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index 909bc80d..e72a8fdd 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -44,7 +44,7 @@ case "$os" in esac case $distribution in - stabletesting|unsatbel|experimental) + stable|testing|unstable|experimental) pin="release a=$distribution" ;; *) From 485283f2e531f5fa23a876fa52411e0030f32589 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 7 Jul 2021 20:47:22 +0300 Subject: [PATCH 121/206] new type: __sed --- cdist/conf/type/__sed/explorer/file | 22 +++++++++ cdist/conf/type/__sed/gencode-remote | 46 +++++++++++++++++ cdist/conf/type/__sed/man.rst | 49 +++++++++++++++++++ cdist/conf/type/__sed/parameter/boolean | 1 + cdist/conf/type/__sed/parameter/optional | 1 + .../type/__sed/parameter/required_multiple | 1 + 6 files changed, 120 insertions(+) create mode 100755 cdist/conf/type/__sed/explorer/file create mode 100755 cdist/conf/type/__sed/gencode-remote create mode 100644 cdist/conf/type/__sed/man.rst create mode 100644 cdist/conf/type/__sed/parameter/boolean create mode 100644 cdist/conf/type/__sed/parameter/optional create mode 100644 cdist/conf/type/__sed/parameter/required_multiple diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file new file mode 100755 index 00000000..a40f71b3 --- /dev/null +++ b/cdist/conf/type/__sed/explorer/file @@ -0,0 +1,22 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/file" ] +then + file="$( cat "$__object/parameter/file" )" +else + file="/$__object_id" +fi + +if [ -f "$file" ] +then + if [ -s "$file" ] + then + cat "$file" + else + echo "$file is empty" >&2 + exit 1 + fi +else + echo "$file do not exist" >&2 + exit 1 +fi diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote new file mode 100755 index 00000000..391e3bc1 --- /dev/null +++ b/cdist/conf/type/__sed/gencode-remote @@ -0,0 +1,46 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/file" ] +then + file="$( cat "$__object/parameter/file" )" +else + file="/$__object_id" +fi + +script="$( cat "$__object/parameter/script" )" + +if [ "$script" = '-' ] +then + script="$( cat "$__object/stdin" )" +elif + [ -f "$script" ] +then + script="$( cat "$script" )" +fi + +file_from_target="$__object/explorer/file" + +sed_cmd='sed' + +if [ -f "$__object/parameter/regexp-extended" ] +then + sed_cmd="$sed_cmd --regexp-extended" +fi + +if ! echo "$script" \ + | "$sed_cmd" -f - "$file_from_target" \ + | diff "$file_from_target" - \ + > /dev/null +then + echo 'tmp="$( mktemp )"' + + echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" + + echo "$script" + + echo 'EOF' + + echo "cp \"\$tmp\" '$file'" + + echo 'rm -f "$tmp"' +fi diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst new file mode 100644 index 00000000..fbac212d --- /dev/null +++ b/cdist/conf/type/__sed/man.rst @@ -0,0 +1,49 @@ +cdist-type__sed(7) +================== + +NAME +---- +cdist-type__sed - Transform text files with ``sed`` + + +DESCRIPTION +----------- +TODO + + +REQUIRED MULTIPLE PARAMETERS +---------------------------- +script + TODO + + +OPTIONAL PARAMETERS +------------------- +file + TODO + + +BOOLEAN PARAMETERS +------------------ +regexp-extended + TODO + + +EXAMPLES +-------- +.. code-block:: sh + + true + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. diff --git a/cdist/conf/type/__sed/parameter/boolean b/cdist/conf/type/__sed/parameter/boolean new file mode 100644 index 00000000..1ad75c5d --- /dev/null +++ b/cdist/conf/type/__sed/parameter/boolean @@ -0,0 +1 @@ +regexp-extended diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional new file mode 100644 index 00000000..f73f3093 --- /dev/null +++ b/cdist/conf/type/__sed/parameter/optional @@ -0,0 +1 @@ +file diff --git a/cdist/conf/type/__sed/parameter/required_multiple b/cdist/conf/type/__sed/parameter/required_multiple new file mode 100644 index 00000000..84f7e31d --- /dev/null +++ b/cdist/conf/type/__sed/parameter/required_multiple @@ -0,0 +1 @@ +script From 7a5896acfad0f992d6e7fe9cbe0bd08035ce19db Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 7 Jul 2021 21:23:25 +0300 Subject: [PATCH 122/206] add --onchange, fix shellcheck --- cdist/conf/type/__sed/gencode-remote | 9 ++++++++- cdist/conf/type/__sed/man.rst | 3 +++ cdist/conf/type/__sed/parameter/optional | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 391e3bc1..6b1bd216 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -32,6 +32,7 @@ if ! echo "$script" \ | diff "$file_from_target" - \ > /dev/null then + # shellcheck disable=SC2016 echo 'tmp="$( mktemp )"' echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" @@ -41,6 +42,12 @@ then echo 'EOF' echo "cp \"\$tmp\" '$file'" - + + # shellcheck disable=SC2016 echo 'rm -f "$tmp"' + + if [ -f "$__object/parameter/onchange" ] + then + cat "$__object/parameter/onchange" + fi fi diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst index fbac212d..4a728184 100644 --- a/cdist/conf/type/__sed/man.rst +++ b/cdist/conf/type/__sed/man.rst @@ -22,6 +22,9 @@ OPTIONAL PARAMETERS file TODO +onchange + TODO + BOOLEAN PARAMETERS ------------------ diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional index f73f3093..fa86f917 100644 --- a/cdist/conf/type/__sed/parameter/optional +++ b/cdist/conf/type/__sed/parameter/optional @@ -1 +1,2 @@ file +onchange From cf0032d667eb4bd8b7fe1bebacec341bbf71ed42 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 7 Jul 2021 21:28:00 +0300 Subject: [PATCH 123/206] add messaging and exit earlier --- cdist/conf/type/__sed/gencode-remote | 42 +++++++++++++++------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 6b1bd216..836506f0 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -27,27 +27,31 @@ then sed_cmd="$sed_cmd --regexp-extended" fi -if ! echo "$script" \ +if echo "$script" \ | "$sed_cmd" -f - "$file_from_target" \ | diff "$file_from_target" - \ > /dev/null then - # shellcheck disable=SC2016 - echo 'tmp="$( mktemp )"' - - echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" - - echo "$script" - - echo 'EOF' - - echo "cp \"\$tmp\" '$file'" - - # shellcheck disable=SC2016 - echo 'rm -f "$tmp"' - - if [ -f "$__object/parameter/onchange" ] - then - cat "$__object/parameter/onchange" - fi + exit 0 +fi + +# shellcheck disable=SC2016 +echo 'tmp="$( mktemp )"' + +echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" + +echo "$script" + +echo 'EOF' + +echo "cp \"\$tmp\" '$file'" + +# shellcheck disable=SC2016 +echo 'rm -f "$tmp"' + +echo 'change' >> "$__messages_out" + +if [ -f "$__object/parameter/onchange" ] +then + cat "$__object/parameter/onchange" fi From 3e76d1cd3fbe53fa77c460cb2ce8416698b101bd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 8 Jul 2021 08:09:05 +0200 Subject: [PATCH 124/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 0c3c64e1..04f826f0 100644 --- a/docs/changelog +++ b/docs/changelog @@ -13,6 +13,7 @@ next: * Type __debconf_set_selections: Add state explorer (Dennis Camera) * Core: Implement usable cdist scan (Timothée Floure) * New type: __snakeoil_cert (Ander Punnar) + * Type __rsync: Honour $__remote_exec env var (Daniel Fancsali) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 77dab4c5c63070aef962875af7fe8b1565f5ba78 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 10 Jul 2021 20:37:02 +0200 Subject: [PATCH 125/206] Release 6.9.7 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 04f826f0..284293c1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) From 65c43d3c1db938ac0063f54b1cb6b5090bb0a665 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 10 Jul 2021 21:02:27 +0200 Subject: [PATCH 126/206] Fix docs code block errors --- cdist/conf/type/__snakeoil_cert/man.rst | 1 + docs/src/cdist-scan.rst | 2 ++ 2 files changed, 3 insertions(+) diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst index 0b547804..b0b0a2e9 100644 --- a/cdist/conf/type/__snakeoil_cert/man.rst +++ b/cdist/conf/type/__snakeoil_cert/man.rst @@ -38,6 +38,7 @@ cert-path EXAMPLES -------- .. code-block:: sh + __snakeoil_cert localhost-rsa \ --common-name localhost \ --key-type rsa:4096 diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst index 064e65ff..86b7fab6 100644 --- a/docs/src/cdist-scan.rst +++ b/docs/src/cdist-scan.rst @@ -57,6 +57,7 @@ resolved name to stdout - if any. The script must be executable. Simplest script: .. code-block:: sh + #!/bin/sh case "$1" in @@ -71,6 +72,7 @@ Simplest script: Resolving name from `PTR` DNS record: .. code-block:: sh + #!/bin/sh for cmd in dig sed; do From 0e611af2a6388572eef5112c2ffaed082803965c Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 13 Jul 2021 00:13:22 +0300 Subject: [PATCH 127/206] [__rsync] rewrite --- cdist/conf/type/__rsync/gencode-local | 123 +++++++++++++----- cdist/conf/type/__rsync/gencode-remote | 37 ------ cdist/conf/type/__rsync/man.rst | 107 +++++---------- cdist/conf/type/__rsync/manifest | 18 --- .../type/__rsync/parameter/default/options | 1 + cdist/conf/type/__rsync/parameter/optional | 4 +- .../type/__rsync/parameter/optional_multiple | 2 +- 7 files changed, 130 insertions(+), 162 deletions(-) delete mode 100755 cdist/conf/type/__rsync/gencode-remote create mode 100644 cdist/conf/type/__rsync/parameter/default/options diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index be4feabb..612d237e 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -1,41 +1,100 @@ #!/bin/sh -e -# -# 2015 Dominique Roux (dominique.roux4 at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -source=$(cat "$__object/parameter/source") -remote_user=$(cat "$__object/parameter/remote-user") +if ! command -v rsync > /dev/null +then + echo 'rsync is missing in local machine' >&2 + exit 1 +fi -if [ -f "$__object/parameter/destination" ]; then - destination=$(cat "$__object/parameter/destination") +src="$( cat "$__object/parameter/source" )" + +if [ ! -e "$src" ] +then + echo "$src not found" >&2 + exit 1 +fi + +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" else - destination="/$__object_id" + dst="/$__object_id" fi -set -- -if [ -f "$__object/parameter/rsync-opts" ]; then - while read -r opts; do - set -- "$@" "--$opts" - done < "$__object/parameter/rsync-opts" +# if source is directory, then make sure that +# source and destination are ending with slash, +# because this is what you almost always want when +# rsyncing two directories. + +if [ -d "$src" ] +then + if ! echo "$src" | grep -Eq '/$' + then + src="$src/" + fi + + if ! echo "$dst" | grep -Eq '/$' + then + dst="$dst/" + fi fi +remote_user="$( cat "$__object/parameter/remote-user" )" + +options="$( cat "$__object/parameter/options" )" + +if [ -f "$__object/parameter/option" ] +then + while read -r l + do + # there's a limitation in argparse: value can't begin with '-'. + # to workaround this, let's prefix opts with '\' in manifest and remove here. + # read more about argparse issue: https://bugs.python.org/issue9334 + + options="$options $( echo "$l" | sed 's/\\//g' )" + done \ + < "$__object/parameter/option" +fi + +if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ] +then + options="$options --chown=" + + if [ -f "$__object/parameter/owner" ] + then + owner="$( cat "$__object/parameter/owner" )" + options="$options$owner" + fi + + if [ -f "$__object/parameter/group" ] + then + group="$( cat "$__object/parameter/group" )" + options="$options:$group" + fi +fi + +if [ -f "$__object/parameter/mode" ] +then + mode="$( cat "$__object/parameter/mode" )" + options="$options --chmod=$mode" +fi + +# IMPORTANT +# +# 1. we first dry-run rsync with change summary to find out +# if there are any changes and code generation is needed. +# 2. normally, to get current state or target host, we run +# such operations in type explorers, but that's not +# possible due to how rsync works. +# 3. redirecting output of dry-run to stderr to ease debugging. +# 4. to understand how that cryptic regex works, please +# open rsync manpage and read about --itemize-changes. + # shellcheck disable=SC2086 -echo rsync -a \ - --no-owner --no-group \ - -e \"${__remote_exec}\" \ - -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" +if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \ + | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2 +then + exit 0 +fi + +echo "rsync $options $src $remote_user@$__target_host:$dst" diff --git a/cdist/conf/type/__rsync/gencode-remote b/cdist/conf/type/__rsync/gencode-remote deleted file mode 100755 index 074246af..00000000 --- a/cdist/conf/type/__rsync/gencode-remote +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -e -# -# 2015 Dominique Roux (dominique.roux4 at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -if [ -f "$__object/parameter/destination" ]; then - destination=$(cat "$__object/parameter/destination") -else - destination="/$__object_id" -fi - -ownergroup="" -if [ -f "$__object/parameter/owner" ]; then - ownergroup=$(cat "$__object/parameter/owner") -fi -if [ -f "$__object/parameter/group" ]; then - ownergroup="${ownergroup}:$(cat "$__object/parameter/group")" -fi - -if [ "$ownergroup" ]; then - echo chown -R "$ownergroup" "$destination" -fi diff --git a/cdist/conf/type/__rsync/man.rst b/cdist/conf/type/__rsync/man.rst index 94b06d63..88019c92 100644 --- a/cdist/conf/type/__rsync/man.rst +++ b/cdist/conf/type/__rsync/man.rst @@ -3,112 +3,73 @@ cdist-type__rsync(7) NAME ---- -cdist-type__rsync - Mirror directories using rsync +cdist-type__rsync - Mirror directories using ``rsync`` DESCRIPTION ----------- -WARNING: This type is of BETA quality: - -- it has not been tested widely -- interfaces *may* change -- if there is a better approach to solve the problem -> the type may even vanish - -If you are fine with these constraints, please read on. - - -This cdist type allows you to mirror local directories to the -target host using rsync. Rsync will be installed in the manifest of the type. -If group or owner are giveng, a recursive chown will be executed on the -target host. - -A slash will be appended to the source directory so that only the contents -of the directory are taken and not the directory name itself. +The purpose of this type is to bring power of ``rsync`` into ``cdist``. REQUIRED PARAMETERS ------------------- source - Where to take files from + Source directory in local machine. + If source is directory, slash (``/``) will be added to source and destination paths. OPTIONAL PARAMETERS ------------------- -group - Group to chgrp to. +destination + Destination directory. Defaults to ``$__object_id``. owner - User to chown to. + Will be passed to ``rsync`` as ``--chown=OWNER``. + Read ``rsync(1)`` for more details. -destination - Use this as the base destination instead of the object id +group + Will be passed to ``rsync`` as ``--chown=:GROUP``. + Read ``rsync(1)`` for more details. + +mode + Will be passed to ``rsync`` as ``--chmod=MODE``. + Read ``rsync(1)`` for more details. + +options + Defaults to ``--recursive --links --perms --times``. + Due to `bug in Python's argparse`_, value must be prefixed with ``\``. remote-user - Use this user instead of the default "root" for rsync operations. + Defaults to ``root``. OPTIONAL MULTIPLE PARAMETERS ---------------------------- -rsync-opts - Use this option to give rsync options with. - See rsync(1) for available options. - Only "--" options are supported. - Write the options without the beginning "--" - Can be specified multiple times. - - -MESSAGES --------- -NONE +option + Pass additional options to ``rsync``. + See ``rsync(1)`` for all possible options. + Due to `bug in Python's argparse`_, value must be prefixed with ``\``. EXAMPLES -------- - .. code-block:: sh - # You can use any source directory - __rsync /tmp/testdir \ - --source /etc - - # Use source from type - __rsync /etc \ - --source "$__type/files/package" - - # Allow multiple __rsync objects to write to the same dir - __rsync mystuff \ - --destination /usr/local/bin \ - --source "$__type/files/package" - - __rsync otherstuff \ - --destination /usr/local/bin \ - --source "$__type/files/package2" - - # Use rsync option --exclude - __rsync /tmp/testdir \ - --source /etc \ - --rsync-opts exclude=sshd_conf - - # Use rsync with multiple options --exclude --dry-run - __rsync /tmp/testing \ - --source /home/tester \ - --rsync-opts exclude=id_rsa \ - --rsync-opts dry-run - - -SEE ALSO --------- -:strong:`rsync`\ (1) + __rsync /var/www/example.com \ + --owner root \ + --group www-data \ + --mode 'D750,F640' \ + --source "$__files/example.com/www" AUTHORS ------- -Nico Schottelius +Ander Punnar COPYING ------- -Copyright \(C) 2015 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. diff --git a/cdist/conf/type/__rsync/manifest b/cdist/conf/type/__rsync/manifest index 9bd44c6d..64fa804e 100755 --- a/cdist/conf/type/__rsync/manifest +++ b/cdist/conf/type/__rsync/manifest @@ -1,21 +1,3 @@ #!/bin/sh -e -# -# 2015 Dominique Roux (dominique.roux4 at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# __package rsync diff --git a/cdist/conf/type/__rsync/parameter/default/options b/cdist/conf/type/__rsync/parameter/default/options new file mode 100644 index 00000000..d967b110 --- /dev/null +++ b/cdist/conf/type/__rsync/parameter/default/options @@ -0,0 +1 @@ +--recursive --links --perms --times diff --git a/cdist/conf/type/__rsync/parameter/optional b/cdist/conf/type/__rsync/parameter/optional index ac2b2390..833e9bbe 100644 --- a/cdist/conf/type/__rsync/parameter/optional +++ b/cdist/conf/type/__rsync/parameter/optional @@ -1,4 +1,6 @@ destination -owner group +mode +options +owner remote-user diff --git a/cdist/conf/type/__rsync/parameter/optional_multiple b/cdist/conf/type/__rsync/parameter/optional_multiple index fdb7cd88..01925a15 100644 --- a/cdist/conf/type/__rsync/parameter/optional_multiple +++ b/cdist/conf/type/__rsync/parameter/optional_multiple @@ -1 +1 @@ -rsync-opts +option From 46b5c24cd240cd9006d93cfcc12a4d81b46a5238 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 18 Jul 2021 16:25:00 +0300 Subject: [PATCH 128/206] use $__remote_exec for RSYNC_RSH --- cdist/conf/type/__rsync/gencode-local | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index 612d237e..e9f3c131 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -90,6 +90,8 @@ fi # 4. to understand how that cryptic regex works, please # open rsync manpage and read about --itemize-changes. +export RSYNC_RSH="$__remote_exec" + # shellcheck disable=SC2086 if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \ | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2 @@ -97,4 +99,6 @@ then exit 0 fi +echo "export RSYNC_RSH='$__remote_exec'" + echo "rsync $options $src $remote_user@$__target_host:$dst" From 5229337611b7e7afb2597729a786c637a8bec1f6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 18 Jul 2021 17:41:29 +0200 Subject: [PATCH 129/206] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 284293c1..f0746218 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __rsync: Rewrite (Ander Punnar) + 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) From de116661613002e91b54194ac8f0d5c3dd3eebbd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 18 Jul 2021 17:45:19 +0200 Subject: [PATCH 130/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index f0746218..ea51ac1b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Type __rsync: Rewrite (Ander Punnar) + * New type: __apt_pin (Daniel Fancsali) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 24c9406ea0f2c6c8edc87e5bbc1be25b5e8e1572 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 19 Jul 2021 12:13:23 +0200 Subject: [PATCH 131/206] [explorer/os_version] Convert Devuan ceres to version number Conversion of Devuan ceres to version numbers is done based on Devuan codenames. The version number is the version number of the final release - 0.01. Analogous to Debian. --- cdist/conf/explorer/os_version | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 3b02dedd..6c94915c 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -1,6 +1,7 @@ -#!/bin/sh +#!/bin/sh -e # # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2020-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,12 +18,11 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# # All os variables are lower case # -# -case "$("$__explorer/os")" in +case $("${__explorer:?}/os") +in amazon) cat /etc/system-release ;; @@ -59,7 +59,23 @@ case "$("$__explorer/os")" in esac ;; devuan) - cat /etc/devuan_version + devuan_version=$(cat /etc/devuan_version) + case ${devuan_version} + in + (*/ceres) + # ceres versions don't have a number, so we decode by codename: + case ${devuan_version} + in + (chimaera/ceres) echo 3.99 ;; + (beowulf/ceres) echo 2.99 ;; + (ascii/ceres) echo 1.99 ;; + (*) exit 1 + esac + ;; + (*) + echo "${devuan_version}" + ;; + esac ;; fedora) cat /etc/fedora-release From fbc9594729ece74b2f8612fe8046ddea966e8c05 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 20 Jul 2021 06:38:46 +0200 Subject: [PATCH 132/206] ++changelog --- docs/changelog | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/changelog b/docs/changelog index ea51ac1b..55d7cb73 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) + * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) @@ -146,7 +147,7 @@ next: * Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham) * Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham) * Configuration: Add notes to cdist.cfg.skeleton (Evil Ham) - * Explorers cpu_cores, memory: Improve *BSD support (Evil Ham) + * Explorers cpu_cores, memory: Improve BSD support (Evil Ham) * Core: Remove debug logging noise (Evil Ham) 6.5.4: 2020-04-11 @@ -211,7 +212,7 @@ next: * Documentation: PreOS english nitpicking (Evil Ham) * Documentation: Add installing from source with signature verification (Darko Poljak) * Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak) - * Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham) + * Type __start_on_boot: Docs: remove unsupported BSD claim (Evil Ham) * New type: __openldap_server (Evil Ham) 6.2.0: 2019-11-30 @@ -1070,9 +1071,9 @@ next: * Removed type __removeline (replaced by __line) (Nico Schottelius) * Type __directory: Parameter --parents and --recursive are now boolean (Nico Schottelius) * Type __package_apt, __package_luarocks, __package_opkg, - __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, - __package_rubygem, __package_yum, __process: - Parameter state accepts only "present" and "absent" (Nico Schottelius) + __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, + __package_rubygem, __package_yum, __process: + Parameter state accepts only "present" and "absent" (Nico Schottelius) * Dist: Initial support for pypi packaging (Nico Schottelius) 2.0.15: 2012-11-02 From c7daaabc6c28cbfaebaf6c620cc3cac130684d2c Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 20 Jul 2021 09:03:16 +0200 Subject: [PATCH 133/206] [docs] Bump copyright year to 2021 --- docs/src/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/conf.py b/docs/src/conf.py index 47765413..a3dfafca 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -56,7 +56,7 @@ master_doc = 'index' # General information about the project. project = 'cdist' -copyright = 'ungleich GmbH 2020' +copyright = 'ungleich GmbH 2021' # author = 'Darko Poljak' # The version info for the project you're documenting, acts as replacement for From fed01ded83b217a3ad147f2403ca863be373eeaa Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 22 Jul 2021 11:17:41 +0200 Subject: [PATCH 134/206] [cdist.log] Define custom log functions on logging.Logger Define out custom logger functions on logging.Logger so that they are passed on to all other loggers. Also, the logger functions need to take a self argument so that they can log on the corrent Logger. --- cdist/log.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cdist/log.py b/cdist/log.py index 113f3b4c..62e457fe 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -36,25 +36,27 @@ import threading logging.OFF = logging.CRITICAL + 10 # disable logging logging.addLevelName(logging.OFF, 'OFF') + logging.VERBOSE = logging.INFO - 5 logging.addLevelName(logging.VERBOSE, 'VERBOSE') -def _verbose(msg, *args, **kwargs): - logging.log(logging.VERBOSE, msg, *args, **kwargs) +def _verbose(self, msg, *args, **kwargs): + self.log(logging.VERBOSE, msg, args, **kwargs) -logging.verbose = _verbose +logging.Logger.verbose = _verbose + logging.TRACE = logging.DEBUG - 5 logging.addLevelName(logging.TRACE, 'TRACE') -def _trace(msg, *args, **kwargs): - logging.log(logging.TRACE, msg, *args, **kwargs) +def _trace(self, msg, *args, **kwargs): + self.log(logging.TRACE, msg, *args, **kwargs) -logging.trace = _trace +logging.Logger.trace = _trace class CdistFormatter(logging.Formatter): From 71fee1fd6b4874abffee5173a96c31842a9583bd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 23 Jul 2021 08:06:45 +0200 Subject: [PATCH 135/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 55d7cb73..1273e432 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) + * Core: Fix logging bug (Dennis Camera) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 67bcc6cae38e2d480c3336b6a4d59d01196190e3 Mon Sep 17 00:00:00 2001 From: Evilham Date: Sat, 24 Jul 2021 02:37:58 +0200 Subject: [PATCH 136/206] Improve Makefile compatibility and build docs We now use `$(MAKE)` for subsequent calls to `make`. This means that systems that do not default to GNU make can run `gmake man` and produce the man pages. While there also document a dependency on the rtd theme for sphinx. --- Makefile | 6 +++--- docs/src/cdist-install.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3712511c..89286310 100644 --- a/Makefile +++ b/Makefile @@ -35,9 +35,9 @@ DOCS_SRC_DIR=./docs/src SPEECHDIR=./docs/speeches TYPEDIR=./cdist/conf/type -SPHINXM=make -C $(DOCS_SRC_DIR) man -SPHINXH=make -C $(DOCS_SRC_DIR) html -SPHINXC=make -C $(DOCS_SRC_DIR) clean +SPHINXM=$(MAKE) -C $(DOCS_SRC_DIR) man +SPHINXH=$(MAKE) -C $(DOCS_SRC_DIR) html +SPHINXC=$(MAKE) -C $(DOCS_SRC_DIR) clean ################################################################################ # Manpages diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst index 18863145..390ab9ec 100644 --- a/docs/src/cdist-install.rst +++ b/docs/src/cdist-install.rst @@ -12,7 +12,7 @@ This is the machine from which you will configure target hosts. * /bin/sh: A POSIX like shell (for instance bash, dash, zsh) * Python >= 3.5 * SSH client - * sphinx (for building html docs and/or the man pages) + * sphinx with the rtd theme (for building html docs and/or the man pages) Target Hosts ~~~~~~~~~~~~ From cb8695cc88478640d7e76992438d90d3d6a68a90 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 24 Jul 2021 12:53:39 +0200 Subject: [PATCH 137/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 1273e432..181c4139 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * New type: __apt_pin (Daniel Fancsali) * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) * Core: Fix logging bug (Dennis Camera) + * Build: Improve Makefile compatibility (Evilham) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 4156fea9001aa267c8b173247cada2f919511c1b Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Wed, 28 Jul 2021 12:56:39 +0200 Subject: [PATCH 138/206] [filesystem] Add ubuntu as supported distribution. --- cdist/conf/type/__filesystem/explorer/lsblk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk index 9be3c575..d376c09f 100644 --- a/cdist/conf/type/__filesystem/explorer/lsblk +++ b/cdist/conf/type/__filesystem/explorer/lsblk @@ -27,7 +27,7 @@ else fi case "$os" in - alpine|centos|fedora|redhat|suse|gentoo) + alpine|centos|fedora|gentoo|redhat|suse|ubuntu) if [ ! -x "$(command -v lsblk)" ]; then echo "lsblk is required for __filesystem type" >&2 exit 1 From 542674dae81ab03a4de2c4ad0b2eaf264ee2c442 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 30 Jul 2021 10:30:33 +0200 Subject: [PATCH 139/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 181c4139..9fc10b20 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) * Core: Fix logging bug (Dennis Camera) * Build: Improve Makefile compatibility (Evilham) + * Type __filesystem: Support ubuntu (Joachim Desroches) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 53334fb4eb311550af7f5b73f279a2e86fa1c504 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 4 Aug 2021 19:50:10 +0200 Subject: [PATCH 140/206] [explorer/os_version] Fix for FreeBSD < 10.0 (again) --- cdist/conf/explorer/os_version | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 6c94915c..cc976608 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -89,7 +89,14 @@ in freebsd) # Apparently uname -r is not a reliable way to get the patch level. # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743 - freebsd-version + if command -v freebsd-version >/dev/null 2>&1 + then + # get userland version + freebsd-version -u + else + # fallback to kernel release for FreeBSD < 10.0 + uname -r + fi ;; *bsd|solaris) uname -r From e108cbc205cb5e7ac0d2e07b82cef4c83eaa285f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 3 Aug 2021 13:20:43 +0200 Subject: [PATCH 141/206] [explorer/os_version] Ubuntu: fall back to os-release/lsb-release files --- cdist/conf/explorer/os_version | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 6c94915c..e70fe7f4 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -21,6 +21,17 @@ # All os variables are lower case # +rc_getvar() { + awk -F= -v varname="$2" ' + function unquote(s) { + if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/) + return substr(s, 2, length(s) - 2) + else + return s + } + $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1" +} + case $("${__explorer:?}/os") in amazon) @@ -114,7 +125,20 @@ in fi ;; ubuntu) - lsb_release -sr + if command -v lsb_release >/dev/null 2>&1 + then + lsb_release -sr + elif test -r /usr/lib/os-release + then + # fallback to /usr/lib/os-release if lsb_release is not present (like + # on minimized Ubuntu installations) + rc_getvar /usr/lib/os-release VERSION_ID + elif test -r /etc/lsb-release + then + # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old + # versions without /usr/lib/os-release. + rc_getvar /etc/lsb-release DISTRIB_RELEASE + fi ;; alpine) cat /etc/alpine-release From 83fe6e9f5b2537db73d5c6a142b6d24eef75ac58 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 3 Aug 2021 19:26:55 +0200 Subject: [PATCH 142/206] [explorer/memory] Fix conversion of large numbers (>= 2GiB) At least mawk uses scientific notation when using print for numbers >=2^31 (INT_MAX of a signed 32-bit int). `printf "%.f\n"` works around this. --- cdist/conf/explorer/memory | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory index 63aba9c6..c6d113cf 100755 --- a/cdist/conf/explorer/memory +++ b/cdist/conf/explorer/memory @@ -27,19 +27,18 @@ str2bytes() { awk -F' ' ' $2 == "B" || !$2 { print $1 } - $2 == "kB" { print $1 * 1000 } - $2 == "MB" { print $1 * 1000 * 1000 } - $2 == "GB" { print $1 * 1000 * 1000 * 1000 } - $2 == "TB" { print $1 * 1000 * 1000 * 1000 * 1000 } - $2 == "kiB" { print $1 * 1024 } - $2 == "MiB" { print $1 * 1024 * 1024 } - $2 == "GiB" { print $1 * 1024 * 1024 * 1024 } - $2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }' + $2 == "kB" { printf "%.f\n", ($1 * 1000) } + $2 == "MB" { printf "%.f\n", ($1 * 1000 * 1000) } + $2 == "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) } + $2 == "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) } + $2 == "kiB" { printf "%.f\n", ($1 * 1024) } + $2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) } + $2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) } + $2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }' } bytes2kib() { - set -- "$(cat)" - test "$1" -gt 0 && echo $(($1 / 1024)) + awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }' } From a7d6481a7ddc7cb72b1a55bfca7fdfed20514a62 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 2 Aug 2021 21:23:50 +0200 Subject: [PATCH 143/206] [type/__update_alternatives] Secure cdist-defined environment variables with :? --- .../__update_alternatives/explorer/alternatives | 2 +- cdist/conf/type/__update_alternatives/explorer/link | 6 +++--- .../type/__update_alternatives/explorer/path_is | 4 ++-- .../explorer/path_should_state | 2 +- .../conf/type/__update_alternatives/gencode-remote | 13 ++++++------- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives index 34aaca56..ecc62f4b 100755 --- a/cdist/conf/type/__update_alternatives/explorer/alternatives +++ b/cdist/conf/type/__update_alternatives/explorer/alternatives @@ -1,4 +1,4 @@ #!/bin/sh -e -update-alternatives --display "$__object_id" 2>/dev/null \ +update-alternatives --display "${__object_id:?}" 2>/dev/null \ | awk -F ' - ' '/priority [0-9]+$/ { print $1 }' diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link index 6519e7c2..c6fd1c98 100755 --- a/cdist/conf/type/__update_alternatives/explorer/link +++ b/cdist/conf/type/__update_alternatives/explorer/link @@ -18,12 +18,12 @@ for altdir in \ /var/lib/dpkg/alternatives \ /var/lib/alternatives do - if [ ! -f "$altdir/$__object_id" ] + if [ ! -f "$altdir/${__object_id:?}" ] then continue fi - link="$( awk 'NR==2' "$altdir/$__object_id" )" + link="$( awk 'NR==2' "$altdir/${__object_id:?}" )" if [ -n "$link" ] then @@ -33,7 +33,7 @@ done if [ -z "$link" ] then - echo "unable to get link for $__object_id" >&2 + echo "unable to get link for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index fc304d5d..a24bd40e 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -1,11 +1,11 @@ #!/bin/sh -e -path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \ +path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \ | awk '/link currently points to/ {print $5}' )" if [ -z "$path_is" ] then - echo "unable to get current path for $__object_id" >&2 + echo "unable to get current path for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state index 59e015c5..b74a7ee8 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_should_state +++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state @@ -1,6 +1,6 @@ #!/bin/sh -e -if [ -f "$( cat "$__object/parameter/path" )" ] +if [ -f "$( cat "${__object:?}/parameter/path" )" ] then echo 'present' else diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index e393cdef..13666805 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -18,26 +18,25 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . -path_is="$( cat "$__object/explorer/path_is" )" +path_is="$( cat "${__object:?}/explorer/path_is" )" -path_should="$( cat "$__object/parameter/path" )" +path_should="$( cat "${__object:?}/parameter/path" )" if [ "$path_is" = "$path_should" ] then exit 0 fi -if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ] +if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \ + && [ -z "${__cdist_dry_run+dry run}" ] then echo "$path_should does not exist in target" >&2 exit 1 fi -name="$__object_id" +name=${__object_id:?} -alternatives="$( cat "$__object/explorer/alternatives" )" - -if ! echo "$alternatives" | grep -Fxq "$path_should" +if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives" then if [ ! -f "$__object/parameter/install" ] then From 0b3b47396f2aafa377e3d5d9a13f51ace2303d41 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 2 Aug 2021 21:25:08 +0200 Subject: [PATCH 144/206] [type/__update_alternatives] dry-run fixes --- cdist/conf/type/__update_alternatives/explorer/link | 5 ++++- .../type/__update_alternatives/explorer/path_is | 5 ++++- .../conf/type/__update_alternatives/gencode-remote | 13 ++++++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link index c6fd1c98..d1087c75 100755 --- a/cdist/conf/type/__update_alternatives/explorer/link +++ b/cdist/conf/type/__update_alternatives/explorer/link @@ -31,8 +31,11 @@ do fi done -if [ -z "$link" ] +if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ] then + # NOTE: ignore error for dry-runs because a package providing the link + # might be managed by another cdist object (which wasn't executed, + # because dry run…). echo "unable to get link for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index a24bd40e..9208df7b 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -3,8 +3,11 @@ path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \ | awk '/link currently points to/ {print $5}' )" -if [ -z "$path_is" ] +if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ] then + # NOTE: ignore error for dry-runs because a package providing the + # alternative might be managed by another cdist object (which + # wasn't executed, because dry run…). echo "unable to get current path for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index 13666805..e91ea78f 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -38,16 +38,19 @@ name=${__object_id:?} if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives" then - if [ ! -f "$__object/parameter/install" ] + if [ -f "${__object:?}/parameter/install" ] then + link="$( cat "${__object:?}/explorer/link" )" + echo "update-alternatives --install '$link' '$name' '$path_should' 1000" + elif [ -z "${__cdist_dry_run+dry run}" ] + then + # NOTE: ignore error for dry-runs because a package providing the link + # to be installed might be managed by another cdist object (which + # wasn't executed, because dry run…). echo "$path_should is not in $name alternatives." >&2 echo 'Please install missing packages or use --install to add path to alternatives.' >&2 exit 1 fi - - link="$( cat "$__object/explorer/link" )" - - echo "update-alternatives --install '$link' '$name' '$path_should' 1000" fi echo "update-alternatives --set '$name' '$path_should'" From bbcc81a9841f2619e1b9e13b25a941337489a681 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 4 Aug 2021 21:44:04 +0200 Subject: [PATCH 145/206] [type/__update_alternatives] Fix for non-English locales Since update-alternatives(1) is localized, screen scraping its output breaks if the locale is set to non-English. --- cdist/conf/type/__update_alternatives/explorer/alternatives | 4 ++-- cdist/conf/type/__update_alternatives/explorer/path_is | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives index ecc62f4b..bb1619a9 100755 --- a/cdist/conf/type/__update_alternatives/explorer/alternatives +++ b/cdist/conf/type/__update_alternatives/explorer/alternatives @@ -1,4 +1,4 @@ #!/bin/sh -e -update-alternatives --display "${__object_id:?}" 2>/dev/null \ - | awk -F ' - ' '/priority [0-9]+$/ { print $1 }' +LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \ +| awk -F ' - ' '/priority [0-9]+$/ { print $1 }' diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index 9208df7b..5cf4fa4b 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -1,7 +1,8 @@ #!/bin/sh -e -path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \ - | awk '/link currently points to/ {print $5}' )" +path_is=$( + LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \ + | awk '/link currently points to/ { print $5 }') if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ] then From 2a0c073d4021206c7459015cefaba218004235ce Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 4 Aug 2021 21:54:17 +0200 Subject: [PATCH 146/206] [explorer/os_version] Fix for legacy Mac OS X versions --- cdist/conf/explorer/os_version | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 3b02dedd..96eca1ee 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -68,7 +68,8 @@ case "$("$__explorer/os")" in cat /etc/gentoo-release ;; macosx) - sw_vers -productVersion + # NOTE: Legacy versions (< 10.3) do not support options + sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }' ;; freebsd) # Apparently uname -r is not a reliable way to get the patch level. From 3ae5a606ca7182ec7fe13134670400506e198ec1 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 5 Aug 2021 10:27:51 +0200 Subject: [PATCH 147/206] ++changelog --- docs/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog b/docs/changelog index 9fc10b20..f9409d7e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,10 @@ next: * Core: Fix logging bug (Dennis Camera) * Build: Improve Makefile compatibility (Evilham) * Type __filesystem: Support ubuntu (Joachim Desroches) + * Explorer os_version: Fall back to os-release/lsb-release file on Ubuntu (Dennis Camera) + * Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera) + * Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera) + * Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From edcac70b2a99ce532dd5bca9a0b8fc5bf6dc5148 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 21 Jul 2021 13:22:34 +0200 Subject: [PATCH 148/206] [explorer/machine_type] Reimplement --- cdist/conf/explorer/machine_type | 986 ++++++++++++++++++++++++++++--- 1 file changed, 904 insertions(+), 82 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 1c84f4d7..29f98849 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -1,8 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2014 Daniel Heule (hda at sfs.biz) -# 2014 Thomas Oettli (otho at sfs.biz) -# 2020 Evilham (contact at evilham.com) +# 2021 Dennis Camera (cdist at dtnr.ch) # # This file is part of cdist. # @@ -19,91 +17,915 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# This explorer tries to determine what type of machine the target to be +# configured is (container, virtual machine, bare-metal). +# +# It will print one line for each layer it can detect. +# The format of all lines is: TYPE[ VERB VENDOR] +# +# VERB does not have a special meaning, it is just for better readability. +# +# e.g. +# container on lxc +# virtual by kvm-spapr +# +# The third word of each line can be composed of different parts concatenated with a `-' +# (minus) character, with each component being a specification of the previous, +# e.g.: +# - lxc-libvirt (LXC container, managed by libvirt) +# - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) +# - xen-hvm / xen-pv (Xen HVM vs para-virtualization) +# +# If this explorer cannot determine any information it will print nothing. +# -os=$("$__explorer/os") +# Add /sbin and /usr/sbin to the path so we can find system +# binaries like dmidecode. +PATH=$(getconf PATH 2>/dev/null) || PATH='/usr/bin:/bin' +PATH="/sbin:/usr/sbin:${PATH}" +export PATH -vendor_string_to_machine_type() { - for vendor in vmware bochs kvm qemu virtualbox bhyve; do - if echo "${1}" | grep -q -i "${vendor}"; then - if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then - vendor="kvm" - fi - echo "virtual_by_${vendor}" - exit - fi - done +arch=$(uname -m | sed -e 's/i.86/i386/' -e 's/arm.*/arm/') +uname_s=$(uname -s) + + +is_command() { command -v "$1" >/dev/null 2>&1; } + +is_oneof() ( + x=$1; shift + for y + do + test "${x}" = "${y}" || continue + return 0 + done + return 1 +) + +tolower() { LC_ALL=C tr '[:upper:]' '[:lower:]'; } + +# shellcheck disable=SC2086 +glob_exists() { set -- $1; test -e "$1"; } + +get_dmi_field() { + if is_oneof "${uname_s}" NetBSD + then + case $1 + in + (system-manufacturer) _mib=machdep.dmi.system-vendor ;; + (system-product-name) _mib=machdep.dmi.system-product ;; + (system-version|system-uuid) _mib=machdep.dmi.$1 ;; + (bios-vendor|bios-version) _mib=machdep.dmi.$1 ;; + (biod-release-date) _mib=machdep.dmi.bios-date ;; + (*) _mib= ;; + esac + + test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return + fi + + if is_command dmidecode + then + dmidecode -s "$1" + elif test -d "${dmi_sysfs-}" + then + case $1 + in + (system-manufacturer) _filename=sys_vendor ;; + (system-product-name) _filename=product_name ;; + (*) _filename=$(echo "$1" | tr - _) ;; + esac + if test -r "${dmi_sysfs-}/${_filename}" + then + cat "${dmi_sysfs}/${_filename}" + fi + unset _filename + elif test "${uname_s}" = OpenBSD + then + # NOTE: something similar to system-manufacutrer and system-product-name + # is available on OpenBSD in sysctl + case $1 + in + (system-manufacturer) _mib=hw.vendor ;; + (system-product-name) _mib=hw.product ;; + (*) _mib= ;; + esac + + test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return + fi + + return 1 } -case "$os" in - "freebsd") - # FreeBSD does not have /proc/cpuinfo even when procfs is used. - # Instead there is a sysctl kern.vm_guest. - # Which is 'none' if physical, else the virtualisation. - vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)" - if [ -n "${vm_guest}" ]; then - if [ "${vm_guest}" = "none" ]; then - echo "physical" - exit - fi - echo "virtual_by_${vm_guest}" - exit - fi - ;; +has_cpuinfo() { test -e /proc/cpuinfo; } - "openbsd") - # OpenBSD can also use the sysctl's: hw.vendor or hw.product. - # Note we can be reasonably sure about a machine being virtualised - # as long as we can identify the virtualisation technology. - # But not so much about it being physical... - # Patches are welcome / reach out if you have better ideas. - for sysctl in hw.vendor hw.product; do - # This exits if we can make a reasonable judgement - vendor_string_to_machine_type "$(sysctl -n "${sysctl}")" - done - ;; +get_sysctl() { + is_command sysctl && sysctl -n "$1" 2>/dev/null +} - *) - # Defaulting to linux for compatibility with previous cdist behaviour - if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then - echo openvz - exit - fi +# Check for container - if [ -e "/proc/1/environ" ] && - tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then - echo lxc - exit - fi +has_ct_pid_1() { + test -r /run/systemd/container -o -r /proc/1/environ +} - if [ -r /proc/cpuinfo ]; then - # this should only exist on virtual guest machines, - # tested on vmware, xen, kvm, bhyve - if grep -q "hypervisor" /proc/cpuinfo; then - # this file is aviable in xen guest systems - if [ -r /sys/hypervisor/type ]; then - if grep -q -i "xen" /sys/hypervisor/type; then - echo virtual_by_xen - exit - fi - else - for vendor_file in /sys/class/dmi/id/product_name \ - /sys/class/dmi/id/sys_vendor \ - /sys/class/dmi/id/chasis_vendor; do - if [ -r ${vendor_file} ]; then - # This exits if we can make a reasonable judgement - vendor_string_to_machine_type "$(cat "${vendor_file}")" - fi - done - fi - echo "virtual_by_unknown" - exit - else - echo "physical" - exit - fi - fi - ;; -esac +translate_container_name() { + case $1 + in + ('lxc') + echo lxc ;; + ('lxc-libvirt') + echo lxc-libvirt ;; + ('podman') + echo podman ;; + ('systemd-nspawn') + echo systemd_nspawn ;; + (*) + return 1 ;; + esac + return 0 +} -echo "unknown" +check_ct_pid_1() { + if test -r /run/systemd/container + then + translate_container_name "$(head -n1 /run/systemd/container)" \ + && return 0 + fi + + if test -r /proc/1/environ + then + translate_container_name "$( + LC_ALL=C tr '\000' '\n' /dev/null + then + # https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 + echo wsl + elif test -d /var/.cagefs + then + # https://docs.cloudlinux.com/cloudlinux_os_components/#cagefs + # CageFS is not "really" a container, but it isn't a chroot either. + echo cagefs + elif test -e /proc/self/status && grep -q -e '^VxID: [0-9]\{1,\}' /proc/self/status + then + # Linux-VServer + if grep -q -x -F 'VxID: 0' /proc/self/status + then + # host + return 1 + else + # guest + echo linux_vserver + fi + else + return 1 + fi +} + +check_ct_os_specific() ( + if jailed=$(get_sysctl security.jail.jailed) && test "${jailed}" = 1 + then + # FreeBSD jail + echo jail + return 0 + fi + + if is_command zonename && test "$(zonename)" != global + then + # Solaris zone + echo zone + return 0 + fi + + return 1 +) + + +# Check for hypervisor + +guess_hypervisor_from_cpu_model() { + case $1 + in + (*\ KVM\ *) + echo kvm ;; + (*\ QEMU\ *|QEMU\ *) + echo qemu ;; + (*) + return 1 ;; + esac +} + +has_vm_cpuinfo() { has_cpuinfo; } + +check_vm_cpuinfo() { + if grep -q -F 'User Mode Linux' /proc/cpuinfo \ + || grep -q -F 'UML' /proc/cpuinfo + then + # User Mode Linux + echo uml + elif grep -q -e '^vendor_id.*: PowerVM Lx86' /proc/cpuinfo + then + # IBM PowerVM Lx86 (Linux/x86 emulator) + echo powervm_lx86 + elif grep -q -e '^vendor_id.*: IBM/S390' /proc/cpuinfo + then + # IBM SystemZ (S/390) + if test -f /proc/sysinfo + then + if grep -q -e '^VM[0-9]* Control Program: KVM/Linux' /proc/sysinfo + then + echo kvm-s390 + return 0 + elif grep -q -e '^VM[0-9]* Control Program: z/VM' /proc/sysinfo + then + echo zvm + return 0 + elif grep -q -e '^LPAR ' /proc/sysinfo + then + echo zvm-lpar + return 0 + fi + fi + return 1 + else + if grep -q -e '^model name.*:' /proc/cpuinfo + then + sed -n -e 's/^model name[^:]*: *//p' /proc/cpuinfo \ + | while read -r _cpu_model + do + guess_hypervisor_from_cpu_model "${_cpu_model}" + done \ + | sort \ + | uniq -c \ + | awk ' + { if ($1 > most_c) { most_c = $1; most_s = $2 } } + END { + if (most_s) print most_s + exit !most_s + }' \ + && return 0 + fi + return 1 + fi +} + +check_vm_arch_specific() { + case ${arch} + in + (ppc64|ppc64le) + # Check PPC64 LPAR, KVM + + # example /proc/cpuinfo line indicating 'not baremetal' + # platform : pSeries + # + # example /proc/ppc64/lparcfg systemtype line + # system_type=IBM pSeries (emulated by qemu) + + if has_cpuinfo && grep -q -e 'platform.**pSeries' /proc/cpuinfo + then + if test -e /proc/ppc64/lparcfg + then + # Assume LPAR, now detect shared or dedicated + if grep -q -x -F 'shared_processor_mode=1' /proc/ppc64/lparcfg + then + echo powervm-shared + return 0 + else + echo powervm-dedicated + return 0 + fi + fi + fi + ;; + (sparc*) + # Check for SPARC LDoms + + if test -e /dev/mdesc + then + if test -d /sys/class/vlds/ctrl -a -d /sys/class/vlds/sp + then + # control LDom + return 1 + else + # guest LDom + echo ldom-sparc + fi + + # MDPROP=/usr/lib/ldoms/mdprop.py + # if test -x "${MDPROP}" + # then + # if test -n "$("${MDPROP}" -v iodevice device-type=pciex)" + # then + # echo ldoms-root + # echo ldoms-io + # elif test -n "$("${MDPROP}" -v iov-device vf-id=0)" + # then + # echo ldoms-io + # fi + # fi + return 0 + fi + ;; + (i?86|x86*|amd64|i86pc) + # Check CPUID + # + # Many fullvirt hypervisors give an indication through CPUID. Use + # the virt-what helper program to get this information if available. + + for CPUID_HELPER in \ + $(command -v virt-what-cpuid-helper 2>/dev/null) \ + /usr/lib/x86_64-*/virt-what-cpuid-helper \ + /usr/lib/i?86-*/virt-what-cpuid-helper \ + /usr/lib/virt-what/virt-what-cpuid-helper + do + if test -x "${CPUID_HELPER:?}"; then break; fi + done + + if test -x "${CPUID_HELPER-}" + then + case $(command "${CPUID_HELPER}") + in + ('bhyve bhyve ') + echo bhyve + ;; + ('LKVMLKVMLKVM') + echo lkvm + ;; + ('KVMKVMKVM') + echo kvm + ;; + ('TCGTCGTCGTCG') + echo qemu-tcg + ;; + ('Microsoft Hv') + # http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx + echo hyperv + ;; + ('OpenBSDVMM58') + # OpenBSD/VMM + echo openbsd_vmm + ;; + ('VMwareVMware') + # check added by Chetan Loke. + echo vmware + ;; + ('XenVMMXenVMM') + if has dmi + then + # https://access.redhat.com/solutions/222903 + echo xen-hvm + else + echo xen-paravirt + fi + ;; + (*) + return 1 ;; + esac + return 0 + fi + + unset CPUID_HELPER + + # VMM CPUID flag denotes that this system is running under a VMM + if is_oneof "${uname_s}" Darwin + then + get_sysctl machdep.cpu.features | tr ' ' '\n' | grep -qixF VMM \ + && return 0 + fi + if has_cpuinfo \ + && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo + then + return 0 + fi + ;; + (ia64) + if test -d /sys/bus/xen -a ! -d /sys/bus/xen-backend + then + # PV-on-HVM drivers installed in a Xen guest + echo xen-hvm + return 0 + fi + ;; + esac + return 1 +} + +has_vm_dmi() { + # Check for various products in SMBIOS/DMI. + # Note that DMI doesn't exist on all architectures (only x86 and some ARM). + # On other architectures the $dmi variable will be empty. + + if test -d /sys/class/dmi/id/ + then + dmi_sysfs=/sys/class/dmi/id + elif test -d /sys/devices/virtual/dmi/id/ + then + dmi_sysfs=/sys/devices/virtual/dmi/id + fi + + # shellcheck disable=SC2015 + { + is_command dmidecode \ + && ( + # dmidecode needs to exit 0 and not print the No SMBIOS/DMI line + dmi_out=$(dmidecode 2>&1) \ + && ! printf '%s\n' "${dmi_out}" \ + | grep -qF 'No SMBIOS nor DMI entry point found, sorry.' + ) \ + || test -d "${dmi_sysfs}" + } +} + +check_vm_dmi() { + case $(get_dmi_field system-product-name) + in + (*.metal) + if test "$(get_dmi_field system-manufacturer)" = 'Amazon EC2' + then + # AWS EC2 bare metal -> no virtualisation + return 1 + fi + ;; + ('BHYVE') + echo bhyve + return 0 + ;; + ('Google Compute Engine') + echo gce + return 0 + ;; + ('RHEV Hypervisor') + # Red Hat Enterprise Virtualization + echo rhev + return 0 + ;; + ('KVM'|'Bochs'|'KVM Virtual Machine') + echo kvm + return 0 + ;; + ('Parallels Virtual Platform') + echo parallels + return 0 + ;; + ('VirtualBox') + echo virtualbox + return 0 + ;; + ('VMware Virtual Platform') + echo vmware + return 0 + ;; + esac + + case $(get_dmi_field system-manufacturer) + in + ('Alibaba'*) + case $(get_dmi_field system-product-name) + in + ('Alibaba Cloud ECS') + echo alibaba-ecs + ;; + (*) + echo alibaba + ;; + esac + return 0 + ;; + ('Amazon EC2') + # AWS on bare-metal or KVM + echo aws-ec2 + return 0 + ;; + ('innotek GmbH'|'Oracle Corporation') + echo virtualbox + return 0 + ;; + ('Joyent') + if test "$(get_dmi_field system-product-name)" = 'SmartDC HVM' + then + # SmartOS KVM + echo kvm-smartdc_hvm + return 0 + fi + ;; + ('Microsoft Corporation'*) + if test "$(get_dmi_field system-product-name)" = 'Virtual Machine' + then + if test -e /proc/irq/7/hyperv \ + || expr "$(get_dmi_field bios-version)" : 'VRTUAL.*' >/dev/null + then + echo hyperv + return 0 + fi + + case $(get_dmi_field system-version) + in + (VPC[0-9]*|VS2005*|*[Vv]irtual*[Pp][Cc]*) + echo virtualpc + return 0 + ;; + (*) + echo hyperv + return 0 + ;; + esac + fi + ;; + ('Nutanix') + # Nutanix AHV. Similar to KVM. + if test "$(get_dmi_field system-product-name)" = 'AHV' + then + echo nutanix_ahv + return 0 + fi + ;; + ('oVirt') + echo ovirt + return 0 + ;; + ('Parallels Software International Inc.') + echo parallels + return 0 + ;; + ('QEMU') + echo qemu + return 0 + ;; + ('VMware, Inc.') + echo vmware + return 0 + ;; + esac + + case $(get_dmi_field bios-vendor) + in + ('Amazon EC2') + # AWS on bare-metal or KVM + echo aws-ec2 + return 0 + ;; + ('BHYVE') + echo bhyve + return 0 + ;; + ('innotek GmbH') + echo virtualbox + return 0 + ;; + ('Parallels Software International Inc.') + echo parallels + return 0 + ;; + ('Xen') + if get_dmi_field bios-version | grep -q -e '\([0-9]\{1,\}\.\)\{2\}amazon' + then + # AWS on Xen + echo aws-xen + return 0 + fi + ;; + esac + + return 1 +} + +check_vm_hyp_specific() { + if is_command vmware-checkvm && vmware-checkvm >/dev/null + then + # vmware-checkvm is provided by VMware's open-vm-tools + echo vmware + return 0 + elif test -d /proc/xen + then + test -r /proc/xen/capabilities && + if grep -q -F 'control_d' /proc/xen/capabilities 2>/dev/null + then + # Xen dom0 + return 1 + else + # Xen domU + echo xen + return 0 + fi + fi + return 1 +} + +has_vm_dt() { + # OpenFirmware/Das U-Boot device-tree + test -d /proc/device-tree +} + +check_vm_dt() { + case ${arch} + in + (arm|aarch64) + if test -r /proc/device-tree/hypervisor/compatible + then + if grep -q -F 'xen' /proc/device-tree/hypervisor/compatible + then + echo xen + return 0 + elif grep -q -F 'vmware' /proc/device-tree/hypervisor/compatible + then + # e.g. VMware ESXi on ARM + echo vmware + return 0 + fi + fi + if glob_exists /proc/device-tree/fw-cfg@*/compatible + then + # qemu,fw-cfg-mmio + sed -e 's/,.*$//' /proc/device-tree/fw-cfg@*/compatible | head -n1 + return 0 + fi + if grep -q -F 'dummy-virt' /proc/device-tree/compatible + then + echo lkvm + return 0 + fi + ;; + (ppc64*) + if test -d /proc/device-tree/hypervisor \ + && grep -qF 'linux,kvm' /proc/device-tree/hypervisor/compatible + then + # We are running as a spapr KVM guest on ppc64 + echo kvm-spapr + return 0 + fi + if test -r /proc/device-tree/ibm,partition-name \ + && test -r /proc/device-tree/hmc-managed\? \ + && test -r /proc/device-tree/chosen/qemu,graphic-width + then + echo powervm + fi + ;; + esac + return 1 +} + +has_vm_sys_hypervisor() { + test -d /sys/hypervisor/ +} + +check_vm_sys_hypervisor() { + test -r /sys/hypervisor/type && + case $(head -n1 /sys/hypervisor/type) + in + (xen) + # Ordinary kernel with pv_ops. There does not seem to be + # enough information at present to tell whether this is dom0 + # or domU. + echo xen + return 0 + ;; + esac + return 1 +} + +check_vm_os_specific() { + _hyp_generic=false + + case ${uname_s} + in + (Darwin) + if hv_vmm_present=$(get_sysctl kern.hv_vmm_present) \ + && test "${hv_vmm_present}" -ne 0 + then + _hyp_generic=true + fi + ;; + (FreeBSD) + # FreeBSD does not have /proc/cpuinfo even when procfs is used. + # Instead there is a sysctl kern.vm_guest. + # Which is 'none' if physical, else the virtualisation. + vm_guest=$(get_sysctl kern.vm_guest | tolower) && + case ${vm_guest} + in + (none) ;; + (generic) _hyp_generic=true ;; + (*) + # kernel could detect hypervisor + case ${vm_guest} + in + (hv) echo hyperv ;; + (vbox) echo virtualbox ;; + (*) echo "${vm_guest}" ;; + esac + return 0 + ;; + esac + ;; + (NetBSD) + machdep_hv=$(get_sysctl machdep.hypervisor | tolower) && + case ${machdep_hv} + in + (none) ;; + (generic) _hyp_generic=true ;; + (*) + # kernel could detect hypervisor + case ${machdep_hv} + in + (hyper-v) echo hyperv ;; + (xenhvm*) echo xen-hvm ;; + (xenpv*) echo xen-pv ;; + (xen*) echo xen ;; + (*) echo "${machdep_hv}" ;; + esac + return 0 + ;; + esac + ;; + (OpenBSD) + if is_command hostctl && glob_exists /dev/pvbus[0-9]* + then + for _pvbus in /dev/pvbus[0-9]* + do + _h_out=$(hostctl -f "${_pvbus}" -t 2>/dev/null) || continue + case $(expr "${_h_out}" : '[^:]*: *\(.*\)$') + in + (KVM) echo kvm ;; + (Hyper-V) echo hyperv ;; + (VMware) echo vmware ;; + (Xen) echo xen ;; + (bhyve) echo bhyve ;; + (OpenBSD) echo openbsd_vmm ;; + esac + return 0 + done + fi + ;; + (SunOS) + diag_conf=$(prtdiag | sed -n -e 's/.*Configuration: *//p' -e '/^$/q') + # NOTE: Don't use -e or -F in Solaris grep + if printf '%s\n' "${diag_conf}" | grep -q -i QEMU + then + echo qemu + return 0 + elif printf '%s\n' "${diag_conf}" | grep -q -i VMware + then + echo vmware + return 0 + fi + ;; + (Linux) + if is_command dmesg + then + while read -r line + do + case ${line} + in + ('Booting paravirtualized kernel on ') + case $(expr "${line}" : '.* kernel on \(.*\)') + in + ('Xen') + echo xen-pv; return 0 ;; + ('bare hardware') + return 1 ;; + esac + ;; + ('Hypervisor detected') + case $(expr "${line}" : '.*: *\(.*\)') + in + ('ACRN') + echo acrn ;; + ('Jailhouse') + echo jailhouse ;; + ('KVM') + echo kvm ;; + ('Microsoft Hyper-V') + echo hyperv ;; + ('VMware') + echo vmware ;; + ('Xen HVM') + echo xen-hvm ;; + ('Xen PV') + echo xen-pv ;; + esac + return 0 + ;; + (lpar:*' under hypervisor') + return 0 ;; + esac + done <<-EOF + $(dmesg 2>/dev/null | awk ' + /Booting paravirtualized kernel on / + /Hypervisor detected: / + /lpar: .* under hypervisor/ + ') + EOF + fi + esac + + # Try to guess hypervisor based on CPU model (sysctl hw.model if available) + if cpu_model=$(get_sysctl hw.model) + then + guess_hypervisor_from_cpu_model "${cpu_model}" && return 0 + fi + + if ${_hyp_generic} + then + # cannot say which hypervisor, but one was detected + return 0 + else + return 1 + fi +} + +run_stage() { + if type "has_$1_$2" >/dev/null 2>&1 + then + "has_$1_$2" + else + true + fi \ + && "check_$1_$2" +} + + +# Execute container stages + +for stage in \ + pid_1 cgroup files os_specific +do + ctengine=$(run_stage ct ${stage}) || continue + is_contained=true + if test -n "${ctengine}" + then + echo container on "${ctengine}" + break + fi +done +if ${is_contained:-false} && test -z "${ctengine}" +then + # none of the stages could determine the specific container engine, but + # we are running in some container. + echo container +fi + + +# Execute virtual machine / hypervisor stages + +for stage in \ + os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific +do + hypervisor=$(run_stage vm ${stage}) || continue + is_virtual=true + if test -n "${hypervisor}" + then + echo virtual by "${hypervisor}" + break + fi +done +if ${is_virtual:-false} && test -z "${hypervisor}" +then + # none of the stages could determine the specific hypervisor, but + # we are virtual. + echo virtual +fi From abc6d009b21b0d1ce3fc5107201e30740f127200 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 19:29:41 +0200 Subject: [PATCH 149/206] [explorer/machine_type] Print top most machine layer as first line (fallback to physical) --- cdist/conf/explorer/machine_type | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 29f98849..90c441da 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -26,17 +26,19 @@ # VERB does not have a special meaning, it is just for better readability. # # e.g. +# container # container on lxc # virtual by kvm-spapr # -# The third word of each line can be composed of different parts concatenated with a `-' -# (minus) character, with each component being a specification of the previous, -# e.g.: +# The third word of each line (except the first) can be composed of different +# parts concatenated with a `-' (minus) character, with each component being +# a specification of the previous, e.g.: # - lxc-libvirt (LXC container, managed by libvirt) # - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) # - xen-hvm / xen-pv (Xen HVM vs para-virtualization) # -# If this explorer cannot determine any information it will print nothing. +# If this explorer cannot collect enough information about virtualization it +# will fall back to 'physical'. # # Add /sbin and /usr/sbin to the path so we can find system @@ -121,6 +123,10 @@ get_sysctl() { is_command sysctl && sysctl -n "$1" 2>/dev/null } +detected_layer() { + test -n "${_toplayer:-}" || echo "${_toplayer:=${1:?}}" +} + # Check for container @@ -895,6 +901,7 @@ for stage in \ pid_1 cgroup files os_specific do ctengine=$(run_stage ct ${stage}) || continue + detected_layer 'container' is_contained=true if test -n "${ctengine}" then @@ -916,6 +923,7 @@ for stage in \ os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific do hypervisor=$(run_stage vm ${stage}) || continue + detected_layer 'virtual machine' is_virtual=true if test -n "${hypervisor}" then @@ -929,3 +937,8 @@ then # we are virtual. echo virtual fi + + +# Fallback + +detected_layer physical From 2ffa895f578f12d615ed65b9a93f3e2ae07f1d07 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 19:57:24 +0200 Subject: [PATCH 150/206] [explorer/machine_type] Remove CPUID check it's a lot of code and depends on a binary helper unlikely to be installed. --- cdist/conf/explorer/machine_type | 59 -------------------------------- 1 file changed, 59 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 90c441da..7ce035e3 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -379,65 +379,6 @@ check_vm_arch_specific() { fi ;; (i?86|x86*|amd64|i86pc) - # Check CPUID - # - # Many fullvirt hypervisors give an indication through CPUID. Use - # the virt-what helper program to get this information if available. - - for CPUID_HELPER in \ - $(command -v virt-what-cpuid-helper 2>/dev/null) \ - /usr/lib/x86_64-*/virt-what-cpuid-helper \ - /usr/lib/i?86-*/virt-what-cpuid-helper \ - /usr/lib/virt-what/virt-what-cpuid-helper - do - if test -x "${CPUID_HELPER:?}"; then break; fi - done - - if test -x "${CPUID_HELPER-}" - then - case $(command "${CPUID_HELPER}") - in - ('bhyve bhyve ') - echo bhyve - ;; - ('LKVMLKVMLKVM') - echo lkvm - ;; - ('KVMKVMKVM') - echo kvm - ;; - ('TCGTCGTCGTCG') - echo qemu-tcg - ;; - ('Microsoft Hv') - # http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx - echo hyperv - ;; - ('OpenBSDVMM58') - # OpenBSD/VMM - echo openbsd_vmm - ;; - ('VMwareVMware') - # check added by Chetan Loke. - echo vmware - ;; - ('XenVMMXenVMM') - if has dmi - then - # https://access.redhat.com/solutions/222903 - echo xen-hvm - else - echo xen-paravirt - fi - ;; - (*) - return 1 ;; - esac - return 0 - fi - - unset CPUID_HELPER - # VMM CPUID flag denotes that this system is running under a VMM if is_oneof "${uname_s}" Darwin then From 23fbfaf0352eadaabd43504d3ee074bb9f696fcd Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 21:29:24 +0200 Subject: [PATCH 151/206] [explorer/machine_type] Use systemd-detect-virt (if available) to detect containers and VMs --- cdist/conf/explorer/machine_type | 49 ++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 7ce035e3..1a38fda0 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -130,6 +130,25 @@ detected_layer() { # Check for container +has_ct_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -c' +} + +check_ct_systemd() ( + _ctengine=$(systemd-detect-virt -c 2>/dev/null) && + case ${_ctengine} + in + (''|'none') + return 1 ;; + ('container-other') + return 0 ;; + ('systemd-nspawn') + echo systemd_nspawn ;; + (*) + echo "${_ctengine}" ;; + esac +) + has_ct_pid_1() { test -r /run/systemd/container -o -r /proc/1/environ } @@ -267,6 +286,32 @@ guess_hypervisor_from_cpu_model() { esac } +has_vm_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -v' +} + +check_vm_systemd() ( + _hypervisor=$(systemd-detect-virt -v 2>/dev/null) && + case ${_hypervisor} + in + (''|'none') + return 1 ;; + ('amazon') + echo aws ;; + ('bochs') + echo kvm ;; + ('microsoft') + # assumption + echo hyperv ;; + ('oracle') + echo virtualbox ;; + ('vm-other') + return 0 ;; + (*) + echo "${_hypervisor}" ;; + esac +) + has_vm_cpuinfo() { has_cpuinfo; } check_vm_cpuinfo() { @@ -839,7 +884,7 @@ run_stage() { # Execute container stages for stage in \ - pid_1 cgroup files os_specific + systemd pid_1 cgroup files os_specific do ctengine=$(run_stage ct ${stage}) || continue detected_layer 'container' @@ -861,7 +906,7 @@ fi # Execute virtual machine / hypervisor stages for stage in \ - os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific + systemd os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific do hypervisor=$(run_stage vm ${stage}) || continue detected_layer 'virtual machine' From 4a05669765c2c00c19af0ef1b607b0f2efa10f42 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 22:01:28 +0200 Subject: [PATCH 152/206] [explorer/machine_type] Implement chroot detection --- cdist/conf/explorer/machine_type | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 1a38fda0..10c914ba 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -53,6 +53,21 @@ uname_s=$(uname -s) is_command() { command -v "$1" >/dev/null 2>&1; } +files_same() { + # shellcheck disable=SC2012 + LC_ALL=C df -P "$1" "$2" 2>/dev/null | { + read -r _ # skip header line + read -r fs1 _ _ _ _ mp1 + read -r fs2 _ _ _ _ mp2 + test "${fs1}" = "${fs2}" -a "${mp1}" = "${mp2}" || return 1 + } && + ls -1Ldi "$1" "$2" 2>/dev/null | { + read -r ino1 _ + read -r ino2 _ + test "${ino1}" = "${ino2}" || return 1 + } +} + is_oneof() ( x=$1; shift for y @@ -128,6 +143,32 @@ detected_layer() { } +# Check for chroot + +has_chroot_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -r' +} + +check_chroot_systemd() { + systemd-detect-virt -r +} + +has_chroot_debian_ischroot() { + is_command ischroot +} + +check_chroot_debian_ischroot() { + ischroot --default-false +} + +has_chroot_procfs() { + test -d /proc/ +} + +check_chroot_procfs() { + test -e /proc/1/root && ! files_same /proc/1/root / +} + # Check for container has_ct_systemd() { @@ -881,6 +922,18 @@ run_stage() { } +# Execute chroot stages + +for stage in \ + procfs debian_ischroot systemd +do + run_stage chroot ${stage} || continue + detected_layer 'chroot' + echo chroot + break +done + + # Execute container stages for stage in \ From 5af1317c2969995871da1d17b951de60dbc3fd09 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 1 Aug 2021 16:40:20 +0200 Subject: [PATCH 153/206] [explorer/machine_type] Try to detect chroot path --- cdist/conf/explorer/machine_type | 38 ++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 10c914ba..fa68ec4d 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -166,7 +166,28 @@ has_chroot_procfs() { } check_chroot_procfs() { - test -e /proc/1/root && ! files_same /proc/1/root / + if test -e /proc/1/root && ! files_same /proc/1/root / + then + # try to determine where the chroot has been mounted + ( + rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') + + if test -e "${rootdev}" + then + # escape chroot to determine where the device containing the + # chroot's / is mounted + rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') + + # shellcheck disable=SC2012 + root_ino=$(ls -1di / | awk '{print $1}') + + # Get mount point + chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" + fi + ) + return 0 + fi + return 1 } # Check for container @@ -927,11 +948,20 @@ run_stage() { for stage in \ procfs debian_ischroot systemd do - run_stage chroot ${stage} || continue + chrootpnt=$(run_stage chroot ${stage}) || continue + is_chrooted=true detected_layer 'chroot' - echo chroot - break + if test -n "${chrootpnt}" + then + echo chroot at "${chrootpnt}" + break + fi done +if ${is_chrooted:-false} && test -z "${chrootpnt}" +then + # could determine chroot, but not its mount point + echo chroot +fi # Execute container stages From 05c2a62191f533ceff1e074df571278a103d75ab Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 1 Aug 2021 23:09:02 +0200 Subject: [PATCH 154/206] [explorer/machine_type] Implement chroot detection using /proc/.../mountinfo --- cdist/conf/explorer/machine_type | 64 +++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index fa68ec4d..00646c75 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -165,30 +165,52 @@ has_chroot_procfs() { test -d /proc/ } -check_chroot_procfs() { +check_chroot_procfs() ( + is_chroot=false # default if test -e /proc/1/root && ! files_same /proc/1/root / then - # try to determine where the chroot has been mounted - ( - rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') - - if test -e "${rootdev}" - then - # escape chroot to determine where the device containing the - # chroot's / is mounted - rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') - - # shellcheck disable=SC2012 - root_ino=$(ls -1di / | awk '{print $1}') - - # Get mount point - chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" - fi - ) - return 0 + is_chroot=true fi - return 1 -} + if test -e /proc/1/mountinfo -a -e /proc/self/mountinfo + then + has_mountinfo=true + cmp -s /proc/1/mountinfo /proc/self/mountinfo || is_chroot=true + fi + + if ${is_chroot} + then + # try to determine where the chroot has been mounted + rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') + + if test -e "${rootdev}" + then + # escape chroot to determine where the device containing the + # chroot's / is mounted + rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') + + # shellcheck disable=SC2012 + root_ino=$(ls -1di / | awk '{print $1}') + + # escape chroot and find mount point by inode + chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" + elif ${has_mountinfo} + then + while read -r mntid _ _ _ cmntpnt _ + do + read -r _ _ _ _ hmntpnt _ <<-EOF + $(grep -e "^$((mntid)) " /proc/1/mountinfo) + EOF + printf '%s\n' "${hmntpnt%${cmntpnt}}" + done Date: Mon, 23 Aug 2021 09:57:20 +0300 Subject: [PATCH 155/206] [explorer/os_version] add new debian code names: bookworm and trixie --- cdist/conf/explorer/os_version | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 7bc6dd6b..aea3c43f 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -54,6 +54,8 @@ in # sid versions don't have a number, so we decode by codename: case $(expr "$debian_version" : '\([a-z]\{1,\}\)/') in + trixie) echo 12.99 ;; + bookworm) echo 11.99 ;; bullseye) echo 10.99 ;; buster) echo 9.99 ;; stretch) echo 8.99 ;; From e1e134899811e51e4694e89e421790e41cb894bf Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Mon, 23 Aug 2021 10:44:48 +0300 Subject: [PATCH 156/206] [explorer/os_version] use 99.99 as fallback for unknown code names in */sid --- cdist/conf/explorer/os_version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index aea3c43f..bbc9e4f0 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -63,7 +63,7 @@ in wheezy) echo 6.99 ;; squeeze) echo 5.99 ;; lenny) echo 4.99 ;; - *) exit 1 + *) echo 99.99 ;; esac ;; *) From 46ed48d546af6ff663b131c803a9027b19798ba1 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 08:09:47 +0200 Subject: [PATCH 157/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index f9409d7e..8507c663 100644 --- a/docs/changelog +++ b/docs/changelog @@ -12,6 +12,7 @@ next: * Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera) * Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera) * Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera) + * Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 0546283d0ebc0058dbff68cec9d18b9281c26edd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:32:44 +0200 Subject: [PATCH 158/206] Update shellcheck disable --- cdist/conf/type/__apt_backports/manifest | 1 + cdist/conf/type/__package_pkg_freebsd/gencode-remote | 1 + cdist/conf/type/__ssh_authorized_keys/explorer/keys | 1 + 3 files changed, 3 insertions(+) diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index bc47d8de..6fcd9212 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -28,6 +28,7 @@ # lsb_release may not be given in all installations codename_os_release() { # shellcheck disable=SC1090 + # shellcheck disable=SC1091 . "$__global/explorer/os_release" printf "%s" "$VERSION_CODENAME" } diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote index 3f88f6bc..ca9aa45a 100755 --- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote @@ -37,6 +37,7 @@ assert () # If condition false, then echo "Assertion failed: \"$1\"" # shellcheck disable=SC2039 + # shellcheck disable=SC3044 echo "File \"$0\", line $lineno, called by $(caller 0)" exit $E_ASSERT_FAILED fi diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys index cec25746..9694a64b 100755 --- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys +++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys @@ -1,6 +1,7 @@ #!/bin/sh -e # shellcheck disable=SC1090 +# shellcheck disable=SC1091 file="$( . "$__type_explorer/file" )" if [ -f "$file" ] From 44741e714b16f7a00bf84bd54ae33eacc7593192 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:25:49 +0200 Subject: [PATCH 159/206] Release 6.9.8 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 8507c663..dcdc4b3d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) From b8eb6e984c1638e8e167394c6b3aa482cb8aad49 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:47:50 +0200 Subject: [PATCH 160/206] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index dcdc4b3d..693d028f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Explorer machine_type: Rewrite (Dennis Camera) + 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) From d7fdc8006f747be86b64a2790266b50219bab507 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 21:54:45 +0300 Subject: [PATCH 161/206] allow empty file --- cdist/conf/type/__sed/explorer/file | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file index a40f71b3..5f0f4edd 100755 --- a/cdist/conf/type/__sed/explorer/file +++ b/cdist/conf/type/__sed/explorer/file @@ -9,13 +9,7 @@ fi if [ -f "$file" ] then - if [ -s "$file" ] - then - cat "$file" - else - echo "$file is empty" >&2 - exit 1 - fi + cat "$file" else echo "$file do not exist" >&2 exit 1 From 0f6e48dbc657a9d9145b965eacf79f021a1419f5 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:24:26 +0300 Subject: [PATCH 162/206] use $__object/tempfile in target instead of mktemp, add comments --- cdist/conf/type/__sed/gencode-remote | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 836506f0..df929b42 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -35,8 +35,11 @@ then exit 0 fi +# we can't use -i, because it's not posix, so we fly with tempfile and cp +# and we use cp because we want to preserve destination file's attributes + # shellcheck disable=SC2016 -echo 'tmp="$( mktemp )"' +echo 'tmp="$__object/tempfile"' echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" From 90488fcebcc00f3b4ea260867f8b31625df61e46 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:27:42 +0300 Subject: [PATCH 163/206] use -e --- cdist/conf/type/__sed/explorer/file | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file index 5f0f4edd..9262b613 100755 --- a/cdist/conf/type/__sed/explorer/file +++ b/cdist/conf/type/__sed/explorer/file @@ -7,10 +7,10 @@ else file="/$__object_id" fi -if [ -f "$file" ] +if [ ! -e "$file" ] then - cat "$file" -else echo "$file do not exist" >&2 exit 1 fi + +cat "$file" From b7f392fa371ff78591abb910f7e224e7587c81be Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:38:55 +0300 Subject: [PATCH 164/206] use -E for better compat (not really sure if it is posix at all) --- cdist/conf/type/__sed/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index df929b42..625448e6 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -24,7 +24,7 @@ sed_cmd='sed' if [ -f "$__object/parameter/regexp-extended" ] then - sed_cmd="$sed_cmd --regexp-extended" + sed_cmd="$sed_cmd -E" fi if echo "$script" \ From aabef7f44a98ad144a7e1fe68bfb562079916ef8 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:40:06 +0300 Subject: [PATCH 165/206] remove reading script from file --- cdist/conf/type/__sed/gencode-remote | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 625448e6..be7690e1 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -12,10 +12,6 @@ script="$( cat "$__object/parameter/script" )" if [ "$script" = '-' ] then script="$( cat "$__object/stdin" )" -elif - [ -f "$script" ] -then - script="$( cat "$script" )" fi file_from_target="$__object/explorer/file" From 5bf0c71e7af49ff6e7c1cccec192f763a019503e Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:45:36 +0300 Subject: [PATCH 166/206] update man --- cdist/conf/type/__sed/man.rst | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst index 4a728184..86789363 100644 --- a/cdist/conf/type/__sed/man.rst +++ b/cdist/conf/type/__sed/man.rst @@ -8,35 +8,40 @@ cdist-type__sed - Transform text files with ``sed`` DESCRIPTION ----------- -TODO +Transform text files with ``sed``. REQUIRED MULTIPLE PARAMETERS ---------------------------- script - TODO + ``sed`` script. + If ``-`` then the script is read from ``stdin``. OPTIONAL PARAMETERS ------------------- file - TODO + Path to the file. Defaults to ``$__object_id``. onchange - TODO + Execute this command if ``sed`` changes file. BOOLEAN PARAMETERS ------------------ regexp-extended - TODO + Use extended regular expressions in the script. + Might not be supported with every ``sed`` version. EXAMPLES -------- + .. code-block:: sh - true + __sed /tmp/foobar --script 's/foo/bar/' + + echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script - AUTHORS From cd4acde67ec9c3967e6e08450734cf65040e787f Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 15 Sep 2021 09:22:27 +0300 Subject: [PATCH 167/206] grammar --- cdist/conf/type/__sed/explorer/file | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file index 9262b613..ec3d0fe8 100755 --- a/cdist/conf/type/__sed/explorer/file +++ b/cdist/conf/type/__sed/explorer/file @@ -9,7 +9,7 @@ fi if [ ! -e "$file" ] then - echo "$file do not exist" >&2 + echo "$file does not exist" >&2 exit 1 fi From 7b6789ddeb74450f70991efe957e3eb0f11043f6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 15:04:12 +0200 Subject: [PATCH 168/206] __package_update_index: fix complain about suite change 2 of 4th fix for ticket #861 --- .../type/__package_update_index/gencode-remote | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote index 803468b5..a10c16d3 100755 --- a/cdist/conf/type/__package_update_index/gencode-remote +++ b/cdist/conf/type/__package_update_index/gencode-remote @@ -41,7 +41,19 @@ fi case "$type" in yum) ;; apt) - echo "apt-get --quiet update" + # There are special arguments to apt(8) to prevent aborts if apt woudn't been + # updated after the 19th April 2021 till the bullseye release. The additional + # arguments acknoledge the happend suite change (the apt(8) update does the + # same by itself). + # + # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter + # allows backward compatablility to pre-buster Debian versions. + # + # See more: ticket #861 + # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 + apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + + echo "apt-get --quiet $apt_opts update" echo "apt-cache updated (age was: $currage)" >> "$__messages_out" ;; pacman) From 12787ffe2c48f5c6d534d10f8faadde30db477ac Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 15:12:20 +0200 Subject: [PATCH 169/206] __apt_source: fix complain about suite change 3 of 4th fix for ticket #861 --- cdist/conf/type/__apt_source/gencode-remote | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_source/gencode-remote b/cdist/conf/type/__apt_source/gencode-remote index 1e8592c6..973b0f6c 100755 --- a/cdist/conf/type/__apt_source/gencode-remote +++ b/cdist/conf/type/__apt_source/gencode-remote @@ -22,7 +22,21 @@ name="$__object_id" destination="/etc/apt/sources.list.d/${name}.list" +# There are special arguments to apt(8) to prevent aborts if apt woudn't been +# updated after the 19th April 2021 till the bullseye release. The additional +# arguments acknoledge the happend suite change (the apt(8) update does the +# same by itself). +# +# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter +# allows backward compatablility to pre-buster Debian versions. +# +# See more: ticket #861 +# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 +apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + +# run 'apt-get update' only if something changed with our sources.list file +# it will be run a second time on error as a redundancy messure to success if grep -q "^__file${destination}" "$__messages_in"; then - printf 'apt-get update || apt-get update\n' + printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts" fi From d246e067109dc8b85ea89668f742affbb221d1a6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 14:49:23 +0200 Subject: [PATCH 170/206] __apt_update_index: fix complain about suite change 1 of 4th fix for ticket #861 --- .../conf/type/__apt_update_index/gencode-remote | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_update_index/gencode-remote b/cdist/conf/type/__apt_update_index/gencode-remote index 70b59710..2d7f9030 100755 --- a/cdist/conf/type/__apt_update_index/gencode-remote +++ b/cdist/conf/type/__apt_update_index/gencode-remote @@ -18,9 +18,23 @@ # along with cdist. If not, see . # + +# There are special arguments to apt(8) to prevent aborts if apt woudn't been +# updated after the 19th April 2021 till the bullseye release. The additional +# arguments acknoledge the happend suite change (the apt(8) update does the +# same by itself). +# +# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter +# allows backward compatablility to pre-buster Debian versions. +# +# See more: ticket #861 +# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 +apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists +# it will be run a second time on error as a redundancy messure to success cat << DONE if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then - apt-get update || apt-get update + apt-get $apt_opts update || apt-get $apt_opts update fi DONE From 3d7b31cbb43067271c2c510281f90f19dc5ec5fb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 15:22:16 +0200 Subject: [PATCH 171/206] __package_apt: fix complain about suite change the last fix for ticket #861 :-) --- cdist/conf/type/__package_apt/gencode-remote | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index fbfca330..79c0d9d3 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -81,12 +81,24 @@ aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::= case "$state_should" in present) + # There are special arguments to apt(8) to prevent aborts if apt woudn't been + # updated after the 19th April 2021 till the bullseye release. The additional + # arguments acknoledge the happend suite change (the apt(8) update does the + # same by itself). + # + # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter + # allows backward compatablility to pre-buster Debian versions. + # + # See more: ticket #861 + # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 + apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + # following is bit ugly, but important hack. # due to how cdist config run works, there isn't # currently better way to do it :( cat << EOF if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ] -then echo apt-get update > /dev/null 2>&1 || true +then echo apt-get $apt_opts update > /dev/null 2>&1 || true fi EOF if [ -n "$version" ]; then From 72ff48154c958ebdedc8b6d919b6f372efa63be2 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 16 Sep 2021 21:36:39 +0300 Subject: [PATCH 172/206] add comments, add -u to diff --- cdist/conf/type/__sed/gencode-remote | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index be7690e1..f99c5a88 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -14,6 +14,8 @@ then script="$( cat "$__object/stdin" )" fi +# since stdin is not available in explorer, we pull file from target with explorer + file_from_target="$__object/explorer/file" sed_cmd='sed' @@ -23,10 +25,10 @@ then sed_cmd="$sed_cmd -E" fi -if echo "$script" \ - | "$sed_cmd" -f - "$file_from_target" \ - | diff "$file_from_target" - \ - > /dev/null +# do sed dry run, diff result and if no change, then there's nothing to do +# also redirect diff's output to stderr for debugging purposes + +if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2 then exit 0 fi From bf222d05435000063930ff199bd0dec94d7d8cd5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 21 Sep 2021 08:55:54 +0200 Subject: [PATCH 173/206] ++changelog --- docs/changelog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog b/docs/changelog index 693d028f..ab637f5f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,11 @@ Changelog next: * Explorer machine_type: Rewrite (Dennis Camera) + * New type: __sed (Ander Punnar) + * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) + * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) + * Type __apt_source: Fix complaint about suite change (Matthias Stecher) + * Type __package_apt: Fix complaint about suite change (Matthias Stecher) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 15c642a9b7af42c368eb5a8e22cab5f07b39b95e Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 12:06:45 +0200 Subject: [PATCH 174/206] [__debconf_set_selections] Fix --file not being supported Even if deprecated, the parameter *must* be supported, which isn't the case right now. This was due to a misunderstanding of how deprecating parameters work, see: https://www.cdi.st/manual/latest/cdist-type.html#deprecated-parameters --- .../conf/type/__debconf_set_selections/parameter/deprecated/file | 1 + .../__debconf_set_selections/parameter/{deprecated => optional} | 0 2 files changed, 1 insertion(+) create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/deprecated/file rename cdist/conf/type/__debconf_set_selections/parameter/{deprecated => optional} (100%) diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file new file mode 100644 index 00000000..09db545a --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file @@ -0,0 +1 @@ +'file' has been deprecated in favour of 'line' in order to provide idempotency. diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated b/cdist/conf/type/__debconf_set_selections/parameter/optional similarity index 100% rename from cdist/conf/type/__debconf_set_selections/parameter/deprecated rename to cdist/conf/type/__debconf_set_selections/parameter/optional From 5b7cca99f73e4ec3dd5cdb90e4dbfd40ec8fe349 Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 12:09:42 +0200 Subject: [PATCH 175/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index ab637f5f..142602c8 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) * Type __apt_source: Fix complaint about suite change (Matthias Stecher) * Type __package_apt: Fix complaint about suite change (Matthias Stecher) + * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From fc9bd40c9a34cf54e2d6cfd2fec089bd25e1b172 Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 13:14:57 +0200 Subject: [PATCH 176/206] Improve bullseye support, perticularly __letsencrypt_cert --- cdist/conf/type/__grafana_dashboard/manifest | 2 +- cdist/conf/type/__letsencrypt_cert/manifest | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index d145c4c3..0d944482 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -15,7 +15,7 @@ case $os in # Differntation not needed anymore apt_source_distribution=stable ;; - 10*) + 10*|11*) # Differntation not needed anymore apt_source_distribution=stable ;; diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 6394f629..638a99e0 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -41,7 +41,7 @@ if [ -z "${certbot_fullpath}" ]; then require="__apt_source/stretch-backports" __package_apt certbot \ --target-release stretch-backports ;; - 10*) + 10*|11*) __package_apt certbot ;; From 560374a6861281695b61c8c2298123be41d92326 Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 13:16:11 +0200 Subject: [PATCH 177/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 142602c8..6f717cf4 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ next: * Type __apt_source: Fix complaint about suite change (Matthias Stecher) * Type __package_apt: Fix complaint about suite change (Matthias Stecher) * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) + * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From c33d99ee120ad180daba144be80a25d77b473f56 Mon Sep 17 00:00:00 2001 From: Evilham Date: Sun, 31 Oct 2021 17:38:10 +0100 Subject: [PATCH 178/206] [__haproxy_dualstack] New type with PROXY protocol support This is backwards compatible with what is already used internally @ungleich, but adds on top of that the ability to customise ports and, most importantly, it adds PROXY protocol support. --- .../conf/type/__haproxy_dualstack/files/http | 8 + .../conf/type/__haproxy_dualstack/files/https | 10 ++ .../conf/type/__haproxy_dualstack/files/imaps | 12 ++ .../conf/type/__haproxy_dualstack/files/smtps | 12 ++ cdist/conf/type/__haproxy_dualstack/man.rst | 121 ++++++++++++++ cdist/conf/type/__haproxy_dualstack/manifest | 155 ++++++++++++++++++ .../parameter/default/protocol | 1 + .../parameter/optional_multiple | 3 + cdist/conf/type/__haproxy_dualstack/singleton | 0 9 files changed, 322 insertions(+) create mode 100644 cdist/conf/type/__haproxy_dualstack/files/http create mode 100644 cdist/conf/type/__haproxy_dualstack/files/https create mode 100644 cdist/conf/type/__haproxy_dualstack/files/imaps create mode 100644 cdist/conf/type/__haproxy_dualstack/files/smtps create mode 100644 cdist/conf/type/__haproxy_dualstack/man.rst create mode 100644 cdist/conf/type/__haproxy_dualstack/manifest create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/default/protocol create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple create mode 100644 cdist/conf/type/__haproxy_dualstack/singleton diff --git a/cdist/conf/type/__haproxy_dualstack/files/http b/cdist/conf/type/__haproxy_dualstack/files/http new file mode 100644 index 00000000..0508a465 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/http @@ -0,0 +1,8 @@ +frontend http + bind BIND@:80 + mode http + option httplog + default_backend http + +backend http + mode http diff --git a/cdist/conf/type/__haproxy_dualstack/files/https b/cdist/conf/type/__haproxy_dualstack/files/https new file mode 100644 index 00000000..73deac46 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/https @@ -0,0 +1,10 @@ +frontend https + bind BIND@:443 + mode tcp + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_ssl_hello_type 1 } + default_backend https + +backend https + mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/files/imaps b/cdist/conf/type/__haproxy_dualstack/files/imaps new file mode 100644 index 00000000..b1ec3793 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/imaps @@ -0,0 +1,12 @@ +frontend imaps + bind BIND@:143 + bind BIND@:993 + + mode tcp + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_ssl_hello_type 1 } + default_backend imaps + +backend imaps + mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/files/smtps b/cdist/conf/type/__haproxy_dualstack/files/smtps new file mode 100644 index 00000000..dce6ed4a --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/smtps @@ -0,0 +1,12 @@ +frontend smtps + bind BIND@:25 + bind BIND@:465 + + mode tcp + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_ssl_hello_type 1 } + default_backend smtps + +backend smtps + mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/man.rst b/cdist/conf/type/__haproxy_dualstack/man.rst new file mode 100644 index 00000000..6c131cbe --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/man.rst @@ -0,0 +1,121 @@ +cdist-type__haproxy_dualstack(7) +================================ + + +NAME +---- +cdist-type__haproxy_dualstack - Proxy services from a dual-stack server + + +DESCRIPTION +----------- +This (singleton) type installs and configures haproxy to act as a dual-stack +proxy for single-stack services. + +This can be useful to add IPv4 support to IPv6-only services while only using +one IPv4 for many such services. + +By default this type uses the plain TCP proxy mode, which means that there is no +need for TLS termination on this host when SNI is supported. +This also means that proxied services will not receive the client's IP address, +but will see the proxy's IP address instead (that of `$__target_host`). + +This can be solved by using the PROXY protocol, but do take into account that, +e.g. nginx cannot serve both regular HTTP(S) and PROXY protocols on the same +port, so you will need to use other ports for that. + +As a recommendation in this type: use TCP ports 8080 and 591 respectively to +serve HTTP and HTTPS using the PROXY protocol. + +See the EXAMPLES for more details. + + +OPTIONAL PARAMETERS +------------------- +v4proxy + Proxy incoming IPv4 connections to the equivalent IPv6 endpoint. + In its simplest use, it must be a NAME with an `AAAA` DNS entry, which is + the IP address actually providing the proxied services. + The full format of this argument is: + `[proxy:]NAME[[:PROTOCOL_1=PORT_1]...[:PROTOCOL_N=PORT_N]]` + Where starting with `proxy:` determines that the PROXY protocol must be + used and each `:PROTOCOL=PORT` (e.g. `:http=8080` or `:https=591`) is a PORT + override for the given PROTOCOL (see `--protocol`), if not present the + PROTOCOL's default port will be used. + + +v6proxy + Proxy incoming IPv6 connections to the equivalent IPv4 endpoint. + In its simplest use, it must be a NAME with an `A` DNS entry, which is + the IP address actually providing the proxied services. + See `--v4proxy` for more options and details. + +protocol + Can be passed multiple times or as a space-separated list of protocols. + Currently supported protocols are: `http`, `https`, `imaps`, `smtps`. + This defaults to: `http https imaps smtps`. + + +EXAMPLES +-------- + +.. code-block:: sh + + # Proxy the IPv6-only services so IPv4-only clients can access them + # This uses HAProxy's TCP mode for http, https, imaps and smtps + __haproxy_dualstack \ + --v4proxy ipv6.chat \ + --v4proxy matrix.ungleich.ch + + # Proxy the IPv6-only HTTP(S) services so IPv4-only clients can access them + # Note this means that the backend IPv6-only server will only see + # the IPv6 address of the haproxy host managed by cdist, which can be + # troublesome if this information is relevant for analytics/security/... + # See the PROXY example below + __haproxy_dualstack \ + --protocol http --protocol https \ + --v4proxy ipv6.chat \ + --v4proxy matrix.ungleich.ch + + # Use the PROXY protocol to proxy the IPv6-only HTTP(S) services enabling + # IPv4-only clients to access them while maintaining the client's IP address + __haproxy_dualstack \ + --protocol http --protocol https \ + --v4proxy proxy:ipv6.chat:http=8080:https=591 \ + --v4proxy proxy:matrix.ungleich.ch:http=8080:https=591 + # Note however that the PROXY protocol is not compatible with regular + # HTTP(S) protocols, so your nginx will have to listen on different ports + # with the PROXY settings. + # Note that you will need to restrict access to the 8080 port to prevent + # Client IP spoofing. + # This can be something like: + # server { + # # listen for regular HTTP connections + # listen [::]:80 default_server; + # listen 80 default_server; + # # listen for PROXY HTTP connections + # listen [::]:8080 proxy_protocol; + # # Accept the Client's IP from the PROXY protocol + # real_ip_header proxy_protocol; + # } + + +SEE ALSO +-------- +- https://www.haproxy.com/blog/enhanced-ssl-load-balancing-with-server-name-indication-sni-tls-extension/ +- https://www.haproxy.com/blog/haproxy/proxy-protocol/ +- https://docs.nginx.com/nginx/admin-guide/load-balancer/using-proxy-protocol/ + + +AUTHORS +------- +ungleich +Evilham + + +COPYING +------- +Copyright \(C) 2021 ungleich glarus ag. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__haproxy_dualstack/manifest b/cdist/conf/type/__haproxy_dualstack/manifest new file mode 100644 index 00000000..d110eea6 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/manifest @@ -0,0 +1,155 @@ +#!/bin/sh -eu + +__package haproxy +require="__package/haproxy" __start_on_boot haproxy + +tmpdir="$__object/files" +mkdir "$tmpdir" +configtmp="$__object/files/haproxy.cfg" + +os=$(cat "$__global/explorer/os") +case $os in + freebsd) + CONFIG_FILE="/usr/local/etc/haproxy.conf" + cat < "$configtmp" +global + maxconn 4000 + user nobody + group nogroup + daemon + +EOF + + ;; + *) + CONFIG_FILE="/etc/haproxy/haproxy.cfg" + cat < "$configtmp" +global + log [::1] local2 + chroot /var/lib/haproxy + pidfile /var/run/haproxy.pid + maxconn 4000 + user haproxy + group haproxy + daemon + + # turn on stats unix socket + stats socket /var/lib/haproxy/stats + +EOF + ;; +esac + +cat <> "$configtmp" +defaults + retries 3 + log global + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 1m + timeout server 1m + timeout http-keep-alive 10s + timeout check 10s +EOF + +dig_cmd="$(command -v dig || true)" +get_ip() { + # Usage: get_ip (ipv4|ipv6) NAME + # uses "dig" if available, else fallback to "host" + case $1 in + ipv4) + if [ -n "${dig_cmd}" ]; then + ${dig_cmd} +short A "$2" + else + host -t A "$2" | cut -d ' ' -f 4 | grep -v 'found:' + fi + ;; + ipv6) + if [ -n "${dig_cmd}" ]; then + ${dig_cmd} +short AAAA "$2" + else + host -t AAAA "$2" | cut -d ' ' -f 5 | grep -v 'NXDOMAIN' + fi + ;; + esac +} + +PROTOCOLS="$(cat "$__object/parameter/protocol")" + +for proxy in v4proxy v6proxy; do + param=$__object/parameter/$proxy + # no backend? skip generating code + if [ ! -f "$param" ]; then + continue + fi + + # turn backend name into bind parameter: v4backend -> ipv4@ + bind=$(echo $proxy | sed -e 's/^/ip/' -e 's/proxy//') + + case $bind in + ipv4) + backendproto=ipv6 + ;; + ipv6) + backendproto=ipv4 + ;; + esac + + for proto in ${PROTOCOLS}; do + # Add protocol "header" + printf "\n# %s %s \n" "${bind}" "${proto}" >> "$configtmp" + + sed -e "s/BIND/$bind/" \ + -e "s/\(frontend[[:space:]].*\)/\1$bind/" \ + -e "s/\(backend[[:space:]].*\)/\\1$bind/" \ + "$__type/files/$proto" >> "$configtmp" + + while read -r hostdefinition; do + if echo "$hostdefinition" | grep -qE '^proxy:'; then + # Proxy protocol was requested + host="$(echo "$hostdefinition" | sed -E 's/^proxy:([^:]+).*$/\1/')" + send_proxy=" send-proxy" + else + # Just use tcp proxy mode + host="$hostdefinition" + send_proxy="" + fi + if echo "$hostdefinition" | grep -qE ":${proto}="; then + # Use custom port definition if requested + port="$(echo "$hostdefinition" | sed -E "s/^(.*:)?${proto}=([0-9]+).*$/:\2/")" + else + # Else use the default + port="" + fi + servername=$host + + res=$(get_ip "$bind" "$servername") + + if [ -z "$res" ]; then + echo "$servername does not resolve - aborting config" >&2 + exit 1 + fi + + # Treat protocols without TLS+SNI specially + if [ "$proto" = http ]; then + echo " use-server $servername if { hdr(host) -i $host }" >> "$configtmp" + else + echo " use-server $servername if { req_ssl_sni -i $host }" >> "$configtmp" + fi + + # Create the "server" itself. + # Note that port and send_proxy will be empty unless + # they were requested by the type user + echo " server $servername ${backendproto}@${host}${port}${send_proxy}" >> "$configtmp" + + done < "$param" + done +done + +# Create config file +require="__package/haproxy" __file ${CONFIG_FILE} --source "$configtmp" --mode 0644 + +require="__file${CONFIG_FILE}" __check_messages "haproxy_reload" \ + --pattern "^__file${CONFIG_FILE}" \ + --execute "service haproxy reload || service haproxy restart" diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol new file mode 100644 index 00000000..dc8bb7bf --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol @@ -0,0 +1 @@ +http https imaps smtps diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple new file mode 100644 index 00000000..8c482bd4 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple @@ -0,0 +1,3 @@ +protocol +v4proxy +v6proxy diff --git a/cdist/conf/type/__haproxy_dualstack/singleton b/cdist/conf/type/__haproxy_dualstack/singleton new file mode 100644 index 00000000..e69de29b From e2500248f2ddc83129e77f2e6b8dffb64904dbae Mon Sep 17 00:00:00 2001 From: Evilham Date: Wed, 3 Nov 2021 11:03:33 +0100 Subject: [PATCH 179/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 6f717cf4..99a8c08b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Explorer machine_type: Rewrite (Dennis Camera) * New type: __sed (Ander Punnar) + * New type: __haproxy_dualstack (Evilham and ungleich) * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) * Type __apt_source: Fix complaint about suite change (Matthias Stecher) From 3a321469a8ba5aea55220bd70bd4900de732e917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 16 Nov 2021 11:11:45 +0100 Subject: [PATCH 180/206] Python 3.10: collections.X -> collections.abc.X --- cdist/integration.py | 2 +- cdist/util/fsproperty.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/integration.py b/cdist/integration.py index 17b65f09..04470ea7 100644 --- a/cdist/integration.py +++ b/cdist/integration.py @@ -84,7 +84,7 @@ def _process_hosts_simple(action, host, manifest, verbose, """ if isinstance(host, str): hosts = [host, ] - elif isinstance(host, collections.Iterable): + elif isinstance(host, collections.abc.Iterable): hosts = host else: raise cdist.Error('Invalid host argument: {}'.format(host)) diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py index 09e9cc19..6bf935e8 100644 --- a/cdist/util/fsproperty.py +++ b/cdist/util/fsproperty.py @@ -33,7 +33,7 @@ class AbsolutePathRequiredError(cdist.Error): return 'Absolute path required, got: {}'.format(self.path) -class FileList(collections.MutableSequence): +class FileList(collections.abc.MutableSequence): """A list that stores it's state in a file. """ @@ -102,7 +102,7 @@ class FileList(collections.MutableSequence): self.__write(lines) -class DirectoryDict(collections.MutableMapping): +class DirectoryDict(collections.abc.MutableMapping): """A dict that stores it's items as files in a directory. """ From 6e3ad11ea0177865e7e0288b1765267df8d5d020 Mon Sep 17 00:00:00 2001 From: Evilham Date: Thu, 23 Dec 2021 20:07:28 +0100 Subject: [PATCH 181/206] [__package_upgrade_all] Add new --apt-with-new-pkgs argument --- cdist/conf/type/__package_upgrade_all/gencode-remote | 6 +++++- cdist/conf/type/__package_upgrade_all/man.rst | 8 ++++++++ cdist/conf/type/__package_upgrade_all/parameter/boolean | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote index 38aa001e..d332e851 100755 --- a/cdist/conf/type/__package_upgrade_all/gencode-remote +++ b/cdist/conf/type/__package_upgrade_all/gencode-remote @@ -28,6 +28,10 @@ apt_clean="$__object/parameter/apt-clean" apt_dist_upgrade="$__object/parameter/apt-dist-upgrade" +if [ -f "$__object/parameter/apt-with-new-pkgs" ]; then + apt_with_new_pkgs="--with-new-pkgs" +fi + if [ -f "$type" ]; then type="$(cat "$type")" else @@ -54,7 +58,7 @@ case "$type" in apt) if [ -f "$apt_dist_upgrade" ] then echo "$aptget dist-upgrade" - else echo "$aptget upgrade" + else echo "$aptget $apt_with_new_pkgs upgrade" fi if [ -f "$apt_clean" ] diff --git a/cdist/conf/type/__package_upgrade_all/man.rst b/cdist/conf/type/__package_upgrade_all/man.rst index e9e2b8ce..0c116bac 100644 --- a/cdist/conf/type/__package_upgrade_all/man.rst +++ b/cdist/conf/type/__package_upgrade_all/man.rst @@ -33,6 +33,14 @@ BOOLEAN PARAMETERS apt-dist-upgrade Do dist-upgrade instead of upgrade. +apt-with-new-pkg + Allow installing new packages when used in conjunction with + upgrade. This is useful if the update of an installed package + requires new dependencies to be installed. Instead of holding the + package back upgrade will upgrade the package and install the new + dependencies. Note that upgrade with this option will never remove + packages, only allow adding new ones. + apt-clean Clean out the local repository of retrieved package files. diff --git a/cdist/conf/type/__package_upgrade_all/parameter/boolean b/cdist/conf/type/__package_upgrade_all/parameter/boolean index 7a56a34b..cd22eb90 100644 --- a/cdist/conf/type/__package_upgrade_all/parameter/boolean +++ b/cdist/conf/type/__package_upgrade_all/parameter/boolean @@ -1,2 +1,3 @@ apt-clean apt-dist-upgrade +apt-with-new-pkgs From c2c5668b704e1648ff6c8fb88219badddd028346 Mon Sep 17 00:00:00 2001 From: Evilham Date: Thu, 23 Dec 2021 20:08:49 +0100 Subject: [PATCH 182/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 99a8c08b..26d89057 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * New type: __haproxy_dualstack (Evilham and ungleich) * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) + * Type __package_upgrade_all: Add new --apt-with-new-pkgs argument (Evilham) * Type __apt_source: Fix complaint about suite change (Matthias Stecher) * Type __package_apt: Fix complaint about suite change (Matthias Stecher) * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) From 08ff41efded6e3112fc462ba13d9166e620b4082 Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Tue, 8 Mar 2022 12:04:58 +0100 Subject: [PATCH 183/206] Added rm of tmpfile. --- cdist/conf/type/__ssh_authorized_key/gencode-remote | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index 61c77fb9..cbffde94 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -40,6 +40,7 @@ if [ -f "$file" ]; then grep -v -F -x '$line' '$file' >\$tmpfile fi cat "\$tmpfile" >"$file" +rm -f "\$tmpfile" DONE } From e0150e779681e232f95bdbefd957c666f05daa89 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 9 Mar 2022 16:16:44 +0100 Subject: [PATCH 184/206] ++changes --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 26d89057..81be51f6 100644 --- a/docs/changelog +++ b/docs/changelog @@ -12,6 +12,8 @@ next: * Type __package_apt: Fix complaint about suite change (Matthias Stecher) * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) + * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) + * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From bd44c023d33eb51a09afad12b18c082f6a3ae36d Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 11 Jun 2021 11:22:31 +0100 Subject: [PATCH 185/206] Fix typos; add default priority; comments in generated files --- cdist/conf/type/__apt_pin/manifest | 5 +++++ cdist/conf/type/__apt_pin/parameter/default/priority | 1 + cdist/conf/type/__apt_pin/parameter/optional | 1 + cdist/conf/type/__apt_pin/parameter/required | 1 - 4 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 cdist/conf/type/__apt_pin/parameter/default/priority diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index e72a8fdd..983b2b42 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -57,6 +57,11 @@ __file "/etc/apt/preferences.d/$name" \ --owner root --group root --mode 0644 \ --state "$state" \ --source - << EOF +# Created by cdist ${__type##*/} +# Do not change. Changes will be overwritten. +# + +# $name Package: $package Pin: $pin Pin-Priority: $priority diff --git a/cdist/conf/type/__apt_pin/parameter/default/priority b/cdist/conf/type/__apt_pin/parameter/default/priority new file mode 100644 index 00000000..1b79f38e --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/default/priority @@ -0,0 +1 @@ +500 diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional index 52f01fd2..847e703d 100644 --- a/cdist/conf/type/__apt_pin/parameter/optional +++ b/cdist/conf/type/__apt_pin/parameter/optional @@ -1,2 +1,3 @@ state package +priority diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required index 4b4e9741..c8572d92 100644 --- a/cdist/conf/type/__apt_pin/parameter/required +++ b/cdist/conf/type/__apt_pin/parameter/required @@ -1,2 +1 @@ distribution -priority From 22039284f57f575defa0bc65c46b8bbcbe016cd8 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sun, 10 Apr 2022 23:52:53 +0200 Subject: [PATCH 186/206] __file: make file uploading and attribute changes more atomic Fixes https://code.ungleich.ch/ungleich-public/cdist/pulls/331 Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 28 +++++++++++++++++++-------- cdist/conf/type/__file/gencode-remote | 14 +++++++++++++- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 231b6927..bea3d79c 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) -# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -89,10 +89,26 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then touch "$__object/files/set-attributes" # upload file to temp location - tempfile_template="${destination}.cdist.XXXXXXXXXX" + upload_destination="$(mktemp -u "${destination}.cdist.XXXXXXXXXX")" + # Yes, we are aware that this is a race condition. + # However: + # a) cdist usually writes to directories that are not user writable + # (probably > 99.9%) + # b) if they are user owned, the user / attacker always wins + # (probably < 0.1%) + # c) the only case which we could improve are tmp directories and we + # don't think managing tmp directories with cdist is a typical case + # ("the rest %)" cat << DONE -destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template")" +$__remote_exec $__target_host test -e $upload_destination && { + echo "Refusing to upload file to existing destination: $upload_destination" >&2 + exit 1 +} DONE + # Tell gencode-remote to where we uploaded the file so it can move + # it to its final destination. + echo "$upload_destination" > "$__object/files/upload-destination" + if [ "$upload_file" ]; then echo upload >> "$__messages_out" # IPv6 fix @@ -103,12 +119,8 @@ DONE my_target_host="${__target_host}" fi cat << DONE -$__remote_copy "$source" "${my_target_host}:\$destination_upload" +$__remote_copy "$source" "${my_target_host}:${upload_destination}" DONE fi -# move uploaded file into place -cat << DONE -$__remote_exec $__target_host "rm -rf \"$destination\"; mv \"\$destination_upload\" \"$destination\"" -DONE fi fi diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index f7a528fd..136520a7 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) -# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -62,6 +62,13 @@ set_mode() { case "$state_should" in present|exists) + if [ -f "$__object/files/upload-destination" ]; then + final_destination="$destination" + # We change the 'global' $destination variable here so we can + # change attributes of the new/uploaded file before moving it + # to it's final destination. + destination="$(cat "$__object/files/upload-destination")" + fi # Note: Mode - needs to happen last as a chown/chgrp can alter mode by # clearing S_ISUID and S_ISGID bits (see chown(2)) for attribute in group owner mode; do @@ -81,6 +88,11 @@ case "$state_should" in fi fi done + if [ -f "$__object/files/upload-destination" ]; then + # move uploaded file into place + printf 'rm -rf "%s"\n' "$final_destination" + printf 'mv -T "%s" "%s"\n' "$destination" "$final_destination" + fi if [ -f "$__object/files/set-attributes" ]; then # set-attributes is created if file is created or uploaded in gencode-local fire_onchange=1 From af54fe6febc186b0db6da00f2d06d5e87f4012f9 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Mon, 11 Apr 2022 00:04:41 +0200 Subject: [PATCH 187/206] changelog++ Signed-off-by: Steven Armstrong --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 81be51f6..3363be27 100644 --- a/docs/changelog +++ b/docs/changelog @@ -14,6 +14,7 @@ next: * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) + * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From cb0fa0f2e4d99891653b8871309e22f62ff393dd Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Fri, 24 Dec 2021 15:39:29 +0100 Subject: [PATCH 188/206] force add-apt-repository to act in non-interactive mode --- cdist/conf/type/__apt_ppa/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index 84ebebfe..094366b9 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -29,7 +29,7 @@ fi case "$state_should" in present) - echo "add-apt-repository '$name'" + echo "add-apt-repository -y '$name'" ;; absent) echo "remove-apt-repository '$name'" From 54a5cb17b762a7fbb61225b36621c8a45e83536b Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Fri, 24 Dec 2021 15:41:03 +0100 Subject: [PATCH 189/206] use add-apt-repository instead of add-apt-repository Remove `remove-apt-repository` which is now no longer needed; use `add-apt-repository` which allow removal through the `-r` flag. --- .../__apt_ppa/files/remove-apt-repository | 55 ------------------- cdist/conf/type/__apt_ppa/gencode-remote | 2 +- cdist/conf/type/__apt_ppa/manifest | 5 -- 3 files changed, 1 insertion(+), 61 deletions(-) delete mode 100755 cdist/conf/type/__apt_ppa/files/remove-apt-repository diff --git a/cdist/conf/type/__apt_ppa/files/remove-apt-repository b/cdist/conf/type/__apt_ppa/files/remove-apt-repository deleted file mode 100755 index 3eb7d491..00000000 --- a/cdist/conf/type/__apt_ppa/files/remove-apt-repository +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# -# Remove the given apt repository. -# -# Exit with: -# 0: if it worked -# 1: if not -# 2: on other error - -import os -import sys -from aptsources import distro, sourceslist -from softwareproperties import ppa -from softwareproperties.SoftwareProperties import SoftwareProperties - - -def remove_if_empty(file_name): - with open(file_name, 'r') as f: - if f.read().strip(): - return - os.unlink(file_name) - -def remove_repository(repository): - #print 'repository:', repository - codename = distro.get_distro().codename - #print 'codename:', codename - (line, file) = ppa.expand_ppa_line(repository.strip(), codename) - #print 'line:', line - #print 'file:', file - deb_source_entry = sourceslist.SourceEntry(line, file) - src_source_entry = sourceslist.SourceEntry('deb-src{}'.format(line[3:]), file) - - try: - sp = SoftwareProperties() - sp.remove_source(deb_source_entry) - try: - # If there's a deb-src entry, remove that too - sp.remove_source(src_source_entry) - except: - pass - remove_if_empty(file) - return True - except ValueError: - print >> sys.stderr, "Error: '%s' doesn't exists in a sourcelist file" % line - return False - -if __name__ == '__main__': - if (len(sys.argv) != 2): - print >> sys.stderr, 'Error: need a repository as argument' - sys.exit(2) - repository = sys.argv[1] - if remove_repository(repository): - sys.exit(0) - else: - sys.exit(1) diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index 094366b9..e41341b8 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -32,6 +32,6 @@ case "$state_should" in echo "add-apt-repository -y '$name'" ;; absent) - echo "remove-apt-repository '$name'" + echo "add-apt-repository -r -y '$name'" ;; esac diff --git a/cdist/conf/type/__apt_ppa/manifest b/cdist/conf/type/__apt_ppa/manifest index c6f4e876..57e85442 100755 --- a/cdist/conf/type/__apt_ppa/manifest +++ b/cdist/conf/type/__apt_ppa/manifest @@ -20,9 +20,4 @@ __package software-properties-common -require="__package/software-properties-common" \ - __file /usr/local/bin/remove-apt-repository \ - --source "$__type/files/remove-apt-repository" \ - --mode 0755 - require="$__object_name" __apt_update_index From 6f8c774cb08e29c8f0282a8a40a43f1d09c7248e Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 14 Apr 2022 00:16:10 +0200 Subject: [PATCH 190/206] workaround mktemp -u checking for write access Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index bea3d79c..42c9d1e7 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -89,7 +89,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then touch "$__object/files/set-attributes" # upload file to temp location - upload_destination="$(mktemp -u "${destination}.cdist.XXXXXXXXXX")" + upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')" # Yes, we are aware that this is a race condition. # However: # a) cdist usually writes to directories that are not user writable From 2df2578e36c688fcbdb32a87609bc263d2e7db9c Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 14 Apr 2022 00:27:28 +0200 Subject: [PATCH 191/206] __file: remove the questionable check for uploadfile existence Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 42c9d1e7..aa7149c1 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -99,12 +99,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then # c) the only case which we could improve are tmp directories and we # don't think managing tmp directories with cdist is a typical case # ("the rest %)" - cat << DONE -$__remote_exec $__target_host test -e $upload_destination && { - echo "Refusing to upload file to existing destination: $upload_destination" >&2 - exit 1 -} -DONE + # Tell gencode-remote to where we uploaded the file so it can move # it to its final destination. echo "$upload_destination" > "$__object/files/upload-destination" From 8b915b15b5a969b7de308e2b1b124d28548349dc Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 14 Apr 2022 00:46:13 +0200 Subject: [PATCH 192/206] __file: make the create-empty-file case work again Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index aa7149c1..f9acdb17 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -72,6 +72,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then if [ "$type" != "file" ]; then # destination is not a regular file, upload source to replace it upload_file=1 + echo upload >> "$__messages_out" else local_cksum="$(cksum < "$source")" remote_cksum="$(cat "$__object/explorer/cksum")" @@ -88,6 +89,14 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then mkdir "$__object/files" touch "$__object/files/set-attributes" + if [ "$create_file" ]; then + # When creating an empty file we create it locally and then + # upload it so that permissions can be set before moving the file + # into place. + source="$__object/files/empty" + touch "$source" + fi + # upload file to temp location upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')" # Yes, we are aware that this is a race condition. @@ -104,18 +113,15 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then # it to its final destination. echo "$upload_destination" > "$__object/files/upload-destination" - if [ "$upload_file" ]; then - echo upload >> "$__messages_out" - # IPv6 fix - if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' - then - my_target_host="[${__target_host}]" - else - my_target_host="${__target_host}" - fi - cat << DONE + # IPv6 fix + if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' + then + my_target_host="[${__target_host}]" + else + my_target_host="${__target_host}" + fi + cat << DONE $__remote_copy "$source" "${my_target_host}:${upload_destination}" DONE - fi fi fi From abbc7dfc376ffb6554bfaad2d5a6ec93955f235f Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sat, 16 Apr 2022 19:05:31 +0200 Subject: [PATCH 193/206] since we already remove the destination, we have no need to use -T on move, fixes #333 Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index 136520a7..1a9ff69c 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -91,7 +91,7 @@ case "$state_should" in if [ -f "$__object/files/upload-destination" ]; then # move uploaded file into place printf 'rm -rf "%s"\n' "$final_destination" - printf 'mv -T "%s" "%s"\n' "$destination" "$final_destination" + printf 'mv "%s" "%s"\n' "$destination" "$final_destination" fi if [ -f "$__object/files/set-attributes" ]; then # set-attributes is created if file is created or uploaded in gencode-local From 6c8c692a22c886bf82b18f51a133f88a8ab547b7 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Mon, 2 May 2022 23:25:59 +0200 Subject: [PATCH 194/206] __file: kiss and fix regression on Mac OSX Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index f9acdb17..5a303308 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -98,7 +98,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then fi # upload file to temp location - upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')" + upload_destination="${destination}.cdist.${__cdist_object_marker}.$$" # Yes, we are aware that this is a race condition. # However: # a) cdist usually writes to directories that are not user writable From 3d58c9b24fede2d1dafb15fc36424c52c466bb0d Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Fri, 20 May 2022 13:48:07 +0200 Subject: [PATCH 195/206] add optional file parameter to allow for use in a loop without object_id clashes --- cdist/conf/type/__dot_file/man.rst | 15 +++++++++++++++ cdist/conf/type/__dot_file/manifest | 10 ++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__dot_file/man.rst b/cdist/conf/type/__dot_file/man.rst index ba7621a1..c8f36712 100644 --- a/cdist/conf/type/__dot_file/man.rst +++ b/cdist/conf/type/__dot_file/man.rst @@ -37,6 +37,12 @@ state source forwarded to :strong:`__file` type +file + forwarded to :strong:`__file` type + This can be used if multiple users need to have a dotfile updated, + which will result in duplicate object id errors. When using the + file parameter the object id can be some unique value. + MESSAGES -------- @@ -61,6 +67,15 @@ EXAMPLES # Install default xmonad config for user 'eve'. Parent directory is created automatically. __dot_file .xmonad/xmonad.hs --user eve --state exists --source "$__files/xmonad.hs" + # install .vimrc for root and some users + for user in root userx usery userz; do + __dot_file "${user}_dot_vimrc" \ + --user $user \ + --file .vimrc \ + --state exists \ + --source "$__files/$user/.vimrc" + done + SEE ALSO -------- diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest index 02dadf05..a38ed943 100755 --- a/cdist/conf/type/__dot_file/manifest +++ b/cdist/conf/type/__dot_file/manifest @@ -20,13 +20,19 @@ user="$(cat "${__object}/parameter/user")" home="$(cat "${__object}/explorer/home")" primary_group="$(cat "${__object}/explorer/primary_group")" dirmode="$(cat "${__object}/parameter/dirmode")" +if [ -f "${__object}/parameter/file" ]; then + file="$(cat "${__object}/parameter/file")" +else + file="${__object_id}" +fi + # Create parent directory. Type __directory has flag 'parents', but it # will leave us with root-owned directory in user home, which is not # acceptable. So we create parent directories one-by-one. XXX: maybe # it should be fixed in '__directory'? set -- -subpath=${__object_id} +subpath=${file} while subpath="$(dirname "${subpath}")" ; do [ "${subpath}" = . ] && break set -- "${subpath}" "$@" @@ -64,4 +70,4 @@ if [ "${source}" = "-" ] ; then fi unset source -__file "${home}/${__object_id}" --owner "$user" --group "$primary_group" "$@" +__file "${home}/${file}" --owner "$user" --group "$primary_group" "$@" From 9839c2d8ec63719c30493870851f1e1ab8079fb1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 20 May 2022 14:55:12 +0200 Subject: [PATCH 196/206] ++changelog Signed-off-by: Nico Schottelius --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 3363be27..29507443 100644 --- a/docs/changelog +++ b/docs/changelog @@ -15,6 +15,7 @@ next: * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) + * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 77d9a757ec0e6986388aa368b77f065ae6be68c8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 20 May 2022 14:58:45 +0200 Subject: [PATCH 197/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 29507443..8c9e1028 100644 --- a/docs/changelog +++ b/docs/changelog @@ -16,6 +16,7 @@ next: * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) + * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 0ae37b3445c6662a968c49972e0102c75d6f14c2 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 1 Jul 2022 14:27:33 +0100 Subject: [PATCH 198/206] Handle signed-by option in __apt_source Allow users to specify a GPG key fingerprint or keyring file to be included as the 'signed-by' option. --- .../type/__apt_source/files/source.list.template | 3 ++- cdist/conf/type/__apt_source/man.rst | 8 ++++++++ cdist/conf/type/__apt_source/manifest | 14 ++++++++++---- cdist/conf/type/__apt_source/parameter/optional | 3 ++- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__apt_source/files/source.list.template b/cdist/conf/type/__apt_source/files/source.list.template index d4420e96..a28bb45f 100755 --- a/cdist/conf/type/__apt_source/files/source.list.template +++ b/cdist/conf/type/__apt_source/files/source.list.template @@ -2,13 +2,14 @@ set -u entry="$uri $distribution $component" + cat << DONE # Created by cdist ${__type##*/} # Do not change. Changes will be overwritten. # # $name -deb ${forcedarch} $entry +deb ${options} $entry DONE if [ -f "$__object/parameter/include-src" ]; then echo "deb-src $entry" diff --git a/cdist/conf/type/__apt_source/man.rst b/cdist/conf/type/__apt_source/man.rst index d1acb388..d317a135 100644 --- a/cdist/conf/type/__apt_source/man.rst +++ b/cdist/conf/type/__apt_source/man.rst @@ -23,6 +23,9 @@ OPTIONAL PARAMETERS arch set this if you need to force and specific arch (ubuntu specific) +signed-by + provide a GPG key fingerprint or keyring path for signature checks + state 'present' or 'absent', defaults to 'present' @@ -56,6 +59,11 @@ EXAMPLES --uri http://archive.canonical.com/ \ --component partner --state present + __apt_source goaccess \ + --uri http://deb.goaccess.io/ \ + --component main \ + --signed-by C03B48887D5E56B046715D3297BD1A0133449C3D + AUTHORS ------- diff --git a/cdist/conf/type/__apt_source/manifest b/cdist/conf/type/__apt_source/manifest index 35f15909..cdb526d3 100755 --- a/cdist/conf/type/__apt_source/manifest +++ b/cdist/conf/type/__apt_source/manifest @@ -31,9 +31,15 @@ fi component="$(cat "$__object/parameter/component")" if [ -f "$__object/parameter/arch" ]; then - forcedarch="[arch=$(cat "$__object/parameter/arch")]" -else - forcedarch="" + options="arch=$(cat "$__object/parameter/arch")" +fi + +if [ -f "$__object/parameter/signed-by" ]; then + options="$options signed-by=$(cat "$__object/parameter/signed-by")" +fi + +if [ "$options" ]; then + options="[$options]" fi # export variables for use in template @@ -41,7 +47,7 @@ export name export uri export distribution export component -export forcedarch +export options # generate file from template mkdir "$__object/files" diff --git a/cdist/conf/type/__apt_source/parameter/optional b/cdist/conf/type/__apt_source/parameter/optional index 87537335..0b5470a1 100644 --- a/cdist/conf/type/__apt_source/parameter/optional +++ b/cdist/conf/type/__apt_source/parameter/optional @@ -1,4 +1,5 @@ state distribution component -arch \ No newline at end of file +arch +signed-by From 339ca9347b8dc3d9167393bf754a92c4dedbd704 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 2 Jul 2022 19:21:27 +0200 Subject: [PATCH 199/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 8c9e1028..3d9cb704 100644 --- a/docs/changelog +++ b/docs/changelog @@ -17,6 +17,7 @@ next: * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) + * Type __apt_source: Add signed-by parameter (Daniel Fancsali) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 483f0c161440a018efe48bc96910dcb5834c32a1 Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Wed, 13 Jul 2022 14:50:17 +0200 Subject: [PATCH 200/206] add Check Point Gaia --- cdist/conf/explorer/lsb_codename | 3 +++ cdist/conf/explorer/lsb_description | 3 +++ cdist/conf/explorer/lsb_id | 3 +++ cdist/conf/explorer/lsb_release | 3 +++ cdist/conf/explorer/os | 7 +++++++ cdist/conf/explorer/os_release | 4 ++++ cdist/conf/explorer/os_version | 3 +++ 7 files changed, 26 insertions(+) diff --git a/cdist/conf/explorer/lsb_codename b/cdist/conf/explorer/lsb_codename index 26bb8e3d..c9fb5cdf 100755 --- a/cdist/conf/explorer/lsb_codename +++ b/cdist/conf/explorer/lsb_codename @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + awk '{printf("%s\n", $(NF-1))}' /etc/cp-release + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_CODENAME") diff --git a/cdist/conf/explorer/lsb_description b/cdist/conf/explorer/lsb_description index b1009627..7279a9c2 100755 --- a/cdist/conf/explorer/lsb_description +++ b/cdist/conf/explorer/lsb_description @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + cat /etc/cp-release + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION") diff --git a/cdist/conf/explorer/lsb_id b/cdist/conf/explorer/lsb_id index 82ff9977..1f91cc40 100755 --- a/cdist/conf/explorer/lsb_id +++ b/cdist/conf/explorer/lsb_id @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + echo "CheckPoint" + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_ID") diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release index 5ebfff1a..d9100569 100755 --- a/cdist/conf/explorer/lsb_release +++ b/cdist/conf/explorer/lsb_release @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + cat /etc/cp-release|sed -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_RELEASE") diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os index 46d87f3e..b9232ee4 100755 --- a/cdist/conf/explorer/os +++ b/cdist/conf/explorer/os @@ -116,6 +116,13 @@ if [ -f /etc/slackware-version ]; then exit 0 fi +# Appliances + +if grep -q '^Check Point Gaia' /etc/cp-release 2>/dev/null; then + echo checkpoint + exit 0 +fi + uname_s="$(uname -s)" # Assume there is no tr on the client -> do lower case ourselves diff --git a/cdist/conf/explorer/os_release b/cdist/conf/explorer/os_release index 6489446b..ec85046f 100644 --- a/cdist/conf/explorer/os_release +++ b/cdist/conf/explorer/os_release @@ -34,5 +34,9 @@ elif test -f /var/run/os-release then # FreeBSD (created by os-release service) cat /var/run/os-release +elif test -f /etc/cp-release +then + # Checkpoint firewall or management (actually linux based) + cat /etc/cp-release fi diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index bbc9e4f0..430200ae 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -41,6 +41,9 @@ in # empty, but well... cat /etc/arch-release ;; + checkpoint) + awk '{version=$NF; printf("%s\n", substr(version, 2))}' /etc/cp-release + ;; debian) debian_version=$(cat /etc/debian_version) case $debian_version From 624316564531cc7575d2ee0504e3a5f684c74e3d Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Thu, 28 Jul 2022 16:27:12 +0200 Subject: [PATCH 201/206] add create and ifexists to line type --- cdist/conf/type/__line/gencode-remote | 12 ++++++++++-- cdist/conf/type/__line/man.rst | 6 ++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index a89886da..95c2360d 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -34,8 +34,16 @@ state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" if [ -z "$state_is" ]; then - printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 - exit 1 + if [ -f "$__object/parameter/create" ]; then + echo "touch $file" + else + # only bark if the file should exists + if [ -f "$__object/parameter/ifexists" ]; then + exit 0 + fi + printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 + exit 1 + fi fi if [ "$state_should" = "$state_is" ] || \ diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst index 70490f68..dd41fc40 100644 --- a/cdist/conf/type/__line/man.rst +++ b/cdist/conf/type/__line/man.rst @@ -21,6 +21,9 @@ OPTIONAL PARAMETERS after Insert the given line after this pattern. +create + It the file does not exist then create an empty file + before Insert the given line before this pattern. @@ -28,6 +31,9 @@ file If supplied, use this as the destination file. Otherwise the object_id is used. +ifexists + Only apply the line if the file exists. + line Specifies the line which should be absent or present. From 7d8fc8a5c38c47a36ea87cd98d798d22f91c3185 Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Thu, 28 Jul 2022 17:18:41 +0200 Subject: [PATCH 202/206] improve checkpoint sed, add __line changes --- cdist/conf/explorer/lsb_release | 2 +- cdist/conf/type/__line/parameter/boolean | 2 ++ cdist/conf/type/__line/parameter/default/create | 1 + cdist/conf/type/__line/parameter/default/ifexists | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 cdist/conf/type/__line/parameter/boolean create mode 100644 cdist/conf/type/__line/parameter/default/create create mode 100644 cdist/conf/type/__line/parameter/default/ifexists diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release index d9100569..0bb9f7fe 100755 --- a/cdist/conf/explorer/lsb_release +++ b/cdist/conf/explorer/lsb_release @@ -22,7 +22,7 @@ set +e case "$("$__explorer/os")" in checkpoint) - cat /etc/cp-release|sed -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' + sed /etc/cp-release -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' ;; openwrt) # shellcheck disable=SC1091 diff --git a/cdist/conf/type/__line/parameter/boolean b/cdist/conf/type/__line/parameter/boolean new file mode 100644 index 00000000..182a5da6 --- /dev/null +++ b/cdist/conf/type/__line/parameter/boolean @@ -0,0 +1,2 @@ +create +ifexists diff --git a/cdist/conf/type/__line/parameter/default/create b/cdist/conf/type/__line/parameter/default/create new file mode 100644 index 00000000..c508d536 --- /dev/null +++ b/cdist/conf/type/__line/parameter/default/create @@ -0,0 +1 @@ +false diff --git a/cdist/conf/type/__line/parameter/default/ifexists b/cdist/conf/type/__line/parameter/default/ifexists new file mode 100644 index 00000000..c508d536 --- /dev/null +++ b/cdist/conf/type/__line/parameter/default/ifexists @@ -0,0 +1 @@ +false From 17466452f0762fc3a259a66742a59943c5af2510 Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Thu, 28 Jul 2022 17:53:41 +0200 Subject: [PATCH 203/206] revert __line for clean PR history --- cdist/conf/type/__line/gencode-remote | 12 ++---------- cdist/conf/type/__line/man.rst | 6 ------ cdist/conf/type/__line/parameter/boolean | 2 -- cdist/conf/type/__line/parameter/default/create | 1 - cdist/conf/type/__line/parameter/default/ifexists | 1 - 5 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 cdist/conf/type/__line/parameter/boolean delete mode 100644 cdist/conf/type/__line/parameter/default/create delete mode 100644 cdist/conf/type/__line/parameter/default/ifexists diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index 95c2360d..a89886da 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -34,16 +34,8 @@ state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" if [ -z "$state_is" ]; then - if [ -f "$__object/parameter/create" ]; then - echo "touch $file" - else - # only bark if the file should exists - if [ -f "$__object/parameter/ifexists" ]; then - exit 0 - fi - printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 - exit 1 - fi + printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 + exit 1 fi if [ "$state_should" = "$state_is" ] || \ diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst index dd41fc40..70490f68 100644 --- a/cdist/conf/type/__line/man.rst +++ b/cdist/conf/type/__line/man.rst @@ -21,9 +21,6 @@ OPTIONAL PARAMETERS after Insert the given line after this pattern. -create - It the file does not exist then create an empty file - before Insert the given line before this pattern. @@ -31,9 +28,6 @@ file If supplied, use this as the destination file. Otherwise the object_id is used. -ifexists - Only apply the line if the file exists. - line Specifies the line which should be absent or present. diff --git a/cdist/conf/type/__line/parameter/boolean b/cdist/conf/type/__line/parameter/boolean deleted file mode 100644 index 182a5da6..00000000 --- a/cdist/conf/type/__line/parameter/boolean +++ /dev/null @@ -1,2 +0,0 @@ -create -ifexists diff --git a/cdist/conf/type/__line/parameter/default/create b/cdist/conf/type/__line/parameter/default/create deleted file mode 100644 index c508d536..00000000 --- a/cdist/conf/type/__line/parameter/default/create +++ /dev/null @@ -1 +0,0 @@ -false diff --git a/cdist/conf/type/__line/parameter/default/ifexists b/cdist/conf/type/__line/parameter/default/ifexists deleted file mode 100644 index c508d536..00000000 --- a/cdist/conf/type/__line/parameter/default/ifexists +++ /dev/null @@ -1 +0,0 @@ -false From d4bf41ce3b7d186e7e85c030c5960ddd38855a29 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 29 Jul 2022 10:57:01 +0200 Subject: [PATCH 204/206] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 3d9cb704..db2faef0 100644 --- a/docs/changelog +++ b/docs/changelog @@ -18,6 +18,7 @@ next: * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) * Type __apt_source: Add signed-by parameter (Daniel Fancsali) + * Explorer: add support for checkpoint (Stephan Leemburg) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From be6e7fcc08d5c4e98f7ce30b157ee087010cd126 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 31 Jul 2022 21:59:35 +0200 Subject: [PATCH 205/206] Prepare release of cdist 7.0.0 --- bin/cdist-build-helper | 2 +- docs/changelog | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper index 0380b3f8..cadddae7 100755 --- a/bin/cdist-build-helper +++ b/bin/cdist-build-helper @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2022 Nico Schottelius (nico-cdist at schottelius.org) # 2016-2019 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. diff --git a/docs/changelog b/docs/changelog index db2faef0..00defc2a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) * New type: __sed (Ander Punnar) * New type: __haproxy_dualstack (Evilham and ungleich) From 90488d2e9e21b4e6bf3b7f0d8c44ebfe29c517b7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 1 Aug 2022 00:03:51 +0200 Subject: [PATCH 206/206] [doc] add release process documentation --- docs/dev/release-process.org | 90 ++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 docs/dev/release-process.org diff --git a/docs/dev/release-process.org b/docs/dev/release-process.org new file mode 100644 index 00000000..42b4f5c5 --- /dev/null +++ b/docs/dev/release-process.org @@ -0,0 +1,90 @@ +* Install requirements (Alpine) + - apk add py3-pycodestyle shellcheck py3-sphinx py3-sphinx_rtd_theme \ + py3-build twine +* Ensure your gpg setup works with the email used in the git commit! + - For me this is nico@nico-notebook.schottelius.org + - Signature / id is on nb2 +* Create ~/.pypirc +[distutils] + index-servers = + pypi + cdist + +[pypi] + username = __token__ + password = ... + +[cdist] + repository = https://upload.pypi.org/legacy/ + username = __token__ + password = ... + +* Add date in docs/changelog +* Run ./bin/cdist-build-helper +* TODO Move to "build" + - python3 -m build +* DONE git tag: when? +CLOSED: [2022-07-31 Sun 23:58] +** Asked during release process: ok +* DONE Pypi error with distutils: do not use distutils anymore +CLOSED: [2022-07-31 Sun 23:58] +python3 setup.py sdist upload +... +Creating tar archive +removing 'cdist-7.0.0' (and everything under it) +running upload +Submitting dist/cdist-7.0.0.tar.gz to https://upload.pypi.org/legacy/ +Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest. +error: Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest. +(venv2) [22:50] nb2:cdist% + +* DONE Pypi error with twine: fixed in twine 4.0.1 +CLOSED: [2022-07-31 Sun 23:58] + +Seeing: + +(venv2) [22:47] nb2:cdist% twine upload dist/cdist-7.0.0* +Uploading distributions to https://upload.pypi.org/legacy/ +Traceback (most recent call last): + File "/usr/bin/twine", line 8, in + sys.exit(main()) + File "/usr/lib/python3.10/site-packages/twine/__main__.py", line 28, in main + result = cli.dispatch(sys.argv[1:]) + File "/usr/lib/python3.10/site-packages/twine/cli.py", line 68, in dispatch + return main(args.args) + File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 197, in main + return upload(upload_settings, parsed_args.dists) + File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 141, in upload + resp = repository.upload(package) + File "/usr/lib/python3.10/site-packages/twine/repository.py", line 189, in upload + resp = self._upload(package) + File "/usr/lib/python3.10/site-packages/twine/repository.py", line 144, in _upload + data = package.metadata_dictionary() + File "/usr/lib/python3.10/site-packages/twine/package.py", line 181, in metadata_dictionary + "dynamic": meta.dynamic, +AttributeError: 'Wheel' object has no attribute 'dynamic' + + +Fix: + + +(venv2) [23:43] nb2:cdist% pipx run twine upload dist/* +⚠️ twine is already on your PATH and installed at /home/nico/venv2/bin/twine. Downloading and running anyway. +Uploading distributions to https://upload.pypi.org/legacy/ +Uploading cdist-7.0.0-py3-none-any.whl +100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 868.6/868.6 kB • 00:04 • 221.3 kB/s +Uploading cdist-7.0.0.tar.gz +100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.5/1.5 MB • 00:08 • 169.3 kB/s + +View at: +https://pypi.org/project/cdist/7.0.0/ +* TODO cdist web + - on staticweb-2022 + - Should be moved to sftp/k8s + + + Manual steps: + + ~/bin/permissions.public html/ + rsync -a html/ staticweb.ungleich.ch:/home/services/www/nico/www.cdi.st/www/manual/7.0.0/ + ssh staticweb.ungleich.ch "cd /home/services/www/nico/www.cdi.st/www/manual; ln -sf 7.0.0 latest"