From e2500248f2ddc83129e77f2e6b8dffb64904dbae Mon Sep 17 00:00:00 2001 From: Evilham Date: Wed, 3 Nov 2021 11:03:33 +0100 Subject: [PATCH 01/45] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 6f717cf4..99a8c08b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Explorer machine_type: Rewrite (Dennis Camera) * New type: __sed (Ander Punnar) + * New type: __haproxy_dualstack (Evilham and ungleich) * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) * Type __apt_source: Fix complaint about suite change (Matthias Stecher) From 3a321469a8ba5aea55220bd70bd4900de732e917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 16 Nov 2021 11:11:45 +0100 Subject: [PATCH 02/45] Python 3.10: collections.X -> collections.abc.X --- cdist/integration.py | 2 +- cdist/util/fsproperty.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/integration.py b/cdist/integration.py index 17b65f09..04470ea7 100644 --- a/cdist/integration.py +++ b/cdist/integration.py @@ -84,7 +84,7 @@ def _process_hosts_simple(action, host, manifest, verbose, """ if isinstance(host, str): hosts = [host, ] - elif isinstance(host, collections.Iterable): + elif isinstance(host, collections.abc.Iterable): hosts = host else: raise cdist.Error('Invalid host argument: {}'.format(host)) diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py index 09e9cc19..6bf935e8 100644 --- a/cdist/util/fsproperty.py +++ b/cdist/util/fsproperty.py @@ -33,7 +33,7 @@ class AbsolutePathRequiredError(cdist.Error): return 'Absolute path required, got: {}'.format(self.path) -class FileList(collections.MutableSequence): +class FileList(collections.abc.MutableSequence): """A list that stores it's state in a file. """ @@ -102,7 +102,7 @@ class FileList(collections.MutableSequence): self.__write(lines) -class DirectoryDict(collections.MutableMapping): +class DirectoryDict(collections.abc.MutableMapping): """A dict that stores it's items as files in a directory. """ From 6e3ad11ea0177865e7e0288b1765267df8d5d020 Mon Sep 17 00:00:00 2001 From: Evilham Date: Thu, 23 Dec 2021 20:07:28 +0100 Subject: [PATCH 03/45] [__package_upgrade_all] Add new --apt-with-new-pkgs argument --- cdist/conf/type/__package_upgrade_all/gencode-remote | 6 +++++- cdist/conf/type/__package_upgrade_all/man.rst | 8 ++++++++ cdist/conf/type/__package_upgrade_all/parameter/boolean | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote index 38aa001e..d332e851 100755 --- a/cdist/conf/type/__package_upgrade_all/gencode-remote +++ b/cdist/conf/type/__package_upgrade_all/gencode-remote @@ -28,6 +28,10 @@ apt_clean="$__object/parameter/apt-clean" apt_dist_upgrade="$__object/parameter/apt-dist-upgrade" +if [ -f "$__object/parameter/apt-with-new-pkgs" ]; then + apt_with_new_pkgs="--with-new-pkgs" +fi + if [ -f "$type" ]; then type="$(cat "$type")" else @@ -54,7 +58,7 @@ case "$type" in apt) if [ -f "$apt_dist_upgrade" ] then echo "$aptget dist-upgrade" - else echo "$aptget upgrade" + else echo "$aptget $apt_with_new_pkgs upgrade" fi if [ -f "$apt_clean" ] diff --git a/cdist/conf/type/__package_upgrade_all/man.rst b/cdist/conf/type/__package_upgrade_all/man.rst index e9e2b8ce..0c116bac 100644 --- a/cdist/conf/type/__package_upgrade_all/man.rst +++ b/cdist/conf/type/__package_upgrade_all/man.rst @@ -33,6 +33,14 @@ BOOLEAN PARAMETERS apt-dist-upgrade Do dist-upgrade instead of upgrade. +apt-with-new-pkg + Allow installing new packages when used in conjunction with + upgrade. This is useful if the update of an installed package + requires new dependencies to be installed. Instead of holding the + package back upgrade will upgrade the package and install the new + dependencies. Note that upgrade with this option will never remove + packages, only allow adding new ones. + apt-clean Clean out the local repository of retrieved package files. diff --git a/cdist/conf/type/__package_upgrade_all/parameter/boolean b/cdist/conf/type/__package_upgrade_all/parameter/boolean index 7a56a34b..cd22eb90 100644 --- a/cdist/conf/type/__package_upgrade_all/parameter/boolean +++ b/cdist/conf/type/__package_upgrade_all/parameter/boolean @@ -1,2 +1,3 @@ apt-clean apt-dist-upgrade +apt-with-new-pkgs From c2c5668b704e1648ff6c8fb88219badddd028346 Mon Sep 17 00:00:00 2001 From: Evilham Date: Thu, 23 Dec 2021 20:08:49 +0100 Subject: [PATCH 04/45] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 99a8c08b..26d89057 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * New type: __haproxy_dualstack (Evilham and ungleich) * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) + * Type __package_upgrade_all: Add new --apt-with-new-pkgs argument (Evilham) * Type __apt_source: Fix complaint about suite change (Matthias Stecher) * Type __package_apt: Fix complaint about suite change (Matthias Stecher) * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) From 08ff41efded6e3112fc462ba13d9166e620b4082 Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Tue, 8 Mar 2022 12:04:58 +0100 Subject: [PATCH 05/45] Added rm of tmpfile. --- cdist/conf/type/__ssh_authorized_key/gencode-remote | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index 61c77fb9..cbffde94 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -40,6 +40,7 @@ if [ -f "$file" ]; then grep -v -F -x '$line' '$file' >\$tmpfile fi cat "\$tmpfile" >"$file" +rm -f "\$tmpfile" DONE } From e0150e779681e232f95bdbefd957c666f05daa89 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 9 Mar 2022 16:16:44 +0100 Subject: [PATCH 06/45] ++changes --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 26d89057..81be51f6 100644 --- a/docs/changelog +++ b/docs/changelog @@ -12,6 +12,8 @@ next: * Type __package_apt: Fix complaint about suite change (Matthias Stecher) * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) + * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) + * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From bd44c023d33eb51a09afad12b18c082f6a3ae36d Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 11 Jun 2021 11:22:31 +0100 Subject: [PATCH 07/45] Fix typos; add default priority; comments in generated files --- cdist/conf/type/__apt_pin/manifest | 5 +++++ cdist/conf/type/__apt_pin/parameter/default/priority | 1 + cdist/conf/type/__apt_pin/parameter/optional | 1 + cdist/conf/type/__apt_pin/parameter/required | 1 - 4 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 cdist/conf/type/__apt_pin/parameter/default/priority diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index e72a8fdd..983b2b42 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -57,6 +57,11 @@ __file "/etc/apt/preferences.d/$name" \ --owner root --group root --mode 0644 \ --state "$state" \ --source - << EOF +# Created by cdist ${__type##*/} +# Do not change. Changes will be overwritten. +# + +# $name Package: $package Pin: $pin Pin-Priority: $priority diff --git a/cdist/conf/type/__apt_pin/parameter/default/priority b/cdist/conf/type/__apt_pin/parameter/default/priority new file mode 100644 index 00000000..1b79f38e --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/default/priority @@ -0,0 +1 @@ +500 diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional index 52f01fd2..847e703d 100644 --- a/cdist/conf/type/__apt_pin/parameter/optional +++ b/cdist/conf/type/__apt_pin/parameter/optional @@ -1,2 +1,3 @@ state package +priority diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required index 4b4e9741..c8572d92 100644 --- a/cdist/conf/type/__apt_pin/parameter/required +++ b/cdist/conf/type/__apt_pin/parameter/required @@ -1,2 +1 @@ distribution -priority From 22039284f57f575defa0bc65c46b8bbcbe016cd8 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sun, 10 Apr 2022 23:52:53 +0200 Subject: [PATCH 08/45] __file: make file uploading and attribute changes more atomic Fixes https://code.ungleich.ch/ungleich-public/cdist/pulls/331 Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 28 +++++++++++++++++++-------- cdist/conf/type/__file/gencode-remote | 14 +++++++++++++- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 231b6927..bea3d79c 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) -# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -89,10 +89,26 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then touch "$__object/files/set-attributes" # upload file to temp location - tempfile_template="${destination}.cdist.XXXXXXXXXX" + upload_destination="$(mktemp -u "${destination}.cdist.XXXXXXXXXX")" + # Yes, we are aware that this is a race condition. + # However: + # a) cdist usually writes to directories that are not user writable + # (probably > 99.9%) + # b) if they are user owned, the user / attacker always wins + # (probably < 0.1%) + # c) the only case which we could improve are tmp directories and we + # don't think managing tmp directories with cdist is a typical case + # ("the rest %)" cat << DONE -destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template")" +$__remote_exec $__target_host test -e $upload_destination && { + echo "Refusing to upload file to existing destination: $upload_destination" >&2 + exit 1 +} DONE + # Tell gencode-remote to where we uploaded the file so it can move + # it to its final destination. + echo "$upload_destination" > "$__object/files/upload-destination" + if [ "$upload_file" ]; then echo upload >> "$__messages_out" # IPv6 fix @@ -103,12 +119,8 @@ DONE my_target_host="${__target_host}" fi cat << DONE -$__remote_copy "$source" "${my_target_host}:\$destination_upload" +$__remote_copy "$source" "${my_target_host}:${upload_destination}" DONE fi -# move uploaded file into place -cat << DONE -$__remote_exec $__target_host "rm -rf \"$destination\"; mv \"\$destination_upload\" \"$destination\"" -DONE fi fi diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index f7a528fd..136520a7 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) -# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -62,6 +62,13 @@ set_mode() { case "$state_should" in present|exists) + if [ -f "$__object/files/upload-destination" ]; then + final_destination="$destination" + # We change the 'global' $destination variable here so we can + # change attributes of the new/uploaded file before moving it + # to it's final destination. + destination="$(cat "$__object/files/upload-destination")" + fi # Note: Mode - needs to happen last as a chown/chgrp can alter mode by # clearing S_ISUID and S_ISGID bits (see chown(2)) for attribute in group owner mode; do @@ -81,6 +88,11 @@ case "$state_should" in fi fi done + if [ -f "$__object/files/upload-destination" ]; then + # move uploaded file into place + printf 'rm -rf "%s"\n' "$final_destination" + printf 'mv -T "%s" "%s"\n' "$destination" "$final_destination" + fi if [ -f "$__object/files/set-attributes" ]; then # set-attributes is created if file is created or uploaded in gencode-local fire_onchange=1 From af54fe6febc186b0db6da00f2d06d5e87f4012f9 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Mon, 11 Apr 2022 00:04:41 +0200 Subject: [PATCH 09/45] changelog++ Signed-off-by: Steven Armstrong --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 81be51f6..3363be27 100644 --- a/docs/changelog +++ b/docs/changelog @@ -14,6 +14,7 @@ next: * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) + * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From cb0fa0f2e4d99891653b8871309e22f62ff393dd Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Fri, 24 Dec 2021 15:39:29 +0100 Subject: [PATCH 10/45] force add-apt-repository to act in non-interactive mode --- cdist/conf/type/__apt_ppa/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index 84ebebfe..094366b9 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -29,7 +29,7 @@ fi case "$state_should" in present) - echo "add-apt-repository '$name'" + echo "add-apt-repository -y '$name'" ;; absent) echo "remove-apt-repository '$name'" From 54a5cb17b762a7fbb61225b36621c8a45e83536b Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Fri, 24 Dec 2021 15:41:03 +0100 Subject: [PATCH 11/45] use add-apt-repository instead of add-apt-repository Remove `remove-apt-repository` which is now no longer needed; use `add-apt-repository` which allow removal through the `-r` flag. --- .../__apt_ppa/files/remove-apt-repository | 55 ------------------- cdist/conf/type/__apt_ppa/gencode-remote | 2 +- cdist/conf/type/__apt_ppa/manifest | 5 -- 3 files changed, 1 insertion(+), 61 deletions(-) delete mode 100755 cdist/conf/type/__apt_ppa/files/remove-apt-repository diff --git a/cdist/conf/type/__apt_ppa/files/remove-apt-repository b/cdist/conf/type/__apt_ppa/files/remove-apt-repository deleted file mode 100755 index 3eb7d491..00000000 --- a/cdist/conf/type/__apt_ppa/files/remove-apt-repository +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# -# Remove the given apt repository. -# -# Exit with: -# 0: if it worked -# 1: if not -# 2: on other error - -import os -import sys -from aptsources import distro, sourceslist -from softwareproperties import ppa -from softwareproperties.SoftwareProperties import SoftwareProperties - - -def remove_if_empty(file_name): - with open(file_name, 'r') as f: - if f.read().strip(): - return - os.unlink(file_name) - -def remove_repository(repository): - #print 'repository:', repository - codename = distro.get_distro().codename - #print 'codename:', codename - (line, file) = ppa.expand_ppa_line(repository.strip(), codename) - #print 'line:', line - #print 'file:', file - deb_source_entry = sourceslist.SourceEntry(line, file) - src_source_entry = sourceslist.SourceEntry('deb-src{}'.format(line[3:]), file) - - try: - sp = SoftwareProperties() - sp.remove_source(deb_source_entry) - try: - # If there's a deb-src entry, remove that too - sp.remove_source(src_source_entry) - except: - pass - remove_if_empty(file) - return True - except ValueError: - print >> sys.stderr, "Error: '%s' doesn't exists in a sourcelist file" % line - return False - -if __name__ == '__main__': - if (len(sys.argv) != 2): - print >> sys.stderr, 'Error: need a repository as argument' - sys.exit(2) - repository = sys.argv[1] - if remove_repository(repository): - sys.exit(0) - else: - sys.exit(1) diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index 094366b9..e41341b8 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -32,6 +32,6 @@ case "$state_should" in echo "add-apt-repository -y '$name'" ;; absent) - echo "remove-apt-repository '$name'" + echo "add-apt-repository -r -y '$name'" ;; esac diff --git a/cdist/conf/type/__apt_ppa/manifest b/cdist/conf/type/__apt_ppa/manifest index c6f4e876..57e85442 100755 --- a/cdist/conf/type/__apt_ppa/manifest +++ b/cdist/conf/type/__apt_ppa/manifest @@ -20,9 +20,4 @@ __package software-properties-common -require="__package/software-properties-common" \ - __file /usr/local/bin/remove-apt-repository \ - --source "$__type/files/remove-apt-repository" \ - --mode 0755 - require="$__object_name" __apt_update_index From 6f8c774cb08e29c8f0282a8a40a43f1d09c7248e Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 14 Apr 2022 00:16:10 +0200 Subject: [PATCH 12/45] workaround mktemp -u checking for write access Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index bea3d79c..42c9d1e7 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -89,7 +89,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then touch "$__object/files/set-attributes" # upload file to temp location - upload_destination="$(mktemp -u "${destination}.cdist.XXXXXXXXXX")" + upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')" # Yes, we are aware that this is a race condition. # However: # a) cdist usually writes to directories that are not user writable From 2df2578e36c688fcbdb32a87609bc263d2e7db9c Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 14 Apr 2022 00:27:28 +0200 Subject: [PATCH 13/45] __file: remove the questionable check for uploadfile existence Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 42c9d1e7..aa7149c1 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -99,12 +99,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then # c) the only case which we could improve are tmp directories and we # don't think managing tmp directories with cdist is a typical case # ("the rest %)" - cat << DONE -$__remote_exec $__target_host test -e $upload_destination && { - echo "Refusing to upload file to existing destination: $upload_destination" >&2 - exit 1 -} -DONE + # Tell gencode-remote to where we uploaded the file so it can move # it to its final destination. echo "$upload_destination" > "$__object/files/upload-destination" From 8b915b15b5a969b7de308e2b1b124d28548349dc Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 14 Apr 2022 00:46:13 +0200 Subject: [PATCH 14/45] __file: make the create-empty-file case work again Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index aa7149c1..f9acdb17 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -72,6 +72,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then if [ "$type" != "file" ]; then # destination is not a regular file, upload source to replace it upload_file=1 + echo upload >> "$__messages_out" else local_cksum="$(cksum < "$source")" remote_cksum="$(cat "$__object/explorer/cksum")" @@ -88,6 +89,14 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then mkdir "$__object/files" touch "$__object/files/set-attributes" + if [ "$create_file" ]; then + # When creating an empty file we create it locally and then + # upload it so that permissions can be set before moving the file + # into place. + source="$__object/files/empty" + touch "$source" + fi + # upload file to temp location upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')" # Yes, we are aware that this is a race condition. @@ -104,18 +113,15 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then # it to its final destination. echo "$upload_destination" > "$__object/files/upload-destination" - if [ "$upload_file" ]; then - echo upload >> "$__messages_out" - # IPv6 fix - if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' - then - my_target_host="[${__target_host}]" - else - my_target_host="${__target_host}" - fi - cat << DONE + # IPv6 fix + if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' + then + my_target_host="[${__target_host}]" + else + my_target_host="${__target_host}" + fi + cat << DONE $__remote_copy "$source" "${my_target_host}:${upload_destination}" DONE - fi fi fi From abbc7dfc376ffb6554bfaad2d5a6ec93955f235f Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sat, 16 Apr 2022 19:05:31 +0200 Subject: [PATCH 15/45] since we already remove the destination, we have no need to use -T on move, fixes #333 Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index 136520a7..1a9ff69c 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -91,7 +91,7 @@ case "$state_should" in if [ -f "$__object/files/upload-destination" ]; then # move uploaded file into place printf 'rm -rf "%s"\n' "$final_destination" - printf 'mv -T "%s" "%s"\n' "$destination" "$final_destination" + printf 'mv "%s" "%s"\n' "$destination" "$final_destination" fi if [ -f "$__object/files/set-attributes" ]; then # set-attributes is created if file is created or uploaded in gencode-local From 6c8c692a22c886bf82b18f51a133f88a8ab547b7 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Mon, 2 May 2022 23:25:59 +0200 Subject: [PATCH 16/45] __file: kiss and fix regression on Mac OSX Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index f9acdb17..5a303308 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -98,7 +98,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then fi # upload file to temp location - upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')" + upload_destination="${destination}.cdist.${__cdist_object_marker}.$$" # Yes, we are aware that this is a race condition. # However: # a) cdist usually writes to directories that are not user writable From 3d58c9b24fede2d1dafb15fc36424c52c466bb0d Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Fri, 20 May 2022 13:48:07 +0200 Subject: [PATCH 17/45] add optional file parameter to allow for use in a loop without object_id clashes --- cdist/conf/type/__dot_file/man.rst | 15 +++++++++++++++ cdist/conf/type/__dot_file/manifest | 10 ++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__dot_file/man.rst b/cdist/conf/type/__dot_file/man.rst index ba7621a1..c8f36712 100644 --- a/cdist/conf/type/__dot_file/man.rst +++ b/cdist/conf/type/__dot_file/man.rst @@ -37,6 +37,12 @@ state source forwarded to :strong:`__file` type +file + forwarded to :strong:`__file` type + This can be used if multiple users need to have a dotfile updated, + which will result in duplicate object id errors. When using the + file parameter the object id can be some unique value. + MESSAGES -------- @@ -61,6 +67,15 @@ EXAMPLES # Install default xmonad config for user 'eve'. Parent directory is created automatically. __dot_file .xmonad/xmonad.hs --user eve --state exists --source "$__files/xmonad.hs" + # install .vimrc for root and some users + for user in root userx usery userz; do + __dot_file "${user}_dot_vimrc" \ + --user $user \ + --file .vimrc \ + --state exists \ + --source "$__files/$user/.vimrc" + done + SEE ALSO -------- diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest index 02dadf05..a38ed943 100755 --- a/cdist/conf/type/__dot_file/manifest +++ b/cdist/conf/type/__dot_file/manifest @@ -20,13 +20,19 @@ user="$(cat "${__object}/parameter/user")" home="$(cat "${__object}/explorer/home")" primary_group="$(cat "${__object}/explorer/primary_group")" dirmode="$(cat "${__object}/parameter/dirmode")" +if [ -f "${__object}/parameter/file" ]; then + file="$(cat "${__object}/parameter/file")" +else + file="${__object_id}" +fi + # Create parent directory. Type __directory has flag 'parents', but it # will leave us with root-owned directory in user home, which is not # acceptable. So we create parent directories one-by-one. XXX: maybe # it should be fixed in '__directory'? set -- -subpath=${__object_id} +subpath=${file} while subpath="$(dirname "${subpath}")" ; do [ "${subpath}" = . ] && break set -- "${subpath}" "$@" @@ -64,4 +70,4 @@ if [ "${source}" = "-" ] ; then fi unset source -__file "${home}/${__object_id}" --owner "$user" --group "$primary_group" "$@" +__file "${home}/${file}" --owner "$user" --group "$primary_group" "$@" From 9839c2d8ec63719c30493870851f1e1ab8079fb1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 20 May 2022 14:55:12 +0200 Subject: [PATCH 18/45] ++changelog Signed-off-by: Nico Schottelius --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 3363be27..29507443 100644 --- a/docs/changelog +++ b/docs/changelog @@ -15,6 +15,7 @@ next: * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) + * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 77d9a757ec0e6986388aa368b77f065ae6be68c8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 20 May 2022 14:58:45 +0200 Subject: [PATCH 19/45] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 29507443..8c9e1028 100644 --- a/docs/changelog +++ b/docs/changelog @@ -16,6 +16,7 @@ next: * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) + * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 0ae37b3445c6662a968c49972e0102c75d6f14c2 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 1 Jul 2022 14:27:33 +0100 Subject: [PATCH 20/45] Handle signed-by option in __apt_source Allow users to specify a GPG key fingerprint or keyring file to be included as the 'signed-by' option. --- .../type/__apt_source/files/source.list.template | 3 ++- cdist/conf/type/__apt_source/man.rst | 8 ++++++++ cdist/conf/type/__apt_source/manifest | 14 ++++++++++---- cdist/conf/type/__apt_source/parameter/optional | 3 ++- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__apt_source/files/source.list.template b/cdist/conf/type/__apt_source/files/source.list.template index d4420e96..a28bb45f 100755 --- a/cdist/conf/type/__apt_source/files/source.list.template +++ b/cdist/conf/type/__apt_source/files/source.list.template @@ -2,13 +2,14 @@ set -u entry="$uri $distribution $component" + cat << DONE # Created by cdist ${__type##*/} # Do not change. Changes will be overwritten. # # $name -deb ${forcedarch} $entry +deb ${options} $entry DONE if [ -f "$__object/parameter/include-src" ]; then echo "deb-src $entry" diff --git a/cdist/conf/type/__apt_source/man.rst b/cdist/conf/type/__apt_source/man.rst index d1acb388..d317a135 100644 --- a/cdist/conf/type/__apt_source/man.rst +++ b/cdist/conf/type/__apt_source/man.rst @@ -23,6 +23,9 @@ OPTIONAL PARAMETERS arch set this if you need to force and specific arch (ubuntu specific) +signed-by + provide a GPG key fingerprint or keyring path for signature checks + state 'present' or 'absent', defaults to 'present' @@ -56,6 +59,11 @@ EXAMPLES --uri http://archive.canonical.com/ \ --component partner --state present + __apt_source goaccess \ + --uri http://deb.goaccess.io/ \ + --component main \ + --signed-by C03B48887D5E56B046715D3297BD1A0133449C3D + AUTHORS ------- diff --git a/cdist/conf/type/__apt_source/manifest b/cdist/conf/type/__apt_source/manifest index 35f15909..cdb526d3 100755 --- a/cdist/conf/type/__apt_source/manifest +++ b/cdist/conf/type/__apt_source/manifest @@ -31,9 +31,15 @@ fi component="$(cat "$__object/parameter/component")" if [ -f "$__object/parameter/arch" ]; then - forcedarch="[arch=$(cat "$__object/parameter/arch")]" -else - forcedarch="" + options="arch=$(cat "$__object/parameter/arch")" +fi + +if [ -f "$__object/parameter/signed-by" ]; then + options="$options signed-by=$(cat "$__object/parameter/signed-by")" +fi + +if [ "$options" ]; then + options="[$options]" fi # export variables for use in template @@ -41,7 +47,7 @@ export name export uri export distribution export component -export forcedarch +export options # generate file from template mkdir "$__object/files" diff --git a/cdist/conf/type/__apt_source/parameter/optional b/cdist/conf/type/__apt_source/parameter/optional index 87537335..0b5470a1 100644 --- a/cdist/conf/type/__apt_source/parameter/optional +++ b/cdist/conf/type/__apt_source/parameter/optional @@ -1,4 +1,5 @@ state distribution component -arch \ No newline at end of file +arch +signed-by From 339ca9347b8dc3d9167393bf754a92c4dedbd704 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 2 Jul 2022 19:21:27 +0200 Subject: [PATCH 21/45] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 8c9e1028..3d9cb704 100644 --- a/docs/changelog +++ b/docs/changelog @@ -17,6 +17,7 @@ next: * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) + * Type __apt_source: Add signed-by parameter (Daniel Fancsali) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 483f0c161440a018efe48bc96910dcb5834c32a1 Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Wed, 13 Jul 2022 14:50:17 +0200 Subject: [PATCH 22/45] add Check Point Gaia --- cdist/conf/explorer/lsb_codename | 3 +++ cdist/conf/explorer/lsb_description | 3 +++ cdist/conf/explorer/lsb_id | 3 +++ cdist/conf/explorer/lsb_release | 3 +++ cdist/conf/explorer/os | 7 +++++++ cdist/conf/explorer/os_release | 4 ++++ cdist/conf/explorer/os_version | 3 +++ 7 files changed, 26 insertions(+) diff --git a/cdist/conf/explorer/lsb_codename b/cdist/conf/explorer/lsb_codename index 26bb8e3d..c9fb5cdf 100755 --- a/cdist/conf/explorer/lsb_codename +++ b/cdist/conf/explorer/lsb_codename @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + awk '{printf("%s\n", $(NF-1))}' /etc/cp-release + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_CODENAME") diff --git a/cdist/conf/explorer/lsb_description b/cdist/conf/explorer/lsb_description index b1009627..7279a9c2 100755 --- a/cdist/conf/explorer/lsb_description +++ b/cdist/conf/explorer/lsb_description @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + cat /etc/cp-release + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION") diff --git a/cdist/conf/explorer/lsb_id b/cdist/conf/explorer/lsb_id index 82ff9977..1f91cc40 100755 --- a/cdist/conf/explorer/lsb_id +++ b/cdist/conf/explorer/lsb_id @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + echo "CheckPoint" + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_ID") diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release index 5ebfff1a..d9100569 100755 --- a/cdist/conf/explorer/lsb_release +++ b/cdist/conf/explorer/lsb_release @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + cat /etc/cp-release|sed -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_RELEASE") diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os index 46d87f3e..b9232ee4 100755 --- a/cdist/conf/explorer/os +++ b/cdist/conf/explorer/os @@ -116,6 +116,13 @@ if [ -f /etc/slackware-version ]; then exit 0 fi +# Appliances + +if grep -q '^Check Point Gaia' /etc/cp-release 2>/dev/null; then + echo checkpoint + exit 0 +fi + uname_s="$(uname -s)" # Assume there is no tr on the client -> do lower case ourselves diff --git a/cdist/conf/explorer/os_release b/cdist/conf/explorer/os_release index 6489446b..ec85046f 100644 --- a/cdist/conf/explorer/os_release +++ b/cdist/conf/explorer/os_release @@ -34,5 +34,9 @@ elif test -f /var/run/os-release then # FreeBSD (created by os-release service) cat /var/run/os-release +elif test -f /etc/cp-release +then + # Checkpoint firewall or management (actually linux based) + cat /etc/cp-release fi diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index bbc9e4f0..430200ae 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -41,6 +41,9 @@ in # empty, but well... cat /etc/arch-release ;; + checkpoint) + awk '{version=$NF; printf("%s\n", substr(version, 2))}' /etc/cp-release + ;; debian) debian_version=$(cat /etc/debian_version) case $debian_version From 624316564531cc7575d2ee0504e3a5f684c74e3d Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Thu, 28 Jul 2022 16:27:12 +0200 Subject: [PATCH 23/45] add create and ifexists to line type --- cdist/conf/type/__line/gencode-remote | 12 ++++++++++-- cdist/conf/type/__line/man.rst | 6 ++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index a89886da..95c2360d 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -34,8 +34,16 @@ state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" if [ -z "$state_is" ]; then - printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 - exit 1 + if [ -f "$__object/parameter/create" ]; then + echo "touch $file" + else + # only bark if the file should exists + if [ -f "$__object/parameter/ifexists" ]; then + exit 0 + fi + printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 + exit 1 + fi fi if [ "$state_should" = "$state_is" ] || \ diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst index 70490f68..dd41fc40 100644 --- a/cdist/conf/type/__line/man.rst +++ b/cdist/conf/type/__line/man.rst @@ -21,6 +21,9 @@ OPTIONAL PARAMETERS after Insert the given line after this pattern. +create + It the file does not exist then create an empty file + before Insert the given line before this pattern. @@ -28,6 +31,9 @@ file If supplied, use this as the destination file. Otherwise the object_id is used. +ifexists + Only apply the line if the file exists. + line Specifies the line which should be absent or present. From 7d8fc8a5c38c47a36ea87cd98d798d22f91c3185 Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Thu, 28 Jul 2022 17:18:41 +0200 Subject: [PATCH 24/45] improve checkpoint sed, add __line changes --- cdist/conf/explorer/lsb_release | 2 +- cdist/conf/type/__line/parameter/boolean | 2 ++ cdist/conf/type/__line/parameter/default/create | 1 + cdist/conf/type/__line/parameter/default/ifexists | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 cdist/conf/type/__line/parameter/boolean create mode 100644 cdist/conf/type/__line/parameter/default/create create mode 100644 cdist/conf/type/__line/parameter/default/ifexists diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release index d9100569..0bb9f7fe 100755 --- a/cdist/conf/explorer/lsb_release +++ b/cdist/conf/explorer/lsb_release @@ -22,7 +22,7 @@ set +e case "$("$__explorer/os")" in checkpoint) - cat /etc/cp-release|sed -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' + sed /etc/cp-release -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' ;; openwrt) # shellcheck disable=SC1091 diff --git a/cdist/conf/type/__line/parameter/boolean b/cdist/conf/type/__line/parameter/boolean new file mode 100644 index 00000000..182a5da6 --- /dev/null +++ b/cdist/conf/type/__line/parameter/boolean @@ -0,0 +1,2 @@ +create +ifexists diff --git a/cdist/conf/type/__line/parameter/default/create b/cdist/conf/type/__line/parameter/default/create new file mode 100644 index 00000000..c508d536 --- /dev/null +++ b/cdist/conf/type/__line/parameter/default/create @@ -0,0 +1 @@ +false diff --git a/cdist/conf/type/__line/parameter/default/ifexists b/cdist/conf/type/__line/parameter/default/ifexists new file mode 100644 index 00000000..c508d536 --- /dev/null +++ b/cdist/conf/type/__line/parameter/default/ifexists @@ -0,0 +1 @@ +false From 17466452f0762fc3a259a66742a59943c5af2510 Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Thu, 28 Jul 2022 17:53:41 +0200 Subject: [PATCH 25/45] revert __line for clean PR history --- cdist/conf/type/__line/gencode-remote | 12 ++---------- cdist/conf/type/__line/man.rst | 6 ------ cdist/conf/type/__line/parameter/boolean | 2 -- cdist/conf/type/__line/parameter/default/create | 1 - cdist/conf/type/__line/parameter/default/ifexists | 1 - 5 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 cdist/conf/type/__line/parameter/boolean delete mode 100644 cdist/conf/type/__line/parameter/default/create delete mode 100644 cdist/conf/type/__line/parameter/default/ifexists diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index 95c2360d..a89886da 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -34,16 +34,8 @@ state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" if [ -z "$state_is" ]; then - if [ -f "$__object/parameter/create" ]; then - echo "touch $file" - else - # only bark if the file should exists - if [ -f "$__object/parameter/ifexists" ]; then - exit 0 - fi - printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 - exit 1 - fi + printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 + exit 1 fi if [ "$state_should" = "$state_is" ] || \ diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst index dd41fc40..70490f68 100644 --- a/cdist/conf/type/__line/man.rst +++ b/cdist/conf/type/__line/man.rst @@ -21,9 +21,6 @@ OPTIONAL PARAMETERS after Insert the given line after this pattern. -create - It the file does not exist then create an empty file - before Insert the given line before this pattern. @@ -31,9 +28,6 @@ file If supplied, use this as the destination file. Otherwise the object_id is used. -ifexists - Only apply the line if the file exists. - line Specifies the line which should be absent or present. diff --git a/cdist/conf/type/__line/parameter/boolean b/cdist/conf/type/__line/parameter/boolean deleted file mode 100644 index 182a5da6..00000000 --- a/cdist/conf/type/__line/parameter/boolean +++ /dev/null @@ -1,2 +0,0 @@ -create -ifexists diff --git a/cdist/conf/type/__line/parameter/default/create b/cdist/conf/type/__line/parameter/default/create deleted file mode 100644 index c508d536..00000000 --- a/cdist/conf/type/__line/parameter/default/create +++ /dev/null @@ -1 +0,0 @@ -false diff --git a/cdist/conf/type/__line/parameter/default/ifexists b/cdist/conf/type/__line/parameter/default/ifexists deleted file mode 100644 index c508d536..00000000 --- a/cdist/conf/type/__line/parameter/default/ifexists +++ /dev/null @@ -1 +0,0 @@ -false From d4bf41ce3b7d186e7e85c030c5960ddd38855a29 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 29 Jul 2022 10:57:01 +0200 Subject: [PATCH 26/45] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 3d9cb704..db2faef0 100644 --- a/docs/changelog +++ b/docs/changelog @@ -18,6 +18,7 @@ next: * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) * Type __apt_source: Add signed-by parameter (Daniel Fancsali) + * Explorer: add support for checkpoint (Stephan Leemburg) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From be6e7fcc08d5c4e98f7ce30b157ee087010cd126 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 31 Jul 2022 21:59:35 +0200 Subject: [PATCH 27/45] Prepare release of cdist 7.0.0 --- bin/cdist-build-helper | 2 +- docs/changelog | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper index 0380b3f8..cadddae7 100755 --- a/bin/cdist-build-helper +++ b/bin/cdist-build-helper @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2022 Nico Schottelius (nico-cdist at schottelius.org) # 2016-2019 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. diff --git a/docs/changelog b/docs/changelog index db2faef0..00defc2a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) * New type: __sed (Ander Punnar) * New type: __haproxy_dualstack (Evilham and ungleich) From 90488d2e9e21b4e6bf3b7f0d8c44ebfe29c517b7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 1 Aug 2022 00:03:51 +0200 Subject: [PATCH 28/45] [doc] add release process documentation --- docs/dev/release-process.org | 90 ++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 docs/dev/release-process.org diff --git a/docs/dev/release-process.org b/docs/dev/release-process.org new file mode 100644 index 00000000..42b4f5c5 --- /dev/null +++ b/docs/dev/release-process.org @@ -0,0 +1,90 @@ +* Install requirements (Alpine) + - apk add py3-pycodestyle shellcheck py3-sphinx py3-sphinx_rtd_theme \ + py3-build twine +* Ensure your gpg setup works with the email used in the git commit! + - For me this is nico@nico-notebook.schottelius.org + - Signature / id is on nb2 +* Create ~/.pypirc +[distutils] + index-servers = + pypi + cdist + +[pypi] + username = __token__ + password = ... + +[cdist] + repository = https://upload.pypi.org/legacy/ + username = __token__ + password = ... + +* Add date in docs/changelog +* Run ./bin/cdist-build-helper +* TODO Move to "build" + - python3 -m build +* DONE git tag: when? +CLOSED: [2022-07-31 Sun 23:58] +** Asked during release process: ok +* DONE Pypi error with distutils: do not use distutils anymore +CLOSED: [2022-07-31 Sun 23:58] +python3 setup.py sdist upload +... +Creating tar archive +removing 'cdist-7.0.0' (and everything under it) +running upload +Submitting dist/cdist-7.0.0.tar.gz to https://upload.pypi.org/legacy/ +Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest. +error: Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest. +(venv2) [22:50] nb2:cdist% + +* DONE Pypi error with twine: fixed in twine 4.0.1 +CLOSED: [2022-07-31 Sun 23:58] + +Seeing: + +(venv2) [22:47] nb2:cdist% twine upload dist/cdist-7.0.0* +Uploading distributions to https://upload.pypi.org/legacy/ +Traceback (most recent call last): + File "/usr/bin/twine", line 8, in + sys.exit(main()) + File "/usr/lib/python3.10/site-packages/twine/__main__.py", line 28, in main + result = cli.dispatch(sys.argv[1:]) + File "/usr/lib/python3.10/site-packages/twine/cli.py", line 68, in dispatch + return main(args.args) + File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 197, in main + return upload(upload_settings, parsed_args.dists) + File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 141, in upload + resp = repository.upload(package) + File "/usr/lib/python3.10/site-packages/twine/repository.py", line 189, in upload + resp = self._upload(package) + File "/usr/lib/python3.10/site-packages/twine/repository.py", line 144, in _upload + data = package.metadata_dictionary() + File "/usr/lib/python3.10/site-packages/twine/package.py", line 181, in metadata_dictionary + "dynamic": meta.dynamic, +AttributeError: 'Wheel' object has no attribute 'dynamic' + + +Fix: + + +(venv2) [23:43] nb2:cdist% pipx run twine upload dist/* +⚠️ twine is already on your PATH and installed at /home/nico/venv2/bin/twine. Downloading and running anyway. +Uploading distributions to https://upload.pypi.org/legacy/ +Uploading cdist-7.0.0-py3-none-any.whl +100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 868.6/868.6 kB • 00:04 • 221.3 kB/s +Uploading cdist-7.0.0.tar.gz +100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.5/1.5 MB • 00:08 • 169.3 kB/s + +View at: +https://pypi.org/project/cdist/7.0.0/ +* TODO cdist web + - on staticweb-2022 + - Should be moved to sftp/k8s + + + Manual steps: + + ~/bin/permissions.public html/ + rsync -a html/ staticweb.ungleich.ch:/home/services/www/nico/www.cdi.st/www/manual/7.0.0/ + ssh staticweb.ungleich.ch "cd /home/services/www/nico/www.cdi.st/www/manual; ln -sf 7.0.0 latest" From c85184dcb48118ea5edbda2799badb5779253f7c Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Sun, 18 Sep 2022 08:49:37 +0200 Subject: [PATCH 29/45] Make sure flag is followed by end of line or space. --- cdist/conf/explorer/machine_type | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 00646c75..0f212e73 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -515,7 +515,7 @@ check_vm_arch_specific() { && return 0 fi if has_cpuinfo \ - && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo + && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)\( \|$\) /proc/cpuinfo then return 0 fi From 62db96bb376354c829cdde290c915cffc24612b0 Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Thu, 29 Sep 2022 16:19:07 +0200 Subject: [PATCH 30/45] Initialise options variable so expansion when running files/source.list.template there will not be an error when the variable is not set. --- cdist/conf/type/__apt_source/manifest | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__apt_source/manifest b/cdist/conf/type/__apt_source/manifest index cdb526d3..fd1ec47f 100755 --- a/cdist/conf/type/__apt_source/manifest +++ b/cdist/conf/type/__apt_source/manifest @@ -21,6 +21,7 @@ name="$__object_id" state="$(cat "$__object/parameter/state")" uri="$(cat "$__object/parameter/uri")" +options="" if [ -f "$__object/parameter/distribution" ]; then distribution="$(cat "$__object/parameter/distribution")" From ffeaafe9b68a944418ec8496c7db9f347dd4b1ee Mon Sep 17 00:00:00 2001 From: marcoduif Date: Fri, 7 Oct 2022 07:22:31 +0000 Subject: [PATCH 31/45] Make grep more specific package name should be an exact match, not a substring --- cdist/conf/type/__apt_mark/explorer/state | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_mark/explorer/state b/cdist/conf/type/__apt_mark/explorer/state index b7fe08fa..b464179a 100755 --- a/cdist/conf/type/__apt_mark/explorer/state +++ b/cdist/conf/type/__apt_mark/explorer/state @@ -24,4 +24,4 @@ else name="$__object_id" fi -apt-mark showhold | grep -Fq "$name" && echo hold || echo unhold +apt-mark showhold | grep -q "^${name}$" && echo hold || echo unhold From f36069754c0101d1eccbef59fa4f060a19784d26 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 20 Dec 2022 18:03:15 +0100 Subject: [PATCH 32/45] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 00defc2a..af41368d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +7.0.1: + * Type __apt_mark: Narrow down grep for hold packages (marcoduif) + 7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) * New type: __sed (Ander Punnar) From bdfd92dc3707004503af4a03aea0819ad587a947 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 21 Dec 2022 09:41:33 +0100 Subject: [PATCH 33/45] ++changes --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index af41368d..e8090131 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog 7.0.1: * Type __apt_mark: Narrow down grep for hold packages (marcoduif) + * Type __apt_source: Set required options variable (Mark Verboom) 7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) From b974969f28f4d007f75904757e80e4f663e3d134 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Dec 2022 20:59:16 +0100 Subject: [PATCH 34/45] Remove double definition of scan parser Fixes #353 --- cdist/argparse.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index f17315e7..8f7bbb85 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -472,9 +472,6 @@ def get_parsers(): parser['info'].set_defaults(func=cdist.info.Info.commandline) # Scan = config + further - parser['scan'] = parser['sub'].add_parser('scan', add_help=False, - parents=[parser['config']]) - parser['scan'] = parser['sub'].add_parser( 'scan', parents=[parser['loglevel'], parser['beta'], From ed3da3c829d47c7c578733482920f111e8bc8869 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Dec 2022 21:02:41 +0100 Subject: [PATCH 35/45] ++changes --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index e8090131..6ade5d47 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog 7.0.1: * Type __apt_mark: Narrow down grep for hold packages (marcoduif) * Type __apt_source: Set required options variable (Mark Verboom) + * Core: Remove double definition of scan parser (Nico Schottelius) 7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) From 08a6b467fa7afcd2f2a0a2497441cee773166d6e Mon Sep 17 00:00:00 2001 From: Michelle Date: Wed, 25 Jan 2023 16:06:35 -0500 Subject: [PATCH 36/45] Added support for Devuan Daedalus Added one line that allows cdist to support Devuan Daedelus version --- cdist/conf/explorer/os_version | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 430200ae..fc59fd14 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -82,6 +82,7 @@ in # ceres versions don't have a number, so we decode by codename: case ${devuan_version} in + (daedalus/ceres) echo 4.99 ;; (chimaera/ceres) echo 3.99 ;; (beowulf/ceres) echo 2.99 ;; (ascii/ceres) echo 1.99 ;; From 1450861e26c8c5dd249517fa14fa8ea0f4961d3d Mon Sep 17 00:00:00 2001 From: Michelle Date: Thu, 2 Feb 2023 18:21:43 -0500 Subject: [PATCH 37/45] Updated the python version of cerbot freebsd The package referenced for cerbot to be install in the FreeBSD platform used python 3.7 package, updated to python 3.9 --- cdist/conf/type/__letsencrypt_cert/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 638a99e0..39067f3b 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -85,7 +85,7 @@ if [ -z "${certbot_fullpath}" ]; then esac ;; freebsd) - __package py37-certbot + __package py39-certbot certbot_fullpath="/usr/local/bin/certbot" ;; ubuntu) From 7dd2d1025ae3b005e5d44d11e1ca62536119f36c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 3 Feb 2023 22:54:13 +0100 Subject: [PATCH 38/45] ++changelog --- docs/changelog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 6ade5d47..4e9a8a6d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,9 +2,12 @@ Changelog --------- 7.0.1: + * Core: Remove double definition of scan parser (Nico Schottelius) * Type __apt_mark: Narrow down grep for hold packages (marcoduif) * Type __apt_source: Set required options variable (Mark Verboom) - * Core: Remove double definition of scan parser (Nico Schottelius) + * Type __letsencrypt_cert: Update python version (Michelle) + * Explorer os_version: Add support for Daedalus (Michelle) + * Explorer machine_type: Correct incorrect VMM matching (Mark Verboom) 7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) From e57cf1e70a3818180b3f54e1e0364ff3b6bbd9cc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 25 Mar 2023 12:53:24 +0100 Subject: [PATCH 39/45] Apply machine_type explorer fix from pedro https://code.ungleich.ch/pedro/cdist/commit/e13939752913eea29b84d6e4659e1ca9e0e9ffeb --- cdist/conf/explorer/machine_type | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 0f212e73..c31f5ca6 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -515,7 +515,7 @@ check_vm_arch_specific() { && return 0 fi if has_cpuinfo \ - && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)\( \|$\) /proc/cpuinfo + && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)\( \|$\)' /proc/cpuinfo then return 0 fi From 2b102f303a9c2a49e1171cddb1b7fb86309ab5e9 Mon Sep 17 00:00:00 2001 From: marcoduif Date: Wed, 20 Dec 2023 08:01:02 +0000 Subject: [PATCH 40/45] changed package state from installed to present --- cdist/conf/type/__package_luarocks/manifest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__package_luarocks/manifest b/cdist/conf/type/__package_luarocks/manifest index 7d8262ca..9e4499b2 100755 --- a/cdist/conf/type/__package_luarocks/manifest +++ b/cdist/conf/type/__package_luarocks/manifest @@ -19,5 +19,5 @@ # along with cdist. If not, see . # -__package luarocks --state installed -__package make --state installed +__package luarocks --state present +__package make --state present From 61fc5e5de8adeb6d2e49c37b2cbb64c750edcb0b Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 8 Jul 2022 16:50:54 +0100 Subject: [PATCH 41/45] Fix typo in __apt_pin docs --- cdist/conf/type/__apt_pin/man.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst index 4229c0cd..e6ec8b51 100644 --- a/cdist/conf/type/__apt_pin/man.rst +++ b/cdist/conf/type/__apt_pin/man.rst @@ -23,7 +23,7 @@ package Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used. priority - The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority) + The priority value to assign to matching packages. Defaults to 500. (To match the default target distro's priority) state Will be passed to underlying `__file` type; see there for valid values and defaults. From b7394ff4c25846be5ad8bd4fa793ceda9efae5f3 Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Thu, 2 Nov 2023 11:01:45 +0100 Subject: [PATCH 42/45] Locally sort remote group information, to prevent differences in sort output. --- cdist/conf/type/__user_groups/explorer/group | 2 +- cdist/conf/type/__user_groups/gencode-remote | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__user_groups/explorer/group b/cdist/conf/type/__user_groups/explorer/group index 5bad9a0b..8a02f219 100755 --- a/cdist/conf/type/__user_groups/explorer/group +++ b/cdist/conf/type/__user_groups/explorer/group @@ -20,4 +20,4 @@ user="$(cat "$__object/parameter/user" 2>/dev/null || echo "$__object_id")" -(id -G -n "$user" | tr ' ' '\n' | sort) 2>/dev/null || true +(id -G -n "$user" | tr ' ' '\n') 2>/dev/null || true diff --git a/cdist/conf/type/__user_groups/gencode-remote b/cdist/conf/type/__user_groups/gencode-remote index 8120761a..0585e90f 100755 --- a/cdist/conf/type/__user_groups/gencode-remote +++ b/cdist/conf/type/__user_groups/gencode-remote @@ -26,13 +26,15 @@ os=$(cat "$__global/explorer/os") mkdir "$__object/files" # file has to be sorted for comparison with `comm` sort "$__object/parameter/group" > "$__object/files/group.sorted" +# Use local sort for remote groups +sort "$__object/explorer/group" > "$__object/files/group-remote.sorted" case "$state_should" in present) - changed_groups="$(comm -13 "$__object/explorer/group" "$__object/files/group.sorted")" + changed_groups="$(comm -13 "$__object/files/group-remote.sorted" "$__object/files/group.sorted")" ;; absent) - changed_groups="$(comm -12 "$__object/explorer/group" "$__object/files/group.sorted")" + changed_groups="$(comm -12 "$__object/files/group-remote.sorted" "$__object/files/group.sorted")" ;; esac From 3e82b0085b693e6b23f2b613e2f1dd975d67e6dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=BDubom=C3=ADr=20Ku=C4=8Dera?= Date: Wed, 1 May 2024 12:11:07 +0000 Subject: [PATCH 43/45] Make development version strings PEP 440 compliant (#366) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With current pip (23.1.2) and setuptools (67.7.2) versions, installation from Git was failing with "Invalid version: '7.0.0-17-ge57cf1e7'" message. The issue can be reproduced by running the following command with the latest pip and setuptools installed: $ pip install git+https://code.ungleich.ch/ungleich-public/cdist.git@e57cf1e70a3818180b3f54e1e0364ff3b6bbd9cc#egg=cdist Reviewed-on: https://code.ungleich.ch/ungleich-public/cdist/pulls/366 Co-authored-by: Ľubomír Kučera Co-committed-by: Ľubomír Kučera --- bin/cdist-build-helper | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper index cadddae7..6f514ef5 100755 --- a/bin/cdist-build-helper +++ b/bin/cdist-build-helper @@ -534,7 +534,8 @@ eof ;; version) - printf "VERSION = \"%s\"\n" "$(git describe)" > cdist/version.py + target_version="$(git describe | sed 's/-/.dev/; s/-/+/g')" + printf "VERSION = \"%s\"\n" "${target_version}" > cdist/version.py ;; target-version) From 27471a4a82ac29c1a1f3062178a509660d55e732 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 2 Aug 2024 12:05:37 +0200 Subject: [PATCH 44/45] [__timezone] add support for openwrt --- cdist/conf/type/__timezone/gencode-remote | 9 +++++++++ cdist/conf/type/__timezone/manifest | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__timezone/gencode-remote b/cdist/conf/type/__timezone/gencode-remote index b685c990..d8612986 100755 --- a/cdist/conf/type/__timezone/gencode-remote +++ b/cdist/conf/type/__timezone/gencode-remote @@ -34,3 +34,12 @@ case "$os" in echo "echo \"$timezone_should\" > /etc/timezone" ;; esac + +case "$os" in + openwrt) + cat <