From 894311a57200c5bc2094890033d6df0867d52c43 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 24 Apr 2019 00:08:43 +0300 Subject: [PATCH 01/37] __acl: if users/groups check fail, log error and exit in explorer --- cdist/conf/type/__acl/explorer/missing_users_groups | 3 ++- cdist/conf/type/__acl/gencode-remote | 8 -------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/cdist/conf/type/__acl/explorer/missing_users_groups b/cdist/conf/type/__acl/explorer/missing_users_groups index b4af614c..2f3b9e6d 100755 --- a/cdist/conf/type/__acl/explorer/missing_users_groups +++ b/cdist/conf/type/__acl/explorer/missing_users_groups @@ -40,7 +40,8 @@ do if ! getent "$getent_db" "$check" > /dev/null then - echo "missing $parameter '$check'" + echo "missing $parameter '$check'" >&2 + exit 1 fi done \ < "$__object/parameter/$parameter" diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote index a0f25a15..c0594497 100755 --- a/cdist/conf/type/__acl/gencode-remote +++ b/cdist/conf/type/__acl/gencode-remote @@ -22,14 +22,6 @@ file_is="$( cat "$__object/explorer/file_is" )" [ "$file_is" = 'missing' ] && exit 0 -missing_users_groups="$( cat "$__object/explorer/missing_users_groups" )" - -if [ -n "$missing_users_groups" ] -then - echo "$missing_users_groups" >&2 - exit 1 -fi - os="$( cat "$__global/explorer/os" )" acl_is="$( cat "$__object/explorer/acl_is" )" From d66b6969f31ea2c7690a0b85a41a76d07b7ac3e0 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 24 Apr 2019 00:09:49 +0300 Subject: [PATCH 02/37] __acl: rename missing_users_groups explorer to more generic name for future checks --- cdist/conf/type/__acl/explorer/{missing_users_groups => checks} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cdist/conf/type/__acl/explorer/{missing_users_groups => checks} (100%) diff --git a/cdist/conf/type/__acl/explorer/missing_users_groups b/cdist/conf/type/__acl/explorer/checks similarity index 100% rename from cdist/conf/type/__acl/explorer/missing_users_groups rename to cdist/conf/type/__acl/explorer/checks From f586937614383ebf67e2bcbf061c0ab7b7847620 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 24 Apr 2019 00:36:53 +0300 Subject: [PATCH 03/37] __acl: drop Solaris because POSIX-draft ACL specification is not supported --- cdist/conf/type/__acl/gencode-remote | 44 ++++++++++++---------------- cdist/conf/type/__acl/man.rst | 6 ++-- 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote index c0594497..61a83c8c 100755 --- a/cdist/conf/type/__acl/gencode-remote +++ b/cdist/conf/type/__acl/gencode-remote @@ -74,34 +74,26 @@ fi if [ -f "$__object/parameter/remove" ] then - if echo "$os" | grep -Fq 'solaris' - then - # Solaris setfacl behaves differently. - # We will not support Solaris for now, because no way to test it. - # But adding support should be easy (use -s instead of -m on modify). - echo "$os setfacl do not support -x flag for ACL remove" >&2 - else - echo "$acl_is" | while read -r acl - do - # Skip wanted ACL entries which already exist - # and skip mask and other entries, because we - # can't actually remove them, but only change. - if echo "$acl_should" | grep -Eq "^$acl" \ - || echo "$acl" | grep -Eq '^(default:)?(mask|other)' - then continue - fi + echo "$acl_is" | while read -r acl + do + # Skip wanted ACL entries which already exist + # and skip mask and other entries, because we + # can't actually remove them, but only change. + if echo "$acl_should" | grep -Eq "^$acl" \ + || echo "$acl" | grep -Eq '^(default:)?(mask|other)' + then continue + fi - if echo "$os" | grep -Eq 'macosx|freebsd' - then - remove="$acl" - else - remove="$( echo "$acl" | sed 's/:...$//' )" - fi + if echo "$os" | grep -Eq 'macosx|freebsd' + then + remove="$acl" + else + remove="$( echo "$acl" | sed 's/:...$//' )" + fi - echo "$setfacl_exec -x \"$remove\" \"$acl_path\"" - echo "removed '$remove'" >> "$__messages_out" - done - fi + echo "$setfacl_exec -x \"$remove\" \"$acl_path\"" + echo "removed '$remove'" >> "$__messages_out" + done fi for acl in $acl_should diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst index 092eb555..e6784c87 100644 --- a/cdist/conf/type/__acl/man.rst +++ b/cdist/conf/type/__acl/man.rst @@ -10,11 +10,9 @@ DESCRIPTION ----------- ACL must be defined as 3-symbol combination, using ``r``, ``w``, ``x`` and ``-``. -Fully supported on Linux (tested on Debian and CentOS). +Fully supported and tested on Linux, partial support for FreeBSD. -Partial support for FreeBSD, OSX and Solaris. - -OpenBSD and NetBSD support is not possible. +OpenBSD, NetBSD and Solaris support is not possible. See ``setfacl`` and ``acl`` manpages for more details. From 4c21983698932e17ca9276873e32a908b4235bf7 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 24 Apr 2019 00:39:54 +0300 Subject: [PATCH 04/37] __acl: remove macosx because no way to properly test --- cdist/conf/type/__acl/gencode-remote | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote index 61a83c8c..ef903816 100755 --- a/cdist/conf/type/__acl/gencode-remote +++ b/cdist/conf/type/__acl/gencode-remote @@ -64,7 +64,7 @@ setfacl_exec='setfacl' if [ -f "$__object/parameter/recursive" ] then - if echo "$os" | grep -Eq 'macosx|freebsd' + if echo "$os" | grep -Fq 'freebsd' then echo "$os setfacl do not support recursive operations" >&2 else @@ -84,7 +84,7 @@ then then continue fi - if echo "$os" | grep -Eq 'macosx|freebsd' + if echo "$os" | grep -Fq 'freebsd' then remove="$acl" else @@ -100,7 +100,7 @@ for acl in $acl_should do if ! echo "$acl_is" | grep -Eq "^$acl" then - if echo "$os" | grep -Eq 'macosx|freebsd' \ + if echo "$os" | grep -Fq 'freebsd' \ && echo "$acl" | grep -Eq '^default:' then echo "setting default ACL in $os is currently not supported. sorry :(" >&2 From ca8bc959ed931734570b2c47d2d99c5b35c754a3 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 May 2019 13:27:43 +0300 Subject: [PATCH 05/37] __acl: add todo note --- cdist/conf/type/__acl/explorer/checks | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/conf/type/__acl/explorer/checks b/cdist/conf/type/__acl/explorer/checks index 2f3b9e6d..2aae13f6 100755 --- a/cdist/conf/type/__acl/explorer/checks +++ b/cdist/conf/type/__acl/explorer/checks @@ -18,6 +18,8 @@ # along with cdist. If not, see . # +# TODO check if filesystem has ACL turned on etc + [ ! -e "/$__object_id" ] && exit 0 for parameter in user group From 7dfc5bc4736b00e8622555169c65800bb7bf4600 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 May 2019 13:41:32 +0300 Subject: [PATCH 06/37] __acl: we do not have to check here if file exist --- cdist/conf/type/__acl/explorer/checks | 2 -- 1 file changed, 2 deletions(-) diff --git a/cdist/conf/type/__acl/explorer/checks b/cdist/conf/type/__acl/explorer/checks index 2aae13f6..a2fcf44d 100755 --- a/cdist/conf/type/__acl/explorer/checks +++ b/cdist/conf/type/__acl/explorer/checks @@ -20,8 +20,6 @@ # TODO check if filesystem has ACL turned on etc -[ ! -e "/$__object_id" ] && exit 0 - for parameter in user group do if [ ! -f "$__object/parameter/$parameter" ] From 7a25ec00edb8458c8a11de3971024a9f99d1ba47 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 May 2019 13:42:56 +0300 Subject: [PATCH 07/37] __acl: add TODO note about dry-run --- cdist/conf/type/__acl/gencode-remote | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote index ef903816..d47469c1 100755 --- a/cdist/conf/type/__acl/gencode-remote +++ b/cdist/conf/type/__acl/gencode-remote @@ -20,6 +20,7 @@ file_is="$( cat "$__object/explorer/file_is" )" +# TODO this check is not needed with dry-run [ "$file_is" = 'missing' ] && exit 0 os="$( cat "$__global/explorer/os" )" From 03f8c3aaed3d106792a05e9e24e14b86f7e0414c Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 19 May 2019 23:22:02 +0300 Subject: [PATCH 08/37] __acl: don't exit on missing file when dry run --- cdist/conf/type/__acl/gencode-remote | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote index d47469c1..36eca3f8 100755 --- a/cdist/conf/type/__acl/gencode-remote +++ b/cdist/conf/type/__acl/gencode-remote @@ -20,8 +20,7 @@ file_is="$( cat "$__object/explorer/file_is" )" -# TODO this check is not needed with dry-run -[ "$file_is" = 'missing' ] && exit 0 +[ "$file_is" = 'missing' ] && [ -z "$__cdist_dry_run" ] && exit 0 os="$( cat "$__global/explorer/os" )" From 02e10b1ffd1d1fa28586caf234f5e007dfe7e606 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 19 May 2019 23:27:58 +0300 Subject: [PATCH 09/37] __acl: fix SC1117 --- cdist/conf/type/__acl/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote index 36eca3f8..f5b0474f 100755 --- a/cdist/conf/type/__acl/gencode-remote +++ b/cdist/conf/type/__acl/gencode-remote @@ -48,7 +48,7 @@ do then [ "$file_is" = 'directory' ] && rep=x || rep=- - acl="$( echo "$acl" | sed "s/\(.*\)X/\1$rep/" )" + acl="$( echo "$acl" | sed "s/\\(.*\\)X/\\1$rep/" )" fi echo "$parameter" | grep -Eq '(mask|other)' && sep=:: || sep=: From e30d76014accf5e1f0860f504c287c5ed025da9c Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 19 May 2019 23:44:20 +0300 Subject: [PATCH 10/37] __acl: update man --- cdist/conf/type/__acl/man.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst index e6784c87..d066aae5 100644 --- a/cdist/conf/type/__acl/man.rst +++ b/cdist/conf/type/__acl/man.rst @@ -10,9 +10,7 @@ DESCRIPTION ----------- ACL must be defined as 3-symbol combination, using ``r``, ``w``, ``x`` and ``-``. -Fully supported and tested on Linux, partial support for FreeBSD. - -OpenBSD, NetBSD and Solaris support is not possible. +Fully supported and tested on Linux (ext4 filesystem), partial support for FreeBSD. See ``setfacl`` and ``acl`` manpages for more details. From c58ae44409b7170ee44187eb3622612baf91ac51 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 22 May 2019 18:34:31 +0200 Subject: [PATCH 11/37] Release 5.1.0 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index ba092840..86ac6c9e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +5.1.0: 2019-05-22 * Type __consul: Add alpine support (Nico Schottelius) * Type __consul: Add version 1.5.0 (Nico Schottelius) * Type __consul_agent: Add alpine support (Nico Schottelius) From 1d57305d359fd773a540adcec8116d785f51e9db Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sat, 25 May 2019 15:58:39 +0200 Subject: [PATCH 12/37] Use gpg key, fallback to deprecated apt-key Fixes #762 --- cdist/conf/type/__apt_key/explorer/state | 18 ++++- cdist/conf/type/__apt_key/gencode-remote | 76 ++++++++++++++++++- cdist/conf/type/__apt_key/man.rst | 17 ++++- cdist/conf/type/__apt_key/manifest | 8 ++ .../type/__apt_key/parameter/default/keydir | 1 + cdist/conf/type/__apt_key/parameter/optional | 2 + 6 files changed, 114 insertions(+), 8 deletions(-) create mode 100755 cdist/conf/type/__apt_key/manifest create mode 100644 cdist/conf/type/__apt_key/parameter/default/keydir diff --git a/cdist/conf/type/__apt_key/explorer/state b/cdist/conf/type/__apt_key/explorer/state index f7940741..38f1bd3c 100755 --- a/cdist/conf/type/__apt_key/explorer/state +++ b/cdist/conf/type/__apt_key/explorer/state @@ -27,6 +27,18 @@ else keyid="$__object_id" fi -apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \ - && echo present \ - || echo absent +keydir="$(cat "$__object/parameter/keydir")" +keyfile="$keydir/$__object_id.gpg" + +if [ -d "$keydir" ] +then + if [ -f "$keyfile" ] + then echo present + else echo absent + fi +else + # fallback to deprecated apt-key + apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \ + && echo present \ + || echo absent +fi diff --git a/cdist/conf/type/__apt_key/gencode-remote b/cdist/conf/type/__apt_key/gencode-remote index 9c4fa00c..47c8bb49 100755 --- a/cdist/conf/type/__apt_key/gencode-remote +++ b/cdist/conf/type/__apt_key/gencode-remote @@ -31,12 +31,84 @@ if [ "$state_should" = "$state_is" ]; then exit 0 fi +keydir="$(cat "$__object/parameter/keydir")" +keyfile="$keydir/$__object_id.gpg" + case "$state_should" in present) keyserver="$(cat "$__object/parameter/keyserver")" - echo "apt-key adv --keyserver \"$keyserver\" --recv-keys \"$keyid\"" + + if [ -f "$__object/parameter/uri" ]; then + uri="$(cat "$__object/parameter/uri")" + + if [ -d "$keydir" ]; then + cat << EOF + +curl -s -L \\ + -o "$keyfile" \\ + "$uri" + +if grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK' \\ + "$keyfile" +then + cat "$keyfile" \\ + | gpg --export > "$keyfile" +fi + +EOF + else + # fallback to deprecated apt-key + echo "curl -s -L '$uri' | apt-key add -" + fi + elif [ -d "$keydir" ]; then + tmp='/tmp/cdist_apt_key_tmp' + + # we need to kill gpg after 30 seconds, because gpg + # can get stuck if keyserver is not responding. + # exporting env var and not exit 1, + # because we need to clean up and kill dirmngr. + cat << EOF + +mkdir -m 700 -p "$tmp" + +if timeout 30s \\ + gpg --homedir "$tmp" \\ + --keyserver "$keyserver" \\ + --recv-keys "$keyid" +then + gpg --homedir "$tmp" \\ + --export "$keyid" \\ + > "$keyfile" +else + export GPG_GOT_STUCK=1 +fi + +GNUPGHOME="$tmp" gpgconf --kill dirmngr + +rm -rf "$tmp" + +if [ -n "\$GPG_GOT_STUCK" ] +then + echo "GPG GOT STUCK - no response from keyserver after 30 seconds" >&2 + exit 1 +fi + +EOF + else + # fallback to deprecated apt-key + echo "apt-key adv --keyserver \"$keyserver\" --recv-keys \"$keyid\"" + fi + + echo "added '$keyid'" >> "$__messages_out" ;; absent) - echo "apt-key del \"$keyid\"" + if [ -f "$keyfile" ]; then + echo "rm '$keyfile'" + else + # fallback to deprecated apt-key + echo "apt-key del \"$keyid\"" + fi + + echo "removed '$keyid'" >> "$__messages_out" ;; esac diff --git a/cdist/conf/type/__apt_key/man.rst b/cdist/conf/type/__apt_key/man.rst index 9009877e..234bc715 100644 --- a/cdist/conf/type/__apt_key/man.rst +++ b/cdist/conf/type/__apt_key/man.rst @@ -28,6 +28,12 @@ keyserver the keyserver from which to fetch the key. If omitted the default set in ./parameter/default/keyserver is used. +keydir + key save location, defaults to ``/etc/apt/trusted.pgp.d`` + +uri + the URI from which to download the key + EXAMPLES -------- @@ -47,15 +53,20 @@ EXAMPLES # same thing with other keyserver __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com + # download key from the internet + __apt_key rabbitmq \ + --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc + AUTHORS ------- Steven Armstrong +Ander Punnar COPYING ------- -Copyright \(C) 2011-2014 Steven Armstrong. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the +Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can +redistribute it and/or modify it under the terms of the GNU General Public +License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_key/manifest b/cdist/conf/type/__apt_key/manifest new file mode 100755 index 00000000..010357cd --- /dev/null +++ b/cdist/conf/type/__apt_key/manifest @@ -0,0 +1,8 @@ +#!/bin/sh -e + +__package gnupg + +if [ -f "$__object/parameter/uri" ] +then __package curl +else __package dirmngr +fi diff --git a/cdist/conf/type/__apt_key/parameter/default/keydir b/cdist/conf/type/__apt_key/parameter/default/keydir new file mode 100644 index 00000000..190eb2de --- /dev/null +++ b/cdist/conf/type/__apt_key/parameter/default/keydir @@ -0,0 +1 @@ +/etc/apt/trusted.gpg.d diff --git a/cdist/conf/type/__apt_key/parameter/optional b/cdist/conf/type/__apt_key/parameter/optional index 18cf2586..de647375 100644 --- a/cdist/conf/type/__apt_key/parameter/optional +++ b/cdist/conf/type/__apt_key/parameter/optional @@ -1,3 +1,5 @@ state keyid keyserver +keydir +uri From 520cfeda989fbcaf12706a3540a9267ad23afdee Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 25 May 2019 16:10:18 +0200 Subject: [PATCH 13/37] ++ --- docs/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog b/docs/changelog index 86ac6c9e..66140bb1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,10 @@ Changelog --------- +next: + * Type __apt_key: Use gpg key, fallback to deprecated apt-key (Ander Punnar) + * Type __acl: Fix and improve (Ander Punnar) + 5.1.0: 2019-05-22 * Type __consul: Add alpine support (Nico Schottelius) * Type __consul: Add version 1.5.0 (Nico Schottelius) From 68837e45cc768fed8a891d22c60cde4ecb3ed62b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 25 May 2019 19:19:47 +0200 Subject: [PATCH 14/37] Document type stdin inside loop caveats --- docs/changelog | 1 + docs/src/cdist-type.rst | 67 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/docs/changelog b/docs/changelog index 66140bb1..1cded1f9 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __apt_key: Use gpg key, fallback to deprecated apt-key (Ander Punnar) * Type __acl: Fix and improve (Ander Punnar) + * Documentation: Document type stdin inside loop caveats (Darko Poljak) 5.1.0: 2019-05-22 * Type __consul: Add alpine support (Nico Schottelius) diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index dfad6fa0..6cf83258 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -216,6 +216,73 @@ In the __file type, stdin is used as source for the file, if - is used for sourc .... +Stdin inside a loop +~~~~~~~~~~~~~~~~~~~ +Since cdist saves type's stdin content in the object as **$__object/stdin**, +so it can be accessed in manifest and gencode-* scripts, this can lead to +unexpected behavior. For example, suppose you have some type with the following +in its manifest: + +.. code-block:: sh + + if [ -f "$__object/parameter/foo" ] + then + while read -r l + do + __file "$l" + echo "$l" >&2 + done < "$__object/parameter/foo" + fi + +and init manifest: + +.. code-block:: sh + + __foo foo --foo a --foo b --foo c + +You expect that manifest stderr content is: + +.. code-block:: sh + + a + b + c + +and that files *a*, *b* and *c* are created. But all you get in manifest stderr +is: + +.. code-block:: sh + + a + +and only *a* file is created. + +When redirecting parameter *foo* file content to while's stdin that means that all +commands in while body have this same stdin. So when *__file* type gets executed, +cdist saves its stdin which means it gets the remaining content of parameter *foo* +file, i.e.: + +.. code-block:: sh + + b + c + +The solution is to make sure that your types inside such loops get their stdin +from somewhere else, e.g. for the above problem *__file* type can get empty +stdin from */dev/null*: + +.. code-block:: sh + + if [ -f "$__object/parameter/foo" ] + then + while read -r l + do + __file "$l" < /dev/null + echo "$l" >&2 + done < "$__object/parameter/foo" + fi + + Writing the manifest -------------------- In the manifest of a type you can use other types, so your type extends From 8315677ad1fc6dff6e10cb78754cbb5748340811 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 28 May 2019 17:53:34 +0200 Subject: [PATCH 15/37] Release 5.1.1 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 1cded1f9..2f20fd84 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +5.1.1: 2019-05-28 * Type __apt_key: Use gpg key, fallback to deprecated apt-key (Ander Punnar) * Type __acl: Fix and improve (Ander Punnar) * Documentation: Document type stdin inside loop caveats (Darko Poljak) From 55ba49efacc962d3233e28483b4237059356f887 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 19 Jun 2019 18:19:32 +0200 Subject: [PATCH 16/37] Add support for deprecated type parameters --- cdist/config.py | 5 ++++ cdist/core/cdist_type.py | 21 ++++++++++++++++ cdist/test/cdist_type/__init__.py | 12 ++++++++++ .../__with_deprecated_parameters/deprecated | 0 .../parameter/deprecated | 2 ++ .../parameter/optional | 4 ++++ docs/src/cdist-type.rst | 24 ++++++++++++++++++- 7 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 cdist/test/cdist_type/fixtures/__with_deprecated_parameters/deprecated create mode 100644 cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated create mode 100644 cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/optional diff --git a/cdist/config.py b/cdist/config.py index 1a0ab4d5..6622823b 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -767,6 +767,11 @@ class Config(object): deprecated) else: self.log.warning("Type %s is deprecated.", cdist_type.name) + for param in cdist_object.parameters: + if param in cdist_type.deprecated_parameters: + self.log.warning(("%s parameter of type %s is deprecated, " + "see man page for details"), + param, cdist_type.name) def object_prepare(self, cdist_object, transfer_type_explorers=True): """Prepare object: Run type explorer + manifest""" diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index 7cabd72f..48926a03 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -69,6 +69,7 @@ class CdistType(object): self.__optional_multiple_parameters = None self.__boolean_parameters = None self.__parameter_defaults = None + self.__deprecated_parameters = None def __hash__(self): return hash(self.name) @@ -275,3 +276,23 @@ class CdistType(object): finally: self.__parameter_defaults = defaults return self.__parameter_defaults + + @property + def deprecated_parameters(self): + """Return a list of deprecated parameters""" + if not self.__deprecated_parameters: + parameters = [] + try: + with open(os.path.join(self.absolute_path, + "parameter", + "deprecated")) as fd: + for line in fd: + line = line.strip() + if line: + parameters.append(line) + except EnvironmentError: + # error ignored + pass + finally: + self.__deprecated_parameters = parameters + return self.__deprecated_parameters diff --git a/cdist/test/cdist_type/__init__.py b/cdist/test/cdist_type/__init__.py index ac84d874..9eee395c 100644 --- a/cdist/test/cdist_type/__init__.py +++ b/cdist/test/cdist_type/__init__.py @@ -200,3 +200,15 @@ class TypeTestCase(test.CdistTestCase): self.assertEqual( list(sorted(cdist_type.parameter_defaults.keys())), ['bar', 'foo']) + + def test_without_deprecated_parameters(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, + '__without_deprecated_parameters') + self.assertEqual(cdist_type.deprecated_parameters, []) + + def test_with_deprecated_parameters(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, '__with_deprecated_parameters') + self.assertEqual(cdist_type.deprecated_parameters, + ['eggs', 'spam']) diff --git a/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/deprecated b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/deprecated new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated new file mode 100644 index 00000000..81830edb --- /dev/null +++ b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated @@ -0,0 +1,2 @@ +eggs +spam diff --git a/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/optional b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/optional new file mode 100644 index 00000000..7c804d5f --- /dev/null +++ b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/optional @@ -0,0 +1,4 @@ +spam +eggs +sausage +spam diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index 6cf83258..623ba481 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -74,7 +74,7 @@ prevents to be run in more than one instance. Deprecated types ----------------- If a type is flagged with 'deprecated' marker then it is considered deprecated. -Upon it's usage cdist writes warning line. If 'deprecated' marker has content +When it is used cdist writes warning line. If 'deprecated' marker has content then this content is printed as a deprecation messages, e.g.: .. code-block:: sh @@ -186,6 +186,28 @@ Example: (e.g. in cdist/conf/type/__nginx_vhost/manifest) fi +Deprecated parameters +--------------------- +To deprecate type parameters one can declare a newline separated file +**parameter/deprecated** where each line contains deprecated parameter. +When it is used cdist writes warning line, e.g.: + +.. code-block:: sh + + $ cat parameter/optional_multiple + foo + spam + eggs + $ cat parameter/deprecated + spam + eggs + $ echo '__foo foo --foo foo --spam spam' | ./bin/cdist config -i - 185.203.112.26 + WARNING: 185.203.112.26: spam parameter of type __foo is deprecated, see man page for details + $ echo '__foo foo --foo foo --eggs eggs --spam spam' | ./bin/cdist config -i - 185.203.112.26 + WARNING: 185.203.112.26: spam parameter of type __foo is deprecated, see man page for details + WARNING: 185.203.112.26: eggs parameter of type __foo is deprecated, see man page for details + + Input from stdin ---------------- Every type can access what has been written on stdin when it has been called. From 3cb4e76175c06370cc7b69a326c9ae1c6c0b0c37 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 19 Jun 2019 18:19:32 +0200 Subject: [PATCH 17/37] Allow custom message for each deprecated parameter --- cdist/config.py | 11 +++++-- cdist/core/cdist_type.py | 24 +++++++------- cdist/test/cdist_type/__init__.py | 9 ++++-- .../parameter/deprecated | 2 -- .../parameter/deprecated/eggs | 1 + .../{deprecated => parameter/deprecated/spam} | 0 .../parameter/optional | 1 - docs/src/cdist-type.rst | 31 ++++++++++--------- 8 files changed, 44 insertions(+), 35 deletions(-) delete mode 100644 cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated create mode 100644 cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated/eggs rename cdist/test/cdist_type/fixtures/__with_deprecated_parameters/{deprecated => parameter/deprecated/spam} (100%) diff --git a/cdist/config.py b/cdist/config.py index 6622823b..26d07fc4 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -769,9 +769,14 @@ class Config(object): self.log.warning("Type %s is deprecated.", cdist_type.name) for param in cdist_object.parameters: if param in cdist_type.deprecated_parameters: - self.log.warning(("%s parameter of type %s is deprecated, " - "see man page for details"), - param, cdist_type.name) + msg = cdist_type.deprecated_parameters[param] + if msg: + format = "%s parameter of type %s is deprecated: %s" + args = [param, cdist_type.name, msg] + else: + format = "%s parameter of type %s is deprecated." + args = [param, cdist_type.name] + self.log.warning(format, *args) def object_prepare(self, cdist_object, transfer_type_explorers=True): """Prepare object: Run type explorer + manifest""" diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index 48926a03..4500f50d 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -279,20 +279,20 @@ class CdistType(object): @property def deprecated_parameters(self): - """Return a list of deprecated parameters""" if not self.__deprecated_parameters: - parameters = [] + deprecated = {} try: - with open(os.path.join(self.absolute_path, - "parameter", - "deprecated")) as fd: - for line in fd: - line = line.strip() - if line: - parameters.append(line) + deprecated_dir = os.path.join(self.absolute_path, + "parameter", + "deprecated") + for name in cdist.core.listdir(deprecated_dir): + try: + with open(os.path.join(deprecated_dir, name)) as fd: + deprecated[name] = fd.read().strip() + except EnvironmentError: + pass # Swallow errors raised by open() or read() except EnvironmentError: - # error ignored - pass + pass # Swallow error raised by os.listdir() finally: - self.__deprecated_parameters = parameters + self.__deprecated_parameters = deprecated return self.__deprecated_parameters diff --git a/cdist/test/cdist_type/__init__.py b/cdist/test/cdist_type/__init__.py index 9eee395c..a51a1e6f 100644 --- a/cdist/test/cdist_type/__init__.py +++ b/cdist/test/cdist_type/__init__.py @@ -205,10 +205,13 @@ class TypeTestCase(test.CdistTestCase): base_path = fixtures cdist_type = core.CdistType(base_path, '__without_deprecated_parameters') - self.assertEqual(cdist_type.deprecated_parameters, []) + self.assertEqual(cdist_type.deprecated_parameters, {}) def test_with_deprecated_parameters(self): base_path = fixtures cdist_type = core.CdistType(base_path, '__with_deprecated_parameters') - self.assertEqual(cdist_type.deprecated_parameters, - ['eggs', 'spam']) + self.assertTrue('eggs' in cdist_type.deprecated_parameters) + self.assertTrue('spam' in cdist_type.deprecated_parameters) + self.assertEqual(cdist_type.deprecated_parameters['eggs'], + 'Deprecated') + self.assertEqual(cdist_type.deprecated_parameters['spam'], '') diff --git a/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated deleted file mode 100644 index 81830edb..00000000 --- a/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated +++ /dev/null @@ -1,2 +0,0 @@ -eggs -spam diff --git a/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated/eggs b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated/eggs new file mode 100644 index 00000000..69d9f456 --- /dev/null +++ b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated/eggs @@ -0,0 +1 @@ +Deprecated diff --git a/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/deprecated b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated/spam similarity index 100% rename from cdist/test/cdist_type/fixtures/__with_deprecated_parameters/deprecated rename to cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated/spam diff --git a/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/optional b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/optional index 7c804d5f..bfe09199 100644 --- a/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/optional +++ b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/optional @@ -1,4 +1,3 @@ spam eggs sausage -spam diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index 623ba481..582c0938 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -188,24 +188,27 @@ Example: (e.g. in cdist/conf/type/__nginx_vhost/manifest) Deprecated parameters --------------------- -To deprecate type parameters one can declare a newline separated file -**parameter/deprecated** where each line contains deprecated parameter. -When it is used cdist writes warning line, e.g.: +To deprecate type parameters one can declare a file for each deprecated +parameter under **parameter/deprecated** directory. + +When such parameter is used cdist writes warning line with deprecation message. +If such file has content then this content is printed as deprecation message. +If there is no content then generic parameter deprecation message is printed. + +Example: .. code-block:: sh - $ cat parameter/optional_multiple - foo - spam - eggs - $ cat parameter/deprecated - spam - eggs - $ echo '__foo foo --foo foo --spam spam' | ./bin/cdist config -i - 185.203.112.26 - WARNING: 185.203.112.26: spam parameter of type __foo is deprecated, see man page for details + $ ls parameter/deprecated/ + eggs spam + $ cat parameter/deprecated/eggs + eggs parameter is deprecated, please use multiple egg parameter. + $ cat parameter/deprecated/spam + $ echo '__foo foo --foo foo --eggs eggs' | ./bin/cdist config -i - 185.203.112.26 + WARNING: 185.203.112.26: eggs parameter of type __foo is deprecated: eggs parameter is deprecated, please use multiple egg parameter. $ echo '__foo foo --foo foo --eggs eggs --spam spam' | ./bin/cdist config -i - 185.203.112.26 - WARNING: 185.203.112.26: spam parameter of type __foo is deprecated, see man page for details - WARNING: 185.203.112.26: eggs parameter of type __foo is deprecated, see man page for details + WARNING: 185.203.112.26: spam parameter of type __foo is deprecated. + WARNING: 185.203.112.26: eggs parameter of type __foo is deprecated: eggs parameter is deprecated, please use multiple egg parameter. Input from stdin From d723f60673c35b91017319a6699c53e3d0f38425 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 20 Jun 2019 18:12:50 +0200 Subject: [PATCH 18/37] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 2f20fd84..a79b116d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Core: Add support for type parameters deprecation (Darko Poljak) + 5.1.1: 2019-05-28 * Type __apt_key: Use gpg key, fallback to deprecated apt-key (Ander Punnar) * Type __acl: Fix and improve (Ander Punnar) From 91a6ecc70173638aae928ddadabd93d9d46f097e Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 30 May 2019 23:04:46 +0300 Subject: [PATCH 19/37] __acl: rewrite --- cdist/conf/type/__acl/explorer/checks | 32 ++++------ cdist/conf/type/__acl/gencode-remote | 54 ++++++++--------- cdist/conf/type/__acl/man.rst | 59 ++++++++++--------- cdist/conf/type/__acl/parameter/optional | 2 - .../type/__acl/parameter/optional_multiple | 2 - .../type/__acl/parameter/required_multiple | 1 + 6 files changed, 66 insertions(+), 84 deletions(-) delete mode 100644 cdist/conf/type/__acl/parameter/optional delete mode 100644 cdist/conf/type/__acl/parameter/optional_multiple create mode 100644 cdist/conf/type/__acl/parameter/required_multiple diff --git a/cdist/conf/type/__acl/explorer/checks b/cdist/conf/type/__acl/explorer/checks index a2fcf44d..5b379a08 100755 --- a/cdist/conf/type/__acl/explorer/checks +++ b/cdist/conf/type/__acl/explorer/checks @@ -20,29 +20,17 @@ # TODO check if filesystem has ACL turned on etc -for parameter in user group +grep -E '^(default:)?(user|group):' "$__object/parameter/acl" \ +| while read -r acl do - if [ ! -f "$__object/parameter/$parameter" ] + param="$( echo "$acl" | awk -F: '{print $(NF-2)}' )" + check="$( echo "$acl" | awk -F: '{print $(NF-1)}' )" + + [ "$param" = 'user' ] && db=passwd || db="$param" + + if ! getent "$db" "$check" > /dev/null then - continue + echo "missing $param '$check'" >&2 + exit 1 fi - - while read -r acl - do - check="$( echo "$acl" | awk -F: '{print $1}' )" - - if [ "$parameter" = 'user' ] - then - getent_db=passwd - else - getent_db="$parameter" - fi - - if ! getent "$getent_db" "$check" > /dev/null - then - echo "missing $parameter '$check'" >&2 - exit 1 - fi - done \ - < "$__object/parameter/$parameter" done diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote index f5b0474f..3c7085f0 100755 --- a/cdist/conf/type/__acl/gencode-remote +++ b/cdist/conf/type/__acl/gencode-remote @@ -24,41 +24,35 @@ file_is="$( cat "$__object/explorer/file_is" )" os="$( cat "$__global/explorer/os" )" -acl_is="$( cat "$__object/explorer/acl_is" )" - acl_path="/$__object_id" -if [ -f "$__object/parameter/default" ] && [ "$file_is" = 'directory' ] +acl_is="$( cat "$__object/explorer/acl_is" )" + +acl_should="$( cat "$__object/parameter/acl" )" + +if [ -f "$__object/parameter/default" ] then - set_default=1 -else - set_default=0 + acl_should="$( echo "$acl_should" \ + | sed 's/^default://' \ + | sort -u \ + | sed 's/\(.*\)/default:\1\n\1/' )" fi -acl_should="$( for parameter in user group mask other -do - if [ ! -f "$__object/parameter/$parameter" ] - then - continue - fi +if [ "$file_is" = 'regular' ] \ + && echo "$acl_should" | grep -Eq '^default:' +then + # only directories can have default ACLs, + # but instead of error, + # let's just remove default entries + acl_should="$( echo "$acl_should" | grep -Ev '^default:' )" +fi - while read -r acl - do - if echo "$acl" | awk -F: '{ print $NF }' | grep -Fq 'X' - then - [ "$file_is" = 'directory' ] && rep=x || rep=- +if echo "$acl_should" | awk -F: '{ print $NF }' | grep -Fq 'X' +then + [ "$file_is" = 'directory' ] && rep=x || rep=- - acl="$( echo "$acl" | sed "s/\\(.*\\)X/\\1$rep/" )" - fi - - echo "$parameter" | grep -Eq '(mask|other)' && sep=:: || sep=: - - echo "$parameter$sep$acl" - - [ "$set_default" = '1' ] && echo "default:$parameter$sep$acl" - done \ - < "$__object/parameter/$parameter" -done )" + acl_should="$( echo "$acl_should" | sed "s/\\(.*\\)X/\\1$rep/" )" +fi setfacl_exec='setfacl' @@ -76,7 +70,7 @@ if [ -f "$__object/parameter/remove" ] then echo "$acl_is" | while read -r acl do - # Skip wanted ACL entries which already exist + # skip wanted ACL entries which already exist # and skip mask and other entries, because we # can't actually remove them, but only change. if echo "$acl_should" | grep -Eq "^$acl" \ @@ -103,7 +97,7 @@ do if echo "$os" | grep -Fq 'freebsd' \ && echo "$acl" | grep -Eq '^default:' then - echo "setting default ACL in $os is currently not supported. sorry :(" >&2 + echo "setting default ACL in $os is currently not supported" >&2 else echo "$setfacl_exec -m \"$acl\" \"$acl_path\"" echo "added '$acl'" >> "$__messages_out" diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst index d066aae5..a71e0d3c 100644 --- a/cdist/conf/type/__acl/man.rst +++ b/cdist/conf/type/__acl/man.rst @@ -8,42 +8,30 @@ cdist-type__acl - Set ACL entries DESCRIPTION ----------- -ACL must be defined as 3-symbol combination, using ``r``, ``w``, ``x`` and ``-``. - Fully supported and tested on Linux (ext4 filesystem), partial support for FreeBSD. See ``setfacl`` and ``acl`` manpages for more details. -OPTIONAL MULTIPLE PARAMETERS +REQUIRED MULTIPLE PARAMETERS ---------------------------- -user - Add user ACL entry. - -group - Add group ACL entry. - - -OPTIONAL PARAMETERS -------------------- -mask - Add mask ACL entry. - -other - Add other ACL entry. +acl + Set ACL entry following ``getfacl`` output syntax. BOOLEAN PARAMETERS ------------------ +default + Set all ACL entries as default too. + Only directories can have default ACLs. + Setting default ACL in FreeBSD is currently not supported. + recursive Make ``setfacl`` recursive (Linux only), but not ``getfacl`` in explorer. -default - Add default ACL entries (FreeBSD not supported). - remove - Remove undefined ACL entries (Solaris not supported). - ACL entries for ``mask`` and ``other`` can't be removed. + Remove undefined ACL entries. + ``mask`` and ``other`` entries can't be removed, but only changed. EXAMPLES @@ -52,15 +40,30 @@ EXAMPLES .. code-block:: sh __acl /srv/project \ + --default \ --recursive \ + --remove \ + --acl user:alice:rwx \ + --acl user:bob:r-x \ + --acl group:project-group:rwx \ + --acl group:some-other-group:r-x \ + --acl mask::r-x \ + --acl other::r-x + + # give Alice read-only access to subdir, + # but don't allow her to see parent content. + + __acl /srv/project2 \ + --remove \ + --acl default:group:secret-project:rwx \ + --acl group:secret-project:rwx \ + --acl user:alice:--x + + __acl /srv/project2/subdir \ --default \ --remove \ - --user alice:rwx \ - --user bob:r-x \ - --group project-group:rwx \ - --group some-other-group:r-x \ - --mask r-x \ - --other r-x + --acl group:secret-project:rwx \ + --acl user:alice:r-x AUTHORS diff --git a/cdist/conf/type/__acl/parameter/optional b/cdist/conf/type/__acl/parameter/optional deleted file mode 100644 index 4b32086b..00000000 --- a/cdist/conf/type/__acl/parameter/optional +++ /dev/null @@ -1,2 +0,0 @@ -mask -other diff --git a/cdist/conf/type/__acl/parameter/optional_multiple b/cdist/conf/type/__acl/parameter/optional_multiple deleted file mode 100644 index 22f5a52c..00000000 --- a/cdist/conf/type/__acl/parameter/optional_multiple +++ /dev/null @@ -1,2 +0,0 @@ -user -group diff --git a/cdist/conf/type/__acl/parameter/required_multiple b/cdist/conf/type/__acl/parameter/required_multiple new file mode 100644 index 00000000..39fead3b --- /dev/null +++ b/cdist/conf/type/__acl/parameter/required_multiple @@ -0,0 +1 @@ +acl From a5df0badaf1d38bc0f47e3e30bfd9f2e683677d1 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 16 Jun 2019 15:18:39 +0300 Subject: [PATCH 20/37] __acl: add compatibility for deprecated parameters --- cdist/conf/type/__acl/explorer/checks | 27 ++++++++++--------- cdist/conf/type/__acl/gencode-remote | 22 ++++++++++++++- cdist/conf/type/__acl/man.rst | 6 +++++ cdist/conf/type/__acl/parameter/optional | 2 ++ .../type/__acl/parameter/optional_multiple | 3 +++ .../type/__acl/parameter/required_multiple | 1 - 6 files changed, 47 insertions(+), 14 deletions(-) create mode 100644 cdist/conf/type/__acl/parameter/optional create mode 100644 cdist/conf/type/__acl/parameter/optional_multiple delete mode 100644 cdist/conf/type/__acl/parameter/required_multiple diff --git a/cdist/conf/type/__acl/explorer/checks b/cdist/conf/type/__acl/explorer/checks index 5b379a08..70bb0412 100755 --- a/cdist/conf/type/__acl/explorer/checks +++ b/cdist/conf/type/__acl/explorer/checks @@ -20,17 +20,20 @@ # TODO check if filesystem has ACL turned on etc -grep -E '^(default:)?(user|group):' "$__object/parameter/acl" \ -| while read -r acl -do - param="$( echo "$acl" | awk -F: '{print $(NF-2)}' )" - check="$( echo "$acl" | awk -F: '{print $(NF-1)}' )" +if [ -f "$__object/parameter/acl" ] +then + grep -E '^(default:)?(user|group):' "$__object/parameter/acl" \ + | while read -r acl + do + param="$( echo "$acl" | awk -F: '{print $(NF-2)}' )" + check="$( echo "$acl" | awk -F: '{print $(NF-1)}' )" - [ "$param" = 'user' ] && db=passwd || db="$param" + [ "$param" = 'user' ] && db=passwd || db="$param" - if ! getent "$db" "$check" > /dev/null - then - echo "missing $param '$check'" >&2 - exit 1 - fi -done + if ! getent "$db" "$check" > /dev/null + then + echo "missing $param '$check'" >&2 + exit 1 + fi + done +fi diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote index 3c7085f0..6dab4d09 100755 --- a/cdist/conf/type/__acl/gencode-remote +++ b/cdist/conf/type/__acl/gencode-remote @@ -28,7 +28,27 @@ acl_path="/$__object_id" acl_is="$( cat "$__object/explorer/acl_is" )" -acl_should="$( cat "$__object/parameter/acl" )" +if [ -f "$__object/parameter/acl" ] +then + acl_should="$( cat "$__object/parameter/acl" )" +elif + [ -f "$__object/parameter/user" ] \ + || [ -f "$__object/parameter/group" ] \ + || [ -f "$__object/parameter/mask" ] \ + || [ -f "$__object/parameter/other" ] +then + acl_should="$( for param in user group mask other + do + [ ! -f "$__object/parameter/$param" ] && continue + + echo "$param" | grep -Eq 'mask|other' && sep=:: || sep=: + + echo "$param$sep$( cat "$__object/parameter/$param" )" + done )" +else + echo 'no parameters set' >&2 + exit 1 +fi if [ -f "$__object/parameter/default" ] then diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst index a71e0d3c..85e946ce 100644 --- a/cdist/conf/type/__acl/man.rst +++ b/cdist/conf/type/__acl/man.rst @@ -34,6 +34,12 @@ remove ``mask`` and ``other`` entries can't be removed, but only changed. +DEPRECATED PARAMETERS +--------------------- +Parameters ``user``, ``group``, ``mask`` and ``other`` are deprecated and they +will be removed in future versions. Please use ``acl`` parameter instead. + + EXAMPLES -------- diff --git a/cdist/conf/type/__acl/parameter/optional b/cdist/conf/type/__acl/parameter/optional new file mode 100644 index 00000000..4b32086b --- /dev/null +++ b/cdist/conf/type/__acl/parameter/optional @@ -0,0 +1,2 @@ +mask +other diff --git a/cdist/conf/type/__acl/parameter/optional_multiple b/cdist/conf/type/__acl/parameter/optional_multiple new file mode 100644 index 00000000..95c25d55 --- /dev/null +++ b/cdist/conf/type/__acl/parameter/optional_multiple @@ -0,0 +1,3 @@ +acl +user +group diff --git a/cdist/conf/type/__acl/parameter/required_multiple b/cdist/conf/type/__acl/parameter/required_multiple deleted file mode 100644 index 39fead3b..00000000 --- a/cdist/conf/type/__acl/parameter/required_multiple +++ /dev/null @@ -1 +0,0 @@ -acl From a4bc051ad9ecb538d5d239a03990b788acf81d88 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 21 Jun 2019 13:02:44 +0300 Subject: [PATCH 21/37] __acl: use type deprecation --- cdist/conf/type/__acl/parameter/deprecated/group | 1 + cdist/conf/type/__acl/parameter/deprecated/mask | 1 + cdist/conf/type/__acl/parameter/deprecated/other | 1 + cdist/conf/type/__acl/parameter/deprecated/user | 1 + 4 files changed, 4 insertions(+) create mode 100644 cdist/conf/type/__acl/parameter/deprecated/group create mode 100644 cdist/conf/type/__acl/parameter/deprecated/mask create mode 100644 cdist/conf/type/__acl/parameter/deprecated/other create mode 100644 cdist/conf/type/__acl/parameter/deprecated/user diff --git a/cdist/conf/type/__acl/parameter/deprecated/group b/cdist/conf/type/__acl/parameter/deprecated/group new file mode 100644 index 00000000..94e14159 --- /dev/null +++ b/cdist/conf/type/__acl/parameter/deprecated/group @@ -0,0 +1 @@ +see manual for details diff --git a/cdist/conf/type/__acl/parameter/deprecated/mask b/cdist/conf/type/__acl/parameter/deprecated/mask new file mode 100644 index 00000000..94e14159 --- /dev/null +++ b/cdist/conf/type/__acl/parameter/deprecated/mask @@ -0,0 +1 @@ +see manual for details diff --git a/cdist/conf/type/__acl/parameter/deprecated/other b/cdist/conf/type/__acl/parameter/deprecated/other new file mode 100644 index 00000000..94e14159 --- /dev/null +++ b/cdist/conf/type/__acl/parameter/deprecated/other @@ -0,0 +1 @@ +see manual for details diff --git a/cdist/conf/type/__acl/parameter/deprecated/user b/cdist/conf/type/__acl/parameter/deprecated/user new file mode 100644 index 00000000..94e14159 --- /dev/null +++ b/cdist/conf/type/__acl/parameter/deprecated/user @@ -0,0 +1 @@ +see manual for details From 8881ff22246afb78e71a7b992fcb032eb4ce1e90 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 21 Jun 2019 12:55:33 +0200 Subject: [PATCH 22/37] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index a79b116d..2548b4ce 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Core: Add support for type parameters deprecation (Darko Poljak) + * Type __acl: Rewrite and improve (Ander Punnar) 5.1.1: 2019-05-28 * Type __apt_key: Use gpg key, fallback to deprecated apt-key (Ander Punnar) From 6915d300153e690e569f3f691d8bdec4952d91a6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 21 Jun 2019 14:13:25 +0200 Subject: [PATCH 23/37] Release 5.1.2 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 2548b4ce..4952991b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +5.1.2: 2019-06-21 * Core: Add support for type parameters deprecation (Darko Poljak) * Type __acl: Rewrite and improve (Ander Punnar) From 5c11c15ae4f7962588f88a9fcdad5bc812d819d0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 23 Jun 2019 19:55:37 +0200 Subject: [PATCH 24/37] Overcome bash CDPATH issue Thanks to Dmitry Bogatov. --- Makefile | 6 +++--- docs/changelog | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b739ab1f..fa3327d1 100644 --- a/Makefile +++ b/Makefile @@ -31,9 +31,9 @@ help: @echo "docs-clean clean documentation" @echo "clean clean" -DOCS_SRC_DIR=docs/src -SPEECHDIR=docs/speeches -TYPEDIR=cdist/conf/type +DOCS_SRC_DIR=./docs/src +SPEECHDIR=./docs/speeches +TYPEDIR=./cdist/conf/type SPHINXM=make -C $(DOCS_SRC_DIR) man SPHINXH=make -C $(DOCS_SRC_DIR) html diff --git a/docs/changelog b/docs/changelog index 4952991b..e4e312fd 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Build: Overcome bash CDPATH when building docs (Dmitry Bogatov) + 5.1.2: 2019-06-21 * Core: Add support for type parameters deprecation (Darko Poljak) * Type __acl: Rewrite and improve (Ander Punnar) From 6bb58f882086f5f46c20355f768ee1095540f4d2 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Wed, 24 Jul 2019 13:32:39 +0200 Subject: [PATCH 25/37] Updated the __grafana_dashboard type for the new package repository - Changed the signing key uri - Changed the repo uri --- cdist/conf/type/__grafana_dashboard/manifest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index 9cd1465d..c1fe9416 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -21,10 +21,10 @@ case $os in __apt_key_uri grafana \ --name 'Grafana Release Signing Key' \ - --uri https://packagecloud.io/gpg.key + --uri https://packages.grafana.com/gpg.key require="$require __apt_key_uri/grafana" __apt_source grafana \ - --uri https://packagecloud.io/grafana/stable/debian/ \ + --uri https://packages.grafana.com/oss/deb \ --distribution $apt_source_distribution \ --component main From a70d2e0af5a7b842d8a1b0ceb03abb90a770e001 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Wed, 24 Jul 2019 13:35:46 +0200 Subject: [PATCH 26/37] Had to change the apt_source_distribution to stable, since the repository doesn't differ in distributions --- cdist/conf/type/__grafana_dashboard/manifest | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index c1fe9416..e652202b 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -8,10 +8,12 @@ case $os in debian|devuan) case $os_version in 8*|jessie) - apt_source_distribution=jessie + # Differntation not needed anymore + apt_source_distribution=stable ;; 9*|ascii/ceres|ascii) - apt_source_distribution=stretch + # Differntation not needed anymore + apt_source_distribution=stable ;; *) echo "Don't know how to install Grafana on $os $os_version. Send us a pull request!" >&2 From f7efde0d0a28cd0e82f6126abcbc7f797a0602b8 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 25 Jul 2019 08:06:58 +0200 Subject: [PATCH 27/37] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index e4e312fd..ae2eac8d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Build: Overcome bash CDPATH when building docs (Dmitry Bogatov) + * Type __grafana_dashboard: Update distribution name, package signing key URI and repository URI (Dominique Roux) 5.1.2: 2019-06-21 * Core: Add support for type parameters deprecation (Darko Poljak) From 031d59c82ccadb931db70bc3c22a013f1104b8e5 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Sun, 4 Aug 2019 21:23:44 +0200 Subject: [PATCH 28/37] Added Devuan support for __docker type --- cdist/conf/type/__docker/manifest | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/cdist/conf/type/__docker/manifest b/cdist/conf/type/__docker/manifest index 04a9ff27..f20337d7 100755 --- a/cdist/conf/type/__docker/manifest +++ b/cdist/conf/type/__docker/manifest @@ -64,6 +64,43 @@ case "$os" in require="__apt_source/docker" __package docker-ce --state "${state}" fi ;; + devuan) + os_version="$(cat "$__global/explorer/os_version")" + + case "$os_version" in + ascii) + distributione="stretch" + ;; + jessie) + distribution="jessie" + ;; + *) + echo "Your devuan release ($os_version) is currently not supported by this type (${__type##*/}).">&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 + ;; + esac + + if [ "${state}" = "present" ]; then + __package apt-transport-https + __package ca-certificates + __package gnupg2 + fi + __apt_key_uri docker --name "Docker Release (CE deb) " \ + --uri "https://download.docker.com/linux/${os}/gpg" --state "${state}" + + require="__apt_key_uri/docker" __apt_source docker \ + --uri "https://download.docker.com/linux/${os}" \ + --distribution "${distribution}" \ + --state "${state}" \ + --component "stable" + if [ "$version" != "latest" ]; then + require="__apt_source/docker" __package docker-ce --version "${version}" --state "${state}" + else + require="__apt_source/docker" __package docker-ce --state "${state}" + fi + + ;; *) echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 echo "Please contribute an implementation for it if you can." >&2 From c6b739b5b6de870e629d421b140587278777b5de Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 4 Aug 2019 21:52:59 +0200 Subject: [PATCH 29/37] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index ae2eac8d..57412999 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Build: Overcome bash CDPATH when building docs (Dmitry Bogatov) * Type __grafana_dashboard: Update distribution name, package signing key URI and repository URI (Dominique Roux) + * Type __docker: Add devuan support (Dominique Roux) 5.1.2: 2019-06-21 * Core: Add support for type parameters deprecation (Darko Poljak) From 2f7dc5a65d264d7f210ab06834980d8782c4a565 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 4 Aug 2019 21:54:13 +0200 Subject: [PATCH 30/37] Fix variable typo --- cdist/conf/type/__docker/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__docker/manifest b/cdist/conf/type/__docker/manifest index f20337d7..6a57d85a 100755 --- a/cdist/conf/type/__docker/manifest +++ b/cdist/conf/type/__docker/manifest @@ -69,7 +69,7 @@ case "$os" in case "$os_version" in ascii) - distributione="stretch" + distribution="stretch" ;; jessie) distribution="jessie" From c0aa2214aa5b34ae1a073682fc3bd6545ceacf36 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 15 Aug 2019 14:19:54 +0200 Subject: [PATCH 31/37] [letsencrypt] add support for devuan/beowulf --- cdist/conf/type/__letsencrypt_cert/manifest | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index d6892c9b..072fa398 100755 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -67,6 +67,9 @@ if [ -z "${certbot_fullpath}" ]; then require="__apt_source/ascii-backports" __package_apt certbot \ --target-release ascii-backports ;; + bewoulf*) + __package_apt certbot + ;; *) echo "Unsupported OS version: $os_version" >&2 exit 1 From 7182de596891f963c6867c1d0b08a5276450c56f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 15 Aug 2019 14:20:14 +0200 Subject: [PATCH 32/37] ++changes --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index ae2eac8d..b13e54fa 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Build: Overcome bash CDPATH when building docs (Dmitry Bogatov) * Type __grafana_dashboard: Update distribution name, package signing key URI and repository URI (Dominique Roux) + * Type __letsencrypt_cert: Add Devuan Beowulf support (Nico Schottelius) 5.1.2: 2019-06-21 * Core: Add support for type parameters deprecation (Darko Poljak) From a86893889b3a9d84712a71a23da60461671ca0e9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 15 Aug 2019 14:24:11 +0200 Subject: [PATCH 33/37] [letsencrypt] devuan/ascii: only install certbot package It seems python-certbot is gone --- cdist/conf/type/__letsencrypt_cert/manifest | 2 -- 1 file changed, 2 deletions(-) diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 072fa398..d598949e 100755 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -62,8 +62,6 @@ if [ -z "${certbot_fullpath}" ]; then --distribution ascii-backports \ --component main - require="__apt_source/ascii-backports" __package_apt python-certbot \ - --target-release ascii-backports require="__apt_source/ascii-backports" __package_apt certbot \ --target-release ascii-backports ;; From 6c780c24c7d1fdf8a316fef98a304c07ffd57d0e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 15 Aug 2019 14:25:16 +0200 Subject: [PATCH 34/37] ++changes --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 6e20c64e..f3b84ba8 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * Build: Overcome bash CDPATH when building docs (Dmitry Bogatov) * Type __grafana_dashboard: Update distribution name, package signing key URI and repository URI (Dominique Roux) * Type __letsencrypt_cert: Add Devuan Beowulf support (Nico Schottelius) + * Type __letsencrypt_cert: Fix Devuan Ascii: support (Nico Schottelius) * Type __docker: Add devuan support (Dominique Roux) 5.1.2: 2019-06-21 From 451dfaffe49fc85f8222da19177b7aa05c864f56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=BDubom=C3=ADr=20Ku=C4=8Dera?= Date: Fri, 23 Aug 2019 11:28:13 +0200 Subject: [PATCH 35/37] __docker_swarm: Fix for Docker 19.03 --- cdist/conf/type/__docker_swarm/explorer/swarm-state | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__docker_swarm/explorer/swarm-state b/cdist/conf/type/__docker_swarm/explorer/swarm-state index 9c1bc32d..2c9fd598 100755 --- a/cdist/conf/type/__docker_swarm/explorer/swarm-state +++ b/cdist/conf/type/__docker_swarm/explorer/swarm-state @@ -18,4 +18,4 @@ # along with cdist. If not, see . # -docker info 2>/dev/null | grep "^Swarm: " | cut -d " " -f 2- +docker info 2>/dev/null | grep '^ *Swarm: ' | awk '{print $2}' From 65b9e1d00f230ca97ef30546e1490a8b51e83c19 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 29 Aug 2019 14:40:16 +0200 Subject: [PATCH 36/37] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index f3b84ba8..b790b983 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * Type __letsencrypt_cert: Add Devuan Beowulf support (Nico Schottelius) * Type __letsencrypt_cert: Fix Devuan Ascii: support (Nico Schottelius) * Type __docker: Add devuan support (Dominique Roux) + * Type __docker_swarm: Fix for Docker 19.03 (Ľubomír Kučera) 5.1.2: 2019-06-21 * Core: Add support for type parameters deprecation (Darko Poljak) From 067d0a62e73d89fef35e1f7183a3b376634da5c0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 30 Aug 2019 08:51:08 +0200 Subject: [PATCH 37/37] Release 5.1.3 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index b790b983..7594a6d4 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +5.1.3: 2019-08-30 * Build: Overcome bash CDPATH when building docs (Dmitry Bogatov) * Type __grafana_dashboard: Update distribution name, package signing key URI and repository URI (Dominique Roux) * Type __letsencrypt_cert: Add Devuan Beowulf support (Nico Schottelius)