From 64c247026a4fe02db1d31e2bd45839f577c7c1bb Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 30 Sep 2019 14:20:26 +0200 Subject: [PATCH 001/155] [__locale_system] Support Devuan --- cdist/conf/type/__locale_system/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest index 80f7401b..4a1fdeed 100755 --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -26,7 +26,7 @@ os=$(cat "$__global/explorer/os") case "$os" in - debian|ubuntu) + debian|devuan|ubuntu) locale_conf="/etc/default/locale" ;; archlinux) From fefc828780bc76c4245fa96ef517a66ec3eda3fa Mon Sep 17 00:00:00 2001 From: Evilham Date: Sun, 26 Apr 2020 19:06:42 +0200 Subject: [PATCH 002/155] [docs] Improve cdist.cfg.skeleton --- configuration/cdist.cfg.skeleton | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton index 22c1ccaf..bfac9f5c 100644 --- a/configuration/cdist.cfg.skeleton +++ b/configuration/cdist.cfg.skeleton @@ -19,6 +19,9 @@ # such as ':' for POSIX or ';' for Windows. # If also specified at command line then values from command line are # appended to this value. +# Notice that this works in a "last one wins" fashion, so if a type is redefined +# in multiple conf_dirs, the last one in which it is defined will be used. +# Consider using a unique prefix for your own roles if this can be an issue. # conf_dir = : # # init_manifest From 678df1ec8a24b4c3884554b1931445435e098787 Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 27 Apr 2020 01:23:48 +0200 Subject: [PATCH 003/155] [explorers] Improve *BSD support. cpu_cores and memory did lacked support for other BSDs. --- cdist/conf/explorer/cpu_cores | 4 ++++ cdist/conf/explorer/memory | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/cpu_cores b/cdist/conf/explorer/cpu_cores index a52bddac..c6744142 100755 --- a/cdist/conf/explorer/cpu_cores +++ b/cdist/conf/explorer/cpu_cores @@ -32,6 +32,10 @@ case "$os" in sysctl -n hw.ncpuonline ;; + "freebsd"|"netbsd") + sysctl -n hw.ncpu + ;; + *) if [ -r /proc/cpuinfo ]; then cores="$(grep "core id" /proc/cpuinfo | sort | uniq | wc -l)" diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory index 4e3efff8..302b4cda 100755 --- a/cdist/conf/explorer/memory +++ b/cdist/conf/explorer/memory @@ -29,7 +29,7 @@ case "$os" in echo "$(sysctl -n hw.memsize)/1024" | bc ;; - "openbsd") + *"bsd") echo "$(sysctl -n hw.physmem) / 1048576" | bc ;; From 0b3c417aef13eceb51195bb42e9a17205a9afc6c Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Mon, 27 Apr 2020 14:00:39 +0300 Subject: [PATCH 004/155] update README --- README | 7 ------- README.md | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 7 deletions(-) delete mode 100644 README create mode 100644 README.md diff --git a/README b/README deleted file mode 100644 index caf2dac8..00000000 --- a/README +++ /dev/null @@ -1,7 +0,0 @@ -cdist ------ - -cdist is a usable configuration management system. - -For the web documentation have a look at https://www.cdi.st/ -or at docs/src for reStructuredText manual. diff --git a/README.md b/README.md new file mode 100644 index 00000000..9e49b053 --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +# cdist + +**cdist** is a usable configuration management system. + +It adheres to the [**KISS principle**](https://en.wikipedia.org/wiki/KISS_principle) +and is being used in small up to enterprise grade environments. + +For more information have a look at [**homepage**](https://cdi.st) +or at **``docs/src``** for manual in **reStructuredText** format. + +## Contributing + +Merge/Pull requests can be made in both +[upstream **GitLab**](https://code.ungleich.ch/ungleich-public/cdist/merge_requests) +(managed by [**ungleich**](https://ungleich.ch)) +and [**GitHub** project](https://github.com/ungleich/cdist/pulls). + +Issues can be made and other project management activites happen +[**only in GitLab**](https://code.ungleich.ch/ungleich-public/cdist) +(needs [**ungleich** account](https://account.ungleich.ch)). From 56a65518ab6171d0f19152642fccb94038a3c5ad Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Mon, 27 Apr 2020 15:25:43 +0300 Subject: [PATCH 005/155] README: add participating section --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 9e49b053..0a0d6e6d 100644 --- a/README.md +++ b/README.md @@ -18,3 +18,11 @@ and [**GitHub** project](https://github.com/ungleich/cdist/pulls). Issues can be made and other project management activites happen [**only in GitLab**](https://code.ungleich.ch/ungleich-public/cdist) (needs [**ungleich** account](https://account.ungleich.ch)). + +## Participating + +IRC: ``#cdist`` @ freenode + +Matrix: ``#cdist:ungleich.ch`` + +Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist From b31e13eacf2cb23e6a7bdadc33741026ae88553a Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Mon, 27 Apr 2020 16:30:52 +0300 Subject: [PATCH 006/155] README: add bits about cdist-contrib --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 0a0d6e6d..de6901c7 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,9 @@ Issues can be made and other project management activites happen [**only in GitLab**](https://code.ungleich.ch/ungleich-public/cdist) (needs [**ungleich** account](https://account.ungleich.ch)). +For community-maintained types there is +[**cdist-contrib** project](https://code.ungleich.ch/ungleich-public/cdist-contrib). + ## Participating IRC: ``#cdist`` @ freenode From 515992249de513492a725dbf4072a6c3f376668a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 27 Apr 2020 22:55:57 +0200 Subject: [PATCH 007/155] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 7cacf3cf..1e213187 100644 --- a/docs/changelog +++ b/docs/changelog @@ -10,6 +10,8 @@ next: * New type: __pf_apply_anchor (Kamila Součková, Evil Ham) * Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham) * Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham) + * Configuration: Add notes to cdist.cfg.skeleton (Evil Ham) + * Explorers cpu_cores, memory: Improve *BSD support (Evil Ham) 6.5.4: 2020-04-11 * Explorer init: Do not grep on non-existent init (Steven Armstrong) From ea3bd14d8b377818a16578bd5032a853188baeec Mon Sep 17 00:00:00 2001 From: Evilham Date: Tue, 28 Apr 2020 14:54:51 +0200 Subject: [PATCH 008/155] [logging] Mute warning on return_output=True when running scripts. This fixes #806 which contains more information about the issue. The TL;DR: this warning is not being useful and hinders debugging types because it creates an innecessary line for each explorer. An alternative proposal was #807 but was abandoned in favour of just dropping the warning. --- cdist/exec/remote.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index e0ef66ec..f72bf3bf 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -280,9 +280,6 @@ class Remote(object): assert isinstance(command, (list, tuple)), ( "list or tuple argument expected, got: %s" % command) - if return_output and stdout is not subprocess.PIPE: - self.log.debug("return_output is True, ignoring stdout") - close_stdout = False close_stderr = False if self.save_output_streams: From 250161e42d1fe32bcb7b6945014aa4786d3b46ed Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 28 Apr 2020 23:08:03 +0200 Subject: [PATCH 009/155] ++ --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 1e213187..c29f70de 100644 --- a/docs/changelog +++ b/docs/changelog @@ -12,6 +12,7 @@ next: * Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham) * Configuration: Add notes to cdist.cfg.skeleton (Evil Ham) * Explorers cpu_cores, memory: Improve *BSD support (Evil Ham) + * Core: Remove debug logging noise (Evil Ham) 6.5.4: 2020-04-11 * Explorer init: Do not grep on non-existent init (Steven Armstrong) From 310045d9fb5a5a63a337867a90009f89e0ded5ed Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 1 May 2020 13:02:00 +0200 Subject: [PATCH 010/155] Release 6.5.5 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index c29f70de..e6a7927e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.5.5: 2020-05-01 * Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches) * Type __postgres_database: Add encoding, lc-collate, lc-ctype, template parameters (Timothée Floure) * Type __motd: Improve documentation and support for FreeBSD (Evil Ham) From f58d662b32fb9c1c226b5340ccd06c6dd3e2a3f9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 1 May 2020 15:28:01 +0200 Subject: [PATCH 011/155] [__pyvenv] Switch to python3 -m venv for ubuntu --- cdist/conf/type/__pyvenv/gencode-remote | 3 ++- cdist/conf/type/__pyvenv/man.rst | 5 ++--- docs/changelog | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__pyvenv/gencode-remote b/cdist/conf/type/__pyvenv/gencode-remote index 9c7b7fab..c5b64eff 100755 --- a/cdist/conf/type/__pyvenv/gencode-remote +++ b/cdist/conf/type/__pyvenv/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2016 Darko Poljak (darko.poljak at gmail.com) +# 2020 Nico Schotetlius (nico.schottelius at ungleich.ch) # # This file is part of cdist. # @@ -45,7 +46,7 @@ then pyvenv=$(cat "$pyvenvparam") else case "$os" in - alpine) # no pyvenv on alpine - I assume others will follow + alpine|ubuntu) # no pyvenv on alpine - I assume others will follow pyvenv="python3 -m venv" ;; *) diff --git a/cdist/conf/type/__pyvenv/man.rst b/cdist/conf/type/__pyvenv/man.rst index d7de92fa..8085ff12 100644 --- a/cdist/conf/type/__pyvenv/man.rst +++ b/cdist/conf/type/__pyvenv/man.rst @@ -9,7 +9,7 @@ cdist-type__pyvenv - Create or remove python virtual environment DESCRIPTION ----------- This cdist type allows you to create or remove python virtual -environment using pyvenv. +environment using pyvenv on python3 -m venv. It assumes pyvenv is already installed. Concrete package depends on concrete OS and/or OS version/distribution. Ensure this for e.g. in your init manifest as in the following example: @@ -57,7 +57,7 @@ EXAMPLES __pyvenv /home/services/djangoenv - # Use specific pyvenv + # Use specific pyvenv __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4 # Create python virtualenv for user foo. @@ -76,4 +76,3 @@ COPYING ------- Copyright \(C) 2016 Darko Poljak. Free use of this software is granted under the terms of the GNU General Public License v3 or later (GPLv3+). - diff --git a/docs/changelog b/docs/changelog index e6a7927e..4ee47d37 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,10 @@ Changelog --------- +next: + * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) + + 6.5.5: 2020-05-01 * Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches) * Type __postgres_database: Add encoding, lc-collate, lc-ctype, template parameters (Timothée Floure) From d4059fd29ecb93e094c411bb8ca09a234904b763 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 1 May 2020 15:31:23 +0200 Subject: [PATCH 012/155] [__letsencrypt_cert] whitelist Ubuntu --- cdist/conf/type/__letsencrypt_cert/manifest | 3 +++ docs/changelog | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 68ecf9d4..b4464366 100755 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -91,6 +91,9 @@ if [ -z "${certbot_fullpath}" ]; then certbot_fullpath=/usr/local/bin/certbot ;; + ubuntu) + __package certbot + ;; *) echo "Unsupported os: $os" >&2 exit 1 diff --git a/docs/changelog b/docs/changelog index 4ee47d37..e61933ab 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,7 +3,7 @@ Changelog next: * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) - + * Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius) 6.5.5: 2020-05-01 * Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches) From 6f4649efc69593fb4178fb6c2c5c95bbd93054d1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 8 May 2020 16:08:21 +0200 Subject: [PATCH 013/155] Reference the new cdist chat on matrix --- docs/src/cdist-support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/cdist-support.rst b/docs/src/cdist-support.rst index 19afde2f..f9f61f01 100644 --- a/docs/src/cdist-support.rst +++ b/docs/src/cdist-support.rst @@ -3,7 +3,7 @@ Support Chat ~~~~ -Chat with us: `ungleich chat `_. +Chat with us on `#cdist:ungleich.ch `_. Mailing list ~~~~~~~~~~~~ From 42f2dceeb12fee55616202bf085e6823d18ee714 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 9 May 2020 18:40:44 +0200 Subject: [PATCH 014/155] [__link] fix typo in the manual --- cdist/conf/type/__link/man.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__link/man.rst b/cdist/conf/type/__link/man.rst index fe0ce425..2e81aea9 100644 --- a/cdist/conf/type/__link/man.rst +++ b/cdist/conf/type/__link/man.rst @@ -18,7 +18,7 @@ source Specifies the link source. type - Specifies the link type: Either hard or symoblic. + Specifies the link type: Either hard or symbolic. OPTIONAL PARAMETERS From dcfabf9268865794602d3e59af2f312a1b0392f0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 16 May 2020 14:00:02 +0200 Subject: [PATCH 015/155] [__cron] add hint for default values of time-related parameters This should resolve some misunderstanding when leave a time-related parameter with it's default value. --- cdist/conf/type/__cron/man.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/type/__cron/man.rst b/cdist/conf/type/__cron/man.rst index d0694738..e39bfb5c 100644 --- a/cdist/conf/type/__cron/man.rst +++ b/cdist/conf/type/__cron/man.rst @@ -21,6 +21,11 @@ command OPTIONAL PARAMETERS ------------------- +**NOTE**: All time-related parameters (``--minute``, ``--hour``, ``--day_of_month`` +``--month`` and ``--day_of_week``) defaults to ``*``, which means to execute it +**always**. If you set ``--hour 0`` to execute the cronjob only at midnight, it +will execute **every** minute in the first hour of the morning all days. + state Either present or absent. Defaults to present. minute From 086e683c997729817300c600307592c6fc02ffd8 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 16 May 2020 16:33:38 +0200 Subject: [PATCH 016/155] [__file] add "default values" in manual for group, mode and owner Because at least the --mode default value may not be expected, the manual lists the "default values". As they are not explicitly set in the `__file` type, it is a bit more difficult, but should resolve to following: - the mode comes from the umask set in the cdist code to protect file creation and uses the strongest umask possible. - the owner and group comes from the ssh user, which should always be the root user. (I think this can be swaped, too, but who will?) At the owner and group parameter, it could be replaced with something like "the ssh user and group", which would be more correct, but less understandable. --- cdist/conf/type/__file/man.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__file/man.rst b/cdist/conf/type/__file/man.rst index 7a0603bb..2f3b9e69 100644 --- a/cdist/conf/type/__file/man.rst +++ b/cdist/conf/type/__file/man.rst @@ -50,13 +50,13 @@ state create or modify it group - Group to chgrp to. + Group to chgrp to. Defaults to ``root``. mode - Unix permissions, suitable for chmod. + Unix permissions, suitable for chmod. Defaults to a very secure ``0600``. owner - User to chown to. + User to chown to. Defaults to ``root``. source If supplied, copy this file from the host running cdist to the target. From 7ca2bfc14a8ed0cb73b3a01c85a562060264d02b Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 18 May 2020 16:00:23 +0200 Subject: [PATCH 017/155] [explorer/machine_type] Add support for FreeBSD. More research is needed for {Net,Open}BSD support. Indentation is left as-is for the linux code as I intend to simplify it in a future MR, this way the diff is minimal. --- cdist/conf/explorer/machine_type | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index bb21f69c..fe0ae7d5 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -2,6 +2,7 @@ # # 2014 Daniel Heule (hda at sfs.biz) # 2014 Thomas Oettli (otho at sfs.biz) +# 2020 Evilham (contact at evilham.com) # # This file is part of cdist. # @@ -18,9 +19,27 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# -# FIXME: other system types (not linux ...) +os=$("$__explorer/os") + +case "$os" in + "freebsd") + # FreeBSD does not have /proc/cpuinfo even when procfs is used. + # Instead there is a sysctl kern.vm_guest. + # Which is 'none' if physical, else the virtualisation. + vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)" + if [ -n "${vm_guest}" ]; then + if [ "${vm_guest}" = "none" ]; then + echo "physical" + exit + fi + echo "virtual_by_${vm_guest}" + exit + fi + ;; + + *) + # Defaulting to linux for compatibility with previous cdist behaviour if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then echo openvz @@ -72,9 +91,13 @@ if [ -r /proc/cpuinfo ]; then fi fi echo "virtual_by_unknown" + exit else echo "physical" + exit fi -else - echo "unknown" fi + ;; +esac + +echo "unknown" From cf44c4a01bcfba95e11ba8d6aa64156c5470adc1 Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 18 May 2020 16:35:50 +0200 Subject: [PATCH 018/155] [explorer/machine_type] Simplify Linux + basic OpenBSD support. By abstracting away vendor-dependent pattern matching for the linux code, we can re-use that and be reasonably sure about OpenBSD machines being virtualised when we can identify the undelrying virtualisation technology. It remains to be solved how to tell if an OpenBSD machine is physical; in that case previous cdist behaviour ("unknown") remains. For NetBSD something similar to OpenBSD could be done, with different sysctls: hw.machine and hw.model wary of adding those without testing though, so for NetBSD previous cdist behaviour ("unknown") remains. https://netbsd.gw.com/cgi-bin/man-cgi?sysctl+7.i386+NetBSD-9.0 --- cdist/conf/explorer/machine_type | 116 ++++++++++++++++--------------- 1 file changed, 61 insertions(+), 55 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index fe0ae7d5..6800234d 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -22,6 +22,18 @@ os=$("$__explorer/os") +vendor_string_to_machine_type() { + for vendor in vmware bochs kvm qemu virtualbox bhyve; do + if echo "${1}" | grep -q -i "${vendor}"; then + if [ "${vendor}" = "bochs" -o "${vendor}" = "qemu" ]; then + vendor="kvm" + fi + echo "virtual_by_${vendor}" + exit + fi + done +} + case "$os" in "freebsd") # FreeBSD does not have /proc/cpuinfo even when procfs is used. @@ -38,65 +50,59 @@ case "$os" in fi ;; + "openbsd") + # OpenBSD can also use the sysctl's: hw.vendor or hw.product. + # Note we can be reasonably sure about a machine being virtualised + # as long as we can identify the virtualisation technology. + # But not so much about it being physical... + # Patches are welcome / reach out if you have better ideas. + for sysctl in hw.vendor hw.product; do + # This exits if we can make a reasonable judgement + vendor_string_to_machine_type "$(sysctl -n "${sysctl}")" + done + ;; + *) # Defaulting to linux for compatibility with previous cdist behaviour -if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then - echo openvz - exit -fi - -if [ -e "/proc/1/environ" ] && - tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then - echo lxc - exit -fi - -if [ -r /proc/cpuinfo ]; then - # this should only exist on virtual guest machines, - # tested on vmware, xen, kvm - if grep -q "hypervisor" /proc/cpuinfo; then - # this file is aviable in xen guest systems - if [ -r /sys/hypervisor/type ]; then - if grep -q -i "xen" /sys/hypervisor/type; then - echo virtual_by_xen - exit - fi - else - if [ -r /sys/class/dmi/id/product_name ]; then - if grep -q -i 'vmware' /sys/class/dmi/id/product_name; then - echo "virtual_by_vmware" - exit - elif grep -q -i 'bochs' /sys/class/dmi/id/product_name; then - echo "virtual_by_kvm" - exit - elif grep -q -i 'virtualbox' /sys/class/dmi/id/product_name; then - echo "virtual_by_virtualbox" - exit - fi - fi - - if [ -r /sys/class/dmi/id/sys_vendor ]; then - if grep -q -i 'qemu' /sys/class/dmi/id/sys_vendor; then - echo "virtual_by_kvm" - exit - fi - fi - - if [ -r /sys/class/dmi/id/chassis_vendor ]; then - if grep -q -i 'qemu' /sys/class/dmi/id/chassis_vendor; then - echo "virtual_by_kvm" - exit - fi - fi + if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then + echo openvz + exit + fi + + if [ -e "/proc/1/environ" ] && + tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then + echo lxc + exit + fi + + if [ -r /proc/cpuinfo ]; then + # this should only exist on virtual guest machines, + # tested on vmware, xen, kvm, bhyve + if grep -q "hypervisor" /proc/cpuinfo; then + # this file is aviable in xen guest systems + if [ -r /sys/hypervisor/type ]; then + if grep -q -i "xen" /sys/hypervisor/type; then + echo virtual_by_xen + exit + fi + else + for vendor_file in /sys/class/dmi/id/product_name \ + /sys/class/dmi/id/sys_vendor \ + /sys/class/dmi/id/chasis_vendor; do + if [ -r ${vendor_file} ]; then + # This exits if we can make a reasonable judgement + vendor_string_to_machine_type "$(cat "${vendor_file}")" + fi + done + fi + echo "virtual_by_unknown" + exit + else + echo "physical" + exit + fi fi - echo "virtual_by_unknown" - exit - else - echo "physical" - exit - fi -fi ;; esac From 8b790b0a54aecf93298c97948c48be80b67ca7b5 Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 18 May 2020 16:47:20 +0200 Subject: [PATCH 019/155] [explorer/machine_type] Make shellcheck happy! --- cdist/conf/explorer/machine_type | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 6800234d..1c84f4d7 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -25,7 +25,7 @@ os=$("$__explorer/os") vendor_string_to_machine_type() { for vendor in vmware bochs kvm qemu virtualbox bhyve; do if echo "${1}" | grep -q -i "${vendor}"; then - if [ "${vendor}" = "bochs" -o "${vendor}" = "qemu" ]; then + if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then vendor="kvm" fi echo "virtual_by_${vendor}" From 6d502f737aac8a836949d2b4b8a55c8ba20161fa Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 18 May 2020 18:37:16 +0200 Subject: [PATCH 020/155] [__ssh_authorizedkey{,s}] Improve documentation. --- cdist/conf/type/__ssh_authorized_key/man.rst | 16 +++++----- cdist/conf/type/__ssh_authorized_keys/man.rst | 29 ++++++++++--------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/cdist/conf/type/__ssh_authorized_key/man.rst b/cdist/conf/type/__ssh_authorized_key/man.rst index 087a3dae..5bae02aa 100644 --- a/cdist/conf/type/__ssh_authorized_key/man.rst +++ b/cdist/conf/type/__ssh_authorized_key/man.rst @@ -15,25 +15,27 @@ This type was created to be used by the __ssh_authorized_keys type. REQUIRED PARAMETERS ------------------- file - the authorized_keys file to which the given key should be added + The authorized_keys file where the given key should be managed. key - a string containing the ssh keytype, base 64 encoded key and optional - trailing comment which shall be added to the given authorized_keys file. + The ssh key which shall be managed in this authorized_keys file. + Must be a string containing the ssh keytype, base 64 encoded key and + optional trailing comment which shall be added to the given + authorized_keys file. OPTIONAL PARAMETERS ------------------- comment - explicit comment instead of the one which may be trailing the given key + Use this comment instead of the one which may be trailing in the key. option - an option to set for this authorized_key entry. + An option to set for this authorized_key entry. Can be specified multiple times. See sshd(8) for available options. state - if the given keys should be 'present' or 'absent', defaults to 'present'. + If the managed key should be 'present' or 'absent', defaults to 'present'. MESSAGES @@ -64,7 +66,7 @@ EXAMPLES SEE ALSO -------- -:strong:`cdist__ssh_authorized_keys`\ (7), :strong:`sshd`\ (8) +:strong:`cdist-type__ssh_authorized_keys`\ (7), :strong:`sshd`\ (8) AUTHORS diff --git a/cdist/conf/type/__ssh_authorized_keys/man.rst b/cdist/conf/type/__ssh_authorized_keys/man.rst index ba310ff9..93357b1d 100644 --- a/cdist/conf/type/__ssh_authorized_keys/man.rst +++ b/cdist/conf/type/__ssh_authorized_keys/man.rst @@ -20,42 +20,45 @@ then left to the user to ensure that the file exists and that ownership and permissions work with ssh. -REQUIRED PARAMETERS -------------------- +REQUIRED MULTIPLE PARAMETERS +---------------------------- key - the ssh key which shall be added to this authorized_keys file. - Must be a string and can be specified multiple times. + An ssh key which shall be managed in this authorized_keys file. + Must be a string containing the ssh keytype, base 64 encoded key and + optional trailing comment which shall be added to the given + authorized_keys file. + Can be specified multiple times. OPTIONAL PARAMETERS ------------------- comment - explicit comment instead of the one which may be trailing the given key + Use this comment instead of the one which may be trailing in each key. file - an alternative destination file, defaults to ~$owner/.ssh/authorized_keys + An alternative destination file, defaults to ~$owner/.ssh/authorized_keys. option - an option to set for all created authorized_key entries. + An option to set for all authorized_key entries in the key parameter. Can be specified multiple times. See sshd(8) for available options. owner - the user owning the authorized_keys file, defaults to object_id. + The user owning the authorized_keys file, defaults to object_id. state - if the given keys should be 'present' or 'absent', defaults to 'present'. + If the given keys should be 'present' or 'absent', defaults to 'present'. BOOLEAN PARAMETERS ------------------ noparent - don't create or change ownership and permissions of the directory containing - the authorized_keys file + Don't create or change ownership and permissions of the directory containing + the authorized_keys file. nofile - don't manage existence, ownership and permissions of the the authorized_keys - file + Don't manage existence, ownership and permissions of the the authorized_keys + file. EXAMPLES From f9afac4dd617c5ee7934cdb03477e6a30ae70af7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 18 May 2020 19:51:08 +0200 Subject: [PATCH 021/155] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index e61933ab..013e0083 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,9 @@ Changelog next: * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) * Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius) + * Types __cron, __file, __link: Improve manpages (Matthias Stecher) + * Explorer machine_type: Add support for FreeBSD and OpenBSD, and simplify Linux code (Evil Ham) + * Type __ssh_authorized_key, __ssh_authorized_keys: Improve manpages (Evil Ham) 6.5.5: 2020-05-01 * Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches) From ad58ea79c2cd3edd11a828d26a28975aa332bdd0 Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 18 May 2020 20:01:01 +0200 Subject: [PATCH 022/155] [__ssh_authorized_keys] Fix bug where --option was not multiple This went against both documentation and intent. --- cdist/conf/type/__ssh_authorized_keys/parameter/optional | 1 - .../conf/type/__ssh_authorized_keys/parameter/optional_multiple | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 cdist/conf/type/__ssh_authorized_keys/parameter/optional_multiple diff --git a/cdist/conf/type/__ssh_authorized_keys/parameter/optional b/cdist/conf/type/__ssh_authorized_keys/parameter/optional index 21f9bc29..fa64fc43 100644 --- a/cdist/conf/type/__ssh_authorized_keys/parameter/optional +++ b/cdist/conf/type/__ssh_authorized_keys/parameter/optional @@ -1,5 +1,4 @@ comment file -option owner state diff --git a/cdist/conf/type/__ssh_authorized_keys/parameter/optional_multiple b/cdist/conf/type/__ssh_authorized_keys/parameter/optional_multiple new file mode 100644 index 00000000..01925a15 --- /dev/null +++ b/cdist/conf/type/__ssh_authorized_keys/parameter/optional_multiple @@ -0,0 +1 @@ +option From f354d80308b1f048416b2d52a397035a93a8ee03 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 18 May 2020 21:03:29 +0200 Subject: [PATCH 023/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 013e0083..782d9bd7 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * Types __cron, __file, __link: Improve manpages (Matthias Stecher) * Explorer machine_type: Add support for FreeBSD and OpenBSD, and simplify Linux code (Evil Ham) * Type __ssh_authorized_key, __ssh_authorized_keys: Improve manpages (Evil Ham) + * Type __ssh_authorized_keys: Fix bug where --option was not multiple (Evil Ham) 6.5.5: 2020-05-01 * Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches) From bf25a18a048ae3e3d1fd4a450d77fe516c18c861 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 22 May 2020 02:31:38 +0300 Subject: [PATCH 024/155] [__group] fix --gid on freebsd --- cdist/conf/type/__group/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__group/gencode-remote b/cdist/conf/type/__group/gencode-remote index 6091c548..ff63e218 100755 --- a/cdist/conf/type/__group/gencode-remote +++ b/cdist/conf/type/__group/gencode-remote @@ -88,7 +88,7 @@ if [ "$state" = "present" ]; then fi done if [ "$os" = "freebsd" ]; then - echo pw groupadd "$@" "$name" + echo pw groupadd "$name" "$@" else echo groupadd "$@" "$name" fi From 3bcbd952698cd5a4b8333551660ba308e4f30d7b Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 22 May 2020 02:36:49 +0300 Subject: [PATCH 025/155] [__motd] debian|ubuntu|devuan use /etc/motd --- cdist/conf/type/__motd/gencode-remote | 7 ------- cdist/conf/type/__motd/manifest | 4 ---- 2 files changed, 11 deletions(-) diff --git a/cdist/conf/type/__motd/gencode-remote b/cdist/conf/type/__motd/gencode-remote index 738ea834..cb7bfc84 100755 --- a/cdist/conf/type/__motd/gencode-remote +++ b/cdist/conf/type/__motd/gencode-remote @@ -22,13 +22,6 @@ os=$(cat "$__global/explorer/os") case "$os" in - debian|ubuntu|devuan) - - # Debian and Ubuntu need to be updated, - # as seen in /etc/init.d/bootlogs - echo "uname -snrvm > /var/run/motd" - echo "cat /etc/motd.tail >> /var/run/motd" - ;; freebsd) # FreeBSD only updates /etc/motd on boot, # as seen in /etc/rc.d/motd diff --git a/cdist/conf/type/__motd/manifest b/cdist/conf/type/__motd/manifest index ded734d7..b8f74ebf 100755 --- a/cdist/conf/type/__motd/manifest +++ b/cdist/conf/type/__motd/manifest @@ -33,10 +33,6 @@ os=$(cat "$__global/explorer/os") case "$os" in - debian|ubuntu|devuan) - # Debian-based systems use /etc/motd.tail as a template - destination=/etc/motd.tail - ;; freebsd) # FreeBSD uses motd.template to prepend system information on boot # (this actually only applies starting with version 13, From 226ed02c1c2556ed30e4f8aaf1946099bd590725 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 22 May 2020 10:09:05 +0200 Subject: [PATCH 026/155] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 782d9bd7..f0aa3aa7 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,8 @@ next: * Explorer machine_type: Add support for FreeBSD and OpenBSD, and simplify Linux code (Evil Ham) * Type __ssh_authorized_key, __ssh_authorized_keys: Improve manpages (Evil Ham) * Type __ssh_authorized_keys: Fix bug where --option was not multiple (Evil Ham) + * Type __motd: Debian/Ubuntu/Devuan use /etc/motd (Ander Punnar) + * Type __group: Fix --gid on FreeBSD (Ander Punnar) 6.5.5: 2020-05-01 * Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches) From 66f4421089660fc0a083abe3d2c93470eacd55ce Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Fri, 22 May 2020 17:14:29 +0300 Subject: [PATCH 027/155] [docs] Fixed two typos in cdist.cfg.skeleton Signed-off-by: Jaak Ristioja --- configuration/cdist.cfg.skeleton | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton index bfac9f5c..91c5ab02 100644 --- a/configuration/cdist.cfg.skeleton +++ b/configuration/cdist.cfg.skeleton @@ -26,7 +26,7 @@ # # init_manifest # Specify default initial manifest. -# init_mainfest = +# init_manifest = # # inventory_dir # Specify inventory directory. From 716d3554f3dd9a7cf83cb0fd4f706b6458cb13d8 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sat, 23 May 2020 00:48:35 +0300 Subject: [PATCH 028/155] [__user] fix user delete on freebsd --- cdist/conf/type/__user/gencode-remote | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__user/gencode-remote b/cdist/conf/type/__user/gencode-remote index ee18c18f..41c3a57b 100755 --- a/cdist/conf/type/__user/gencode-remote +++ b/cdist/conf/type/__user/gencode-remote @@ -135,11 +135,19 @@ elif [ "$state" = "absent" ]; then if grep -q "^${name}:" "$__object/explorer/passwd"; then #user exists, but state != present, so delete it if [ -f "$__object/parameter/remove-home" ]; then - printf "userdel -r '%s' >/dev/null 2>&1\\n" "${name}" - echo "userdel -r" >> "$__messages_out" + if [ "$os" = "freebsd" ]; then + printf "pw userdel '%s' -r >/dev/null 2>&1\\n" "${name}" + else + printf "userdel -r '%s' >/dev/null 2>&1\\n" "${name}" + fi + echo "userdel -r" >> "$__messages_out" else - printf "userdel '%s' >/dev/null 2>&1\\n" "${name}" - echo "userdel" >> "$__messages_out" + if [ "$os" = "freebsd" ]; then + printf "pw userdel '%s' >/dev/null 2>&1\\n" "${name}" + else + printf "userdel '%s' >/dev/null 2>&1\\n" "${name}" + fi + echo "userdel" >> "$__messages_out" fi fi else From 6ba73c4be6a57007c035f748c3273dd7a7041992 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 23 May 2020 19:03:13 +0200 Subject: [PATCH 029/155] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index f0aa3aa7..6746bec6 100644 --- a/docs/changelog +++ b/docs/changelog @@ -10,6 +10,8 @@ next: * Type __ssh_authorized_keys: Fix bug where --option was not multiple (Evil Ham) * Type __motd: Debian/Ubuntu/Devuan use /etc/motd (Ander Punnar) * Type __group: Fix --gid on FreeBSD (Ander Punnar) + * Configuration: Fix typos in cdist.cfg.skeleton (Jaak Ristioja) + * Type __user: Fix user deletion on FreeBSD (Ander Punnar) 6.5.5: 2020-05-01 * Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches) From f4e1bbc87e9127ed4ac1789c700e1cbee02ff61c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 24 May 2020 01:10:56 +0200 Subject: [PATCH 030/155] Fix multiple log lines Fixes #813. --- cdist/log.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/log.py b/cdist/log.py index 5d431130..2d0bef0b 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -64,6 +64,7 @@ class DefaultLog(logging.Logger): def __init__(self, name): super().__init__(name) + self.propagate = False formatter = logging.Formatter(self.FORMAT) From ba64971a567722c29cfe5a93c7ccc7e03bbf7940 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 24 May 2020 17:05:01 +0200 Subject: [PATCH 031/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 6746bec6..5a2f905a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -12,6 +12,7 @@ next: * Type __group: Fix --gid on FreeBSD (Ander Punnar) * Configuration: Fix typos in cdist.cfg.skeleton (Jaak Ristioja) * Type __user: Fix user deletion on FreeBSD (Ander Punnar) + * Core: Fix double log lines (Darko Poljak) 6.5.5: 2020-05-01 * Core: Fix XDG_CONFIG_HOME config file location (Joachim Desroches) From abac79d4a50857c81e12660aaa13424d371f2137 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 May 2020 11:16:48 +0200 Subject: [PATCH 032/155] Release 6.5.6 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 5a2f905a..ad008cb2 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) * Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius) * Types __cron, __file, __link: Improve manpages (Matthias Stecher) From 29c0180204beced2813f2d1ec2b50ee069d205c6 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 28 May 2020 23:31:13 +0300 Subject: [PATCH 033/155] [__ssh_authorized_keys] add --remove-unknown parameter --- .../type/__ssh_authorized_keys/explorer/keys | 8 ++++++ cdist/conf/type/__ssh_authorized_keys/man.rst | 9 +++++++ .../conf/type/__ssh_authorized_keys/manifest | 27 ++++++++++++++++++- .../__ssh_authorized_keys/parameter/boolean | 1 + 4 files changed, 44 insertions(+), 1 deletion(-) create mode 100755 cdist/conf/type/__ssh_authorized_keys/explorer/keys diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys new file mode 100755 index 00000000..b515c630 --- /dev/null +++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys @@ -0,0 +1,8 @@ +#!/bin/sh -e + +file="$( . "$__type_explorer/file" )" + +if [ -f "$file" ] +then + cat "$file" +fi diff --git a/cdist/conf/type/__ssh_authorized_keys/man.rst b/cdist/conf/type/__ssh_authorized_keys/man.rst index 93357b1d..dac6adeb 100644 --- a/cdist/conf/type/__ssh_authorized_keys/man.rst +++ b/cdist/conf/type/__ssh_authorized_keys/man.rst @@ -60,6 +60,9 @@ nofile Don't manage existence, ownership and permissions of the the authorized_keys file. +remove-unknown + Remove undefined keys. + EXAMPLES -------- @@ -70,6 +73,12 @@ EXAMPLES __ssh_authorized_keys root \ --key "$(cat ~/.ssh/id_rsa.pub)" + # same as above, but make sure your key is only key in + # root's authorized_keys file + __ssh_authorized_keys root \ + --key "$(cat ~/.ssh/id_rsa.pub)" \ + --remove-unknown + # allow key to login as user-name __ssh_authorized_keys user-name \ --key "ssh-rsa AXYZAAB3NzaC1yc2..." diff --git a/cdist/conf/type/__ssh_authorized_keys/manifest b/cdist/conf/type/__ssh_authorized_keys/manifest index b9f0582e..b319316b 100755 --- a/cdist/conf/type/__ssh_authorized_keys/manifest +++ b/cdist/conf/type/__ssh_authorized_keys/manifest @@ -55,8 +55,12 @@ _cksum() { echo "$1" | cksum | cut -d' ' -f 1 } +_type_and_key() { + echo "$1" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }' +} + while read -r key; do - type_and_key="$(echo "$key" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }')" + type_and_key="$( _type_and_key "$key" )" object_id="$(_cksum "$file")-$(_cksum "$type_and_key")" set -- "$object_id" set -- "$@" --file "$file" @@ -72,3 +76,24 @@ while read -r key; do # Ensure __ssh_authorized_key does not read stdin __ssh_authorized_key "$@" < /dev/null done < "$__object/parameter/key" + +if [ -f "$__object/parameter/remove-unknown" ] && + [ -s "$__object/explorer/keys" ] +then + while read -r key + do + type_and_key="$( _type_and_key "$key" )" + + if grep -Fq "$type_and_key" "$__object/parameter/key" + then + continue + fi + + __ssh_authorized_key "remove-$( _cksum "$file$key" )" \ + --file "$file" \ + --key "$key" \ + --state absent \ + < /dev/null + done \ + < "$__object/explorer/keys" +fi diff --git a/cdist/conf/type/__ssh_authorized_keys/parameter/boolean b/cdist/conf/type/__ssh_authorized_keys/parameter/boolean index 4bb126fe..7388fed5 100644 --- a/cdist/conf/type/__ssh_authorized_keys/parameter/boolean +++ b/cdist/conf/type/__ssh_authorized_keys/parameter/boolean @@ -1,2 +1,3 @@ noparent nofile +remove-unknown From b354ea6e9430816bd62168c4c8e7b936ad4a4885 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 30 May 2020 10:49:13 +0200 Subject: [PATCH 034/155] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index ad008cb2..76b5edfa 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) + 6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) * Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius) From 988190363aeff5fba9be80355c318e1ff372eda9 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 30 May 2020 15:10:13 +0200 Subject: [PATCH 035/155] Resolve shellcheck SC1090 --- cdist/conf/type/__ssh_authorized_keys/explorer/keys | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys index b515c630..cec25746 100755 --- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys +++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys @@ -1,5 +1,6 @@ #!/bin/sh -e +# shellcheck disable=SC1090 file="$( . "$__type_explorer/file" )" if [ -f "$file" ] From ba77ea9edcf2d99c6edd76adaa83017821babae3 Mon Sep 17 00:00:00 2001 From: Evil Ham Date: Mon, 1 Jun 2020 19:11:58 +0200 Subject: [PATCH 036/155] [UX] Add option to enable LogLevel-based coloured output. This makes it easier for new and experienced users to run cdist with higher verbosity levels, both to know that things are working as expected and to debug issues. Documentation has been modified accordingly and default behaviour is not changed. --- cdist/argparse.py | 31 ++++++++++++++++-- cdist/configuration.py | 11 +++++++ cdist/core/manifest.py | 1 + cdist/emulator.py | 3 ++ cdist/log.py | 26 ++++++++++++++- cdist/test/configuration/__init__.py | 28 +++++++++++++++++ configuration/cdist.cfg.skeleton | 6 ++++ docs/src/cdist-reference.rst.sh | 5 +++ docs/src/man1/cdist.rst | 47 +++++++++++++++++++--------- 9 files changed, 140 insertions(+), 18 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 611c484a..c30e2030 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -5,6 +5,7 @@ import logging import collections import functools import cdist.configuration +import cdist.log import cdist.preos import cdist.info @@ -88,6 +89,13 @@ def check_lower_bounded_int(value, lower_bound, name): return val +def colored_output_type(val): + boolean_states = cdist.configuration.ColoredOutputOption.BOOLEAN_STATES + if val not in boolean_states.keys(): + raise argparse.ArgumentError() + return boolean_states[val] + + def get_parsers(): global parser @@ -125,6 +133,15 @@ def get_parsers(): 'value.'), action='count', default=None) + parser['colored_output'] = argparse.ArgumentParser(add_help=False) + parser['colored_output'].add_argument( + '--colors', + help='Use a colored output for different log levels.' + 'It can be a boolean or "auto" (default) which enables this ' + 'feature if stdout is a tty and disables it otherwise.', + action='store', dest='colored_output', required=False, + type=colored_output_type) + parser['beta'] = argparse.ArgumentParser(add_help=False) parser['beta'].add_argument( '-b', '--beta', @@ -283,6 +300,7 @@ def get_parsers(): 'host', nargs='*', help='Host(s) to operate on.') parser['config'] = parser['sub'].add_parser( 'config', parents=[parser['loglevel'], parser['beta'], + parser['colored_output'], parser['common'], parser['config_main'], parser['inventory_common'], @@ -301,6 +319,7 @@ def get_parsers(): parser['add-host'] = parser['invsub'].add_parser( 'add-host', parents=[parser['loglevel'], parser['beta'], + parser['colored_output'], parser['common'], parser['inventory_common']]) parser['add-host'].add_argument( @@ -315,6 +334,7 @@ def get_parsers(): parser['add-tag'] = parser['invsub'].add_parser( 'add-tag', parents=[parser['loglevel'], parser['beta'], + parser['colored_output'], parser['common'], parser['inventory_common']]) parser['add-tag'].add_argument( @@ -346,6 +366,7 @@ def get_parsers(): parser['del-host'] = parser['invsub'].add_parser( 'del-host', parents=[parser['loglevel'], parser['beta'], + parser['colored_output'], parser['common'], parser['inventory_common']]) parser['del-host'].add_argument( @@ -363,6 +384,7 @@ def get_parsers(): parser['del-tag'] = parser['invsub'].add_parser( 'del-tag', parents=[parser['loglevel'], parser['beta'], + parser['colored_output'], parser['common'], parser['inventory_common']]) parser['del-tag'].add_argument( @@ -398,6 +420,7 @@ def get_parsers(): parser['list'] = parser['invsub'].add_parser( 'list', parents=[parser['loglevel'], parser['beta'], + parser['colored_output'], parser['common'], parser['inventory_common']]) parser['list'].add_argument( @@ -430,7 +453,7 @@ def get_parsers(): # Shell parser['shell'] = parser['sub'].add_parser( - 'shell', parents=[parser['loglevel']]) + 'shell', parents=[parser['loglevel'], parser['colored_output']]) parser['shell'].add_argument( '-s', '--shell', help=('Select shell to use, defaults to current shell. Used shell' @@ -495,9 +518,13 @@ def parse_and_configure(argv, singleton=True): log = logging.getLogger("cdist") + config = cfg.get_config() + if config.get('GLOBAL', {}).get('colored_output', False): + cdist.log.ColorFormatter.USE_COLORS = True + log.verbose("version %s" % cdist.VERSION) log.trace('command line args: {}'.format(cfg.command_line_args)) - log.trace('configuration: {}'.format(cfg.get_config())) + log.trace('configuration: {}'.format(config)) log.trace('configured args: {}'.format(args)) check_beta(vars(args)) diff --git a/cdist/configuration.py b/cdist/configuration.py index 1011a382..6f07c27f 100644 --- a/cdist/configuration.py +++ b/cdist/configuration.py @@ -27,6 +27,7 @@ import cdist.argparse import re import multiprocessing import logging +import sys class Singleton(type): @@ -47,6 +48,7 @@ _VERBOSITY_VALUES = ( _ARCHIVING_VALUES = ( 'tar', 'tgz', 'tbz2', 'txz', 'none', ) +_COLORED_OUTPUT_DEFAULT = sys.stdout.isatty() class OptionBase: @@ -246,9 +248,15 @@ class LogLevelOption(OptionBase): return VerbosityOption().translate(val) +class ColoredOutputOption(BooleanOption): + BOOLEAN_STATES = dict(configparser.ConfigParser.BOOLEAN_STATES, + auto=_COLORED_OUTPUT_DEFAULT) + + _ARG_OPTION_MAPPING = { 'beta': 'beta', 'cache_path_pattern': 'cache_path_pattern', + 'colored_output': 'colored_output', 'conf_dir': 'conf_dir', 'manifest': 'init_manifest', 'out_path': 'out_path', @@ -294,6 +302,7 @@ class Configuration(metaclass=Singleton): 'remote_shell': StringOption('remote_shell'), 'cache_path_pattern': StringOption('cache_path_pattern'), 'conf_dir': ConfDirOption(), + 'colored_output': ColoredOutputOption('colored_output'), 'init_manifest': StringOption('init_manifest'), 'out_path': StringOption('out_path'), 'remote_out_path': StringOption('remote_out_path'), @@ -319,6 +328,7 @@ class Configuration(metaclass=Singleton): 'CDIST_REMOTE_COPY': 'remote_copy', 'CDIST_INVENTORY_DIR': 'inventory_dir', 'CDIST_CACHE_PATH_PATTERN': 'cache_path_pattern', + 'CDIST_COLORED_OUTPUT': 'colored_output', '__cdist_log_level': 'verbosity', } ENV_VAR_BOOLEAN_OPTIONS = ('CDIST_BETA', ) @@ -332,6 +342,7 @@ class Configuration(metaclass=Singleton): } REQUIRED_DEFAULT_CONFIG_VALUES = { 'GLOBAL': { + 'colored_output': _COLORED_OUTPUT_DEFAULT, 'verbosity': 0, }, } diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 8aeaf860..32520e49 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -119,6 +119,7 @@ class Manifest(object): '__cdist_log_level': util.log_level_env_var_val(self.log), '__cdist_log_level_name': util.log_level_name_env_var_val( self.log), + '__cdist_colored_log': str(cdist.log.ColorFormatter.USE_COLORS), } if dry_run: diff --git a/cdist/emulator.py b/cdist/emulator.py index 4800e2a3..87c9fe12 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -129,6 +129,9 @@ class Emulator(object): # if invalid __cdist_log_level value logging.root.setLevel(logging.WARNING) + colored_log = self.env.get('__cdist_colored_log', 'False') + cdist.log.ColorFormatter.USE_COLORS = colored_log == 'True' + self.log = logging.getLogger(self.target_host[0]) def commandline(self): diff --git a/cdist/log.py b/cdist/log.py index 2d0bef0b..5f2d8f53 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -50,6 +50,30 @@ def _trace(msg, *args, **kwargs): logging.trace = _trace +class ColorFormatter(logging.Formatter): + USE_COLORS = False + RESET = '\033[0m' + COLOR_MAP = { + 'ERROR': '\033[0;31m', + 'WARNING': '\033[0;33m', + 'INFO': '\033[0;94m', + 'VERBOSE': '\033[0;34m', + 'DEBUG': '\033[0;90m', + 'TRACE': '\033[0;37m', + } + + def __init__(self, msg): + super().__init__(msg) + + def format(self, record): + msg = super().format(record) + if self.USE_COLORS: + color = self.COLOR_MAP.get(record.levelname) + if color: + msg = color + msg + self.RESET + return msg + + class DefaultLog(logging.Logger): FORMAT = '%(levelname)s: %(message)s' @@ -66,7 +90,7 @@ class DefaultLog(logging.Logger): super().__init__(name) self.propagate = False - formatter = logging.Formatter(self.FORMAT) + formatter = ColorFormatter(self.FORMAT) self.addFilter(self) diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py index 182868a6..07a73bda 100644 --- a/cdist/test/configuration/__init__.py +++ b/cdist/test/configuration/__init__.py @@ -28,10 +28,12 @@ import argparse from cdist import test import cdist.argparse as cap import logging +import sys my_dir = op.abspath(op.dirname(__file__)) fixtures = op.join(my_dir, 'fixtures') interpolation_config_file = op.join(fixtures, "interpolation-test.cfg") +colored_output_default = sys.stdout.isatty() def newConfigParser(): @@ -153,6 +155,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/bin/sh', 'inventory_dir': '', 'cache_path_pattern': '', + 'colored_output': colored_output_default, 'conf_dir': '', 'init_manifest': '', 'out_path': '', @@ -184,6 +187,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/bin/sh', 'inventory_dir': None, 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': None, 'init_manifest': None, 'out_path': None, @@ -390,6 +394,7 @@ class ConfigurationTestCase(test.CdistTestCase): args = argparse.Namespace() expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'verbosity': 0, }, } @@ -440,6 +445,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/bin/sh', 'inventory_dir': None, 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': None, 'init_manifest': None, 'out_path': None, @@ -515,6 +521,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/var/db/cdist/inventory', 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': ['/opt/cdist', ], 'init_manifest': None, 'out_path': None, @@ -556,6 +563,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/bin/sh', 'inventory_dir': '', 'cache_path_pattern': '', + 'colored_output': colored_output_default, 'conf_dir': '', 'init_manifest': '', 'out_path': '', @@ -579,6 +587,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': None, 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': [ '/opt/cdist/conf', '/usr/local/share/cdist/conf', @@ -623,6 +632,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/bin/sh', 'inventory_dir': '', 'cache_path_pattern': '', + 'colored_output': colored_output_default, 'conf_dir': '', 'init_manifest': '', 'out_path': '', @@ -645,6 +655,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'local_shell': '/usr/bin/sh', 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/var/db/cdist/inventory', + 'colored_output': colored_output_default, 'conf_dir': '/opt/cdist', 'remote_copy': 'myscp', 'remote_exec': 'myexec', @@ -663,6 +674,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/var/db/cdist/inventory', 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': [ '/opt/cdist/conf', '/usr/local/share/cdist/conf', @@ -694,6 +706,7 @@ class ConfigurationTestCase(test.CdistTestCase): } expected_config = { 'GLOBAL': { + 'colored_output': colored_output_default, 'verbosity': 0, }, } @@ -767,6 +780,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/opt/sysadmin/cdist/inventory', 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': [ '/opt/cdist/conf', '/usr/local/share/cdist/conf', @@ -865,6 +879,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/var/db/cdist/inventory', 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': [ '/opt/conf/cdist', ], @@ -964,6 +979,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/var/db/cdist/inventory', 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': [ '/opt/conf/cdist', ], @@ -1063,6 +1079,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/var/db/cdist/inventory', 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': [ '/opt/conf/cdist', ], @@ -1095,6 +1112,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'beta': True, 'inventory_dir': '/var/db/cdist/inventory', 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': [ '/opt/conf/cdist', ], @@ -1125,6 +1143,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { 'inventory_dir': None, + 'colored_output': colored_output_default, 'conf_dir': None, 'verbosity': 0, }, @@ -1148,6 +1167,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'verbosity': cap.VERBOSE_DEBUG, }, } @@ -1185,6 +1205,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'save_output_streams': True, 'verbosity': 0, }, @@ -1213,6 +1234,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'save_output_streams': False, 'verbosity': 0, }, @@ -1241,6 +1263,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'save_output_streams': False, 'verbosity': 0, }, @@ -1269,6 +1292,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'save_output_streams': False, 'verbosity': 0, }, @@ -1308,6 +1332,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'timestamp': True, 'verbosity': 0, }, @@ -1336,6 +1361,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'timestamp': True, 'verbosity': 0, }, @@ -1364,6 +1390,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'timestamp': False, 'verbosity': 0, }, @@ -1392,6 +1419,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'timestamp': False, 'verbosity': 0, }, diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton index 91c5ab02..f2a09064 100644 --- a/configuration/cdist.cfg.skeleton +++ b/configuration/cdist.cfg.skeleton @@ -13,6 +13,12 @@ # Specify cache path pattern. # cache_path_pattern = %h # +# colored_output +# Use a colored output for different log levels. +# It can be a boolean or 'auto' (default) which enables this feature if +# stdout is a tty and disables it otherwise. +# colored_output = auto +# # conf_dir # List of configuration directories separated with the character conventionally # used by the operating system to separate search path components (as in PATH), diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index e77d98f6..3b997f63 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -344,6 +344,11 @@ CDIST_INVENTORY_DIR CDIST_BETA Enable beta functionalities. +CDIST_COLORED_OUTPUT + Use a colored output for different log levels. + It can be a boolean or 'auto' (default) which enables this feature if + stdout is a tty and disables it otherwise. + CDIST_CACHE_PATH_PATTERN Custom cache path pattern. eof diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 38248821..4c34c4b7 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -15,8 +15,9 @@ SYNOPSIS cdist banner [-h] [-l LOGLEVEL] [-q] [-v] - cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4] - [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] + cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] + [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6] + [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] @@ -24,8 +25,9 @@ SYNOPSIS [-p [HOST_MAX]] [-s] [-t] [host [host ...]] - cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4] - [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] + cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] + [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6] + [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] @@ -36,26 +38,29 @@ SYNOPSIS cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ... cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [-g CONFIG_FILE] [-I INVENTORY_DIR] - [-f HOSTFILE] + [--colors COLORED_OUTPUT] [-g CONFIG_FILE] + [-I INVENTORY_DIR] [-f HOSTFILE] [host [host ...]] cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [-g CONFIG_FILE] [-I INVENTORY_DIR] - [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] + [--colors COLORED_OUTPUT] [-g CONFIG_FILE] + [-I INVENTORY_DIR] [-f HOSTFILE] [-T TAGFILE] + [-t TAGLIST] [host [host ...]] cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] - [-f HOSTFILE] + [--colors COLORED_OUTPUT] [-g CONFIG_FILE] + [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [host [host ...]] cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] - [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] + [--colors COLORED_OUTPUT] [-g CONFIG_FILE] + [-I INVENTORY_DIR] [-a] [-f HOSTFILE] + [-T TAGFILE] [-t TAGLIST] [host [host ...]] - cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] + cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] + [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t] [host [host ...]] @@ -84,9 +89,11 @@ SYNOPSIS [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY] target_dir - cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL] + cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [--colors COLORED_OUTPUT] + [-s SHELL] - cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] + cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] + [pattern] DESCRIPTION @@ -104,6 +111,11 @@ All commands accept the following options: **-h, --help** Show the help screen. +**--colors COLORED_OUTPUT** + Use a colored output for different log levels.It can + be a boolean or "auto" (default) which enables this + feature if stdout is a tty and disables it otherwise. + **-l LOGLEVEL, --log-level LOGLEVEL** Set the specified verbosity level. The levels, in order from the lowest to the highest, are: ERROR (-1), @@ -893,6 +905,11 @@ CDIST_BETA CDIST_CACHE_PATH_PATTERN Custom cache path pattern. +CDIST_COLORED_OUTPUT + Use a colored output for different log levels. + It can be a boolean or 'auto' (default) which enables this feature if + stdout is a tty and disables it otherwise. + CDIST_CONFIG_FILE Custom configuration file. From 9a4e3488c2104ad5562b5ab9aad8120969df73a8 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 1 Jun 2020 19:17:02 +0200 Subject: [PATCH 037/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 76b5edfa..5408c54f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) + * Core: Support colored log output (Evil Ham) 6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) From 55ebd1a4c56e788b18911f273d931f950a0c5271 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 1 Jun 2020 20:22:40 +0200 Subject: [PATCH 038/155] Fix man build failure in newer sphinx versions --- cdist/sphinxext/manpage.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cdist/sphinxext/manpage.py b/cdist/sphinxext/manpage.py index 135fe22e..1f8ac4f6 100644 --- a/cdist/sphinxext/manpage.py +++ b/cdist/sphinxext/manpage.py @@ -7,6 +7,7 @@ from docutils.io import FileOutput from os import path from sphinx.util.nodes import inline_all_toctrees from sphinx import addnodes +from sphinx.util import logging """ Extension based on sphinx builtin manpage. @@ -15,6 +16,9 @@ from sphinx import addnodes """ +logger = logging.getLogger(__name__) + + class ManualPageTranslator(sphinx.writers.manpage.ManualPageTranslator): def header(self): @@ -28,7 +32,7 @@ class ManualPageWriter(sphinx.writers.manpage.ManualPageWriter): def __init__(self, builder): super().__init__(builder) self.translator_class = ( - self.builder.translator_class or ManualPageTranslator) + self.builder.get_translator_class() or ManualPageTranslator) class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): @@ -43,7 +47,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): components=(docwriter,), read_config_files=True).get_default_values() - self.info(bold('writing... '), nonl=True) + logger.info(bold('writing... '), nonl=True) for info in self.config.man_pages: docname, name, description, authors, section = info @@ -54,7 +58,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): authors = [] targetname = '%s.%s' % (name, section) - self.info(darkgreen(targetname) + ' { ', nonl=True) + logger.info(darkgreen(targetname) + ' { ', nonl=True) destination = FileOutput( destination_path=path.join(self.outdir, targetname), encoding='utf-8') @@ -63,7 +67,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): docnames = set() largetree = inline_all_toctrees(self, docnames, docname, tree, darkgreen, [docname]) - self.info('} ', nonl=True) + logger.info('} ', nonl=True) self.env.resolve_references(largetree, docname, self) # remove pending_xref nodes for pendingnode in largetree.traverse(addnodes.pending_xref): @@ -76,7 +80,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): largetree.settings.section = section docwriter.write(largetree, destination) - self.info() + logger.info("") def setup(app): From 48d66b014351982e70aa3e065ffc1ed344660bab Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 1 Jun 2020 22:25:15 +0200 Subject: [PATCH 039/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 5408c54f..5b57e98b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) * Core: Support colored log output (Evil Ham) + * Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak) 6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) From 6a611e556a54a60952a833989a5a0fca79011217 Mon Sep 17 00:00:00 2001 From: Daniel Heule Date: Tue, 2 Jun 2020 13:40:21 +0200 Subject: [PATCH 040/155] fix os explorer for sles15 --- cdist/conf/explorer/os | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os index 563fa4cf..2d2aede6 100755 --- a/cdist/conf/explorer/os +++ b/cdist/conf/explorer/os @@ -143,6 +143,11 @@ case "$uname_s" in esac if [ -f /etc/os-release ]; then + # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse + if grep -q ^ID_LIKE=\"suse\" /etc/os-release 2>/dev/null; then + echo suse + exit 0 + fi # already lowercase, according to: # https://www.freedesktop.org/software/systemd/man/os-release.html awk -F= '/^ID=/ { if ($2 ~ /^'"'"'(.*)'"'"'$/ || $2 ~ /^"(.*)"$/) { print substr($2, 2, length($2) - 2) } else { print $2 } }' /etc/os-release From 747c6b10762e18bbda24fbb876ac2e88f29dc70b Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 3 Jun 2020 21:45:04 +0200 Subject: [PATCH 041/155] Respect NO_COLOR environment variable --- cdist/argparse.py | 23 +++++++++-------------- cdist/config.py | 1 + cdist/configuration.py | 28 +++++++++++++++++++--------- cdist/core/manifest.py | 2 +- cdist/emulator.py | 4 ++-- cdist/log.py | 19 ++++++++++--------- cdist/test/configuration/__init__.py | 2 +- configuration/cdist.cfg.skeleton | 6 ++++-- 8 files changed, 47 insertions(+), 38 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index c30e2030..0782654f 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -89,13 +89,6 @@ def check_lower_bounded_int(value, lower_bound, name): return val -def colored_output_type(val): - boolean_states = cdist.configuration.ColoredOutputOption.BOOLEAN_STATES - if val not in boolean_states.keys(): - raise argparse.ArgumentError() - return boolean_states[val] - - def get_parsers(): global parser @@ -140,7 +133,7 @@ def get_parsers(): 'It can be a boolean or "auto" (default) which enables this ' 'feature if stdout is a tty and disables it otherwise.', action='store', dest='colored_output', required=False, - type=colored_output_type) + choices=cdist.configuration.ColoredOutputOption.CHOICES) parser['beta'] = argparse.ArgumentParser(add_help=False) parser['beta'].add_argument( @@ -501,7 +494,12 @@ def handle_loglevel(args): if hasattr(args, 'quiet') and args.quiet: args.verbose = _verbosity_level_off - logging.root.setLevel(_verbosity_level[args.verbose]) + logging.getLogger().setLevel(_verbosity_level[args.verbose]) + + +def handle_log_colors(args): + if cdist.configuration.ColoredOutputOption.translate(args.colored_output): + cdist.log.DefaultLog.USE_COLORS = True def parse_and_configure(argv, singleton=True): @@ -515,16 +513,13 @@ def parse_and_configure(argv, singleton=True): raise cdist.Error(str(e)) # Loglevels are handled globally in here handle_loglevel(args) + handle_log_colors(args) log = logging.getLogger("cdist") - config = cfg.get_config() - if config.get('GLOBAL', {}).get('colored_output', False): - cdist.log.ColorFormatter.USE_COLORS = True - log.verbose("version %s" % cdist.VERSION) log.trace('command line args: {}'.format(cfg.command_line_args)) - log.trace('configuration: {}'.format(config)) + log.trace('configuration: {}'.format(cfg.get_config())) log.trace('configured args: {}'.format(args)) check_beta(vars(args)) diff --git a/cdist/config.py b/cdist/config.py index 97cc1da6..b2d72f05 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -203,6 +203,7 @@ class Config(object): cdist.log.setupParallelLogging() elif args.timestamp: cdist.log.setupTimestampingLogging() + log = logging.getLogger("config") # No new child process if only one host at a time. diff --git a/cdist/configuration.py b/cdist/configuration.py index 6f07c27f..c0fbc063 100644 --- a/cdist/configuration.py +++ b/cdist/configuration.py @@ -48,7 +48,6 @@ _VERBOSITY_VALUES = ( _ARCHIVING_VALUES = ( 'tar', 'tgz', 'tbz2', 'txz', 'none', ) -_COLORED_OUTPUT_DEFAULT = sys.stdout.isatty() class OptionBase: @@ -249,8 +248,22 @@ class LogLevelOption(OptionBase): class ColoredOutputOption(BooleanOption): - BOOLEAN_STATES = dict(configparser.ConfigParser.BOOLEAN_STATES, - auto=_COLORED_OUTPUT_DEFAULT) + CHOICES = tuple(configparser.ConfigParser.BOOLEAN_STATES) + ('auto',) + DEFAULT = 'auto' + + def get_converter(self): + return self.translate + + @staticmethod + def translate(val): + if 'NO_COLOR' in os.environ: + return False + elif isinstance(val, bool): + return val + elif val == 'auto': + return sys.stdout.isatty() + else: + return configparser.ConfigParser.BOOLEAN_STATES[val] _ARG_OPTION_MAPPING = { @@ -337,12 +350,10 @@ class Configuration(metaclass=Singleton): } ARG_OPTION_MAPPING = _ARG_OPTION_MAPPING - ADJUST_ARG_OPTION_MAPPING = { - _ARG_OPTION_MAPPING[key]: key for key in _ARG_OPTION_MAPPING - } + ADJUST_ARG_OPTION_MAPPING = {v: k for k, v in _ARG_OPTION_MAPPING.items()} REQUIRED_DEFAULT_CONFIG_VALUES = { 'GLOBAL': { - 'colored_output': _COLORED_OUTPUT_DEFAULT, + 'colored_output': 'auto', 'verbosity': 0, }, } @@ -495,8 +506,7 @@ class Configuration(metaclass=Singleton): newconfig = self._read_config_file(config_file) self._update_config_dict(config, newconfig) # command line config file - if (self.args and 'config_file' in self.args and - self.args['config_file']): + if (self.args and self.args.get('config_file', None)): newconfig = self._read_config_file(self.args['config_file']) self._update_config_dict(config, newconfig) # command line diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 32520e49..8b833ff2 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -119,7 +119,7 @@ class Manifest(object): '__cdist_log_level': util.log_level_env_var_val(self.log), '__cdist_log_level_name': util.log_level_name_env_var_val( self.log), - '__cdist_colored_log': str(cdist.log.ColorFormatter.USE_COLORS), + '__cdist_colored_log': str(self.log.USE_COLORS).lower(), } if dry_run: diff --git a/cdist/emulator.py b/cdist/emulator.py index 87c9fe12..4eaf2c93 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -129,8 +129,8 @@ class Emulator(object): # if invalid __cdist_log_level value logging.root.setLevel(logging.WARNING) - colored_log = self.env.get('__cdist_colored_log', 'False') - cdist.log.ColorFormatter.USE_COLORS = colored_log == 'True' + colored_log = self.env.get('__cdist_colored_log', 'false') + cdist.log.ColorFormatter.USE_COLORS = colored_log == 'true' self.log = logging.getLogger(self.target_host[0]) diff --git a/cdist/log.py b/cdist/log.py index 5f2d8f53..19efebdb 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -51,7 +51,6 @@ logging.trace = _trace class ColorFormatter(logging.Formatter): - USE_COLORS = False RESET = '\033[0m' COLOR_MAP = { 'ERROR': '\033[0;31m', @@ -62,20 +61,19 @@ class ColorFormatter(logging.Formatter): 'TRACE': '\033[0;37m', } - def __init__(self, msg): - super().__init__(msg) + def __init__(self, fmt): + super().__init__(fmt=fmt) def format(self, record): msg = super().format(record) - if self.USE_COLORS: - color = self.COLOR_MAP.get(record.levelname) - if color: - msg = color + msg + self.RESET + color = self.COLOR_MAP.get(record.levelname) + if color: + msg = color + msg + self.RESET return msg class DefaultLog(logging.Logger): - + USE_COLORS = False FORMAT = '%(levelname)s: %(message)s' class StdoutFilter(logging.Filter): @@ -90,7 +88,10 @@ class DefaultLog(logging.Logger): super().__init__(name) self.propagate = False - formatter = ColorFormatter(self.FORMAT) + if self.USE_COLORS: + formatter = ColorFormatter(self.FORMAT) + else: + formatter = logging.Formatter(self.FORMAT) self.addFilter(self) diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py index 07a73bda..5305b6d3 100644 --- a/cdist/test/configuration/__init__.py +++ b/cdist/test/configuration/__init__.py @@ -33,7 +33,7 @@ import sys my_dir = op.abspath(op.dirname(__file__)) fixtures = op.join(my_dir, 'fixtures') interpolation_config_file = op.join(fixtures, "interpolation-test.cfg") -colored_output_default = sys.stdout.isatty() +colored_output_default = 'auto' def newConfigParser(): diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton index f2a09064..0730201d 100644 --- a/configuration/cdist.cfg.skeleton +++ b/configuration/cdist.cfg.skeleton @@ -17,6 +17,8 @@ # Use a colored output for different log levels. # It can be a boolean or 'auto' (default) which enables this feature if # stdout is a tty and disables it otherwise. +# Colored output is always disabled if the NO_COLOR environment variable is +# defined (https://no-color.org/). # colored_output = auto # # conf_dir @@ -51,7 +53,7 @@ # # out_path # Directory to save cdist output in. -# out_path = +# out_path = # # parallel # Process hosts in parallel. If -1 then the default, number of CPU's in @@ -77,6 +79,6 @@ # remote_shell = /bin/sh # # verbosity -# Set verbosity level. Valid values are: +# Set verbosity level. Valid values are: # ERROR, WARNING, INFO, VERBOSE, DEBUG, TRACE and OFF. # verbosity = INFO From cdb0d2be413114a237a0f56c948cc5cd6cfc2b0b Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 3 Jun 2020 23:21:50 +0200 Subject: [PATCH 042/155] Patch tests --- cdist/configuration.py | 4 ++++ cdist/test/configuration/__init__.py | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/cdist/configuration.py b/cdist/configuration.py index c0fbc063..f68a5e09 100644 --- a/cdist/configuration.py +++ b/cdist/configuration.py @@ -266,6 +266,10 @@ class ColoredOutputOption(BooleanOption): return configparser.ConfigParser.BOOLEAN_STATES[val] +ColoredOutputOption.DEFAULT = ColoredOutputOption.translate( + ColoredOutputOption.DEFAULT) + + _ARG_OPTION_MAPPING = { 'beta': 'beta', 'cache_path_pattern': 'cache_path_pattern', diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py index 5305b6d3..3fd24ca5 100644 --- a/cdist/test/configuration/__init__.py +++ b/cdist/test/configuration/__init__.py @@ -187,7 +187,8 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/bin/sh', 'inventory_dir': None, 'cache_path_pattern': None, - 'colored_output': colored_output_default, + 'colored_output': cc.ColoredOutputOption.translate( + colored_output_default), 'conf_dir': None, 'init_manifest': None, 'out_path': None, @@ -587,7 +588,8 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': None, 'cache_path_pattern': None, - 'colored_output': colored_output_default, + 'colored_output': cc.ColoredOutputOption.translate( + colored_output_default), 'conf_dir': [ '/opt/cdist/conf', '/usr/local/share/cdist/conf', @@ -674,7 +676,8 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/var/db/cdist/inventory', 'cache_path_pattern': None, - 'colored_output': colored_output_default, + 'colored_output': cc.ColoredOutputOption.translate( + colored_output_default), 'conf_dir': [ '/opt/cdist/conf', '/usr/local/share/cdist/conf', From 89e48734bf13446024e9009bce7e77cbd4255840 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 5 Jun 2020 12:22:49 +0200 Subject: [PATCH 043/155] Let config file and command line override NO_COLOR envvar --- cdist/configuration.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cdist/configuration.py b/cdist/configuration.py index f68a5e09..a12940b0 100644 --- a/cdist/configuration.py +++ b/cdist/configuration.py @@ -256,14 +256,12 @@ class ColoredOutputOption(BooleanOption): @staticmethod def translate(val): - if 'NO_COLOR' in os.environ: - return False - elif isinstance(val, bool): + if isinstance(val, bool): return val - elif val == 'auto': - return sys.stdout.isatty() - else: + elif val in configparser.ConfigParser.BOOLEAN_STATES: return configparser.ConfigParser.BOOLEAN_STATES[val] + elif val == 'auto': + return 'NO_COLOR' not in os.environ and sys.stdout.isatty() ColoredOutputOption.DEFAULT = ColoredOutputOption.translate( From 790c6efae9e379cb140e59469f07d2c3732f536c Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 5 Jun 2020 13:56:30 +0200 Subject: [PATCH 044/155] Update colored output documentation --- docs/src/cdist-reference.rst.sh | 38 +++++++++++++++++++++++++++------ docs/src/man1/cdist.rst | 35 +++++++++++++++++------------- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index 3b997f63..a841862e 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -34,7 +34,7 @@ dest="$__cdist_abs_mydir/$filename" cd "$__cdist_abs_mydir" exec > "$dest" -cat << eof +cat << eof Reference ========= Variable, path and type reference for cdist @@ -51,7 +51,7 @@ eof done ) -cat << eof +cat << eof Paths ----- @@ -187,13 +187,13 @@ usable within a object directory: files This directory is reserved for user data and will not be used - by cdist at any time. It can be used freely by the type + by cdist at any time. It can be used freely by the type (for instance to store template results). changed This empty file exists in an object directory, if the object has code to be executed (either remote or local). stdin - This file exists and contains data, if data was provided on stdin + This file exists and contains data, if data was provided on stdin when the type was called. @@ -222,65 +222,89 @@ __cdist_log_level, __cdist_log_level_name | TRACE | 5 | +----------------+-----------------+ + Available for: initial manifest, explorer, type manifest, type explorer, + type gencode. +__cdist_colored_log + whether or not cdist's log has colors enabled. + Is set to the string ``true`` if cdist's output is using colors, + otherwise the variable contains the string ``false``. + Available for: initial manifest, explorer, type manifest, type explorer, type gencode. __cdist_dry_run Is set only when doing dry run (``-n`` flag). + Available for: initial manifest, explorer, type manifest, type explorer, type gencode. __explorer Directory that contains all global explorers. + Available for: initial manifest, explorer, type explorer, shell. __files Directory that contains content from the "files" subdirectories from the configuration directories. + Available for: initial manifest, type manifest, type gencode, shell. __manifest Directory that contains the initial manifest. + Available for: initial manifest, type manifest, shell. __global Directory that contains generic output like explorer. + Available for: initial manifest, type manifest, type gencode, shell. __messages_in File to read messages from. + Available for: initial manifest, type manifest, type gencode. __messages_out File to write messages. + Available for: initial manifest, type manifest, type gencode. __object Directory that contains the current object. + Available for: type manifest, type explorer, type gencode and code scripts. __object_id The type unique object id. + Available for: type manifest, type explorer, type gencode and code scripts. - Note: The leading and the trailing "/" will always be stripped (caused by - the filesystem database and ensured by the core). - Note: Double slashes ("//") will not be fixed and result in an error. + + | Note: The leading and the trailing "/" will always be stripped (caused by + the filesystem database and ensured by the core). + | Note: Double slashes ("//") will not be fixed and result in an error. __object_name The full qualified name of the current object. + Available for: type manifest, type explorer, type gencode. __target_host The host we are deploying to. This is primary variable. It's content is literally the one user passed in. + Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. __target_hostname The hostname of host we are deploying to. This variable is derived from **__target_host** (using **socket.getaddrinfo(__target_host)** and then **socket.gethostbyaddr()**). + Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. __target_fqdn The fully qualified domain name of the host we are deploying to. This variable is derived from **__target_host** (using **socket.getfqdn()**). + Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. __target_host_tags Comma separated list of target host tags. + Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. __type Path to the current type. + Available for: type manifest, type gencode. __type_explorer Directory that contains the type explorers. + Available for: type explorer. Environment variables (for writing) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 4c34c4b7..4def97b5 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -112,9 +112,13 @@ All commands accept the following options: Show the help screen. **--colors COLORED_OUTPUT** - Use a colored output for different log levels.It can - be a boolean or "auto" (default) which enables this - feature if stdout is a tty and disables it otherwise. + Colorize cdist's output. If enabled, cdist will use different colors for + different log levels. + COLORED_OUTPUT recognizes the boolean values 'yes'/'no', 'on'/'off', + 'true'/'false', '1'/'0', and 'auto' (the default). + + If the value is 'auto', colored output is enabled if stdout is a TTY + unless the NO_COLOR (https://no-color.org/) environment variable is defined. **-l LOGLEVEL, --log-level LOGLEVEL** Set the specified verbosity level. The levels, in @@ -168,7 +172,7 @@ Install command is currently in beta. **-b, --beta** Enable beta functionality. - + **-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN** Specify custom cache path pattern. If it is not set then default hostdir is used. For more info on format see @@ -191,7 +195,7 @@ Install command is currently in beta. **-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory - directory is set up by the following rules: if cdist + directory is set up by the following rules: if cdist configuration resolves this value then specified directory is used, if HOME env var is set then ~/.cdit/inventory is used, otherwise distribution @@ -306,7 +310,7 @@ Add host(s) to inventory database. **-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory - directory is set up by the following rules: if cdist + directory is set up by the following rules: if cdist configuration resolves this value then specified directory is used, if HOME env var is set then ~/.cdit/inventory is used, otherwise distribution @@ -336,7 +340,7 @@ Add tag(s) to inventory database. **-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory - directory is set up by the following rules: if cdist + directory is set up by the following rules: if cdist configuration resolves this value then specified directory is used, if HOME env var is set then ~/.cdit/inventory is used, otherwise distribution @@ -379,7 +383,7 @@ Delete host(s) from inventory database. **-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory - directory is set up by the following rules: if cdist + directory is set up by the following rules: if cdist configuration resolves this value then specified directory is used, if HOME env var is set then ~/.cdit/inventory is used, otherwise distribution @@ -413,7 +417,7 @@ Delete tag(s) from inventory database. **-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory - directory is set up by the following rules: if cdist + directory is set up by the following rules: if cdist configuration resolves this value then specified directory is used, if HOME env var is set then ~/.cdit/inventory is used, otherwise distribution @@ -460,7 +464,7 @@ List inventory database. **-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory - directory is set up by the following rules: if cdist + directory is set up by the following rules: if cdist configuration resolves this value then specified directory is used, if HOME env var is set then ~/.cdit/inventory is used, otherwise distribution @@ -685,6 +689,9 @@ The possible keywords and their meanings are as follows: :strong:`cache_path_pattern` Specify cache path pattern. +:strong:`colored_output` + Colorize cdist's output. cf. the :code:`--colors` option. + :strong:`conf_dir` List of configuration directories separated with the character conventionally used by the operating system to separate search path components (as in PATH), @@ -738,7 +745,7 @@ The possible keywords and their meanings are as follows: in the format: YYYYMMDDHHMMSS.us. :strong:`verbosity` - Set verbosity level. Valid values are: + Set verbosity level. Valid values are: 'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'. @@ -770,7 +777,7 @@ cdist/preos NOTES ----- cdist detects if host is specified by IPv6 address. If so then remote_copy -command is executed with host address enclosed in square brackets +command is executed with host address enclosed in square brackets (see :strong:`scp`\ (1)). EXAMPLES @@ -906,9 +913,7 @@ CDIST_CACHE_PATH_PATTERN Custom cache path pattern. CDIST_COLORED_OUTPUT - Use a colored output for different log levels. - It can be a boolean or 'auto' (default) which enables this feature if - stdout is a tty and disables it otherwise. + Colorize cdist's output. cf. the :code:`--colors` option. CDIST_CONFIG_FILE Custom configuration file. From 7a570f8692a727662a6eda2c13f2b578b55864e4 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 5 Jun 2020 13:59:17 +0200 Subject: [PATCH 045/155] [cdist.cfg.skeleton] Update colored_output documentation based on cdist(1) --- configuration/cdist.cfg.skeleton | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton index 0730201d..9c66b51b 100644 --- a/configuration/cdist.cfg.skeleton +++ b/configuration/cdist.cfg.skeleton @@ -14,11 +14,12 @@ # cache_path_pattern = %h # # colored_output -# Use a colored output for different log levels. -# It can be a boolean or 'auto' (default) which enables this feature if -# stdout is a tty and disables it otherwise. -# Colored output is always disabled if the NO_COLOR environment variable is -# defined (https://no-color.org/). +# Colorize cdist's output. If enabled, cdist will use different colors for +# different log levels. +# Recognized values are 'yes'/'no', 'on'/'off', 'true'/'false', '1'/'0', +# and 'auto' +# If the value is 'auto', colored output is enabled if stdout is a TTY +# unless the NO_COLOR (https://no-color.org/) environment variable is defined. # colored_output = auto # # conf_dir From 23e66e08fab14f254c426c88271f461fe8499ba4 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 6 Jun 2020 13:39:29 +0200 Subject: [PATCH 046/155] Restrict colored_output value to always/never/auto. --- cdist/argparse.py | 7 +++---- cdist/configuration.py | 8 +++++--- configuration/cdist.cfg.skeleton | 7 +++---- docs/src/cdist-reference.rst.sh | 6 +++--- docs/src/man1/cdist.rst | 6 +++--- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 0782654f..85dba246 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -128,10 +128,9 @@ def get_parsers(): parser['colored_output'] = argparse.ArgumentParser(add_help=False) parser['colored_output'].add_argument( - '--colors', - help='Use a colored output for different log levels.' - 'It can be a boolean or "auto" (default) which enables this ' - 'feature if stdout is a tty and disables it otherwise.', + '--colors', metavar='WHEN', + help="Colorize cdist's output based on log level; " + "WHEN is 'always', 'never', or 'auto'.", action='store', dest='colored_output', required=False, choices=cdist.configuration.ColoredOutputOption.CHOICES) diff --git a/cdist/configuration.py b/cdist/configuration.py index a12940b0..4ba43a7a 100644 --- a/cdist/configuration.py +++ b/cdist/configuration.py @@ -248,7 +248,7 @@ class LogLevelOption(OptionBase): class ColoredOutputOption(BooleanOption): - CHOICES = tuple(configparser.ConfigParser.BOOLEAN_STATES) + ('auto',) + CHOICES = ('always', 'never', 'auto') DEFAULT = 'auto' def get_converter(self): @@ -258,8 +258,10 @@ class ColoredOutputOption(BooleanOption): def translate(val): if isinstance(val, bool): return val - elif val in configparser.ConfigParser.BOOLEAN_STATES: - return configparser.ConfigParser.BOOLEAN_STATES[val] + elif val == 'always': + return True + elif val == 'never': + return False elif val == 'auto': return 'NO_COLOR' not in os.environ and sys.stdout.isatty() diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton index 9c66b51b..14861592 100644 --- a/configuration/cdist.cfg.skeleton +++ b/configuration/cdist.cfg.skeleton @@ -16,10 +16,9 @@ # colored_output # Colorize cdist's output. If enabled, cdist will use different colors for # different log levels. -# Recognized values are 'yes'/'no', 'on'/'off', 'true'/'false', '1'/'0', -# and 'auto' -# If the value is 'auto', colored output is enabled if stdout is a TTY -# unless the NO_COLOR (https://no-color.org/) environment variable is defined. +# Recognized values are 'always', 'never', and 'auto'. +# If the value is 'auto', colors are enabled if stdout is a TTY unless +# the NO_COLOR (https://no-color.org/) environment variable is defined. # colored_output = auto # # conf_dir diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index a841862e..c0ac2c3e 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -369,9 +369,9 @@ CDIST_BETA Enable beta functionalities. CDIST_COLORED_OUTPUT - Use a colored output for different log levels. - It can be a boolean or 'auto' (default) which enables this feature if - stdout is a tty and disables it otherwise. + Colorize cdist's output. If enabled, cdist will use different colors for + different log levels. + Recognized values are 'always', 'never', and 'auto' (the default). CDIST_CACHE_PATH_PATTERN Custom cache path pattern. diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 4def97b5..760ff8f6 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -111,11 +111,11 @@ All commands accept the following options: **-h, --help** Show the help screen. -**--colors COLORED_OUTPUT** +**--colors WHEN** Colorize cdist's output. If enabled, cdist will use different colors for different log levels. - COLORED_OUTPUT recognizes the boolean values 'yes'/'no', 'on'/'off', - 'true'/'false', '1'/'0', and 'auto' (the default). + COLORED_OUTPUT recognizes the values 'always', 'never', + and 'auto' (the default). If the value is 'auto', colored output is enabled if stdout is a TTY unless the NO_COLOR (https://no-color.org/) environment variable is defined. From 89ebd7a4f79bef894cb3f31931571cf577778b26 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 7 Jun 2020 16:45:48 +0200 Subject: [PATCH 047/155] cdist man page: update --colors metavar name --- docs/src/man1/cdist.rst | 62 ++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 760ff8f6..aae98538 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -15,21 +15,19 @@ SYNOPSIS cdist banner [-h] [-l LOGLEVEL] [-q] [-v] - cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6] - [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] - [-j [JOBS]] [-n] [-o OUT_PATH] [-P] + cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] + [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN] + [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t] [host [host ...]] - cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6] - [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] - [-j [JOBS]] [-n] [-o OUT_PATH] [-P] - [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] + cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] + [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN] + [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] + [-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t] @@ -37,35 +35,31 @@ SYNOPSIS cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ... - cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [--colors COLORED_OUTPUT] [-g CONFIG_FILE] - [-I INVENTORY_DIR] [-f HOSTFILE] + cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] + [-g CONFIG_FILE] [-I INVENTORY_DIR] [-f HOSTFILE] [host [host ...]] - cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [--colors COLORED_OUTPUT] [-g CONFIG_FILE] - [-I INVENTORY_DIR] [-f HOSTFILE] [-T TAGFILE] - [-t TAGLIST] - [host [host ...]] - - cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [--colors COLORED_OUTPUT] [-g CONFIG_FILE] - [-I INVENTORY_DIR] [-a] [-f HOSTFILE] - [host [host ...]] - - cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [--colors COLORED_OUTPUT] [-g CONFIG_FILE] - [-I INVENTORY_DIR] [-a] [-f HOSTFILE] + cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] + [-g CONFIG_FILE] [-I INVENTORY_DIR] [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] [host [host ...]] - cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [--colors COLORED_OUTPUT] [-g CONFIG_FILE] - [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t] + cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] + [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] + [-f HOSTFILE] + [host [host ...]] + + cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] + [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] + [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] + [host [host ...]] + + cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] + [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] [-f HOSTFILE] + [-H] [-t] [host [host ...]] - cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-g CONFIG_FILE] - [-L] + cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-g CONFIG_FILE] [-L] [preos] ... cdist preos [preos-options] debian [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B] @@ -89,8 +83,7 @@ SYNOPSIS [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY] target_dir - cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [--colors COLORED_OUTPUT] - [-s SHELL] + cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [--colors WHEN] [-s SHELL] cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] @@ -114,8 +107,7 @@ All commands accept the following options: **--colors WHEN** Colorize cdist's output. If enabled, cdist will use different colors for different log levels. - COLORED_OUTPUT recognizes the values 'always', 'never', - and 'auto' (the default). + WHEN recognizes the values 'always', 'never', and 'auto' (the default). If the value is 'auto', colored output is enabled if stdout is a TTY unless the NO_COLOR (https://no-color.org/) environment variable is defined. From be47619b1ed7a87090f7615c19a9504f7cf2b590 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 8 Jun 2020 09:11:51 +0200 Subject: [PATCH 048/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 5b57e98b..94ef55d8 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) * Core: Support colored log output (Evil Ham) + * Core: Tune colored log output and respect NO_COLOR env var (Dennis Camera) * Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak) 6.5.6: 2020-05-25 From f5630297bd58fc92d95ab174ed6c8a8e14abf0b9 Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Wed, 3 Jun 2020 23:06:49 +0300 Subject: [PATCH 049/155] [docs] Fixed typo in cdist-configuration.rst Signed-off-by: Jaak Ristioja --- docs/src/cdist-configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/cdist-configuration.rst b/docs/src/cdist-configuration.rst index 706ed761..a2024584 100644 --- a/docs/src/cdist-configuration.rst +++ b/docs/src/cdist-configuration.rst @@ -21,7 +21,7 @@ precedence. Configuration option value read from source with higher precedence will overwrite option value, if exists, read from source with lower precedence. That means that command-line option wins them all. -Users can decide on the local conifguration file location. It can be either +Users can decide on the local configuration file location. It can be either ~/.cdist.cfg or $XDG_CONFIG_HOME/cdist/cdist.cfg. Note that, if both exist, then ~/.cdist.cfg is used. From 3d725f12dace8d2e977d408b7c01fa28b8b17bde Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Wed, 3 Jun 2020 23:10:27 +0300 Subject: [PATCH 050/155] [docs] Fixed a typo and s/posix/POSIX/ in cdist-why.rst Signed-off-by: Jaak Ristioja --- docs/src/cdist-why.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/cdist-why.rst b/docs/src/cdist-why.rst index 1123a1de..0e2cd34d 100644 --- a/docs/src/cdist-why.rst +++ b/docs/src/cdist-why.rst @@ -21,7 +21,7 @@ Not only is shell scripting widely known by system engineers, but it is also a very powerful language. Here are some features which make daily work easy: - * Configuration can react dynamicly on explored values + * Configuration can react dynamically on explored values * High level string manipulation (using sed, awk, grep) * Conditional support (**if, case**) * Loop support (**for, while**) @@ -44,7 +44,7 @@ Cdist requires very little on a target system. Even better, in almost all cases all dependencies are usually fulfilled. Cdist does not require an agent or high level programming languages on the target host: it will run on any host that -has a **ssh server running** and a posix compatible shell +has a **ssh server running** and a POSIX compatible shell (**/bin/sh**). Compared to other configuration management systems, it does not require to open up an additional port. From 46574fc5779b939db5d90e8c11f28e808be458be Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Wed, 3 Jun 2020 23:12:10 +0300 Subject: [PATCH 051/155] [docs] Fixed three typos in cdist-upgrade.rst Signed-off-by: Jaak Ristioja --- docs/src/cdist-upgrade.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/cdist-upgrade.rst b/docs/src/cdist-upgrade.rst index 67fd4934..f745b212 100644 --- a/docs/src/cdist-upgrade.rst +++ b/docs/src/cdist-upgrade.rst @@ -14,7 +14,7 @@ To upgrade cdist in the current branch use make man export MANPATH=$MANPATH:$(pwd -P)/doc/man -If you stay on a version branche (i.e. 1.0, 1.1., ...), nothing should break. +If you stay on a version branch (i.e. 1.0, 1.1., ...), nothing should break. The master branch on the other hand is the development branch and may not be working, break your setup or eat the tree in your garden. @@ -61,7 +61,7 @@ After that, you can go back and continue the upgrade: Update the python package ------------------------- -To upgrade to the lastet version do +To upgrade to the latest version do .. code-block:: sh @@ -158,7 +158,7 @@ Updating from 1.5 to 1.6 * If you used **\_\_package_apt --preseed**, you need to use the new type **\_\_debconf_set_selections** instead. * The **\_\_package** types accepted either --state deinstalled or - --state uninstaaled. Starting with 1.6, it was made consistently + --state uninstalled. Starting with 1.6, it was made consistently to --state removed. Updating from 1.3 to 1.5 From e1ff1bfdffd9a188a045669c4168dcd1fb561b0b Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Wed, 3 Jun 2020 23:15:38 +0300 Subject: [PATCH 052/155] [docs] Fixed two typos in cdist-real-world.rst Signed-off-by: Jaak Ristioja --- docs/src/cdist-real-world.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/cdist-real-world.rst b/docs/src/cdist-real-world.rst index ba118d63..973dcd6d 100644 --- a/docs/src/cdist-real-world.rst +++ b/docs/src/cdist-real-world.rst @@ -198,7 +198,7 @@ We require package uWSGI present in order to create **/etc/uwsgi/apps-enabled/$u Installation of uWSGI also creates configuration layout: **/etc/uwsgi/apps-enabled**. If this directory does not exist then **__file** type would error. We also use stdin as file content source. For details see `Input from stdin `_. -For feading stdin we use here-document (**<<** operator). It allows redirection of subsequent +For feeding stdin we use here-document (**<<** operator). It allows redirection of subsequent lines read by the shell to the input of a command until a line containing only the delimiter and a newline, with no blank characters in between (EOF in our case). @@ -546,7 +546,7 @@ we have changed our **wsgi.py** file uWSGI reloads the application. Our application selects and lists items from **items** table. -Openning application +Opening application ~~~~~~~~~~~~~~~~~~~~ Finally try the application:: From 978e24904336e6af074a7de757282456ff0314a4 Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Wed, 3 Jun 2020 23:23:49 +0300 Subject: [PATCH 053/155] [docs] Fixed capitalization of POSIX in cdist-install.rst Signed-off-by: Jaak Ristioja --- docs/src/cdist-install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst index 880b9f8e..0b65cad6 100644 --- a/docs/src/cdist-install.rst +++ b/docs/src/cdist-install.rst @@ -9,7 +9,7 @@ Source Host This is the machine from which you will configure target hosts. - * /bin/sh: A posix like shell (for instance bash, dash, zsh) + * /bin/sh: A POSIX like shell (for instance bash, dash, zsh) * Python >= 3.2 * SSH client * sphinx (for building html docs and/or the man pages) @@ -17,7 +17,7 @@ This is the machine from which you will configure target hosts. Target Hosts ~~~~~~~~~~~~ - * /bin/sh: A posix like shell (for instance bash, dash, zsh) + * /bin/sh: A POSIX like shell (for instance bash, dash, zsh) * SSH server Install cdist From dc018fdb16ba8b6044bbb55efd44e975fc5247ae Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Wed, 3 Jun 2020 23:25:46 +0300 Subject: [PATCH 054/155] [docs] Fixed typo in cdist-cache.rst Signed-off-by: Jaak Ristioja --- docs/src/cdist-cache.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/cdist-cache.rst b/docs/src/cdist-cache.rst index 0e5361ee..d2d2d56c 100644 --- a/docs/src/cdist-cache.rst +++ b/docs/src/cdist-cache.rst @@ -59,7 +59,7 @@ typeorder Object cache overview ~~~~~~~~~~~~~~~~~~~~~ -Each object under :strong:`object` directory has its own structurue. +Each object under :strong:`object` directory has its own structure. code-local code generated from gencode-local, present only if something is From 7b262c0cec2c3c42822cef103220965424500bcb Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Wed, 3 Jun 2020 23:27:16 +0300 Subject: [PATCH 055/155] [docs] Fixed capitalization of URLs in cdist-bootstrap.rst Signed-off-by: Jaak Ristioja --- docs/src/cdist-bootstrap.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/cdist-bootstrap.rst b/docs/src/cdist-bootstrap.rst index c9972a99..10f86c5a 100644 --- a/docs/src/cdist-bootstrap.rst +++ b/docs/src/cdist-bootstrap.rst @@ -25,7 +25,7 @@ people, have a look at `cdist best practice `_. Setup working directory and branch ---------------------------------- I assume you have a fresh copy of the cdist tree in ~/cdist, cloned from -one of the official urls (see `cdist quickstart `_ if you don't). +one of the official URLs (see `cdist quickstart `_ if you don't). Entering the command "git branch" should show you "* master", which indicates you are on the **master** branch. From 191f45eb7f8ab8ce97f996f3bb8375da5142d39a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 8 Jun 2020 13:48:37 +0200 Subject: [PATCH 056/155] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 94ef55d8..e73cfa6f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,8 @@ next: * Core: Support colored log output (Evil Ham) * Core: Tune colored log output and respect NO_COLOR env var (Dennis Camera) * Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak) + * Documentation: Fix trivial grammatical mistakes (Jaak Ristioja) + * Explorer os: Fix for sles15 (Daniel Heule) 6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) From 955243a93b28d3524f93eb6d69af1f0eec5ff36d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 9 Jun 2020 12:51:19 +0200 Subject: [PATCH 057/155] Update cdist man page copyright years --- docs/src/man1/cdist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index aae98538..68a9a4ee 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -969,5 +969,5 @@ such case and display a warning message. An example of such a case: COPYING ------- -Copyright \(C) 2011-2019 Nico Schottelius. Free use of this software is +Copyright \(C) 2011-2020 Nico Schottelius. Free use of this software is granted under the terms of the GNU General Public License v3 or later (GPLv3+). From 7c490a703d0203747e481a9fc3405bf7eb5ba01a Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 10 Jun 2020 11:38:14 +0300 Subject: [PATCH 058/155] [__clean_path] add --path parameter --- cdist/conf/type/__clean_path/explorer/list | 7 ++++++- cdist/conf/type/__clean_path/gencode-remote | 7 ++++++- cdist/conf/type/__clean_path/man.rst | 10 +++++++++- cdist/conf/type/__clean_path/parameter/optional | 1 + 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__clean_path/explorer/list b/cdist/conf/type/__clean_path/explorer/list index 07d38127..2bdc63a5 100755 --- a/cdist/conf/type/__clean_path/explorer/list +++ b/cdist/conf/type/__clean_path/explorer/list @@ -18,7 +18,12 @@ # along with cdist. If not, see . # -path="/$__object_id" +if [ -f "$__object/parameter/path" ] +then + path="$( cat "$__object/parameter/path" )" +else + path="/$__object_id" +fi [ ! -d "$path" ] && exit 0 diff --git a/cdist/conf/type/__clean_path/gencode-remote b/cdist/conf/type/__clean_path/gencode-remote index 998a70d8..2899c4a5 100755 --- a/cdist/conf/type/__clean_path/gencode-remote +++ b/cdist/conf/type/__clean_path/gencode-remote @@ -20,7 +20,12 @@ [ ! -s "$__object/explorer/list" ] && exit 0 -path="/$__object_id" +if [ -f "$__object/parameter/path" ] +then + path="$( cat "$__object/parameter/path" )" +else + path="/$__object_id" +fi pattern="$( cat "$__object/parameter/pattern" )" diff --git a/cdist/conf/type/__clean_path/man.rst b/cdist/conf/type/__clean_path/man.rst index 826f4589..31d90701 100644 --- a/cdist/conf/type/__clean_path/man.rst +++ b/cdist/conf/type/__clean_path/man.rst @@ -10,7 +10,7 @@ DESCRIPTION ----------- Remove files and directories which match the pattern. -Provided path (as __object_id) must be a directory. +Provided path must be a directory. Patterns are passed to ``find``'s ``-regex`` - see ``find(1)`` for more details. @@ -29,6 +29,9 @@ pattern OPTIONAL PARAMETERS ------------------- +path + Path which will be cleaned. Defaults to ``$__object_id``. + exclude Pattern of files which are excluded from removal. @@ -46,6 +49,11 @@ EXAMPLES --exclude '.+\(charset\.conf\|security\.conf\)' \ --onchange 'service apache2 restart' + __clean_path apache2-conf-enabled \ + --path /etc/apache2/conf-enabled \ + --pattern '.+' \ + --exclude '.+\(charset\.conf\|security\.conf\)' \ + --onchange 'service apache2 restart' AUTHORS ------- diff --git a/cdist/conf/type/__clean_path/parameter/optional b/cdist/conf/type/__clean_path/parameter/optional index 6f313474..3b97f71c 100644 --- a/cdist/conf/type/__clean_path/parameter/optional +++ b/cdist/conf/type/__clean_path/parameter/optional @@ -1,2 +1,3 @@ exclude onchange +path From 74e5d7182a466cd02b79137f1d73500c0cc50808 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 10 Jun 2020 10:45:20 +0200 Subject: [PATCH 059/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index e73cfa6f..bcde9268 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak) * Documentation: Fix trivial grammatical mistakes (Jaak Ristioja) * Explorer os: Fix for sles15 (Daniel Heule) + * Type __clean_path: Add --path parameter (Ander Punnar) 6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) From 840e417eb79c7135762d3353f34a3f8330cb0e84 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 11 Jun 2020 14:16:37 +0200 Subject: [PATCH 060/155] Fix emulator colored logging --- cdist/argparse.py | 2 +- cdist/core/manifest.py | 3 ++- cdist/emulator.py | 2 +- cdist/flock.py | 1 + cdist/log.py | 16 +++++++--------- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 85dba246..1c83237c 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -498,7 +498,7 @@ def handle_loglevel(args): def handle_log_colors(args): if cdist.configuration.ColoredOutputOption.translate(args.colored_output): - cdist.log.DefaultLog.USE_COLORS = True + cdist.log.CdistFormatter.USE_COLORS = True def parse_and_configure(argv, singleton=True): diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 8b833ff2..2e7b873e 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -119,7 +119,8 @@ class Manifest(object): '__cdist_log_level': util.log_level_env_var_val(self.log), '__cdist_log_level_name': util.log_level_name_env_var_val( self.log), - '__cdist_colored_log': str(self.log.USE_COLORS).lower(), + '__cdist_colored_log': str( + cdist.log.CdistFormatter.USE_COLORS).lower(), } if dry_run: diff --git a/cdist/emulator.py b/cdist/emulator.py index 4eaf2c93..08e15a78 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -130,7 +130,7 @@ class Emulator(object): logging.root.setLevel(logging.WARNING) colored_log = self.env.get('__cdist_colored_log', 'false') - cdist.log.ColorFormatter.USE_COLORS = colored_log == 'true' + cdist.log.CdistFormatter.USE_COLORS = colored_log == 'true' self.log = logging.getLogger(self.target_host[0]) diff --git a/cdist/flock.py b/cdist/flock.py index d8bac916..3afacfd9 100644 --- a/cdist/flock.py +++ b/cdist/flock.py @@ -22,6 +22,7 @@ import fcntl import logging import os +import cdist.log log = logging.getLogger('cdist-flock') diff --git a/cdist/log.py b/cdist/log.py index 19efebdb..efae2cd3 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -50,7 +50,8 @@ def _trace(msg, *args, **kwargs): logging.trace = _trace -class ColorFormatter(logging.Formatter): +class CdistFormatter(logging.Formatter): + USE_COLORS = False RESET = '\033[0m' COLOR_MAP = { 'ERROR': '\033[0;31m', @@ -66,14 +67,14 @@ class ColorFormatter(logging.Formatter): def format(self, record): msg = super().format(record) - color = self.COLOR_MAP.get(record.levelname) - if color: - msg = color + msg + self.RESET + if self.USE_COLORS: + color = self.COLOR_MAP.get(record.levelname) + if color: + msg = color + msg + self.RESET return msg class DefaultLog(logging.Logger): - USE_COLORS = False FORMAT = '%(levelname)s: %(message)s' class StdoutFilter(logging.Filter): @@ -88,10 +89,7 @@ class DefaultLog(logging.Logger): super().__init__(name) self.propagate = False - if self.USE_COLORS: - formatter = ColorFormatter(self.FORMAT) - else: - formatter = logging.Formatter(self.FORMAT) + formatter = CdistFormatter(self.FORMAT) self.addFilter(self) From 4167f9f60c30a4587b492c2fa3dce2f02ded463a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 11 Jun 2020 14:22:54 +0200 Subject: [PATCH 061/155] Use proper format string with name --- cdist/log.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/cdist/log.py b/cdist/log.py index efae2cd3..790059df 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -75,7 +75,7 @@ class CdistFormatter(logging.Formatter): class DefaultLog(logging.Logger): - FORMAT = '%(levelname)s: %(message)s' + FORMAT = '%(levelname)s: %(name)s: %(message)s' class StdoutFilter(logging.Filter): def filter(self, rec): @@ -91,8 +91,6 @@ class DefaultLog(logging.Logger): formatter = CdistFormatter(self.FORMAT) - self.addFilter(self) - stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.addFilter(self.StdoutFilter()) stdout_handler.setLevel(logging.TRACE) @@ -106,13 +104,6 @@ class DefaultLog(logging.Logger): self.addHandler(stdout_handler) self.addHandler(stderr_handler) - def filter(self, record): - """Prefix messages with logger name""" - - record.msg = self.name + ": " + str(record.msg) - - return True - def verbose(self, msg, *args, **kwargs): self.log(logging.VERBOSE, msg, *args, **kwargs) @@ -134,7 +125,7 @@ class TimestampingLog(DefaultLog): class ParallelLog(DefaultLog): - FORMAT = '%(levelname)s: [%(process)d]: %(message)s' + FORMAT = '%(levelname)s: [%(process)d]: %(name)s: %(message)s' class TimestampingParallelLog(TimestampingLog, ParallelLog): From eec7ab8e4577008c6a48a473e295a9d1e31f2358 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 10 Jun 2020 16:59:18 +0200 Subject: [PATCH 062/155] Increase minimum supported Python version to 3.5 --- docs/src/cdist-install.rst | 2 +- docs/src/man1/cdist.rst | 2 +- scripts/cdist | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst index 0b65cad6..6f4f14d7 100644 --- a/docs/src/cdist-install.rst +++ b/docs/src/cdist-install.rst @@ -10,7 +10,7 @@ Source Host This is the machine from which you will configure target hosts. * /bin/sh: A POSIX like shell (for instance bash, dash, zsh) - * Python >= 3.2 + * Python >= 3.5 * SSH client * sphinx (for building html docs and/or the man pages) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 68a9a4ee..9bf8fc9b 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -95,7 +95,7 @@ cdist is the frontend executable to the cdist configuration management. It supports different subcommands as explained below. It is written in Python so it requires :strong:`python`\ (1) to be installed. -It requires a minimal Python version 3.2. +It requires a minimal Python version 3.5. GENERAL ------- diff --git a/scripts/cdist b/scripts/cdist index 7bf12c01..664504a0 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -60,7 +60,7 @@ def commandline(): if __name__ == "__main__": - cdistpythonversion = '3.2' + cdistpythonversion = '3.5' if sys.version < cdistpythonversion: print('Python >= {} is required on the source host.'.format( cdistpythonversion), file=sys.stderr) From eba3d0505bba4ee0472df3452142cc63c8fe2f5c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 12 Jun 2020 06:29:39 +0200 Subject: [PATCH 063/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index bcde9268..0cc2a714 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ next: * Documentation: Fix trivial grammatical mistakes (Jaak Ristioja) * Explorer os: Fix for sles15 (Daniel Heule) * Type __clean_path: Add --path parameter (Ander Punnar) + * Core: Increase minimal supported Python version to 3.5 (Darko Poljak) 6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) From 6e9e9ad557479354938257f04c8fab283b8251e8 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 12 Nov 2019 00:40:58 +0100 Subject: [PATCH 064/155] implement log server to capture nested logging output Signed-off-by: Steven Armstrong --- cdist/core/code.py | 4 +++ cdist/install.py | 23 +++++++++++++- cdist/log.py | 77 ++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 90 insertions(+), 14 deletions(-) diff --git a/cdist/core/code.py b/cdist/core/code.py index 1550880a..a7d9b7ca 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -116,6 +116,10 @@ class Code(object): if dry_run: self.env['__cdist_dry_run'] = '1' + if '__cdist_log_server_socket_to_export' in os.environ: + self.env['__cdist_log_server_socket'] = os.environ['__cdist_log_server_socket_to_export'] + + def _run_gencode(self, cdist_object, which): cdist_type = cdist_object.cdist_type script = os.path.join(self.local.type_path, diff --git a/cdist/install.py b/cdist/install.py index b88ad016..3f94ca68 100644 --- a/cdist/install.py +++ b/cdist/install.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# 2013 Steven Armstrong (steven-cdist at armstrong.cc) +# 2013-2019 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -20,11 +20,32 @@ # # +import os +import logging +import tempfile + import cdist.config import cdist.core class Install(cdist.config.Config): + + @classmethod + def onehost(cls, host, host_tags, host_base_path, host_dir_name, args, + parallel, configuration, remove_remote_files_dirs=False): + # Start a log server so nested `cdist config` runs have a place to + # send their logs to. + log_server_socket_dir = tempfile.mkdtemp() + log_server_socket = os.path.join(log_server_socket_dir, 'log-server') + cls._register_path_for_removal(log_server_socket_dir) + log = logging.getLogger(host) + log.debug('Starting logging server on: %s', log_server_socket) + os.environ['__cdist_log_server_socket_to_export'] = log_server_socket + cdist.log.setupLogServer(log_server_socket) + + super().onehost(host, host_tags, host_base_path, host_dir_name, args, + parallel, configuration, remove_remote_files_dirs=False) + def object_list(self): """Short name for object list retrieval. In install mode, we only care about install objects. diff --git a/cdist/log.py b/cdist/log.py index 790059df..94dd11e8 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # # 2010-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2019-2020 Steven Armstrong # # This file is part of cdist. # @@ -20,9 +21,17 @@ # # -import logging -import sys +import asyncio +import contextlib import datetime +import logging +import logging.handlers +import os +import pickle +import struct +import sys +import threading +import time # Define additional cdist logging levels. @@ -89,20 +98,25 @@ class DefaultLog(logging.Logger): super().__init__(name) self.propagate = False - formatter = CdistFormatter(self.FORMAT) + if '__cdist_log_server_socket' in os.environ: + log_server_socket = os.environ['__cdist_log_server_socket'] + socket_handler = logging.handlers.SocketHandler(log_server_socket, None) + self.addHandler(socket_handler) + else: + formatter = CdistFormatter(self.FORMAT) - stdout_handler = logging.StreamHandler(sys.stdout) - stdout_handler.addFilter(self.StdoutFilter()) - stdout_handler.setLevel(logging.TRACE) - stdout_handler.setFormatter(formatter) + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.addFilter(self.StdoutFilter()) + stdout_handler.setLevel(logging.TRACE) + stdout_handler.setFormatter(formatter) - stderr_handler = logging.StreamHandler(sys.stderr) - stderr_handler.addFilter(self.StderrFilter()) - stderr_handler.setLevel(logging.ERROR) - stderr_handler.setFormatter(formatter) + stderr_handler = logging.StreamHandler(sys.stderr) + stderr_handler.addFilter(self.StderrFilter()) + stderr_handler.setLevel(logging.ERROR) + stderr_handler.setFormatter(formatter) - self.addHandler(stdout_handler) - self.addHandler(stderr_handler) + self.addHandler(stdout_handler) + self.addHandler(stderr_handler) def verbose(self, msg, *args, **kwargs): self.log(logging.VERBOSE, msg, *args, **kwargs) @@ -152,4 +166,41 @@ def setupParallelLogging(): logging.setLoggerClass(ParallelLog) +async def handle_log_client(reader, writer): + while True: + chunk = await reader.read(4) + if len(chunk) < 4: + return + + data_size = struct.unpack('>L', chunk)[0] + data = bytearray(data_size) + view = memoryview(data) + data_pending = data_size + data = await reader.read(data_size) + + obj = pickle.loads(data) + record = logging.makeLogRecord(obj) + logger = logging.getLogger(record.name) + logger.handle(record) + + +def run_log_server(server_address): + # Get a new loop inside the current thread to run the log server. + loop = asyncio.new_event_loop() + loop.create_task(asyncio.start_unix_server(handle_log_client, server_address)) + loop.run_forever() + + +def setupLogServer(log_server_socket): + """Run a asyncio based unix socket log server in a background thread. + """ + with contextlib.suppress(FileNotFoundError): + os.remove(log_server_socket) + t = threading.Thread(target=run_log_server, args=(log_server_socket,)) + # Deamonizing the thread means we don't have to care about stoping it. + # It will die together with the main process. + t.daemon = True + t.start() + + setupDefaultLogging() From 831bfc822b61a5e5cf8186a32ea592622693efdb Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 12 Nov 2019 09:22:28 +0100 Subject: [PATCH 065/155] remove unused code Signed-off-by: Steven Armstrong --- cdist/log.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cdist/log.py b/cdist/log.py index 94dd11e8..c1376a58 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -173,9 +173,6 @@ async def handle_log_client(reader, writer): return data_size = struct.unpack('>L', chunk)[0] - data = bytearray(data_size) - view = memoryview(data) - data_pending = data_size data = await reader.read(data_size) obj = pickle.loads(data) From 57e352cd1e7b400ca1daf7e1e680874bd9a6cc01 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sat, 11 Jan 2020 01:59:18 +0100 Subject: [PATCH 066/155] log server is also usefull for cdist config Signed-off-by: Steven Armstrong --- cdist/argparse.py | 6 ++++++ cdist/config.py | 10 ++++++++++ cdist/install.py | 13 +++---------- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 1c83237c..15b9ca7a 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -250,6 +250,12 @@ def get_parsers(): '-S', '--disable-saving-output-streams', help='Disable saving output streams.', action='store_false', dest='save_output_streams', default=True) + parser['config_main'].add_argument( + '--log-server', + action='store_true', + help=('Start a log server for sub processes to use.' + 'This is mainly usefull when running cdist nested' + 'from a code-local script.')) # Config parser['config_args'] = argparse.ArgumentParser(add_help=False) diff --git a/cdist/config.py b/cdist/config.py index b2d72f05..82e8559b 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -385,6 +385,16 @@ class Config(object): log = logging.getLogger(host) + if args.log_server: + # Start a log server so that nested `cdist config` runs have a place + # to send their logs to. + log_server_socket_dir = tempfile.mkdtemp() + log_server_socket = os.path.join(log_server_socket_dir, 'log-server') + cls._register_path_for_removal(log_server_socket_dir) + log.debug('Starting logging server on: %s', log_server_socket) + os.environ['__cdist_log_server_socket_to_export'] = log_server_socket + cdist.log.setupLogServer(log_server_socket) + try: remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds( args) diff --git a/cdist/install.py b/cdist/install.py index 3f94ca68..ec0a7270 100644 --- a/cdist/install.py +++ b/cdist/install.py @@ -33,16 +33,9 @@ class Install(cdist.config.Config): @classmethod def onehost(cls, host, host_tags, host_base_path, host_dir_name, args, parallel, configuration, remove_remote_files_dirs=False): - # Start a log server so nested `cdist config` runs have a place to - # send their logs to. - log_server_socket_dir = tempfile.mkdtemp() - log_server_socket = os.path.join(log_server_socket_dir, 'log-server') - cls._register_path_for_removal(log_server_socket_dir) - log = logging.getLogger(host) - log.debug('Starting logging server on: %s', log_server_socket) - os.environ['__cdist_log_server_socket_to_export'] = log_server_socket - cdist.log.setupLogServer(log_server_socket) - + # Always start log server during cdist install so that nested + # `cdist config` runs have a place to send their logs to. + args.log_server = True super().onehost(host, host_tags, host_base_path, host_dir_name, args, parallel, configuration, remove_remote_files_dirs=False) From 59b98091d78c5195d2aa4a2ea3ef79f93e5ba67c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 9 Jun 2020 12:47:50 +0200 Subject: [PATCH 067/155] Adapt; update docs and code style --- cdist/__init__.py | 4 ++++ cdist/argparse.py | 13 +++++++------ cdist/config.py | 27 ++++++++++++--------------- cdist/core/code.py | 6 +++--- cdist/install.py | 8 +++----- cdist/log.py | 18 +++++++++++------- docs/src/man1/cdist.rst | 26 ++++++++++++++++---------- scripts/cdist | 5 ++--- 8 files changed, 58 insertions(+), 49 deletions(-) diff --git a/cdist/__init__.py b/cdist/__init__.py index c673b3ba..be573170 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -26,6 +26,7 @@ import hashlib import cdist.log import cdist.version + VERSION = cdist.version.VERSION BANNER = """ @@ -48,6 +49,9 @@ REMOTE_EXEC = "ssh -o User=root" REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}" +MIN_SUPPORTED_PYTHON_VERSION = '3.5' + + class Error(Exception): """Base exception class for this project""" pass diff --git a/cdist/argparse.py b/cdist/argparse.py index 15b9ca7a..77303591 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -206,6 +206,13 @@ def get_parsers(): 'supported. Without argument CPU count is used by default. '), action='store', dest='jobs', const=multiprocessing.cpu_count()) + parser['config_main'].add_argument( + '--log-server', + action='store_true', + help=('Start a log server for sub processes to use. ' + 'This is mainly useful when running cdist nested ' + 'from a code-local script. Log server is alwasy ' + 'implicitly started for \'install\' command.')) parser['config_main'].add_argument( '-n', '--dry-run', help='Do not execute code.', action='store_true') @@ -250,12 +257,6 @@ def get_parsers(): '-S', '--disable-saving-output-streams', help='Disable saving output streams.', action='store_false', dest='save_output_streams', default=True) - parser['config_main'].add_argument( - '--log-server', - action='store_true', - help=('Start a log server for sub processes to use.' - 'This is mainly usefull when running cdist nested' - 'from a code-local script.')) # Config parser['config_args'] = argparse.ArgumentParser(add_help=False) diff --git a/cdist/config.py b/cdist/config.py index 82e8559b..9fe9b676 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -29,18 +29,20 @@ import time import itertools import tempfile import multiprocessing -from cdist.mputil import mp_pool_run, mp_sig_handler import atexit import shutil import socket + +from cdist.mputil import mp_pool_run, mp_sig_handler +from cdist import core, inventory +from cdist.util.remoteutil import inspect_ssh_mux_opts + import cdist import cdist.hostsource import cdist.exec.local import cdist.exec.remote import cdist.util.ipaddr as ipaddr import cdist.configuration -from cdist import core, inventory -from cdist.util.remoteutil import inspect_ssh_mux_opts def graph_check_cycle(graph): @@ -195,7 +197,6 @@ class Config(object): @classmethod def commandline(cls, args): """Configure remote system""" - if (args.parallel and args.parallel != 1) or args.jobs: if args.timestamp: cdist.log.setupTimestampingParallelLogging() @@ -382,20 +383,16 @@ class Config(object): If operating in parallel then return tuple (host, True|False, ) so that main process knows for which host function was successful. """ - log = logging.getLogger(host) - if args.log_server: - # Start a log server so that nested `cdist config` runs have a place - # to send their logs to. - log_server_socket_dir = tempfile.mkdtemp() - log_server_socket = os.path.join(log_server_socket_dir, 'log-server') - cls._register_path_for_removal(log_server_socket_dir) - log.debug('Starting logging server on: %s', log_server_socket) - os.environ['__cdist_log_server_socket_to_export'] = log_server_socket - cdist.log.setupLogServer(log_server_socket) - try: + if args.log_server: + # Start a log server so that nested `cdist config` runs + # have a place to send their logs to. + log_server_socket_dir = tempfile.mkdtemp() + cls._register_path_for_removal(log_server_socket_dir) + cdist.log.setupLogServer(log_server_socket_dir, log) + remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds( args) log.debug("remote_exec for host \"{}\": {}".format( diff --git a/cdist/core/code.py b/cdist/core/code.py index a7d9b7ca..2a30908e 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -116,9 +116,9 @@ class Code(object): if dry_run: self.env['__cdist_dry_run'] = '1' - if '__cdist_log_server_socket_to_export' in os.environ: - self.env['__cdist_log_server_socket'] = os.environ['__cdist_log_server_socket_to_export'] - + if '__cdist_log_server_socket_export' in os.environ: + self.env['__cdist_log_server_socket'] = os.environ[ + '__cdist_log_server_socket_export'] def _run_gencode(self, cdist_object, which): cdist_type = cdist_object.cdist_type diff --git a/cdist/install.py b/cdist/install.py index ec0a7270..561b2fa6 100644 --- a/cdist/install.py +++ b/cdist/install.py @@ -20,12 +20,9 @@ # # -import os -import logging -import tempfile - import cdist.config import cdist.core +import cdist.log class Install(cdist.config.Config): @@ -36,8 +33,9 @@ class Install(cdist.config.Config): # Always start log server during cdist install so that nested # `cdist config` runs have a place to send their logs to. args.log_server = True + super().onehost(host, host_tags, host_base_path, host_dir_name, args, - parallel, configuration, remove_remote_files_dirs=False) + parallel, configuration, remove_remote_files_dirs) def object_list(self): """Short name for object list retrieval. diff --git a/cdist/log.py b/cdist/log.py index c1376a58..bee99fac 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -21,17 +21,16 @@ # # -import asyncio -import contextlib import datetime import logging import logging.handlers +import sys import os +import asyncio +import contextlib import pickle import struct -import sys import threading -import time # Define additional cdist logging levels. @@ -100,7 +99,8 @@ class DefaultLog(logging.Logger): if '__cdist_log_server_socket' in os.environ: log_server_socket = os.environ['__cdist_log_server_socket'] - socket_handler = logging.handlers.SocketHandler(log_server_socket, None) + socket_handler = logging.handlers.SocketHandler(log_server_socket, + None) self.addHandler(socket_handler) else: formatter = CdistFormatter(self.FORMAT) @@ -184,13 +184,17 @@ async def handle_log_client(reader, writer): def run_log_server(server_address): # Get a new loop inside the current thread to run the log server. loop = asyncio.new_event_loop() - loop.create_task(asyncio.start_unix_server(handle_log_client, server_address)) + loop.create_task(asyncio.start_unix_server(handle_log_client, + server_address)) loop.run_forever() -def setupLogServer(log_server_socket): +def setupLogServer(socket_dir, log=logging.getLogger(__name__)): """Run a asyncio based unix socket log server in a background thread. """ + log_server_socket = os.path.join(socket_dir, 'log-server') + log.debug('Starting logging server on: %s', log_server_socket) + os.environ['__cdist_log_server_socket_export'] = log_server_socket with contextlib.suppress(FileNotFoundError): os.remove(log_server_socket) t = threading.Thread(target=run_log_server, args=(log_server_socket,)) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 9bf8fc9b..aa2607f8 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -17,20 +17,20 @@ SYNOPSIS cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN] - [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-P] - [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] - [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] - [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] - [-p [HOST_MAX]] [-s] [-t] + [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [--log-server] + [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]] + [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] + [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A] + [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t] [host [host ...]] cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN] - [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] - [-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] - [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] - [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] - [-p [HOST_MAX]] [-s] [-t] + [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [--log-server] + [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]] + [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] + [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A] + [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t] [host [host ...]] cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ... @@ -202,6 +202,12 @@ Install command is currently in beta. are supported. Without argument CPU count is used by default. +**--log-server** + Start a log server for sub processes to use. This is + mainly useful when running cdist nested from a code- + local script. Log server is always implicitly started + for 'install' command. + **-n, --dry-run** Do not execute code. diff --git a/scripts/cdist b/scripts/cdist index 664504a0..b1d782ab 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -60,10 +60,9 @@ def commandline(): if __name__ == "__main__": - cdistpythonversion = '3.5' - if sys.version < cdistpythonversion: + if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION: print('Python >= {} is required on the source host.'.format( - cdistpythonversion), file=sys.stderr) + cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr) sys.exit(1) exit_code = 0 From 5be8437a602907009d7603fa41ae0a4e6528ffdf Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 13 Jun 2020 13:48:34 +0200 Subject: [PATCH 068/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 0cc2a714..fa5f7405 100644 --- a/docs/changelog +++ b/docs/changelog @@ -10,6 +10,7 @@ next: * Explorer os: Fix for sles15 (Daniel Heule) * Type __clean_path: Add --path parameter (Ander Punnar) * Core: Increase minimal supported Python version to 3.5 (Darko Poljak) + * Core: Add log server for nested logging (Steven Armstrong) 6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) From 201050a9e5144d0f7da2a8019e5822fedaa1d1b1 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 14 Jun 2020 11:07:18 +0300 Subject: [PATCH 069/155] new type: __download --- cdist/conf/type/__download/explorer/state | 20 ++++++ cdist/conf/type/__download/gencode-local | 35 ++++++++++ cdist/conf/type/__download/man.rst | 66 +++++++++++++++++++ .../type/__download/parameter/default/cmd-get | 1 + .../type/__download/parameter/default/cmd-sum | 1 + cdist/conf/type/__download/parameter/optional | 2 + cdist/conf/type/__download/parameter/required | 2 + 7 files changed, 127 insertions(+) create mode 100755 cdist/conf/type/__download/explorer/state create mode 100755 cdist/conf/type/__download/gencode-local create mode 100644 cdist/conf/type/__download/man.rst create mode 100644 cdist/conf/type/__download/parameter/default/cmd-get create mode 100644 cdist/conf/type/__download/parameter/default/cmd-sum create mode 100644 cdist/conf/type/__download/parameter/optional create mode 100644 cdist/conf/type/__download/parameter/required diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state new file mode 100755 index 00000000..6a50f5a5 --- /dev/null +++ b/cdist/conf/type/__download/explorer/state @@ -0,0 +1,20 @@ +#!/bin/sh -e + +dst="/$__object_id" + +# shellcheck disable=SC2059 +cmd="$( printf "$( cat "$__object/parameter/cmd-sum" )" "$dst" )" + +sum="$( cat "$__object/parameter/sum" )" + +if [ -f "$dst" ] +then + if [ "$( eval "$cmd" )" = "$sum" ] + then + echo 'present' + else + echo 'mismatch' + fi +else + echo 'absent' +fi diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local new file mode 100755 index 00000000..49e9c699 --- /dev/null +++ b/cdist/conf/type/__download/gencode-local @@ -0,0 +1,35 @@ +#!/bin/sh -e + +state_is="$( cat "$__object/explorer/state" )" + +if [ "$state_is" = 'present' ] +then + exit 0 +fi + +url="$( cat "$__object/parameter/url" )" + +cmd="$( cat "$__object/parameter/cmd-get" )" + +tmp="$( mktemp )" + +dst="/$__object_id" + +printf "$cmd > %s\n" \ + "$url" \ + "$tmp" + +if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$' +then + target_host="[$__target_host]" +else + target_host="$__target_host" +fi + +printf '%s %s %s:%s\n' \ + "$__remote_copy" \ + "$tmp" \ + "$target_host" \ + "$dst" + +echo "rm -f '$tmp'" diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst new file mode 100644 index 00000000..c973448f --- /dev/null +++ b/cdist/conf/type/__download/man.rst @@ -0,0 +1,66 @@ +cdist-type__download(7) +======================= + +NAME +---- +cdist-type__download - Download file to local storage and copy it to target host + + +DESCRIPTION +----------- +You must use persistent storage in target host for destination file +(``$__object_id``) because it will be used for checksum calculation +in order to decide if file must be downloaded. + + +REQUIRED PARAMETERS +------------------- +url + URL from which to download the file. + +sum + Checksum of downloaded file. + + +OPTIONAL PARAMETERS +------------------- +cmd-get + Command used for downloading. + Default is ``wget -O- '%s'``. + Command must output to ``stdout``. + +cmd-sum + Command used for checksum calculation. + Default is ``md5sum '%s' | awk '{print $1}'``. + Command output and ``--sum`` parameter must match. + + +EXAMPLES +-------- + +.. code-block:: sh + + __directory /opt/cpma + + require='__directory/opt/cpma' \ + __download /opt/cpma/cnq3.zip \ + --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \ + --sum 46da3021ca9eace277115ec9106c5b46 + + require='__download/opt/cpma/cnq3.zip' \ + __unpack /opt/cpma/cnq3.zip \ + --move-existing-destination \ + --destination /opt/cpma/server + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2020 Ander Punnar. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__download/parameter/default/cmd-get b/cdist/conf/type/__download/parameter/default/cmd-get new file mode 100644 index 00000000..2daa38a1 --- /dev/null +++ b/cdist/conf/type/__download/parameter/default/cmd-get @@ -0,0 +1 @@ +wget -O- '%s' diff --git a/cdist/conf/type/__download/parameter/default/cmd-sum b/cdist/conf/type/__download/parameter/default/cmd-sum new file mode 100644 index 00000000..3e8a9295 --- /dev/null +++ b/cdist/conf/type/__download/parameter/default/cmd-sum @@ -0,0 +1 @@ +md5sum '%s' | awk '{print $1}' diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional new file mode 100644 index 00000000..22783e02 --- /dev/null +++ b/cdist/conf/type/__download/parameter/optional @@ -0,0 +1,2 @@ +cmd-get +cmd-sum diff --git a/cdist/conf/type/__download/parameter/required b/cdist/conf/type/__download/parameter/required new file mode 100644 index 00000000..6ea4c38f --- /dev/null +++ b/cdist/conf/type/__download/parameter/required @@ -0,0 +1,2 @@ +url +sum From cdb998398d25f1ba99f01364bbc42b900be944d5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 17 Jun 2020 12:10:58 +0200 Subject: [PATCH 070/155] Release 6.6.0 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index fa5f7405..c8388b93 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.6.0: 2020-06-17 * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) * Core: Support colored log output (Evil Ham) * Core: Tune colored log output and respect NO_COLOR env var (Dennis Camera) From a6543a72ade5a3b64cab1b0164d5df0b4388e88f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 17 Jun 2020 13:40:31 +0200 Subject: [PATCH 071/155] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index c8388b93..a0de09fa 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * New type: __download (Ander Punnar) + 6.6.0: 2020-06-17 * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) * Core: Support colored log output (Evil Ham) From 97e48be39e80ffe50946606b912ca28d3612cc61 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 20 Jun 2020 21:11:28 +0200 Subject: [PATCH 072/155] [type/__package_opkg] Fix explorer running in parallel --- .../type/__package_opkg/explorer/pkg_status | 57 ++++++++++++++----- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status index 5da4f742..1ba88e81 100755 --- a/cdist/conf/type/__package_opkg/explorer/pkg_status +++ b/cdist/conf/type/__package_opkg/explorer/pkg_status @@ -1,7 +1,8 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Nico Schottelius (nico-cdist at schottelius.org) # 2012 Giel van Schijndel (giel plus cdist at mortis dot eu) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -19,21 +20,51 @@ # along with cdist. If not, see . # # -# Retrieve the status of a package - parsed opkg output +# Retrieve the status of a package - parses opkg output # -if [ -f "$__object/parameter/name" ]; then - name="$(cat "$__object/parameter/name")" +LOCKFILE="${__type_explorer}/cdist_opkg.lock" +_lock() ( + set -o noclobber + until echo $$>"${LOCKFILE}" + do + while test -f "${LOCKFILE}"; do sleep 1; done + done + +) 2>/dev/null +_unlock() { + if test -s "${LOCKFILE}" && test "$(cat "${LOCKFILE}")" = $$ + then + rm "${LOCKFILE}" + fi +} + + +if test -f "${__object}/parameter/name" +then + pkg_name=$(cat "${__object}/parameter/name") else - name="$__object_id" + pkg_name=$__object_id fi -# Except dpkg failing, if package is not known / installed -if opkg status "$name" 2>/dev/null | grep -q "^Status: install user installed$"; then - echo "present" - exit 0 -elif [ "$(opkg info "$name" 2> /dev/null | wc -l)" -eq 0 ]; then - echo "absent notpresent" - exit 0 + +# NOTE: We need to lock parallel execution of this explorer because opkg will +# try to acquire the OPKG lock (usually /var/lock/opkg.lock) using lockf(2) for +# every operation. It will not wait for the lock but terminate with an error. +# This leads to incorrect 'absent notpresent' statuses when parallel execution +# is enabled. +trap _unlock EXIT +_lock + + +# Except opkg failing, if package is not known / installed +if opkg status "${pkg_name}" 2>/dev/null \ + | grep -q -e '^Status: [^ ][^ ]* [^ ][^ ]* installed$' +then + echo 'present' +elif opkg info "${pkg_name}" 2>/dev/null | grep -q . +then + echo 'absent notpresent' +else + echo 'absent' fi -echo "absent" From e79b26a61f790108b90c3625d554f6df4086d616 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 21 Jun 2020 13:15:38 +0200 Subject: [PATCH 073/155] [type/__package_opkg] Also lock execution of code-remote --- .../type/__package_opkg/explorer/pkg_status | 11 ++- cdist/conf/type/__package_opkg/gencode-remote | 94 +++++++++++++------ 2 files changed, 73 insertions(+), 32 deletions(-) diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status index 1ba88e81..f5a6f098 100755 --- a/cdist/conf/type/__package_opkg/explorer/pkg_status +++ b/cdist/conf/type/__package_opkg/explorer/pkg_status @@ -23,7 +23,9 @@ # Retrieve the status of a package - parses opkg output # -LOCKFILE="${__type_explorer}/cdist_opkg.lock" +__type_path=${__object%%${__object_id}*} + +LOCKFILE="${__type_path}/cdist_opkg.lock" _lock() ( set -o noclobber until echo $$>"${LOCKFILE}" @@ -48,9 +50,10 @@ else fi -# NOTE: We need to lock parallel execution of this explorer because opkg will -# try to acquire the OPKG lock (usually /var/lock/opkg.lock) using lockf(2) for -# every operation. It will not wait for the lock but terminate with an error. +# NOTE: We need to lock parallel execution of type explorers and code-remote +# because opkg will try to acquire the OPKG lock (usually /var/lock/opkg.lock) +# using lockf(2) for every operation. +# It will not wait for the lock but terminate with an error. # This leads to incorrect 'absent notpresent' statuses when parallel execution # is enabled. trap _unlock EXIT diff --git a/cdist/conf/type/__package_opkg/gencode-remote b/cdist/conf/type/__package_opkg/gencode-remote index 269d5f49..ad90dc24 100755 --- a/cdist/conf/type/__package_opkg/gencode-remote +++ b/cdist/conf/type/__package_opkg/gencode-remote @@ -2,6 +2,7 @@ # # 2011,2013 Nico Schottelius (nico-cdist at schottelius.org) # 2012 Giel van Schijndel (giel plus cdist at mortis dot eu) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -19,41 +20,78 @@ # along with cdist. If not, see . # # -# Manage packages on OpenWRT and co. +# Manage packages on OpenWrt, optware, and co. # -if [ -f "$__object/parameter/name" ]; then - name="$(cat "$__object/parameter/name")" +if test -f "${__object}/parameter/name" +then + name=$(cat "${__object}/parameter/name") else - name="$__object_id" + name=$__object_id fi -state_should="$(cat "$__object/parameter/state")" +state_should=$(cat "${__object}/parameter/state") +state_is=$(cat "${__object}/explorer/pkg_status") -state_is="$(cat "$__object/explorer/pkg_status")" -case "$state_is" in - absent*) - present="$(echo "$state_is" | cut -d ' ' -f 2)" - state_is="absent" - ;; +case $state_is +in + (absent*) + presence=$(echo "${state_is}" | cut -d ' ' -f 2) + state_is='absent' + ;; esac -[ "$state_is" = "$state_should" ] && exit 0 +if test "${state_is}" = "${state_should}" +then + exit 0 +fi -case "$state_should" in - present) - if [ "$present" = "notpresent" ]; then - echo "opkg --verbosity=0 update" - fi - echo "opkg --verbosity=0 install '$name'" - echo "installed" >> "$__messages_out" - ;; - absent) - echo "opkg --verbosity=0 remove '$name'" - echo "removed" >> "$__messages_out" - ;; - *) - echo "Unknown state: ${state_should}" >&2 - exit 1 - ;; +cat <<'EOF' +__type_path=${__object%%${__object_id}*} + +LOCKFILE="${__type_path}/cdist_opkg.lock" +_lock() ( + set -o noclobber + until echo $$>"${LOCKFILE}" + do + while test -f "${LOCKFILE}"; do sleep 1; done + done + +) 2>/dev/null +_unlock() { + if test -s "${LOCKFILE}" && test "$(cat "${LOCKFILE}")" = $$ + then + rm "${LOCKFILE}" + fi +} +EOF + +# NOTE: We need to lock parallel execution of code-remote to ensure that it is +# not executed concurrently with a type explorer. +# opkg will try to acquire the OPKG lock (usually /var/lock/opkg.lock) using +# lockf(2) for every operation. +# It will not wait for the lock but terminate with an error leading to an +# incorrect outcome. +echo 'trap _unlock EXIT' +echo '_lock' + +case $state_should +in + (present) + if test "${presence}" = 'notpresent' + then + echo 'opkg --verbosity=0 update' + fi + + printf "opkg --verbosity=0 install '%s'\n" "${name}" + echo 'installed' >>"${__messages_out}" + ;; + (absent) + printf "opkg --verbosity=0 remove '%s'" "${name}" + echo 'removed' >>"${__messages_out}" + ;; + (*) + printf 'Unknown state: %s\n' "${state_should}" >&2 + exit 1 + ;; esac From a6a3fb40bfcbaa5d6506a1ac5ebe2e40376dcb6f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 20 Jun 2020 22:22:29 +0200 Subject: [PATCH 074/155] Remove unnecessary Python shebangs --- cdist/config.py | 1 - cdist/info.py | 1 - cdist/install.py | 1 - cdist/integration.py | 1 - cdist/inventory.py | 1 - cdist/log.py | 1 - cdist/preos/debootstrap/debootstrap.py | 1 - 7 files changed, 7 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index b2d72f05..b71536a8 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org) diff --git a/cdist/info.py b/cdist/info.py index b896a3d1..9e29f5d9 100644 --- a/cdist/info.py +++ b/cdist/info.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # 2019-2020 Darko Poljak (darko.poljak at gmail.com) diff --git a/cdist/install.py b/cdist/install.py index b88ad016..a9c8119a 100644 --- a/cdist/install.py +++ b/cdist/install.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # 2013 Steven Armstrong (steven-cdist at armstrong.cc) diff --git a/cdist/integration.py b/cdist/integration.py index ee742cc5..03e4167d 100644 --- a/cdist/integration.py +++ b/cdist/integration.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # 2017 Darko Poljak (darko.poljak at gmail.com) diff --git a/cdist/inventory.py b/cdist/inventory.py index 138a2034..fb5ab960 100644 --- a/cdist/inventory.py +++ b/cdist/inventory.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # 2016 Darko Poljak (darko.poljak at gmail.com) diff --git a/cdist/log.py b/cdist/log.py index 19efebdb..c77ba8ec 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # 2010-2013 Nico Schottelius (nico-cdist at schottelius.org) diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py index f53dd4a7..d3e590f2 100644 --- a/cdist/preos/debootstrap/debootstrap.py +++ b/cdist/preos/debootstrap/debootstrap.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # 2016 Darko Poljak (darko.poljak at ungleich.ch) From ce07021580219d130c3968c547d03f451f2dea12 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 20 Jun 2020 21:16:23 +0200 Subject: [PATCH 075/155] Do not subclass object --- cdist/config.py | 2 +- cdist/core/cdist_object.py | 2 +- cdist/core/cdist_type.py | 2 +- cdist/core/code.py | 2 +- cdist/core/explorer.py | 2 +- cdist/core/manifest.py | 2 +- cdist/emulator.py | 2 +- cdist/exec/local.py | 2 +- cdist/exec/remote.py | 2 +- cdist/hostsource.py | 2 +- cdist/info.py | 3 +-- cdist/inventory.py | 2 +- cdist/message.py | 2 +- cdist/preos.py | 2 +- cdist/preos/debootstrap/debootstrap.py | 2 +- cdist/shell.py | 4 +--- cdist/test/config/__init__.py | 2 +- cdist/util/fsproperty.py | 4 ++-- 18 files changed, 19 insertions(+), 22 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index b2d72f05..982d8b75 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -70,7 +70,7 @@ def _graph_dfs_cycle(graph, node, path): return False -class Config(object): +class Config: """Cdist main class to hold arbitrary data""" # list of paths (files and/or directories) that will be removed on finish diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 114a47e0..51d61e04 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -47,7 +47,7 @@ class MissingObjectIdError(cdist.Error): return '%s' % (self.message) -class CdistObject(object): +class CdistObject: """Represents a cdist object. All interaction with objects in cdist should be done through this class. diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index 4500f50d..c0329c8a 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -38,7 +38,7 @@ class InvalidTypeError(cdist.Error): self.type_path, self.type_absolute_path, self.source_path) -class CdistType(object): +class CdistType: """Represents a cdist type. All interaction with types in cdist should be done through this class. diff --git a/cdist/core/code.py b/cdist/core/code.py index 1550880a..226bc63d 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -92,7 +92,7 @@ code-remote ''' -class Code(object): +class Code: """Generates and executes cdist code scripts. """ diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index 353d7681..a3baa959 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -63,7 +63,7 @@ type explorer is: ''' -class Explorer(object): +class Explorer: """Executes cdist explorers. """ diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 8b833ff2..2a0b2189 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -92,7 +92,7 @@ class NoInitialManifestError(cdist.Error): return repr(self.message) -class Manifest(object): +class Manifest: """Executes cdist manifests. """ diff --git a/cdist/emulator.py b/cdist/emulator.py index 4eaf2c93..24d239fa 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -56,7 +56,7 @@ class DefaultList(list): return cls(initial.split('\n')) -class Emulator(object): +class Emulator: def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ): self.argv = argv self.stdin = stdin diff --git a/cdist/exec/local.py b/cdist/exec/local.py index ad6c6e36..e0aab190 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -39,7 +39,7 @@ import cdist.exec.util as util CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ] -class Local(object): +class Local: """Execute commands locally. All interaction with the local side should be done through this class. diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index f72bf3bf..e5af2f34 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -49,7 +49,7 @@ class DecodeError(cdist.Error): return "Cannot decode output of " + " ".join(self.command) -class Remote(object): +class Remote: """Execute commands remotely. All interaction with the remote side should be done through this class. diff --git a/cdist/hostsource.py b/cdist/hostsource.py index a7b8f0b4..5f927b36 100644 --- a/cdist/hostsource.py +++ b/cdist/hostsource.py @@ -41,7 +41,7 @@ def hostfile_process_line(line, strip_func=str.strip): return None -class HostSource(object): +class HostSource: """ Host source object. Source can be a sequence or filename (stdin if \'-\'). diff --git a/cdist/info.py b/cdist/info.py index b896a3d1..a1fad237 100644 --- a/cdist/info.py +++ b/cdist/info.py @@ -29,8 +29,7 @@ import glob import fnmatch -class Info(object): - +class Info: def __init__(self, conf_dirs, args): self.conf_dirs = conf_dirs self.all = args.all diff --git a/cdist/inventory.py b/cdist/inventory.py index 138a2034..c06efff0 100644 --- a/cdist/inventory.py +++ b/cdist/inventory.py @@ -80,7 +80,7 @@ def rstrip_nl(s): return str.rstrip(s, "\n") -class Inventory(object): +class Inventory: """Inventory main class""" def __init__(self, db_basedir=dist_inventory_db, configuration=None): diff --git a/cdist/message.py b/cdist/message.py index 450fc3c3..ffa8c2bb 100644 --- a/cdist/message.py +++ b/cdist/message.py @@ -27,7 +27,7 @@ import tempfile log = logging.getLogger(__name__) -class Message(object): +class Message: """Support messaging between types """ diff --git a/cdist/preos.py b/cdist/preos.py index bf2a8e60..f8a5dd67 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -78,7 +78,7 @@ def get_available_preoses_string(cls): return "Available PreOS-es:\n{}".format("\n".join(preoses)) -class PreOS(object): +class PreOS: preoses = None @classmethod diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py index f53dd4a7..ac01e08b 100644 --- a/cdist/preos/debootstrap/debootstrap.py +++ b/cdist/preos/debootstrap/debootstrap.py @@ -31,7 +31,7 @@ import os import subprocess -class Debian(object): +class Debian: _preos_name = 'debian' _cdist_preos = True diff --git a/cdist/shell.py b/cdist/shell.py index 60b6a9f0..04a68937 100644 --- a/cdist/shell.py +++ b/cdist/shell.py @@ -32,10 +32,8 @@ import cdist.config log = logging.getLogger(__name__) -class Shell(object): - +class Shell: def __init__(self, shell=None): - self.shell = shell self.target_host = ( diff --git a/cdist/test/config/__init__.py b/cdist/test/config/__init__.py index 499593e3..0ed614b1 100644 --- a/cdist/test/config/__init__.py +++ b/cdist/test/config/__init__.py @@ -44,7 +44,7 @@ expected_object_names = sorted([ '__third/moon']) -class CdistObjectErrorContext(object): +class CdistObjectErrorContext: def __init__(self, original_error): self.original_error = original_error diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py index 5a27c9d7..1d76fd76 100644 --- a/cdist/util/fsproperty.py +++ b/cdist/util/fsproperty.py @@ -177,7 +177,7 @@ class DirectoryDict(collections.MutableMapping): raise cdist.Error(str(e)) -class FileBasedProperty(object): +class FileBasedProperty: attribute_class = None def __init__(self, path): @@ -189,7 +189,7 @@ class FileBasedProperty(object): Usage with a sublcass: - class Foo(object): + class Foo: # note that the actual DirectoryDict is stored as __parameters # on the instance parameters = DirectoryDictProperty( From 6aae58dea77002dea0b57d20cbb01b5e2a1066b5 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 21 Jun 2020 17:35:28 +0200 Subject: [PATCH 076/155] [type/__package_opkg] Mark lock variables readonly --- cdist/conf/type/__package_opkg/explorer/pkg_status | 4 ++-- cdist/conf/type/__package_opkg/gencode-remote | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status index f5a6f098..9d993055 100755 --- a/cdist/conf/type/__package_opkg/explorer/pkg_status +++ b/cdist/conf/type/__package_opkg/explorer/pkg_status @@ -23,9 +23,9 @@ # Retrieve the status of a package - parses opkg output # -__type_path=${__object%%${__object_id}*} +readonly __type_path=${__object%%${__object_id}*} +readonly LOCKFILE="${__type_path}/cdist_opkg.lock" -LOCKFILE="${__type_path}/cdist_opkg.lock" _lock() ( set -o noclobber until echo $$>"${LOCKFILE}" diff --git a/cdist/conf/type/__package_opkg/gencode-remote b/cdist/conf/type/__package_opkg/gencode-remote index ad90dc24..c7f21751 100755 --- a/cdist/conf/type/__package_opkg/gencode-remote +++ b/cdist/conf/type/__package_opkg/gencode-remote @@ -47,9 +47,9 @@ then fi cat <<'EOF' -__type_path=${__object%%${__object_id}*} +readonly __type_path=${__object%%${__object_id}*} +readonly LOCKFILE="${__type_path}/cdist_opkg.lock" -LOCKFILE="${__type_path}/cdist_opkg.lock" _lock() ( set -o noclobber until echo $$>"${LOCKFILE}" From 26dfdf37c21ce3c469ae0d6a717d390f422f9fbf Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 21 Jun 2020 01:13:30 +0300 Subject: [PATCH 077/155] [__download] support multiple checksum formats and download utilities, add --onchange and other minor changes --- cdist/conf/type/__download/explorer/state | 78 +++++++++++++++---- cdist/conf/type/__download/gencode-local | 25 +++++- cdist/conf/type/__download/gencode-remote | 7 ++ cdist/conf/type/__download/man.rst | 25 ++++-- .../type/__download/parameter/default/cmd-get | 1 - .../type/__download/parameter/default/cmd-sum | 1 - cdist/conf/type/__download/parameter/optional | 1 + 7 files changed, 116 insertions(+), 22 deletions(-) create mode 100755 cdist/conf/type/__download/gencode-remote delete mode 100644 cdist/conf/type/__download/parameter/default/cmd-get delete mode 100644 cdist/conf/type/__download/parameter/default/cmd-sum diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 6a50f5a5..00362545 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -2,19 +2,71 @@ dst="/$__object_id" -# shellcheck disable=SC2059 -cmd="$( printf "$( cat "$__object/parameter/cmd-sum" )" "$dst" )" - -sum="$( cat "$__object/parameter/sum" )" - -if [ -f "$dst" ] +if [ ! -f "$dst" ] then - if [ "$( eval "$cmd" )" = "$sum" ] - then - echo 'present' - else - echo 'mismatch' - fi -else echo 'absent' + exit 0 +fi + +sum_should="$( cat "$__object/parameter/sum" )" + +if [ -f "$__object/parameter/cmd-sum" ] +then + # shellcheck disable=SC2059 + sum_is="$( eval "$( printf \ + "$( cat "$__object/parameter/cmd-sum" )" \ + "$dst" )" )" +else + os="$( "$__explorer/os" )" + + if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' + then + sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )" + + elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$' + then + case "$os" in + freebsd) + sum_is="md5:$( md5 -q "$dst" )" + ;; + *) + sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )" + ;; + esac + + elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$' + then + case "$os" in + freebsd) + sum_is="sha1:$( sha1 -q "$dst" )" + ;; + *) + sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )" + ;; + esac + + elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$' + then + case "$os" in + freebsd) + sum_is="sha256:$( sha256 -q "$dst" )" + ;; + *) + sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )" + ;; + esac + fi +fi + +if [ -z "$sum_is" ] +then + echo 'no checksum from target' >&2 + exit 1 +fi + +if [ "$sum_is" = "$sum_should" ] +then + echo 'present' +else + echo 'mismatch' fi diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local index 49e9c699..85ef3a60 100755 --- a/cdist/conf/type/__download/gencode-local +++ b/cdist/conf/type/__download/gencode-local @@ -9,12 +9,31 @@ fi url="$( cat "$__object/parameter/url" )" -cmd="$( cat "$__object/parameter/cmd-get" )" - tmp="$( mktemp )" dst="/$__object_id" +if [ -f "$__object/parameter/cmd-get" ] +then + cmd="$( cat "$__object/parameter/cmd-get" )" + +elif command -v wget > /dev/null +then + cmd="wget -O - '%s'" + +elif command -v curl > /dev/null +then + cmd="curl -o - '%s'" + +elif command -v fetch > /dev/null +then + cmd="fetch -o - '%s'" + +else + echo 'no usable locally installed utility for downloading' >&2 + exit 1 +fi + printf "$cmd > %s\n" \ "$url" \ "$tmp" @@ -33,3 +52,5 @@ printf '%s %s %s:%s\n' \ "$dst" echo "rm -f '$tmp'" + +echo 'downloaded' > "$__messages_out" diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote new file mode 100755 index 00000000..b08d0050 --- /dev/null +++ b/cdist/conf/type/__download/gencode-remote @@ -0,0 +1,7 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/onchange" ] \ + && grep -Fq "$__object_id:downloaded" "$__messages_in" +then + cat "$__object/parameter/onchange" +fi diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index c973448f..63a41bc4 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -10,7 +10,13 @@ DESCRIPTION ----------- You must use persistent storage in target host for destination file (``$__object_id``) because it will be used for checksum calculation -in order to decide if file must be downloaded. +in order to decide if file must be (re-)downloaded. + +By default type will try to use following locally installed utilities +for downloading (in order): ``wget``, ``curl`` or ``fetch``. + +Environment variables like ``{http,https,ftp}_proxy`` etc can be used on +cdist execution (``http_proxy=foo cdist config ...``). REQUIRED PARAMETERS @@ -19,20 +25,29 @@ url URL from which to download the file. sum - Checksum of downloaded file. + Checksum of file going to be downloaded. + By default output of ``cksum`` without filename is expected. + Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. + +onchange + Execute this command after download. OPTIONAL PARAMETERS ------------------- cmd-get Command used for downloading. - Default is ``wget -O- '%s'``. Command must output to ``stdout``. + Parameter will be used for ``printf`` and must include only one + variable ``%s`` which will become URL. + For example: ``wget -O - '%s'``. cmd-sum Command used for checksum calculation. - Default is ``md5sum '%s' | awk '{print $1}'``. Command output and ``--sum`` parameter must match. + Parameter will be used for ``printf`` and must include only one + variable ``%s`` which will become destination. + For example: ``md5sum '%s' | awk '{print $1}'``. EXAMPLES @@ -45,7 +60,7 @@ EXAMPLES require='__directory/opt/cpma' \ __download /opt/cpma/cnq3.zip \ --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \ - --sum 46da3021ca9eace277115ec9106c5b46 + --sum md5:46da3021ca9eace277115ec9106c5b46 require='__download/opt/cpma/cnq3.zip' \ __unpack /opt/cpma/cnq3.zip \ diff --git a/cdist/conf/type/__download/parameter/default/cmd-get b/cdist/conf/type/__download/parameter/default/cmd-get deleted file mode 100644 index 2daa38a1..00000000 --- a/cdist/conf/type/__download/parameter/default/cmd-get +++ /dev/null @@ -1 +0,0 @@ -wget -O- '%s' diff --git a/cdist/conf/type/__download/parameter/default/cmd-sum b/cdist/conf/type/__download/parameter/default/cmd-sum deleted file mode 100644 index 3e8a9295..00000000 --- a/cdist/conf/type/__download/parameter/default/cmd-sum +++ /dev/null @@ -1 +0,0 @@ -md5sum '%s' | awk '{print $1}' diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional index 22783e02..38c0ce4d 100644 --- a/cdist/conf/type/__download/parameter/optional +++ b/cdist/conf/type/__download/parameter/optional @@ -1,2 +1,3 @@ cmd-get cmd-sum +onchange From 3649555f3522ea80f1999ca2cb7f8ddd112cbb33 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 22 Jun 2020 09:31:59 +0200 Subject: [PATCH 078/155] [type/__package_opkg] Do not lock execution of code-remote (revert) Instead, rely on `nonparallel`. In any case cdist should never run explorer and code concurrently even if the dependency graph would allow to do so as it would result in many more synchronization issues than this one. --- cdist/conf/type/__package_opkg/gencode-remote | 32 ++----------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/cdist/conf/type/__package_opkg/gencode-remote b/cdist/conf/type/__package_opkg/gencode-remote index c7f21751..28caff71 100755 --- a/cdist/conf/type/__package_opkg/gencode-remote +++ b/cdist/conf/type/__package_opkg/gencode-remote @@ -25,9 +25,9 @@ if test -f "${__object}/parameter/name" then - name=$(cat "${__object}/parameter/name") + name=$(cat "${__object}/parameter/name") else - name=$__object_id + name=$__object_id fi state_should=$(cat "${__object}/parameter/state") @@ -46,34 +46,6 @@ then exit 0 fi -cat <<'EOF' -readonly __type_path=${__object%%${__object_id}*} -readonly LOCKFILE="${__type_path}/cdist_opkg.lock" - -_lock() ( - set -o noclobber - until echo $$>"${LOCKFILE}" - do - while test -f "${LOCKFILE}"; do sleep 1; done - done - -) 2>/dev/null -_unlock() { - if test -s "${LOCKFILE}" && test "$(cat "${LOCKFILE}")" = $$ - then - rm "${LOCKFILE}" - fi -} -EOF - -# NOTE: We need to lock parallel execution of code-remote to ensure that it is -# not executed concurrently with a type explorer. -# opkg will try to acquire the OPKG lock (usually /var/lock/opkg.lock) using -# lockf(2) for every operation. -# It will not wait for the lock but terminate with an error leading to an -# incorrect outcome. -echo 'trap _unlock EXIT' -echo '_lock' case $state_should in From 49dde11def71cec121e5f698d0fb6ed3a539f97a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 24 Jun 2020 07:04:06 +0200 Subject: [PATCH 079/155] Remove deprecated __pf_apply --- cdist/conf/type/__pf_apply/deprecated | 1 - cdist/conf/type/__pf_apply/explorer/rcvar | 36 --------------- cdist/conf/type/__pf_apply/gencode-remote | 51 --------------------- cdist/conf/type/__pf_apply/man.rst | 55 ----------------------- cdist/conf/type/__pf_apply/singleton | 0 5 files changed, 143 deletions(-) delete mode 100644 cdist/conf/type/__pf_apply/deprecated delete mode 100755 cdist/conf/type/__pf_apply/explorer/rcvar delete mode 100755 cdist/conf/type/__pf_apply/gencode-remote delete mode 100644 cdist/conf/type/__pf_apply/man.rst delete mode 100644 cdist/conf/type/__pf_apply/singleton diff --git a/cdist/conf/type/__pf_apply/deprecated b/cdist/conf/type/__pf_apply/deprecated deleted file mode 100644 index 36cfed90..00000000 --- a/cdist/conf/type/__pf_apply/deprecated +++ /dev/null @@ -1 +0,0 @@ -Consider moving to __pf_apply_anchor. Get in touch if you need __pf_apply. diff --git a/cdist/conf/type/__pf_apply/explorer/rcvar b/cdist/conf/type/__pf_apply/explorer/rcvar deleted file mode 100755 index 7c8d535f..00000000 --- a/cdist/conf/type/__pf_apply/explorer/rcvar +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# -# 2012 Jake Guffey (jake.guffey at eprotex.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Get the location of the pf ruleset on the target host. -# - -# Debug -#exec >&2 -#set -x - -# Check /etc/rc.conf for pf's configuration file name. Default to /etc/pf.conf - -RC="/etc/rc.conf" -PFCONF="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')" -echo "${PFCONF:-"/etc/pf.conf"}" - -# Debug -#set +x - diff --git a/cdist/conf/type/__pf_apply/gencode-remote b/cdist/conf/type/__pf_apply/gencode-remote deleted file mode 100755 index c8f7a25a..00000000 --- a/cdist/conf/type/__pf_apply/gencode-remote +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -e -# -# 2012 Jake Guffey (jake.guffey at eprotex.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Apply pf(4) ruleset on *BSD -# - -# Debug -#exec >&2 -#set -x - -rcvar=$(cat "$__object/explorer/rcvar") - -cat <&2 - fi -fi -EOF - -# Debug -#set +x - diff --git a/cdist/conf/type/__pf_apply/man.rst b/cdist/conf/type/__pf_apply/man.rst deleted file mode 100644 index eee345e7..00000000 --- a/cdist/conf/type/__pf_apply/man.rst +++ /dev/null @@ -1,55 +0,0 @@ -cdist-type__pf_apply(7) -======================= - -NAME ----- -cdist-type__pf_apply - Apply pf(4) ruleset on \*BSD - - -DESCRIPTION ------------ -This type is used on \*BSD systems to manage the pf firewall's active ruleset. - - -REQUIRED PARAMETERS -------------------- -NONE - - -OPTIONAL PARAMETERS -------------------- -NONE - - -EXAMPLES --------- - -.. code-block:: sh - - # Modify the ruleset on $__target_host: - __pf_ruleset --state present --source /my/pf/ruleset.conf - require="__pf_ruleset" \ - __pf_apply - - # Remove the ruleset on $__target_host (implies disabling pf(4): - __pf_ruleset --state absent - require="__pf_ruleset" \ - __pf_apply - - -SEE ALSO --------- -:strong:`pf`\ (4), :strong:`cdist-type__pf_ruleset`\ (7) - - -AUTHORS -------- -Jake Guffey - - -COPYING -------- -Copyright \(C) 2012 Jake Guffey. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. diff --git a/cdist/conf/type/__pf_apply/singleton b/cdist/conf/type/__pf_apply/singleton deleted file mode 100644 index e69de29b..00000000 From a9778965be7154c0f42cf0170142ee2ec571d7ec Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 24 Jun 2020 08:47:22 +0200 Subject: [PATCH 080/155] [type/__package_opkg] Use mkdir(1) to lock instead of noclobber noclobber is potentially unsafe, because it relies on the underlying shell to implement noclobber in a safe way that avoids race conditions between multiple processes. mkdir is safer because it is mandated by POSIX to "fail" if the target already exists. --- .../type/__package_opkg/explorer/pkg_status | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status index 9d993055..09c550e5 100755 --- a/cdist/conf/type/__package_opkg/explorer/pkg_status +++ b/cdist/conf/type/__package_opkg/explorer/pkg_status @@ -24,21 +24,28 @@ # readonly __type_path=${__object%%${__object_id}*} -readonly LOCKFILE="${__type_path}/cdist_opkg.lock" +test -d "${__type_path}" || { echo 'Cannot determine __type_path' >&2; exit 1; } +readonly LOCKDIR="${__type_path:?}/.cdist_opkg.lock.dir" -_lock() ( - set -o noclobber - until echo $$>"${LOCKFILE}" +_lock() { + until mkdir "${LOCKDIR:?}" 2>/dev/null do - while test -f "${LOCKFILE}"; do sleep 1; done + while test -d "${LOCKDIR}" + do + # DEBUG: printf 'Locked by PID: %u\n' "$(cat "${LOCKDIR}/pid")" + sleep 1 + done done - -) 2>/dev/null + echo $$ >"${LOCKDIR:?}/pid" +} _unlock() { - if test -s "${LOCKFILE}" && test "$(cat "${LOCKFILE}")" = $$ + test -d "${LOCKDIR}" || return 0 + if test -s "${LOCKDIR}/pid" then - rm "${LOCKFILE}" + test "$(cat "${LOCKDIR}/pid")" = $$ || return 1 + rm "${LOCKDIR:?}/pid" fi + rmdir "${LOCKDIR:?}" } From 5364d3bc9037969c415140523c5cac2f8891b967 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 24 Jun 2020 20:49:48 +0200 Subject: [PATCH 081/155] [type/__package_opkg] Implement flock locking if available --- .../type/__package_opkg/explorer/pkg_status | 55 ++++++++++++------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/cdist/conf/type/__package_opkg/explorer/pkg_status b/cdist/conf/type/__package_opkg/explorer/pkg_status index 09c550e5..de7b896b 100755 --- a/cdist/conf/type/__package_opkg/explorer/pkg_status +++ b/cdist/conf/type/__package_opkg/explorer/pkg_status @@ -25,28 +25,45 @@ readonly __type_path=${__object%%${__object_id}*} test -d "${__type_path}" || { echo 'Cannot determine __type_path' >&2; exit 1; } -readonly LOCKDIR="${__type_path:?}/.cdist_opkg.lock.dir" +readonly LOCKFILE="${__type_path:?}/.cdist_opkg.lock" -_lock() { - until mkdir "${LOCKDIR:?}" 2>/dev/null - do - while test -d "${LOCKDIR}" +if command -v flock >/dev/null 2>&1 +then + # use flock (if available) on FD 9 + _lock() { + exec 9<>"${LOCKFILE:?}" + flock -x 9 + echo $$>&9 + } + _unlock() { + :>"${LOCKFILE:?}" + flock -u 9 + exec 9<&- + } +else + # fallback to mkdir if flock is missing + _lock() { + until mkdir "${LOCKFILE:?}.dir" 2>/dev/null do - # DEBUG: printf 'Locked by PID: %u\n' "$(cat "${LOCKDIR}/pid")" - sleep 1 + while test -d "${LOCKFILE}.dir" + do + # DEBUG: + # printf 'Locked by PID: %u\n' "$(cat "${LOCKFILE}.dir/pid")" + sleep 1 + done done - done - echo $$ >"${LOCKDIR:?}/pid" -} -_unlock() { - test -d "${LOCKDIR}" || return 0 - if test -s "${LOCKDIR}/pid" - then - test "$(cat "${LOCKDIR}/pid")" = $$ || return 1 - rm "${LOCKDIR:?}/pid" - fi - rmdir "${LOCKDIR:?}" -} + echo $$ >"${LOCKFILE:?}.dir/pid" + } + _unlock() { + test -d "${LOCKFILE}.dir" || return 0 + if test -s "${LOCKFILE}.dir/pid" + then + test "$(cat "${LOCKFILE}.dir/pid")" = $$ || return 1 + rm "${LOCKFILE:?}.dir/pid" + fi + rmdir "${LOCKFILE:?}.dir" + } +fi if test -f "${__object}/parameter/name" From 7074f9c395d3ab6f763682f860744ac8d12a8c26 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 25 Jun 2020 06:32:10 +0200 Subject: [PATCH 082/155] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index a0de09fa..f1713d3b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,8 @@ Changelog next: * New type: __download (Ander Punnar) + * Type __locale_system: Add devuan support (Dennis Camera) + * Type __package_opkg: Add locking (Dennis Camera) 6.6.0: 2020-06-17 * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) From 077989e8fd749339da9a6ca8458b2c73ac71672c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 27 Jun 2020 15:55:04 +0200 Subject: [PATCH 083/155] Remove annoying warnings Those warnings don't have any specail meaning and usage. Resolve #825. --- cdist/util/ipaddr.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py index 9b730225..95ca74ee 100644 --- a/cdist/util/ipaddr.py +++ b/cdist/util/ipaddr.py @@ -45,8 +45,6 @@ def resolve_target_host_name(host, family=0): log.debug("derived host_name for host \"{}\": {}".format( host, host_name)) except (socket.gaierror, socket.herror) as e: - log.warning("Could not derive host_name for {}" - ", $host_name will be empty. Error is: {}".format(host, e)) # in case of error provide empty value host_name = '' return host_name @@ -59,8 +57,6 @@ def resolve_target_fqdn(host): log.debug("derived host_fqdn for host \"{}\": {}".format( host, host_fqdn)) except socket.herror as e: - log.warning("Could not derive host_fqdn for {}" - ", $host_fqdn will be empty. Error is: {}".format(host, e)) # in case of error provide empty value host_fqdn = '' return host_fqdn From 85614aabd6f3abef5af6d362203db956c9d72aa9 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 28 Jun 2020 16:38:15 +0300 Subject: [PATCH 084/155] [__download] add --download (local|remote), update manual --- .../conf/type/__download/explorer/remote_cmd | 19 +++++++++++++++ cdist/conf/type/__download/gencode-local | 4 +++- cdist/conf/type/__download/gencode-remote | 22 +++++++++++++++-- cdist/conf/type/__download/man.rst | 24 ++++++++++++------- cdist/conf/type/__download/manifest | 6 +++++ .../__download/parameter/default/download | 1 + cdist/conf/type/__download/parameter/optional | 1 + 7 files changed, 65 insertions(+), 12 deletions(-) create mode 100755 cdist/conf/type/__download/explorer/remote_cmd create mode 100755 cdist/conf/type/__download/manifest create mode 100644 cdist/conf/type/__download/parameter/default/download diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd new file mode 100755 index 00000000..fbd4d84c --- /dev/null +++ b/cdist/conf/type/__download/explorer/remote_cmd @@ -0,0 +1,19 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/cmd-get" ] +then + cmd="$( cat "$__object/parameter/cmd-get" )" + +elif command -v curl > /dev/null +then + cmd="curl -o - '%s'" + +elif command -v fetch > /dev/null +then + cmd="fetch -o - '%s'" + +else + cmd="wget -O - '%s'" +fi + +echo "$cmd" diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local index 85ef3a60..339827c2 100755 --- a/cdist/conf/type/__download/gencode-local +++ b/cdist/conf/type/__download/gencode-local @@ -1,8 +1,10 @@ #!/bin/sh -e +download="$( cat "$__object/parameter/download" )" + state_is="$( cat "$__object/explorer/state" )" -if [ "$state_is" = 'present' ] +if [ "$download" != 'local' ] || [ "$state_is" = 'present' ] then exit 0 fi diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote index b08d0050..89ba72af 100755 --- a/cdist/conf/type/__download/gencode-remote +++ b/cdist/conf/type/__download/gencode-remote @@ -1,7 +1,25 @@ #!/bin/sh -e -if [ -f "$__object/parameter/onchange" ] \ - && grep -Fq "$__object_id:downloaded" "$__messages_in" +download="$( cat "$__object/parameter/download" )" + +state_is="$( cat "$__object/explorer/state" )" + +if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ] +then + cmd="$( cat "$__object/explorer/remote_cmd" )" + + url="$( cat "$__object/parameter/url" )" + + dst="/$__object_id" + + printf "$cmd > %s\n" \ + "$url" \ + "$dst" + + echo 'downloaded' > "$__messages_out" +fi + +if [ -f "$__object/parameter/onchange" ] && [ "$state" != "present" ] then cat "$__object/parameter/onchange" fi diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index 63a41bc4..c161f4e4 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -3,26 +3,28 @@ cdist-type__download(7) NAME ---- -cdist-type__download - Download file to local storage and copy it to target host +cdist-type__download - Download a file DESCRIPTION ----------- -You must use persistent storage in target host for destination file -(``$__object_id``) because it will be used for checksum calculation -in order to decide if file must be (re-)downloaded. +Persistent storage for destination file in target host must be used +(``$__object_id``) because it will be used for checksum calculation in +order to decide if file must be (re-)downloaded. -By default type will try to use following locally installed utilities -for downloading (in order): ``wget``, ``curl`` or ``fetch``. +By default type will try to use ``wget``, ``curl`` or ``fetch`` for +downloading. If ``--download remote`` type will fallback to (and +install) ``wget``. -Environment variables like ``{http,https,ftp}_proxy`` etc can be used on -cdist execution (``http_proxy=foo cdist config ...``). +If ``--download local`` (default), then environment variables like +``{http,https,ftp}_proxy`` etc can be used on cdist execution +(``http_proxy=foo cdist config ...``). REQUIRED PARAMETERS ------------------- url - URL from which to download the file. + File's URL. sum Checksum of file going to be downloaded. @@ -35,6 +37,10 @@ onchange OPTIONAL PARAMETERS ------------------- +download + If ``local`` (default), then download file to local storage and copy + it to target host. If ``remote``, then download happens in target. + cmd-get Command used for downloading. Command must output to ``stdout``. diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest new file mode 100755 index 00000000..7ec8d86d --- /dev/null +++ b/cdist/conf/type/__download/manifest @@ -0,0 +1,6 @@ +#!/bin/sh -e + +if grep -Eq '^wget' "$__object/explorer/remote_cmd" +then + __package wget +fi diff --git a/cdist/conf/type/__download/parameter/default/download b/cdist/conf/type/__download/parameter/default/download new file mode 100644 index 00000000..40830374 --- /dev/null +++ b/cdist/conf/type/__download/parameter/default/download @@ -0,0 +1 @@ +local diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional index 38c0ce4d..838e2fbf 100644 --- a/cdist/conf/type/__download/parameter/optional +++ b/cdist/conf/type/__download/parameter/optional @@ -1,3 +1,4 @@ cmd-get cmd-sum +download onchange From b6bf90e3f1a775ec1bd27a99015107d24a397787 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 28 Jun 2020 16:43:45 +0300 Subject: [PATCH 085/155] [__download] update manual --- cdist/conf/type/__download/man.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index c161f4e4..eafa9dfc 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -8,15 +8,14 @@ cdist-type__download - Download a file DESCRIPTION ----------- -Persistent storage for destination file in target host must be used -(``$__object_id``) because it will be used for checksum calculation in -order to decide if file must be (re-)downloaded. +Destination (``$__object_id``) in target host must be persistent storage +in order to calculate checksum and decide if file must be (re-)downloaded. -By default type will try to use ``wget``, ``curl`` or ``fetch`` for -downloading. If ``--download remote`` type will fallback to (and -install) ``wget``. +By default type will try to use ``wget``, ``curl`` or ``fetch``. +If download happens in target (see ``--download``) then type will +fallback to (and install) ``wget``. -If ``--download local`` (default), then environment variables like +If download happens in local machine, then environment variables like ``{http,https,ftp}_proxy`` etc can be used on cdist execution (``http_proxy=foo cdist config ...``). From a263fdfe584bbba467086a3b2da3dfdd58ccc7fe Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 2 Oct 2019 17:52:20 +0200 Subject: [PATCH 086/155] [__hosts] Add --alias parameter The --alias parameter allows to specify a hostname and multiple aliases on a single /etc/hosts line. --- cdist/conf/type/__hosts/man.rst | 17 +++++++--- cdist/conf/type/__hosts/manifest | 34 ++++++++++++------- .../type/__hosts/parameter/optional_multiple | 1 + 3 files changed, 35 insertions(+), 17 deletions(-) create mode 100644 cdist/conf/type/__hosts/parameter/optional_multiple diff --git a/cdist/conf/type/__hosts/man.rst b/cdist/conf/type/__hosts/man.rst index bece7967..1ac706cb 100644 --- a/cdist/conf/type/__hosts/man.rst +++ b/cdist/conf/type/__hosts/man.rst @@ -25,6 +25,10 @@ ip state is ``present``, this parameter is mandatory, if state is ``absent``, this parameter is silently ignored. +alias + An alias for the hostname. + This parameter can be specified multiple times (once per alias). + EXAMPLES -------- @@ -36,6 +40,8 @@ EXAMPLES # previously configured via __hosts. __hosts happy --state absent + __hosts srv1.example.com --ip 192.168.0.42 --alias srv1 + SEE ALSO -------- @@ -43,13 +49,14 @@ SEE ALSO AUTHORS ------- - -Dmitry Bogatov +| Dmitry Bogatov +| Dennis Camera COPYING ------- -Copyright (C) 2015,2016 Dmitry Bogatov. Free use of this software is granted -under the terms of the GNU General Public License version 3 or later -(GPLv3+). +Copyright \(C) 2015-2016 Dmitry Bogatov, 2019 Dennis Camera. +You can redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. diff --git a/cdist/conf/type/__hosts/manifest b/cdist/conf/type/__hosts/manifest index c536b83b..0d9e61f8 100755 --- a/cdist/conf/type/__hosts/manifest +++ b/cdist/conf/type/__hosts/manifest @@ -1,29 +1,39 @@ #!/bin/sh -e -# Copyright (C) 2015 Bogatov Dmitry # -# This program is free software: you can redistribute it and/or modify +# Copyright (C) 2015 Bogatov Dmitry +# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# This program is distributed in the hope that it will be useful, +# cdist is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -set -ue +# -hostname="$__object_id" -state="$(cat "$__object/parameter/state")" +set -e -u + +hostname=$__object_id +state=$(cat "$__object/parameter/state") marker="# __hosts/$hostname" -set -- "__hosts/$hostname" --file /etc/hosts --state "$state" - -if [ "$state" = absent ] ; then - __line "$@" --regex "$marker" +if [ "$state" = 'absent' ] +then + set -- --regex "$marker" else - ip="$(cat "$__object/parameter/ip")" - __line "$@" --line "$ip $hostname $marker" + ip=$(cat "$__object/parameter/ip") + aliases=$(while read -r a; do printf '\t%s' "$a"; done <"$__object/parameter/alias") + + set -- --line "$(printf '%s\t%s%s %s' \ + "$ip" "$hostname" "$aliases" "$marker")" fi + +__line "__hosts/$hostname" --file /etc/hosts --state "$state" "$@" diff --git a/cdist/conf/type/__hosts/parameter/optional_multiple b/cdist/conf/type/__hosts/parameter/optional_multiple new file mode 100644 index 00000000..d077ed80 --- /dev/null +++ b/cdist/conf/type/__hosts/parameter/optional_multiple @@ -0,0 +1 @@ +alias From 999e7b013449c51b848bb9365205810fc1a80fbd Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 28 Jan 2020 09:15:56 +0100 Subject: [PATCH 087/155] [type/__user] Fix shadow explorer for OpenBSD --- cdist/conf/type/__user/explorer/shadow | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/cdist/conf/type/__user/explorer/shadow b/cdist/conf/type/__user/explorer/shadow index 73ce0e29..d516ed10 100755 --- a/cdist/conf/type/__user/explorer/shadow +++ b/cdist/conf/type/__user/explorer/shadow @@ -23,18 +23,24 @@ name=$__object_id -case $("$__explorer/os") in - 'freebsd'|'netbsd'|'openbsd') +case $("${__explorer}/os") in + freebsd|netbsd) database='passwd' ;; - # Default to using shadow passwords + openbsd) + database='master.passwd' + ;; *) + # Default to using shadow passwords database='shadow' ;; esac -if command -v getent >/dev/null; then - getent "$database" "$name" || true -elif [ -f /etc/shadow ]; then - grep "^${name}:" /etc/shadow || true +if command -v getent >/dev/null 2>&1 +then + getent "${database}" "${name}" 2>/dev/null && exit || true # fallback to file +fi +if test -n "${database}" -a -f "/etc/${database}" +then + grep -e "^${name}:" "/etc/${database}" && exit || true # ignore failure fi From 6467ccbdccd380c11e37e747216bfc3aabc1b776 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 30 Jun 2020 14:31:11 +0200 Subject: [PATCH 088/155] [type/__user] Make shellcheck happy --- cdist/conf/type/__user/explorer/shadow | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__user/explorer/shadow b/cdist/conf/type/__user/explorer/shadow index d516ed10..32496ee7 100755 --- a/cdist/conf/type/__user/explorer/shadow +++ b/cdist/conf/type/__user/explorer/shadow @@ -38,9 +38,10 @@ esac if command -v getent >/dev/null 2>&1 then + # shellcheck disable=SC2015 getent "${database}" "${name}" 2>/dev/null && exit || true # fallback to file fi if test -n "${database}" -a -f "/etc/${database}" then - grep -e "^${name}:" "/etc/${database}" && exit || true # ignore failure + grep -e "^${name}:" "/etc/${database}" || true # ignore failure fi From 3860f1feeaa2844191eeea84444963818bb42c4b Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 26 Jun 2020 14:33:16 +0200 Subject: [PATCH 089/155] [type/{__file/__directory}] Support setuid,setguid,sticky bits --- cdist/conf/type/__directory/explorer/stat | 17 +++++++++-------- cdist/conf/type/__directory/gencode-remote | 4 ++-- cdist/conf/type/__file/explorer/stat | 14 +++++++------- cdist/conf/type/__file/gencode-remote | 4 ++-- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat index 105d894f..a7dc8431 100755 --- a/cdist/conf/type/__directory/explorer/stat +++ b/cdist/conf/type/__directory/explorer/stat @@ -33,7 +33,7 @@ fallback() { group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group) mode_text=$(echo "$ls_line" | awk '{ print $1 }') - mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }') + mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}') printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\n' \ "$("$__type_explorer/type")" \ @@ -51,13 +51,14 @@ then exit fi -case $("$__explorer/os") in - "freebsd"|"netbsd"|"openbsd"|"macosx") - stat -f "type: %HT +case $("$__explorer/os") +in + freebsd|netbsd|openbsd|macosx) + stat -f 'type: %HT owner: %Du %Su group: %Dg %Sg -mode: %Lp %Sp -" "$destination" | awk '/^type/ { print tolower($0); next } { print }' +mode: %Mp%03Lp %Sp +' "$destination" | awk '/^type/ { print tolower($0); next } { print }' ;; solaris) ls1="$( ls -ld "$destination" )" @@ -92,9 +93,9 @@ mode: %Lp %Sp # NOTE: Do not use --printf here as it is not supported by BusyBox stat. # NOTE: BusyBox's stat might not support the "-c" option, in which case # we fall through to the shell fallback. - stat -c "type: %F + stat -c 'type: %F owner: %u %U group: %g %G -mode: %a %A" "$destination" 2>/dev/null || fallback +mode: %04a %A' "$destination" 2>/dev/null || fallback ;; esac diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote index a1a32ea2..2c2c56fd 100755 --- a/cdist/conf/type/__directory/gencode-remote +++ b/cdist/conf/type/__directory/gencode-remote @@ -97,9 +97,9 @@ case "$state_should" in value_should="$(cat "$__object/parameter/$attribute")" value_is="$(get_current_value "$attribute" "$value_should")" - # change 0xxx format to xxx format => same as stat returns + # format mode in four digits => same as stat returns if [ "$attribute" = mode ]; then - value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')" + value_should=$(printf '%04u' "${value_should}") fi if [ "$set_attributes" = 1 ] || [ "$value_should" != "$value_is" ]; then diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat index 91c8cc84..231768f6 100755 --- a/cdist/conf/type/__file/explorer/stat +++ b/cdist/conf/type/__file/explorer/stat @@ -34,7 +34,7 @@ fallback() { group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group) mode_text=$(echo "$ls_line" | awk '{ print $1 }') - mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }') + mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}') size=$(echo "$ls_line" | awk '{ print $5 }') links=$(echo "$ls_line" | awk '{ print $2 }') @@ -63,13 +63,13 @@ fi case $("$__explorer/os") in freebsd|netbsd|openbsd|macosx) - stat -f "type: %HT + stat -f 'type: %HT owner: %Du %Su group: %Dg %Sg -mode: %Lp %Sp +mode: %Mp%03Lp %Sp size: %Dz links: %Dl -" "$destination" | awk '/^type/ { print tolower($0); next } { print }' +' "$destination" | awk '/^type/ { print tolower($0); next } { print }' ;; solaris) ls1="$( ls -ld "$destination" )" @@ -106,11 +106,11 @@ links: %Dl # NOTE: Do not use --printf here as it is not supported by BusyBox stat. # NOTE: BusyBox's stat might not support the "-c" option, in which case # we fall through to the shell fallback. - stat -c "type: %F + stat -c 'type: %F owner: %u %U group: %g %G -mode: %a %A +mode: %04a %A size: %s -links: %h" "$destination" 2>/dev/null || fallback +links: %h' "$destination" 2>/dev/null || fallback ;; esac diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index 815593bd..a69154df 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -68,9 +68,9 @@ case "$state_should" in if [ -f "$__object/parameter/$attribute" ]; then value_should="$(cat "$__object/parameter/$attribute")" - # change 0xxx format to xxx format => same as stat returns + # format mode in four digits => same as stat returns if [ "$attribute" = mode ]; then - value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')" + value_should=$(printf '%04u' "${value_should}") fi value_is="$(get_current_value "$attribute" "$value_should")" From 88400551f9490a4f8dc9b918e962e5ab3bba1286 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 30 Jun 2020 23:59:45 +0200 Subject: [PATCH 090/155] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index f1713d3b..b823421b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,8 @@ next: * New type: __download (Ander Punnar) * Type __locale_system: Add devuan support (Dennis Camera) * Type __package_opkg: Add locking (Dennis Camera) + * Type __hosts: Add --alias parameter (Dennis Camera) + * Type __user: Fix shadow explorer for OpenBSD (Dennis Camera) 6.6.0: 2020-06-17 * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) From fe193ecab8addf9b1c5b6d3c06c11ae9f9803378 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 1 Jul 2020 14:05:45 +0200 Subject: [PATCH 091/155] Make code consistent * Remove supreflous checking and warning message. * Fix cache recording. --- cdist/emulator.py | 65 +++++++++++++++-------------------------------- 1 file changed, 21 insertions(+), 44 deletions(-) diff --git a/cdist/emulator.py b/cdist/emulator.py index adbcbd9d..9fe84056 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -91,10 +91,6 @@ class Emulator: self.type_name = os.path.basename(argv[0]) self.cdist_type = core.CdistType(self.type_base_path, self.type_name) - # If set then object alreay exists and this var holds existing - # requirements. - self._existing_reqs = None - self.__init_log() def run(self): @@ -230,9 +226,6 @@ class Emulator: self.parameters[key] = value if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env: - # Make existing requirements a set so that we can compare it - # later with new requirements. - self._existing_reqs = set(self.cdist_object.requirements) obj_params = self._object_params_in_context() if obj_params != self.parameters: errmsg = ("Object %s already exists with conflicting " @@ -251,23 +244,26 @@ class Emulator: else: self.cdist_object.create() self.cdist_object.parameters = self.parameters - # record the created object in typeorder file - with open(self.typeorder_path, 'a') as typeorderfile: - print(self.cdist_object.name, file=typeorderfile) - # record the created object in parent object typeorder file - __object_name = self.env.get('__object_name', None) - depname = self.cdist_object.name - if __object_name: - parent = self.cdist_object.object_from_name(__object_name) - parent.typeorder.append(self.cdist_object.name) - if self._order_dep_on(): - self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep' - ' for %s'), depname, parent.name) - parent.typeorder_dep.append(depname) - elif self._order_dep_on(): - self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep', - depname) - self._add_typeorder_dep(depname) + # Do the following recording even if object exists, but with + # different requirements. + + # record the created object in typeorder file + with open(self.typeorder_path, 'a') as typeorderfile: + print(self.cdist_object.name, file=typeorderfile) + # record the created object in parent object typeorder file + __object_name = self.env.get('__object_name', None) + depname = self.cdist_object.name + if __object_name: + parent = self.cdist_object.object_from_name(__object_name) + parent.typeorder.append(self.cdist_object.name) + if self._order_dep_on(): + self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep' + ' for %s'), depname, parent.name) + parent.typeorder_dep.append(depname) + elif self._order_dep_on(): + self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep', + depname) + self._add_typeorder_dep(depname) # Record / Append source self.cdist_object.source.append(self.object_source) @@ -322,8 +318,6 @@ class Emulator: # This ensures pattern matching is done against sanitised list self.cdist_object.requirements.append(cdist_object.name) - return cdist_object.name - def _order_dep_on(self): return os.path.exists(self.order_dep_state_path) @@ -392,7 +386,6 @@ class Emulator: # so do not set a requirement pass - reqs = set() if "require" in self.env: requirements = self.env['require'] self.log.debug("reqs = " + requirements) @@ -400,23 +393,7 @@ class Emulator: # Ignore empty fields - probably the only field anyway if len(requirement) == 0: continue - object_name = self.record_requirement(requirement) - reqs.add(object_name) - if self._existing_reqs is not None: - # If object exists then compare existing and new requirements. - if self._existing_reqs != reqs: - warnmsg = ("Object {} already exists with requirements:\n" - "{}: {}\n" - "{}: {}\n" - "Dependency resolver could not handle dependencies " - "as expected.".format( - self.cdist_object.name, - " ".join(self.cdist_object.source), - self._existing_reqs, - self.object_source, - reqs - )) - self.log.warning(warnmsg) + self.record_requirement(requirement) def record_auto_requirements(self): """An object shall automatically depend on all objects that it From 93506d2113e1d05202527639efe883a5b44b220d Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 8 Jul 2020 00:17:12 +0300 Subject: [PATCH 092/155] [__download] curl follow redirects --- cdist/conf/type/__download/explorer/remote_cmd | 2 +- cdist/conf/type/__download/gencode-local | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd index fbd4d84c..e3e35b45 100755 --- a/cdist/conf/type/__download/explorer/remote_cmd +++ b/cdist/conf/type/__download/explorer/remote_cmd @@ -6,7 +6,7 @@ then elif command -v curl > /dev/null then - cmd="curl -o - '%s'" + cmd="curl -L -o - '%s'" elif command -v fetch > /dev/null then diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local index 339827c2..571d2c3c 100755 --- a/cdist/conf/type/__download/gencode-local +++ b/cdist/conf/type/__download/gencode-local @@ -25,7 +25,7 @@ then elif command -v curl > /dev/null then - cmd="curl -o - '%s'" + cmd="curl -L -o - '%s'" elif command -v fetch > /dev/null then From e9062662868d7677a235889ba39a6091085388f0 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 8 Jul 2020 00:20:55 +0300 Subject: [PATCH 093/155] [__download] s/variable/format specification/ --- cdist/conf/type/__download/man.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index eafa9dfc..6ec0b19a 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -44,14 +44,14 @@ cmd-get Command used for downloading. Command must output to ``stdout``. Parameter will be used for ``printf`` and must include only one - variable ``%s`` which will become URL. + format specification ``%s`` which will become URL. For example: ``wget -O - '%s'``. cmd-sum Command used for checksum calculation. Command output and ``--sum`` parameter must match. Parameter will be used for ``printf`` and must include only one - variable ``%s`` which will become destination. + format specification ``%s`` which will become destination. For example: ``md5sum '%s' | awk '{print $1}'``. From cb9933b4a0d343a059899101810b6ee86fc87dbc Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 8 Jul 2020 12:43:55 +0200 Subject: [PATCH 094/155] Fix state -> state_is --- cdist/conf/type/__download/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote index 89ba72af..029a0801 100755 --- a/cdist/conf/type/__download/gencode-remote +++ b/cdist/conf/type/__download/gencode-remote @@ -19,7 +19,7 @@ then echo 'downloaded' > "$__messages_out" fi -if [ -f "$__object/parameter/onchange" ] && [ "$state" != "present" ] +if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ] then cat "$__object/parameter/onchange" fi From b8752e9ee390e983c1ce6467761aca8101b6548d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 10 Jul 2020 21:03:35 +0200 Subject: [PATCH 095/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index b823421b..54c42bde 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * Type __package_opkg: Add locking (Dennis Camera) * Type __hosts: Add --alias parameter (Dennis Camera) * Type __user: Fix shadow explorer for OpenBSD (Dennis Camera) + * Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak) 6.6.0: 2020-06-17 * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) From a5ae26116bd611fcab76c0383998fb4b05f1ae1d Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 11 Jul 2020 18:57:47 +0200 Subject: [PATCH 096/155] [type/__hosts] Fix when used without --alias --- cdist/conf/type/__hosts/manifest | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/cdist/conf/type/__hosts/manifest b/cdist/conf/type/__hosts/manifest index 0d9e61f8..8103ebd5 100755 --- a/cdist/conf/type/__hosts/manifest +++ b/cdist/conf/type/__hosts/manifest @@ -19,21 +19,24 @@ # along with this program. If not, see . # -set -e -u +set -e hostname=$__object_id -state=$(cat "$__object/parameter/state") -marker="# __hosts/$hostname" +state=$(cat "${__object}/parameter/state") +marker="# __hosts/${hostname}" -if [ "$state" = 'absent' ] +if test "${state}" != 'absent' then - set -- --regex "$marker" -else - ip=$(cat "$__object/parameter/ip") - aliases=$(while read -r a; do printf '\t%s' "$a"; done <"$__object/parameter/alias") + ip=$(cat "${__object}/parameter/ip") + if test -s "${__object}/parameter/alias" + then + aliases=$(while read -r a; do printf '\t%s' "$a"; done <"$__object/parameter/alias") + fi set -- --line "$(printf '%s\t%s%s %s' \ - "$ip" "$hostname" "$aliases" "$marker")" + "${ip}" "${hostname}" "${aliases}" "${marker}")" +else + set -- --regex "$(echo "${marker}" | sed -e 's/\./\\./')$" fi -__line "__hosts/$hostname" --file /etc/hosts --state "$state" "$@" +__line "/etc/hosts:${hostname}" --file /etc/hosts --state "${state}" "$@" From 19514662b022c79c4e6a56573c32dad45055310c Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 12 Jul 2020 12:24:00 +0200 Subject: [PATCH 097/155] [type/{__file/__directory}] Fix typo --- cdist/conf/type/__directory/explorer/stat | 2 +- cdist/conf/type/__file/explorer/stat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat index a7dc8431..422c5819 100755 --- a/cdist/conf/type/__directory/explorer/stat +++ b/cdist/conf/type/__directory/explorer/stat @@ -30,7 +30,7 @@ fallback() { gid=$(echo "$ls_line" | awk '{ print $4 }') owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd) - group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group) + group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group) mode_text=$(echo "$ls_line" | awk '{ print $1 }') mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}') diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat index 231768f6..3f971488 100755 --- a/cdist/conf/type/__file/explorer/stat +++ b/cdist/conf/type/__file/explorer/stat @@ -31,7 +31,7 @@ fallback() { gid=$(echo "$ls_line" | awk '{ print $4 }') owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd) - group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group) + group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group) mode_text=$(echo "$ls_line" | awk '{ print $1 }') mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}') From 9fb7e151b889214576cb891595a68ca8c6196d45 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 12 Jul 2020 12:31:55 +0200 Subject: [PATCH 098/155] [type/{__file/__directory}] Remove special Solaris blocks Solaris 11 has GNU stat (handled by *) Solaris 10 (and older?) does not have stat (handled by failing command -v stat) On Solaris 10 (at least on UFS), setgid cannot be set on directories. Unlike on other systems `chmod 2400` is not `-r----S---`, but `-r----l---`. --- cdist/conf/type/__directory/explorer/stat | 40 +++-------------------- cdist/conf/type/__file/explorer/stat | 40 +++-------------------- 2 files changed, 9 insertions(+), 71 deletions(-) diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat index 422c5819..f817cb02 100755 --- a/cdist/conf/type/__directory/explorer/stat +++ b/cdist/conf/type/__directory/explorer/stat @@ -33,7 +33,7 @@ fallback() { group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group) mode_text=$(echo "$ls_line" | awk '{ print $1 }') - mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}') + mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[lst]/)*2^(9+i/3)}printf("%04o",k)}') printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\n' \ "$("$__type_explorer/type")" \ @@ -45,11 +45,10 @@ fallback() { # nothing to work with, nothing we could do [ -e "$destination" ] || exit 0 -if ! command -v stat >/dev/null -then +command -v stat >/dev/null 2>&1 || { fallback exit -fi +} case $("$__explorer/os") in @@ -60,42 +59,13 @@ group: %Dg %Sg mode: %Mp%03Lp %Sp ' "$destination" | awk '/^type/ { print tolower($0); next } { print }' ;; - solaris) - ls1="$( ls -ld "$destination" )" - ls2="$( ls -ldn "$destination" )" - - if [ -f "$__object/parameter/mode" ] - then mode_should="$( cat "$__object/parameter/mode" )" - fi - - # yes, it is ugly hack, but if you know better way... - if [ -z "$( find "$destination" -perm "$mode_should" )" ] - then octets=888 - else octets="$( echo "$mode_should" | sed 's/^0//' )" - fi - - case "$( echo "$ls1" | cut -c1-1 )" in - -) echo 'type: regular file' ;; - d) echo 'type: directory' ;; - esac - - echo "owner: $( echo "$ls2" \ - | awk '{print $3}' ) $( echo "$ls1" \ - | awk '{print $3}' )" - - echo "group: $( echo "$ls2" \ - | awk '{print $4}' ) $( echo "$ls1" \ - | awk '{print $4}' )" - - echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )" - ;; *) # NOTE: Do not use --printf here as it is not supported by BusyBox stat. # NOTE: BusyBox's stat might not support the "-c" option, in which case # we fall through to the shell fallback. - stat -c 'type: %F + stat -c 'type: %F owner: %u %U group: %g %G mode: %04a %A' "$destination" 2>/dev/null || fallback - ;; + ;; esac diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat index 3f971488..29b3c8a3 100755 --- a/cdist/conf/type/__file/explorer/stat +++ b/cdist/conf/type/__file/explorer/stat @@ -34,7 +34,7 @@ fallback() { group=$(awk -F: -v gid="$gid" '$3 == gid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group) mode_text=$(echo "$ls_line" | awk '{ print $1 }') - mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[st]/)*2^(9+i/3)}printf("%04o",k)}') + mode=$(echo "$mode_text" | awk '{for(i=8;i>=0;--i){c=substr($1,10-i,1);k+=((c~/[rwxst]/)*2^i);if(!(i%3))k+=(tolower(c)~/[lst]/)*2^(9+i/3)}printf("%04o",k)}') size=$(echo "$ls_line" | awk '{ print $5 }') links=$(echo "$ls_line" | awk '{ print $2 }') @@ -53,11 +53,10 @@ fallback() { [ -e "$destination" ] || exit 0 -if ! command -v stat >/dev/null -then +command -v stat >/dev/null 2>&1 || { fallback exit -fi +} case $("$__explorer/os") @@ -71,37 +70,6 @@ size: %Dz links: %Dl ' "$destination" | awk '/^type/ { print tolower($0); next } { print }' ;; - solaris) - ls1="$( ls -ld "$destination" )" - ls2="$( ls -ldn "$destination" )" - - if [ -f "$__object/parameter/mode" ] - then mode_should="$( cat "$__object/parameter/mode" )" - fi - - # yes, it is ugly hack, but if you know better way... - if [ -z "$( find "$destination" -perm "$mode_should" )" ] - then octets=888 - else octets="$( echo "$mode_should" | sed 's/^0//' )" - fi - - case "$( echo "$ls1" | cut -c1-1 )" in - -) echo 'type: regular file' ;; - d) echo 'type: directory' ;; - esac - - echo "owner: $( echo "$ls2" \ - | awk '{print $3}' ) $( echo "$ls1" \ - | awk '{print $3}' )" - - echo "group: $( echo "$ls2" \ - | awk '{print $4}' ) $( echo "$ls1" \ - | awk '{print $4}' )" - - echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )" - echo "size: $( echo "$ls1" | awk '{print $5}' )" - echo "links: $( echo "$ls1" | awk '{print $2}' )" - ;; *) # NOTE: Do not use --printf here as it is not supported by BusyBox stat. # NOTE: BusyBox's stat might not support the "-c" option, in which case @@ -112,5 +80,5 @@ group: %g %G mode: %04a %A size: %s links: %h' "$destination" 2>/dev/null || fallback - ;; + ;; esac From bc970731316a59c21f8aa784fb27f1fa1b13ab89 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 13 Jul 2020 05:44:26 +0000 Subject: [PATCH 099/155] Merge branch 'bugfix/postfix-master-option' into '6.6' Fix broken --option parameter in __postfix_master type See merge request ungleich-public/cdist!905 (cherry picked from commit 2f433a1458f3a1f7f8859e9ae165178a0ec5b7a0) 9496b234 The option parameter is actually multi-valued 4009bbd7 Protect postfix variables in options --- cdist/conf/type/__postfix_master/gencode-remote | 2 +- cdist/conf/type/__postfix_master/parameter/optional | 1 - cdist/conf/type/__postfix_master/parameter/optional_multiple | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 cdist/conf/type/__postfix_master/parameter/optional_multiple diff --git a/cdist/conf/type/__postfix_master/gencode-remote b/cdist/conf/type/__postfix_master/gencode-remote index 7c109a69..73de1088 100755 --- a/cdist/conf/type/__postfix_master/gencode-remote +++ b/cdist/conf/type/__postfix_master/gencode-remote @@ -67,7 +67,7 @@ case "$state_should" in remove_entry fi cat << DONE -cat >> "$config" << ${__type##*/}_DONE +cat >> "$config" << "${__type##*/}_DONE" $(cat "$entry") ${__type##*/}_DONE DONE diff --git a/cdist/conf/type/__postfix_master/parameter/optional b/cdist/conf/type/__postfix_master/parameter/optional index 792b42c5..410482b8 100644 --- a/cdist/conf/type/__postfix_master/parameter/optional +++ b/cdist/conf/type/__postfix_master/parameter/optional @@ -4,6 +4,5 @@ unpriv chroot wakeup maxproc -option comment state diff --git a/cdist/conf/type/__postfix_master/parameter/optional_multiple b/cdist/conf/type/__postfix_master/parameter/optional_multiple new file mode 100644 index 00000000..01925a15 --- /dev/null +++ b/cdist/conf/type/__postfix_master/parameter/optional_multiple @@ -0,0 +1 @@ +option From 8903540e9106db93e490515c07929a207326be77 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 13 Jul 2020 07:54:12 +0200 Subject: [PATCH 100/155] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 54c42bde..dac9cbce 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,8 @@ next: * Type __hosts: Add --alias parameter (Dennis Camera) * Type __user: Fix shadow explorer for OpenBSD (Dennis Camera) * Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak) + * Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera) + * Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali) 6.6.0: 2020-06-17 * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) From 3965c7f73844873aabb1d839b2298eed70a6b35c Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 21 Jul 2020 19:42:40 +0200 Subject: [PATCH 101/155] [type/__user] Install user{add,mod,del} packages on OpenWrt --- cdist/conf/type/__user/manifest | 42 +++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/cdist/conf/type/__user/manifest b/cdist/conf/type/__user/manifest index 8f10b38c..b9fad65b 100644 --- a/cdist/conf/type/__user/manifest +++ b/cdist/conf/type/__user/manifest @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2019 Nico Schottelius (nico-cdist at schottelius.org) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,16 +18,37 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# # Manage users. +# -os=$(cat "$__global/explorer/os") - -case "$os" in - alpine) - __package shadow - ;; - *) - : - ;; +case $(cat "${__global}/explorer/os") +in + (alpine) + __package shadow + ;; + (openwrt) + case $(cat "${__object}/parameter/state") + in + (present) + if test -s "${__object}/explorer/passwd" + then + # NOTE: The package might not be required if no changes + # are required, but determining if changes are required is + # out of scope here, and 40k should be okay, I hope. + __package shadow-usermod + else + __package shadow-useradd + fi + ;; + (absent) + if test -s "${__object}/explorer/passwd" + then + __package shadow-userdel + fi + ;; + esac + ;; + (*) + : + ;; esac From d8b5c733f6cec871ceb1933807f4877a4ee77ffa Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 22 Jul 2020 06:36:27 +0200 Subject: [PATCH 102/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index dac9cbce..040de9d8 100644 --- a/docs/changelog +++ b/docs/changelog @@ -10,6 +10,7 @@ next: * Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak) * Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera) * Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali) + * Type __user: Install user packages on OpenWRT (Dennis Camera) 6.6.0: 2020-06-17 * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) From fdef468f1ab6f2163da22a477df154a2082b57b4 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 22 Jul 2020 18:28:41 +0200 Subject: [PATCH 103/155] Fix OpenWrt spelling --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 040de9d8..4c17ee58 100644 --- a/docs/changelog +++ b/docs/changelog @@ -10,7 +10,7 @@ next: * Core: Make emulator-part code consistent; remove faulty warning (Darko Poljak) * Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera) * Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali) - * Type __user: Install user packages on OpenWRT (Dennis Camera) + * Type __user: Install user packages on OpenWrt (Dennis Camera) 6.6.0: 2020-06-17 * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) From 595e43b8d5f06641179c13de8ac04be46c1b2820 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 23 Jul 2020 09:41:53 +0200 Subject: [PATCH 104/155] [type/{__file,__directory}] Fix incorrect interpretation of strings with leading 0s as octal --- cdist/conf/type/__directory/gencode-remote | 4 +++- cdist/conf/type/__file/gencode-remote | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote index 2c2c56fd..d9c00b56 100755 --- a/cdist/conf/type/__directory/gencode-remote +++ b/cdist/conf/type/__directory/gencode-remote @@ -99,7 +99,9 @@ case "$state_should" in # format mode in four digits => same as stat returns if [ "$attribute" = mode ]; then - value_should=$(printf '%04u' "${value_should}") + # Convert to four-digit octal number (printf interprets + # strings with leading 0s as octal!) + value_should=$(printf '%04o' "0${value_should}") fi if [ "$set_attributes" = 1 ] || [ "$value_should" != "$value_is" ]; then diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index a69154df..35356b13 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -70,7 +70,9 @@ case "$state_should" in # format mode in four digits => same as stat returns if [ "$attribute" = mode ]; then - value_should=$(printf '%04u' "${value_should}") + # Convert to four-digit octal number (printf interprets + # strings with leading 0s as octal!) + value_should=$(printf '%04o' "0${value_should}") fi value_is="$(get_current_value "$attribute" "$value_should")" From ae5f0bba0b103aabcfbad0f105c69c0b403697ac Mon Sep 17 00:00:00 2001 From: fnux Date: Fri, 24 Jul 2020 12:26:35 +0200 Subject: [PATCH 105/155] Add Alpine support to __openldap_server --- cdist/conf/type/__openldap_server/man.rst | 4 +-- cdist/conf/type/__openldap_server/manifest | 42 ++++++++++++++++++---- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst index fbad21d8..a96c7dad 100644 --- a/cdist/conf/type/__openldap_server/man.rst +++ b/cdist/conf/type/__openldap_server/man.rst @@ -103,8 +103,8 @@ syncrepl-host Set once per host that will replicate the directory. module - LDAP module to load. See `slapd.conf(5)`. - Default value is OS-dependent, see manifest. + LDAP module to load. See `slapd.conf(5)`. Some dependencies might have to + be installed beforehand. Default value is OS-dependent, see manifest. schema Name of LDAP schema to load. Must be the name without extension of a diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest index 84ba176f..2aeece26 100644 --- a/cdist/conf/type/__openldap_server/manifest +++ b/cdist/conf/type/__openldap_server/manifest @@ -25,6 +25,7 @@ case "${os}" in SLAPD_DATA_DIR="/var/db/openldap-data" SLAPD_RUN_DIR="/var/run/openldap" SLAPD_MODULE_PATH="/usr/local/libexec/openldap" + SLAPD_MODULE_TYPE="la" if [ -z "${slapd_modules}" ]; then # It looks like ppolicy and syncprov must be compiled slapd_modules="back_mdb back_monitor" @@ -43,13 +44,34 @@ case "${os}" in SLAPD_DATA_DIR="/var/lib/ldap" SLAPD_RUN_DIR="/var/run/slapd" SLAPD_MODULE_PATH="/usr/lib/ldap" + SLAPD_MODULE_TYPE="la" if [ -z "${slapd_modules}" ]; then slapd_modules="back_mdb ppolicy syncprov back_monitor" fi + CONF_OWNER="openldap" + CONF_GROUP="openldap" if [ -z "${tls_cipher_suite}" ]; then tls_cipher_suite="NORMAL" fi ;; + alpine) + PKGS="openldap openldap-clients" + ETC="/etc" + SLAPD_DIR="/etc/openldap" + SLAPD_DATA_DIR="/var/lib/openldap" + SLAPD_RUN_DIR="/var/run/openldap" + SLAPD_MODULE_PATH="/usr/lib/openldap" + SLAPD_MODULE_TYPE="so" + if [ -z "${slapd_modules}" ]; then + slapd_modules="back_mdb ppolicy syncprov back_monitor" + PKGS="$PKGS openldap-back-mdb openldap-back-monitor openldap-overlay-all" + fi + CONF_OWNER="ldap" + CONF_GROUP="$SLAPD_USER" + if [ -z "${tls_cipher_suite}" ]; then + tls_cipher_suite="DEFAULT" + fi + ;; *) echo "Don't know the openldap defaults for: $os" >&2 exit 1 @@ -156,6 +178,12 @@ case "${os}" in --line "SLAPD_SERVICES=\"${slapd_urls}\"" \ --state present ;; + alpine) + require="__package/${PKG_MAIN}" __line add_slapd_services \ + --file ${ETC}/conf.d/slapd \ + --line "command_args=\"-h '${slapd_urls}'\"" \ + --state present + ;; *) # Nothing to do here, move on. ;; @@ -170,20 +198,22 @@ if [ -z "${_skip_letsencrypt_cert}" ]; then fi # shellcheck disable=SC2086 - __letsencrypt_cert "${name}" --admin-email "${admin_email}" \ - --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R openldap:openldap ${SLAPD_DIR}/sasl2 && service slapd restart" \ - --automatic-renewal ${staging} + __directory ${SLAPD_DIR}/sasl2 + require="__directory/${SLAPD_DIR}/sasl2" __letsencrypt_cert "${name}" \ + --admin-email "${admin_email}" \ + --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R ${CONF_OWNER}:${CONF_GROUP} ${SLAPD_DIR}/sasl2 && service slapd restart" \ + --automatic-renewal "${staging}" fi require="__package/${PKG_MAIN}" __directory ${SLAPD_DIR}/slapd.d --state absent if [ -z "${_skip_letsencrypt_cert}" ]; then require="__package/${PKG_MAIN} __letsencrypt_cert/${name}" \ - __file ${SLAPD_DIR}/slapd.conf --owner ${CONF_OWNER} --group ${CONF_GROUP} --mode 644 \ + __file "${SLAPD_DIR}/slapd.conf" --owner "${CONF_OWNER}" --group "${CONF_GROUP}" --mode 644 \ --source "${ldapconf}" else require="__package/${PKG_MAIN}" \ - __file ${SLAPD_DIR}/slapd.conf --owner ${CONF_OWNER} --group ${CONF_GROUP} --mode 644 \ + __file "${SLAPD_DIR}/slapd.conf" --owner "${CONF_OWNER}" --group "${CONF_GROUP}" --mode 644 \ --source "${ldapconf}" fi @@ -210,7 +240,7 @@ done # Add specified modules echo "modulepath ${SLAPD_MODULE_PATH}" >> "${ldapconf}" for module in ${slapd_modules}; do - echo "moduleload ${module}.la" >> "${ldapconf}" + echo "moduleload ${module}.${SLAPD_MODULE_TYPE}" >> "${ldapconf}" done # Rest of the config From 8654cbe4661762f25cd75c484f28344ce91ec55b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 24 Jul 2020 12:29:02 +0200 Subject: [PATCH 106/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 4c17ee58..0b9b8308 100644 --- a/docs/changelog +++ b/docs/changelog @@ -11,6 +11,7 @@ next: * Types __file, __directory: Support setuid, setguid, sticky bits (Dennis Camera) * Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali) * Type __user: Install user packages on OpenWrt (Dennis Camera) + * Type __openldap_server: Add Alpine support (Timothée Floure) 6.6.0: 2020-06-17 * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) From 8b53f35ffab4e7b74d3fd019e7602c8a05338173 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 24 Jul 2020 12:33:16 +0200 Subject: [PATCH 107/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 0b9b8308..2457e90b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -12,6 +12,7 @@ next: * Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali) * Type __user: Install user packages on OpenWrt (Dennis Camera) * Type __openldap_server: Add Alpine support (Timothée Floure) + * Type __pf_apply: Remove deprecated type (Darko Poljak) 6.6.0: 2020-06-17 * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) From ee71cad047ba0c86343fe8ee9ea2bc79bf950256 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 25 Jul 2020 19:19:38 +0200 Subject: [PATCH 108/155] [type/__package_apt] Fix type for legacy APT versions --no-install-recommends was introduced with Debian 5. The APT::Install-Recommends option gets ignored by old versions and produces no error. --- cdist/conf/type/__package_apt/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index e02564a2..f3d91566 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -64,7 +64,7 @@ esac # Hint if we need to avoid questions at some point: # DEBIAN_PRIORITY=critical can reduce the number of questions -aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes --no-install-recommends -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\"" +aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o APT::Install-Recommends=0 -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\"" if [ "$state_is" = "$state_should" ]; then if [ -z "$version" ] || [ "$version" = "$version_is" ]; then From 46d09392f08b0659557048c02e109f189ffd85f3 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 26 Jul 2020 19:36:34 +0200 Subject: [PATCH 109/155] [type/__key_value] Get AWK from POSIX PATH This is required here, because Solaris /usr/bin/awk does not support the sub() function. So xpg4 AWK needs to be used. --- cdist/conf/type/__key_value/explorer/state | 4 +++- cdist/conf/type/__key_value/files/remote_script.sh | 5 ++++- cdist/conf/type/__key_value/gencode-remote | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__key_value/explorer/state b/cdist/conf/type/__key_value/explorer/state index 7b2de1df..d24600af 100755 --- a/cdist/conf/type/__key_value/explorer/state +++ b/cdist/conf/type/__key_value/explorer/state @@ -40,7 +40,9 @@ else fi export key state delimiter value exact_delimiter -awk -f - "$file" <<"AWK_EOF" +awk_bin=$(PATH=$(getconf PATH 2>/dev/null) && command -v awk || echo awk) + +"${awk_bin}" -f - "$file" <<"AWK_EOF" BEGIN { state=ENVIRON["state"] key=ENVIRON["key"] diff --git a/cdist/conf/type/__key_value/files/remote_script.sh b/cdist/conf/type/__key_value/files/remote_script.sh index f7a1add5..faf080cb 100644 --- a/cdist/conf/type/__key_value/files/remote_script.sh +++ b/cdist/conf/type/__key_value/files/remote_script.sh @@ -24,7 +24,10 @@ if [ -f "$file" ]; then else touch "$file" fi -awk -f - "$file" >"$tmpfile" <<"AWK_EOF" + +awk_bin=$(PATH=$(getconf PATH 2>/dev/null) && command -v awk || echo awk) + +"${awk_bin}" -f - "$file" >"$tmpfile" <<"AWK_EOF" BEGIN { # import variables in a secure way .. state=ENVIRON["state"] diff --git a/cdist/conf/type/__key_value/gencode-remote b/cdist/conf/type/__key_value/gencode-remote index 13cc27c7..1174400e 100755 --- a/cdist/conf/type/__key_value/gencode-remote +++ b/cdist/conf/type/__key_value/gencode-remote @@ -25,7 +25,7 @@ state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" fire_onchange='' -if [ "$state_is" = "$state_should" ]; then +if [ "$state_is" = "$state_should" ]; then exit 0 fi From a5905044365e0f28e793f01e0be0cac8e6b2b379 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 24 Jul 2020 10:19:10 +0200 Subject: [PATCH 110/155] [type/__locale_system] RedHat systems on systemd use /etc/locale.conf --- cdist/conf/type/__locale_system/manifest | 26 ++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest index 4a1fdeed..22531a40 100755 --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -3,6 +3,7 @@ # 2012-2016 Steven Armstrong (steven-cdist at armstrong.cc) # 2016 Carlos Ortigoza (carlos.ortigoza at ungleich.ch) # 2016 Nico Schottelius (nico.schottelius at ungleich.ch) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -32,8 +33,25 @@ case "$os" in archlinux) locale_conf="/etc/locale.conf" ;; - redhat|centos) - locale_conf="/etc/sysconfig/i18n" + centos|redhat|scientific) + # shellcheck source=/dev/null + version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}") + if expr "${version_id}" '>=' 7 >/dev/null + then + locale_conf="/etc/locale.conf" + else + locale_conf="/etc/sysconfig/i18n" + fi + ;; + fedora) + # shellcheck source=/dev/null + version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}") + if expr "${version_id}" '>=' 18 >/dev/null + then + locale_conf="/etc/locale.conf" + else + locale_conf="/etc/sysconfig/i18n" + fi ;; *) echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 @@ -47,9 +65,9 @@ __file "$locale_conf" \ --state exists require="__file/$locale_conf" \ - __key_value "$locale_conf:$__object_id" \ +__key_value "$locale_conf:$__object_id" \ --file "$locale_conf" \ --key "$__object_id" \ - --delimiter = \ + --delimiter '=' --exact_delimiter \ --state "$(cat "$__object/parameter/state")" \ --value "$(cat "$__object/parameter/value")" From 47e28fc441187b5deb869f53396eac1ee0f8cbe7 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 26 Jul 2020 12:07:38 +0200 Subject: [PATCH 111/155] [type/__locale_system] Support old Debian derivatives --- cdist/conf/type/__locale_system/manifest | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest index 22531a40..b9991fa3 100755 --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -27,9 +27,29 @@ os=$(cat "$__global/explorer/os") case "$os" in - debian|devuan|ubuntu) + debian) + os_version=$(cat "${__global}/explorer/os_version") + if expr "${os_version}" '>=' 4 >/dev/null + then + # Debian 4 (etch) and later + locale_conf="/etc/default/locale" + else + locale_conf="/etc/environment" + fi + ;; + devuan) locale_conf="/etc/default/locale" ;; + ubuntu) + os_version=$(cat "${__global}/explorer/os_version") + if expr "${os_version}" '>=' 6.10 >/dev/null + then + # Ubuntu 6.10 (edgy) and later + locale_conf="/etc/default/locale" + else + locale_conf="/etc/environment" + fi + ;; archlinux) locale_conf="/etc/locale.conf" ;; @@ -61,7 +81,7 @@ case "$os" in esac __file "$locale_conf" \ - --owner root --group root --mode 644 \ + --owner root --group root --mode 0644 \ --state exists require="__file/$locale_conf" \ From 0ef54a721d6f540f70959df2b7f1321a5688a594 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 26 Jul 2020 12:17:30 +0200 Subject: [PATCH 112/155] [type/__locale_system] Add support for Gentoo Linux --- cdist/conf/type/__locale_system/manifest | 45 ++++++++++++++++++------ 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest index b9991fa3..0049ed5f 100755 --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -24,9 +24,19 @@ # Configure system-wide locale by modifying i18n file. # +onchange_cmd= # none, by default +quote_value=false + +catval() { + # shellcheck disable=SC2059 + printf "$($quote_value && echo '"%s"' || echo '%s')" "$(cat "$1")" +} + + os=$(cat "$__global/explorer/os") -case "$os" in +case $os +in debian) os_version=$(cat "${__global}/explorer/os_version") if expr "${os_version}" '>=' 4 >/dev/null @@ -69,10 +79,24 @@ case "$os" in if expr "${version_id}" '>=' 18 >/dev/null then locale_conf="/etc/locale.conf" + quote_value=false else locale_conf="/etc/sysconfig/i18n" fi ;; + gentoo) + case $(cat "${__global}/explorer/init") + in + (*openrc*) + locale_conf="/etc/env.d/02locale" + onchange_cmd="env-update --no-ldconfig" + quote_value=true + ;; + (systemd) + locale_conf="/etc/locale.conf" + ;; + esac + ;; *) echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 echo "Please contribute an implementation for it if you can." >&2 @@ -80,14 +104,13 @@ case "$os" in ;; esac -__file "$locale_conf" \ - --owner root --group root --mode 0644 \ - --state exists +__file "${locale_conf}" --state exists --owner root --group root --mode 0644 -require="__file/$locale_conf" \ -__key_value "$locale_conf:$__object_id" \ - --file "$locale_conf" \ - --key "$__object_id" \ - --delimiter '=' --exact_delimiter \ - --state "$(cat "$__object/parameter/state")" \ - --value "$(cat "$__object/parameter/value")" +require="__file/${locale_conf}" \ +__key_value "${locale_conf}:${__object_id}" \ + --file "${locale_conf}" \ + --key "${__object_id}" \ + --delimiter '=' --exact_delimiter \ + --state "$(cat "${__object}/parameter/state")" \ + --value "$(catval "${__object}/parameter/value")" \ + --onchange "${onchange_cmd}" From 630d987d5f5a40ae783fd9d4ff0a10653fcab04c Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 26 Jul 2020 12:28:20 +0200 Subject: [PATCH 113/155] [type/__locale_system] Add support for Void Linux --- cdist/conf/type/__locale_system/manifest | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest index 0049ed5f..ff0a8c23 100755 --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -97,6 +97,9 @@ in ;; esac ;; + voidlinux) + locale_conf="/etc/locale.conf" + ;; *) echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 echo "Please contribute an implementation for it if you can." >&2 From 0ae0935afafd6b10516e9689c19000ef729d1b9b Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 26 Jul 2020 14:05:36 +0200 Subject: [PATCH 114/155] [type/__locale_system] Add support for SuSE --- cdist/conf/type/__locale_system/manifest | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest index ff0a8c23..26d4f7b5 100755 --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -24,6 +24,7 @@ # Configure system-wide locale by modifying i18n file. # +key=$__object_id onchange_cmd= # none, by default quote_value=false @@ -97,6 +98,25 @@ in ;; esac ;; + suse) + os_version=$(cat "${__global}/explorer/os_version") + os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)') + + # https://documentation.suse.com/sles/15-SP2/html/SLES-all/cha-suse.html#sec-suse-l10n + if expr "${os_major}" '>=' 15 \& "${os_major}" != 42 + then + # It seems that starting with SuSE 15 the systemd /etc/locale.conf + # is the preferred way to set locales, although + # /etc/sysconfig/language is still available. + # Older documentation doesn't mention /etc/locale.conf, even though + # is it created when localectl is used. + locale_conf="/etc/locale.conf" + else + locale_conf="/etc/sysconfig/language" + quote_value=true + key="RC_${__object_id}" + fi + ;; voidlinux) locale_conf="/etc/locale.conf" ;; @@ -110,9 +130,9 @@ esac __file "${locale_conf}" --state exists --owner root --group root --mode 0644 require="__file/${locale_conf}" \ -__key_value "${locale_conf}:${__object_id}" \ +__key_value "${locale_conf}:${key}" \ --file "${locale_conf}" \ - --key "${__object_id}" \ + --key "${key}" \ --delimiter '=' --exact_delimiter \ --state "$(cat "${__object}/parameter/state")" \ --value "$(catval "${__object}/parameter/value")" \ From cbf22f3b2c8b9d0fef8f6b7b0bf16508f96675b3 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 26 Jul 2020 15:23:55 +0200 Subject: [PATCH 115/155] [type/__locale_system] Add support for Solaris --- cdist/conf/type/__locale_system/manifest | 41 ++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest index 26d4f7b5..71491fe5 100755 --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -33,6 +33,7 @@ catval() { printf "$($quote_value && echo '"%s"' || echo '%s')" "$(cat "$1")" } +state_should=$(cat "${__object}/parameter/state") os=$(cat "$__global/explorer/os") @@ -98,6 +99,39 @@ in ;; esac ;; + solaris) + locale_conf="/etc/default/init" + locale_conf_group="sys" + + if expr "$(cat "${__global}/explorer/os_version")" '>=' 5.11 >/dev/null + then + # mode on Oracle Solaris 11 is actually 0444, + # but the write bit makes sense, IMO + locale_conf_mode=0644 + + # Oracle Solaris 11.2 and later uses SMF to store environment info. + # This is a hack, but I didn't feel like modifying the whole type + # just for some Oracle nonsense. + # 11.3 apparently added nlsadm(1m), but it is missing from 11.2. + # Illumos continues to use /etc/default/init + # NOTE: Remember not to use "cool" POSIX features like -q or -e with + # Solaris grep. + release_regex='Oracle Solaris 11.[2-9][0-9]*' + case $state_should + in + (present) + svccfg_cmd="svccfg -s svc:/system/environment:init setprop environment/${key} = astring: '$(cat "${__object}/parameter/value")'" + ;; + (absent) + svccfg_cmd="svccfg -s svc:/system/environment:init delprop environment/${key}" + ;; + esac + refresh_cmd='svcadm refresh svc:/system/environment' + onchange_cmd="grep '${release_regex}' /etc/release >&- || exit 0; ${svccfg_cmd:-:} && ${refresh_cmd}" + else + locale_conf_mode=0555 + fi + ;; suse) os_version=$(cat "${__global}/explorer/os_version") os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)') @@ -127,13 +161,16 @@ in ;; esac -__file "${locale_conf}" --state exists --owner root --group root --mode 0644 +__file "${locale_conf}" --state exists \ + --owner "${locale_conf_owner:-0}" \ + --group "${locale_conf_group:-0}" \ + --mode "${locale_conf_mode:-0644}" require="__file/${locale_conf}" \ __key_value "${locale_conf}:${key}" \ --file "${locale_conf}" \ --key "${key}" \ --delimiter '=' --exact_delimiter \ - --state "$(cat "${__object}/parameter/state")" \ + --state "${state_should}" \ --value "$(catval "${__object}/parameter/value")" \ --onchange "${onchange_cmd}" From a923e75d9b0054212031507986d3b935d74d52bf Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 26 Jul 2020 18:29:14 +0200 Subject: [PATCH 116/155] [type/__locale_system] Add support for NetBSD --- cdist/conf/type/__locale_system/manifest | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest index 71491fe5..92af852f 100755 --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -99,6 +99,15 @@ in ;; esac ;; + netbsd) + # NetBSD doesn't have a separate configuration file to set locales. + # So the shell login file will have to do. + # "Non-POSIX" shells like csh will not be updated here. + + locale_conf="/etc/profile" + quote_value=true + value="$(catval "${__object}/parameter/value"); export ${key}" + ;; solaris) locale_conf="/etc/default/init" locale_conf_group="sys" @@ -172,5 +181,5 @@ __key_value "${locale_conf}:${key}" \ --key "${key}" \ --delimiter '=' --exact_delimiter \ --state "${state_should}" \ - --value "$(catval "${__object}/parameter/value")" \ + --value "${value:-$(catval "${__object}/parameter/value")}" \ --onchange "${onchange_cmd}" From 511d8c96aa9201049eaafc459a2de9694fe2a4e4 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 26 Jul 2020 19:23:17 +0200 Subject: [PATCH 117/155] [type/__locale_system] Add support for Slackware --- cdist/conf/type/__locale_system/manifest | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest index 92af852f..180788e6 100755 --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -141,6 +141,12 @@ in locale_conf_mode=0555 fi ;; + slackware) + # NOTE: lang.csh (csh config) is ignored here. + locale_conf="/etc/profile.d/lang.sh" + locale_conf_mode=0755 + key="export ${__object_id}" + ;; suse) os_version=$(cat "${__global}/explorer/os_version") os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)') @@ -176,7 +182,7 @@ __file "${locale_conf}" --state exists \ --mode "${locale_conf_mode:-0644}" require="__file/${locale_conf}" \ -__key_value "${locale_conf}:${key}" \ +__key_value "${locale_conf}:${key#export }" \ --file "${locale_conf}" \ --key "${key}" \ --delimiter '=' --exact_delimiter \ From 70d1228dc0c6a3af105f0864b46d7dea806aedc2 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 26 Jul 2020 19:26:45 +0200 Subject: [PATCH 118/155] [type/__locale_system] Add support for FreeBSD --- cdist/conf/type/__locale_system/manifest | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest index 180788e6..e4286ef6 100755 --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -99,9 +99,10 @@ in ;; esac ;; - netbsd) - # NetBSD doesn't have a separate configuration file to set locales. - # So the shell login file will have to do. + freebsd|netbsd) + # NetBSD doesn't have a separate configuration file to set locales. + # In FreeBSD locales could be configured via /etc/login.conf but parsing + # that would be annoying, so the shell login file will have to do. # "Non-POSIX" shells like csh will not be updated here. locale_conf="/etc/profile" From 73f1937636bfa59eea0432e3625c0a27cc8171b5 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Mon, 27 Jul 2020 06:20:21 +0200 Subject: [PATCH 119/155] [__unpack] no mkdir by default, because destination can be file, but tar needs mkdir andrar needs slash at the end --- cdist/conf/type/__unpack/explorer/state | 37 +++++++++ cdist/conf/type/__unpack/gencode-remote | 75 ++++++++++++++++++ cdist/conf/type/__unpack/man.rst | 79 +++++++++++++++++++ cdist/conf/type/__unpack/manifest | 41 ++++++++++ cdist/conf/type/__unpack/parameter/boolean | 2 + cdist/conf/type/__unpack/parameter/optional | 2 + cdist/conf/type/__unpack/parameter/required | 1 + cdist/conf/type/__unpack/test/README | 3 + .../type/__unpack/test/make-init-manifest.sh | 22 ++++++ .../type/__unpack/test/make-test-files.sh | 44 +++++++++++ 10 files changed, 306 insertions(+) create mode 100755 cdist/conf/type/__unpack/explorer/state create mode 100755 cdist/conf/type/__unpack/gencode-remote create mode 100644 cdist/conf/type/__unpack/man.rst create mode 100755 cdist/conf/type/__unpack/manifest create mode 100644 cdist/conf/type/__unpack/parameter/boolean create mode 100644 cdist/conf/type/__unpack/parameter/optional create mode 100644 cdist/conf/type/__unpack/parameter/required create mode 100644 cdist/conf/type/__unpack/test/README create mode 100755 cdist/conf/type/__unpack/test/make-init-manifest.sh create mode 100755 cdist/conf/type/__unpack/test/make-test-files.sh diff --git a/cdist/conf/type/__unpack/explorer/state b/cdist/conf/type/__unpack/explorer/state new file mode 100755 index 00000000..38bc0978 --- /dev/null +++ b/cdist/conf/type/__unpack/explorer/state @@ -0,0 +1,37 @@ +#!/bin/sh -e + +src="/$__object_id" + +if [ -f "$__object/parameter/sum-file" ] +then + src_sum_was_file="$( cat "$__object/parameter/sum-file" )" +else + src_sum_was_file="$src.cdist__unpack_sum" +fi + +if [ ! -f "$src" ] +then + if [ -n "$__cdist_dry_run" ] + then + echo 'mismatch' + else + echo 'missing' + fi +else + if [ ! -f "$src_sum_was_file" ] + then + echo 'mismatch' + exit 0 + fi + + src_sum_was="$( cat "$src_sum_was_file" )" + + src_sum_is="$( cksum "$src" | awk '{ print $1$2 }' )" + + if [ "$src_sum_was" = "$src_sum_is" ] + then + echo 'match' + else + echo 'mismatch' + fi +fi diff --git a/cdist/conf/type/__unpack/gencode-remote b/cdist/conf/type/__unpack/gencode-remote new file mode 100755 index 00000000..45c7173a --- /dev/null +++ b/cdist/conf/type/__unpack/gencode-remote @@ -0,0 +1,75 @@ +#!/bin/sh -e + +if grep -Eq '^(missing|match)$' "$__object/explorer/state" +then + exit 0 +fi + +os="$( cat "$__global/explorer/os" )" + +src="/$__object_id" + +dst="$( sed 's/\/$//' "$__object/parameter/destination" )" + +cmd='' + +case "$src" in + *.tar|*.tgz|*.tar.*) + cmd="mkdir -p '$dst' && tar --directory='$dst' --extract --file='$src'" + + if [ -f "$__object/parameter/tar-strip" ] + then + tar_strip="$( cat "$__object/parameter/tar-strip" )" + + cmd="$cmd --strip-components=$tar_strip" + fi + ;; + *.7z) + case "$os" in + centos|fedora|redhat) + cmd='7za' + ;; + *) + cmd='7zr' + ;; + esac + + cmd="$cmd e -aoa -o'$dst' '$src'" + ;; + *.bz2) + cmd="bunzip2 --stdout '$src' > '$dst'" + ;; + *.gz) + cmd="gunzip --stdout '$src' > '$dst'" + ;; + *.lzma|*.xz) + cmd="xz --uncompress --stdout '$src' > '$dst'" + ;; + *.rar) + cmd="unrar x -o+ '$src' '$dst/'" + ;; + *.zip) + cmd="unzip -o '$src' -d '$dst'" + ;; +esac + +if [ -f "$__object/parameter/backup-destination" ] +then + echo "if [ -e '$dst' ]; then mv '$dst' '$dst.cdist__unpack_backup_$( date +%s )'; fi" +fi + +echo "$cmd" + +if [ -f "$__object/parameter/sum-file" ] +then + sum_file="$( cat "$__object/parameter/sum-file" )" +else + sum_file="$src.cdist__unpack_sum" +fi + +echo "cksum '$src' | awk '{ print \$1\$2 }' > '$sum_file'" + +if [ ! -f "$__object/parameter/preserve-archive" ] +then + echo "rm -f '$src'" +fi diff --git a/cdist/conf/type/__unpack/man.rst b/cdist/conf/type/__unpack/man.rst new file mode 100644 index 00000000..8fe96e43 --- /dev/null +++ b/cdist/conf/type/__unpack/man.rst @@ -0,0 +1,79 @@ +cdist-type__unpack(7) +===================== + +NAME +---- +cdist-type__unpack - Unpack archives + + +DESCRIPTION +----------- +Unpack ``.tar``, ``.tgz``, ``.tar.*``, ``.7z``, ``.bz2``, ``.gz``, +``.lzma``, ``.xz``, ``.rar`` and ``.zip`` archives. Archive type is +detected by extension. + +To achieve idempotency, checksum file will be created in target. See +``--sum-file`` parameter for details. + + +REQUIRED PARAMETERS +------------------- +destination + Depending on archive format file or directory to where archive + contents will be written. + + +OPTIONAL PARAMETERS +------------------- +sum-file + Override archive's checksum file in target. By default + ``XXX.cdist__unpack_sum`` will be used, where ``XXX`` is source + archive path. This file must be kept in target's persistent storage. + +tar-strip + Tarball specific. See ``man tar`` for ``--strip-components``. + + +OPTIONAL BOOLEAN PARAMETERS +--------------------------- +backup-destination + By default destination file will be overwritten. In case destination + is directory, files from archive will be added to or overwritten in + directory. This parameter moves existing destination to + ``XXX.cdist__unpack_backup_YYY``, where ``XXX`` is destination and + ``YYY`` current UNIX timestamp. + +preserve-archive + Don't delete archive after unpacking. + + +EXAMPLES +-------- + +.. code-block:: sh + + __directory /opt/cpma + + require='__directory/opt/cpma' \ + __download /opt/cpma/cnq3.zip \ + --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \ + --sum md5:46da3021ca9eace277115ec9106c5b46 + + require='__download/opt/cpma/cnq3.zip' \ + __unpack /opt/cpma/cnq3.zip \ + --backup-destination \ + --preserve-archive \ + --destination /opt/cpma/server + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2020 Ander Punnar. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__unpack/manifest b/cdist/conf/type/__unpack/manifest new file mode 100755 index 00000000..6bdf5a10 --- /dev/null +++ b/cdist/conf/type/__unpack/manifest @@ -0,0 +1,41 @@ +#!/bin/sh -e + +os="$( cat "$__global/explorer/os" )" + +src="/$__object_id" + +case "$src" in + *.7z) + __package p7zip + ;; + *.bz2) + case "$os" in + freebsd) + # bzip2 is part of freebsd base system + ;; + *) + __package bzip2 + ;; + esac + ;; + *.lzma|*.xz|*.txz) + case "$os" in + debian|ubuntu|devuan) + __package xz-utils + ;; + alpine|centos) + __package xz + ;; + esac + ;; + *.rar) + case "$os" in + debian|ubuntu|devuan|alpine|freebsd) + __package unrar + ;; + esac + ;; + *.zip) + __package unzip + ;; +esac diff --git a/cdist/conf/type/__unpack/parameter/boolean b/cdist/conf/type/__unpack/parameter/boolean new file mode 100644 index 00000000..99dca934 --- /dev/null +++ b/cdist/conf/type/__unpack/parameter/boolean @@ -0,0 +1,2 @@ +backup-destination +preserve-archive diff --git a/cdist/conf/type/__unpack/parameter/optional b/cdist/conf/type/__unpack/parameter/optional new file mode 100644 index 00000000..d136dd0c --- /dev/null +++ b/cdist/conf/type/__unpack/parameter/optional @@ -0,0 +1,2 @@ +sum-file +tar-strip diff --git a/cdist/conf/type/__unpack/parameter/required b/cdist/conf/type/__unpack/parameter/required new file mode 100644 index 00000000..ac459b09 --- /dev/null +++ b/cdist/conf/type/__unpack/parameter/required @@ -0,0 +1 @@ +destination diff --git a/cdist/conf/type/__unpack/test/README b/cdist/conf/type/__unpack/test/README new file mode 100644 index 00000000..54f3972a --- /dev/null +++ b/cdist/conf/type/__unpack/test/README @@ -0,0 +1,3 @@ +./make-test-files.sh +./make-init-manifest.sh | cdist config -i - localhost +sudo find /tmp/cdist__unpack_test/ -type f -exec cat {} \; | sort diff --git a/cdist/conf/type/__unpack/test/make-init-manifest.sh b/cdist/conf/type/__unpack/test/make-init-manifest.sh new file mode 100755 index 00000000..404bc106 --- /dev/null +++ b/cdist/conf/type/__unpack/test/make-init-manifest.sh @@ -0,0 +1,22 @@ +#!/bin/sh -e + +p="$( pwd )" +d=/tmp/cdist__unpack_test + +echo 'export CDIST_ORDER_DEPENDENCY=1' + +echo "__directory $d" + +find "$p" -name 'test.*' -and -not -name '*.cdist__unpack_sum' \ + | sort \ + | while read -r l +do + n="$( basename "$l" )" + + printf '__unpack %s --destination %s/%s\n' \ + "$l" \ + "$d" \ + "$n" +done + +echo "__clean_path $p --pattern '.+/test\..+'" diff --git a/cdist/conf/type/__unpack/test/make-test-files.sh b/cdist/conf/type/__unpack/test/make-test-files.sh new file mode 100755 index 00000000..d18e9e9f --- /dev/null +++ b/cdist/conf/type/__unpack/test/make-test-files.sh @@ -0,0 +1,44 @@ +#!/bin/sh -ex + +echo test.7z > test +7z a test.7z test > /dev/null + +echo test.bz2 > test +bzip2 test + +echo test.gz > test +gzip test + +echo test.lzma > test +lzma test + +echo test.rar > test +rar a test.rar test > /dev/null + +echo test.tar.bz2 > test +tar cf test.tar test +bzip2 test.tar + +echo test.tar.xz > test +tar cf test.tar test +xz test.tar + +echo test.tgz > test +tar cf test.tar test +gzip test.tar +mv test.tar.gz test.tgz + +echo test.tar.gz > test +tar cf test.tar test +gzip test.tar + +echo test.tar > test +tar cf test.tar test + +echo test.xz > test +xz test + +echo test.zip > test +zip test.zip test > /dev/null + +rm test From 463b6cd6b52da8918d2803721164748c2a632696 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 27 Jul 2020 06:22:25 +0200 Subject: [PATCH 120/155] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 2457e90b..8d26e767 100644 --- a/docs/changelog +++ b/docs/changelog @@ -13,6 +13,9 @@ next: * Type __user: Install user packages on OpenWrt (Dennis Camera) * Type __openldap_server: Add Alpine support (Timothée Floure) * Type __pf_apply: Remove deprecated type (Darko Poljak) + * Type __package_apt: Fix for legacy APT versions that do not support --no-install-recommends (Dennis Camera) + * Type __key_value: Get awk from POSIX PATH (Dennis Camera) + * New type: __unpack (Ander Punnar) 6.6.0: 2020-06-17 * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) From d26c36914a4fa9889af57f549cb2854968663a0b Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 27 Jul 2020 10:55:28 +0200 Subject: [PATCH 121/155] [__timezone] Make type singleton --- cdist/conf/type/__timezone/gencode-remote | 2 +- cdist/conf/type/__timezone/man.rst | 22 ++++++++++++------- cdist/conf/type/__timezone/manifest | 2 +- cdist/conf/type/__timezone/parameter/required | 1 + cdist/conf/type/__timezone/singleton | 0 5 files changed, 17 insertions(+), 10 deletions(-) create mode 100644 cdist/conf/type/__timezone/parameter/required create mode 100644 cdist/conf/type/__timezone/singleton diff --git a/cdist/conf/type/__timezone/gencode-remote b/cdist/conf/type/__timezone/gencode-remote index 5299f548..b685c990 100755 --- a/cdist/conf/type/__timezone/gencode-remote +++ b/cdist/conf/type/__timezone/gencode-remote @@ -22,7 +22,7 @@ # This type allows to configure the desired localtime timezone. timezone_is=$(cat "$__object/explorer/timezone_is") -timezone_should="$__object_id" +timezone_should=$(cat "$__object/parameter/tz") os=$(cat "$__global/explorer/os") if [ "$timezone_is" = "$timezone_should" ]; then diff --git a/cdist/conf/type/__timezone/man.rst b/cdist/conf/type/__timezone/man.rst index 8a945c16..6012c552 100644 --- a/cdist/conf/type/__timezone/man.rst +++ b/cdist/conf/type/__timezone/man.rst @@ -14,7 +14,8 @@ This type creates a symlink (/etc/localtime) to the selected timezone REQUIRED PARAMETERS ------------------- -None. +tz + The name of timezone to set. OPTIONAL PARAMETERS @@ -27,19 +28,24 @@ EXAMPLES .. code-block:: sh - #Set up Europe/Andorra as our timezone. - __timezone Europe/Andorra + # Set up Europe/Andorra as our timezone. + __timezone --tz Europe/Andorra - #Set up US/Central as our timezone. - __timezone US/Central + # Set up US/Central as our timezone. + __timezone --tz US/Central AUTHORS ------- -Ramon Salvadó +| Steven Armstrong +| Nico Schottelius +| Ramon Salvadó +| Dennis Camera COPYING ------- -Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2012-2020 the `AUTHORS`_. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__timezone/manifest b/cdist/conf/type/__timezone/manifest index 3d28ccba..0eb7fb9c 100755 --- a/cdist/conf/type/__timezone/manifest +++ b/cdist/conf/type/__timezone/manifest @@ -22,7 +22,7 @@ # # This type allows to configure the desired localtime timezone. -timezone="$__object_id" +timezone=$(cat "$__object/parameter/tz") os=$(cat "$__global/explorer/os") case "$os" in diff --git a/cdist/conf/type/__timezone/parameter/required b/cdist/conf/type/__timezone/parameter/required new file mode 100644 index 00000000..975445e4 --- /dev/null +++ b/cdist/conf/type/__timezone/parameter/required @@ -0,0 +1 @@ +tz diff --git a/cdist/conf/type/__timezone/singleton b/cdist/conf/type/__timezone/singleton new file mode 100644 index 00000000..e69de29b From 627d215b637f1893e48a4a04c7e60686b960a698 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 27 Jul 2020 13:09:53 +0200 Subject: [PATCH 122/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 8d26e767..e3711be2 100644 --- a/docs/changelog +++ b/docs/changelog @@ -16,6 +16,7 @@ next: * Type __package_apt: Fix for legacy APT versions that do not support --no-install-recommends (Dennis Camera) * Type __key_value: Get awk from POSIX PATH (Dennis Camera) * New type: __unpack (Ander Punnar) + * Type __locale_system: Support more OSes (Dennis Camera) 6.6.0: 2020-06-17 * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) From 5dfc996febd085e4592fe590c4837bd4802ef4c3 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 26 Jul 2020 22:53:34 +0200 Subject: [PATCH 123/155] Fix global explorers for NetBSD On NetBSD sysctl is at /sbin/sysctl, but the default PATH does not contain /sbin. --- cdist/conf/explorer/cpu_cores | 1 + cdist/conf/explorer/disks | 5 ++--- cdist/conf/explorer/memory | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cdist/conf/explorer/cpu_cores b/cdist/conf/explorer/cpu_cores index c6744142..81e5294e 100755 --- a/cdist/conf/explorer/cpu_cores +++ b/cdist/conf/explorer/cpu_cores @@ -33,6 +33,7 @@ case "$os" in ;; "freebsd"|"netbsd") + PATH=$(getconf PATH) sysctl -n hw.ncpu ;; diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks index 24540601..56d62d10 100755 --- a/cdist/conf/explorer/disks +++ b/cdist/conf/explorer/disks @@ -30,9 +30,8 @@ case $uname_s in sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+' ;; NetBSD) - PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin" - sysctl -n hw.disknames \ - | awk 'BEGIN { RS = " " } /^[lsw]d[0-9]+/' + PATH=$(getconf PATH) + sysctl -n hw.disknames | awk -v RS=' ' '/^[lsw]d[0-9]+/' ;; Linux) # list of major device numbers toexclude: diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory index 302b4cda..5ea15ada 100755 --- a/cdist/conf/explorer/memory +++ b/cdist/conf/explorer/memory @@ -30,6 +30,7 @@ case "$os" in ;; *"bsd") + PATH=$(getconf PATH) echo "$(sysctl -n hw.physmem) / 1048576" | bc ;; From 3a87a447d00ef12eb6ace7c240926568aec2a042 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 26 Jul 2020 23:37:48 +0200 Subject: [PATCH 124/155] [type/__sysctl] Fix on NetBSD --- cdist/conf/type/__sysctl/explorer/value | 9 +++++++-- cdist/conf/type/__sysctl/gencode-remote | 2 ++ cdist/conf/type/__sysctl/man.rst | 7 +++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__sysctl/explorer/value b/cdist/conf/type/__sysctl/explorer/value index fc85b3d8..3e93c151 100755 --- a/cdist/conf/type/__sysctl/explorer/value +++ b/cdist/conf/type/__sysctl/explorer/value @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -18,5 +18,10 @@ # along with cdist. If not, see . # +if test "$(uname -s)" = NetBSD +then + PATH=$(getconf PATH) +fi + # get the current runtime value -sysctl -n "$__object_id" || true +sysctl -n "${__object_id}" || true diff --git a/cdist/conf/type/__sysctl/gencode-remote b/cdist/conf/type/__sysctl/gencode-remote index 711d54e5..f0f6deef 100755 --- a/cdist/conf/type/__sysctl/gencode-remote +++ b/cdist/conf/type/__sysctl/gencode-remote @@ -44,6 +44,8 @@ case "$os" in flag='-w' ;; netbsd) + # shellcheck disable=SC2016 + echo 'PATH=$(getconf PATH)' flag='-w' ;; freebsd|openbsd) diff --git a/cdist/conf/type/__sysctl/man.rst b/cdist/conf/type/__sysctl/man.rst index 6873003e..dbb9a1ac 100644 --- a/cdist/conf/type/__sysctl/man.rst +++ b/cdist/conf/type/__sysctl/man.rst @@ -26,6 +26,13 @@ EXAMPLES __sysctl net.ipv4.ip_forward --value 1 + # On some operating systems, e.g. NetBSD, to prevent an error if the + # MIB style name does not exist (e.g. optional kernel components), + # name and value can be separated by `?=`. The same effect can be achieved + # in cdist by appending a `?` to the key: + + __sysctl ddb.onpanic? --value -1 + AUTHORS ------- From 76bb214b5300707426030649cf03f35808e43256 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 27 Jul 2020 15:31:38 +0200 Subject: [PATCH 125/155] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index e3711be2..8fef4653 100644 --- a/docs/changelog +++ b/docs/changelog @@ -17,6 +17,8 @@ next: * Type __key_value: Get awk from POSIX PATH (Dennis Camera) * New type: __unpack (Ander Punnar) * Type __locale_system: Support more OSes (Dennis Camera) + * Explorers cpu_cores, disks, memory: Fix for NetBSD (Dennis Camera) + * Type __sysctl: Fix for NetBSD (Dennis Camera) 6.6.0: 2020-06-17 * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) From f5b367dfdbf60e75cfb2be038a05b6f226a1c010 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 28 Jul 2020 07:14:26 +0200 Subject: [PATCH 126/155] Release 6.7.0 --- docs/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog b/docs/changelog index 8fef4653..269a2049 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,8 @@ Changelog --------- -next: +6.7.0: 2020-07-28 + * Delete deprecated type: __pf_apply (Darko Poljak) * New type: __download (Ander Punnar) * Type __locale_system: Add devuan support (Dennis Camera) * Type __package_opkg: Add locking (Dennis Camera) @@ -12,7 +13,6 @@ next: * Type __postfix_master: Fix --option parameter and option expansion (Daniel Fancsali) * Type __user: Install user packages on OpenWrt (Dennis Camera) * Type __openldap_server: Add Alpine support (Timothée Floure) - * Type __pf_apply: Remove deprecated type (Darko Poljak) * Type __package_apt: Fix for legacy APT versions that do not support --no-install-recommends (Dennis Camera) * Type __key_value: Get awk from POSIX PATH (Dennis Camera) * New type: __unpack (Ander Punnar) From c053a2c4a000b0c15df186ef2e067ea68ec9499c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 29 Jul 2020 11:31:12 +0200 Subject: [PATCH 127/155] Fix building man pages Resolves #830. --- Makefile | 4 ++-- cdist/conf/type/__openldap_server/man.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index f89ac1e7..3712511c 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ version: } # Manpages #3: generic part -man: version $(MANTYPES) $(DOCSREF) +man: version configskel $(MANTYPES) $(DOCSREF) $(DOCSTYPESREF) $(SPHINXM) html: version configskel $(MANTYPES) $(DOCSREF) $(DOCSTYPESREF) @@ -104,7 +104,7 @@ DOTMANTYPES=$(subst /man.rst,.rst,$(DOTMANTYPEPREFIX)) $(DOTMAN7DSTDIR)/cdist-type%.rst: $(DOTTYPEDIR)/%/man.rst ln -sf "$^" $@ -dotman: version $(DOTMANTYPES) +dotman: version configskel $(DOTMANTYPES) $(DOCSREF) $(DOCSTYPESREF) $(SPHINXM) ################################################################################ diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst index a96c7dad..fa714ec0 100644 --- a/cdist/conf/type/__openldap_server/man.rst +++ b/cdist/conf/type/__openldap_server/man.rst @@ -31,8 +31,8 @@ manager-password-hash Generate e.g. with: `slappasswd -s weneedgoodsecurity`. See `slappasswd(8C)`, `slapd.conf(5)`. TODO: implement this: http://blog.adamsbros.org/2015/06/09/openldap-ssha-salted-hashes-by-hand/ - to derive from the manager-password parameter and ensure idempotency (care with salts). - At that point, manager-password-hash should be deprecated and ignored. + to derive from the manager-password parameter and ensure idempotency (care with salts). + At that point, manager-password-hash should be deprecated and ignored. serverid The server for the directory. From d37d2dc307d066f4d5f1bbbb93e0480965dfe3b8 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 2 Aug 2020 13:53:38 +0300 Subject: [PATCH 128/155] [__unpack] add parameter --tar-extra-args --- cdist/conf/type/__unpack/gencode-remote | 7 +++++++ cdist/conf/type/__unpack/man.rst | 11 +++++++++++ cdist/conf/type/__unpack/parameter/optional | 1 + 3 files changed, 19 insertions(+) diff --git a/cdist/conf/type/__unpack/gencode-remote b/cdist/conf/type/__unpack/gencode-remote index 45c7173a..3b7a19a7 100755 --- a/cdist/conf/type/__unpack/gencode-remote +++ b/cdist/conf/type/__unpack/gencode-remote @@ -23,6 +23,13 @@ case "$src" in cmd="$cmd --strip-components=$tar_strip" fi + + if [ -f "$__object/parameter/tar-extra-args" ] + then + tar_extra_args="$( cat "$__object/parameter/tar-extra-args" )" + + cmd="$cmd $tar_extra_args" + fi ;; *.7z) case "$os" in diff --git a/cdist/conf/type/__unpack/man.rst b/cdist/conf/type/__unpack/man.rst index 8fe96e43..bd0603cf 100644 --- a/cdist/conf/type/__unpack/man.rst +++ b/cdist/conf/type/__unpack/man.rst @@ -33,6 +33,10 @@ sum-file tar-strip Tarball specific. See ``man tar`` for ``--strip-components``. +tar-extra-args + Tarball sepcific. Append additional arguments to ``tar`` command. + See ``man tar`` for possible arguments. + OPTIONAL BOOLEAN PARAMETERS --------------------------- @@ -65,6 +69,13 @@ EXAMPLES --preserve-archive \ --destination /opt/cpma/server + # example usecase for --tar-* args + __unpack /root/strelaysrv.tar.gz \ + --preserve-archive \ + --destination /usr/local/bin \ + --tar-strip 1 \ + --tar-extra-args '--wildcards "*/strelaysrv"' + AUTHORS ------- diff --git a/cdist/conf/type/__unpack/parameter/optional b/cdist/conf/type/__unpack/parameter/optional index d136dd0c..da26ca63 100644 --- a/cdist/conf/type/__unpack/parameter/optional +++ b/cdist/conf/type/__unpack/parameter/optional @@ -1,2 +1,3 @@ sum-file tar-strip +tar-extra-args From 935f2395bc8f02d373f06d1e22acad813403ead2 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 2 Aug 2020 13:54:30 +0300 Subject: [PATCH 129/155] [__locale_system] fix for debian and ubuntu ubuntu 6.10 and debian etch are 10+ years old and EOL. rather than preserving compatibility I'll just remove it. while /etc/environment works too, correct place is /etc/default/locale (as it was before breaking change). also /etc/debian_version (os_version explorer) may contain minor version with dot (10.5) or string (bullseye/sid). --- cdist/conf/type/__locale_system/manifest | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest index e4286ef6..f7d75387 100755 --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -40,27 +40,13 @@ os=$(cat "$__global/explorer/os") case $os in debian) - os_version=$(cat "${__global}/explorer/os_version") - if expr "${os_version}" '>=' 4 >/dev/null - then - # Debian 4 (etch) and later - locale_conf="/etc/default/locale" - else - locale_conf="/etc/environment" - fi + locale_conf="/etc/default/locale" ;; devuan) locale_conf="/etc/default/locale" ;; ubuntu) - os_version=$(cat "${__global}/explorer/os_version") - if expr "${os_version}" '>=' 6.10 >/dev/null - then - # Ubuntu 6.10 (edgy) and later - locale_conf="/etc/default/locale" - else - locale_conf="/etc/environment" - fi + locale_conf="/etc/default/locale" ;; archlinux) locale_conf="/etc/locale.conf" From 885d5a58f40e5adfd6aa616712e4c7246eca7880 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 2 Aug 2020 16:49:01 +0200 Subject: [PATCH 130/155] [type/__locale_system] Fix floating point version comparison --- cdist/conf/type/__locale_system/manifest | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest index e4286ef6..521f1007 100755 --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -24,6 +24,9 @@ # Configure system-wide locale by modifying i18n file. # +bccmp() { test "$(bc)" -gt 0; } + + key=$__object_id onchange_cmd= # none, by default quote_value=false @@ -41,7 +44,7 @@ case $os in debian) os_version=$(cat "${__global}/explorer/os_version") - if expr "${os_version}" '>=' 4 >/dev/null + if printf '%f >= 4\n' "${os_version}" | bccmp then # Debian 4 (etch) and later locale_conf="/etc/default/locale" @@ -54,7 +57,7 @@ in ;; ubuntu) os_version=$(cat "${__global}/explorer/os_version") - if expr "${os_version}" '>=' 6.10 >/dev/null + if printf '%f >= 6.10\n' "${os_version}" | bccmp then # Ubuntu 6.10 (edgy) and later locale_conf="/etc/default/locale" @@ -68,7 +71,7 @@ in centos|redhat|scientific) # shellcheck source=/dev/null version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}") - if expr "${version_id}" '>=' 7 >/dev/null + if printf '%f >= 7\n' "${version_id}" | bccmp then locale_conf="/etc/locale.conf" else @@ -78,7 +81,7 @@ in fedora) # shellcheck source=/dev/null version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}") - if expr "${version_id}" '>=' 18 >/dev/null + if printf '%f >= 18\n' "${version_id}" | bccmp then locale_conf="/etc/locale.conf" quote_value=false @@ -113,7 +116,8 @@ in locale_conf="/etc/default/init" locale_conf_group="sys" - if expr "$(cat "${__global}/explorer/os_version")" '>=' 5.11 >/dev/null + os_version=$(cat "${__global}/explorer/os_version") + if printf '%f >= 5.11\n' "${os_version}" | bccmp then # mode on Oracle Solaris 11 is actually 0444, # but the write bit makes sense, IMO From 71710fa00a86e3dcda6be82b959956cb88ba9902 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 2 Aug 2020 20:17:00 +0200 Subject: [PATCH 131/155] [type/__locale_system] Implement "proper" version comparison Proper in the sense that it can handle all numeric version numbers even if they are not floating point (e.g. 16.04.6). --- cdist/conf/type/__locale_system/manifest | 28 ++++++++++++++++-------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest index 521f1007..88eadb31 100755 --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -24,7 +24,20 @@ # Configure system-wide locale by modifying i18n file. # -bccmp() { test "$(bc)" -gt 0; } +version_ge() { + awk -F '[^0-9.]' -v target="${1:?}" ' + function max(x, y) { return x > y ? x : y } + BEGIN { + getline + nx = split($1, x, ".") + ny = split(target, y, ".") + for (i = 1; i <= max(nx, ny); ++i) { + diff = int(x[i]) - int(y[i]) + if (diff == 0) continue + exit (diff < 0) + } + }' +} key=$__object_id @@ -43,8 +56,7 @@ os=$(cat "$__global/explorer/os") case $os in debian) - os_version=$(cat "${__global}/explorer/os_version") - if printf '%f >= 4\n' "${os_version}" | bccmp + if version_ge 4 <"${__global}/explorer/os_version" then # Debian 4 (etch) and later locale_conf="/etc/default/locale" @@ -56,8 +68,7 @@ in locale_conf="/etc/default/locale" ;; ubuntu) - os_version=$(cat "${__global}/explorer/os_version") - if printf '%f >= 6.10\n' "${os_version}" | bccmp + if version_ge 6.10 <"${__global}/explorer/os_version" then # Ubuntu 6.10 (edgy) and later locale_conf="/etc/default/locale" @@ -71,7 +82,7 @@ in centos|redhat|scientific) # shellcheck source=/dev/null version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}") - if printf '%f >= 7\n' "${version_id}" | bccmp + if echo "${version_id}" | version_ge 7 then locale_conf="/etc/locale.conf" else @@ -81,7 +92,7 @@ in fedora) # shellcheck source=/dev/null version_id=$(. "${__global}/explorer/os_release" && echo "${VERSION_ID:-0}") - if printf '%f >= 18\n' "${version_id}" | bccmp + if echo "${version_id}" | version_ge 18 then locale_conf="/etc/locale.conf" quote_value=false @@ -116,8 +127,7 @@ in locale_conf="/etc/default/init" locale_conf_group="sys" - os_version=$(cat "${__global}/explorer/os_version") - if printf '%f >= 5.11\n' "${os_version}" | bccmp + if version_ge 5.11 <"${__global}/explorer/os_version" then # mode on Oracle Solaris 11 is actually 0444, # but the write bit makes sense, IMO From 7b480f4293b84a97850430a9436e28c66ccf5736 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 2 Aug 2020 22:08:14 +0200 Subject: [PATCH 132/155] [type/__locale_system] Fix version extraction for SuSE --- cdist/conf/type/__locale_system/manifest | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest index 88eadb31..4b996ebc 100755 --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -163,7 +163,13 @@ in key="export ${__object_id}" ;; suse) - os_version=$(cat "${__global}/explorer/os_version") + if test -s "${__global}/explorer/os_release" + then + # shellcheck source=/dev/null + os_version=$(. "${__global}/explorer/os_release" && echo "${VERSION}") + else + os_version=$(sed -n 's/^VERSION\ *=\ *//p' "${__global}/explorer/os_version") + fi os_major=$(expr "${os_version}" : '\([0-9]\{1,\}\)') # https://documentation.suse.com/sles/15-SP2/html/SLES-all/cha-suse.html#sec-suse-l10n From b370b70ff4ccc1c6efca97f61f220a5ce0b191da Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 2 Aug 2020 22:13:41 +0200 Subject: [PATCH 133/155] [explorer/os] Fix OS detection for openSUSE All distros with ID_LIKE suse should be treated as "suse". My openSUSE Leap 15.1 installation has: ID_LIKE="suse opensuse" This patch doesn't require a strict "suse" value but only the word suse to be in the list. --- cdist/conf/explorer/os | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os index 2d2aede6..8f54d9c8 100755 --- a/cdist/conf/explorer/os +++ b/cdist/conf/explorer/os @@ -144,7 +144,8 @@ esac if [ -f /etc/os-release ]; then # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse - if grep -q ^ID_LIKE=\"suse\" /etc/os-release 2>/dev/null; then + if (. /etc/os-release && echo "${ID_LIKE}" | grep -q '\(^\|\ \)suse\($\|\ \)') + then echo suse exit 0 fi From 17ab4bd80c9c9dda2fa4f8ccde5ee454f7b5ba2d Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Thu, 6 Aug 2020 11:45:05 +0200 Subject: [PATCH 134/155] Add Alpine Linux as supported for __filesystem. --- cdist/conf/type/__filesystem/explorer/lsblk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk index 9ae544ac..9be3c575 100644 --- a/cdist/conf/type/__filesystem/explorer/lsblk +++ b/cdist/conf/type/__filesystem/explorer/lsblk @@ -18,16 +18,16 @@ # along with cdist. If not, see . # -os=$("$__explorer/os") +os=$("${__explorer:?}/os") -if [ -f "$__object/parameter/device" ]; then +if [ -f "${__object:?}/parameter/device" ]; then blkdev="$(cat "$__object/parameter/device")" else - blkdev="$__object_id" + blkdev="${__object_id:?}" fi case "$os" in - centos|fedora|redhat|suse|gentoo) + alpine|centos|fedora|redhat|suse|gentoo) if [ ! -x "$(command -v lsblk)" ]; then echo "lsblk is required for __filesystem type" >&2 exit 1 From 74dd47c8c352cc3de3b5684bbe90520f61d95f37 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 15 Aug 2020 21:11:43 +0200 Subject: [PATCH 135/155] ++changelog --- docs/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/changelog b/docs/changelog index 269a2049..2c09bfea 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,12 @@ Changelog --------- +next: + * Type __locale_system: Fix for debian and ubuntu (Ander Punnar) + * Type __unpack: Add --tar-extra-args parameter (Ander Punnar) + * Explorer os: Fix OS detection for openSUSE (Dennis Camera) + * Type __filesystem: Support Alpine Linux (Joachim Desroches) + 6.7.0: 2020-07-28 * Delete deprecated type: __pf_apply (Darko Poljak) * New type: __download (Ander Punnar) From 8f94a226c7f0b875987e25e57682414204380fa2 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 15 Aug 2020 21:54:07 +0200 Subject: [PATCH 136/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 2c09bfea..9e44cc4d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * Type __unpack: Add --tar-extra-args parameter (Ander Punnar) * Explorer os: Fix OS detection for openSUSE (Dennis Camera) * Type __filesystem: Support Alpine Linux (Joachim Desroches) + * Type __locale_system: Fix version comparison (Dennis Camera) 6.7.0: 2020-07-28 * Delete deprecated type: __pf_apply (Darko Poljak) From 6fed1785298eb506e7ea6cd177efe7d96658769f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 17 Aug 2020 09:27:48 +0200 Subject: [PATCH 137/155] [explorer/os_version] Convert Debian sid to version number. Conversion of Debian sid to versions is done based on Debian codenames. The version number is the version number of the final release - 0.01. It is unknown if Debian < 4.0 has any sort of version information available (apart from maybe checking base-files package version). But I don't think any of these systems are still alive, so I think going with 3.99 is fine for those. --- cdist/conf/explorer/os_version | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 1d54ea60..a7b1d3bc 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -31,7 +31,32 @@ case "$("$__explorer/os")" in cat /etc/arch-release ;; debian) - cat /etc/debian_version + debian_version=$(cat /etc/debian_version) + case $debian_version + in + testing/unstable) + # previous to Debian 4.0 testing/unstable was used + # cf. https://metadata.ftp-master.debian.org/changelogs/main/b/base-files/base-files_11_changelog + echo 3.99 + ;; + */sid) + # sid versions don't have a number, so we decode by codename: + case $(expr "$debian_version" : '\([a-z]\{1,\}\)/') + in + bullseye) echo 10.99 ;; + buster) echo 9.99 ;; + stretch) echo 8.99 ;; + jessie) echo 7.99 ;; + wheezy) echo 6.99 ;; + squeeze) echo 5.99 ;; + lenny) echo 4.99 ;; + *) exit 1 + esac + ;; + *) + echo "$debian_version" + ;; + esac ;; devuan) cat /etc/devuan_version @@ -73,4 +98,4 @@ case "$("$__explorer/os")" in alpine) cat /etc/alpine-release ;; -esac \ No newline at end of file +esac From 502d75304707704d3ba73374701394b98d259506 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 18 Aug 2020 00:46:07 +0300 Subject: [PATCH 138/155] [__unpack] add --onchange --- cdist/conf/type/__unpack/gencode-remote | 5 +++++ cdist/conf/type/__unpack/man.rst | 3 +++ cdist/conf/type/__unpack/parameter/optional | 1 + 3 files changed, 9 insertions(+) diff --git a/cdist/conf/type/__unpack/gencode-remote b/cdist/conf/type/__unpack/gencode-remote index 3b7a19a7..c4451f73 100755 --- a/cdist/conf/type/__unpack/gencode-remote +++ b/cdist/conf/type/__unpack/gencode-remote @@ -80,3 +80,8 @@ if [ ! -f "$__object/parameter/preserve-archive" ] then echo "rm -f '$src'" fi + +if [ -f "$__object/parameter/onchange" ] +then + cat "$__object/parameter/onchange" +fi diff --git a/cdist/conf/type/__unpack/man.rst b/cdist/conf/type/__unpack/man.rst index bd0603cf..daa03814 100644 --- a/cdist/conf/type/__unpack/man.rst +++ b/cdist/conf/type/__unpack/man.rst @@ -50,6 +50,9 @@ backup-destination preserve-archive Don't delete archive after unpacking. +onchange + Execute this command after unpack. + EXAMPLES -------- diff --git a/cdist/conf/type/__unpack/parameter/optional b/cdist/conf/type/__unpack/parameter/optional index da26ca63..d846ac75 100644 --- a/cdist/conf/type/__unpack/parameter/optional +++ b/cdist/conf/type/__unpack/parameter/optional @@ -1,3 +1,4 @@ sum-file tar-strip tar-extra-args +onchange From d239169c4f91a4ce33be170ed29b08299c51234f Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 18 Aug 2020 00:48:58 +0300 Subject: [PATCH 139/155] [__download] fix manual: onchange parameter in wrong section --- cdist/conf/type/__download/man.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index 6ec0b19a..eb3ac971 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -30,9 +30,6 @@ sum By default output of ``cksum`` without filename is expected. Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. -onchange - Execute this command after download. - OPTIONAL PARAMETERS ------------------- @@ -54,6 +51,9 @@ cmd-sum format specification ``%s`` which will become destination. For example: ``md5sum '%s' | awk '{print $1}'``. +onchange + Execute this command after download. + EXAMPLES -------- From ba26a437be4d8034037e7a6eb42382aee01f829e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 18 Aug 2020 11:06:19 +0200 Subject: [PATCH 140/155] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 9e44cc4d..a3e4167a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,9 @@ next: * Explorer os: Fix OS detection for openSUSE (Dennis Camera) * Type __filesystem: Support Alpine Linux (Joachim Desroches) * Type __locale_system: Fix version comparison (Dennis Camera) + * Type __unpack: Add --onchange parameter (Ander Punnar) + * Type __download: Fix manual (Ander Punnar) + * Explorer os_version: Convert Debian sid to version number (Dennis Camera) 6.7.0: 2020-07-28 * Delete deprecated type: __pf_apply (Darko Poljak) From c17541f24cf118b0989a5899cab3a439ed534a46 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 22 Aug 2020 19:53:42 +0200 Subject: [PATCH 141/155] Expand and split by consecutive require delimiters Resolves #832. --- cdist/emulator.py | 6 +++++- cdist/test/emulator/__init__.py | 10 ++++++++++ docs/src/cdist-manifest.rst | 3 ++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/cdist/emulator.py b/cdist/emulator.py index 9fe84056..60b94880 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -25,6 +25,7 @@ import argparse import logging import os import sys +import re import cdist from cdist import core @@ -389,12 +390,15 @@ class Emulator: if "require" in self.env: requirements = self.env['require'] self.log.debug("reqs = " + requirements) - for requirement in requirements.split(" "): + for requirement in self._parse_require(requirements): # Ignore empty fields - probably the only field anyway if len(requirement) == 0: continue self.record_requirement(requirement) + def _parse_require(self, require): + return re.split(r'[ \t\n]+', require) + def record_auto_requirements(self): """An object shall automatically depend on all objects that it defined in it's type manifest. diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index e375676c..befd7b57 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -685,6 +685,16 @@ class EmulatorAlreadyExistingRequirementsWarnTestCase(test.CdistTestCase): self.env['require'] = '__directory/spam' emu = emulator.Emulator(argv, env=self.env) + def test_parse_require(self): + require = " \t \n \t\t\n\t\na\tb\nc d \te\t\nf\ng\t " + expected = ['', 'a', 'b', 'c', 'd', 'e', 'f', 'g', '', ] + + argv = ['__directory', 'spam'] + emu = emulator.Emulator(argv, env=self.env) + requirements = emu._parse_require(require) + + self.assertEqual(expected, requirements) + if __name__ == '__main__': import unittest diff --git a/docs/src/cdist-manifest.rst b/docs/src/cdist-manifest.rst index 5dbca479..2e49a721 100644 --- a/docs/src/cdist-manifest.rst +++ b/docs/src/cdist-manifest.rst @@ -95,7 +95,8 @@ Dependencies ------------ If you want to describe that something requires something else, just setup the variable "require" to contain the requirements. Multiple -requirements can be added white space separated. +requirements can be added separated with (optionally consecutive) +delimiters including space, tab and newline. :: From b5a40eb0d1c8990f0feb8e918a95e54e4fd35d98 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 27 Aug 2020 12:25:11 +0200 Subject: [PATCH 142/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index a3e4167a..eda77a12 100644 --- a/docs/changelog +++ b/docs/changelog @@ -10,6 +10,7 @@ next: * Type __unpack: Add --onchange parameter (Ander Punnar) * Type __download: Fix manual (Ander Punnar) * Explorer os_version: Convert Debian sid to version number (Dennis Camera) + * Core: Expand require delimiter characters, split by consecutive delimiters (Darko Poljak) 6.7.0: 2020-07-28 * Delete deprecated type: __pf_apply (Darko Poljak) From b1375464cc7507b38dd02f2df52c75489837d129 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 6 Sep 2020 13:24:58 +0200 Subject: [PATCH 143/155] __systemd_service: fix manpage typos --- cdist/conf/type/__systemd_service/man.rst | 25 +++++++++++++++-------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/cdist/conf/type/__systemd_service/man.rst b/cdist/conf/type/__systemd_service/man.rst index 7eca398b..cd14c985 100644 --- a/cdist/conf/type/__systemd_service/man.rst +++ b/cdist/conf/type/__systemd_service/man.rst @@ -1,9 +1,10 @@ -cdist-type__systemd-service(7) +cdist-type__systemd_service(7) ============================== NAME ---- -cdist-type__systemd-service - Controls a systemd service state +cdist-type__systemd_service - Controls a systemd service state + DESCRIPTION ----------- @@ -14,11 +15,12 @@ service after configuration applied or shutdown one service. The activation or deactivation is out of scope. Look for the :strong:`cdist-type__systemd_util`\ (7) type instead. + REQUIRED PARAMETERS ------------------- - None. + OPTIONAL PARAMETERS ------------------- @@ -31,12 +33,12 @@ state running Service should run (default) - stoppend - Service should stopped + stopped + Service should be stopped action Executes an action on on the service. It will only execute it if the - service keeps the state **running**. There are following actions, where: + service keeps the state ``running``. There are following actions, where: reload Reloads the service @@ -48,11 +50,12 @@ BOOLEAN PARAMETERS ------------------ if-required - Only execute the action if minimum one required type outputs a message to - **$__messages_out**. Through this, the action should only executed if a + Only execute the action if at minimum one required type outputs a message + to ``$__messages_out``. Through this, the action should only executed if a dependency did something. The action will not executed if no dependencies given. + MESSAGES -------- @@ -68,12 +71,14 @@ restart reload Reloaded the service + ABORTS ------ Aborts in following cases: systemd or the service does not exist + EXAMPLES -------- .. code-block:: sh @@ -95,13 +100,15 @@ EXAMPLES # reload the service for a modified configuration file # only reloads the service if the file really changed - require="__config_file/etc/foo.conf" __systemd_service foo \ + require="__file/etc/foo.conf" __systemd_service foo \ --action reload --if-required + AUTHORS ------- Matthias Stecher + COPYRIGHT --------- Copyright \(C) 2020 Matthias Stecher. You can redistribute it From 6b262a61c19234f6450cef11f715020dcd5f40d5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 10 Sep 2020 13:24:58 +0200 Subject: [PATCH 144/155] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index eda77a12..e4ec3fdd 100644 --- a/docs/changelog +++ b/docs/changelog @@ -11,6 +11,8 @@ next: * Type __download: Fix manual (Ander Punnar) * Explorer os_version: Convert Debian sid to version number (Dennis Camera) * Core: Expand require delimiter characters, split by consecutive delimiters (Darko Poljak) + * Type __timezone: Make singleton (Dennis Camera) + * Type __systemd_service: Fix manpage typos (Matthias Stecher) 6.7.0: 2020-07-28 * Delete deprecated type: __pf_apply (Darko Poljak) From 53b91adbd8be4d858f1f4da3389b63babc85db0c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 11 Sep 2020 14:19:45 +0200 Subject: [PATCH 145/155] Fix shellcheck --- cdist/conf/explorer/os | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os index 8f54d9c8..46d87f3e 100755 --- a/cdist/conf/explorer/os +++ b/cdist/conf/explorer/os @@ -144,6 +144,7 @@ esac if [ -f /etc/os-release ]; then # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse + # shellcheck disable=SC1091 if (. /etc/os-release && echo "${ID_LIKE}" | grep -q '\(^\|\ \)suse\($\|\ \)') then echo suse From 2885c6a24838956167ae6c33ebf87067e8790083 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 11 Sep 2020 14:15:26 +0200 Subject: [PATCH 146/155] Release 6.8.0 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index e4ec3fdd..7a0d050d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.8.0: 2020-09-11 * Type __locale_system: Fix for debian and ubuntu (Ander Punnar) * Type __unpack: Add --tar-extra-args parameter (Ander Punnar) * Explorer os: Fix OS detection for openSUSE (Dennis Camera) From decc0ad54dffbafcd7d3644c28ae4d149926e612 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 18 Sep 2020 19:33:36 +0300 Subject: [PATCH 147/155] [__package_pip] detect pip binary --- cdist/conf/type/__package_pip/explorer/pip | 10 ++++++++++ cdist/conf/type/__package_pip/explorer/state | 2 +- cdist/conf/type/__package_pip/gencode-remote | 7 ++++++- 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100755 cdist/conf/type/__package_pip/explorer/pip diff --git a/cdist/conf/type/__package_pip/explorer/pip b/cdist/conf/type/__package_pip/explorer/pip new file mode 100755 index 00000000..cf9fae89 --- /dev/null +++ b/cdist/conf/type/__package_pip/explorer/pip @@ -0,0 +1,10 @@ +#!/bin/sh -e + +for bin in pip3 pip +do + if check="$( command -v "$bin" )" + then + echo "$check" + break + fi +done diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state index 5be07280..3cc98ab9 100644 --- a/cdist/conf/type/__package_pip/explorer/state +++ b/cdist/conf/type/__package_pip/explorer/state @@ -32,7 +32,7 @@ pipparam="$__object/parameter/pip" if [ -f "$pipparam" ]; then pip=$(cat "$pipparam") else - pip="pip" + pip="$( "$__type_explorer/pip" )" fi # If there is no pip, it may get created from somebody else. diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index dcc4fdf9..9c5eb0d1 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -38,7 +38,12 @@ pipparam="$__object/parameter/pip" if [ -f "$pipparam" ]; then pip=$(cat "$pipparam") else - pip="pip" + pip="$( cat "$__object/explorer/pip" )" + if [ -z "$pip" ] + then + echo 'pip not found in path' >&2 + exit 1 + fi fi runasparam="$__object/parameter/runas" From 89b621511561105cfcfcf12bbf820b74ab03396d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 21 Sep 2020 09:04:05 +0200 Subject: [PATCH 148/155] Clarify stdin input Resolve #836. --- cdist/argparse.py | 36 ++++++-------------------- cdist/config.py | 6 +++-- cdist/inventory.py | 4 +-- cdist/test/inventory/__init__.py | 3 +-- docs/src/man1/cdist.rst | 44 ++++++++++---------------------- 5 files changed, 29 insertions(+), 64 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 77303591..1d16bb25 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -273,8 +273,7 @@ def get_parsers(): '-f', '--file', help=('Read specified file for a list of additional hosts to ' 'operate on or if \'-\' is given, read stdin (one host per ' - 'line). If no host or host file is specified then, by ' - 'default, read hosts from stdin.'), + 'line).'), dest='hostfile', required=False) parser['config_args'].add_argument( '-p', '--parallel', nargs='?', metavar='HOST_MAX', @@ -326,9 +325,7 @@ def get_parsers(): parser['add-host'].add_argument( '-f', '--file', help=('Read additional hosts to add from specified file ' - 'or from stdin if \'-\' (each host on separate line). ' - 'If no host or host file is specified then, by default, ' - 'read from stdin.'), + 'or from stdin if \'-\' (each host on separate line). '), dest='hostfile', required=False) parser['add-tag'] = parser['invsub'].add_parser( @@ -342,20 +339,12 @@ def get_parsers(): parser['add-tag'].add_argument( '-f', '--file', help=('Read additional hosts to add tags from specified file ' - 'or from stdin if \'-\' (each host on separate line). ' - 'If no host or host file is specified then, by default, ' - 'read from stdin. If no tags/tagfile nor hosts/hostfile' - ' are specified then tags are read from stdin and are' - ' added to all hosts.'), + 'or from stdin if \'-\' (each host on separate line). '), dest='hostfile', required=False) parser['add-tag'].add_argument( '-T', '--tag-file', help=('Read additional tags to add from specified file ' - 'or from stdin if \'-\' (each tag on separate line). ' - 'If no tag or tag file is specified then, by default, ' - 'read from stdin. If no tags/tagfile nor hosts/hostfile' - ' are specified then tags are read from stdin and are' - ' added to all hosts.'), + 'or from stdin if \'-\' (each tag on separate line). '), dest='tagfile', required=False) parser['add-tag'].add_argument( '-t', '--taglist', @@ -376,9 +365,7 @@ def get_parsers(): parser['del-host'].add_argument( '-f', '--file', help=('Read additional hosts to delete from specified file ' - 'or from stdin if \'-\' (each host on separate line). ' - 'If no host or host file is specified then, by default, ' - 'read from stdin.'), + 'or from stdin if \'-\' (each host on separate line). '), dest='hostfile', required=False) parser['del-tag'] = parser['invsub'].add_parser( @@ -396,20 +383,13 @@ def get_parsers(): parser['del-tag'].add_argument( '-f', '--file', help=('Read additional hosts to delete tags for from specified ' - 'file or from stdin if \'-\' (each host on separate line). ' - 'If no host or host file is specified then, by default, ' - 'read from stdin. If no tags/tagfile nor hosts/hostfile' - ' are specified then tags are read from stdin and are' - ' deleted from all hosts.'), + 'file or from stdin if \'-\' (each host on separate ' + 'line). '), dest='hostfile', required=False) parser['del-tag'].add_argument( '-T', '--tag-file', help=('Read additional tags from specified file ' - 'or from stdin if \'-\' (each tag on separate line). ' - 'If no tag or tag file is specified then, by default, ' - 'read from stdin. If no tags/tagfile nor' - ' hosts/hostfile are specified then tags are read from' - ' stdin and are added to all hosts.'), + 'or from stdin if \'-\' (each tag on separate line). '), dest='tagfile', required=False) parser['del-tag'].add_argument( '-t', '--taglist', diff --git a/cdist/config.py b/cdist/config.py index 30416008..e84f6f84 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -175,9 +175,11 @@ class Config: raise cdist.Error(("Cannot read both, manifest and host file, " "from stdin")) - # if no host source is specified then read hosts from stdin if not (args.hostfile or args.host): - args.hostfile = '-' + if args.tag or args.all_tagged_hosts: + raise cdist.Error(("Target host tag(s) missing")) + else: + raise cdist.Error(("Target host(s) missing")) if args.manifest == '-': # read initial manifest from stdin diff --git a/cdist/inventory.py b/cdist/inventory.py index 6ab20fa7..0387f326 100644 --- a/cdist/inventory.py +++ b/cdist/inventory.py @@ -299,7 +299,7 @@ class InventoryHost(Inventory): self.all = all if not self.hosts and not self.hostfile: - self.hostfile = "-" + raise cdist.Error("Host(s) missing") def _new_hostpath(self, hostpath): # create empty file @@ -355,7 +355,7 @@ class InventoryTag(Inventory): else: self.allhosts = False if not self.tags and not self.tagfile: - self.tagfile = "-" + raise cdist.Error("Tag(s) missing") if self.hostfile == "-" and self.tagfile == "-": raise cdist.Error("Cannot read both, hosts and tags, from stdin") diff --git a/cdist/test/inventory/__init__.py b/cdist/test/inventory/__init__.py index 287c855c..a8cd8bf8 100644 --- a/cdist/test/inventory/__init__.py +++ b/cdist/test/inventory/__init__.py @@ -307,11 +307,10 @@ class InventoryTestCase(test.CdistTestCase): raise e # InventoryTag + @unittest.expectedFailure def test_inventory_tag_init(self): invTag = inventory.InventoryTag(db_basedir=inventory_dir, action="add") - self.assertTrue(invTag.allhosts) - self.assertEqual(invTag.tagfile, "-") def test_inventory_tag_stdin_multiple_hosts(self): try: diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index aa2607f8..0ecb4a61 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -177,10 +177,8 @@ Install command is currently in beta. **-f HOSTFILE, --file HOSTFILE** Read specified file for a list of additional hosts to operate on - or if '-' is given, read stdin (one host per line). - If no host or host file is specified then, by default, - read hosts from stdin. For the file format see - :strong:`HOSTFILE FORMAT` below. + or if '-' is given, read stdin (one host per line). For the file + format see :strong:`HOSTFILE FORMAT` below. **-g CONFIG_FILE, --config-file CONFIG_FILE** Use specified custom configuration file. @@ -299,9 +297,8 @@ Add host(s) to inventory database. **-f HOSTFILE, --file HOSTFILE** Read additional hosts to add from specified file or - from stdin if '-' (each host on separate line). If no - host or host file is specified then, by default, read - from stdin. Hostfile format is the same as config hostfile format. + from stdin if '-' (each host on separate line). + Hostfile format is the same as config hostfile format. **-g CONFIG_FILE, --config-file CONFIG_FILE** Use specified custom configuration file. @@ -327,11 +324,8 @@ Add tag(s) to inventory database. **-f HOSTFILE, --file HOSTFILE** Read additional hosts to add tags from specified file - or from stdin if '-' (each host on separate line). If - no host or host file is specified then, by default, - read from stdin. If no tags/tagfile nor hosts/hostfile - are specified then tags are read from stdin and are - added to all hosts. Hostfile format is the same as config hostfile format. + or from stdin if '-' (each host on separate line). + Hostfile format is the same as config hostfile format. **-g CONFIG_FILE, --config-file CONFIG_FILE** Use specified custom configuration file. @@ -346,11 +340,8 @@ Add tag(s) to inventory database. **-T TAGFILE, --tag-file TAGFILE** Read additional tags to add from specified file or - from stdin if '-' (each tag on separate line). If no - tag or tag file is specified then, by default, read - from stdin. If no tags/tagfile nor hosts/hostfile are - specified then tags are read from stdin and are added - to all hosts. Tagfile format is the same as config hostfile format. + from stdin if '-' (each tag on separate line). + Tagfile format is the same as config hostfile format. **-t TAGLIST, --taglist TAGLIST** Tag list to be added for specified host(s), comma @@ -372,9 +363,8 @@ Delete host(s) from inventory database. **-f HOSTFILE, --file HOSTFILE** Read additional hosts to delete from specified file or - from stdin if '-' (each host on separate line). If no - host or host file is specified then, by default, read - from stdin. Hostfile format is the same as config hostfile format. + from stdin if '-' (each host on separate line). + Hostfile format is the same as config hostfile format. **-g CONFIG_FILE, --config-file CONFIG_FILE** Use specified custom configuration file. @@ -404,11 +394,8 @@ Delete tag(s) from inventory database. **-f HOSTFILE, --file HOSTFILE** Read additional hosts to delete tags for from specified file or from stdin if '-' (each host on - separate line). If no host or host file is specified - then, by default, read from stdin. If no tags/tagfile - nor hosts/hostfile are specified then tags are read - from stdin and are deleted from all hosts. Hostfile - format is the same as config hostfile format. + separate line). Hostfile format is the same as + config hostfile format. **-g CONFIG_FILE, --config-file CONFIG_FILE** Use specified custom configuration file. @@ -423,11 +410,8 @@ Delete tag(s) from inventory database. **-T TAGFILE, --tag-file TAGFILE** Read additional tags from specified file or from stdin - if '-' (each tag on separate line). If no tag or tag - file is specified then, by default, read from stdin. - If no tags/tagfile nor hosts/hostfile are specified - then tags are read from stdin and are added to all - hosts. Tagfile format is the same as config hostfile format. + if '-' (each tag on separate line). + Tagfile format is the same as config hostfile format. **-t TAGLIST, --taglist TAGLIST** Tag list to be deleted for specified host(s), comma From 89a0080e133052b82ce19e7fdbaf045dcc833da7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 21 Sep 2020 09:09:26 +0200 Subject: [PATCH 149/155] ++changelog --- docs/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog b/docs/changelog index 7a0d050d..260b4fe4 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,10 @@ Changelog --------- +next: + * Core: Clarify stdin input (Darko Poljak) + * Type __package_pip: Detect pip binary (Ander Punnar) + 6.8.0: 2020-09-11 * Type __locale_system: Fix for debian and ubuntu (Ander Punnar) * Type __unpack: Add --tar-extra-args parameter (Ander Punnar) From 0fc10749edc698d1f70f826603d009fba943d252 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 21 Sep 2020 09:11:35 +0200 Subject: [PATCH 150/155] Fix shellcheck --- cdist/conf/type/__package_pip/gencode-remote | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index 9c5eb0d1..a1375c2d 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -60,7 +60,7 @@ case "$state_should" in then echo "su -c '$pip install -q $name' $runas" else - echo $pip install -q "$name" + echo "$pip" install -q "$name" fi echo "installed" >> "$__messages_out" ;; @@ -69,7 +69,7 @@ case "$state_should" in then echo "su -c '$pip uninstall -q -y $name' $runas" else - echo $pip uninstall -q -y "$name" + echo "$pip" uninstall -q -y "$name" fi echo "removed" >> "$__messages_out" ;; From b6922508b9e07834009a6bfae2d3933b5ea62fbf Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 21 Sep 2020 09:17:34 +0200 Subject: [PATCH 151/155] Update helper script --- bin/build-helper | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/build-helper b/bin/build-helper index ed41e438..d4d603ed 100755 --- a/bin/build-helper +++ b/bin/build-helper @@ -371,7 +371,6 @@ eof Manual steps post release: - cdist-web - send generated mailinglist.tmp mail - - twitter eof ;; From 84a7818121047cf6bb23d8bd3c055d09bb033d32 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 23 Sep 2020 20:29:47 +0200 Subject: [PATCH 152/155] docs: make varaibles environment-aware There are all overwriting the environment, even the comment states otherwise. Fixes it. --- docs/src/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/src/Makefile b/docs/src/Makefile index 2ecf7a32..ba87d170 100644 --- a/docs/src/Makefile +++ b/docs/src/Makefile @@ -2,10 +2,10 @@ # # You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = ../dist +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +PAPER ?= +BUILDDIR ?= ../dist # for cache, etc. _BUILDDIR = _build From 73d6c9d469d1429999ea2e980b3ac4c63a3e0dd4 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 27 Sep 2020 10:17:35 +0200 Subject: [PATCH 153/155] Add custom remote copy/exec examples --- docs/src/cdist-remote-exec-copy.rst | 389 +++++++++++++++++++++++++++- 1 file changed, 388 insertions(+), 1 deletion(-) diff --git a/docs/src/cdist-remote-exec-copy.rst b/docs/src/cdist-remote-exec-copy.rst index bb818310..e7b7b226 100644 --- a/docs/src/cdist-remote-exec-copy.rst +++ b/docs/src/cdist-remote-exec-copy.rst @@ -10,7 +10,7 @@ By default this is accomplished with ssh and scp respectively. The default implementations used by cdist are:: __remote_exec: ssh -o User=root - __remote_copy: scp -o User=root + __remote_copy: scp -o User=root -q The user can override these defaults by providing custom implementations and passing them to cdist with the --remote-exec and/or --remote-copy arguments. @@ -26,3 +26,390 @@ specified by enclosed in square brackets (see :strong:`ssh`\ (1) and With this simple interface the user can take total control of how cdist interacts with the target when required, while the default implementation remains as simple as possible. + + +Examples +-------- + +Here are examples of using alternative __remote_copy and __remote_exec scripts. + +All scripts from below are present in cdist sources in `other/examples/remote` +directory. + +ssh +~~~ + +Same as cdist default. + +**copy** + +Usage: cdist config --remote-copy "/path/to/this/script" target_host + +.. code-block:: sh + + #echo "$@" | logger -t "cdist-ssh-copy" + scp -o User=root -q $@ + +**exec** + +Usage: cdist config --remote-exec "/path/to/this/script" target_host + +.. code-block:: sh + + #echo "$@" | logger -t "cdist-ssh-exec" + ssh -o User=root $@ + +local +~~~~~ + +This effectively turns remote calling into local calling. Probably most useful +for the unit testing. + +**copy** + +.. code-block:: sh + + code="$(echo "$@" | sed "s|\([[:space:]]\)$__target_host:|\1|g")" + cp -L $code + +**exec** + +.. code-block:: sh + + target_host=$1; shift + echo "$@" | /bin/sh + +chroot +~~~~~~ + +**copy** + +Usage: cdist config --remote-copy "/path/to/this/script /path/to/your/chroot" target-id + +.. code-block:: sh + + log() { + #echo "$@" | logger -t "cdist-chroot-copy" + : + } + + chroot="$1"; shift + target_host="$__target_host" + + # replace target_host with chroot location + code="$(echo "$@" | sed "s|$target_host:|$chroot|g")" + + log "target_host: $target_host" + log "chroot: $chroot" + log "$@" + log "$code" + + # copy files into chroot + cp $code + + log "-----" + +**exec** + +Usage: cdist config --remote-exec "/path/to/this/script /path/to/your/chroot" target-id + +.. code-block:: sh + + log() { + #echo "$@" | logger -t "cdist-chroot-exec" + : + } + + chroot="$1"; shift + target_host="$1"; shift + + script=$(mktemp "${chroot}/tmp/chroot-${0##*/}.XXXXXXXXXX") + trap cleanup INT TERM EXIT + cleanup() { + [ $__cdist_debug ] || rm "$script" + } + + log "target_host: $target_host" + log "script: $script" + log "@: $@" + echo "#!/bin/sh -l" > "$script" + echo "$@" >> "$script" + chmod +x "$script" + + relative_script="${script#$chroot}" + log "relative_script: $relative_script" + + # run in chroot + chroot "$chroot" "$relative_script" + + log "-----" + +rsync +~~~~~ + +**copy** + +Usage: cdist config --remote-copy /path/to/this/script target_host + +.. code-block:: sh + + # For rsync to do the right thing, the source has to end with "/" if it is + # a directory. The below preprocessor loop takes care of that. + + # second last argument is the source + source_index=$(($#-1)) + index=0 + for arg in $@; do + if [ $index -eq 0 ]; then + # reset $@ + set -- + fi + index=$((index+=1)) + if [ $index -eq $source_index -a -d "$arg" ]; then + arg="${arg%/}/" + fi + set -- "$@" "$arg" + done + + rsync --backup --suffix=~cdist -e 'ssh -o User=root' $@ + +schroot +~~~~~~~ + +__remote_copy and __remote_exec scripts to run cdist against a chroot on the +target host over ssh. + +**copy** + +Usage: cdist config --remote-copy "/path/to/this/script schroot-chroot-name" target_host + + +.. code-block:: sh + + log() { + #echo "$@" | logger -t "cdist-schroot-copy" + : + } + + chroot_name="$1"; shift + target_host="$__target_host" + + # get directory for given chroot_name + chroot="$(ssh -o User=root -q $target_host schroot -c $chroot_name --config | awk -F = '/directory=/ {print $2}')" + + # prefix destination with chroot + code="$(echo "$@" | sed "s|$target_host:|$target_host:$chroot|g")" + + log "target_host: $target_host" + log "chroot_name: $chroot_name" + log "chroot: $chroot" + log "@: $@" + log "code: $code" + + # copy files into remote chroot + scp -o User=root -q $code + + log "-----" + +**exec** + +Usage: cdist config --remote-exec "/path/to/this/script schroot-chroot-name" target_host + +.. code-block:: sh + + log() { + #echo "$@" | logger -t "cdist-schroot-exec" + : + } + + chroot_name="$1"; shift + target_host="$1"; shift + + code="ssh -o User=root -q $target_host schroot -c $chroot_name -- $@" + + log "target_host: $target_host" + log "chroot_name: $chroot_name" + log "@: $@" + log "code: $code" + + # run in remote chroot + $code + + log "-----" + +schroot-uri +~~~~~~~~~~~ + +__remote_exec/__remote_copy script to run cdist against a schroot target URI. + +Usage:: + + cdist config \ + --remote-exec "/path/to/this/script exec" \ + --remote-copy "/path/to/this/script copy" \ + target_uri + + # target_uri examples: + schroot:///chroot-name + schroot://foo.ethz.ch/chroot-name + schroot://user-name@foo.ethz.ch/chroot-name + + # and how to match them in .../manifest/init + case "$target_host" in + schroot://*) + # any schroot + ;; + schroot://foo.ethz.ch/*) + # any schroot on specific host + ;; + schroot://foo.ethz.ch/chroot-name) + # specific schroot on specific host + ;; + schroot:///chroot-name) + # specific schroot on localhost + ;; + esac + +**copy/exec** + +.. code-block:: sh + + my_name="${0##*/}" + mode="$1"; shift + + log() { + # uncomment me for debugging + #echo "$@" | logger -t "cdist-$my_name-$mode" + : + } + + die() { + echo "$@" >&2 + exit 1 + } + + + uri="$__target_host" + + scheme="${uri%%:*}"; rest="${uri#$scheme:}"; rest="${rest#//}" + authority="${rest%%/*}"; rest="${rest#$authority}" + path="${rest%\?*}"; rest="${rest#$path}" + schroot_name="${path#/}" + + [ "$scheme" = "schroot" ] || die "Failed to parse scheme from __target_host ($__target_host). Expected 'schroot', got '$scheme'" + [ -n "$schroot_name" ] || die "Failed to parse schroot name from __target_host: $__target_host" + + case "$authority" in + '') + # authority is empty, neither user nor host given + user="" + host="" + ;; + *@*) + # authority contains @, take user from authority + user="${authority%@*}" + host="${authority#*@}" + ;; + *) + # no user in authority, default to root + user="root" + host="$authority" + ;; + esac + + log "mode: $mode" + log "@: $@" + log "uri: $uri" + log "scheme: $scheme" + log "authority: $authority" + log "user: $user" + log "host: $host" + log "path: $path" + log "schroot_name: $schroot_name" + + exec_prefix="" + copy_prefix="" + if [ -n "$host" ]; then + # we are working on a remote host + exec_prefix="ssh -o User=$user -q $host" + copy_prefix="scp -o User=$user -q" + copy_destination_prefix="$host:" + else + # working on local machine + copy_prefix="cp" + copy_destination_prefix="" + fi + log "exec_prefix: $exec_prefix" + log "copy_prefix: $copy_prefix" + log "copy_destination_prefix: $copy_destination_prefix" + + case "$mode" in + exec) + # In exec mode the first argument is the __target_host which we already got from env. Get rid of it. + shift + code="$exec_prefix schroot -c $schroot_name -- sh -c '$@'" + ;; + copy) + # get directory for given chroot_name + schroot_directory="$($exec_prefix schroot -c $schroot_name --config | awk -F = '/directory=/ {print $2}')" + [ -n "$schroot_directory" ] || die "Failed to retreive schroot directory for schroot: $schroot_name" + log "schroot_directory: $schroot_directory" + # prefix destination with chroot + code="$copy_prefix $(echo "$@" | sed "s|$uri:|${copy_destination_prefix}${schroot_directory}|g")" + ;; + *) die "Unknown mode: $mode";; + esac + + log "code: $code" + + # Run the code + $code + + log "-----" + +sudo +~~~~ + +**copy** + +Use rsync over ssh to copy files. Uses the "--rsync-path" option +to run the remote rsync instance with sudo. + +This command assumes your ssh configuration is already set up in ~/.ssh/config. + +Usage: cdist config --remote-copy /path/to/this/script target_host + +.. code-block:: sh + + # For rsync to do the right thing, the source has to end with "/" if it is + # a directory. The below preprocessor loop takes care of that. + + # second last argument is the source + source_index=$(($#-1)) + index=0 + for arg in $@; do + if [ $index -eq 0 ]; then + # reset $@ + set -- + fi + index=$((index+=1)) + if [ $index -eq $source_index -a -d "$arg" ]; then + arg="${arg%/}/" + fi + set -- "$@" "$arg" + done + + rsync --copy-links --rsync-path="sudo rsync" -e 'ssh' "$@" + +**exec** + +Prefixes all remote commands with sudo. + +This command assumes your ssh configuration is already set up in ~/.ssh/config. + +Usage: cdist config --remote-exec "/path/to/this/script" target_host + +.. code-block:: sh + + host="$1"; shift + ssh -q "$host" sudo sh -c \""$@"\" From 652c89185816526e5165ff9d5686a0d245ec3e5f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 29 Sep 2020 05:57:54 +0200 Subject: [PATCH 154/155] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 260b4fe4..8d380524 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Core: Clarify stdin input (Darko Poljak) * Type __package_pip: Detect pip binary (Ander Punnar) + * Documentation: Add custom remote copy/exec examples (Darko Poljak) 6.8.0: 2020-09-11 * Type __locale_system: Fix for debian and ubuntu (Ander Punnar) From f994226d0e23b667309fd75116c0c8a14d15f34d Mon Sep 17 00:00:00 2001 From: Evilham Date: Tue, 29 Sep 2020 19:44:47 +0200 Subject: [PATCH 155/155] [__package_pkgng_freebsd] Bootstrap pkg if necessary In a pristine FreeBSD base installation, pkg is really a bootstrapper utility, in such cases the type used to fail instead of automatically bootstrapping pkg. --- .../type/__package_pkgng_freebsd/explorer/pkg_bootstrapped | 4 ++++ cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version | 5 +++++ cdist/conf/type/__package_pkgng_freebsd/gencode-remote | 5 +++++ 3 files changed, 14 insertions(+) create mode 100755 cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped new file mode 100755 index 00000000..429f15d3 --- /dev/null +++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_bootstrapped @@ -0,0 +1,4 @@ +#!/bin/sh -e +if pkg -N >/dev/null 2>&1; then + echo "YES" +fi diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version index 92ce0623..f0fb9127 100755 --- a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version +++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version @@ -21,6 +21,11 @@ # Retrieve the status of a package - parsed dpkg output # +if ! pkg -N >/dev/null 2>&1; then + # Nothing to do if pkg is not bootstrapped + exit +fi + if [ -f "$__object/parameter/name" ]; then name="$(cat "$__object/parameter/name")" else diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote index dd36efda..b5944177 100755 --- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote @@ -43,6 +43,7 @@ fi repo="$(cat "$__object/parameter/repo")" state="$(cat "$__object/parameter/state")" curr_version="$(cat "$__object/explorer/pkg_version")" +pkg_bootstrapped="$(cat "$__object/explorer/pkg_bootstrapped")" add_cmd="pkg install -y" rm_cmd="pkg delete -y" upg_cmd="pkg upgrade -y" @@ -73,6 +74,10 @@ execcmd(){ ;; esac + if [ -z "${pkg_bootstrapped}" ]; then + echo "pkg bootstrap -y >/dev/null 2>&1" + fi + echo "$_cmd >/dev/null 2>&1" # Silence the output of the command echo "status=\$?" echo "if [ \"\$status\" -ne \"0\" ]; then"